fix🐛: fail readiness while the database is behind the migrations

The process started, both probes passed, and the first sign that the
schema did not match was a login failing with a driver-level encoding
error - go-admin#919, where an operator upgraded the binary and
restarted the API without running migrate. Nothing between those two
events had an opinion about the schema.

Readiness is where this belongs. Liveness asks "restart me", and a
process whose database is on the wrong schema comes back to the same
schema. Readiness asks "send me requests", and the answer is no. A
rolling update then stalls at the deploy - new instances never become
ready, the old ones keep serving - rather than at somebody's login, and
running migrate clears it without a restart because the check is
evaluated per request.

Any tenant database being behind fails the check, not only the one being
served: migrations are applied to every database in one run, so one
behind means that run did not finish, and serving the rest would let a
half-applied deploy look like a partial success.

A missing sys_migration table is nothing applied rather than an error.
That is a first deploy, where every migration is pending and the
operator can act on being told so.

The one test that matters is the one that cannot be written normally.
The registry is filled by init() in packages cmd/api does not import, so
a test that imported them to look at it would pass whatever the real
binary links - and a binary that links none of them gives a check that
reports every database current, forever, with every other test here
still green. TestTheServingBinaryLinksTheMigrationRegistry asks the
build instead, with a negative control so that a query matching
everything fails rather than passes.

Closes #920.

.github/workflows/build.yml

@@ -1,12 +1,25 @@
 name: Build
 
-# Documentation-only changes skip this workflow entirely.
+# Documentation-only changes, and changes confined to the Kubernetes
+# manifests, skip this workflow entirely.
 #
 # A push to master here does not just build - it pushes an image, runs the
 # migrations and restarts the demo container, so the site takes a short outage.
 # Paying that for a README edit is waste at best; at worst a deploy fails for a
 # reason unrelated to anything in the change. Code coverage is unaffected,
 # because go.yml still builds every push and pull request.
+#
+# scripts/k8s holds deploy.yml, storage.yml and prerun.sh, and the deploy below
+# reads none of them - it is an ssh into one host that runs docker, building the
+# Dockerfile at the repository root. Those manifests are for people deploying to
+# a cluster of their own. The pattern is scripts/k8s/** rather than scripts/**
+# because scripts/Dockerfile is a build input: go.yml builds the release image
+# from it on a tag.
+#
+# A file outside these patterns still runs the workflow even when the rest of
+# the change is ignorable: paths-ignore skips only when every changed path
+# matches. Editing this file is one such case, on purpose - a deploy script
+# that is never exercised by the change that broke it is worse than an outage.
 on:
   push:
     branches: [ master ]
@@ -15,6 +28,7 @@ on:
       - 'docs/**'
       - 'LICENSE*'
       - '.github/ISSUE_TEMPLATE/**'
+      - 'scripts/k8s/**'
   pull_request:
     branches: [ master ]
     paths-ignore:
@@ -22,6 +36,7 @@ on:
       - 'docs/**'
       - 'LICENSE*'
       - '.github/ISSUE_TEMPLATE/**'
+      - 'scripts/k8s/**'
 
 # One deploy at a time. Two merges seconds apart raced here: both runs did
 # docker rm -f then docker run, the second removed the container the first had

app/other/router/demo_guard_test.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/gin-gonic/gin"
 	jwt "github.com/go-admin-team/go-admin-core/v2/jwtauth"
+	"github.com/go-admin-team/go-admin-core/v2/sdk/config"
 
 	"go-admin/common/middleware"
 )
@@ -12,13 +13,28 @@ import (
 // registeredRoutes builds the generator's routes on an engine of its own and
 // reports the patterns they were registered under.
 //
+// The mode has to be given rather than inherited, because it now decides what
+// gets registered: a test that leaves it at the zero value would be asking
+// about a mode no deployment runs in, and would pass whether or not the gate
+// works.
+//
+// It is put back before this returns, not at the end of the test. t.Cleanup
+// would leave the mode set for everything the caller does afterwards, so a
+// caller that went on to assert something mode-dependent would be reading a
+// value this helper left behind rather than one it chose. A caller that does
+// want the mode set has to set it, which is visible where it happens.
+//
 // The JWT middleware is a zero value. MiddlewareFunc only closes over the
 // receiver and is never called here - no request is served, the engine is
 // asked what it has - so nothing dereferences it.
-func registeredRoutes(t *testing.T) map[string]bool {
+func registeredRoutes(t *testing.T, mode string) map[string]bool {
 	t.Helper()
 	gin.SetMode(gin.TestMode)
 
+	previous := config.ApplicationConfig.Mode
+	defer func() { config.ApplicationConfig.Mode = previous }()
+	config.ApplicationConfig.Mode = mode
+
 	r := gin.New()
 	v1 := r.Group("/api/v1")
 	sysNoCheckRoleRouter(v1, &jwt.GinJWTMiddleware{})
@@ -38,7 +54,7 @@ func registeredRoutes(t *testing.T) map[string]bool {
 // stops matching anything, demo mode silently starts serving it again, and
 // nothing else would say so.
 func TestEveryRouteDemoModeRefusesStillExists(t *testing.T) {
-	routes := registeredRoutes(t)
+	routes := registeredRoutes(t, "demo")
 	for _, guarded := range middleware.DemoWriteRoutes() {
 		if !routes[guarded] {
 			t.Errorf("demo mode refuses %q, but no route is registered under that pattern - "+
@@ -64,7 +80,7 @@ func TestTheGeneratorsReadOnlyRoutesAreNotRefused(t *testing.T) {
 		"/api/v1/db/tables/page",
 		"/api/v1/db/columns/page",
 	} {
-		if !registeredRoutes(t)[readOnly] {
+		if !registeredRoutes(t, "demo")[readOnly] {
 			t.Fatalf("%s is not registered, so this test is asserting against nothing", readOnly)
 		}
 		if refused[readOnly] {

app/other/router/gen_router.go

@@ -3,15 +3,49 @@ package router
 import (
 	"github.com/gin-gonic/gin"
 	jwt "github.com/go-admin-team/go-admin-core/v2/jwtauth"
+	"github.com/go-admin-team/go-admin-core/v2/sdk/config"
+
 	"go-admin/app/admin/apis"
 	"go-admin/app/other/apis/tools"
 )
 
+// GenWriteRoutesEnabled reports whether the code generator's writing endpoints
+// are registered in this process.
+//
+// Three of the generator's endpoints do not read. /gen/toproject writes seven
+// Go and Vue source files onto this host, one of them under the path
+// gen.frontpath names; /gen/apitofile writes a migration; /gen/todb inserts
+// menus and APIs. All three are GET, all three are listed in CasbinExclude,
+// and AuthCheckRole skips what is on that list - so Enforce never runs for
+// them and any account that can log in may call them. That is a bargain a
+// workstation can make and a deployment cannot.
+//
+// dev is the shipped default and is where the generator is meant to be used.
+// demo keeps them because demo mode already has a better answer than a 404:
+// DemoEvn refuses these three by name and explains itself, which is the thing
+// the demo exists to show. test and prod get nothing.
+//
+// The mode is read once, while the routes are being built. Changing
+// application.mode in a running process adds and removes nothing - a
+// configuration reload rebuilds neither the engine nor its routes.
+//
+// core has constants for dev, test and prod but none for demo, which this
+// repository spells as a literal in common/middleware/demo.go. Both are
+// literals here so that the two read as one set rather than two conventions.
+func GenWriteRoutesEnabled() bool {
+	switch config.ApplicationConfig.Mode {
+	case "dev", "demo":
+		return true
+	default:
+		return false
+	}
+}
+
 func init() {
 	routerCheckRole = append(routerCheckRole, sysNoCheckRoleRouter, registerDBRouter, registerSysTableRouter)
 }
 
-func sysNoCheckRoleRouter(v1 *gin.RouterGroup ,authMiddleware *jwt.GinJWTMiddleware) {
+func sysNoCheckRoleRouter(v1 *gin.RouterGroup, authMiddleware *jwt.GinJWTMiddleware) {
 	r1 := v1.Group("")
 	{
 		sys := apis.System{}
@@ -22,9 +56,11 @@ func sysNoCheckRoleRouter(v1 *gin.RouterGroup ,authMiddleware *jwt.GinJWTMiddlew
 	{
 		gen := tools.Gen{}
 		r.GET("/gen/preview/:tableId", gen.Preview)
-		r.GET("/gen/toproject/:tableId", gen.GenCode)
-		r.GET("/gen/apitofile/:tableId", gen.GenApiToFile)
-		r.GET("/gen/todb/:tableId", gen.GenMenuAndApi)
+		if GenWriteRoutesEnabled() {
+			r.GET("/gen/toproject/:tableId", gen.GenCode)
+			r.GET("/gen/apitofile/:tableId", gen.GenApiToFile)
+			r.GET("/gen/todb/:tableId", gen.GenMenuAndApi)
+		}
 		sysTable := tools.SysTable{}
 		r.GET("/gen/tabletree", sysTable.GetSysTablesTree)
 	}
@@ -53,4 +89,4 @@ func registerSysTableRouter(v1 *gin.RouterGroup, authMiddleware *jwt.GinJWTMiddl
 			tablesInfo.GET("", sysTable.GetSysTablesInfo)
 		}
 	}
-}
+}

app/other/router/gen_router_test.go

@@ -0,0 +1,151 @@
+package router
+
+import (
+	"testing"
+
+	"github.com/go-admin-team/go-admin-core/v2/sdk/config"
+)
+
+// genWritingRoutes are the three that do not read. They write Go and Vue
+// source onto the host, a migration, and rows in sys_menu.
+var genWritingRoutes = []string{
+	"/api/v1/gen/toproject/:tableId",
+	"/api/v1/gen/apitofile/:tableId",
+	"/api/v1/gen/todb/:tableId",
+}
+
+// genReadingRoutes are the rest of the generator's surface. Gating the three
+// above must not cost any of these: a deployment that cannot list its tables
+// or preview a template has lost the feature, not secured it.
+var genReadingRoutes = []string{
+	"/api/v1/gen/preview/:tableId",
+	"/api/v1/gen/tabletree",
+	"/api/v1/db/tables/page",
+	"/api/v1/db/columns/page",
+}
+
+// The endpoints that write are registered where the mode says development and
+// nowhere else.
+//
+// They are in CasbinExclude, so Enforce never runs for them and any account
+// that can log in may call them. dev is the shipped default and is where the
+// generator is meant to be used; demo keeps them because DemoEvn refuses these
+// three by name and saying so is the thing the demo is for. Everything else,
+// including the empty mode a process gets when nothing set one, is refused by
+// not existing.
+func TestGeneratorWritingRoutesExistOnlyWhereTheModeAllowsIt(t *testing.T) {
+	for _, tc := range []struct {
+		mode     string
+		expected bool
+		why      string
+	}{
+		{"dev", true, "the shipped default, and where the generator is used"},
+		{"demo", true, "registered so demo mode can refuse them by name"},
+		{"test", false, "a deployment, however much it is called a test"},
+		{"prod", false, "a deployment"},
+		{"", false, "no mode configured is not a reason to trust the caller"},
+	} {
+		t.Run("mode="+tc.mode, func(t *testing.T) {
+			routes := registeredRoutes(t, tc.mode)
+			for _, writing := range genWritingRoutes {
+				if got := routes[writing]; got != tc.expected {
+					t.Errorf("mode %q: %s registered = %v, want %v (%s)",
+						tc.mode, writing, got, tc.expected, tc.why)
+				}
+			}
+		})
+	}
+}
+
+// The other direction. Refusing too much is as much of a defect as refusing
+// too little, and the read-only half of the generator is what a demo shows.
+func TestGeneratorReadingRoutesExistInEveryMode(t *testing.T) {
+	for _, mode := range []string{"dev", "demo", "test", "prod", ""} {
+		t.Run("mode="+mode, func(t *testing.T) {
+			routes := registeredRoutes(t, mode)
+			for _, reading := range genReadingRoutes {
+				if !routes[reading] {
+					t.Errorf("mode %q: %s is not registered - the gate took a route that only reads",
+						mode, reading)
+				}
+			}
+		})
+	}
+}
+
+// GenWriteRoutesEnabled is what cmd/api reads to decide whether to warn at
+// start-up. If it and the registration ever disagree, the log says one thing
+// and the engine does another, so they are checked against each other rather
+// than each against a list.
+func TestGenWriteRoutesEnabledAgreesWithWhatWasRegistered(t *testing.T) {
+	for _, mode := range []string{"dev", "demo", "test", "prod", ""} {
+		t.Run("mode="+mode, func(t *testing.T) {
+			routes := registeredRoutes(t, mode)
+
+			// registeredRoutes puts the mode back before it returns, so ask
+			// the predicate under a mode set here - about the same value the
+			// engine was just built under.
+			previous := config.ApplicationConfig.Mode
+			t.Cleanup(func() { config.ApplicationConfig.Mode = previous })
+			config.ApplicationConfig.Mode = mode
+
+			claimed := GenWriteRoutesEnabled()
+			actual := routes["/api/v1/gen/todb/:tableId"]
+			if claimed != actual {
+				t.Errorf("mode %q: GenWriteRoutesEnabled() = %v but the route was registered = %v",
+					mode, claimed, actual)
+			}
+		})
+	}
+}
+
+// The helper restores the mode before it returns, so nothing it was asked
+// about leaks into what the caller does next.
+//
+// Worth a test of its own because the failure is silent: a helper that left
+// the mode set would make every assertion after the call read a value the
+// caller did not choose, and each of those assertions would still pass for as
+// long as the leaked value happened to be the right one.
+func TestRegisteredRoutesRestoresTheModeBeforeReturning(t *testing.T) {
+	const sentinel = "not-a-mode"
+
+	previous := config.ApplicationConfig.Mode
+	t.Cleanup(func() { config.ApplicationConfig.Mode = previous })
+	config.ApplicationConfig.Mode = sentinel
+
+	registeredRoutes(t, "prod")
+
+	if got := config.ApplicationConfig.Mode; got != sentinel {
+		t.Errorf("mode after the helper returned = %q, want %q - it was left set to what "+
+			"the helper was asked about", got, sentinel)
+	}
+}
+
+// Changing the mode after the routes were built unregisters nothing.
+//
+// buildRouter has one call site, in run(), and route registration is not on
+// any phase or reload callback - so a configuration reload moves
+// config.ApplicationConfig.Mode without moving the routes. From that moment
+// GenWriteRoutesEnabled answers about a mode the engine was not built under.
+//
+// That gap is why the start-up warning tells the reader to restart rather than
+// only to change the mode. This pins it: if registration ever becomes dynamic,
+// this test fails and the message it justifies has to be revisited.
+func TestChangingTheModeDoesNotUnregisterWhatWasAlreadyBuilt(t *testing.T) {
+	built := registeredRoutes(t, "dev")
+	if !built["/api/v1/gen/todb/:tableId"] {
+		t.Fatal("built under dev without the writing routes, so this test asserts nothing")
+	}
+
+	previous := config.ApplicationConfig.Mode
+	t.Cleanup(func() { config.ApplicationConfig.Mode = previous })
+	config.ApplicationConfig.Mode = "prod"
+
+	if GenWriteRoutesEnabled() {
+		t.Fatal("the predicate still allows prod, so the disagreement below is not the one meant")
+	}
+	if !built["/api/v1/gen/todb/:tableId"] {
+		t.Error("the route left the engine when the mode changed - registration has become " +
+			"dynamic, and the start-up warning's advice to restart is now wrong")
+	}
+}

cmd/api/gen_warning_test.go

@@ -0,0 +1,38 @@
+package api
+
+import (
+	"testing"
+
+	"github.com/go-admin-team/go-admin-core/v2/sdk/config"
+)
+
+// The warning fires exactly where the generator's writing endpoints are served
+// and nothing else refuses them.
+//
+// dev is the case the warning exists for: it is the shipped default, so it is
+// the mode a deployment that changed nothing is running in. demo serves the
+// routes too, but DemoEvn refuses all three by name, so warning there would
+// describe an exposure that is not there.
+func TestGeneratorWriteRoutesWarningFiresWhereTheExposureIs(t *testing.T) {
+	for _, tc := range []struct {
+		mode string
+		want bool
+		why  string
+	}{
+		{"dev", true, "shipped default, endpoints served and not refused"},
+		{"demo", false, "served, but DemoEvn refuses all three"},
+		{"test", false, "not served"},
+		{"prod", false, "not served"},
+		{"", false, "not served"},
+	} {
+		t.Run("mode="+tc.mode, func(t *testing.T) {
+			previous := config.ApplicationConfig.Mode
+			t.Cleanup(func() { config.ApplicationConfig.Mode = previous })
+			config.ApplicationConfig.Mode = tc.mode
+
+			if got := generatorWriteRoutesNeedWarning(); got != tc.want {
+				t.Errorf("mode %q: warning = %v, want %v (%s)", tc.mode, got, tc.want, tc.why)
+			}
+		})
+	}
+}

cmd/api/schema.go

@@ -0,0 +1,117 @@
+package api
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/go-admin-team/go-admin-core/v2/sdk"
+	"gorm.io/gorm"
+
+	"go-admin/cmd/migrate/migration"
+	"go-admin/common/health"
+	commonmodels "go-admin/common/models"
+)
+
+// schemaCheckName is what a failing schema reports itself as in /ready's body.
+const schemaCheckName = "schema"
+
+// registerSchemaCheck adds the pending-migration check to readiness.
+//
+// Readiness rather than a refusal to start, and rather than a log line alone.
+// The two probes answer different questions: liveness is "restart me", and a
+// process whose database is on the wrong schema comes back to the same schema,
+// so restarting is not the answer. Readiness is "send me requests", and with a
+// schema the binary does not match the answer is no.
+//
+// Issue #919 is what the absence of this looked like: the process started,
+// both probes passed, and the first sign of trouble was a login failing with a
+// driver-level encoding error. Refusing to start would have been the wrong fix
+// - a process that exits tells an operator less than one that runs and says
+// why, and under an orchestrator it crash-loops - while a log line alone is
+// not something an orchestrator can act on.
+func registerSchemaCheck() {
+	health.Register(schemaCheckName, schemaCheck)
+}
+
+// schemaCheck fails while any tenant database is behind the migrations this
+// binary registers.
+//
+// Any one of them, rather than only the tenant being served: migrations are
+// applied to every database in one run, so one database behind means that run
+// did not finish. Serving the rest would let a half-applied deploy look like a
+// partial success.
+//
+// Evaluated per request rather than decided at start-up, so that running
+// migrate clears it without a restart.
+func schemaCheck(ctx context.Context) error {
+	registered := migration.RegisteredVersions()
+	if len(registered) == 0 {
+		// Nothing registered means nothing can be pending, which is the honest
+		// answer for a tree with no migrations. It is also what a broken build
+		// would produce - the registry is filled by init() in packages the
+		// binary has to link - so cmd/api's dependency test asserts the real
+		// binary links them.
+		return nil
+	}
+
+	behind := make([]string, 0, 2)
+	for name, db := range sdk.Runtime.GetAllDb() {
+		applied, err := appliedVersions(ctx, db)
+		if err != nil {
+			return fmt.Errorf("reading applied migrations for %q: %w", name, err)
+		}
+		if pending := pendingVersions(registered, applied); len(pending) > 0 {
+			behind = append(behind, fmt.Sprintf("%s is %d behind, first pending %s",
+				name, len(pending), pending[0]))
+		}
+	}
+	if len(behind) == 0 {
+		return nil
+	}
+	sort.Strings(behind)
+	return fmt.Errorf("%s; run `go-admin migrate -c <config>` and see `go-admin migrate status`",
+		strings.Join(behind, "; "))
+}
+
+// appliedVersions reads what sys_migration records for one database.
+//
+// A missing table is not an error: a database that has never been migrated has
+// applied nothing, which is exactly what the caller needs to hear, and is the
+// state a first deploy is in.
+func appliedVersions(ctx context.Context, db *gorm.DB) (map[string]bool, error) {
+	if db == nil {
+		return nil, fmt.Errorf("no database")
+	}
+	db = db.WithContext(ctx)
+	if !db.Migrator().HasTable(&commonmodels.Migration{}) {
+		return map[string]bool{}, nil
+	}
+	var rows []commonmodels.Migration
+	if err := db.Select("version").Find(&rows).Error; err != nil {
+		return nil, err
+	}
+	out := make(map[string]bool, len(rows))
+	for _, r := range rows {
+		out[r.Version] = true
+	}
+	return out, nil
+}
+
+// pendingVersions returns the registered versions applied does not contain.
+//
+// Split out and taking both sides as arguments because the registry is
+// process-wide and filled by init() in packages cmd/api does not import: a
+// test in this package cannot arrange it, so the arranging part is the part
+// that is not tested here.
+func pendingVersions(registered []string, applied map[string]bool) []string {
+	out := make([]string, 0)
+	for _, v := range registered {
+		if !applied[v] {
+			out = append(out, v)
+		}
+	}
+	sort.Strings(out)
+	return out
+}

cmd/api/schema_test.go

@@ -0,0 +1,124 @@
+package api
+
+import (
+	"context"
+	"os/exec"
+	"strings"
+	"testing"
+
+	"github.com/glebarez/sqlite"
+	"gorm.io/gorm"
+
+	commonmodels "go-admin/common/models"
+)
+
+func TestPendingVersionsReportsOnlyWhatIsNotApplied(t *testing.T) {
+	registered := []string{"1000_a", "2000_b", "3000_c"}
+	applied := map[string]bool{"1000_a": true, "3000_c": true}
+
+	got := pendingVersions(registered, applied)
+	if len(got) != 1 || got[0] != "2000_b" {
+		t.Errorf("pending = %v, want [2000_b]", got)
+	}
+}
+
+func TestPendingVersionsIsEmptyWhenTheDatabaseIsCurrent(t *testing.T) {
+	registered := []string{"1000_a", "2000_b"}
+	applied := map[string]bool{"1000_a": true, "2000_b": true}
+
+	if got := pendingVersions(registered, applied); len(got) != 0 {
+		t.Errorf("pending = %v, want none", got)
+	}
+}
+
+// A row recorded that this binary no longer registers is not pending. It is
+// the orphan `migrate status` already reports, and readiness has nothing to
+// say about it: the schema is ahead, not behind, and requests will be served
+// correctly.
+func TestPendingVersionsIgnoresAppliedRowsNothingRegisters(t *testing.T) {
+	registered := []string{"1000_a"}
+	applied := map[string]bool{"1000_a": true, "9999_gone": true}
+
+	if got := pendingVersions(registered, applied); len(got) != 0 {
+		t.Errorf("pending = %v, want none - an orphaned row is not a pending migration", got)
+	}
+}
+
+func memoryDB(t *testing.T) *gorm.DB {
+	t.Helper()
+	db, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), &gorm.Config{})
+	if err != nil {
+		t.Fatalf("open: %v", err)
+	}
+	t.Cleanup(func() { db.Migrator().DropTable(&commonmodels.Migration{}) })
+	return db
+}
+
+// A first deploy has no sys_migration table. That is "nothing applied", not an
+// error: reporting it as one would make the check fail for a reason the
+// operator cannot act on, on the one deployment where every migration really
+// is pending.
+func TestAppliedVersionsTreatsAMissingTableAsNothingApplied(t *testing.T) {
+	db := memoryDB(t)
+	db.Migrator().DropTable(&commonmodels.Migration{})
+
+	got, err := appliedVersions(context.Background(), db)
+	if err != nil {
+		t.Fatalf("appliedVersions: %v", err)
+	}
+	if len(got) != 0 {
+		t.Errorf("applied = %v, want empty", got)
+	}
+}
+
+func TestAppliedVersionsReadsWhatTheTableHolds(t *testing.T) {
+	db := memoryDB(t)
+	if err := db.AutoMigrate(&commonmodels.Migration{}); err != nil {
+		t.Fatalf("automigrate: %v", err)
+	}
+	db.Create(&commonmodels.Migration{Version: "1000_a"})
+	db.Create(&commonmodels.Migration{Version: "2000_b"})
+
+	got, err := appliedVersions(context.Background(), db)
+	if err != nil {
+		t.Fatalf("appliedVersions: %v", err)
+	}
+	if !got["1000_a"] || !got["2000_b"] || len(got) != 2 {
+		t.Errorf("applied = %v, want the two rows written", got)
+	}
+}
+
+// The check is only worth anything if the registry it reads is populated in
+// the binary that serves requests, and it is filled by init() in packages
+// cmd/api does not import - cmd/migrate blank-imports them, and cmd wires both
+// subcommands into one binary.
+//
+// This cannot be asserted from an ordinary test: importing the version package
+// to look at the registry would put it in the test binary's dependency graph
+// and pass whatever the real binary links. So ask the build instead.
+//
+// Without this, dropping those blank imports leaves a check that reports
+// "nothing pending" for every database forever, and every test above still
+// passes.
+func TestTheServingBinaryLinksTheMigrationRegistry(t *testing.T) {
+	out, err := exec.Command("go", "list", "-deps", "go-admin").Output()
+	if err != nil {
+		t.Skipf("go list unavailable: %v", err)
+	}
+	deps := string(out)
+
+	const versions = "go-admin/cmd/migrate/migration/version"
+	if !strings.Contains(deps, versions+"\n") {
+		t.Errorf("the main package does not link %s, so the schema check would "+
+			"read an empty registry and report every database as current", versions)
+	}
+
+	// Negative control: a package the binary genuinely must not link, so that a
+	// `deps` that somehow contained everything would fail here rather than pass
+	// the assertion above for the wrong reason.
+	const notLinked = "go-admin/tools/checksilent"
+	if strings.Contains(deps, notLinked+"\n") {
+		t.Errorf("%s is in the binary's dependency closure, so this test cannot "+
+			"tell a real link from a query that matches anything", notLinked)
+	}
+}

cmd/api/server.go

@@ -82,6 +82,11 @@ func setup() {
 	// can call the API, which is only true once the socket is accepting.
 	sdk.Runtime.SetPhase(runtime.AfterListen, startCronJobs)
 
+	// Registered before the configuration is read, because it registers a
+	// callback rather than reading anything: the check runs per request and
+	// asks the databases that exist then.
+	registerSchemaCheck()
+
 	//1. 读取配置
 	bootstrap.SetupConfig(
 		file.NewSource(file.WithPath(configYml)),
@@ -177,6 +182,7 @@ func run() error {
 		gin.SetMode(gin.ReleaseMode)
 	}
 	buildRouter()
+	reportGeneratorWriteRoutes()
 
 	srv := &http.Server{
 		Addr:         fmt.Sprintf("%s:%d", config.ApplicationConfig.Host, config.ApplicationConfig.Port),
@@ -356,6 +362,39 @@ const (
 	kubernetesGraceSeconds = 30
 )
 
+// reportGeneratorWriteRoutes says whether this process serves the code
+// generator's writing endpoints, and to whom.
+//
+// The endpoints are gated on the mode, and the shipped configuration says dev -
+// so the deployment most likely to be exposed is the one that changed nothing,
+// and the one least likely to go looking. Silence there would leave the gate
+// technically correct and practically useless.
+//
+// Nothing is said in demo mode. The routes are registered, but DemoEvn refuses
+// all three by name, so a warning would describe an exposure that is not there.
+func reportGeneratorWriteRoutes() {
+	if !generatorWriteRoutesNeedWarning() {
+		return
+	}
+	log.Warnf("the code generator's writing endpoints are served in mode %q: "+
+		"/api/v1/gen/{toproject,apitofile,todb} write Go and Vue source onto this host and rows "+
+		"into this database, and they are in CasbinExclude, so any account that can log in may "+
+		"call them. Set application.mode to prod or test on anything that is not a workstation, "+
+		"then restart: these routes were registered at start-up and a configuration reload does "+
+		"not rebuild them.",
+		config.ApplicationConfig.Mode)
+}
+
+// generatorWriteRoutesNeedWarning reports whether there is an exposure to warn
+// about: the endpoints are served, and nothing else is refusing them.
+//
+// Split from the logging so the decision can be tested. A warning nobody can
+// make fire is indistinguishable from no warning at all, and this one exists
+// precisely for the case nobody is looking at.
+func generatorWriteRoutesNeedWarning() bool {
+	return otherrouter.GenWriteRoutesEnabled() && config.ApplicationConfig.Mode != "demo"
+}
+
 // reportShutdownBudget states what a shutdown will spend and whether it fits.
 //
 // The sum is taken from the resolved values, not from the configuration file:

cmd/migrate/migration/init.go

@@ -185,6 +185,28 @@ func (e *Migration) mergedEntries() map[string]versionEntry {
 	return out
 }
 
+// RegisteredVersions returns every migration version this binary registers,
+// sorted, without touching a database.
+//
+// Status answers a richer question - what is registered, what is applied, and
+// what is applied while nothing registers it - and needs a database to do it.
+// This is the half that can be asked of the process alone, which is what a
+// readiness check needs: the check holds the databases it is asking about, and
+// reusing Status would mean calling SetDb from a request handler, writing this
+// package's shared state from a request path.
+func (e *Migration) RegisteredVersions() []string {
+	all := e.mergedEntries()
+	out := make([]string, 0, len(all))
+	for k := range all {
+		out = append(out, k)
+	}
+	sort.Strings(out)
+	return out
+}
+
+// RegisteredVersions reports what the process-wide registry holds.
+func RegisteredVersions() []string { return Migrate.RegisteredVersions() }
+
 // StatusEntry is one row of migrate status.
 type StatusEntry struct {
 	AppCode    string

common/health/health.go

@@ -41,6 +41,7 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -68,6 +69,55 @@ type Check struct {
 	Err  string `json:"error,omitempty"`
 }
 
+// extra are checks a host registers that this package cannot make itself.
+//
+// The direction is why this exists. Whether the schema matches what the binary
+// expects is answered by the migration registry, which lives under cmd/ - and
+// common/ has never imported cmd/. Rather than start, the host registers the
+// check from where both are already in scope.
+var (
+	extraMu sync.RWMutex
+	extra   []namedCheck
+)
+
+type namedCheck struct {
+	name string
+	fn   func(context.Context) error
+}
+
+// Register adds a check to what Ready asks.
+//
+// It panics on a duplicate name rather than replacing or appending: two checks
+// under one name make the failing one impossible to identify from the response,
+// and registering the same one twice is a wiring mistake worth hearing about at
+// start-up rather than never.
+func Register(name string, fn func(context.Context) error) {
+	if name == "" {
+		panic("health: a registered check needs a name")
+	}
+	if fn == nil {
+		panic("health: check " + name + " is nil")
+	}
+	extraMu.Lock()
+	defer extraMu.Unlock()
+	for _, c := range extra {
+		if c.name == name {
+			panic("health: check " + name + " is already registered")
+		}
+	}
+	extra = append(extra, namedCheck{name: name, fn: fn})
+}
+
+// registered returns the checks a host has added, copied so that Ready is not
+// iterating the slice while another goroutine appends to it.
+func registered() []namedCheck {
+	extraMu.RLock()
+	defer extraMu.RUnlock()
+	out := make([]namedCheck, len(extra))
+	copy(out, extra)
+	return out
+}
+
 // Ready asks every dependency this process cannot serve a request without.
 //
 // The queue is deliberately absent. Nothing on AdapterQueue answers "are you
@@ -75,10 +125,14 @@ type Check struct {
 // a queue that is down degrades logging rather than stopping requests - which
 // is a reason to alert, not a reason to leave the load balancer pool.
 func Ready(ctx context.Context) []Check {
-	return []Check{
+	checks := []Check{
 		safely("database", func() error { return pingDB(ctx) }),
 		safely("cache", probeCache),
 	}
+	for _, c := range registered() {
+		checks = append(checks, safely(c.name, func() error { return c.fn(ctx) }))
+	}
+	return checks
 }
 
 // safely turns a panic into a failed check.

common/health/register_test.go

@@ -0,0 +1,125 @@
+package health
+
+import (
+	"context"
+	"errors"
+	"testing"
+)
+
+// isolate empties the registered checks and puts them back, so one test in
+// this package cannot decide what the next one sees.
+func isolate(t *testing.T) {
+	t.Helper()
+	extraMu.Lock()
+	previous := extra
+	extra = nil
+	extraMu.Unlock()
+	t.Cleanup(func() {
+		extraMu.Lock()
+		extra = previous
+		extraMu.Unlock()
+	})
+}
+
+func findCheck(checks []Check, name string) (Check, bool) {
+	for _, c := range checks {
+		if c.Name == name {
+			return c, true
+		}
+	}
+	return Check{}, false
+}
+
+// A registered check has to reach Ready's answer, or the host has wired
+// something that never gets asked.
+func TestARegisteredCheckIsAsked(t *testing.T) {
+	isolate(t)
+	Register("schema", func(context.Context) error { return errors.New("two behind") })
+
+	checks := Ready(context.Background())
+	c, ok := findCheck(checks, "schema")
+	if !ok {
+		t.Fatal("Ready did not ask the registered check")
+	}
+	if c.OK {
+		t.Error("the check returned an error and was still reported OK")
+	}
+	if c.Err != "two behind" {
+		t.Errorf("Err = %q, want the check's own message", c.Err)
+	}
+	if Healthy(checks) {
+		t.Error("Healthy said yes while a registered check was failing")
+	}
+}
+
+// The context Ready is given has to reach the check: it carries the probe's
+// deadline, and a check that ignores it can hold the handler past it.
+func TestTheRegisteredCheckIsGivenReadysContext(t *testing.T) {
+	isolate(t)
+	type key struct{}
+	Register("ctx", func(ctx context.Context) error {
+		if ctx.Value(key{}) != "carried" {
+			return errors.New("the check was handed a different context")
+		}
+		return nil
+	})
+
+	checks := Ready(context.WithValue(context.Background(), key{}, "carried"))
+	c, ok := findCheck(checks, "ctx")
+	if !ok {
+		t.Fatal("the registered check was not asked")
+	}
+	if !c.OK {
+		t.Errorf("check failed: %s", c.Err)
+	}
+}
+
+// A check that panics must not take the process down through the probe, the
+// same guarantee the built-in checks have.
+func TestARegisteredCheckThatPanicsFailsRatherThanCrashes(t *testing.T) {
+	isolate(t)
+	Register("boom", func(context.Context) error { panic("registry unreachable") })
+
+	checks := Ready(context.Background())
+	c, ok := findCheck(checks, "boom")
+	if !ok {
+		t.Fatal("the registered check was not asked")
+	}
+	if c.OK {
+		t.Error("a panicking check was reported OK")
+	}
+}
+
+func TestRegisteringTheSameNameTwicePanics(t *testing.T) {
+	isolate(t)
+	Register("dup", func(context.Context) error { return nil })
+
+	defer func() {
+		if recover() == nil {
+			t.Error("registering a duplicate name did not panic; two checks under " +
+				"one name make the failing one impossible to identify")
+		}
+	}()
+	Register("dup", func(context.Context) error { return nil })
+}
+
+func TestRegisterRefusesAnEmptyNameOrNilCheck(t *testing.T) {
+	isolate(t)
+	for _, tc := range []struct {
+		name string
+		fn   func(context.Context) error
+		why  string
+	}{
+		{"", func(context.Context) error { return nil }, "empty name"},
+		{"nilfn", nil, "nil function"},
+	} {
+		func() {
+			defer func() {
+				if recover() == nil {
+					t.Errorf("%s did not panic", tc.why)
+				}
+			}()
+			Register(tc.name, tc.fn)
+		}()
+	}
+}

common/storage/initialize.go

@@ -8,10 +8,12 @@
 package storage
 
 import (
+	"context"
 	"log"
 	"sync"
 
 	"github.com/go-admin-team/go-admin-core/v2/captcha"
+	corelog "github.com/go-admin-team/go-admin-core/v2/logger"
 	"github.com/go-admin-team/go-admin-core/v2/sdk"
 	"github.com/go-admin-team/go-admin-core/v2/sdk/config"
 )
@@ -21,6 +23,7 @@ func Setup() {
 	setupCache()
 	setupCaptcha()
 	setupQueue()
+	registerQueueDrain()
 }
 
 func setupCache() {
@@ -41,8 +44,123 @@ var (
 	// shut it down, and counted so a consumer can tell one from the next.
 	installed    interface{ Shutdown() }
 	installedGen uint64
+	// drainRegistered records that the BeforeExit callback is on the runtime,
+	// so that a reload does not add another one.
+	drainRegistered bool
 )
 
+// setShutdown is sdk.Runtime.SetShutdown, indirected so that registering can
+// be observed.
+//
+// It has to be: the runtime does not report how many callbacks a phase holds,
+// and shutdownQueue takes the adapter on its first run, so every registration
+// after the first returns immediately and changes nothing anybody can see. A
+// reload adding one callback per round would therefore be invisible from the
+// outside - which is exactly how it would survive.
+var setShutdown = func(f func(context.Context)) { sdk.Runtime.SetShutdown(f) }
+
+// registerQueueDrain puts shutdownQueue on the BeforeExit phase, once.
+//
+// Setup is one of the callbacks bootstrap.SetupConfig re-runs on every
+// configuration change, so registering from it without a guard would leave one
+// callback per reload - each shutting down the same adapter, each reported
+// separately when the budget runs out.
+//
+// A flag under the existing mutex rather than a sync.Once: the tests in this
+// package already save and restore installed and installedGen to keep one test
+// from deciding what the next one sees, and a sync.Once cannot be put back.
+func registerQueueDrain() {
+	queueMu.Lock()
+	first := !drainRegistered
+	drainRegistered = true
+	queueMu.Unlock()
+
+	if first {
+		setShutdown(shutdownQueue)
+	}
+}
+
+// drainedInTime shuts q down and reports whether it finished before ctx expired.
+func drainedInTime(ctx context.Context, q interface{ Shutdown() }) bool {
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		q.Shutdown()
+	}()
+	return finishedBeforeDeadline(ctx, done)
+}
+
+// finishedBeforeDeadline waits for done or for ctx, and resolves a tie in
+// favour of done.
+//
+// The tie is the reason this is a function of its own rather than one select
+// inline. Both channels can be ready when the select runs, select picks at
+// random among ready cases, and so a single look reports an overrun for a
+// drain that completed - about half the times it lands there, which is exactly
+// often enough to be dismissed as noise. core's own RunShutdown re-checks for
+// this reason.
+//
+// Taking channels rather than a queue is what makes it testable: a closed done
+// and an expired ctx can be handed in together, which is the state a race
+// would otherwise have to be caught in.
+func finishedBeforeDeadline(ctx context.Context, done <-chan struct{}) bool {
+	select {
+	case <-done:
+		return true
+	case <-ctx.Done():
+	}
+
+	select {
+	case <-done:
+		return true
+	default:
+		return false
+	}
+}
+
+// shutdownQueue drains the queue this package installed, on the way out.
+//
+// Nothing used to. core's Memory.Shutdown closes the queue and waits for every
+// consumer to finish what it is holding, and the legacy adapter cancels its
+// context and closes the underlying queue - but neither ran at exit, so the
+// process left with the login log, the operation log and the API sync still
+// buffered, and left reporting success.
+//
+// The adapter is read here rather than captured at registration because a
+// reload replaces it. Registration happens once per process; this runs against
+// whatever is current when the signal arrives.
+//
+// Only an adapter this package installed. sdk.Runtime.GetQueueAdapter never
+// returns nil - with no queue section configured the runtime wraps its own
+// fallback queue - so going through that accessor would shut down a queue this
+// package neither built nor started.
+//
+// The adapter is taken, not read: after this the package owns nothing, so a
+// reload arriving mid-shutdown builds a new one instead of being handed a
+// closed one to shut down again. Both implementations tolerate a second
+// Shutdown, so this is about who owns it rather than about a crash.
+func shutdownQueue(ctx context.Context) {
+	queueMu.Lock()
+	q := installed
+	installed = nil
+	queueMu.Unlock()
+
+	if q == nil {
+		return
+	}
+
+	if drainedInTime(ctx, q) {
+		corelog.Info("queue: drained")
+		return
+	}
+	// The wait is what the budget bounds, not the work: Shutdown takes no
+	// context and is still running in that goroutine. Saying so here names what
+	// is being lost, which the generic overrun message cannot.
+	corelog.Warnf("queue: the shutdown budget ran out while the queue was still draining - " +
+		"whatever it had not delivered goes with the process. Raise extend.shutdown.cleanup " +
+		"if this recurs.")
+}
+
 // QueueGeneration reports how many times this package has installed a queue
 // adapter. It changes every time setupQueue builds a new one, which is on
 // every configuration reload, and stays 0 for as long as the configuration has

common/storage/queue_drain_test.go

@@ -0,0 +1,333 @@
+package storage
+
+import (
+	"context"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/go-admin-team/go-admin-core/v2/sdk"
+	"github.com/go-admin-team/go-admin-core/v2/sdk/config"
+	"github.com/go-admin-team/go-admin-core/v2/sdk/runtime"
+	corestorage "github.com/go-admin-team/go-admin-core/v2/storage"
+)
+
+// countingQueue stands in for an installed adapter. Only Shutdown is exercised
+// - the drain callback never publishes or consumes - so the rest of
+// AdapterQueue is deliberately absent: `installed` is typed on Shutdown alone,
+// and widening the fake would only invite it to be used for something else.
+type countingQueue struct {
+	calls atomic.Int32
+	block chan struct{}
+
+	// started is closed on the way into Shutdown, so a test can wait for the
+	// call rather than assume the goroutine that makes it was scheduled. The
+	// caller returns on its own deadline while Shutdown is still running, so
+	// reading calls straight after that return is a race with the increment.
+	startOnce sync.Once
+	started   chan struct{}
+}
+
+func newCountingQueue() *countingQueue {
+	return &countingQueue{started: make(chan struct{})}
+}
+
+func (q *countingQueue) Shutdown() {
+	q.startOnce.Do(func() { close(q.started) })
+	q.calls.Add(1)
+	if q.block != nil {
+		<-q.block
+	}
+}
+
+// waitStarted blocks until Shutdown has been entered, or fails the test.
+func (q *countingQueue) waitStarted(t *testing.T) {
+	t.Helper()
+	select {
+	case <-q.started:
+	case <-time.After(5 * time.Second):
+		t.Fatal("Shutdown was never called")
+	}
+}
+
+// isolate gives the test its own runtime and its own view of what this package
+// has installed, and puts the process-wide state back afterwards.
+//
+// Same isolation as TestSetupBumpsTheQueueGenerationOnEveryReload, plus
+// drainRegistered: it is what stops a reload registering a second callback, so
+// leaving it set would make every later test in this binary see a package that
+// has already registered.
+func isolate(t *testing.T) {
+	t.Helper()
+
+	prevQ, prevC := config.QueueConfig, config.CacheConfig
+	prevRuntime := sdk.Runtime
+	queueMu.Lock()
+	prevInstalled, prevGen, prevRegistered := installed, installedGen, drainRegistered
+	queueMu.Unlock()
+
+	t.Cleanup(func() {
+		config.QueueConfig, config.CacheConfig = prevQ, prevC
+		sdk.Runtime = prevRuntime
+		queueMu.Lock()
+		installed, installedGen, drainRegistered = prevInstalled, prevGen, prevRegistered
+		queueMu.Unlock()
+	})
+
+	sdk.Runtime = runtime.NewConfig()
+	queueMu.Lock()
+	installed, drainRegistered = nil, false
+	queueMu.Unlock()
+}
+
+// setInstalled puts a fake where setupQueue would have left the real adapter.
+//
+// Legitimate because the callback reads `installed` when it runs rather than
+// capturing it at registration - that is the property that lets a reload
+// replace the adapter and still have the right one drained.
+func setInstalled(q interface{ Shutdown() }) {
+	queueMu.Lock()
+	installed = q
+	queueMu.Unlock()
+}
+
+func currentInstalled() interface{ Shutdown() } {
+	queueMu.Lock()
+	defer queueMu.Unlock()
+	return installed
+}
+
+// Issue #911: nothing shut the queue down at exit, so whatever was buffered
+// went with the process.
+//
+// This is the half that matters most - a callback is on BeforeExit and it
+// reaches the adapter this package installed. It says nothing about how many
+// times the callback was registered; see the test below for that.
+func TestSetupPutsTheQueueDrainOnBeforeExit(t *testing.T) {
+	isolate(t)
+	config.CacheConfig = &config.Cache{Memory: struct{}{}}
+	config.QueueConfig = &config.Queue{Memory: &config.QueueMemory{PoolSize: 10}}
+
+	Setup()
+	Setup()
+	Setup()
+
+	q := newCountingQueue()
+	setInstalled(q)
+
+	if err := sdk.Runtime.RunShutdown(context.Background()); err != nil {
+		t.Fatalf("RunShutdown: %v", err)
+	}
+
+	if got := q.calls.Load(); got != 1 {
+		t.Errorf("Shutdown called %d times, want 1 - 0 means nothing registered the drain", got)
+	}
+}
+
+// Setup is re-run on every configuration change, so registering from it has to
+// be guarded: a callback per reload would leave the shutdown phase holding a
+// row of identical entries, each timed and each eligible to be named as the one
+// that overran the budget.
+//
+// Counted at the seam rather than through the effect. The test above cannot see
+// this - shutdownQueue takes the adapter on its first run, so the second and
+// third callbacks find nothing and return, and three registrations produce
+// exactly the same observable result as one. That is a good property of the
+// callback and a blind spot for any test that goes through it.
+func TestSetupRegistersTheDrainOncePerProcessHoweverManyReloads(t *testing.T) {
+	isolate(t)
+
+	previous := setShutdown
+	t.Cleanup(func() { setShutdown = previous })
+	registrations := 0
+	setShutdown = func(func(context.Context)) { registrations++ }
+
+	config.CacheConfig = &config.Cache{Memory: struct{}{}}
+	config.QueueConfig = &config.Queue{Memory: &config.QueueMemory{PoolSize: 10}}
+
+	Setup()
+	Setup()
+	Setup()
+
+	if registrations != 1 {
+		t.Errorf("three reloads registered the drain %d times, want 1", registrations)
+	}
+}
+
+// The drain reaches the adapter that is current when the signal arrives, not
+// one captured while wiring up. A reload replaces the adapter, and draining the
+// one that was installed at start-up would drain something nobody has published
+// to since.
+func TestTheDrainRunsAgainstTheAdapterInstalledLast(t *testing.T) {
+	isolate(t)
+	config.CacheConfig = &config.Cache{Memory: struct{}{}}
+	config.QueueConfig = &config.Queue{Memory: &config.QueueMemory{PoolSize: 10}}
+	Setup()
+
+	first, second := newCountingQueue(), newCountingQueue()
+	setInstalled(first)
+	setInstalled(second)
+
+	if err := sdk.Runtime.RunShutdown(context.Background()); err != nil {
+		t.Fatalf("RunShutdown: %v", err)
+	}
+
+	if first.calls.Load() != 0 {
+		t.Error("the adapter that was replaced was shut down; the callback captured it instead of " +
+			"reading it when it ran")
+	}
+	if second.calls.Load() != 1 {
+		t.Errorf("the current adapter was shut down %d times, want 1", second.calls.Load())
+	}
+}
+
+// Nothing installed is the shipped default: settings.yml has no queue section,
+// so setupQueue returns early and the runtime's own fallback queue is what
+// callers get. Shutting that down would close a queue this package neither
+// built nor started.
+func TestTheDrainDoesNothingWhenThisPackageInstalledNothing(t *testing.T) {
+	isolate(t)
+	config.CacheConfig = &config.Cache{Memory: struct{}{}}
+	config.QueueConfig = &config.Queue{}
+	Setup()
+
+	if currentInstalled() != nil {
+		t.Fatal("an empty queue configuration installed an adapter, so this test asserts nothing")
+	}
+	if err := sdk.Runtime.RunShutdown(context.Background()); err != nil {
+		t.Fatalf("RunShutdown: %v", err)
+	}
+}
+
+// The budget bounds the wait, not the work. A consumer that never finishes must
+// not hold the process past its grace period - SIGKILL would arrive mid-write
+// instead of at a point of the process's choosing.
+func TestTheDrainStopsWaitingWhenTheBudgetIsGone(t *testing.T) {
+	isolate(t)
+
+	blocked := newCountingQueue()
+	blocked.block = make(chan struct{})
+	t.Cleanup(func() { close(blocked.block) })
+	setInstalled(blocked)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
+	defer cancel()
+
+	returned := make(chan struct{})
+	go func() {
+		defer close(returned)
+		shutdownQueue(ctx)
+	}()
+
+	select {
+	case <-returned:
+	case <-time.After(5 * time.Second):
+		t.Fatal("shutdownQueue did not return after its context expired - it waits on a Shutdown " +
+			"that takes no context, so the wait has to be bounded here")
+	}
+
+	// Waited for rather than read straight after the return: Shutdown runs on a
+	// goroutine that the caller does not join, so the increment is not ordered
+	// against the caller giving up on its deadline.
+	blocked.waitStarted(t)
+	if blocked.calls.Load() != 1 {
+		t.Errorf("Shutdown called %d times, want 1 - the drain has to be attempted even when it "+
+			"cannot be waited out", blocked.calls.Load())
+	}
+}
+
+// After the drain this package owns nothing. A reload arriving mid-shutdown
+// then builds a new adapter rather than being handed a closed one as its
+// `previous` to shut down again.
+func TestTheDrainGivesUpOwnershipOfTheAdapter(t *testing.T) {
+	isolate(t)
+	setInstalled(newCountingQueue())
+
+	shutdownQueue(context.Background())
+
+	if got := currentInstalled(); got != nil {
+		t.Errorf("installed is %T after the drain, want nil", got)
+	}
+}
+
+// runtimeQueue is a full AdapterQueue, so it can be handed to the runtime
+// rather than only to this package's own record of what it installed.
+type runtimeQueue struct {
+	countingQueue
+}
+
+func (q *runtimeQueue) String() string                            { return "runtime-fake" }
+func (q *runtimeQueue) Append(corestorage.Messager) error         { return nil }
+func (q *runtimeQueue) Register(string, corestorage.ConsumerFunc) {}
+func (q *runtimeQueue) Run()                                      {}
+
+// The drain must not reach a queue this package did not install.
+//
+// sdk.Runtime.GetQueueAdapter never returns nil: with no queue section
+// configured it wraps the runtime's own fallback, and the wrapper's Shutdown
+// forwards. Reaching for the accessor would therefore look like it worked and
+// would close a queue this package neither built nor started - the same shape
+// as the `if q := GetQueueAdapter(); q != nil` that setupQueue already had to
+// drop.
+//
+// The previous test's empty-configuration case cannot see this: it only checks
+// that RunShutdown returns, which it would either way.
+func TestTheDrainNeverReachesTheRuntimesOwnQueue(t *testing.T) {
+	isolate(t)
+
+	onTheRuntime := &runtimeQueue{countingQueue: *newCountingQueue()}
+	sdk.Runtime.SetQueueAdapter(onTheRuntime)
+
+	if currentInstalled() != nil {
+		t.Fatal("this package installed something, so the distinction under test is not set up")
+	}
+	if sdk.Runtime.GetQueueAdapter() == nil {
+		t.Fatal("the accessor returned nil, so it is no longer the trap this guards")
+	}
+
+	shutdownQueue(context.Background())
+
+	if got := onTheRuntime.calls.Load(); got != 0 {
+		t.Errorf("the runtime's queue was shut down %d times - the drain went through "+
+			"GetQueueAdapter instead of the adapter this package installed", got)
+	}
+}
+
+// A drain that finishes in the same instant the budget expires counts as
+// finished.
+//
+// Both channels are ready when the select runs, and select picks at random
+// among ready cases, so a single look reports an overrun for a drain that
+// completed - roughly half the times it lands here. The repetition is what
+// makes that visible: one iteration passes either way.
+func TestATieBetweenTheDeadlineAndTheDrainGoesToTheDrain(t *testing.T) {
+	expired, cancel := context.WithCancel(context.Background())
+	cancel()
+	<-expired.Done()
+
+	done := make(chan struct{})
+	close(done)
+
+	for i := 0; i < 1000; i++ {
+		if !finishedBeforeDeadline(expired, done) {
+			t.Fatalf("iteration %d of 1000: both the deadline and the drain were ready and the "+
+				"deadline won - a drain that completed is being reported as an overrun", i)
+		}
+	}
+}
+
+// The other side of it. A drain that really has not finished has to be
+// reported, or the warning never fires and the tie-break above has quietly
+// turned into "always say it drained".
+func TestADrainThatHasNotFinishedIsReportedAsAnOverrun(t *testing.T) {
+	expired, cancel := context.WithCancel(context.Background())
+	cancel()
+	<-expired.Done()
+
+	stillRunning := make(chan struct{}) // never closed
+
+	if finishedBeforeDeadline(expired, stillRunning) {
+		t.Error("an unfinished drain was reported as having finished in time")
+	}
+}

scripts/k8s/deploy.yml

@@ -23,10 +23,20 @@ metadata:
     version: v1
 spec:
   # One replica, and the drain window below buys nothing at one replica: there
-  # is nowhere to send the traffic this pod stops taking. Raising it needs one
-  # more change than the number - the volume below is shared by every replica,
-  # and the log path in settings.yml lives on it, so a second pod would append
-  # to the same rotating file.
+  # is nowhere to send the traffic this pod stops taking. Raising it needs two
+  # changes that are not this number:
+  #
+  # The volume below is shared by every replica, and the log path in
+  # settings.yml lives on it, so a second pod would append to the same
+  # rotating file.
+  #
+  # The job scheduler is per process while its handle on a job is one shared
+  # column. Startup runs `UPDATE sys_job SET entry_id = 0 WHERE entry_id > 0`
+  # across the whole table (app/jobs/jobbase.go), so a second pod erases the
+  # first pod's ids and writes its own, and every pod registers the whole
+  # enabled list in its own scheduler. Neither symptom logs anything: an
+  # enabled job fires once per pod, and stopping one from the UI removes an
+  # entry from the wrong process and still answers 200. See #915.
   replicas: 1
   selector:
     matchLabels: