test✅: add an end-to-end load test harness

Skipped unless GOADMIN_BENCH_ADDR points at a running server, so `go test
./...` is unaffected.

Reports latency percentiles rather than an average, which is what capacity
planning needs, and a status-code distribution - that last part is how the rate
limiter's 200-on-rejection was found, since throughput alone looked excellent
while nothing reached a handler.

Includes a routing-floor control case. When a business endpoint matches it, the
measurement has stopped describing the endpoint and started describing the
transport, or the load generator when both share a machine.

test/loadtest/loadtest_test.go

@@ -0,0 +1,367 @@
+// Package loadtest measures what one go-admin process sustains over HTTP.
+//
+// It is skipped unless GOADMIN_BENCH_ADDR points at a running server, so
+// `go test ./...` is unaffected. Start a server and run:
+//
+//	GOADMIN_BENCH_ADDR=http://127.0.0.1:8000 go test ./test/loadtest/ -v -run TestLoadProfile
+//
+// Unlike a Go benchmark this reports latency percentiles, which is what
+// capacity planning needs: an average hides the tail that users actually feel.
+//
+// Two caveats when reading the numbers. The load generator runs on the same
+// machine as the server unless GOADMIN_BENCH_ADDR is remote, so both compete
+// for the same cores - a split deployment measures higher. And the figures
+// describe the configured backend: sqlite and MySQL differ by more than the
+// framework does.
+package loadtest
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+const (
+	addrEnv  = "GOADMIN_BENCH_ADDR"
+	tokenEnv = "GOADMIN_BENCH_TOKEN"
+	userEnv  = "GOADMIN_BENCH_USER"
+	passEnv  = "GOADMIN_BENCH_PASS"
+
+	// Each concurrency level runs for this long. Long enough to get past
+	// connection setup and let the scheduler settle, short enough that the
+	// whole sweep stays interactive.
+	levelDuration = 3 * time.Second
+)
+
+// concurrencyLevels sweeps from a single client to well past core count, so
+// the point where added concurrency stops buying throughput is visible rather
+// than assumed. Peak throughput and peak concurrency are not the same number:
+// past the peak a server takes more work than it can finish and both
+// throughput and latency get worse, so the sweep has to bracket the turn
+// rather than stop at the top.
+//
+// GOADMIN_BENCH_LEVELS overrides it, comma separated.
+var concurrencyLevels = parseLevels(os.Getenv("GOADMIN_BENCH_LEVELS"), []int{1, 2, 4, 8, 16, 32, 64, 128, 256, 512})
+
+func parseLevels(spec string, fallback []int) []int {
+	if spec == "" {
+		return fallback
+	}
+	out := make([]int, 0, 8)
+	for _, f := range strings.Split(spec, ",") {
+		n, err := strconv.Atoi(strings.TrimSpace(f))
+		if err != nil || n <= 0 {
+			continue
+		}
+		out = append(out, n)
+	}
+	if len(out) == 0 {
+		return fallback
+	}
+	return out
+}
+
+func addr(t testing.TB) string {
+	t.Helper()
+	a := os.Getenv(addrEnv)
+	if a == "" {
+		t.Skipf("%s not set; skipping load test", addrEnv)
+	}
+	return a
+}
+
+// newClient returns a client whose pool is large enough that the generator
+// does not become the bottleneck it is trying to measure.
+func newClient(maxConns int) *http.Client {
+	return &http.Client{
+		Timeout: 30 * time.Second,
+		Transport: &http.Transport{
+			MaxIdleConns:        maxConns * 2,
+			MaxIdleConnsPerHost: maxConns * 2,
+			MaxConnsPerHost:     maxConns * 2,
+			IdleConnTimeout:     90 * time.Second,
+			DisableCompression:  true,
+		},
+	}
+}
+
+// result is one completed request.
+type result struct {
+	latency time.Duration
+	err     bool
+	status  int
+}
+
+// report is the summary of one concurrency level.
+type report struct {
+	concurrency        int
+	total              int64
+	failed             int64
+	elapsed            time.Duration
+	p50, p95, p99, max time.Duration
+	statuses           map[int]int64
+}
+
+func (r report) qps() float64 {
+	if r.elapsed == 0 {
+		return 0
+	}
+	return float64(r.total) / r.elapsed.Seconds()
+}
+
+func (r report) String() string {
+	codes := make([]int, 0, len(r.statuses))
+	for c := range r.statuses {
+		codes = append(codes, c)
+	}
+	sort.Ints(codes)
+	dist := make([]string, 0, len(codes))
+	for _, c := range codes {
+		dist = append(dist, fmt.Sprintf("%d:%d", c, r.statuses[c]))
+	}
+	return fmt.Sprintf("c=%-4d %9.0f req/s  p50=%-9s p95=%-9s p99=%-9s max=%-9s failed=%-7d %s",
+		r.concurrency, r.qps(),
+		r.p50.Round(time.Microsecond), r.p95.Round(time.Microsecond),
+		r.p99.Round(time.Microsecond), r.max.Round(time.Microsecond), r.failed,
+		strings.Join(dist, " "))
+}
+
+// drive runs `concurrency` workers against req for levelDuration and collects
+// every latency. Bodies are drained and closed - skipping that silently caps
+// throughput at the point connections stop being reused.
+func drive(t testing.TB, concurrency int, want int, mk func() *http.Request) report {
+	t.Helper()
+
+	client := newClient(concurrency)
+	defer client.CloseIdleConnections()
+
+	ctx, cancel := context.WithTimeout(context.Background(), levelDuration)
+	defer cancel()
+
+	var (
+		mu       sync.Mutex
+		samples  []time.Duration
+		statuses = map[int]int64{}
+		failed   atomic.Int64
+		total    atomic.Int64
+		wg       sync.WaitGroup
+	)
+
+	start := time.Now()
+	for i := 0; i < concurrency; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			local := make([]time.Duration, 0, 1024)
+			localStatus := map[int]int64{}
+			for ctx.Err() == nil {
+				req := mk()
+				t0 := time.Now()
+				resp, err := client.Do(req.WithContext(ctx))
+				d := time.Since(t0)
+				if err != nil {
+					if ctx.Err() != nil {
+						break
+					}
+					failed.Add(1)
+					total.Add(1)
+					continue
+				}
+				_, _ = io.Copy(io.Discard, resp.Body)
+				_ = resp.Body.Close()
+				localStatus[resp.StatusCode]++
+				if resp.StatusCode != want {
+					failed.Add(1)
+				}
+				total.Add(1)
+				local = append(local, d)
+			}
+			mu.Lock()
+			samples = append(samples, local...)
+			for code, n := range localStatus {
+				statuses[code] += n
+			}
+			mu.Unlock()
+		}()
+	}
+	wg.Wait()
+	elapsed := time.Since(start)
+
+	sort.Slice(samples, func(i, j int) bool { return samples[i] < samples[j] })
+	r := report{
+		concurrency: concurrency,
+		total:       total.Load(),
+		failed:      failed.Load(),
+		elapsed:     elapsed,
+		statuses:    statuses,
+	}
+	if n := len(samples); n > 0 {
+		r.p50 = samples[n*50/100]
+		r.p95 = samples[min(n*95/100, n-1)]
+		r.p99 = samples[min(n*99/100, n-1)]
+		r.max = samples[n-1]
+	}
+	return r
+}
+
+func min(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}
+
+// login obtains a token. GOADMIN_BENCH_TOKEN short-circuits it, which is how a
+// server in prod mode is reached - there the login endpoint demands a captcha.
+func login(t testing.TB, base string) string {
+	t.Helper()
+	if tok := os.Getenv(tokenEnv); tok != "" {
+		return tok
+	}
+
+	user, pass := os.Getenv(userEnv), os.Getenv(passEnv)
+	if user == "" {
+		user, pass = "admin", "123456"
+	}
+
+	body, _ := json.Marshal(map[string]string{
+		"username": user,
+		"password": pass,
+		"code":     "0",
+		"uuid":     "0",
+	})
+	resp, err := http.Post(base+"/api/v1/login", "application/json", bytes.NewReader(body))
+	if err != nil {
+		t.Fatalf("login request failed: %v", err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+
+	raw, _ := io.ReadAll(resp.Body)
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("login returned %d: %s\n(a server in prod mode requires a captcha; set %s instead)",
+			resp.StatusCode, raw, tokenEnv)
+	}
+	var out struct {
+		Token string `json:"token"`
+	}
+	if err := json.Unmarshal(raw, &out); err != nil || out.Token == "" {
+		t.Fatalf("no token in login response: %s", raw)
+	}
+	return out.Token
+}
+
+// TestLoadProfile sweeps concurrency against three endpoints chosen for what
+// they isolate:
+//
+//   - captcha: no auth, no business query. The routing and image-generation
+//     floor.
+//   - dept list: the full authenticated path - JWT parse, casbin check, data
+//     permission scope, database read. This is what a real page costs.
+//   - login: bcrypt. Deliberately slow, and the one endpoint whose ceiling is
+//     set by design rather than by the framework.
+func TestLoadProfile(t *testing.T) {
+	base := addr(t)
+	token := login(t, base)
+
+	cases := []struct {
+		name string
+		want int
+		mk   func() *http.Request
+	}{
+		{
+			// The control. An unrouted path exercises the HTTP stack, gin's
+			// tree lookup and nothing else, so it bounds every other row here.
+			// When a business endpoint reaches this number, the measurement has
+			// stopped describing the endpoint and started describing the
+			// transport - or the load generator, when both share a machine.
+			name: "404 (http+routing floor)",
+			want: 404,
+			mk: func() *http.Request {
+				req, _ := http.NewRequest(http.MethodGet, base+"/api/v1/__no_such_route__", nil)
+				return req
+			},
+		},
+		{
+			// The framework on its own: global middleware chain, route lookup,
+			// and a handler that only sets a status. No database, no cache.
+			// Against the 404 row this isolates what the chain costs; against
+			// the rows below it, what the business path adds.
+			//
+			// Numbers from any endpoint that touches a database describe the
+			// database, the driver and the pool as much as the framework - the
+			// MySQL sweeps here moved from collapsing at c=64 to 19k req/s at
+			// c=512 on a pool setting alone, with the framework untouched.
+			name: "health (framework only, no db)",
+			want: 200,
+			mk: func() *http.Request {
+				req, _ := http.NewRequest(http.MethodGet, base+"/api/v1/health", nil)
+				return req
+			},
+		},
+		{
+			name: "captcha (no auth)",
+			want: 200,
+			mk: func() *http.Request {
+				req, _ := http.NewRequest(http.MethodGet, base+"/api/v1/captcha", nil)
+				return req
+			},
+		},
+		{
+			name: "dept list (jwt+casbin+db)",
+			want: 200,
+			mk: func() *http.Request {
+				req, _ := http.NewRequest(http.MethodGet, base+"/api/v1/dept", nil)
+				req.Header.Set("Authorization", "Bearer "+token)
+				return req
+			},
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			for _, c := range concurrencyLevels {
+				t.Log(drive(t, c, tc.want, tc.mk))
+			}
+		})
+	}
+}
+
+// TestLoginThroughput is separated because bcrypt saturates the CPU: running
+// it alongside the others would distort them. It also writes a login-log row
+// per attempt when logger.enableddb is on, so the number moves with that
+// setting.
+func TestLoginThroughput(t *testing.T) {
+	base := addr(t)
+	if os.Getenv(tokenEnv) != "" {
+		t.Skip("token supplied; login endpoint presumably needs a captcha")
+	}
+
+	user, pass := os.Getenv(userEnv), os.Getenv(passEnv)
+	if user == "" {
+		user, pass = "admin", "123456"
+	}
+	body, _ := json.Marshal(map[string]string{
+		"username": user, "password": pass, "code": "0", "uuid": "0",
+	})
+
+	mk := func() *http.Request {
+		req, _ := http.NewRequest(http.MethodPost, base+"/api/v1/login", bytes.NewReader(body))
+		req.Header.Set("Content-Type", "application/json")
+		return req
+	}
+
+	for _, c := range []int{1, 4, 8, 16, 32, 64, 128} {
+		t.Log(drive(t, c, http.StatusOK, mk))
+	}
+}