|
|
@@ -0,0 +1,232 @@
|
|
|
+package main
|
|
|
+
|
|
|
+// =============================================================================
|
|
|
+// idx_crash_chaos — 二级索引一致性断电混沌
|
|
|
+//
|
|
|
+// 针对生产事故 (DockerManager): kill 后出现"行在盘上、二级索引永远查不到"的
|
|
|
+// 半操作固化。根因是脏页阈值触发的中途提交把"插入一行 (数据页+索引页)"切成
|
|
|
+// 两笔 WAL。本压测就是那个事故形态的故障注入:
|
|
|
+//
|
|
|
+// 子进程: 建带二级索引的表, 关掉防抖、脏页阈值调到 1 (每写一页都顶爆阈值,
|
|
|
+// 提交压力拉满), 然后不停单行插入 —— 不开事务、不手动 Commit。
|
|
|
+// 父进程: 随机 20~250ms 强杀 → 重开 (WAL 恢复) → 逐行校验【反向一致性】:
|
|
|
+// 全表扫到的每一行, 用它自己的索引列值走二级索引等值查, 必须命中自己。
|
|
|
+// 少一条 = 事故复现 = 失败。
|
|
|
+//
|
|
|
+// 默认 60 轮 (TSDB_IDX_ROUNDS), 种子固定可复现 (TSDB_IDX_SEED)。
|
|
|
+// =============================================================================
|
|
|
+
|
|
|
+import (
|
|
|
+ "fmt"
|
|
|
+ "math/rand"
|
|
|
+ "os"
|
|
|
+ "os/exec"
|
|
|
+ "path/filepath"
|
|
|
+ "time"
|
|
|
+
|
|
|
+ "git.ycl.pub/yl32940/TsDB2/DB/core"
|
|
|
+ "git.ycl.pub/yl32940/TsDB2/DB/core/page"
|
|
|
+ "git.ycl.pub/yl32940/TsDB2/DB/core/types"
|
|
|
+)
|
|
|
+
|
|
|
+const (
|
|
|
+ envChild = "TSDB_IDX_CHAOS_CHILD"
|
|
|
+ envDB = "TSDB_IDX_DB"
|
|
|
+ envRounds = "TSDB_IDX_ROUNDS"
|
|
|
+ envSeed = "TSDB_IDX_SEED"
|
|
|
+ envWindow = "TSDB_IDX_WINDOW" // 子进程随机存活上限 ms (默认 250)
|
|
|
+ envThresh = "TSDB_IDX_THRESH" // 脏页阈值 (默认 1, 每页都触发)
|
|
|
+)
|
|
|
+
|
|
|
+func main() {
|
|
|
+ if os.Getenv(envChild) == "1" {
|
|
|
+ runChild()
|
|
|
+ return
|
|
|
+ }
|
|
|
+ runParent()
|
|
|
+}
|
|
|
+
|
|
|
+// =============================================================================
|
|
|
+// 子进程: 建库建表, 关防抖 + 阈值 1, 无限单行插入直到被强杀
|
|
|
+// =============================================================================
|
|
|
+
|
|
|
+func runChild() {
|
|
|
+ path := os.Getenv(envDB)
|
|
|
+ db, err := core.Open(path)
|
|
|
+ if err != nil {
|
|
|
+ os.Exit(2)
|
|
|
+ }
|
|
|
+ db.SetAutoCommitDelay(0) // 防抖关掉: 提交完全由阈值驱动
|
|
|
+ db.SetDirtyThreshold(envInt(envThresh, 1)) // 阈值 1: 每个写操作都在顶爆阈值
|
|
|
+ db.SetBufferPoolCapacity(4096)
|
|
|
+
|
|
|
+ tbl, err := db.CreateTable("t", []types.FieldInfo{
|
|
|
+ {FieldType: types.FieldType_Uint64, FieldName: []byte("id")},
|
|
|
+ {FieldType: types.FieldType_String, Flag: types.WidthCodeFor(16), FieldName: []byte("svc")},
|
|
|
+ })
|
|
|
+ if err != nil {
|
|
|
+ os.Exit(2)
|
|
|
+ }
|
|
|
+ if _, err := tbl.CreateIndex("pk", page.IndexTypePrimary, []byte{0}); err != nil {
|
|
|
+ os.Exit(2)
|
|
|
+ }
|
|
|
+ if _, err := tbl.CreateIndex("idx_svc", page.IndexTypeBTree, []byte{1}); err != nil {
|
|
|
+ os.Exit(2)
|
|
|
+ }
|
|
|
+
|
|
|
+ for i := uint64(1); ; i++ {
|
|
|
+ svc := fmt.Sprintf("s%013d", i) // 每行一个唯一值, 反查才精确
|
|
|
+ if _, err := tbl.InsertRowWithID(i, []types.FieldValue{
|
|
|
+ {FieldName: []byte("id"), Value: i},
|
|
|
+ {FieldName: []byte("svc"), Value: svc},
|
|
|
+ }); err != nil {
|
|
|
+ os.Exit(2)
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+// =============================================================================
|
|
|
+// 父进程: 循环 起子进程 → 随机强杀 → 重开 → 反向一致性校验
|
|
|
+// =============================================================================
|
|
|
+
|
|
|
+func runParent() {
|
|
|
+ rounds := envInt(envRounds, 60)
|
|
|
+ baseSeed := int64(envInt(envSeed, 1))
|
|
|
+ rng := rand.New(rand.NewSource(baseSeed))
|
|
|
+ killWindow := envInt(envWindow, 250)
|
|
|
+
|
|
|
+ fmt.Printf("二级索引断电混沌: %d 轮, 种子 %d (复现: %s=%d %s=轮数 go run main.go)\n",
|
|
|
+ rounds, baseSeed, envSeed, baseSeed, envRounds)
|
|
|
+ for r := 1; r <= rounds; r++ {
|
|
|
+ if err := oneRound(rng, killWindow); err != nil {
|
|
|
+ fmt.Printf("❌ 第 %d 轮失败 (种子 %d): %v\n", r, baseSeed, err)
|
|
|
+ fmt.Printf(" 复现: %s=%d %s=%d go run main.go\n", envSeed, baseSeed, envRounds, r)
|
|
|
+ os.Exit(1)
|
|
|
+ }
|
|
|
+ if r%10 == 0 {
|
|
|
+ fmt.Printf(" 进度 %d/%d 通过\n", r, rounds)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ fmt.Printf("✅ 二级索引断电混沌通过: %d 轮, 每行随机时刻强杀后 —— 存活行全部可被二级索引命中 (无半操作固化)\n", rounds)
|
|
|
+}
|
|
|
+
|
|
|
+func oneRound(rng *rand.Rand, killWindow int) error {
|
|
|
+ dir, err := os.MkdirTemp("", "tsdb-idx-chaos-")
|
|
|
+ if err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ path := filepath.Join(dir, "chaos.db")
|
|
|
+
|
|
|
+ exe, err := os.Executable()
|
|
|
+ if err != nil {
|
|
|
+ os.RemoveAll(dir)
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ cmd := exec.Command(exe)
|
|
|
+ cmd.Env = append(os.Environ(), envChild+"=1", envDB+"="+path,
|
|
|
+ envThresh+"="+os.Getenv(envThresh))
|
|
|
+ if err := cmd.Start(); err != nil {
|
|
|
+ os.RemoveAll(dir)
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ // 杀点均匀覆盖: 提交前 / WAL 写链中 / 页回写中 / 两 fsync 之间 / 操作页写入中途
|
|
|
+ time.Sleep(time.Duration(20+rng.Intn(killWindow-20)) * time.Millisecond)
|
|
|
+ _ = cmd.Process.Kill()
|
|
|
+ _ = cmd.Wait() // 回收句柄, 等文件锁释放
|
|
|
+
|
|
|
+ if _, err := os.Stat(path); os.IsNotExist(err) {
|
|
|
+ os.RemoveAll(dir)
|
|
|
+ return nil // 建文件前被杀, 无可验对象
|
|
|
+ }
|
|
|
+ defer os.RemoveAll(dir)
|
|
|
+ return verify(path, dir)
|
|
|
+}
|
|
|
+
|
|
|
+// =============================================================================
|
|
|
+// 反向一致性校验: 全表扫到的每一行, 用它自己的 svc 值走二级索引等值查,
|
|
|
+// 必须命中自己。少一条即"行在盘上、索引未命中"—— 事故复现。
|
|
|
+// =============================================================================
|
|
|
+
|
|
|
+func verify(path, dir string) error {
|
|
|
+ db, err := core.Open(path) // RecoverWAL + 孤儿 WAL 自愈在这里跑
|
|
|
+ if err != nil {
|
|
|
+ return fmt.Errorf("崩溃后重开失败: %w (现场: %s)", err, dir)
|
|
|
+ }
|
|
|
+ defer db.Close()
|
|
|
+
|
|
|
+ tbl, ok := db.FindTable("t")
|
|
|
+ if !ok {
|
|
|
+ return nil // 建表窗口内被杀
|
|
|
+ }
|
|
|
+
|
|
|
+ // 1) 全表扫 (主键路径) 收集全部存活行 id → svc
|
|
|
+ rows := map[uint64]string{}
|
|
|
+ if err := tbl.Query().Scan(func(vals []types.FieldValue) bool {
|
|
|
+ var id uint64
|
|
|
+ var svc string
|
|
|
+ for _, fv := range vals {
|
|
|
+ switch string(fv.FieldName) {
|
|
|
+ case "id":
|
|
|
+ id, _ = fv.Value.(uint64)
|
|
|
+ case "svc":
|
|
|
+ switch v := fv.Value.(type) {
|
|
|
+ case string:
|
|
|
+ svc = v
|
|
|
+ case []byte:
|
|
|
+ svc = string(v)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ rows[id] = svc
|
|
|
+ return true
|
|
|
+ }); err != nil {
|
|
|
+ return fmt.Errorf("全表扫失败: %w (现场: %s)", err, dir)
|
|
|
+ }
|
|
|
+ if total := tbl.TotalRows(); total != uint64(len(rows)) {
|
|
|
+ return fmt.Errorf("TotalRows=%d 但实际扫到 %d (幽灵行/洞行) (现场: %s)", total, len(rows), dir)
|
|
|
+ }
|
|
|
+ if len(rows) == 0 {
|
|
|
+ return nil
|
|
|
+ }
|
|
|
+
|
|
|
+ // 2) 逐行反查: 用 svc 值走二级索引等值查, 结果集必须包含自己
|
|
|
+ bad := 0
|
|
|
+ for id, svc := range rows {
|
|
|
+ found := false
|
|
|
+ err := tbl.Query().Where(core.Eq("svc", svc, types.FieldType(0))).
|
|
|
+ Scan(func(vals []types.FieldValue) bool {
|
|
|
+ for _, fv := range vals {
|
|
|
+ if string(fv.FieldName) == "id" {
|
|
|
+ if rid, _ := fv.Value.(uint64); rid == id {
|
|
|
+ found = true
|
|
|
+ return false
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return true
|
|
|
+ })
|
|
|
+ if err != nil {
|
|
|
+ return fmt.Errorf("二级索引等值查 %q 报错: %w (现场: %s)", svc, err, dir)
|
|
|
+ }
|
|
|
+ if !found {
|
|
|
+ bad++
|
|
|
+ if bad <= 5 {
|
|
|
+ fmt.Printf(" ✗ 行 pkID=%d svc=%q 按二级索引查不到 (现场: %s)\n", id, svc, dir)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if bad > 0 {
|
|
|
+ return fmt.Errorf("%d 行存活但二级索引未命中 —— 半操作已固化 (现场保留: %s)", bad, dir)
|
|
|
+ }
|
|
|
+ return nil
|
|
|
+}
|
|
|
+
|
|
|
+func envInt(key string, def int) int {
|
|
|
+ if v := os.Getenv(key); v != "" {
|
|
|
+ var n int
|
|
|
+ if _, err := fmt.Sscanf(v, "%d", &n); err == nil && n > 0 {
|
|
|
+ return n
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return def
|
|
|
+}
|
