main.go 62 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755
  1. // ============================================================================
  2. // TsDB2 千万级功能 + 性能测试
  3. //
  4. // 覆盖:
  5. // P2 千万级写入 (分档进度 / 分档 Commit / 磁盘与内存曲线)
  6. // P3 查询性能 (主键点查 / 二级索引等值 / 范围 / 分页 / 排序 / 投影 / 聚合)
  7. // P4 多表 (部门维度表 + 订单表 + 索引关联计数校验)
  8. // P5 更新 (单行全列重写 / core 批量更新 / 索引列更新后索引同步)
  9. // P6 删除 (逐行删 10 万行 / 批量删除全表重建) + 全量一致性校验
  10. // P7 Schema 变更 (ADD COLUMN / DROP COLUMN / 改列名) + 数据正确性校验
  11. // P8 引擎语义探针 (独立库, 不污染主库)
  12. // P9 关库重开后的持久化与 schema 校验
  13. // P10 汇总 (性能总表 / 校验总表 / 数据问题清单)
  14. //
  15. // 规模可用环境变量调整 (默认千万级):
  16. // TSDB_BENCH_N 主表行数 默认 10000000 (向下取整到 1000 的倍数)
  17. // TSDB_BENCH_ORDERS 订单表行数 默认 1000000
  18. // TSDB_BENCH_DB 主库文件名 默认 bench_10m.db (相对当前工作目录)
  19. //
  20. // 运行: cd demo/bench_10m; go run .
  21. // 注意: 启动时会删除同前缀旧库文件; 运行结束后库文件保留, 可交给 analyzer 审计。
  22. // ============================================================================
  23. package main
  24. import (
  25. "encoding/binary"
  26. "fmt"
  27. "os"
  28. "path/filepath"
  29. "runtime"
  30. "strconv"
  31. "strings"
  32. "time"
  33. DB "git.ycl.pub/yl32940/TsDB2/DB"
  34. core "git.ycl.pub/yl32940/TsDB2/DB/core"
  35. "git.ycl.pub/yl32940/TsDB2/DB/core/page"
  36. "git.ycl.pub/yl32940/TsDB2/DB/core/types"
  37. )
  38. // ============================== 规模与派生参数 ==============================
  39. const (
  40. deptCount = 100
  41. scoreMod = 1000
  42. ageBase = 20
  43. ageMod = 50
  44. bulkAgeAt = 500
  45. bulkAgeTo = 99
  46. bulkDeptAt = 321
  47. )
  48. const bulkDeptTo = "Dept099"
  49. var (
  50. nEmp int64
  51. nOrder int64
  52. nJoinMod int64
  53. delCount int64
  54. rowUpdCnt int64
  55. rowUpdAt int64
  56. delFrom int64
  57. delTo int64
  58. sampleNo int64 // P7/P9 抽样行号, 随规模派生, 保证存在且不落在更新/删除区间
  59. extraRows int64 = 3
  60. dbPath string
  61. probePath string
  62. )
  63. var (
  64. perfs []perfRow
  65. checks []checkRow
  66. problems []problemRow
  67. )
  68. // ============================== 数据公式 ==============================
  69. func empName(no int64) string { return fmt.Sprintf("Emp%08d", no) }
  70. func expName(no int64) string {
  71. if no >= rowUpdAt && no < rowUpdAt+rowUpdCnt {
  72. return "UP" + empName(no)
  73. }
  74. return empName(no)
  75. }
  76. func expAge(no int64) int64 {
  77. if no%scoreMod == bulkAgeAt {
  78. return bulkAgeTo
  79. }
  80. return ageBase + no%ageMod
  81. }
  82. func expScore(no int64) int64 { return no % scoreMod }
  83. func expDept(no int64) string {
  84. if no%scoreMod == bulkDeptAt {
  85. return bulkDeptTo
  86. }
  87. return fmt.Sprintf("Dept%03d", no%deptCount)
  88. }
  89. func expSalary(no int64) float64 { return float64(no % 100000) }
  90. func expActive(no int64) bool { return no%2 == 0 }
  91. func expActiveByte(no int64) byte {
  92. if expActive(no) {
  93. return 1
  94. }
  95. return 0
  96. }
  97. func pkIDOf(no int64) uint64 {
  98. if no <= nEmp {
  99. return uint64(no)
  100. }
  101. return uint64(nEmp-delCount + (no - nEmp))
  102. }
  103. func isDeletedNo(no int64) bool { return no >= delFrom && no <= delTo }
  104. func expectRows() int64 { return nEmp - delCount + extraRows }
  105. func expectSalarySum(n int64) float64 {
  106. full := n / 100000
  107. rem := n % 100000
  108. return float64(full)*4999950000.0 + float64(rem)*float64(rem+1)/2
  109. }
  110. // ============================== 通用小工具 ==============================
  111. func must(err error) {
  112. if err != nil {
  113. fmt.Printf("\n[FATAL] %v\n", err)
  114. os.Exit(1)
  115. }
  116. }
  117. // softMust 供探针使用: 出错时记录为数据问题并返回 true, 不终止整个进程。
  118. // 探针的目的是"发现并报告引擎语义/缺陷", 单步失败不应让整场测试无结果。
  119. func softMust(stage string, err error) bool {
  120. if err == nil {
  121. return false
  122. }
  123. fmt.Printf(" [探针错误] %s → %v\n", stage, err)
  124. problem("探针", stage, fmt.Sprintf("%v", err))
  125. return true
  126. }
  127. func phase(title string) {
  128. fmt.Printf("\n%s\n%s\n%s\n", strings.Repeat("=", 104), title, strings.Repeat("=", 104))
  129. }
  130. func sub(title string) { fmt.Printf("\n--- %s ---\n", title) }
  131. func envInt64(key string, def int64) int64 {
  132. if s := os.Getenv(key); s != "" {
  133. if v, err := strconv.ParseInt(s, 10, 64); err == nil && v > 0 {
  134. return v
  135. }
  136. }
  137. return def
  138. }
  139. func human(v float64) string {
  140. switch {
  141. case v >= 1e6:
  142. return fmt.Sprintf("%.2fM", v/1e6)
  143. case v >= 1e3:
  144. return fmt.Sprintf("%.2fk", v/1e3)
  145. default:
  146. return fmt.Sprintf("%.0f", v)
  147. }
  148. }
  149. func mbOf(v uint64) float64 { return float64(v) / (1 << 20) }
  150. func memLine() string {
  151. var m runtime.MemStats
  152. runtime.ReadMemStats(&m)
  153. return fmt.Sprintf("GoAlloc=%.0fMB Sys=%.0fMB HeapInuse=%.0fMB NumGC=%d",
  154. mbOf(m.Alloc), mbOf(m.Sys), mbOf(m.HeapInuse), m.NumGC)
  155. }
  156. func dbSize(path string) int64 {
  157. dir := filepath.Dir(path)
  158. stem := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))
  159. entries, err := os.ReadDir(dir)
  160. if err != nil {
  161. return 0
  162. }
  163. var total int64
  164. for _, e := range entries {
  165. if e.IsDir() || !strings.HasPrefix(e.Name(), stem) {
  166. continue
  167. }
  168. if fi, err := e.Info(); err == nil {
  169. total += fi.Size()
  170. }
  171. }
  172. return total
  173. }
  174. func cleanOld(path string) {
  175. dir := filepath.Dir(path)
  176. stem := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))
  177. entries, err := os.ReadDir(dir)
  178. if err != nil {
  179. return
  180. }
  181. for _, e := range entries {
  182. if !e.IsDir() && strings.HasPrefix(e.Name(), stem) {
  183. p := filepath.Join(dir, e.Name())
  184. if err := os.Remove(p); err == nil {
  185. fmt.Printf(" [清理旧库] %s\n", p)
  186. }
  187. }
  188. }
  189. }
  190. func pkKey(id uint64) []byte {
  191. b := make([]byte, 8)
  192. binary.BigEndian.PutUint64(b, id)
  193. return b
  194. }
  195. func contains(xs []string, s string) bool {
  196. for _, x := range xs {
  197. if x == s {
  198. return true
  199. }
  200. }
  201. return false
  202. }
  203. func dumpVal(v interface{}) string {
  204. if b, ok := v.([]byte); ok {
  205. return strings.TrimRight(string(b), "\x00")
  206. }
  207. return fmt.Sprint(v)
  208. }
  209. func dumpRow(row []types.FieldValue) string {
  210. parts := make([]string, 0, len(row))
  211. for _, fv := range row {
  212. parts = append(parts, fmt.Sprintf("%s=%s", string(fv.FieldName), dumpVal(fv.Value)))
  213. }
  214. return strings.Join(parts, " ")
  215. }
  216. func rowMap(row []types.FieldValue) map[string]interface{} {
  217. m := make(map[string]interface{}, len(row))
  218. for _, fv := range row {
  219. m[string(fv.FieldName)] = fv.Value
  220. }
  221. return m
  222. }
  223. func rawEqual(got, want map[string]interface{}) (bool, string) {
  224. for k, wv := range want {
  225. gv, ok := got[k]
  226. if !ok {
  227. return false, "缺少列 " + k
  228. }
  229. if dumpVal(gv) != dumpVal(wv) {
  230. return false, fmt.Sprintf("%s 期望=%s 实际=%s", k, dumpVal(wv), dumpVal(gv))
  231. }
  232. }
  233. return true, ""
  234. }
  235. func fieldIndexByName(t *core.Table, name string) int {
  236. for i, f := range t.GetFields() {
  237. if string(f.FieldName) == name {
  238. return i
  239. }
  240. }
  241. return -1
  242. }
  243. func coreRow(t *core.Table, pkIdx string, pk uint64) map[string]interface{} {
  244. row, err := t.SearchByIndex(pkIdx, pkKey(pk))
  245. if err != nil {
  246. return nil
  247. }
  248. return rowMap(row)
  249. }
  250. func sampleIDs(count, max int64) []uint64 {
  251. ids := make([]uint64, 0, count)
  252. x := uint64(0x9E3779B97F4A7C15)
  253. for int64(len(ids)) < count {
  254. x ^= x << 13
  255. x ^= x >> 7
  256. x ^= x << 17
  257. ids = append(ids, uint64(x%uint64(max))+1)
  258. }
  259. return ids
  260. }
  261. // ============================== 报告设施 ==============================
  262. type perfRow struct {
  263. Phase, Op string
  264. Rows int64
  265. Cost time.Duration
  266. Note string
  267. }
  268. type checkRow struct {
  269. Name, Expect, Actual string
  270. Pass bool
  271. }
  272. type problemRow struct {
  273. Level string
  274. Title string
  275. Detail string
  276. }
  277. func perf(ph, op string, rows int64, d time.Duration) {
  278. perfs = append(perfs, perfRow{ph, op, rows, d, ""})
  279. rate := "-"
  280. if d > 0 && rows > 0 {
  281. rate = human(float64(rows)/d.Seconds()) + "/s"
  282. }
  283. fmt.Printf(" %-52s 行数=%-9d 耗时=%-11s %s\n", op, rows, d.Round(time.Millisecond), rate)
  284. }
  285. func chk(name string, pass bool, expect, actual string) bool {
  286. checks = append(checks, checkRow{name, expect, actual, pass})
  287. if !pass {
  288. fmt.Printf(" [FAIL] %s | 期望=%s 实际=%s\n", name, expect, actual)
  289. }
  290. return pass
  291. }
  292. func problem(level, title, detail string) {
  293. problems = append(problems, problemRow{level, title, detail})
  294. fmt.Printf(" [%s] %s: %s\n", level, title, detail)
  295. }
  296. // ============================== 表结构 ==============================
  297. type Emp struct {
  298. DB.BaseModel
  299. No int64 `vrtx:"index:idx_emp_no"`
  300. Name string `vrtx:"maxlen:24"`
  301. Age int64 `vrtx:"index:idx_emp_age"`
  302. Score int64 // 故意不建索引: 用于对比"无索引条件 = 全表扫描"
  303. Dept string `vrtx:"maxlen:12,index:idx_emp_dept"`
  304. Salary float64 // float64 → 专用浮点列 (保序编码), 小数不会被截断
  305. Active bool
  306. }
  307. type DeptDim struct {
  308. DB.BaseModel
  309. Code string `vrtx:"maxlen:12,index:idx_dept_code"`
  310. Title string `vrtx:"maxlen:32"`
  311. Level int64
  312. }
  313. type Order struct {
  314. DB.BaseModel
  315. OrderNo int64 `vrtx:"index:idx_order_no"`
  316. EmpNo int64 `vrtx:"index:idx_order_empno"`
  317. Amount int64
  318. Status int64
  319. }
  320. func deptCode(i int64) string { return fmt.Sprintf("Dept%03d", i) }
  321. // ============================== P0 环境 ==============================
  322. func phase0() {
  323. phase("P0 环境与参数")
  324. fmt.Printf(" 时间 : %s\n", time.Now().Format("2006-01-02 15:04:05"))
  325. fmt.Printf(" Go : %s | CPU %d 核 (GOMAXPROCS=%d) | %s/%s\n",
  326. runtime.Version(), runtime.NumCPU(), runtime.GOMAXPROCS(0), runtime.GOOS, runtime.GOARCH)
  327. fmt.Printf(" 规模 : 主表 %d 行 | 订单 %d 行 | 部门 %d 行 | 单行更新 %d 次 | 逐行删除 %d 行\n",
  328. nEmp, nOrder, deptCount, rowUpdCnt, delCount)
  329. fmt.Printf(" 参数 : Age=20+No%%50 | Score=No%%1000 | Dept=Dept(No%%100) | Salary=No%%100000\n")
  330. fmt.Printf(" 索引 : pk_emp + idx_emp_no/BTree + idx_emp_age/BTree + idx_emp_dept/BTree\n")
  331. fmt.Printf(" 提交策略 : preset=batch(脏页阈值 500 页), 自动提交防抖禁用, 每 100 万行显式 Commit\n")
  332. fmt.Printf(" 主库 : %s\n", dbPath)
  333. fmt.Printf(" 探针库 : %s\n", probePath)
  334. fmt.Printf(" %s\n", memLine())
  335. }
  336. // ============================== P1 建表 ==============================
  337. func phase1(db *DB.DB) (*DB.Table[Emp], *DB.Table[DeptDim], *DB.Table[Order], *core.Table) {
  338. phase("P1 建表 / 打开表 + 索引")
  339. t := time.Now()
  340. empT, err := DB.UseTable[Emp](db, "emp")
  341. must(err)
  342. deptT, err := DB.UseTable[DeptDim](db, "dept_dim")
  343. must(err)
  344. orderT, err := DB.UseTable[Order](db, "orders")
  345. must(err)
  346. empCore, ok := db.CoreDB().FindTable("emp")
  347. if !ok {
  348. must(fmt.Errorf("core 层找不到 emp 表"))
  349. }
  350. perf("P1 建表", "UseTable × 3 (含建表/建索引)", 3, time.Since(t))
  351. fmt.Printf(" emp 活跃列 : %v\n", empCore.GetActiveFieldNames())
  352. fmt.Printf(" emp 字段数 : 总 %d / 活跃 %d | TotalRows=%d\n",
  353. len(empCore.GetFields()), len(empCore.GetActiveFieldNames()), empCore.TotalRows())
  354. for _, nm := range []string{"pk_emp", "idx_emp_no", "idx_emp_age", "idx_emp_dept"} {
  355. if m, ok := empCore.IndexMeta().FindIndex(nm); ok {
  356. ids := make([]string, 0, m.FieldCnt)
  357. for k := 0; k < int(m.FieldCnt); k++ {
  358. ids = append(ids, strconv.Itoa(int(m.FieldIDs[k])))
  359. }
  360. fmt.Printf(" 索引 %-14s type=%-2d 分量=%d fieldIDs=[%s]\n",
  361. nm, m.IndexType, m.FieldCnt, strings.Join(ids, ","))
  362. } else {
  363. fmt.Printf(" 索引 %-14s 缺失!\n", nm)
  364. }
  365. }
  366. return empT, deptT, orderT, empCore
  367. }
  368. // ============================== P2 千万级写入 ==============================
  369. func phase2InsertEmp(db *DB.DB, empT *DB.Table[Emp], empCore *core.Table) {
  370. phase(fmt.Sprintf("P2 千万级写入: 单行 Insert × %d (含进度 / 分档 Commit)", nEmp))
  371. t0 := time.Now()
  372. last, lastN := t0, int64(0)
  373. e := &Emp{}
  374. for i := int64(1); i <= nEmp; i++ {
  375. *e = Emp{
  376. BaseModel: DB.BaseModel{ID: 0},
  377. No: i,
  378. Name: empName(i),
  379. Age: int64(ageBase + i%ageMod),
  380. Score: expScore(i),
  381. Dept: fmt.Sprintf("Dept%03d", i%deptCount),
  382. Salary: expSalary(i),
  383. Active: expActive(i),
  384. }
  385. if err := empT.Insert(e); err != nil {
  386. must(fmt.Errorf("插入第 %d 行失败: %w", i, err))
  387. }
  388. if i%200000 == 0 {
  389. now := time.Now()
  390. rate := float64(i-lastN) / now.Sub(last).Seconds()
  391. eta := time.Duration(float64(nEmp-i) / rate * float64(time.Second))
  392. fmt.Printf(" [%9d 行] 累计%7.1fs | 瞬时 %8s/s | 磁盘 %7.1fMB | ETA %6s | %s\n",
  393. i, now.Sub(t0).Seconds(), human(rate), float64(dbSize(dbPath))/(1<<20),
  394. eta.Round(time.Second), memLine())
  395. last, lastN = now, i
  396. }
  397. if i%1000000 == 0 {
  398. ct := time.Now()
  399. must(db.Commit())
  400. fmt.Printf(" [%9d 行] 显式 Commit %s | 磁盘 %.1fMB\n",
  401. i, time.Since(ct).Round(time.Millisecond), float64(dbSize(dbPath))/(1<<20))
  402. }
  403. }
  404. ct := time.Now()
  405. must(db.Commit())
  406. total := time.Since(t0)
  407. perf("P2 写入", fmt.Sprintf("单行 Insert × %d + 分档 Commit", nEmp), nEmp, total)
  408. fmt.Printf(" 最终 Commit %s | 磁盘 %.1fMB | 文件 %.2fGB | %s\n",
  409. time.Since(ct).Round(time.Millisecond), float64(dbSize(dbPath))/(1<<20),
  410. float64(dbSize(dbPath))/(1<<30), memLine())
  411. chk("2.1 TotalRows == N", empCore.TotalRows() == uint64(nEmp),
  412. strconv.FormatInt(nEmp, 10), strconv.FormatUint(empCore.TotalRows(), 10))
  413. scanned, bad := verifyEmpInserted(empT, "写入后全量校验(插入态)", nEmp)
  414. chk("2.2 全表扫描行数 == N", scanned == nEmp, strconv.FormatInt(nEmp, 10), strconv.FormatInt(scanned, 10))
  415. chk("2.3 全表逐行公式校验 0 错", bad == 0, "0", strconv.FormatInt(bad, 10))
  416. }
  417. // verifyEmpORM 用 ORM 映射全表扫描并按公式逐行校验 (跳过已删除区间)
  418. func verifyEmpORM(empT *DB.Table[Emp], label string, want int64) (int64, int64) {
  419. t := time.Now()
  420. var scanned, bad int64
  421. var first string
  422. err := empT.Select(func(e *Emp) bool {
  423. scanned++
  424. no := e.No
  425. if isDeletedNo(no) {
  426. bad++
  427. if first == "" {
  428. first = fmt.Sprintf("已删除行仍存活 No=%d", no)
  429. }
  430. return true
  431. }
  432. badCols := make([]string, 0, 4)
  433. if e.Name != expName(no) {
  434. badCols = append(badCols, "Name")
  435. }
  436. if e.Age != expAge(no) {
  437. badCols = append(badCols, "Age")
  438. }
  439. if e.Score != expScore(no) {
  440. badCols = append(badCols, "Score")
  441. }
  442. if e.Dept != expDept(no) {
  443. badCols = append(badCols, "Dept")
  444. }
  445. if e.Salary != expSalary(no) {
  446. badCols = append(badCols, "Salary")
  447. }
  448. if e.Active != expActive(no) {
  449. badCols = append(badCols, "Active")
  450. }
  451. if e.ID != pkIDOf(no) {
  452. badCols = append(badCols, fmt.Sprintf("_id(%d!=%d)", e.ID, pkIDOf(no)))
  453. }
  454. if len(badCols) > 0 {
  455. bad++
  456. if first == "" {
  457. first = fmt.Sprintf("No=%d 列不符: %s", no, strings.Join(badCols, ","))
  458. }
  459. }
  460. return true
  461. })
  462. must(err)
  463. perf("校验", "全表扫描 "+label, scanned, time.Since(t))
  464. if first != "" {
  465. fmt.Printf(" 首个错行: %s\n", first)
  466. }
  467. if want >= 0 {
  468. chk("全表扫描 "+label+" 行数", scanned == want,
  469. strconv.FormatInt(want, 10), strconv.FormatInt(scanned, 10))
  470. }
  471. return scanned, bad
  472. }
  473. // verifyEmpInserted P2 专用: 只校验"刚插入时应有的原始值", 不含后续 P5 更新 / P6 删除的最终态。
  474. // (verifyEmpORM 的公式里带有更新覆盖与删除区间, 在 P2 阶段尚不成立)
  475. func verifyEmpInserted(empT *DB.Table[Emp], label string, want int64) (int64, int64) {
  476. t := time.Now()
  477. var scanned, bad int64
  478. var first string
  479. err := empT.Select(func(e *Emp) bool {
  480. scanned++
  481. no := e.No
  482. badCols := make([]string, 0, 4)
  483. if e.Name != empName(no) {
  484. badCols = append(badCols, "Name")
  485. }
  486. if e.Age != ageBase+no%ageMod {
  487. badCols = append(badCols, "Age")
  488. }
  489. if e.Score != no%scoreMod {
  490. badCols = append(badCols, "Score")
  491. }
  492. if e.Dept != fmt.Sprintf("Dept%03d", no%deptCount) {
  493. badCols = append(badCols, "Dept")
  494. }
  495. if e.Salary != float64(no%100000) {
  496. badCols = append(badCols, "Salary")
  497. }
  498. if e.Active != (no%2 == 0) {
  499. badCols = append(badCols, "Active")
  500. }
  501. if e.ID != uint64(no) {
  502. badCols = append(badCols, fmt.Sprintf("_id(%d!=%d)", e.ID, no))
  503. }
  504. if len(badCols) > 0 {
  505. bad++
  506. if first == "" {
  507. first = fmt.Sprintf("No=%d 列不符: %s", no, strings.Join(badCols, ","))
  508. }
  509. }
  510. return true
  511. })
  512. must(err)
  513. perf("校验", "全表扫描 "+label, scanned, time.Since(t))
  514. if first != "" {
  515. fmt.Printf(" 首个错行: %s\n", first)
  516. }
  517. if want >= 0 {
  518. chk("全表扫描 "+label+" 行数", scanned == want,
  519. strconv.FormatInt(want, 10), strconv.FormatInt(scanned, 10))
  520. }
  521. return scanned, bad
  522. }
  523. // ============================== P3 查询 ==============================
  524. func phase3Queries(empT *DB.Table[Emp], empCore *core.Table) {
  525. phase("P3 查询性能 (主键点查 / 二级索引等值 / 范围 / 分页 / 排序 / 投影 / 聚合)")
  526. const qn = 20000
  527. // 3.1 主键点查
  528. {
  529. ids := sampleIDs(qn, nEmp)
  530. bad := int64(0)
  531. t := time.Now()
  532. for _, id := range ids {
  533. e, err := empT.Find(id)
  534. if err != nil || e.ID != id || e.No != int64(id) || e.Name != empName(int64(id)) {
  535. bad++
  536. }
  537. }
  538. perf("P3 查询", fmt.Sprintf("主键点查 Find(pkID) × %d", qn), qn, time.Since(t))
  539. chk("3.1 主键点查结果全对", bad == 0, "0 错", fmt.Sprintf("%d 错", bad))
  540. }
  541. // 3.2 二级索引等值
  542. {
  543. ids := sampleIDs(qn, nEmp)
  544. bad := int64(0)
  545. t := time.Now()
  546. for _, id := range ids {
  547. no := int64(id)
  548. e, err := empT.Where("No", "=", no).First()
  549. if err != nil || e.No != no || e.ID != uint64(no) {
  550. bad++
  551. }
  552. }
  553. perf("P3 查询", fmt.Sprintf("二级索引等值 Where(No=..) × %d", qn), qn, time.Since(t))
  554. chk("3.2 索引等值结果全对", bad == 0, "0 错", fmt.Sprintf("%d 错", bad))
  555. }
  556. // 3.3 索引等值 Count (Age)
  557. {
  558. const times = 20
  559. want := nEmp / ageMod
  560. bad := int64(0)
  561. t := time.Now()
  562. for k := int64(0); k < times; k++ {
  563. c, err := empT.Where("Age", "=", ageBase+k).Count()
  564. if err != nil || c != want {
  565. bad++
  566. }
  567. }
  568. perf("P3 查询", fmt.Sprintf("索引等值 Count(Age=..) × %d (每次 %d 行)", times, want), want*times, time.Since(t))
  569. chk("3.3 索引等值计数正确", bad == 0, "0 错", fmt.Sprintf("%d 错", bad))
  570. }
  571. // 3.4 索引范围 Count
  572. {
  573. want := nEmp * 5 / ageMod
  574. t := time.Now()
  575. c, err := empT.Where("Age", ">=", 30).Where("Age", "<=", 34).Count()
  576. perf("P3 查询", fmt.Sprintf("索引范围 Count(Age 30..34, %d 行)", want), want, time.Since(t))
  577. chk("3.4 索引范围计数正确", err == nil && c == want,
  578. strconv.FormatInt(want, 10), fmt.Sprintf("%d err=%v", c, err))
  579. }
  580. // 3.5 字符串索引等值 Count
  581. {
  582. d := deptCode(7)
  583. want := nEmp / deptCount
  584. t := time.Now()
  585. c, err := empT.Where("Dept", "=", d).Count()
  586. perf("P3 查询", fmt.Sprintf("字符串索引 Count(Dept=%s, %d 行)", d, want), want, time.Since(t))
  587. chk("3.5 字符串索引计数正确", err == nil && c == want,
  588. strconv.FormatInt(want, 10), fmt.Sprintf("%d err=%v", c, err))
  589. }
  590. // 3.6 无索引列条件 → 全表扫描
  591. {
  592. want := nEmp / scoreMod
  593. t := time.Now()
  594. c, err := empT.Where("Score", "=", 123).Count()
  595. perf("P3 查询", fmt.Sprintf("无索引条件 Count(Score=123, %d 行) 全表扫描", want), nEmp, time.Since(t))
  596. chk("3.6 无索引条件计数正确", err == nil && c == want,
  597. strconv.FormatInt(want, 10), fmt.Sprintf("%d err=%v", c, err))
  598. }
  599. // 3.7 分页
  600. {
  601. d := deptCode(7)
  602. rows, err := empT.Where("Dept", "=", d).Limit(20).Find()
  603. bad := 0
  604. for _, e := range rows {
  605. if e.Dept != d {
  606. bad++
  607. }
  608. }
  609. chk("3.7 Limit 分页", err == nil && len(rows) == 20 && bad == 0,
  610. "20 行且全为 "+d, fmt.Sprintf("%d 行 %d 不符 err=%v", len(rows), bad, err))
  611. // Offset 分页: 偏移量随规模派生, 避免小规模下 dept 总行数 < 偏移量导致取空
  612. deptSize := nEmp / deptCount
  613. off := int64(500)
  614. if deptSize > 0 && off > deptSize/2 {
  615. off = deptSize / 4
  616. }
  617. if off < 0 {
  618. off = 0
  619. }
  620. wantFull := int(off) + 10
  621. t := time.Now()
  622. rows2, err2 := empT.Where("Dept", "=", d).Limit(10).Offset(int(off)).Find()
  623. perf("P3 查询", fmt.Sprintf("Limit(10).Offset(%d)", off), 10, time.Since(t))
  624. full, _ := empT.Where("Dept", "=", d).Limit(wantFull).Find()
  625. match := len(rows2) == 10 && len(full) == wantFull
  626. if match {
  627. for k := 0; k < 10; k++ {
  628. if full[int(off)+k].No != rows2[k].No {
  629. match = false
  630. break
  631. }
  632. }
  633. }
  634. chk("3.7 Offset 分页与全量前缀一致", match && err2 == nil, "一致",
  635. fmt.Sprintf("一致=%v off=%d full=%d rows2=%d err=%v", match, off, len(full), len(rows2), err2))
  636. }
  637. // 3.8 排序
  638. {
  639. t := time.Now()
  640. rows, err := empT.Where("Age", ">=", 30).Where("Age", "<=", 34).OrderBy("Age").Limit(10).Find()
  641. perf("P3 查询", "OrderBy(索引列 Age) + Limit(10) 流式取前 N", 10, time.Since(t))
  642. ordered := true
  643. for i := 1; i < len(rows); i++ {
  644. if rows[i].Age < rows[i-1].Age {
  645. ordered = false
  646. }
  647. }
  648. chk("3.8 索引序排序正确", err == nil && len(rows) == 10 && ordered,
  649. "10 行且 Age 非降", fmt.Sprintf("%d 行 ordered=%v err=%v", len(rows), ordered, err))
  650. part := nEmp / deptCount
  651. t = time.Now()
  652. rows2, err2 := empT.Where("Dept", "=", deptCode(7)).OrderBy("Score").Limit(10).Find()
  653. perf("P3 查询", fmt.Sprintf("OrderBy(非索引列 Score) %d 行参与内存排序", part), part, time.Since(t))
  654. ordered2 := true
  655. for i := 1; i < len(rows2); i++ {
  656. if rows2[i].Score < rows2[i-1].Score {
  657. ordered2 = false
  658. }
  659. }
  660. chk("3.8 非索引列排序正确", err2 == nil && len(rows2) == 10 && ordered2,
  661. "10 行且 Score 非降", fmt.Sprintf("%d 行 ordered=%v err=%v", len(rows2), ordered2, err2))
  662. }
  663. // 3.9 投影与 FindOne
  664. {
  665. rows, err := empCore.Query().Select("No", "Age").Limit(5).Find()
  666. ok := err == nil && len(rows) == 5
  667. for _, r := range rows {
  668. if len(r) < 2 {
  669. ok = false
  670. }
  671. }
  672. if len(rows) > 0 {
  673. fmt.Printf(" Select(No,Age) 首行返回: %s\n", dumpRow(rows[0]))
  674. }
  675. chk("3.9 Select 投影返回指定列", ok, "5 行且含 No/Age",
  676. fmt.Sprintf("%d 行 err=%v", len(rows), err))
  677. one, err2 := empCore.Query().Where(core.Eq("No", int64(1), types.FieldType(0))).FindOne()
  678. chk("3.9 FindOne 命中", err2 == nil && len(one) > 0 && dumpVal(rowMap(one)["No"]) == "1",
  679. "No=1", fmt.Sprintf("%v err=%v", rowMap(one)["No"], err2))
  680. }
  681. // 3.10 聚合
  682. {
  683. t := time.Now()
  684. sum, err := empCore.Query().Sum("Salary")
  685. perf("P3 聚合", "Sum(Salary) 全表", nEmp, time.Since(t))
  686. want := expectSalarySum(nEmp)
  687. chk("3.10 Sum 正确", err == nil && sum == want,
  688. fmt.Sprintf("%.0f", want), fmt.Sprintf("%.0f err=%v", sum, err))
  689. t = time.Now()
  690. mx, err2 := empCore.Query().Max("No")
  691. perf("P3 聚合", "Max(No)", nEmp, time.Since(t))
  692. chk("3.10 Max 正确", err2 == nil && dumpVal(mx) == strconv.FormatInt(nEmp, 10),
  693. strconv.FormatInt(nEmp, 10), fmt.Sprintf("%v err=%v", mx, err2))
  694. mn, err3 := empCore.Query().Min("No")
  695. chk("3.10 Min 正确", err3 == nil && dumpVal(mn) == "1", "1", fmt.Sprintf("%v err=%v", mn, err3))
  696. }
  697. _ = coreRow
  698. }
  699. // ============================== P4 多表 ==============================
  700. func phase4Multi(db *DB.DB, empT *DB.Table[Emp], deptT *DB.Table[DeptDim], orderT *DB.Table[Order],
  701. empCore *core.Table) (map[int64]int64, map[int64]int64) {
  702. phase("P4 多表: 部门维度表 + 订单表 + 索引关联计数校验")
  703. // 4.1 维度表
  704. t := time.Now()
  705. for i := int64(0); i < deptCount; i++ {
  706. d := &DeptDim{Code: deptCode(i), Title: "部门-" + deptCode(i), Level: i % 5}
  707. must(deptT.Insert(d))
  708. }
  709. must(db.Commit())
  710. perf("P4 多表", fmt.Sprintf("维度表 dept_dim 写入 %d 行", deptCount), deptCount, time.Since(t))
  711. c, err := deptT.Count()
  712. chk("4.1 维度表行数", err == nil && c == deptCount, strconv.Itoa(deptCount), fmt.Sprintf("%d err=%v", c, err))
  713. d7, err := deptT.Where("Code", "=", deptCode(7)).First()
  714. chk("4.1 维度表按 Code 索引查询", err == nil && d7 != nil && d7.Title == "部门-"+deptCode(7),
  715. "部门-"+deptCode(7), fmt.Sprintf("%v err=%v", d7, err))
  716. // 4.2 订单表
  717. orderCore, _ := db.CoreDB().FindTable("orders")
  718. t0 := time.Now()
  719. o := &Order{}
  720. for i := int64(1); i <= nOrder; i++ {
  721. *o = Order{OrderNo: i, EmpNo: 1 + i%nJoinMod, Amount: 100 + i%9900, Status: i % 5}
  722. if err := orderT.Insert(o); err != nil {
  723. must(fmt.Errorf("插入第 %d 条订单失败: %w", i, err))
  724. }
  725. if i%(nOrder/5) == 0 {
  726. rate := float64(i) / time.Since(t0).Seconds()
  727. fmt.Printf(" [订单 %8d 行] 累计%6.1fs | %8s/s | 磁盘 %.1fMB\n",
  728. i, time.Since(t0).Seconds(), human(rate), float64(dbSize(dbPath))/(1<<20))
  729. }
  730. }
  731. must(db.Commit())
  732. perf("P4 多表", fmt.Sprintf("订单表 orders 写入 %d 行", nOrder), nOrder, time.Since(t0))
  733. chk("4.2 订单表行数", orderCore.TotalRows() == uint64(nOrder),
  734. strconv.FormatInt(nOrder, 10), strconv.FormatUint(orderCore.TotalRows(), 10))
  735. // 4.3 关联期望 (公式推导: EmpNo = 1 + OrderNo%nJoinMod)
  736. orderExpect := map[int64]int64{}
  737. orderAfterDel := map[int64]int64{}
  738. for i := int64(1); i <= nOrder; i++ {
  739. e := 1 + i%nJoinMod
  740. orderExpect[e]++
  741. if i%5 != 0 { // P6 会删掉 Status==0 (i%5==0) 的订单
  742. orderAfterDel[e]++
  743. }
  744. }
  745. // 4.4 员工 → 订单 (走 idx_order_empno)
  746. sample := make([]int64, 0, 20)
  747. for k := int64(0); k < 20; k++ {
  748. if k%2 == 0 {
  749. sample = append(sample, 1+k*nJoinMod/40) // 落在 [1, nJoinMod] 内
  750. } else {
  751. sample = append(sample, nJoinMod+1+k) // 落在 (nJoinMod, nEmp] 内
  752. }
  753. }
  754. bad := int64(0)
  755. t = time.Now()
  756. for _, e := range sample {
  757. c, err := orderT.Where("EmpNo", "=", e).Count()
  758. if err != nil || c != orderExpect[e] {
  759. bad++
  760. }
  761. }
  762. perf("P4 多表", "关联计数: 订单表按 EmpNo 索引 × 20", 20, time.Since(t))
  763. chk("4.4 员工→订单 关联计数正确", bad == 0, "0 错", fmt.Sprintf("%d 错", bad))
  764. // 4.5 部门 → 员工 (走 idx_emp_dept, 100 个部门全查)
  765. want := nEmp / deptCount
  766. bad = 0
  767. t = time.Now()
  768. for i := int64(0); i < deptCount; i++ {
  769. c, err := empT.Where("Dept", "=", deptCode(i)).Count()
  770. if err != nil || c != want {
  771. bad++
  772. fmt.Printf(" [部门计数异常] %s 期望=%d 实际=%d err=%v\n", deptCode(i), want, c, err)
  773. }
  774. }
  775. perf("P4 多表", fmt.Sprintf("关联计数: 员工表按 Dept 索引 × %d (每次 %d 行)", deptCount, want),
  776. want*deptCount, time.Since(t))
  777. chk("4.5 部门→员工 关联计数全对", bad == 0, "0 错", fmt.Sprintf("%d 错", bad))
  778. chk("4.5 部门计数之和 == 全表行数", want*deptCount == nEmp,
  779. strconv.FormatInt(nEmp, 10), strconv.FormatInt(want*deptCount, 10))
  780. return orderExpect, orderAfterDel
  781. }
  782. // empSample 员工抽样: 一半落在 [1, nJoinMod] (有订单), 一半落在其后 (无订单)
  783. func empSample() []int64 {
  784. s := make([]int64, 0, 20)
  785. for k := int64(0); k < 20; k++ {
  786. if k%2 == 0 {
  787. s = append(s, 1+k*nJoinMod/40)
  788. } else {
  789. s = append(s, nJoinMod+1+k)
  790. }
  791. }
  792. return s
  793. }
  794. // ============================== P5 更新 ==============================
  795. func phase5Update(db *DB.DB, empT *DB.Table[Emp], empCore *core.Table) {
  796. phase("P5 更新 (单行整行重写 / core 批量更新 / 索引列更新后索引是否同步)")
  797. // 5.1 单行更新
  798. {
  799. bad := int64(0)
  800. t := time.Now()
  801. for k := int64(0); k < rowUpdCnt; k++ {
  802. no := rowUpdAt + k
  803. e, err := empT.Find(uint64(no))
  804. if err != nil {
  805. bad++
  806. continue
  807. }
  808. e.Name = "UP" + empName(no)
  809. if err := empT.Update(e); err != nil {
  810. bad++
  811. }
  812. }
  813. must(db.Commit())
  814. perf("P5 更新", fmt.Sprintf("单行更新 Update(整行重写) × %d", rowUpdCnt), rowUpdCnt, time.Since(t))
  815. chk("5.1 单行更新全部成功", bad == 0, "0 错", fmt.Sprintf("%d 错", bad))
  816. e, err := empT.Find(uint64(rowUpdAt))
  817. chk("5.1 更新后读回正确", err == nil && e.Name == "UP"+empName(rowUpdAt) && e.No == rowUpdAt &&
  818. e.Age == expAge(rowUpdAt) && e.Dept == expDept(rowUpdAt) && e.Score == expScore(rowUpdAt),
  819. "UP"+empName(rowUpdAt),
  820. fmt.Sprintf("Name=%s No=%d Age=%d Dept=%s Score=%d err=%v", e.Name, e.No, e.Age, e.Dept, e.Score, err))
  821. }
  822. // 5.2 core 批量更新 (整数索引列 Age)
  823. {
  824. want := nEmp / scoreMod
  825. t := time.Now()
  826. cnt, err := empCore.Update().Set("Age", int64(bulkAgeTo)).
  827. Where(core.Eq("Score", int64(bulkAgeAt), types.FieldType(0))).Exec()
  828. perf("P5 更新", fmt.Sprintf("core 批量更新 Updater.Exec (Score=%d → Age=%d)", bulkAgeAt, bulkAgeTo),
  829. cnt, time.Since(t))
  830. chk("5.2 批量更新影响行数", err == nil && cnt == want,
  831. strconv.FormatInt(want, 10), fmt.Sprintf("%d err=%v", cnt, err))
  832. t = time.Now()
  833. c99, err2 := empT.Where("Age", "=", int64(bulkAgeTo)).Count()
  834. perf("P5 校验", "索引列更新后 Count(Age=99)", want, time.Since(t))
  835. chk("5.2 索引列更新后索引已同步", err2 == nil && c99 == want,
  836. strconv.FormatInt(want, 10), fmt.Sprintf("%d err=%v", c99, err2))
  837. want20 := nEmp/ageMod - nEmp/scoreMod
  838. c20, err3 := empT.Where("Age", "=", int64(ageBase)).Count()
  839. chk("5.2 原 Age 桶计数同步减少", err3 == nil && c20 == want20,
  840. strconv.FormatInt(want20, 10), fmt.Sprintf("%d err=%v", c20, err3))
  841. }
  842. // 5.3 core 批量更新 (字符串索引列 Dept)
  843. {
  844. want := nEmp / scoreMod
  845. t := time.Now()
  846. // 直接写原始字节: v9 起字符串列按【实际长度】存储且超宽报错 (不再静默截断),
  847. // 不能再按 types.GetSize(ft) (类型默认档 256B) 造 256B 值 —— 那会把 maxlen:12 的
  848. // 列 (16B) 喂成超宽, 整批更新失败。列宽由引擎按列定义把关。
  849. cnt, err := empCore.Update().Set("Dept", []byte(bulkDeptTo)).
  850. Where(core.Eq("Score", int64(bulkDeptAt), types.FieldType(0))).Exec()
  851. perf("P5 更新", fmt.Sprintf("core 批量更新字符串索引列 (Score=%d → Dept=%s)", bulkDeptAt, bulkDeptTo),
  852. cnt, time.Since(t))
  853. chk("5.3 字符串列批量更新影响行数", err == nil && cnt == want,
  854. strconv.FormatInt(want, 10), fmt.Sprintf("%d err=%v", cnt, err))
  855. want99 := nEmp/deptCount + nEmp/scoreMod
  856. c99, err2 := empT.Where("Dept", "=", bulkDeptTo).Count()
  857. chk("5.3 字符串索引同步 (并入 "+bulkDeptTo+")", err2 == nil && c99 == want99,
  858. strconv.FormatInt(want99, 10), fmt.Sprintf("%d err=%v", c99, err2))
  859. want21 := nEmp/deptCount - nEmp/scoreMod
  860. c21, err3 := empT.Where("Dept", "=", deptCode(21)).Count()
  861. chk("5.3 原字符串桶计数同步减少", err3 == nil && c21 == want21,
  862. strconv.FormatInt(want21, 10), fmt.Sprintf("%d err=%v", c21, err3))
  863. }
  864. // 5.4 负数测试: 更新不存在的主键
  865. {
  866. ghost := &Emp{BaseModel: DB.BaseModel{ID: 1 << 60}, No: 1, Name: "ghost"}
  867. err := empT.Update(ghost)
  868. chk("5.4 更新不存在的主键应报错", err != nil, "error", fmt.Sprintf("%v", err))
  869. }
  870. }
  871. // ============================== P6 删除 + 一致性校验 ==============================
  872. func phase6Delete(db *DB.DB, empT *DB.Table[Emp], orderT *DB.Table[Order], empCore *core.Table,
  873. orderAfterDel map[int64]int64) {
  874. phase(fmt.Sprintf("P6 删除 (逐行删 %d 行 / 批量删除全表重建) + 全量一致性校验", delCount))
  875. // 6.1 逐行删除
  876. {
  877. t := time.Now()
  878. ok := int64(0)
  879. for id := uint64(delFrom); id <= uint64(delTo); id++ {
  880. if err := empT.Delete(id); err != nil {
  881. fmt.Printf(" [删除失败] pkID=%d err=%v\n", id, err)
  882. } else {
  883. ok++
  884. }
  885. }
  886. perf("P6 删除", fmt.Sprintf("逐行 Delete(pkID) × %d", delCount), ok, time.Since(t))
  887. chk("6.1 逐行删除全部成功", ok == delCount,
  888. strconv.FormatInt(delCount, 10), strconv.FormatInt(ok, 10))
  889. chk("6.1 TotalRows 已减少", empCore.TotalRows() == uint64(nEmp-delCount),
  890. strconv.FormatInt(nEmp-delCount, 10), strconv.FormatUint(empCore.TotalRows(), 10))
  891. _, err := empT.Find(uint64(delFrom))
  892. chk("6.1 已删除行 Find 应 not found", err != nil, "error", fmt.Sprintf("%v", err))
  893. alive := delFrom - 1
  894. e, err2 := empT.Find(uint64(alive))
  895. chk("6.1 边界外一行仍存活", err2 == nil && e.No == alive,
  896. fmt.Sprintf("No=%d", alive), fmt.Sprintf("%v err=%v", e, err2))
  897. }
  898. // 6.2 批量删除 orders (Status==0 无索引列 → 收集后全表重建)
  899. {
  900. orderCore, _ := db.CoreDB().FindTable("orders")
  901. want := nOrder / 5
  902. t := time.Now()
  903. cnt, err := orderCore.Delete().Where(core.Eq("Status", int64(0), types.FieldType(0))).Exec()
  904. perf("P6 删除", fmt.Sprintf("批量删除 orders (Status=0, %d 行) 触发全表重建", want), want, time.Since(t))
  905. chk("6.2 批量删除影响行数", err == nil && cnt == want,
  906. strconv.FormatInt(want, 10), fmt.Sprintf("%d err=%v", cnt, err))
  907. chk("6.2 订单表行数已减少", orderCore.TotalRows() == uint64(nOrder-want),
  908. strconv.FormatInt(nOrder-want, 10), strconv.FormatUint(orderCore.TotalRows(), 10))
  909. bad := 0
  910. t2 := time.Now()
  911. for _, e := range empSample() {
  912. c, err := orderT.Where("EmpNo", "=", e).Count()
  913. if err != nil || c != orderAfterDel[e] {
  914. bad++
  915. }
  916. }
  917. perf("P6 校验", "批量删除后 员工→订单 关联计数 × 20", 20, time.Since(t2))
  918. chk("6.2 批量删除后关联计数正确", bad == 0, "0 错", fmt.Sprintf("%d 错", bad))
  919. }
  920. // 6.3 全量一致性校验 (期望值全部由公式推导)
  921. {
  922. sub("6.3 一致性校验 (更新 + 删除之后, 期望值由公式推导)")
  923. scanned, bad := verifyEmpORM(empT, "删除后全量校验(公式)", nEmp-delCount)
  924. chk("6.3 全表逐行公式校验 0 错", bad == 0, "0", strconv.FormatInt(bad, 10))
  925. deptWant := map[string]int64{}
  926. ageWant := map[int64]int64{}
  927. for i := int64(1); i <= nEmp; i++ {
  928. if isDeletedNo(i) {
  929. continue
  930. }
  931. deptWant[expDept(i)]++
  932. ageWant[expAge(i)]++
  933. }
  934. badD := 0
  935. t := time.Now()
  936. for i := int64(0); i < deptCount; i++ {
  937. c, err := empT.Where("Dept", "=", deptCode(i)).Count()
  938. if err != nil || c != deptWant[deptCode(i)] {
  939. badD++
  940. fmt.Printf(" [部门计数异常] %s 期望=%d 实际=%d err=%v\n", deptCode(i), deptWant[deptCode(i)], c, err)
  941. }
  942. }
  943. perf("P6 校验", fmt.Sprintf("索引计数复核 Dept × %d", deptCount), nEmp-delCount, time.Since(t))
  944. chk("6.3 全部部门索引计数正确", badD == 0, "0 错", fmt.Sprintf("%d 错", badD))
  945. badA := 0
  946. for _, a := range []int64{int64(bulkAgeTo), int64(ageBase), int64(ageBase) + 1, 40, 55} {
  947. c, err := empT.Where("Age", "=", a).Count()
  948. if err != nil || c != ageWant[a] {
  949. badA++
  950. fmt.Printf(" [Age 计数异常] %d 期望=%d 实际=%d err=%v\n", a, ageWant[a], c, err)
  951. }
  952. }
  953. chk("6.3 抽样 Age 索引计数正确", badA == 0, "0 错", fmt.Sprintf("%d 错", badA))
  954. sum := int64(0)
  955. for _, v := range deptWant {
  956. sum += v
  957. }
  958. chk("6.3 部门计数之和 == 存活行数", sum == nEmp-delCount,
  959. strconv.FormatInt(nEmp-delCount, 10), strconv.FormatInt(sum, 10))
  960. _ = scanned
  961. }
  962. }
  963. // ============================== P7 Schema 变更 ==============================
  964. func expectEmpRaw(no int64) map[string]interface{} {
  965. return map[string]interface{}{
  966. "_id": pkIDOf(no),
  967. "No": uint64(no),
  968. "Name": []byte(expName(no)),
  969. "Age": uint64(expAge(no)),
  970. "Score": uint64(expScore(no)),
  971. "Dept": []byte(expDept(no)),
  972. "Salary": uint64(expSalary(no)),
  973. "Active": expActive(no),
  974. }
  975. }
  976. func expectEmpVals(no int64, pk, level uint64) []types.FieldValue {
  977. return []types.FieldValue{
  978. {FieldName: []byte("_id"), Value: pk},
  979. {FieldName: []byte("No"), Value: uint64(no)},
  980. {FieldName: []byte("Name"), Value: []byte(expName(no))},
  981. {FieldName: []byte("Age"), Value: uint64(expAge(no))},
  982. {FieldName: []byte("Score"), Value: uint64(expScore(no))},
  983. {FieldName: []byte("Dept"), Value: []byte(expDept(no))},
  984. {FieldName: []byte("Salary"), Value: uint64(expSalary(no))},
  985. {FieldName: []byte("Active"), Value: expActiveByte(no)},
  986. {FieldName: []byte("Level"), Value: level},
  987. }
  988. }
  989. func calcDeptWant(nExtra int64) map[string]int64 {
  990. m := map[string]int64{}
  991. for i := int64(1); i <= nEmp; i++ {
  992. if isDeletedNo(i) {
  993. continue
  994. }
  995. m[expDept(i)]++
  996. }
  997. for k := int64(1); k <= nExtra; k++ {
  998. m[expDept(nEmp+k)]++
  999. }
  1000. return m
  1001. }
  1002. // verifyEmpCoreScan 纯 core 视角全表扫描校验 (不经过 ORM 结构体映射)
  1003. func verifyEmpCoreScan(empCore *core.Table, label string, wantRows int64, wantDept bool) (int64, int64) {
  1004. t := time.Now()
  1005. var scanned, bad int64
  1006. var first string
  1007. must(empCore.Query().Scan(func(r []types.FieldValue) bool {
  1008. scanned++
  1009. m := rowMap(r)
  1010. // No 列现在是 int64 (保序 Int64 编码), 旧实现按 uint64 断言会拿到零值,
  1011. // 导致整表每行都被判错。兼容两种整数形态。
  1012. var ino int64
  1013. switch v := m["No"].(type) {
  1014. case int64:
  1015. ino = v
  1016. case uint64:
  1017. ino = int64(v)
  1018. }
  1019. if isDeletedNo(ino) {
  1020. bad++
  1021. if first == "" {
  1022. first = fmt.Sprintf("已删除行仍存活 No=%d", ino)
  1023. }
  1024. return true
  1025. }
  1026. cols := make([]string, 0, 7)
  1027. if dumpVal(m["Name"]) != expName(ino) {
  1028. cols = append(cols, "Name")
  1029. }
  1030. if dumpVal(m["Age"]) != strconv.FormatInt(expAge(ino), 10) {
  1031. cols = append(cols, "Age")
  1032. }
  1033. if dumpVal(m["Score"]) != strconv.FormatInt(expScore(ino), 10) {
  1034. cols = append(cols, "Score")
  1035. }
  1036. if wantDept && dumpVal(m["Dept"]) != expDept(ino) {
  1037. cols = append(cols, "Dept")
  1038. }
  1039. if dumpVal(m["Salary"]) != strconv.FormatUint(uint64(expSalary(ino)), 10) {
  1040. cols = append(cols, "Salary")
  1041. }
  1042. if dumpVal(m["Active"]) != dumpVal(expActive(ino)) {
  1043. cols = append(cols, "Active")
  1044. }
  1045. if dumpVal(m["_id"]) != strconv.FormatUint(pkIDOf(ino), 10) {
  1046. cols = append(cols, "_id")
  1047. }
  1048. if len(cols) > 0 {
  1049. bad++
  1050. if first == "" {
  1051. first = fmt.Sprintf("No=%d 列不符: %s", ino, strings.Join(cols, ","))
  1052. }
  1053. }
  1054. return true
  1055. }))
  1056. perf("校验", "core 全表扫描 "+label, scanned, time.Since(t))
  1057. if first != "" {
  1058. fmt.Printf(" 首个错行: %s\n", first)
  1059. }
  1060. chk("core 全表扫描 "+label+" 行数", scanned == wantRows,
  1061. strconv.FormatInt(wantRows, 10), strconv.FormatInt(scanned, 10))
  1062. return scanned, bad
  1063. }
  1064. func phase7Schema(db *DB.DB, empT *DB.Table[Emp], empCore *core.Table) {
  1065. phase("P7 Schema 变更: 增列 / 删列 / 改列名 (含数据正确性校验)")
  1066. rowsBefore := empCore.TotalRows()
  1067. sub("7.0 变更前基线")
  1068. fmt.Printf(" 活跃列: %v\n", empCore.GetActiveFieldNames())
  1069. fmt.Printf(" TotalRows=%d | 字段数 总%d/活跃%d\n",
  1070. rowsBefore, len(empCore.GetFields()), len(empCore.GetActiveFieldNames()))
  1071. row0 := coreRow(empCore, "pk_emp", uint64(sampleNo))
  1072. fmt.Printf(" 基线行 No=%d → %s\n", sampleNo, dumpRow2(row0))
  1073. // ---- 7.1 ADD COLUMN ----
  1074. sub("7.1 ADD COLUMN Level (8 字节新列)")
  1075. must(empCore.AddField(types.FieldInfo{FieldType: types.FieldType_Uint64, FieldName: []byte("Level")}))
  1076. cols := empCore.GetActiveFieldNames()
  1077. chk("7.1 活跃列含 Level", contains(cols, "Level"), "含 Level", fmt.Sprintf("%v", cols))
  1078. row := coreRow(empCore, "pk_emp", uint64(sampleNo))
  1079. _, hasLevel := row["Level"]
  1080. chk("7.1 老行不返回新增列(缺失而非零值)", !hasLevel, "无 Level", fmt.Sprintf("有=%v", hasLevel))
  1081. eq, diff := rawEqual(row, expectEmpRaw(sampleNo))
  1082. chk("7.1 老行其它列不受加列影响", eq, "全等", diff)
  1083. sampled, missLevel := int64(0), int64(0)
  1084. must(empCore.Query().Limit(2000).Scan(func(r []types.FieldValue) bool {
  1085. sampled++
  1086. if _, ok := rowMap(r)["Level"]; !ok {
  1087. missLevel++
  1088. }
  1089. return true
  1090. }))
  1091. chk("7.1 抽样老行均不含 Level", missLevel == sampled,
  1092. fmt.Sprintf("%d/%d", sampled, sampled), fmt.Sprintf("%d/%d", missLevel, sampled))
  1093. bad := int64(0)
  1094. newPKs := make([]uint64, 0, extraRows)
  1095. for k := int64(1); k <= extraRows; k++ {
  1096. no := nEmp + k
  1097. // 这里要的是"插入一条指定 pkID 的新行", 用显式主键 API。
  1098. // 不能再用 TotalRows()+1 去凑: 自增编号只增不减, 删除后不会回退 (见 T4 探针),
  1099. // 靠行数现算出来的号会与它期望的 pkID 对不上。
  1100. pk := pkIDOf(no)
  1101. got, err := empCore.InsertRowWithID(pk, expectEmpVals(no, pk, 777))
  1102. if err != nil || got != pk {
  1103. bad++
  1104. fmt.Printf(" [新行插入异常] No=%d pk=%d err=%v\n", no, pk, err)
  1105. continue
  1106. }
  1107. newPKs = append(newPKs, pk)
  1108. }
  1109. chk("7.1 加列后新行插入成功", bad == 0, "0 错", fmt.Sprintf("%d 错", bad))
  1110. r1 := coreRow(empCore, "pk_emp", newPKs[0])
  1111. lv, okLv := r1["Level"]
  1112. chk("7.1 新行返回 Level=777", okLv && dumpVal(lv) == "777", "777", dumpVal(lv))
  1113. eq1, diff1 := rawEqual(r1, expectEmpRaw(nEmp+1))
  1114. chk("7.1 新行其它列值正确", eq1, "全等", diff1)
  1115. // ---- 7.2 在新增列上建索引 ----
  1116. sub("7.2 在新增列上建索引")
  1117. lvIdx := fieldIndexByName(empCore, "Level")
  1118. if lvIdx < 0 {
  1119. must(fmt.Errorf("找不到 Level 字段的 master 索引"))
  1120. }
  1121. _, err := empCore.CreateIndex("idx_emp_level", page.IndexTypeBTree, []byte{byte(lvIdx)})
  1122. must(err)
  1123. c7, err7 := empCore.Query().Where(core.Eq("Level", uint64(777), types.FieldType(0))).Count()
  1124. perf("P7 校验", "新增列索引 Count(Level=777)", extraRows, time.Millisecond)
  1125. chk("7.2 新增列索引能查到新行", err7 == nil && c7 == extraRows,
  1126. strconv.FormatInt(extraRows, 10), fmt.Sprintf("%d err=%v", c7, err7))
  1127. if err7 == nil && c7 == 0 {
  1128. problem("重要限制", "CreateIndex 不回填存量行",
  1129. "在已有数据的表上新建二级索引, 索引中不含任何存量行: 本例新行(3 行 Level=777)在 CreateIndex 之前已插入, 建好后 Count(Level=777) 仍为 0。只有建索引之后新插入的行才进索引; 引擎未提供公开的『重建全部二级索引』接口。")
  1130. }
  1131. c0, err0 := empCore.Query().Where(core.Eq("Level", uint64(0), types.FieldType(0))).Count()
  1132. chk("7.2 老行在新增列上视为无值(不落索引)", err0 == nil && c0 == 0,
  1133. "0", fmt.Sprintf("%d err=%v", c0, err0))
  1134. // ---- 7.3 DROP COLUMN ----
  1135. sub("7.3 DROP COLUMN Level (先 DropIndex 再 DropField)")
  1136. must(empCore.DropIndex("idx_emp_level"))
  1137. must(empCore.DropField("Level"))
  1138. cols = empCore.GetActiveFieldNames()
  1139. chk("7.3 活跃列不含 Level", !contains(cols, "Level"), "不含 Level", fmt.Sprintf("%v", cols))
  1140. _, stillThere := empCore.IndexMeta().FindIndex("idx_emp_level")
  1141. chk("7.3 索引已删除", !stillThere, "不存在", fmt.Sprintf("仍存在=%v", stillThere))
  1142. row = coreRow(empCore, "pk_emp", uint64(sampleNo))
  1143. eq2, diff2 := rawEqual(row, expectEmpRaw(sampleNo))
  1144. chk("7.3 删列后老行其它列完好", eq2, "全等", diff2)
  1145. _, hasAfterDrop := row["Level"]
  1146. chk("7.3 删列后老行不再有 Level", !hasAfterDrop, "无 Level", fmt.Sprintf("有=%v", hasAfterDrop))
  1147. r1 = coreRow(empCore, "pk_emp", newPKs[0])
  1148. _, has := r1["Level"]
  1149. eq3, diff3 := rawEqual(r1, expectEmpRaw(nEmp+1))
  1150. chk("7.3 新行的 Level 列已不可见", !has, "无 Level", fmt.Sprintf("有=%v", has))
  1151. chk("7.3 新行其它列完好", eq3, "全等", diff3)
  1152. chk("7.3 删列不影响行数", empCore.TotalRows() == uint64(int64(rowsBefore)+extraRows),
  1153. strconv.FormatInt(int64(rowsBefore)+extraRows, 10), strconv.FormatUint(empCore.TotalRows(), 10))
  1154. // ---- 7.4 改列名 ----
  1155. sub("7.4 改列名 (引擎无 RENAME API, 只能 DROP 旧列 + ADD 新列)")
  1156. oldDept := dumpVal(row0["Dept"])
  1157. // 复活软删除的列必须沿用【原类型 + 原宽度码】: master registry 的类型/参数是所有历史
  1158. // 版本共用的行偏移依据 (见 getVersionedOffset / rowDataSizeForVersion), 换成别的类型/
  1159. // 宽度会让已落盘的旧行整段错位, 引擎会直接拒。所以先把原定义抄下来, 7.5 用它复活。
  1160. deptDef := empCore.Field[fieldIndexByName(empCore, "Dept")]
  1161. must(empCore.DropIndex("idx_emp_dept"))
  1162. must(empCore.DropField("Dept"))
  1163. must(empCore.AddField(types.FieldInfo{FieldType: types.FieldType_String, Flag: types.WidthCodeFor(32), FieldName: []byte("DeptName")}))
  1164. r := coreRow(empCore, "pk_emp", uint64(sampleNo))
  1165. _, hasOld := r["Dept"]
  1166. _, hasNew := r["DeptName"]
  1167. chk("7.4 改名后老列名不可读", !hasOld, "无 Dept", fmt.Sprintf("有=%v", hasOld))
  1168. chk("7.4 改名后新列名在老行上无值", !hasNew, "无 DeptName", fmt.Sprintf("有=%v", hasNew))
  1169. fmt.Printf(" 证据: 老行 Dept 原值=%q, 改名后既读不到 Dept 也读不到 DeptName\n", oldDept)
  1170. problem("设计约束", "引擎没有 RENAME COLUMN / RENAME TABLE API",
  1171. "改列名只能 DROP 旧列 + ADD 新列; 老行原值不会被迁移到新列名(数据仍在行内, 只是列名不可达), 需要业务侧自行回填")
  1172. // ---- 7.5 复活同名列 ----
  1173. // 用 7.4 抄下来的原定义 (类型 + 宽度码) 复活; 只有"同名同类型"才允许复活,
  1174. // 想换宽度必须改用新列名 (见 Table.AddField 的复活分支)。
  1175. sub("7.5 重新加回同名 Dept 列 (DropField 实为软删除)")
  1176. must(empCore.AddField(deptDef))
  1177. r = coreRow(empCore, "pk_emp", uint64(sampleNo))
  1178. got, okGot := r["Dept"]
  1179. chk("7.5 同名列复活后老行数据可见", okGot, "可见", fmt.Sprintf("可见=%v", okGot))
  1180. chk("7.5 复活值与删列前完全一致", dumpVal(got) == oldDept, oldDept, dumpVal(got))
  1181. eq4, diff4 := rawEqual(r, expectEmpRaw(sampleNo))
  1182. chk("7.5 复活后老行全列正确", eq4, "全等", diff4)
  1183. problem("重要语义", "DROP COLUMN 是软删除, 数据仍留在行内; 重新 ADD 同名列会\"复活\"老行数据",
  1184. "这解释了为什么改列名(删+加新名)后数据看似丢失, 但加回旧列名后又能读到 → 敏感数据无法靠删列来清除, 需要重建表/覆写")
  1185. // ---- 7.6 重建索引 + 全表校验 ----
  1186. sub("7.6 重建 Dept 索引 + core 层全表校验")
  1187. dIdx := fieldIndexByName(empCore, "Dept")
  1188. _, ciErr := empCore.CreateIndex("idx_emp_dept", page.IndexTypeBTree, []byte{byte(dIdx)})
  1189. must(ciErr)
  1190. want := calcDeptWant(extraRows)
  1191. badD := 0
  1192. t := time.Now()
  1193. for i := int64(0); i < deptCount; i++ {
  1194. c, err := empT.Where("Dept", "=", deptCode(i)).Count()
  1195. if err != nil || c != want[deptCode(i)] {
  1196. badD++
  1197. fmt.Printf(" [部门计数异常] %s 期望=%d 实际=%d err=%v\n", deptCode(i), want[deptCode(i)], c, err)
  1198. }
  1199. }
  1200. perf("P7 校验", fmt.Sprintf("重建索引后 索引计数复核 Dept × %d", deptCount), expectRows(), time.Since(t))
  1201. chk("7.6 重建索引后部门计数全对", badD == 0, "0 错", fmt.Sprintf("%d 错", badD))
  1202. if badD == int(deptCount) {
  1203. problem("重要限制", "DROP 后重建的二级索引同样不回填存量行",
  1204. "idx_emp_dept 删除后重新 CreateIndex, 对全部存量行按索引计数均为 0; 而紧随其后的 core 逐行全表扫描校验 0 错 → 数据本身完好, 只是索引不含存量行。")
  1205. }
  1206. _, badRows := verifyEmpCoreScan(empCore, "P7 变更后全量校验", expectRows(), true)
  1207. chk("7.6 P7 变更后逐行公式校验 0 错", badRows == 0, "0", strconv.FormatInt(badRows, 10))
  1208. _ = db
  1209. }
  1210. func dumpRow2(m map[string]interface{}) string {
  1211. keys := make([]string, 0, len(m))
  1212. for k := range m {
  1213. keys = append(keys, k)
  1214. }
  1215. // 固定顺序输出, 便于对比
  1216. order := []string{"_id", "No", "Name", "Age", "Score", "Dept", "Salary", "Active", "Level", "DeptName"}
  1217. parts := make([]string, 0, len(keys))
  1218. for _, k := range order {
  1219. if v, ok := m[k]; ok {
  1220. parts = append(parts, fmt.Sprintf("%s=%s", k, dumpVal(v)))
  1221. }
  1222. }
  1223. return strings.Join(parts, " ")
  1224. }
  1225. // ============================== P8 引擎语义探针 ==============================
  1226. func coreRows(pdb *DB.DB, name string) uint64 {
  1227. if t, ok := pdb.CoreDB().FindTable(name); ok {
  1228. return t.TotalRows()
  1229. }
  1230. return 0
  1231. }
  1232. func phase8Probes() {
  1233. phase("P8 引擎语义探针 (独立库 " + filepath.Base(probePath) + ", 不影响主库)")
  1234. pdb, err := DB.Open(probePath)
  1235. must(err)
  1236. defer func() { must(pdb.Close()) }()
  1237. probeLayoutAssumption(pdb)
  1238. probeDanglingIndex(pdb)
  1239. probeExplicitPK(pdb)
  1240. probePKReuse(pdb)
  1241. probeFloatTrunc(pdb)
  1242. }
  1243. func probeLayoutAssumption(pdb *DB.DB) {
  1244. sub("T1 core 建表(无 _id 列) 时, Delete/Update 如何确定 pkID")
  1245. ct, err := pdb.CoreDB().CreateTable("probe_layout", []types.FieldInfo{
  1246. {FieldType: types.FieldType_Uint64, FieldName: []byte("No")},
  1247. {FieldType: types.FieldType_String, Flag: types.WidthCodeFor(32), FieldName: []byte("Label")},
  1248. })
  1249. if softMust("CreateTable(probe_layout)", err) {
  1250. return
  1251. }
  1252. _, err = ct.CreateIndex("pk_probe_layout", page.IndexTypePrimary, []byte{0})
  1253. if softMust("CreateIndex(pk_probe_layout)", err) {
  1254. return
  1255. }
  1256. for i, no := range []uint64{5, 5, 5, 6, 7} {
  1257. _, err := ct.InsertRow([]types.FieldValue{
  1258. {FieldName: []byte("No"), Value: no},
  1259. {FieldName: []byte("Label"), Value: []byte(fmt.Sprintf("L%d", i+1))},
  1260. })
  1261. if softMust(fmt.Sprintf("InsertRow(No=%d)", no), err) {
  1262. return
  1263. }
  1264. }
  1265. one, err := ct.SearchByIndex("pk_probe_layout", pkKey(1))
  1266. if softMust("SearchByIndex(pkID=1)", err) {
  1267. one = nil
  1268. }
  1269. fmt.Printf(" 表 = [No(8B), Label(32B)], 无 _id 列; 引擎反序列化出的行: %s\n", dumpRow(one))
  1270. fmt.Printf(" 数据: pkID 1→No5, 2→No5, 3→No5, 4→No6, 5→No7\n")
  1271. cnt, errD := ct.Delete().Where(core.Eq("No", uint64(5), types.FieldType(0))).Exec()
  1272. alive := map[uint64]uint64{}
  1273. for pk := uint64(1); pk <= 5; pk++ {
  1274. if r, err := ct.SearchByIndex("pk_probe_layout", pkKey(pk)); err == nil {
  1275. if v, ok := rowMap(r)["No"].(uint64); ok {
  1276. alive[pk] = v
  1277. }
  1278. }
  1279. }
  1280. fmt.Printf(" Delete(No==5) → cnt=%d err=%v | 剩余: ", cnt, errD)
  1281. for pk := uint64(1); pk <= 5; pk++ {
  1282. if v, ok := alive[pk]; ok {
  1283. fmt.Printf("pkID%d(No=%d) ", pk, v)
  1284. }
  1285. }
  1286. fmt.Printf("| TotalRows=%d\n", ct.TotalRows())
  1287. if len(alive) == 0 && ct.TotalRows() > 0 {
  1288. problem("严重", "core 主键索引建在非 _id 列上时, 行按 pkID 不可达",
  1289. "表 [No(8B), Label(32B)] 建 IndexTypePrimary(field 0=No) 并插入 5 行: TotalRows=5、Query().Count()=5 数据可见, 但 SearchByIndex(pk_..., pkKey(1..5)) 全部 key not found, 只有 1 个键能命中 → 主键 B+ 树未按 pkID 建立完整条目。纯 core 表应把 _id 作为首列主键。")
  1290. }
  1291. aliveNo5 := int64(0)
  1292. for _, v := range alive {
  1293. if v == 5 {
  1294. aliveNo5++
  1295. }
  1296. }
  1297. chk("T1 批量删除影响行数应等于匹配行数 3", cnt == 3, "3", strconv.FormatInt(cnt, 10))
  1298. chk("T1 被匹配的行(No=5)应全部删除", aliveNo5 == 0, "0 行", fmt.Sprintf("%d 行", aliveNo5))
  1299. _, keep5 := alive[5]
  1300. chk("T1 未被匹配的行(pkID=5,No=7)不应被删", keep5, "存活", fmt.Sprintf("存活=%v", keep5))
  1301. if cnt != 3 || aliveNo5 != 0 || !keep5 {
  1302. problem("严重", "core 层 Deleter.Exec 用\"行内第一个 uint64 列的值\"当 pkID",
  1303. fmt.Sprintf("条件 No==5 匹配 pkID 1/2/3 三行, 但 Deleter 把匹配行第一个 uint64 列的值 5 当成 pkID, 于是删掉了 pkID=5 的行(其 No=7, 未匹配), 真正匹配的 3 行一行未删 (cnt=%d)", cnt))
  1304. }
  1305. cnt2, errU := ct.Update().Set("Label", []byte("X")).
  1306. Where(core.Eq("No", uint64(6), types.FieldType(0))).Exec()
  1307. fmt.Printf(" Update(No==6 → Label=X) → cnt=%d err=%v\n", cnt2, errU)
  1308. if errU != nil {
  1309. problem("严重", "同一假设影响 core 层 Updater.Exec",
  1310. fmt.Sprintf("匹配行的第一个 uint64 值(6)被当成 pkID 去 UpdateRow → err=%v", errU))
  1311. } else if cnt2 != 1 {
  1312. problem("严重", "同一假设影响 core 层 Updater.Exec",
  1313. fmt.Sprintf("影响行数 %d != 匹配数 1", cnt2))
  1314. }
  1315. fmt.Printf(" 对照: ORM 表的行首列是 _id(=pkID), 因此 ORM 路径不受影响; 纯 core 表需自行保证首列语义\n")
  1316. }
  1317. func probeDanglingIndex(pdb *DB.DB) {
  1318. sub("T2 只删列不删索引: 残留索引的行为")
  1319. ct, err := pdb.CoreDB().CreateTable("probe_dangling", []types.FieldInfo{
  1320. {FieldType: types.FieldType_Uint64, FieldName: []byte("No")},
  1321. {FieldType: types.FieldType_Uint64, FieldName: []byte("Tag")},
  1322. })
  1323. if softMust("CreateTable(probe_dangling)", err) {
  1324. return
  1325. }
  1326. _, err = ct.CreateIndex("pk_probe_dangling", page.IndexTypePrimary, []byte{0})
  1327. if softMust("CreateIndex(pk_probe_dangling)", err) {
  1328. return
  1329. }
  1330. _, err = ct.CreateIndex("idx_probe_tag", page.IndexTypeBTree, []byte{1})
  1331. if softMust("CreateIndex(idx_probe_tag)", err) {
  1332. return
  1333. }
  1334. for i := uint64(1); i <= 5; i++ {
  1335. _, err := ct.InsertRow([]types.FieldValue{
  1336. {FieldName: []byte("No"), Value: i},
  1337. {FieldName: []byte("Tag"), Value: i * 10},
  1338. })
  1339. if softMust(fmt.Sprintf("InsertRow(No=%d)", i), err) {
  1340. return
  1341. }
  1342. }
  1343. c0, err0 := ct.Query().Where(core.Eq("Tag", uint64(20), types.FieldType(0))).Count()
  1344. fmt.Printf(" 删列前 Count(Tag==20)=%d err=%v\n", c0, err0)
  1345. if softMust("DropField(Tag)", ct.DropField("Tag")) {
  1346. return
  1347. }
  1348. fmt.Printf(" 删列后活跃列=%v\n", ct.GetActiveFieldNames())
  1349. _, dangling := ct.IndexMeta().FindIndex("idx_probe_tag")
  1350. fmt.Printf(" 残留索引 idx_probe_tag 仍存在=%v\n", dangling)
  1351. _, errQ := ct.Query().Where(core.Eq("Tag", uint64(20), types.FieldType(0))).Count()
  1352. fmt.Printf(" 删列后按残留索引查询 Tag==20 → err=%v\n", errQ)
  1353. _, errI := ct.InsertRow([]types.FieldValue{{FieldName: []byte("No"), Value: uint64(6)}})
  1354. fmt.Printf(" 删列后插入不含 Tag 的新行 → err=%v TotalRows=%d\n", errI, ct.TotalRows())
  1355. r6, err6 := ct.SearchByIndex("pk_probe_dangling", pkKey(6))
  1356. fmt.Printf(" 读回新行: %s err=%v\n", dumpRow(r6), err6)
  1357. chk("T2 残留索引不影响插入(缺失列应被跳过)", errI == nil, "nil", fmt.Sprintf("%v", errI))
  1358. if errQ != nil {
  1359. problem("使用约束", "DropField 不会自动删除引用该列的索引",
  1360. fmt.Sprintf("残留索引会导致按该列查询直接报错(err=%v); 正确顺序是先 DropIndex 再 DropField", errQ))
  1361. }
  1362. }
  1363. func probeExplicitPK(pdb *DB.DB) {
  1364. sub("T3 ORM 显式指定 _id ≠ 引擎 pkID")
  1365. pt, err := DB.UseTable[Emp](pdb, "probe_emp")
  1366. if softMust("UseTable(probe_emp)", err) {
  1367. return
  1368. }
  1369. e := &Emp{BaseModel: DB.BaseModel{ID: 999999}, No: 1, Name: "P1", Age: 30, Score: 1, Dept: "Dept000", Salary: 1, Active: true}
  1370. errI := pt.Insert(e)
  1371. fmt.Printf(" Insert(ID=999999) → err=%v, 对象 ID=%d\n", errI, e.ID)
  1372. _, errF := pt.Find(999999)
  1373. fmt.Printf(" Find(999999) → err=%v\n", errF)
  1374. real, errR := pt.Find(1)
  1375. if errR == nil {
  1376. fmt.Printf(" Find(1) → ID=%d No=%d (说明引擎分配的 pkID 是 1, 与 _id 列 999999 不一致)\n", real.ID, real.No)
  1377. } else {
  1378. fmt.Printf(" Find(1) → err=%v\n", errR)
  1379. }
  1380. chk("T3 按指定的 _id 应能查到该行", errF == nil, "查得到", fmt.Sprintf("err=%v", errF))
  1381. if errF != nil {
  1382. problem("风险", "ORM 显式指定的主键值会被静默忽略, 产生\"插入成功但按主键查不到\"的行",
  1383. fmt.Sprintf("引擎 pkID 恒为 TotalRows+1; 写 ID=999999 后 Find(999999) 失败, 只有 Find(1) 能查到, 且该行 _id 列值仍是 999999 → _id 与真实 pkID 失配, 之后 Delete/批量删除会按 _id 操作到错误的行 (err=%v)", errF))
  1384. }
  1385. }
  1386. func probePKReuse(pdb *DB.DB) {
  1387. sub("T4 删除中间 pkID 后再插入: pkID 复用与碰撞")
  1388. pr, err := DB.UseTable[Emp](pdb, "probe_reuse")
  1389. if softMust("UseTable(probe_reuse)", err) {
  1390. return
  1391. }
  1392. for i := int64(1); i <= 3; i++ {
  1393. if softMust(fmt.Sprintf("Insert(No=%d)", i), pr.Insert(&Emp{No: i, Name: empName(i), Age: 20 + i, Score: i, Dept: "Dept000", Salary: float64(i), Active: true})) {
  1394. return
  1395. }
  1396. }
  1397. if softMust("Delete(pkID=2)", pr.Delete(2)) {
  1398. return
  1399. }
  1400. cnt, _ := pr.Count()
  1401. fmt.Printf(" 插 3 行(pkID 1,2,3) → 删 pkID=2 → Count=%d TotalRows=%d\n", cnt, coreRows(pdb, "probe_reuse"))
  1402. if softMust("Insert(No=99)", pr.Insert(&Emp{No: 99, Name: empName(99), Age: 99, Score: 9, Dept: "Dept000", Salary: 9, Active: true})) {
  1403. return
  1404. }
  1405. type kv struct {
  1406. ID uint64
  1407. No int64
  1408. }
  1409. rows := make([]kv, 0, 4)
  1410. if softMust("Select(probe_reuse)", pr.Select(func(e *Emp) bool { rows = append(rows, kv{e.ID, e.No}); return true })) {
  1411. return
  1412. }
  1413. fmt.Printf(" 再插 1 行(No=99) 后存活: ")
  1414. for _, r := range rows {
  1415. fmt.Printf("{_id=%d,No=%d} ", r.ID, r.No)
  1416. }
  1417. fmt.Printf("| Count=%d TotalRows=%d\n", len(rows), coreRows(pdb, "probe_reuse"))
  1418. hasNo3 := false
  1419. for _, r := range rows {
  1420. if r.No == 3 {
  1421. hasNo3 = true
  1422. }
  1423. }
  1424. chk("T4 新插入不应覆盖既有存活行(No=3 应仍在, 共 3 行)", hasNo3 && len(rows) == 3,
  1425. "3 行且含 No=3", fmt.Sprintf("%d 行 含No3=%v", len(rows), hasNo3))
  1426. if !hasNo3 || len(rows) != 3 {
  1427. problem("严重", "pkID = TotalRows+1 且删除会回退 TotalRows → 删中间行后再插入会复用已占用的 pkID",
  1428. fmt.Sprintf("删 pkID=2 后 TotalRows 回退为 2, 下一条插入分到 pkID=3, 与仍存活的 pkID=3 冲突, 实测存活 %d 行(期望 3, 含 No=3)", len(rows)))
  1429. }
  1430. }
  1431. func probeFloatTrunc(pdb *DB.DB) {
  1432. sub("T5 float64 列的小数精度")
  1433. pt, err := DB.UseTable[Emp](pdb, "probe_float")
  1434. if softMust("UseTable(probe_float)", err) {
  1435. return
  1436. }
  1437. softMust("Insert(Salary=123.75)", pt.Insert(&Emp{No: 1, Name: "F", Age: 1, Score: 1, Dept: "Dept000", Salary: 123.75, Active: true}))
  1438. got, err2 := pt.Find(1)
  1439. if err2 == nil {
  1440. fmt.Printf(" 写入 Salary=123.75 → 读回 %v\n", got.Salary)
  1441. }
  1442. chk("T5 float64 的小数部分应保留", err2 == nil && got.Salary == 123.75,
  1443. "123.75", fmt.Sprintf("%v err=%v", got, err2))
  1444. if got != nil && got.Salary != 123.75 {
  1445. problem("风险", "ORM 把 float64 映射成 8 字节整数列, 小数被静默截断",
  1446. fmt.Sprintf("写入 123.75 读回 %v; 金额/比例等小数数据不能直接存 float64 列(需业务侧放大成整数)", got.Salary))
  1447. }
  1448. }
  1449. // ============================== P9 关库重开 ==============================
  1450. func phase9Reopen(path string) {
  1451. phase("P9 关库重开: 持久化 / schema / 索引 校验")
  1452. db2, err := DB.Open(path)
  1453. must(err)
  1454. defer func() { must(db2.Close()) }()
  1455. empCore, ok := db2.CoreDB().FindTable("emp")
  1456. if !ok {
  1457. must(fmt.Errorf("重开后找不到 emp 表"))
  1458. }
  1459. cols := empCore.GetActiveFieldNames()
  1460. fmt.Printf(" 重开后 emp 活跃列: %v\n", cols)
  1461. fmt.Printf(" TotalRows=%d (期望 %d) | 磁盘 %.1fMB\n",
  1462. empCore.TotalRows(), expectRows(), float64(dbSize(path))/(1<<20))
  1463. chk("9.1 重开后 TotalRows 一致", empCore.TotalRows() == uint64(expectRows()),
  1464. strconv.FormatInt(expectRows(), 10), strconv.FormatUint(empCore.TotalRows(), 10))
  1465. chk("9.1 重开后 schema 保留 P7 结果", contains(cols, "Dept") && contains(cols, "DeptName"),
  1466. "含 Dept 与 DeptName", fmt.Sprintf("%v", cols))
  1467. _, bad := verifyEmpCoreScan(empCore, "重开后 core 全量校验", expectRows(), true)
  1468. chk("9.1 重开后逐行公式校验 0 错", bad == 0, "0", strconv.FormatInt(bad, 10))
  1469. idxOK := 0
  1470. for _, nm := range []string{"pk_emp", "idx_emp_no", "idx_emp_age", "idx_emp_dept"} {
  1471. if _, ok := empCore.IndexMeta().FindIndex(nm); ok {
  1472. idxOK++
  1473. }
  1474. }
  1475. chk("9.1 主键+三个二级索引均存在", idxOK == 4, "4", strconv.Itoa(idxOK))
  1476. sub("9.2 ORM 重开表 (UseTable 会按 struct 同步 schema)")
  1477. empT2, err := DB.UseTable[Emp](db2, "emp")
  1478. must(err)
  1479. cols2 := empCore.GetActiveFieldNames()
  1480. fmt.Printf(" UseTable[Emp] 之后活跃列: %v\n", cols2)
  1481. if contains(cols, "DeptName") && !contains(cols2, "DeptName") {
  1482. problem("风险", "ORM 重开表会按 struct 强制同步 schema, 静默删除 struct 里没有的列",
  1483. fmt.Sprintf("纯 core 视角活跃列=%v 含 DeptName, 调用 UseTable[Emp] 后活跃列=%v, DeptName 被自动 DropField; 手工加过列的表被 ORM 打开就会\"丢列\"(数据仍在行内, 再加回同名列可复活)", cols, cols2))
  1484. }
  1485. scanned, bad2 := verifyEmpORM(empT2, "重开后 ORM 全量校验", expectRows())
  1486. chk("9.2 ORM 重开后可读全表", scanned == expectRows() && bad2 == 0,
  1487. fmt.Sprintf("%d 行 0 错", expectRows()), fmt.Sprintf("%d 行 %d 错", scanned, bad2))
  1488. e, errF := empT2.Find(1)
  1489. chk("9.2 重开后主键点查可用", errF == nil && e.No == 1, "No=1", fmt.Sprintf("%v err=%v", e, errF))
  1490. c, errC := empT2.Where("No", "=", int64(sampleNo)).Count()
  1491. chk("9.2 重开后索引等值查询可用", errC == nil && c == 1, "1", fmt.Sprintf("%d err=%v", c, errC))
  1492. want := int64(0)
  1493. for i := int64(1); i <= nEmp; i++ {
  1494. if !isDeletedNo(i) && expAge(i) == int64(bulkAgeTo) {
  1495. want++
  1496. }
  1497. }
  1498. for k := int64(1); k <= extraRows; k++ {
  1499. if expAge(nEmp+k) == int64(bulkAgeTo) {
  1500. want++
  1501. }
  1502. }
  1503. c2, errC2 := empT2.Where("Age", "=", int64(bulkAgeTo)).Count()
  1504. chk("9.2 重开后 Age 索引计数正确", errC2 == nil && c2 == want,
  1505. strconv.FormatInt(want, 10), fmt.Sprintf("%d err=%v", c2, errC2))
  1506. }
  1507. // ============================== P10 汇总 ==============================
  1508. func report(start time.Time) {
  1509. phase("P10 汇总报告")
  1510. fmt.Printf("\n【性能总表】共 %d 项\n", len(perfs))
  1511. fmt.Printf("%-10s %-52s %12s %13s %14s\n", "阶段", "操作", "行数", "耗时", "速率")
  1512. fmt.Printf("%s\n", strings.Repeat("-", 108))
  1513. for _, p := range perfs {
  1514. rate := "-"
  1515. if p.Cost > 0 && p.Rows > 0 {
  1516. rate = human(float64(p.Rows)/p.Cost.Seconds()) + "/s"
  1517. }
  1518. fmt.Printf("%-10s %-52s %12d %13s %14s\n", p.Phase, p.Op, p.Rows, p.Cost.Round(time.Millisecond), rate)
  1519. }
  1520. pass, fail := 0, 0
  1521. for _, c := range checks {
  1522. if c.Pass {
  1523. pass++
  1524. } else {
  1525. fail++
  1526. }
  1527. }
  1528. fmt.Printf("\n【校验总表】共 %d 项: 通过 %d, 失败 %d\n", len(checks), pass, fail)
  1529. if fail > 0 {
  1530. fmt.Printf("%-66s %-24s %s\n", "失败检查项", "期望", "实际")
  1531. for _, c := range checks {
  1532. if !c.Pass {
  1533. fmt.Printf("%-66s %-24s %s\n", c.Name, c.Expect, c.Actual)
  1534. }
  1535. }
  1536. }
  1537. fmt.Printf("\n【数据问题清单】共 %d 项\n", len(problems))
  1538. for i, p := range problems {
  1539. fmt.Printf(" %d. [%s] %s\n %s\n", i+1, p.Level, p.Title, p.Detail)
  1540. }
  1541. fmt.Printf("\n【结论】总耗时 %s | 校验 %d/%d 通过 | 数据问题 %d 项\n",
  1542. time.Since(start).Round(time.Second), pass, len(checks), len(problems))
  1543. fmt.Printf(" 主库保留: %s (%.1fMB) | 探针库: %s (%.1fMB)\n",
  1544. dbPath, float64(dbSize(dbPath))/(1<<20), probePath, float64(dbSize(probePath))/(1<<20))
  1545. fmt.Printf(" 可将上述 .db 交给 analyzer 审计\n")
  1546. }
  1547. // ============================== main ==============================
  1548. func deriveScale() {
  1549. delCount = nEmp / 100
  1550. if delCount > 100000 {
  1551. delCount = 100000
  1552. }
  1553. if delCount < 100 {
  1554. delCount = 100
  1555. }
  1556. delCount = delCount / 100 * 100
  1557. delFrom, delTo = nEmp-delCount+1, nEmp
  1558. rowUpdCnt = nEmp / 5000
  1559. if rowUpdCnt > 2000 {
  1560. rowUpdCnt = 2000
  1561. }
  1562. if rowUpdCnt < 10 {
  1563. rowUpdCnt = 10
  1564. }
  1565. rowUpdAt = nEmp/4 + 1
  1566. // 抽样行号: 取 nEmp/3, 保证存在且不落在 rowUpd 区间(仅 10 行)或尾部删除区间内
  1567. sampleNo = nEmp / 3
  1568. if sampleNo < 1 {
  1569. sampleNo = 1
  1570. }
  1571. nJoinMod = nOrder / 5
  1572. if nJoinMod > 200000 {
  1573. nJoinMod = 200000
  1574. }
  1575. if nJoinMod < 5 {
  1576. nJoinMod = 5
  1577. }
  1578. }
  1579. func main() {
  1580. start := time.Now()
  1581. nEmp = envInt64("TSDB_BENCH_N", 10_000_000)
  1582. nEmp = nEmp / 1000 * 1000
  1583. if nEmp < 1000 {
  1584. nEmp = 1000
  1585. }
  1586. nOrder = envInt64("TSDB_BENCH_ORDERS", 1_000_000)
  1587. name := os.Getenv("TSDB_BENCH_DB")
  1588. if name == "" {
  1589. name = "bench_10m.db"
  1590. }
  1591. abs, err := filepath.Abs(name)
  1592. must(err)
  1593. dbPath = abs
  1594. probePath = filepath.Join(filepath.Dir(abs), strings.TrimSuffix(filepath.Base(abs), ".db")+"_probe.db")
  1595. deriveScale()
  1596. phase0()
  1597. fmt.Printf("\n 清理同前缀旧库文件...\n")
  1598. cleanOld(dbPath)
  1599. cleanOld(probePath)
  1600. db, err := DB.Open(dbPath)
  1601. must(err)
  1602. db.SetCommitPreset("batch")
  1603. db.SetAutoCommitDelay(0)
  1604. empT, deptT, orderT, empCore := phase1(db)
  1605. phase2InsertEmp(db, empT, empCore)
  1606. phase3Queries(empT, empCore)
  1607. _, orderAfterDel := phase4Multi(db, empT, deptT, orderT, empCore)
  1608. phase5Update(db, empT, empCore)
  1609. phase6Delete(db, empT, orderT, empCore, orderAfterDel)
  1610. phase7Schema(db, empT, empCore)
  1611. must(db.Commit())
  1612. must(db.Close())
  1613. fmt.Printf("\n [已关库] 磁盘 %.1fMB | %s\n", float64(dbSize(dbPath))/(1<<20), memLine())
  1614. phase8Probes()
  1615. phase9Reopen(dbPath)
  1616. report(start)
  1617. }