main.go 62 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762
  1. // ============================================================================
  2. // TsDB2 千万级功能 + 性能测试
  3. //
  4. // 覆盖:
  5. // P2 千万级写入 (分档进度 / 分档 Commit / 磁盘与内存曲线)
  6. // P3 查询性能 (主键点查 / 二级索引等值 / 范围 / 分页 / 排序 / 投影 / 聚合)
  7. // P4 多表 (部门维度表 + 订单表 + 索引关联计数校验)
  8. // P5 更新 (单行全列重写 / core 批量更新 / 索引列更新后索引同步)
  9. // P6 删除 (逐行删 10 万行 / 批量删除全表重建) + 全量一致性校验
  10. // P7 Schema 变更 (ADD COLUMN / DROP COLUMN / 改列名) + 数据正确性校验
  11. // P8 引擎语义探针 (独立库, 不污染主库)
  12. // P9 关库重开后的持久化与 schema 校验
  13. // P10 汇总 (性能总表 / 校验总表 / 数据问题清单)
  14. //
  15. // 规模可用环境变量调整 (默认千万级):
  16. // TSDB_BENCH_N 主表行数 默认 10000000 (向下取整到 1000 的倍数)
  17. // TSDB_BENCH_ORDERS 订单表行数 默认 1000000
  18. // TSDB_BENCH_DB 主库文件名 默认 bench_10m.db (相对当前工作目录)
  19. //
  20. // 运行: cd demo/bench_10m; go run .
  21. // 注意: 启动时会删除同前缀旧库文件; 运行结束后库文件保留, 可交给 analyzer 审计。
  22. // ============================================================================
  23. package main
  24. import (
  25. "encoding/binary"
  26. "fmt"
  27. "os"
  28. "path/filepath"
  29. "runtime"
  30. "strconv"
  31. "strings"
  32. "time"
  33. DB "TsDB2/DB"
  34. core "TsDB2/DB/core"
  35. "TsDB2/DB/core/page"
  36. "TsDB2/DB/core/types"
  37. )
  38. // ============================== 规模与派生参数 ==============================
  39. const (
  40. deptCount = 100
  41. scoreMod = 1000
  42. ageBase = 20
  43. ageMod = 50
  44. bulkAgeAt = 500
  45. bulkAgeTo = 99
  46. bulkDeptAt = 321
  47. )
  48. const bulkDeptTo = "Dept099"
  49. var (
  50. nEmp int64
  51. nOrder int64
  52. nJoinMod int64
  53. delCount int64
  54. rowUpdCnt int64
  55. rowUpdAt int64
  56. delFrom int64
  57. delTo int64
  58. sampleNo int64 // P7/P9 抽样行号, 随规模派生, 保证存在且不落在更新/删除区间
  59. extraRows int64 = 3
  60. dbPath string
  61. probePath string
  62. )
  63. var (
  64. perfs []perfRow
  65. checks []checkRow
  66. problems []problemRow
  67. )
  68. // ============================== 数据公式 ==============================
  69. func empName(no int64) string { return fmt.Sprintf("Emp%08d", no) }
  70. func expName(no int64) string {
  71. if no >= rowUpdAt && no < rowUpdAt+rowUpdCnt {
  72. return "UP" + empName(no)
  73. }
  74. return empName(no)
  75. }
  76. func expAge(no int64) int64 {
  77. if no%scoreMod == bulkAgeAt {
  78. return bulkAgeTo
  79. }
  80. return ageBase + no%ageMod
  81. }
  82. func expScore(no int64) int64 { return no % scoreMod }
  83. func expDept(no int64) string {
  84. if no%scoreMod == bulkDeptAt {
  85. return bulkDeptTo
  86. }
  87. return fmt.Sprintf("Dept%03d", no%deptCount)
  88. }
  89. func expSalary(no int64) float64 { return float64(no % 100000) }
  90. func expActive(no int64) bool { return no%2 == 0 }
  91. func expActiveByte(no int64) byte {
  92. if expActive(no) {
  93. return 1
  94. }
  95. return 0
  96. }
  97. func pkIDOf(no int64) uint64 {
  98. if no <= nEmp {
  99. return uint64(no)
  100. }
  101. return uint64(nEmp-delCount + (no - nEmp))
  102. }
  103. func isDeletedNo(no int64) bool { return no >= delFrom && no <= delTo }
  104. func expectRows() int64 { return nEmp - delCount + extraRows }
  105. func expectSalarySum(n int64) float64 {
  106. full := n / 100000
  107. rem := n % 100000
  108. return float64(full)*4999950000.0 + float64(rem)*float64(rem+1)/2
  109. }
  110. // ============================== 通用小工具 ==============================
  111. func must(err error) {
  112. if err != nil {
  113. fmt.Printf("\n[FATAL] %v\n", err)
  114. os.Exit(1)
  115. }
  116. }
  117. // softMust 供探针使用: 出错时记录为数据问题并返回 true, 不终止整个进程。
  118. // 探针的目的是"发现并报告引擎语义/缺陷", 单步失败不应让整场测试无结果。
  119. func softMust(stage string, err error) bool {
  120. if err == nil {
  121. return false
  122. }
  123. fmt.Printf(" [探针错误] %s → %v\n", stage, err)
  124. problem("探针", stage, fmt.Sprintf("%v", err))
  125. return true
  126. }
  127. func phase(title string) {
  128. fmt.Printf("\n%s\n%s\n%s\n", strings.Repeat("=", 104), title, strings.Repeat("=", 104))
  129. }
  130. func sub(title string) { fmt.Printf("\n--- %s ---\n", title) }
  131. func envInt64(key string, def int64) int64 {
  132. if s := os.Getenv(key); s != "" {
  133. if v, err := strconv.ParseInt(s, 10, 64); err == nil && v > 0 {
  134. return v
  135. }
  136. }
  137. return def
  138. }
  139. func human(v float64) string {
  140. switch {
  141. case v >= 1e6:
  142. return fmt.Sprintf("%.2fM", v/1e6)
  143. case v >= 1e3:
  144. return fmt.Sprintf("%.2fk", v/1e3)
  145. default:
  146. return fmt.Sprintf("%.0f", v)
  147. }
  148. }
  149. func mbOf(v uint64) float64 { return float64(v) / (1 << 20) }
  150. func memLine() string {
  151. var m runtime.MemStats
  152. runtime.ReadMemStats(&m)
  153. return fmt.Sprintf("GoAlloc=%.0fMB Sys=%.0fMB HeapInuse=%.0fMB NumGC=%d",
  154. mbOf(m.Alloc), mbOf(m.Sys), mbOf(m.HeapInuse), m.NumGC)
  155. }
  156. func dbSize(path string) int64 {
  157. dir := filepath.Dir(path)
  158. stem := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))
  159. entries, err := os.ReadDir(dir)
  160. if err != nil {
  161. return 0
  162. }
  163. var total int64
  164. for _, e := range entries {
  165. if e.IsDir() || !strings.HasPrefix(e.Name(), stem) {
  166. continue
  167. }
  168. if fi, err := e.Info(); err == nil {
  169. total += fi.Size()
  170. }
  171. }
  172. return total
  173. }
  174. func cleanOld(path string) {
  175. dir := filepath.Dir(path)
  176. stem := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))
  177. entries, err := os.ReadDir(dir)
  178. if err != nil {
  179. return
  180. }
  181. for _, e := range entries {
  182. if !e.IsDir() && strings.HasPrefix(e.Name(), stem) {
  183. p := filepath.Join(dir, e.Name())
  184. if err := os.Remove(p); err == nil {
  185. fmt.Printf(" [清理旧库] %s\n", p)
  186. }
  187. }
  188. }
  189. }
  190. func pkKey(id uint64) []byte {
  191. b := make([]byte, 8)
  192. binary.BigEndian.PutUint64(b, id)
  193. return b
  194. }
  195. func contains(xs []string, s string) bool {
  196. for _, x := range xs {
  197. if x == s {
  198. return true
  199. }
  200. }
  201. return false
  202. }
  203. func dumpVal(v interface{}) string {
  204. if b, ok := v.([]byte); ok {
  205. return strings.TrimRight(string(b), "\x00")
  206. }
  207. return fmt.Sprint(v)
  208. }
  209. func dumpRow(row []types.FieldValue) string {
  210. parts := make([]string, 0, len(row))
  211. for _, fv := range row {
  212. parts = append(parts, fmt.Sprintf("%s=%s", string(fv.FieldName), dumpVal(fv.Value)))
  213. }
  214. return strings.Join(parts, " ")
  215. }
  216. func rowMap(row []types.FieldValue) map[string]interface{} {
  217. m := make(map[string]interface{}, len(row))
  218. for _, fv := range row {
  219. m[string(fv.FieldName)] = fv.Value
  220. }
  221. return m
  222. }
  223. func rawEqual(got, want map[string]interface{}) (bool, string) {
  224. for k, wv := range want {
  225. gv, ok := got[k]
  226. if !ok {
  227. return false, "缺少列 " + k
  228. }
  229. if dumpVal(gv) != dumpVal(wv) {
  230. return false, fmt.Sprintf("%s 期望=%s 实际=%s", k, dumpVal(wv), dumpVal(gv))
  231. }
  232. }
  233. return true, ""
  234. }
  235. func fieldIndexByName(t *core.Table, name string) int {
  236. for i, f := range t.GetFields() {
  237. if string(f.FieldName) == name {
  238. return i
  239. }
  240. }
  241. return -1
  242. }
  243. func fixedBytes(t *core.Table, col, s string) []byte {
  244. ft, ok := t.GetActiveFieldType(col)
  245. if !ok {
  246. return []byte(s)
  247. }
  248. b := make([]byte, int(types.GetSize(ft)))
  249. copy(b, s)
  250. return b
  251. }
  252. func coreRow(t *core.Table, pkIdx string, pk uint64) map[string]interface{} {
  253. row, err := t.SearchByIndex(pkIdx, pkKey(pk))
  254. if err != nil {
  255. return nil
  256. }
  257. return rowMap(row)
  258. }
  259. func sampleIDs(count, max int64) []uint64 {
  260. ids := make([]uint64, 0, count)
  261. x := uint64(0x9E3779B97F4A7C15)
  262. for int64(len(ids)) < count {
  263. x ^= x << 13
  264. x ^= x >> 7
  265. x ^= x << 17
  266. ids = append(ids, uint64(x%uint64(max))+1)
  267. }
  268. return ids
  269. }
  270. // ============================== 报告设施 ==============================
  271. type perfRow struct {
  272. Phase, Op string
  273. Rows int64
  274. Cost time.Duration
  275. Note string
  276. }
  277. type checkRow struct {
  278. Name, Expect, Actual string
  279. Pass bool
  280. }
  281. type problemRow struct {
  282. Level string
  283. Title string
  284. Detail string
  285. }
  286. func perf(ph, op string, rows int64, d time.Duration) {
  287. perfs = append(perfs, perfRow{ph, op, rows, d, ""})
  288. rate := "-"
  289. if d > 0 && rows > 0 {
  290. rate = human(float64(rows)/d.Seconds()) + "/s"
  291. }
  292. fmt.Printf(" %-52s 行数=%-9d 耗时=%-11s %s\n", op, rows, d.Round(time.Millisecond), rate)
  293. }
  294. func chk(name string, pass bool, expect, actual string) bool {
  295. checks = append(checks, checkRow{name, expect, actual, pass})
  296. if !pass {
  297. fmt.Printf(" [FAIL] %s | 期望=%s 实际=%s\n", name, expect, actual)
  298. }
  299. return pass
  300. }
  301. func problem(level, title, detail string) {
  302. problems = append(problems, problemRow{level, title, detail})
  303. fmt.Printf(" [%s] %s: %s\n", level, title, detail)
  304. }
  305. // ============================== 表结构 ==============================
  306. type Emp struct {
  307. DB.BaseModel
  308. No int64 `vrtx:"index:idx_emp_no"`
  309. Name string `vrtx:"maxlen:24"`
  310. Age int64 `vrtx:"index:idx_emp_age"`
  311. Score int64 // 故意不建索引: 用于对比"无索引条件 = 全表扫描"
  312. Dept string `vrtx:"maxlen:12,index:idx_emp_dept"`
  313. Salary float64 // 注意: ORM 把 float64 映射成 8byte 整数列, 小数会被截断
  314. Active bool
  315. }
  316. type DeptDim struct {
  317. DB.BaseModel
  318. Code string `vrtx:"maxlen:12,index:idx_dept_code"`
  319. Title string `vrtx:"maxlen:32"`
  320. Level int64
  321. }
  322. type Order struct {
  323. DB.BaseModel
  324. OrderNo int64 `vrtx:"index:idx_order_no"`
  325. EmpNo int64 `vrtx:"index:idx_order_empno"`
  326. Amount int64
  327. Status int64
  328. }
  329. func deptCode(i int64) string { return fmt.Sprintf("Dept%03d", i) }
  330. // ============================== P0 环境 ==============================
  331. func phase0() {
  332. phase("P0 环境与参数")
  333. fmt.Printf(" 时间 : %s\n", time.Now().Format("2006-01-02 15:04:05"))
  334. fmt.Printf(" Go : %s | CPU %d 核 (GOMAXPROCS=%d) | %s/%s\n",
  335. runtime.Version(), runtime.NumCPU(), runtime.GOMAXPROCS(0), runtime.GOOS, runtime.GOARCH)
  336. fmt.Printf(" 规模 : 主表 %d 行 | 订单 %d 行 | 部门 %d 行 | 单行更新 %d 次 | 逐行删除 %d 行\n",
  337. nEmp, nOrder, deptCount, rowUpdCnt, delCount)
  338. fmt.Printf(" 参数 : Age=20+No%%50 | Score=No%%1000 | Dept=Dept(No%%100) | Salary=No%%100000\n")
  339. fmt.Printf(" 索引 : pk_emp + idx_emp_no/BTree + idx_emp_age/BTree + idx_emp_dept/BTree\n")
  340. fmt.Printf(" 提交策略 : preset=batch(脏页阈值 500 页), 自动提交防抖禁用, 每 100 万行显式 Commit\n")
  341. fmt.Printf(" 主库 : %s\n", dbPath)
  342. fmt.Printf(" 探针库 : %s\n", probePath)
  343. fmt.Printf(" %s\n", memLine())
  344. }
  345. // ============================== P1 建表 ==============================
  346. func phase1(db *DB.DB) (*DB.Table[Emp], *DB.Table[DeptDim], *DB.Table[Order], *core.Table) {
  347. phase("P1 建表 / 打开表 + 索引")
  348. t := time.Now()
  349. empT, err := DB.UseTable[Emp](db, "emp")
  350. must(err)
  351. deptT, err := DB.UseTable[DeptDim](db, "dept_dim")
  352. must(err)
  353. orderT, err := DB.UseTable[Order](db, "orders")
  354. must(err)
  355. empCore, ok := db.CoreDB().FindTable("emp")
  356. if !ok {
  357. must(fmt.Errorf("core 层找不到 emp 表"))
  358. }
  359. perf("P1 建表", "UseTable × 3 (含建表/建索引)", 3, time.Since(t))
  360. fmt.Printf(" emp 活跃列 : %v\n", empCore.GetActiveFieldNames())
  361. fmt.Printf(" emp 字段数 : 总 %d / 活跃 %d | TotalRows=%d\n",
  362. len(empCore.GetFields()), len(empCore.GetActiveFieldNames()), empCore.TotalRows())
  363. for _, nm := range []string{"pk_emp", "idx_emp_no", "idx_emp_age", "idx_emp_dept"} {
  364. if m, ok := empCore.IndexMeta().FindIndex(nm); ok {
  365. ids := make([]string, 0, m.FieldCnt)
  366. for k := 0; k < int(m.FieldCnt); k++ {
  367. ids = append(ids, strconv.Itoa(int(m.FieldIDs[k])))
  368. }
  369. fmt.Printf(" 索引 %-14s type=%-2d 分量=%d fieldIDs=[%s]\n",
  370. nm, m.IndexType, m.FieldCnt, strings.Join(ids, ","))
  371. } else {
  372. fmt.Printf(" 索引 %-14s 缺失!\n", nm)
  373. }
  374. }
  375. return empT, deptT, orderT, empCore
  376. }
  377. // ============================== P2 千万级写入 ==============================
  378. func phase2InsertEmp(db *DB.DB, empT *DB.Table[Emp], empCore *core.Table) {
  379. phase(fmt.Sprintf("P2 千万级写入: 单行 Insert × %d (含进度 / 分档 Commit)", nEmp))
  380. t0 := time.Now()
  381. last, lastN := t0, int64(0)
  382. e := &Emp{}
  383. for i := int64(1); i <= nEmp; i++ {
  384. *e = Emp{
  385. BaseModel: DB.BaseModel{ID: 0},
  386. No: i,
  387. Name: empName(i),
  388. Age: int64(ageBase + i%ageMod),
  389. Score: expScore(i),
  390. Dept: fmt.Sprintf("Dept%03d", i%deptCount),
  391. Salary: expSalary(i),
  392. Active: expActive(i),
  393. }
  394. if err := empT.Insert(e); err != nil {
  395. must(fmt.Errorf("插入第 %d 行失败: %w", i, err))
  396. }
  397. if i%200000 == 0 {
  398. now := time.Now()
  399. rate := float64(i-lastN) / now.Sub(last).Seconds()
  400. eta := time.Duration(float64(nEmp-i) / rate * float64(time.Second))
  401. fmt.Printf(" [%9d 行] 累计%7.1fs | 瞬时 %8s/s | 磁盘 %7.1fMB | ETA %6s | %s\n",
  402. i, now.Sub(t0).Seconds(), human(rate), float64(dbSize(dbPath))/(1<<20),
  403. eta.Round(time.Second), memLine())
  404. last, lastN = now, i
  405. }
  406. if i%1000000 == 0 {
  407. ct := time.Now()
  408. must(db.Commit())
  409. fmt.Printf(" [%9d 行] 显式 Commit %s | 磁盘 %.1fMB\n",
  410. i, time.Since(ct).Round(time.Millisecond), float64(dbSize(dbPath))/(1<<20))
  411. }
  412. }
  413. ct := time.Now()
  414. must(db.Commit())
  415. total := time.Since(t0)
  416. perf("P2 写入", fmt.Sprintf("单行 Insert × %d + 分档 Commit", nEmp), nEmp, total)
  417. fmt.Printf(" 最终 Commit %s | 磁盘 %.1fMB | 文件 %.2fGB | %s\n",
  418. time.Since(ct).Round(time.Millisecond), float64(dbSize(dbPath))/(1<<20),
  419. float64(dbSize(dbPath))/(1<<30), memLine())
  420. chk("2.1 TotalRows == N", empCore.TotalRows() == uint64(nEmp),
  421. strconv.FormatInt(nEmp, 10), strconv.FormatUint(empCore.TotalRows(), 10))
  422. scanned, bad := verifyEmpInserted(empT, "写入后全量校验(插入态)", nEmp)
  423. chk("2.2 全表扫描行数 == N", scanned == nEmp, strconv.FormatInt(nEmp, 10), strconv.FormatInt(scanned, 10))
  424. chk("2.3 全表逐行公式校验 0 错", bad == 0, "0", strconv.FormatInt(bad, 10))
  425. }
  426. // verifyEmpORM 用 ORM 映射全表扫描并按公式逐行校验 (跳过已删除区间)
  427. func verifyEmpORM(empT *DB.Table[Emp], label string, want int64) (int64, int64) {
  428. t := time.Now()
  429. var scanned, bad int64
  430. var first string
  431. err := empT.Select(func(e *Emp) bool {
  432. scanned++
  433. no := e.No
  434. if isDeletedNo(no) {
  435. bad++
  436. if first == "" {
  437. first = fmt.Sprintf("已删除行仍存活 No=%d", no)
  438. }
  439. return true
  440. }
  441. badCols := make([]string, 0, 4)
  442. if e.Name != expName(no) {
  443. badCols = append(badCols, "Name")
  444. }
  445. if e.Age != expAge(no) {
  446. badCols = append(badCols, "Age")
  447. }
  448. if e.Score != expScore(no) {
  449. badCols = append(badCols, "Score")
  450. }
  451. if e.Dept != expDept(no) {
  452. badCols = append(badCols, "Dept")
  453. }
  454. if e.Salary != expSalary(no) {
  455. badCols = append(badCols, "Salary")
  456. }
  457. if e.Active != expActive(no) {
  458. badCols = append(badCols, "Active")
  459. }
  460. if e.ID != pkIDOf(no) {
  461. badCols = append(badCols, fmt.Sprintf("_id(%d!=%d)", e.ID, pkIDOf(no)))
  462. }
  463. if len(badCols) > 0 {
  464. bad++
  465. if first == "" {
  466. first = fmt.Sprintf("No=%d 列不符: %s", no, strings.Join(badCols, ","))
  467. }
  468. }
  469. return true
  470. })
  471. must(err)
  472. perf("校验", "全表扫描 "+label, scanned, time.Since(t))
  473. if first != "" {
  474. fmt.Printf(" 首个错行: %s\n", first)
  475. }
  476. if want >= 0 {
  477. chk("全表扫描 "+label+" 行数", scanned == want,
  478. strconv.FormatInt(want, 10), strconv.FormatInt(scanned, 10))
  479. }
  480. return scanned, bad
  481. }
  482. // verifyEmpInserted P2 专用: 只校验"刚插入时应有的原始值", 不含后续 P5 更新 / P6 删除的最终态。
  483. // (verifyEmpORM 的公式里带有更新覆盖与删除区间, 在 P2 阶段尚不成立)
  484. func verifyEmpInserted(empT *DB.Table[Emp], label string, want int64) (int64, int64) {
  485. t := time.Now()
  486. var scanned, bad int64
  487. var first string
  488. err := empT.Select(func(e *Emp) bool {
  489. scanned++
  490. no := e.No
  491. badCols := make([]string, 0, 4)
  492. if e.Name != empName(no) {
  493. badCols = append(badCols, "Name")
  494. }
  495. if e.Age != ageBase+no%ageMod {
  496. badCols = append(badCols, "Age")
  497. }
  498. if e.Score != no%scoreMod {
  499. badCols = append(badCols, "Score")
  500. }
  501. if e.Dept != fmt.Sprintf("Dept%03d", no%deptCount) {
  502. badCols = append(badCols, "Dept")
  503. }
  504. if e.Salary != float64(no%100000) {
  505. badCols = append(badCols, "Salary")
  506. }
  507. if e.Active != (no%2 == 0) {
  508. badCols = append(badCols, "Active")
  509. }
  510. if e.ID != uint64(no) {
  511. badCols = append(badCols, fmt.Sprintf("_id(%d!=%d)", e.ID, no))
  512. }
  513. if len(badCols) > 0 {
  514. bad++
  515. if first == "" {
  516. first = fmt.Sprintf("No=%d 列不符: %s", no, strings.Join(badCols, ","))
  517. }
  518. }
  519. return true
  520. })
  521. must(err)
  522. perf("校验", "全表扫描 "+label, scanned, time.Since(t))
  523. if first != "" {
  524. fmt.Printf(" 首个错行: %s\n", first)
  525. }
  526. if want >= 0 {
  527. chk("全表扫描 "+label+" 行数", scanned == want,
  528. strconv.FormatInt(want, 10), strconv.FormatInt(scanned, 10))
  529. }
  530. return scanned, bad
  531. }
  532. // ============================== P3 查询 ==============================
  533. func phase3Queries(empT *DB.Table[Emp], empCore *core.Table) {
  534. phase("P3 查询性能 (主键点查 / 二级索引等值 / 范围 / 分页 / 排序 / 投影 / 聚合)")
  535. const qn = 20000
  536. // 3.1 主键点查
  537. {
  538. ids := sampleIDs(qn, nEmp)
  539. bad := int64(0)
  540. t := time.Now()
  541. for _, id := range ids {
  542. e, err := empT.Find(id)
  543. if err != nil || e.ID != id || e.No != int64(id) || e.Name != empName(int64(id)) {
  544. bad++
  545. }
  546. }
  547. perf("P3 查询", fmt.Sprintf("主键点查 Find(pkID) × %d", qn), qn, time.Since(t))
  548. chk("3.1 主键点查结果全对", bad == 0, "0 错", fmt.Sprintf("%d 错", bad))
  549. }
  550. // 3.2 二级索引等值
  551. {
  552. ids := sampleIDs(qn, nEmp)
  553. bad := int64(0)
  554. t := time.Now()
  555. for _, id := range ids {
  556. no := int64(id)
  557. e, err := empT.Where("No", "=", no).First()
  558. if err != nil || e.No != no || e.ID != uint64(no) {
  559. bad++
  560. }
  561. }
  562. perf("P3 查询", fmt.Sprintf("二级索引等值 Where(No=..) × %d", qn), qn, time.Since(t))
  563. chk("3.2 索引等值结果全对", bad == 0, "0 错", fmt.Sprintf("%d 错", bad))
  564. }
  565. // 3.3 索引等值 Count (Age)
  566. {
  567. const times = 20
  568. want := nEmp / ageMod
  569. bad := int64(0)
  570. t := time.Now()
  571. for k := int64(0); k < times; k++ {
  572. c, err := empT.Where("Age", "=", ageBase+k).Count()
  573. if err != nil || c != want {
  574. bad++
  575. }
  576. }
  577. perf("P3 查询", fmt.Sprintf("索引等值 Count(Age=..) × %d (每次 %d 行)", times, want), want*times, time.Since(t))
  578. chk("3.3 索引等值计数正确", bad == 0, "0 错", fmt.Sprintf("%d 错", bad))
  579. }
  580. // 3.4 索引范围 Count
  581. {
  582. want := nEmp * 5 / ageMod
  583. t := time.Now()
  584. c, err := empT.Where("Age", ">=", 30).Where("Age", "<=", 34).Count()
  585. perf("P3 查询", fmt.Sprintf("索引范围 Count(Age 30..34, %d 行)", want), want, time.Since(t))
  586. chk("3.4 索引范围计数正确", err == nil && c == want,
  587. strconv.FormatInt(want, 10), fmt.Sprintf("%d err=%v", c, err))
  588. }
  589. // 3.5 字符串索引等值 Count
  590. {
  591. d := deptCode(7)
  592. want := nEmp / deptCount
  593. t := time.Now()
  594. c, err := empT.Where("Dept", "=", d).Count()
  595. perf("P3 查询", fmt.Sprintf("字符串索引 Count(Dept=%s, %d 行)", d, want), want, time.Since(t))
  596. chk("3.5 字符串索引计数正确", err == nil && c == want,
  597. strconv.FormatInt(want, 10), fmt.Sprintf("%d err=%v", c, err))
  598. }
  599. // 3.6 无索引列条件 → 全表扫描
  600. {
  601. want := nEmp / scoreMod
  602. t := time.Now()
  603. c, err := empT.Where("Score", "=", 123).Count()
  604. perf("P3 查询", fmt.Sprintf("无索引条件 Count(Score=123, %d 行) 全表扫描", want), nEmp, time.Since(t))
  605. chk("3.6 无索引条件计数正确", err == nil && c == want,
  606. strconv.FormatInt(want, 10), fmt.Sprintf("%d err=%v", c, err))
  607. }
  608. // 3.7 分页
  609. {
  610. d := deptCode(7)
  611. rows, err := empT.Where("Dept", "=", d).Limit(20).Find()
  612. bad := 0
  613. for _, e := range rows {
  614. if e.Dept != d {
  615. bad++
  616. }
  617. }
  618. chk("3.7 Limit 分页", err == nil && len(rows) == 20 && bad == 0,
  619. "20 行且全为 "+d, fmt.Sprintf("%d 行 %d 不符 err=%v", len(rows), bad, err))
  620. // Offset 分页: 偏移量随规模派生, 避免小规模下 dept 总行数 < 偏移量导致取空
  621. deptSize := nEmp / deptCount
  622. off := int64(500)
  623. if deptSize > 0 && off > deptSize/2 {
  624. off = deptSize / 4
  625. }
  626. if off < 0 {
  627. off = 0
  628. }
  629. wantFull := int(off) + 10
  630. t := time.Now()
  631. rows2, err2 := empT.Where("Dept", "=", d).Limit(10).Offset(int(off)).Find()
  632. perf("P3 查询", fmt.Sprintf("Limit(10).Offset(%d)", off), 10, time.Since(t))
  633. full, _ := empT.Where("Dept", "=", d).Limit(wantFull).Find()
  634. match := len(rows2) == 10 && len(full) == wantFull
  635. if match {
  636. for k := 0; k < 10; k++ {
  637. if full[int(off)+k].No != rows2[k].No {
  638. match = false
  639. break
  640. }
  641. }
  642. }
  643. chk("3.7 Offset 分页与全量前缀一致", match && err2 == nil, "一致",
  644. fmt.Sprintf("一致=%v off=%d full=%d rows2=%d err=%v", match, off, len(full), len(rows2), err2))
  645. }
  646. // 3.8 排序
  647. {
  648. t := time.Now()
  649. rows, err := empT.Where("Age", ">=", 30).Where("Age", "<=", 34).OrderBy("Age").Limit(10).Find()
  650. perf("P3 查询", "OrderBy(索引列 Age) + Limit(10) 流式取前 N", 10, time.Since(t))
  651. ordered := true
  652. for i := 1; i < len(rows); i++ {
  653. if rows[i].Age < rows[i-1].Age {
  654. ordered = false
  655. }
  656. }
  657. chk("3.8 索引序排序正确", err == nil && len(rows) == 10 && ordered,
  658. "10 行且 Age 非降", fmt.Sprintf("%d 行 ordered=%v err=%v", len(rows), ordered, err))
  659. part := nEmp / deptCount
  660. t = time.Now()
  661. rows2, err2 := empT.Where("Dept", "=", deptCode(7)).OrderBy("Score").Limit(10).Find()
  662. perf("P3 查询", fmt.Sprintf("OrderBy(非索引列 Score) %d 行参与内存排序", part), part, time.Since(t))
  663. ordered2 := true
  664. for i := 1; i < len(rows2); i++ {
  665. if rows2[i].Score < rows2[i-1].Score {
  666. ordered2 = false
  667. }
  668. }
  669. chk("3.8 非索引列排序正确", err2 == nil && len(rows2) == 10 && ordered2,
  670. "10 行且 Score 非降", fmt.Sprintf("%d 行 ordered=%v err=%v", len(rows2), ordered2, err2))
  671. }
  672. // 3.9 投影与 FindOne
  673. {
  674. rows, err := empCore.Query().Select("No", "Age").Limit(5).Find()
  675. ok := err == nil && len(rows) == 5
  676. for _, r := range rows {
  677. if len(r) < 2 {
  678. ok = false
  679. }
  680. }
  681. if len(rows) > 0 {
  682. fmt.Printf(" Select(No,Age) 首行返回: %s\n", dumpRow(rows[0]))
  683. }
  684. chk("3.9 Select 投影返回指定列", ok, "5 行且含 No/Age",
  685. fmt.Sprintf("%d 行 err=%v", len(rows), err))
  686. one, err2 := empCore.Query().Where(core.Eq("No", int64(1), types.FieldType(0))).FindOne()
  687. chk("3.9 FindOne 命中", err2 == nil && len(one) > 0 && dumpVal(rowMap(one)["No"]) == "1",
  688. "No=1", fmt.Sprintf("%v err=%v", rowMap(one)["No"], err2))
  689. }
  690. // 3.10 聚合
  691. {
  692. t := time.Now()
  693. sum, err := empCore.Query().Sum("Salary")
  694. perf("P3 聚合", "Sum(Salary) 全表", nEmp, time.Since(t))
  695. want := expectSalarySum(nEmp)
  696. chk("3.10 Sum 正确", err == nil && sum == want,
  697. fmt.Sprintf("%.0f", want), fmt.Sprintf("%.0f err=%v", sum, err))
  698. t = time.Now()
  699. mx, err2 := empCore.Query().Max("No")
  700. perf("P3 聚合", "Max(No)", nEmp, time.Since(t))
  701. chk("3.10 Max 正确", err2 == nil && dumpVal(mx) == strconv.FormatInt(nEmp, 10),
  702. strconv.FormatInt(nEmp, 10), fmt.Sprintf("%v err=%v", mx, err2))
  703. mn, err3 := empCore.Query().Min("No")
  704. chk("3.10 Min 正确", err3 == nil && dumpVal(mn) == "1", "1", fmt.Sprintf("%v err=%v", mn, err3))
  705. }
  706. _ = coreRow
  707. }
  708. // ============================== P4 多表 ==============================
  709. func phase4Multi(db *DB.DB, empT *DB.Table[Emp], deptT *DB.Table[DeptDim], orderT *DB.Table[Order],
  710. empCore *core.Table) (map[int64]int64, map[int64]int64) {
  711. phase("P4 多表: 部门维度表 + 订单表 + 索引关联计数校验")
  712. // 4.1 维度表
  713. t := time.Now()
  714. for i := int64(0); i < deptCount; i++ {
  715. d := &DeptDim{Code: deptCode(i), Title: "部门-" + deptCode(i), Level: i % 5}
  716. must(deptT.Insert(d))
  717. }
  718. must(db.Commit())
  719. perf("P4 多表", fmt.Sprintf("维度表 dept_dim 写入 %d 行", deptCount), deptCount, time.Since(t))
  720. c, err := deptT.Count()
  721. chk("4.1 维度表行数", err == nil && c == deptCount, strconv.Itoa(deptCount), fmt.Sprintf("%d err=%v", c, err))
  722. d7, err := deptT.Where("Code", "=", deptCode(7)).First()
  723. chk("4.1 维度表按 Code 索引查询", err == nil && d7 != nil && d7.Title == "部门-"+deptCode(7),
  724. "部门-"+deptCode(7), fmt.Sprintf("%v err=%v", d7, err))
  725. // 4.2 订单表
  726. orderCore, _ := db.CoreDB().FindTable("orders")
  727. t0 := time.Now()
  728. o := &Order{}
  729. for i := int64(1); i <= nOrder; i++ {
  730. *o = Order{OrderNo: i, EmpNo: 1 + i%nJoinMod, Amount: 100 + i%9900, Status: i % 5}
  731. if err := orderT.Insert(o); err != nil {
  732. must(fmt.Errorf("插入第 %d 条订单失败: %w", i, err))
  733. }
  734. if i%(nOrder/5) == 0 {
  735. rate := float64(i) / time.Since(t0).Seconds()
  736. fmt.Printf(" [订单 %8d 行] 累计%6.1fs | %8s/s | 磁盘 %.1fMB\n",
  737. i, time.Since(t0).Seconds(), human(rate), float64(dbSize(dbPath))/(1<<20))
  738. }
  739. }
  740. must(db.Commit())
  741. perf("P4 多表", fmt.Sprintf("订单表 orders 写入 %d 行", nOrder), nOrder, time.Since(t0))
  742. chk("4.2 订单表行数", orderCore.TotalRows() == uint64(nOrder),
  743. strconv.FormatInt(nOrder, 10), strconv.FormatUint(orderCore.TotalRows(), 10))
  744. // 4.3 关联期望 (公式推导: EmpNo = 1 + OrderNo%nJoinMod)
  745. orderExpect := map[int64]int64{}
  746. orderAfterDel := map[int64]int64{}
  747. for i := int64(1); i <= nOrder; i++ {
  748. e := 1 + i%nJoinMod
  749. orderExpect[e]++
  750. if i%5 != 0 { // P6 会删掉 Status==0 (i%5==0) 的订单
  751. orderAfterDel[e]++
  752. }
  753. }
  754. // 4.4 员工 → 订单 (走 idx_order_empno)
  755. sample := make([]int64, 0, 20)
  756. for k := int64(0); k < 20; k++ {
  757. if k%2 == 0 {
  758. sample = append(sample, 1+k*nJoinMod/40) // 落在 [1, nJoinMod] 内
  759. } else {
  760. sample = append(sample, nJoinMod+1+k) // 落在 (nJoinMod, nEmp] 内
  761. }
  762. }
  763. bad := int64(0)
  764. t = time.Now()
  765. for _, e := range sample {
  766. c, err := orderT.Where("EmpNo", "=", e).Count()
  767. if err != nil || c != orderExpect[e] {
  768. bad++
  769. }
  770. }
  771. perf("P4 多表", "关联计数: 订单表按 EmpNo 索引 × 20", 20, time.Since(t))
  772. chk("4.4 员工→订单 关联计数正确", bad == 0, "0 错", fmt.Sprintf("%d 错", bad))
  773. // 4.5 部门 → 员工 (走 idx_emp_dept, 100 个部门全查)
  774. want := nEmp / deptCount
  775. bad = 0
  776. t = time.Now()
  777. for i := int64(0); i < deptCount; i++ {
  778. c, err := empT.Where("Dept", "=", deptCode(i)).Count()
  779. if err != nil || c != want {
  780. bad++
  781. fmt.Printf(" [部门计数异常] %s 期望=%d 实际=%d err=%v\n", deptCode(i), want, c, err)
  782. }
  783. }
  784. perf("P4 多表", fmt.Sprintf("关联计数: 员工表按 Dept 索引 × %d (每次 %d 行)", deptCount, want),
  785. want*deptCount, time.Since(t))
  786. chk("4.5 部门→员工 关联计数全对", bad == 0, "0 错", fmt.Sprintf("%d 错", bad))
  787. chk("4.5 部门计数之和 == 全表行数", want*deptCount == nEmp,
  788. strconv.FormatInt(nEmp, 10), strconv.FormatInt(want*deptCount, 10))
  789. return orderExpect, orderAfterDel
  790. }
  791. // empSample 员工抽样: 一半落在 [1, nJoinMod] (有订单), 一半落在其后 (无订单)
  792. func empSample() []int64 {
  793. s := make([]int64, 0, 20)
  794. for k := int64(0); k < 20; k++ {
  795. if k%2 == 0 {
  796. s = append(s, 1+k*nJoinMod/40)
  797. } else {
  798. s = append(s, nJoinMod+1+k)
  799. }
  800. }
  801. return s
  802. }
  803. // ============================== P5 更新 ==============================
  804. func phase5Update(db *DB.DB, empT *DB.Table[Emp], empCore *core.Table) {
  805. phase("P5 更新 (单行整行重写 / core 批量更新 / 索引列更新后索引是否同步)")
  806. // 5.1 单行更新
  807. {
  808. bad := int64(0)
  809. t := time.Now()
  810. for k := int64(0); k < rowUpdCnt; k++ {
  811. no := rowUpdAt + k
  812. e, err := empT.Find(uint64(no))
  813. if err != nil {
  814. bad++
  815. continue
  816. }
  817. e.Name = "UP" + empName(no)
  818. if err := empT.Update(e); err != nil {
  819. bad++
  820. }
  821. }
  822. must(db.Commit())
  823. perf("P5 更新", fmt.Sprintf("单行更新 Update(整行重写) × %d", rowUpdCnt), rowUpdCnt, time.Since(t))
  824. chk("5.1 单行更新全部成功", bad == 0, "0 错", fmt.Sprintf("%d 错", bad))
  825. e, err := empT.Find(uint64(rowUpdAt))
  826. chk("5.1 更新后读回正确", err == nil && e.Name == "UP"+empName(rowUpdAt) && e.No == rowUpdAt &&
  827. e.Age == expAge(rowUpdAt) && e.Dept == expDept(rowUpdAt) && e.Score == expScore(rowUpdAt),
  828. "UP"+empName(rowUpdAt),
  829. fmt.Sprintf("Name=%s No=%d Age=%d Dept=%s Score=%d err=%v", e.Name, e.No, e.Age, e.Dept, e.Score, err))
  830. }
  831. // 5.2 core 批量更新 (整数索引列 Age)
  832. {
  833. want := nEmp / scoreMod
  834. t := time.Now()
  835. cnt, err := empCore.Update().Set("Age", int64(bulkAgeTo)).
  836. Where(core.Eq("Score", int64(bulkAgeAt), types.FieldType(0))).Exec()
  837. perf("P5 更新", fmt.Sprintf("core 批量更新 Updater.Exec (Score=%d → Age=%d)", bulkAgeAt, bulkAgeTo),
  838. cnt, time.Since(t))
  839. chk("5.2 批量更新影响行数", err == nil && cnt == want,
  840. strconv.FormatInt(want, 10), fmt.Sprintf("%d err=%v", cnt, err))
  841. t = time.Now()
  842. c99, err2 := empT.Where("Age", "=", int64(bulkAgeTo)).Count()
  843. perf("P5 校验", "索引列更新后 Count(Age=99)", want, time.Since(t))
  844. chk("5.2 索引列更新后索引已同步", err2 == nil && c99 == want,
  845. strconv.FormatInt(want, 10), fmt.Sprintf("%d err=%v", c99, err2))
  846. want20 := nEmp/ageMod - nEmp/scoreMod
  847. c20, err3 := empT.Where("Age", "=", int64(ageBase)).Count()
  848. chk("5.2 原 Age 桶计数同步减少", err3 == nil && c20 == want20,
  849. strconv.FormatInt(want20, 10), fmt.Sprintf("%d err=%v", c20, err3))
  850. }
  851. // 5.3 core 批量更新 (字符串索引列 Dept)
  852. {
  853. want := nEmp / scoreMod
  854. t := time.Now()
  855. cnt, err := empCore.Update().Set("Dept", fixedBytes(empCore, "Dept", bulkDeptTo)).
  856. Where(core.Eq("Score", int64(bulkDeptAt), types.FieldType(0))).Exec()
  857. perf("P5 更新", fmt.Sprintf("core 批量更新字符串索引列 (Score=%d → Dept=%s)", bulkDeptAt, bulkDeptTo),
  858. cnt, time.Since(t))
  859. chk("5.3 字符串列批量更新影响行数", err == nil && cnt == want,
  860. strconv.FormatInt(want, 10), fmt.Sprintf("%d err=%v", cnt, err))
  861. want99 := nEmp/deptCount + nEmp/scoreMod
  862. c99, err2 := empT.Where("Dept", "=", bulkDeptTo).Count()
  863. chk("5.3 字符串索引同步 (并入 "+bulkDeptTo+")", err2 == nil && c99 == want99,
  864. strconv.FormatInt(want99, 10), fmt.Sprintf("%d err=%v", c99, err2))
  865. want21 := nEmp/deptCount - nEmp/scoreMod
  866. c21, err3 := empT.Where("Dept", "=", deptCode(21)).Count()
  867. chk("5.3 原字符串桶计数同步减少", err3 == nil && c21 == want21,
  868. strconv.FormatInt(want21, 10), fmt.Sprintf("%d err=%v", c21, err3))
  869. }
  870. // 5.4 负数测试: 更新不存在的主键
  871. {
  872. ghost := &Emp{BaseModel: DB.BaseModel{ID: 1 << 60}, No: 1, Name: "ghost"}
  873. err := empT.Update(ghost)
  874. chk("5.4 更新不存在的主键应报错", err != nil, "error", fmt.Sprintf("%v", err))
  875. }
  876. }
  877. // ============================== P6 删除 + 一致性校验 ==============================
  878. func phase6Delete(db *DB.DB, empT *DB.Table[Emp], orderT *DB.Table[Order], empCore *core.Table,
  879. orderAfterDel map[int64]int64) {
  880. phase(fmt.Sprintf("P6 删除 (逐行删 %d 行 / 批量删除全表重建) + 全量一致性校验", delCount))
  881. // 6.1 逐行删除
  882. {
  883. t := time.Now()
  884. ok := int64(0)
  885. for id := uint64(delFrom); id <= uint64(delTo); id++ {
  886. if err := empT.Delete(id); err != nil {
  887. fmt.Printf(" [删除失败] pkID=%d err=%v\n", id, err)
  888. } else {
  889. ok++
  890. }
  891. }
  892. perf("P6 删除", fmt.Sprintf("逐行 Delete(pkID) × %d", delCount), ok, time.Since(t))
  893. chk("6.1 逐行删除全部成功", ok == delCount,
  894. strconv.FormatInt(delCount, 10), strconv.FormatInt(ok, 10))
  895. chk("6.1 TotalRows 已减少", empCore.TotalRows() == uint64(nEmp-delCount),
  896. strconv.FormatInt(nEmp-delCount, 10), strconv.FormatUint(empCore.TotalRows(), 10))
  897. _, err := empT.Find(uint64(delFrom))
  898. chk("6.1 已删除行 Find 应 not found", err != nil, "error", fmt.Sprintf("%v", err))
  899. alive := delFrom - 1
  900. e, err2 := empT.Find(uint64(alive))
  901. chk("6.1 边界外一行仍存活", err2 == nil && e.No == alive,
  902. fmt.Sprintf("No=%d", alive), fmt.Sprintf("%v err=%v", e, err2))
  903. }
  904. // 6.2 批量删除 orders (Status==0 无索引列 → 收集后全表重建)
  905. {
  906. orderCore, _ := db.CoreDB().FindTable("orders")
  907. want := nOrder / 5
  908. t := time.Now()
  909. cnt, err := orderCore.Delete().Where(core.Eq("Status", int64(0), types.FieldType(0))).Exec()
  910. perf("P6 删除", fmt.Sprintf("批量删除 orders (Status=0, %d 行) 触发全表重建", want), want, time.Since(t))
  911. chk("6.2 批量删除影响行数", err == nil && cnt == want,
  912. strconv.FormatInt(want, 10), fmt.Sprintf("%d err=%v", cnt, err))
  913. chk("6.2 订单表行数已减少", orderCore.TotalRows() == uint64(nOrder-want),
  914. strconv.FormatInt(nOrder-want, 10), strconv.FormatUint(orderCore.TotalRows(), 10))
  915. bad := 0
  916. t2 := time.Now()
  917. for _, e := range empSample() {
  918. c, err := orderT.Where("EmpNo", "=", e).Count()
  919. if err != nil || c != orderAfterDel[e] {
  920. bad++
  921. }
  922. }
  923. perf("P6 校验", "批量删除后 员工→订单 关联计数 × 20", 20, time.Since(t2))
  924. chk("6.2 批量删除后关联计数正确", bad == 0, "0 错", fmt.Sprintf("%d 错", bad))
  925. }
  926. // 6.3 全量一致性校验 (期望值全部由公式推导)
  927. {
  928. sub("6.3 一致性校验 (更新 + 删除之后, 期望值由公式推导)")
  929. scanned, bad := verifyEmpORM(empT, "删除后全量校验(公式)", nEmp-delCount)
  930. chk("6.3 全表逐行公式校验 0 错", bad == 0, "0", strconv.FormatInt(bad, 10))
  931. deptWant := map[string]int64{}
  932. ageWant := map[int64]int64{}
  933. for i := int64(1); i <= nEmp; i++ {
  934. if isDeletedNo(i) {
  935. continue
  936. }
  937. deptWant[expDept(i)]++
  938. ageWant[expAge(i)]++
  939. }
  940. badD := 0
  941. t := time.Now()
  942. for i := int64(0); i < deptCount; i++ {
  943. c, err := empT.Where("Dept", "=", deptCode(i)).Count()
  944. if err != nil || c != deptWant[deptCode(i)] {
  945. badD++
  946. fmt.Printf(" [部门计数异常] %s 期望=%d 实际=%d err=%v\n", deptCode(i), deptWant[deptCode(i)], c, err)
  947. }
  948. }
  949. perf("P6 校验", fmt.Sprintf("索引计数复核 Dept × %d", deptCount), nEmp-delCount, time.Since(t))
  950. chk("6.3 全部部门索引计数正确", badD == 0, "0 错", fmt.Sprintf("%d 错", badD))
  951. badA := 0
  952. for _, a := range []int64{int64(bulkAgeTo), int64(ageBase), int64(ageBase) + 1, 40, 55} {
  953. c, err := empT.Where("Age", "=", a).Count()
  954. if err != nil || c != ageWant[a] {
  955. badA++
  956. fmt.Printf(" [Age 计数异常] %d 期望=%d 实际=%d err=%v\n", a, ageWant[a], c, err)
  957. }
  958. }
  959. chk("6.3 抽样 Age 索引计数正确", badA == 0, "0 错", fmt.Sprintf("%d 错", badA))
  960. sum := int64(0)
  961. for _, v := range deptWant {
  962. sum += v
  963. }
  964. chk("6.3 部门计数之和 == 存活行数", sum == nEmp-delCount,
  965. strconv.FormatInt(nEmp-delCount, 10), strconv.FormatInt(sum, 10))
  966. _ = scanned
  967. }
  968. }
  969. // ============================== P7 Schema 变更 ==============================
  970. func expectEmpRaw(no int64) map[string]interface{} {
  971. return map[string]interface{}{
  972. "_id": pkIDOf(no),
  973. "No": uint64(no),
  974. "Name": []byte(expName(no)),
  975. "Age": uint64(expAge(no)),
  976. "Score": uint64(expScore(no)),
  977. "Dept": []byte(expDept(no)),
  978. "Salary": uint64(expSalary(no)),
  979. "Active": expActive(no),
  980. }
  981. }
  982. func expectEmpVals(no int64, pk, level uint64) []types.FieldValue {
  983. return []types.FieldValue{
  984. {FieldName: []byte("_id"), Value: pk},
  985. {FieldName: []byte("No"), Value: uint64(no)},
  986. {FieldName: []byte("Name"), Value: []byte(expName(no))},
  987. {FieldName: []byte("Age"), Value: uint64(expAge(no))},
  988. {FieldName: []byte("Score"), Value: uint64(expScore(no))},
  989. {FieldName: []byte("Dept"), Value: []byte(expDept(no))},
  990. {FieldName: []byte("Salary"), Value: uint64(expSalary(no))},
  991. {FieldName: []byte("Active"), Value: expActiveByte(no)},
  992. {FieldName: []byte("Level"), Value: level},
  993. }
  994. }
  995. func calcDeptWant(nExtra int64) map[string]int64 {
  996. m := map[string]int64{}
  997. for i := int64(1); i <= nEmp; i++ {
  998. if isDeletedNo(i) {
  999. continue
  1000. }
  1001. m[expDept(i)]++
  1002. }
  1003. for k := int64(1); k <= nExtra; k++ {
  1004. m[expDept(nEmp+k)]++
  1005. }
  1006. return m
  1007. }
  1008. // verifyEmpCoreScan 纯 core 视角全表扫描校验 (不经过 ORM 结构体映射)
  1009. func verifyEmpCoreScan(empCore *core.Table, label string, wantRows int64, wantDept bool) (int64, int64) {
  1010. t := time.Now()
  1011. var scanned, bad int64
  1012. var first string
  1013. must(empCore.Query().Scan(func(r []types.FieldValue) bool {
  1014. scanned++
  1015. m := rowMap(r)
  1016. // No 列现在是 int64 (保序 Int64 编码), 旧实现按 uint64 断言会拿到零值,
  1017. // 导致整表每行都被判错。兼容两种整数形态。
  1018. var ino int64
  1019. switch v := m["No"].(type) {
  1020. case int64:
  1021. ino = v
  1022. case uint64:
  1023. ino = int64(v)
  1024. }
  1025. if isDeletedNo(ino) {
  1026. bad++
  1027. if first == "" {
  1028. first = fmt.Sprintf("已删除行仍存活 No=%d", ino)
  1029. }
  1030. return true
  1031. }
  1032. cols := make([]string, 0, 7)
  1033. if dumpVal(m["Name"]) != expName(ino) {
  1034. cols = append(cols, "Name")
  1035. }
  1036. if dumpVal(m["Age"]) != strconv.FormatInt(expAge(ino), 10) {
  1037. cols = append(cols, "Age")
  1038. }
  1039. if dumpVal(m["Score"]) != strconv.FormatInt(expScore(ino), 10) {
  1040. cols = append(cols, "Score")
  1041. }
  1042. if wantDept && dumpVal(m["Dept"]) != expDept(ino) {
  1043. cols = append(cols, "Dept")
  1044. }
  1045. if dumpVal(m["Salary"]) != strconv.FormatUint(uint64(expSalary(ino)), 10) {
  1046. cols = append(cols, "Salary")
  1047. }
  1048. if dumpVal(m["Active"]) != dumpVal(expActive(ino)) {
  1049. cols = append(cols, "Active")
  1050. }
  1051. if dumpVal(m["_id"]) != strconv.FormatUint(pkIDOf(ino), 10) {
  1052. cols = append(cols, "_id")
  1053. }
  1054. if len(cols) > 0 {
  1055. bad++
  1056. if first == "" {
  1057. first = fmt.Sprintf("No=%d 列不符: %s", ino, strings.Join(cols, ","))
  1058. }
  1059. }
  1060. return true
  1061. }))
  1062. perf("校验", "core 全表扫描 "+label, scanned, time.Since(t))
  1063. if first != "" {
  1064. fmt.Printf(" 首个错行: %s\n", first)
  1065. }
  1066. chk("core 全表扫描 "+label+" 行数", scanned == wantRows,
  1067. strconv.FormatInt(wantRows, 10), strconv.FormatInt(scanned, 10))
  1068. return scanned, bad
  1069. }
  1070. func phase7Schema(db *DB.DB, empT *DB.Table[Emp], empCore *core.Table) {
  1071. phase("P7 Schema 变更: 增列 / 删列 / 改列名 (含数据正确性校验)")
  1072. rowsBefore := empCore.TotalRows()
  1073. sub("7.0 变更前基线")
  1074. fmt.Printf(" 活跃列: %v\n", empCore.GetActiveFieldNames())
  1075. fmt.Printf(" TotalRows=%d | 字段数 总%d/活跃%d\n",
  1076. rowsBefore, len(empCore.GetFields()), len(empCore.GetActiveFieldNames()))
  1077. row0 := coreRow(empCore, "pk_emp", uint64(sampleNo))
  1078. fmt.Printf(" 基线行 No=%d → %s\n", sampleNo, dumpRow2(row0))
  1079. // ---- 7.1 ADD COLUMN ----
  1080. sub("7.1 ADD COLUMN Level (8 字节新列)")
  1081. must(empCore.AddField(types.FieldInfo{FieldType: types.FieldType_8byte, FieldName: []byte("Level")}))
  1082. cols := empCore.GetActiveFieldNames()
  1083. chk("7.1 活跃列含 Level", contains(cols, "Level"), "含 Level", fmt.Sprintf("%v", cols))
  1084. row := coreRow(empCore, "pk_emp", uint64(sampleNo))
  1085. _, hasLevel := row["Level"]
  1086. chk("7.1 老行不返回新增列(缺失而非零值)", !hasLevel, "无 Level", fmt.Sprintf("有=%v", hasLevel))
  1087. eq, diff := rawEqual(row, expectEmpRaw(sampleNo))
  1088. chk("7.1 老行其它列不受加列影响", eq, "全等", diff)
  1089. sampled, missLevel := int64(0), int64(0)
  1090. must(empCore.Query().Limit(2000).Scan(func(r []types.FieldValue) bool {
  1091. sampled++
  1092. if _, ok := rowMap(r)["Level"]; !ok {
  1093. missLevel++
  1094. }
  1095. return true
  1096. }))
  1097. chk("7.1 抽样老行均不含 Level", missLevel == sampled,
  1098. fmt.Sprintf("%d/%d", sampled, sampled), fmt.Sprintf("%d/%d", missLevel, sampled))
  1099. bad := int64(0)
  1100. newPKs := make([]uint64, 0, extraRows)
  1101. for k := int64(1); k <= extraRows; k++ {
  1102. no := nEmp + k
  1103. // 这里要的是"插入一条指定 pkID 的新行", 用显式主键 API。
  1104. // 不能再用 TotalRows()+1 去凑: 自增编号只增不减, 删除后不会回退 (见 T4 探针),
  1105. // 靠行数现算出来的号会与它期望的 pkID 对不上。
  1106. pk := pkIDOf(no)
  1107. got, err := empCore.InsertRowWithID(pk, expectEmpVals(no, pk, 777))
  1108. if err != nil || got != pk {
  1109. bad++
  1110. fmt.Printf(" [新行插入异常] No=%d pk=%d err=%v\n", no, pk, err)
  1111. continue
  1112. }
  1113. newPKs = append(newPKs, pk)
  1114. }
  1115. chk("7.1 加列后新行插入成功", bad == 0, "0 错", fmt.Sprintf("%d 错", bad))
  1116. r1 := coreRow(empCore, "pk_emp", newPKs[0])
  1117. lv, okLv := r1["Level"]
  1118. chk("7.1 新行返回 Level=777", okLv && dumpVal(lv) == "777", "777", dumpVal(lv))
  1119. eq1, diff1 := rawEqual(r1, expectEmpRaw(nEmp+1))
  1120. chk("7.1 新行其它列值正确", eq1, "全等", diff1)
  1121. // ---- 7.2 在新增列上建索引 ----
  1122. sub("7.2 在新增列上建索引")
  1123. lvIdx := fieldIndexByName(empCore, "Level")
  1124. if lvIdx < 0 {
  1125. must(fmt.Errorf("找不到 Level 字段的 master 索引"))
  1126. }
  1127. _, err := empCore.CreateIndex("idx_emp_level", page.IndexTypeBTree, []byte{byte(lvIdx)})
  1128. must(err)
  1129. c7, err7 := empCore.Query().Where(core.Eq("Level", uint64(777), types.FieldType(0))).Count()
  1130. perf("P7 校验", "新增列索引 Count(Level=777)", extraRows, time.Millisecond)
  1131. chk("7.2 新增列索引能查到新行", err7 == nil && c7 == extraRows,
  1132. strconv.FormatInt(extraRows, 10), fmt.Sprintf("%d err=%v", c7, err7))
  1133. if err7 == nil && c7 == 0 {
  1134. problem("重要限制", "CreateIndex 不回填存量行",
  1135. "在已有数据的表上新建二级索引, 索引中不含任何存量行: 本例新行(3 行 Level=777)在 CreateIndex 之前已插入, 建好后 Count(Level=777) 仍为 0。只有建索引之后新插入的行才进索引; 引擎未提供公开的『重建全部二级索引』接口。")
  1136. }
  1137. c0, err0 := empCore.Query().Where(core.Eq("Level", uint64(0), types.FieldType(0))).Count()
  1138. chk("7.2 老行在新增列上视为无值(不落索引)", err0 == nil && c0 == 0,
  1139. "0", fmt.Sprintf("%d err=%v", c0, err0))
  1140. // ---- 7.3 DROP COLUMN ----
  1141. sub("7.3 DROP COLUMN Level (先 DropIndex 再 DropField)")
  1142. must(empCore.DropIndex("idx_emp_level"))
  1143. must(empCore.DropField("Level"))
  1144. cols = empCore.GetActiveFieldNames()
  1145. chk("7.3 活跃列不含 Level", !contains(cols, "Level"), "不含 Level", fmt.Sprintf("%v", cols))
  1146. _, stillThere := empCore.IndexMeta().FindIndex("idx_emp_level")
  1147. chk("7.3 索引已删除", !stillThere, "不存在", fmt.Sprintf("仍存在=%v", stillThere))
  1148. row = coreRow(empCore, "pk_emp", uint64(sampleNo))
  1149. eq2, diff2 := rawEqual(row, expectEmpRaw(sampleNo))
  1150. chk("7.3 删列后老行其它列完好", eq2, "全等", diff2)
  1151. _, hasAfterDrop := row["Level"]
  1152. chk("7.3 删列后老行不再有 Level", !hasAfterDrop, "无 Level", fmt.Sprintf("有=%v", hasAfterDrop))
  1153. r1 = coreRow(empCore, "pk_emp", newPKs[0])
  1154. _, has := r1["Level"]
  1155. eq3, diff3 := rawEqual(r1, expectEmpRaw(nEmp+1))
  1156. chk("7.3 新行的 Level 列已不可见", !has, "无 Level", fmt.Sprintf("有=%v", has))
  1157. chk("7.3 新行其它列完好", eq3, "全等", diff3)
  1158. chk("7.3 删列不影响行数", empCore.TotalRows() == uint64(int64(rowsBefore)+extraRows),
  1159. strconv.FormatInt(int64(rowsBefore)+extraRows, 10), strconv.FormatUint(empCore.TotalRows(), 10))
  1160. // ---- 7.4 改列名 ----
  1161. sub("7.4 改列名 (引擎无 RENAME API, 只能 DROP 旧列 + ADD 新列)")
  1162. oldDept := dumpVal(row0["Dept"])
  1163. // 复活软删除的列必须沿用【原类型 + 原宽度码】: master registry 的类型/参数是所有历史
  1164. // 版本共用的行偏移依据 (见 getVersionedOffset / rowDataSizeForVersion), 换成别的类型/
  1165. // 宽度会让已落盘的旧行整段错位, 引擎会直接拒。所以先把原定义抄下来, 7.5 用它复活。
  1166. deptDef := empCore.Field[fieldIndexByName(empCore, "Dept")]
  1167. must(empCore.DropIndex("idx_emp_dept"))
  1168. must(empCore.DropField("Dept"))
  1169. must(empCore.AddField(types.FieldInfo{FieldType: types.FieldType_32byte, FieldName: []byte("DeptName")}))
  1170. r := coreRow(empCore, "pk_emp", uint64(sampleNo))
  1171. _, hasOld := r["Dept"]
  1172. _, hasNew := r["DeptName"]
  1173. chk("7.4 改名后老列名不可读", !hasOld, "无 Dept", fmt.Sprintf("有=%v", hasOld))
  1174. chk("7.4 改名后新列名在老行上无值", !hasNew, "无 DeptName", fmt.Sprintf("有=%v", hasNew))
  1175. fmt.Printf(" 证据: 老行 Dept 原值=%q, 改名后既读不到 Dept 也读不到 DeptName\n", oldDept)
  1176. problem("设计约束", "引擎没有 RENAME COLUMN / RENAME TABLE API",
  1177. "改列名只能 DROP 旧列 + ADD 新列; 老行原值不会被迁移到新列名(数据仍在行内, 只是列名不可达), 需要业务侧自行回填")
  1178. // ---- 7.5 复活同名列 ----
  1179. // 用 7.4 抄下来的原定义 (类型 + 宽度码) 复活; 只有"同名同类型"才允许复活,
  1180. // 想换宽度必须改用新列名 (见 Table.AddField 的复活分支)。
  1181. sub("7.5 重新加回同名 Dept 列 (DropField 实为软删除)")
  1182. must(empCore.AddField(deptDef))
  1183. r = coreRow(empCore, "pk_emp", uint64(sampleNo))
  1184. got, okGot := r["Dept"]
  1185. chk("7.5 同名列复活后老行数据可见", okGot, "可见", fmt.Sprintf("可见=%v", okGot))
  1186. chk("7.5 复活值与删列前完全一致", dumpVal(got) == oldDept, oldDept, dumpVal(got))
  1187. eq4, diff4 := rawEqual(r, expectEmpRaw(sampleNo))
  1188. chk("7.5 复活后老行全列正确", eq4, "全等", diff4)
  1189. problem("重要语义", "DROP COLUMN 是软删除, 数据仍留在行内; 重新 ADD 同名列会\"复活\"老行数据",
  1190. "这解释了为什么改列名(删+加新名)后数据看似丢失, 但加回旧列名后又能读到 → 敏感数据无法靠删列来清除, 需要重建表/覆写")
  1191. // ---- 7.6 重建索引 + 全表校验 ----
  1192. sub("7.6 重建 Dept 索引 + core 层全表校验")
  1193. dIdx := fieldIndexByName(empCore, "Dept")
  1194. _, ciErr := empCore.CreateIndex("idx_emp_dept", page.IndexTypeBTree, []byte{byte(dIdx)})
  1195. must(ciErr)
  1196. want := calcDeptWant(extraRows)
  1197. badD := 0
  1198. t := time.Now()
  1199. for i := int64(0); i < deptCount; i++ {
  1200. c, err := empT.Where("Dept", "=", deptCode(i)).Count()
  1201. if err != nil || c != want[deptCode(i)] {
  1202. badD++
  1203. fmt.Printf(" [部门计数异常] %s 期望=%d 实际=%d err=%v\n", deptCode(i), want[deptCode(i)], c, err)
  1204. }
  1205. }
  1206. perf("P7 校验", fmt.Sprintf("重建索引后 索引计数复核 Dept × %d", deptCount), expectRows(), time.Since(t))
  1207. chk("7.6 重建索引后部门计数全对", badD == 0, "0 错", fmt.Sprintf("%d 错", badD))
  1208. if badD == int(deptCount) {
  1209. problem("重要限制", "DROP 后重建的二级索引同样不回填存量行",
  1210. "idx_emp_dept 删除后重新 CreateIndex, 对全部存量行按索引计数均为 0; 而紧随其后的 core 逐行全表扫描校验 0 错 → 数据本身完好, 只是索引不含存量行。")
  1211. }
  1212. _, badRows := verifyEmpCoreScan(empCore, "P7 变更后全量校验", expectRows(), true)
  1213. chk("7.6 P7 变更后逐行公式校验 0 错", badRows == 0, "0", strconv.FormatInt(badRows, 10))
  1214. _ = db
  1215. }
  1216. func dumpRow2(m map[string]interface{}) string {
  1217. keys := make([]string, 0, len(m))
  1218. for k := range m {
  1219. keys = append(keys, k)
  1220. }
  1221. // 固定顺序输出, 便于对比
  1222. order := []string{"_id", "No", "Name", "Age", "Score", "Dept", "Salary", "Active", "Level", "DeptName"}
  1223. parts := make([]string, 0, len(keys))
  1224. for _, k := range order {
  1225. if v, ok := m[k]; ok {
  1226. parts = append(parts, fmt.Sprintf("%s=%s", k, dumpVal(v)))
  1227. }
  1228. }
  1229. return strings.Join(parts, " ")
  1230. }
  1231. // ============================== P8 引擎语义探针 ==============================
  1232. func coreRows(pdb *DB.DB, name string) uint64 {
  1233. if t, ok := pdb.CoreDB().FindTable(name); ok {
  1234. return t.TotalRows()
  1235. }
  1236. return 0
  1237. }
  1238. func phase8Probes() {
  1239. phase("P8 引擎语义探针 (独立库 " + filepath.Base(probePath) + ", 不影响主库)")
  1240. pdb, err := DB.Open(probePath)
  1241. must(err)
  1242. defer func() { must(pdb.Close()) }()
  1243. probeLayoutAssumption(pdb)
  1244. probeDanglingIndex(pdb)
  1245. probeExplicitPK(pdb)
  1246. probePKReuse(pdb)
  1247. probeFloatTrunc(pdb)
  1248. }
  1249. func probeLayoutAssumption(pdb *DB.DB) {
  1250. sub("T1 core 建表(无 _id 列) 时, Delete/Update 如何确定 pkID")
  1251. ct, err := pdb.CoreDB().CreateTable("probe_layout", []types.FieldInfo{
  1252. {FieldType: types.FieldType_8byte, FieldName: []byte("No")},
  1253. {FieldType: types.FieldType_32byte, FieldName: []byte("Label")},
  1254. })
  1255. if softMust("CreateTable(probe_layout)", err) {
  1256. return
  1257. }
  1258. _, err = ct.CreateIndex("pk_probe_layout", page.IndexTypePrimary, []byte{0})
  1259. if softMust("CreateIndex(pk_probe_layout)", err) {
  1260. return
  1261. }
  1262. for i, no := range []uint64{5, 5, 5, 6, 7} {
  1263. _, err := ct.InsertRow([]types.FieldValue{
  1264. {FieldName: []byte("No"), Value: no},
  1265. {FieldName: []byte("Label"), Value: []byte(fmt.Sprintf("L%d", i+1))},
  1266. })
  1267. if softMust(fmt.Sprintf("InsertRow(No=%d)", no), err) {
  1268. return
  1269. }
  1270. }
  1271. one, err := ct.SearchByIndex("pk_probe_layout", pkKey(1))
  1272. if softMust("SearchByIndex(pkID=1)", err) {
  1273. one = nil
  1274. }
  1275. fmt.Printf(" 表 = [No(8B), Label(32B)], 无 _id 列; 引擎反序列化出的行: %s\n", dumpRow(one))
  1276. fmt.Printf(" 数据: pkID 1→No5, 2→No5, 3→No5, 4→No6, 5→No7\n")
  1277. cnt, errD := ct.Delete().Where(core.Eq("No", uint64(5), types.FieldType(0))).Exec()
  1278. alive := map[uint64]uint64{}
  1279. for pk := uint64(1); pk <= 5; pk++ {
  1280. if r, err := ct.SearchByIndex("pk_probe_layout", pkKey(pk)); err == nil {
  1281. if v, ok := rowMap(r)["No"].(uint64); ok {
  1282. alive[pk] = v
  1283. }
  1284. }
  1285. }
  1286. fmt.Printf(" Delete(No==5) → cnt=%d err=%v | 剩余: ", cnt, errD)
  1287. for pk := uint64(1); pk <= 5; pk++ {
  1288. if v, ok := alive[pk]; ok {
  1289. fmt.Printf("pkID%d(No=%d) ", pk, v)
  1290. }
  1291. }
  1292. fmt.Printf("| TotalRows=%d\n", ct.TotalRows())
  1293. if len(alive) == 0 && ct.TotalRows() > 0 {
  1294. problem("严重", "core 主键索引建在非 _id 列上时, 行按 pkID 不可达",
  1295. "表 [No(8B), Label(32B)] 建 IndexTypePrimary(field 0=No) 并插入 5 行: TotalRows=5、Query().Count()=5 数据可见, 但 SearchByIndex(pk_..., pkKey(1..5)) 全部 key not found, 只有 1 个键能命中 → 主键 B+ 树未按 pkID 建立完整条目。纯 core 表应把 _id 作为首列主键。")
  1296. }
  1297. aliveNo5 := int64(0)
  1298. for _, v := range alive {
  1299. if v == 5 {
  1300. aliveNo5++
  1301. }
  1302. }
  1303. chk("T1 批量删除影响行数应等于匹配行数 3", cnt == 3, "3", strconv.FormatInt(cnt, 10))
  1304. chk("T1 被匹配的行(No=5)应全部删除", aliveNo5 == 0, "0 行", fmt.Sprintf("%d 行", aliveNo5))
  1305. _, keep5 := alive[5]
  1306. chk("T1 未被匹配的行(pkID=5,No=7)不应被删", keep5, "存活", fmt.Sprintf("存活=%v", keep5))
  1307. if cnt != 3 || aliveNo5 != 0 || !keep5 {
  1308. problem("严重", "core 层 Deleter.Exec 用\"行内第一个 uint64 列的值\"当 pkID",
  1309. fmt.Sprintf("条件 No==5 匹配 pkID 1/2/3 三行, 但 Deleter 把匹配行第一个 uint64 列的值 5 当成 pkID, 于是删掉了 pkID=5 的行(其 No=7, 未匹配), 真正匹配的 3 行一行未删 (cnt=%d)", cnt))
  1310. }
  1311. cnt2, errU := ct.Update().Set("Label", []byte("X")).
  1312. Where(core.Eq("No", uint64(6), types.FieldType(0))).Exec()
  1313. fmt.Printf(" Update(No==6 → Label=X) → cnt=%d err=%v\n", cnt2, errU)
  1314. if errU != nil {
  1315. problem("严重", "同一假设影响 core 层 Updater.Exec",
  1316. fmt.Sprintf("匹配行的第一个 uint64 值(6)被当成 pkID 去 UpdateRow → err=%v", errU))
  1317. } else if cnt2 != 1 {
  1318. problem("严重", "同一假设影响 core 层 Updater.Exec",
  1319. fmt.Sprintf("影响行数 %d != 匹配数 1", cnt2))
  1320. }
  1321. fmt.Printf(" 对照: ORM 表的行首列是 _id(=pkID), 因此 ORM 路径不受影响; 纯 core 表需自行保证首列语义\n")
  1322. }
  1323. func probeDanglingIndex(pdb *DB.DB) {
  1324. sub("T2 只删列不删索引: 残留索引的行为")
  1325. ct, err := pdb.CoreDB().CreateTable("probe_dangling", []types.FieldInfo{
  1326. {FieldType: types.FieldType_8byte, FieldName: []byte("No")},
  1327. {FieldType: types.FieldType_8byte, FieldName: []byte("Tag")},
  1328. })
  1329. if softMust("CreateTable(probe_dangling)", err) {
  1330. return
  1331. }
  1332. _, err = ct.CreateIndex("pk_probe_dangling", page.IndexTypePrimary, []byte{0})
  1333. if softMust("CreateIndex(pk_probe_dangling)", err) {
  1334. return
  1335. }
  1336. _, err = ct.CreateIndex("idx_probe_tag", page.IndexTypeBTree, []byte{1})
  1337. if softMust("CreateIndex(idx_probe_tag)", err) {
  1338. return
  1339. }
  1340. for i := uint64(1); i <= 5; i++ {
  1341. _, err := ct.InsertRow([]types.FieldValue{
  1342. {FieldName: []byte("No"), Value: i},
  1343. {FieldName: []byte("Tag"), Value: i * 10},
  1344. })
  1345. if softMust(fmt.Sprintf("InsertRow(No=%d)", i), err) {
  1346. return
  1347. }
  1348. }
  1349. c0, err0 := ct.Query().Where(core.Eq("Tag", uint64(20), types.FieldType(0))).Count()
  1350. fmt.Printf(" 删列前 Count(Tag==20)=%d err=%v\n", c0, err0)
  1351. if softMust("DropField(Tag)", ct.DropField("Tag")) {
  1352. return
  1353. }
  1354. fmt.Printf(" 删列后活跃列=%v\n", ct.GetActiveFieldNames())
  1355. _, dangling := ct.IndexMeta().FindIndex("idx_probe_tag")
  1356. fmt.Printf(" 残留索引 idx_probe_tag 仍存在=%v\n", dangling)
  1357. _, errQ := ct.Query().Where(core.Eq("Tag", uint64(20), types.FieldType(0))).Count()
  1358. fmt.Printf(" 删列后按残留索引查询 Tag==20 → err=%v\n", errQ)
  1359. _, errI := ct.InsertRow([]types.FieldValue{{FieldName: []byte("No"), Value: uint64(6)}})
  1360. fmt.Printf(" 删列后插入不含 Tag 的新行 → err=%v TotalRows=%d\n", errI, ct.TotalRows())
  1361. r6, err6 := ct.SearchByIndex("pk_probe_dangling", pkKey(6))
  1362. fmt.Printf(" 读回新行: %s err=%v\n", dumpRow(r6), err6)
  1363. chk("T2 残留索引不影响插入(缺失列应被跳过)", errI == nil, "nil", fmt.Sprintf("%v", errI))
  1364. if errQ != nil {
  1365. problem("使用约束", "DropField 不会自动删除引用该列的索引",
  1366. fmt.Sprintf("残留索引会导致按该列查询直接报错(err=%v); 正确顺序是先 DropIndex 再 DropField", errQ))
  1367. }
  1368. }
  1369. func probeExplicitPK(pdb *DB.DB) {
  1370. sub("T3 ORM 显式指定 _id ≠ 引擎 pkID")
  1371. pt, err := DB.UseTable[Emp](pdb, "probe_emp")
  1372. if softMust("UseTable(probe_emp)", err) {
  1373. return
  1374. }
  1375. e := &Emp{BaseModel: DB.BaseModel{ID: 999999}, No: 1, Name: "P1", Age: 30, Score: 1, Dept: "Dept000", Salary: 1, Active: true}
  1376. errI := pt.Insert(e)
  1377. fmt.Printf(" Insert(ID=999999) → err=%v, 对象 ID=%d\n", errI, e.ID)
  1378. _, errF := pt.Find(999999)
  1379. fmt.Printf(" Find(999999) → err=%v\n", errF)
  1380. real, errR := pt.Find(1)
  1381. if errR == nil {
  1382. fmt.Printf(" Find(1) → ID=%d No=%d (说明引擎分配的 pkID 是 1, 与 _id 列 999999 不一致)\n", real.ID, real.No)
  1383. } else {
  1384. fmt.Printf(" Find(1) → err=%v\n", errR)
  1385. }
  1386. chk("T3 按指定的 _id 应能查到该行", errF == nil, "查得到", fmt.Sprintf("err=%v", errF))
  1387. if errF != nil {
  1388. problem("风险", "ORM 显式指定的主键值会被静默忽略, 产生\"插入成功但按主键查不到\"的行",
  1389. fmt.Sprintf("引擎 pkID 恒为 TotalRows+1; 写 ID=999999 后 Find(999999) 失败, 只有 Find(1) 能查到, 且该行 _id 列值仍是 999999 → _id 与真实 pkID 失配, 之后 Delete/批量删除会按 _id 操作到错误的行 (err=%v)", errF))
  1390. }
  1391. }
  1392. func probePKReuse(pdb *DB.DB) {
  1393. sub("T4 删除中间 pkID 后再插入: pkID 复用与碰撞")
  1394. pr, err := DB.UseTable[Emp](pdb, "probe_reuse")
  1395. if softMust("UseTable(probe_reuse)", err) {
  1396. return
  1397. }
  1398. for i := int64(1); i <= 3; i++ {
  1399. if softMust(fmt.Sprintf("Insert(No=%d)", i), pr.Insert(&Emp{No: i, Name: empName(i), Age: 20 + i, Score: i, Dept: "Dept000", Salary: float64(i), Active: true})) {
  1400. return
  1401. }
  1402. }
  1403. if softMust("Delete(pkID=2)", pr.Delete(2)) {
  1404. return
  1405. }
  1406. cnt, _ := pr.Count()
  1407. fmt.Printf(" 插 3 行(pkID 1,2,3) → 删 pkID=2 → Count=%d TotalRows=%d\n", cnt, coreRows(pdb, "probe_reuse"))
  1408. if softMust("Insert(No=99)", pr.Insert(&Emp{No: 99, Name: empName(99), Age: 99, Score: 9, Dept: "Dept000", Salary: 9, Active: true})) {
  1409. return
  1410. }
  1411. type kv struct {
  1412. ID uint64
  1413. No int64
  1414. }
  1415. rows := make([]kv, 0, 4)
  1416. if softMust("Select(probe_reuse)", pr.Select(func(e *Emp) bool { rows = append(rows, kv{e.ID, e.No}); return true })) {
  1417. return
  1418. }
  1419. fmt.Printf(" 再插 1 行(No=99) 后存活: ")
  1420. for _, r := range rows {
  1421. fmt.Printf("{_id=%d,No=%d} ", r.ID, r.No)
  1422. }
  1423. fmt.Printf("| Count=%d TotalRows=%d\n", len(rows), coreRows(pdb, "probe_reuse"))
  1424. hasNo3 := false
  1425. for _, r := range rows {
  1426. if r.No == 3 {
  1427. hasNo3 = true
  1428. }
  1429. }
  1430. chk("T4 新插入不应覆盖既有存活行(No=3 应仍在, 共 3 行)", hasNo3 && len(rows) == 3,
  1431. "3 行且含 No=3", fmt.Sprintf("%d 行 含No3=%v", len(rows), hasNo3))
  1432. if !hasNo3 || len(rows) != 3 {
  1433. problem("严重", "pkID = TotalRows+1 且删除会回退 TotalRows → 删中间行后再插入会复用已占用的 pkID",
  1434. fmt.Sprintf("删 pkID=2 后 TotalRows 回退为 2, 下一条插入分到 pkID=3, 与仍存活的 pkID=3 冲突, 实测存活 %d 行(期望 3, 含 No=3)", len(rows)))
  1435. }
  1436. }
  1437. func probeFloatTrunc(pdb *DB.DB) {
  1438. sub("T5 float64 列的小数精度")
  1439. pt, err := DB.UseTable[Emp](pdb, "probe_float")
  1440. if softMust("UseTable(probe_float)", err) {
  1441. return
  1442. }
  1443. softMust("Insert(Salary=123.75)", pt.Insert(&Emp{No: 1, Name: "F", Age: 1, Score: 1, Dept: "Dept000", Salary: 123.75, Active: true}))
  1444. got, err2 := pt.Find(1)
  1445. if err2 == nil {
  1446. fmt.Printf(" 写入 Salary=123.75 → 读回 %v\n", got.Salary)
  1447. }
  1448. chk("T5 float64 的小数部分应保留", err2 == nil && got.Salary == 123.75,
  1449. "123.75", fmt.Sprintf("%v err=%v", got, err2))
  1450. if got != nil && got.Salary != 123.75 {
  1451. problem("风险", "ORM 把 float64 映射成 8 字节整数列, 小数被静默截断",
  1452. fmt.Sprintf("写入 123.75 读回 %v; 金额/比例等小数数据不能直接存 float64 列(需业务侧放大成整数)", got.Salary))
  1453. }
  1454. }
  1455. // ============================== P9 关库重开 ==============================
  1456. func phase9Reopen(path string) {
  1457. phase("P9 关库重开: 持久化 / schema / 索引 校验")
  1458. db2, err := DB.Open(path)
  1459. must(err)
  1460. defer func() { must(db2.Close()) }()
  1461. empCore, ok := db2.CoreDB().FindTable("emp")
  1462. if !ok {
  1463. must(fmt.Errorf("重开后找不到 emp 表"))
  1464. }
  1465. cols := empCore.GetActiveFieldNames()
  1466. fmt.Printf(" 重开后 emp 活跃列: %v\n", cols)
  1467. fmt.Printf(" TotalRows=%d (期望 %d) | 磁盘 %.1fMB\n",
  1468. empCore.TotalRows(), expectRows(), float64(dbSize(path))/(1<<20))
  1469. chk("9.1 重开后 TotalRows 一致", empCore.TotalRows() == uint64(expectRows()),
  1470. strconv.FormatInt(expectRows(), 10), strconv.FormatUint(empCore.TotalRows(), 10))
  1471. chk("9.1 重开后 schema 保留 P7 结果", contains(cols, "Dept") && contains(cols, "DeptName"),
  1472. "含 Dept 与 DeptName", fmt.Sprintf("%v", cols))
  1473. _, bad := verifyEmpCoreScan(empCore, "重开后 core 全量校验", expectRows(), true)
  1474. chk("9.1 重开后逐行公式校验 0 错", bad == 0, "0", strconv.FormatInt(bad, 10))
  1475. idxOK := 0
  1476. for _, nm := range []string{"pk_emp", "idx_emp_no", "idx_emp_age", "idx_emp_dept"} {
  1477. if _, ok := empCore.IndexMeta().FindIndex(nm); ok {
  1478. idxOK++
  1479. }
  1480. }
  1481. chk("9.1 主键+三个二级索引均存在", idxOK == 4, "4", strconv.Itoa(idxOK))
  1482. sub("9.2 ORM 重开表 (UseTable 会按 struct 同步 schema)")
  1483. empT2, err := DB.UseTable[Emp](db2, "emp")
  1484. must(err)
  1485. cols2 := empCore.GetActiveFieldNames()
  1486. fmt.Printf(" UseTable[Emp] 之后活跃列: %v\n", cols2)
  1487. if contains(cols, "DeptName") && !contains(cols2, "DeptName") {
  1488. problem("风险", "ORM 重开表会按 struct 强制同步 schema, 静默删除 struct 里没有的列",
  1489. fmt.Sprintf("纯 core 视角活跃列=%v 含 DeptName, 调用 UseTable[Emp] 后活跃列=%v, DeptName 被自动 DropField; 手工加过列的表被 ORM 打开就会\"丢列\"(数据仍在行内, 再加回同名列可复活)", cols, cols2))
  1490. }
  1491. scanned, bad2 := verifyEmpORM(empT2, "重开后 ORM 全量校验", expectRows())
  1492. chk("9.2 ORM 重开后可读全表", scanned == expectRows() && bad2 == 0,
  1493. fmt.Sprintf("%d 行 0 错", expectRows()), fmt.Sprintf("%d 行 %d 错", scanned, bad2))
  1494. e, errF := empT2.Find(1)
  1495. chk("9.2 重开后主键点查可用", errF == nil && e.No == 1, "No=1", fmt.Sprintf("%v err=%v", e, errF))
  1496. c, errC := empT2.Where("No", "=", int64(sampleNo)).Count()
  1497. chk("9.2 重开后索引等值查询可用", errC == nil && c == 1, "1", fmt.Sprintf("%d err=%v", c, errC))
  1498. want := int64(0)
  1499. for i := int64(1); i <= nEmp; i++ {
  1500. if !isDeletedNo(i) && expAge(i) == int64(bulkAgeTo) {
  1501. want++
  1502. }
  1503. }
  1504. for k := int64(1); k <= extraRows; k++ {
  1505. if expAge(nEmp+k) == int64(bulkAgeTo) {
  1506. want++
  1507. }
  1508. }
  1509. c2, errC2 := empT2.Where("Age", "=", int64(bulkAgeTo)).Count()
  1510. chk("9.2 重开后 Age 索引计数正确", errC2 == nil && c2 == want,
  1511. strconv.FormatInt(want, 10), fmt.Sprintf("%d err=%v", c2, errC2))
  1512. }
  1513. // ============================== P10 汇总 ==============================
  1514. func report(start time.Time) {
  1515. phase("P10 汇总报告")
  1516. fmt.Printf("\n【性能总表】共 %d 项\n", len(perfs))
  1517. fmt.Printf("%-10s %-52s %12s %13s %14s\n", "阶段", "操作", "行数", "耗时", "速率")
  1518. fmt.Printf("%s\n", strings.Repeat("-", 108))
  1519. for _, p := range perfs {
  1520. rate := "-"
  1521. if p.Cost > 0 && p.Rows > 0 {
  1522. rate = human(float64(p.Rows)/p.Cost.Seconds()) + "/s"
  1523. }
  1524. fmt.Printf("%-10s %-52s %12d %13s %14s\n", p.Phase, p.Op, p.Rows, p.Cost.Round(time.Millisecond), rate)
  1525. }
  1526. pass, fail := 0, 0
  1527. for _, c := range checks {
  1528. if c.Pass {
  1529. pass++
  1530. } else {
  1531. fail++
  1532. }
  1533. }
  1534. fmt.Printf("\n【校验总表】共 %d 项: 通过 %d, 失败 %d\n", len(checks), pass, fail)
  1535. if fail > 0 {
  1536. fmt.Printf("%-66s %-24s %s\n", "失败检查项", "期望", "实际")
  1537. for _, c := range checks {
  1538. if !c.Pass {
  1539. fmt.Printf("%-66s %-24s %s\n", c.Name, c.Expect, c.Actual)
  1540. }
  1541. }
  1542. }
  1543. fmt.Printf("\n【数据问题清单】共 %d 项\n", len(problems))
  1544. for i, p := range problems {
  1545. fmt.Printf(" %d. [%s] %s\n %s\n", i+1, p.Level, p.Title, p.Detail)
  1546. }
  1547. fmt.Printf("\n【结论】总耗时 %s | 校验 %d/%d 通过 | 数据问题 %d 项\n",
  1548. time.Since(start).Round(time.Second), pass, len(checks), len(problems))
  1549. fmt.Printf(" 主库保留: %s (%.1fMB) | 探针库: %s (%.1fMB)\n",
  1550. dbPath, float64(dbSize(dbPath))/(1<<20), probePath, float64(dbSize(probePath))/(1<<20))
  1551. fmt.Printf(" 可将上述 .db 交给 analyzer 审计\n")
  1552. }
  1553. // ============================== main ==============================
  1554. func deriveScale() {
  1555. delCount = nEmp / 100
  1556. if delCount > 100000 {
  1557. delCount = 100000
  1558. }
  1559. if delCount < 100 {
  1560. delCount = 100
  1561. }
  1562. delCount = delCount / 100 * 100
  1563. delFrom, delTo = nEmp-delCount+1, nEmp
  1564. rowUpdCnt = nEmp / 5000
  1565. if rowUpdCnt > 2000 {
  1566. rowUpdCnt = 2000
  1567. }
  1568. if rowUpdCnt < 10 {
  1569. rowUpdCnt = 10
  1570. }
  1571. rowUpdAt = nEmp/4 + 1
  1572. // 抽样行号: 取 nEmp/3, 保证存在且不落在 rowUpd 区间(仅 10 行)或尾部删除区间内
  1573. sampleNo = nEmp / 3
  1574. if sampleNo < 1 {
  1575. sampleNo = 1
  1576. }
  1577. nJoinMod = nOrder / 5
  1578. if nJoinMod > 200000 {
  1579. nJoinMod = 200000
  1580. }
  1581. if nJoinMod < 5 {
  1582. nJoinMod = 5
  1583. }
  1584. }
  1585. func main() {
  1586. start := time.Now()
  1587. nEmp = envInt64("TSDB_BENCH_N", 10_000_000)
  1588. nEmp = nEmp / 1000 * 1000
  1589. if nEmp < 1000 {
  1590. nEmp = 1000
  1591. }
  1592. nOrder = envInt64("TSDB_BENCH_ORDERS", 1_000_000)
  1593. name := os.Getenv("TSDB_BENCH_DB")
  1594. if name == "" {
  1595. name = "bench_10m.db"
  1596. }
  1597. abs, err := filepath.Abs(name)
  1598. must(err)
  1599. dbPath = abs
  1600. probePath = filepath.Join(filepath.Dir(abs), strings.TrimSuffix(filepath.Base(abs), ".db")+"_probe.db")
  1601. deriveScale()
  1602. phase0()
  1603. fmt.Printf("\n 清理同前缀旧库文件...\n")
  1604. cleanOld(dbPath)
  1605. cleanOld(probePath)
  1606. db, err := DB.Open(dbPath)
  1607. must(err)
  1608. db.SetCommitPreset("batch")
  1609. db.SetAutoCommitDelay(0)
  1610. empT, deptT, orderT, empCore := phase1(db)
  1611. phase2InsertEmp(db, empT, empCore)
  1612. phase3Queries(empT, empCore)
  1613. _, orderAfterDel := phase4Multi(db, empT, deptT, orderT, empCore)
  1614. phase5Update(db, empT, empCore)
  1615. phase6Delete(db, empT, orderT, empCore, orderAfterDel)
  1616. phase7Schema(db, empT, empCore)
  1617. must(db.Commit())
  1618. must(db.Close())
  1619. fmt.Printf("\n [已关库] 磁盘 %.1fMB | %s\n", float64(dbSize(dbPath))/(1<<20), memLine())
  1620. phase8Probes()
  1621. phase9Reopen(dbPath)
  1622. report(start)
  1623. }