From 5d4b02ac64a9895df2a284fe978e77098225eaf5 Mon Sep 17 00:00:00 2001 From: VioletQwQ-0 <1659648118@qq.com> Date: Mon, 3 Aug 2026 12:36:28 +0800 Subject: [PATCH 1/2] fix fulltext tokenization with nullable columns --- .../table_function/fulltext_tokenize.go | 22 ++-- .../table_function/fulltext_tokenize_test.go | 110 ++++++++++++++++++ .../cases/fulltext/fulltext.result | 25 ++-- .../cases/fulltext/fulltext_bm25.result | 31 ++--- .../fulltext_update_consistency.result | 90 ++++++++++++++ .../fulltext/fulltext_update_consistency.sql | 71 +++++++++++ .../distributed/cases/fulltext/gojieba.result | 31 ++--- 7 files changed, 328 insertions(+), 52 deletions(-) diff --git a/pkg/sql/colexec/table_function/fulltext_tokenize.go b/pkg/sql/colexec/table_function/fulltext_tokenize.go index b15abd8fcda48..fcf09c1ecf5f0 100644 --- a/pkg/sql/colexec/table_function/fulltext_tokenize.go +++ b/pkg/sql/colexec/table_function/fulltext_tokenize.go @@ -135,21 +135,16 @@ func (u *tokenizeState) start(tf *TableFunction, proc *process.Process, nthRow i idVec := tf.ctr.argVecs[0] id := vector.GetAny(idVec, nthRow, true) - isnull := false - for i := 1; i < vlen; i++ { - isnull = (isnull || tf.ctr.argVecs[i].IsNull(uint64(nthRow))) - } - - if isnull { - return nil - } - switch u.param.Parser { case "", "ngram", "default", "gojieba": var content bytes.Buffer + hasContentColumn := false for i := 1; i < vlen; i++ { - if i > 1 { + if tf.ctr.argVecs[i].IsNull(uint64(nthRow)) { + continue + } + if hasContentColumn { content.WriteByte('\n') } data := tf.ctr.argVecs[i].GetStringAt(nthRow) @@ -172,6 +167,7 @@ func (u *tokenizeState) start(tf *TableFunction, proc *process.Process, nthRow i } else { content.WriteString(data) } + hasContentColumn = true } var tok tokenizer.Tokenizer @@ -201,6 +197,9 @@ func (u *tokenizeState) start(tf *TableFunction, proc *process.Process, nthRow i case "json": joffset := int32(0) for i := 1; i < vlen; i++ { + if tf.ctr.argVecs[i].IsNull(uint64(nthRow)) { + continue + } c := tf.ctr.argVecs[i].GetRawBytesAt(nthRow) var bj bytejson.ByteJson @@ -238,6 +237,9 @@ func (u *tokenizeState) start(tf *TableFunction, proc *process.Process, nthRow i case "json_value": joffset := int32(0) for i := 1; i < vlen; i++ { + if tf.ctr.argVecs[i].IsNull(uint64(nthRow)) { + continue + } c := tf.ctr.argVecs[i].GetRawBytesAt(nthRow) var bj bytejson.ByteJson diff --git a/pkg/sql/colexec/table_function/fulltext_tokenize_test.go b/pkg/sql/colexec/table_function/fulltext_tokenize_test.go index dfc50e7cfaf6a..642713b5efc47 100644 --- a/pkg/sql/colexec/table_function/fulltext_tokenize_test.go +++ b/pkg/sql/colexec/table_function/fulltext_tokenize_test.go @@ -279,6 +279,116 @@ func TestFullTextTokenizeCallJSONValue(t *testing.T) { ut.arg.ctr.state.free(ut.arg, ut.proc, false, nil) } +func TestFullTextTokenizeSkipsNullContentColumns(t *testing.T) { + testCases := []struct { + name string + param string + left string + right string + words []string + }{ + {name: "ordinary", left: "lefttoken", right: "righttoken", words: []string{"lefttoken", "righttoken"}}, + {name: "default", param: `{"parser":"default"}`, left: "lefttoken", right: "righttoken", words: []string{"lefttoken", "righttoken"}}, + {name: "ngram", param: `{"parser":"ngram"}`, left: "lefttoken", right: "righttoken", words: []string{"lefttoken", "righttoken"}}, + {name: "gojieba", param: `{"parser":"gojieba"}`, left: "北京", right: "清华大学", words: []string{"北京", "清华大学"}}, + {name: "json", param: `{"parser":"json"}`, left: `{"k":"lefttoken"}`, right: `{"k":"righttoken"}`, words: []string{"lefttoken", "righttoken"}}, + {name: "json_value", param: `{"parser":"json_value"}`, left: `{"k":"lefttoken"}`, right: `{"k":"righttoken"}`, words: []string{"lefttoken", "righttoken"}}, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + ut := newFTTTestCase(t, mpool.MustNewZero(), fttdefaultAttrs, testCase.param) + ut.arg.Args = fullTextTokenizeThreeArgExprs() + inbat := makeNullableFullTextBatch(t, ut.proc, testCase.left, testCase.right) + defer inbat.Clean(ut.proc.Mp()) + + state := &tokenizeState{} + ut.arg.ctr.argVecs = inbat.Vecs + defer state.free(ut.arg, ut.proc, false, nil) + + for row := 0; row < inbat.RowCount(); row++ { + require.NoError(t, state.start(ut.arg, ut.proc, row, nil)) + + if row == 2 { + require.Empty(t, state.doc.Words) + result, err := state.call(ut.arg, ut.proc) + require.NoError(t, err) + require.Equal(t, vm.CancelResult.Status, result.Status) + continue + } + + expectedWords := testCase.words + if row < 2 { + expectedWords = []string{testCase.words[1-row]} + } + require.Len(t, state.doc.Words, len(expectedWords)+1) + for i, expectedWord := range expectedWords { + require.Equal(t, expectedWord, state.doc.Words[i].Word) + } + require.Equal(t, int32(0), state.doc.Words[0].Pos) + require.Equal(t, "__DocLen", state.doc.Words[len(expectedWords)].Word) + require.Equal(t, int32(len(expectedWords)), state.doc.Words[len(expectedWords)].Pos) + } + }) + } +} + +func TestFullTextTokenizeSkipsConstNullContentColumn(t *testing.T) { + ut := newFTTTestCase(t, mpool.MustNewZero(), fttdefaultAttrs, "") + ut.arg.Args = fullTextTokenizeThreeArgExprs() + + idVec, err := vector.NewConstFixed(types.T_int32.ToType(), int32(1), 1, ut.proc.Mp()) + require.NoError(t, err) + leftVec := vector.NewConstNull(types.T_varchar.ToType(), 1, ut.proc.Mp()) + rightVec, err := vector.NewConstBytes(types.T_varchar.ToType(), []byte("righttoken"), 1, ut.proc.Mp()) + require.NoError(t, err) + for _, vec := range []*vector.Vector{idVec, leftVec, rightVec} { + defer vec.Free(ut.proc.Mp()) + } + ut.arg.ctr.argVecs = []*vector.Vector{idVec, leftVec, rightVec} + + state := &tokenizeState{} + defer state.free(ut.arg, ut.proc, false, nil) + require.NoError(t, state.start(ut.arg, ut.proc, 0, nil)) + require.Len(t, state.doc.Words, 2) + require.Equal(t, "righttoken", state.doc.Words[0].Word) + require.Equal(t, int32(0), state.doc.Words[0].Pos) + require.Equal(t, "__DocLen", state.doc.Words[1].Word) +} + +func fullTextTokenizeThreeArgExprs() []*plan.Expr { + return []*plan.Expr{ + {Typ: plan.Type{Id: int32(types.T_int32)}}, + {Typ: plan.Type{Id: int32(types.T_varchar), Width: 128}}, + {Typ: plan.Type{Id: int32(types.T_varchar), Width: 128}}, + } +} + +func makeNullableFullTextBatch(t *testing.T, proc *process.Process, left, right string) *batch.Batch { + t.Helper() + bat := batch.NewWithSize(3) + bat.Vecs[0] = vector.NewVec(types.T_int32.ToType()) + bat.Vecs[1] = vector.NewVec(types.T_varchar.ToType()) + bat.Vecs[2] = vector.NewVec(types.T_varchar.ToType()) + + rows := []struct { + leftNull bool + rightNull bool + }{ + {leftNull: true}, + {rightNull: true}, + {leftNull: true, rightNull: true}, + {}, + } + for i, row := range rows { + require.NoError(t, vector.AppendFixed(bat.Vecs[0], int32(i+1), false, proc.Mp())) + require.NoError(t, vector.AppendBytes(bat.Vecs[1], []byte(left), row.leftNull, proc.Mp())) + require.NoError(t, vector.AppendBytes(bat.Vecs[2], []byte(right), row.rightNull, proc.Mp())) + } + bat.SetRowCount(len(rows)) + return bat +} + // create const input exprs func makeConstInputExprsFTT() []*plan.Expr { diff --git a/test/distributed/cases/fulltext/fulltext.result b/test/distributed/cases/fulltext/fulltext.result index 235362d8b00d2..f38cba1a9651b 100644 --- a/test/distributed/cases/fulltext/fulltext.result +++ b/test/distributed/cases/fulltext/fulltext.result @@ -78,8 +78,8 @@ id body title 4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 select *, match(body, title) against('遠東兒童中文' in natural language mode) as score from src; id body title score -4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 1.4499049 -5 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 遠東兒童中文 0.72495246 +4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 1.9542363 +5 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 遠東兒童中文 0.97711813 select *, match(body) against('遠東兒童中文' in natural language mode) as score from src; id body title score 4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 1.8211576 @@ -125,8 +125,8 @@ select * from src where match(body, title) against('"blue is red"' in boolean mo id body title select src.*, match(body, title) against('blue') from src; id body title MATCH (body, title) AGAINST (blue) -2 sky is blue no limit 0.36247623 -3 blue is not red colorful 0.36247623 +2 sky is blue no limit 0.48855907 +3 blue is not red colorful 0.48855907 select count(*) from src where match(title, body) against('red'); count(*) 2 @@ -185,8 +185,8 @@ id body title 4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 select *, match(body, title) against('遠東兒童中文' in natural language mode) as score from src; id body title score -4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 1.4499049 -5 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 遠東兒童中文 0.72495246 +4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 1.9542363 +5 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 遠東兒童中文 0.97711813 select *, match(body) against('遠東兒童中文' in natural language mode) as score from src; id body title score 4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 1.8211576 @@ -219,8 +219,8 @@ id body title 3 blue is not red colorful select src.*, match(body, title) against('blue') from src; id body title MATCH (body, title) AGAINST (blue) -2 sky is blue no limit 0.36247623 -3 blue is not red colorful 0.36247623 +2 sky is blue no limit 0.48855907 +3 blue is not red colorful 0.48855907 select count(*) from src where match(title, body) against('red'); count(*) 2 @@ -366,8 +366,8 @@ id body title 4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 select *, match(body, title) against('遠東兒童中文' in natural language mode) as score from src; id body title score -4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 1.4499049 -5 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 遠東兒童中文 0.72495246 +4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 1.9542363 +5 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 遠東兒童中文 0.97711813 select * from src where match(body, title) against('+red blue' in boolean mode); id body title 3 blue is not red colorful @@ -397,8 +397,8 @@ id body title 3 blue is not red colorful select src.*, match(body, title) against('blue') from src; id body title MATCH (body, title) AGAINST (blue) -2 sky is blue no limit 0.36247623 -3 blue is not red colorful 0.36247623 +2 sky is blue no limit 0.48855907 +3 blue is not red colorful 0.48855907 select count(*) from src where match(title, body) against('red'); count(*) 2 @@ -424,6 +424,7 @@ id body title insert into src (id, body) values (11, 'color is brown'); select * from src where match(body, title) against('brown'); id body title +11 color is brown null update src set title='a good title' where id=11; select * from src where match(body, title) against('brown'); id body title diff --git a/test/distributed/cases/fulltext/fulltext_bm25.result b/test/distributed/cases/fulltext/fulltext_bm25.result index 43964fa3dae8a..80e6d0dd3ddc8 100644 --- a/test/distributed/cases/fulltext/fulltext_bm25.result +++ b/test/distributed/cases/fulltext/fulltext_bm25.result @@ -60,7 +60,7 @@ id body title 3 blue is not red colorful select *, match(body, title) against('is red' in natural language mode) as score from src; id body title score -0 color is red t1 2.6239278 +0 color is red t1 3.1378522 select * from src where match(body, title) against('教學指引'); id body title 6 各個單元主題內容涵蓋中華文化及生活應用的介紹。本套教材含課本、教學指引、生字卡、學生作業本與CD，中英對照，精美大字版。本系列有繁體字及簡體字兩種版本印行。 中文短篇小說 @@ -76,8 +76,8 @@ id body title 4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 select *, match(body, title) against('遠東兒童中文' in natural language mode) as score from src; id body title score -4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 0.82570016 -5 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 遠東兒童中文 0.5491443 +4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 1.0109811 +5 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 遠東兒童中文 0.6560832 select *, match(body) against('遠東兒童中文' in natural language mode) as score from src; id body title score 4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 1.2537473 @@ -123,8 +123,8 @@ select * from src where match(body, title) against('"blue is red"' in boolean mo id body title select src.*, match(body, title) against('blue') from src; id body title MATCH (body, title) AGAINST (blue) -2 sky is blue no limit 0.56676567 -3 blue is not red colorful 0.56676567 +2 sky is blue no limit 0.7411132 +3 blue is not red colorful 0.7411132 select count(*) from src where match(title, body) against('red'); count(*) 2 @@ -167,7 +167,7 @@ id body title 3 blue is not red colorful select *, match(body, title) against('is red' in natural language mode) as score from src; id body title score -0 color is red t1 2.6239278 +0 color is red t1 3.1378522 select * from src where match(body, title) against('教學指引'); id body title 6 各個單元主題內容涵蓋中華文化及生活應用的介紹。本套教材含課本、教學指引、生字卡、學生作業本與CD，中英對照，精美大字版。本系列有繁體字及簡體字兩種版本印行。 中文短>篇小說 @@ -183,8 +183,8 @@ id body title 4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 select *, match(body, title) against('遠東兒童中文' in natural language mode) as score from src; id body title score -4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 0.82570016 -5 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 遠東兒童中文 0.5491443 +4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 1.0109811 +5 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 遠東兒童中文 0.6560832 select *, match(body) against('遠東兒童中文' in natural language mode) as score from src; id body title score 4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 1.2537473 @@ -217,8 +217,8 @@ id body title 3 blue is not red colorful select src.*, match(body, title) against('blue') from src; id body title MATCH (body, title) AGAINST (blue) -2 sky is blue no limit 0.56676567 -3 blue is not red colorful 0.56676567 +2 sky is blue no limit 0.7411132 +3 blue is not red colorful 0.7411132 select count(*) from src where match(title, body) against('red'); count(*) 2 @@ -348,7 +348,7 @@ id body title 3 blue is not red colorful select *, match(body, title) against('is red' in natural language mode) as score from src; id body title score -0 color is red t1 2.6239278 +0 color is red t1 3.1378522 select * from src where match(body, title) against('教學指引'); id body title 6 各個單元主題內容涵蓋中華文化及生活應用的介紹。本套教材含課本、教學指引、生字卡、學生作業本與CD，中英對照，精美大字版。本系列有繁體字及簡體字兩種版本印行。 中文短篇小說\n @@ -364,8 +364,8 @@ id body title 4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 select *, match(body, title) against('遠東兒童中文' in natural language mode) as score from src; id body title score -4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 0.82570016 -5 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 遠東兒童中文 0.5491443 +4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 1.0109811 +5 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 遠東兒童中文 0.6560832 select * from src where match(body, title) against('+red blue' in boolean mode); id body title 3 blue is not red colorful @@ -395,8 +395,8 @@ id body title 3 blue is not red colorful select src.*, match(body, title) against('blue') from src; id body title MATCH (body, title) AGAINST (blue) -2 sky is blue no limit 0.56676567 -3 blue is not red colorful 0.56676567 +2 sky is blue no limit 0.7411132 +3 blue is not red colorful 0.7411132 select count(*) from src where match(title, body) against('red'); count(*) 2 @@ -422,6 +422,7 @@ id body title insert into src (id, body) values (11, 'color is brown'); select * from src where match(body, title) against('brown'); id body title +11 color is brown null update src set title='a good title' where id=11; select * from src where match(body, title) against('brown'); id body title diff --git a/test/distributed/cases/fulltext/fulltext_update_consistency.result b/test/distributed/cases/fulltext/fulltext_update_consistency.result index 839e640e3cb0d..9374ca6bfa7e2 100644 --- a/test/distributed/cases/fulltext/fulltext_update_consistency.result +++ b/test/distributed/cases/fulltext/fulltext_update_consistency.result @@ -66,4 +66,94 @@ update ft_async_composite set tenant = 'b', id = 2 where tenant = 'a' and id = 1 select tenant, id from ft_async_composite where match(body) against('composite') order by tenant, id; ➤ tenant[12,-1,0] ¦ id[-5,64,0] 𝄀 b ¦ 2 +create table ft_null_default(id int primary key, left_text text, right_text text); +insert into ft_null_default values +(1, null, 'defaultrighttoken'), +(2, 'defaultlefttoken', null), +(3, null, null); +create fulltext index ft_body on ft_null_default(left_text, right_text); +select id from ft_null_default where match(left_text, right_text) against('defaultrighttoken') order by id; +➤ id[4,32,0] 𝄀 +1 +select id from ft_null_default where match(left_text, right_text) against('defaultlefttoken') order by id; +➤ id[4,32,0] 𝄀 +2 +create table ft_null_ngram(id int primary key, left_text text, right_text text); +insert into ft_null_ngram values (1, null, '風月無情'), (2, '神雕俠侶', null), (3, null, null); +create fulltext index ft_body on ft_null_ngram(left_text, right_text) with parser ngram; +select id from ft_null_ngram where match(left_text, right_text) against('風月無情') order by id; +➤ id[4,32,0] 𝄀 +1 +select id from ft_null_ngram where match(left_text, right_text) against('神雕俠侶') order by id; +➤ id[4,32,0] 𝄀 +2 +create table ft_null_gojieba(id int primary key, left_text text, right_text text); +insert into ft_null_gojieba values (1, null, '清华大学'), (2, '北京', null), (3, null, null); +create fulltext index ft_body on ft_null_gojieba(left_text, right_text) with parser gojieba; +select id from ft_null_gojieba where match(left_text, right_text) against('清华大学') order by id; +➤ id[4,32,0] 𝄀 +1 +select id from ft_null_gojieba where match(left_text, right_text) against('北京') order by id; +➤ id[4,32,0] 𝄀 +2 +create table ft_null_json(id int primary key, left_doc json, right_doc json); +insert into ft_null_json values +(1, null, '{"k":"jsonrighttoken"}'), +(2, '{"k":"jsonlefttoken"}', null), +(3, null, null); +create fulltext index ft_body on ft_null_json(left_doc, right_doc) with parser json; +select id from ft_null_json where match(left_doc, right_doc) against('jsonrighttoken') order by id; +➤ id[4,32,0] 𝄀 +1 +select id from ft_null_json where match(left_doc, right_doc) against('jsonlefttoken') order by id; +➤ id[4,32,0] 𝄀 +2 +create table ft_null_json_value(id int primary key, left_doc json, right_doc json); +insert into ft_null_json_value values +(1, null, '{"k":"jsonvaluerighttoken"}'), +(2, '{"k":"jsonvaluelefttoken"}', null), +(3, null, null); +create fulltext index ft_body on ft_null_json_value(left_doc, right_doc) with parser json_value; +select id from ft_null_json_value where match(left_doc, right_doc) against('jsonvaluerighttoken') order by id; +➤ id[4,32,0] 𝄀 +1 +select id from ft_null_json_value where match(left_doc, right_doc) against('jsonvaluelefttoken') order by id; +➤ id[4,32,0] 𝄀 +2 +create table ft_null_dml( +id int primary key, +left_text text, +right_text text, +fulltext ft_body(left_text, right_text) +); +insert into ft_null_dml(id, right_text) values (1, 'insertrighttoken'); +select id from ft_null_dml where match(left_text, right_text) against('insertrighttoken') order by id; +➤ id[4,32,0] 𝄀 +1 +update ft_null_dml set left_text = 'updatelefttoken', right_text = null where id = 1; +select id from ft_null_dml where match(left_text, right_text) against('insertrighttoken') order by id; +➤ id[4,32,0] +select id from ft_null_dml where match(left_text, right_text) against('updatelefttoken') order by id; +➤ id[4,32,0] 𝄀 +1 +update ft_null_dml set left_text = null, right_text = null where id = 1; +select id from ft_null_dml where match(left_text, right_text) against('updatelefttoken') order by id; +➤ id[4,32,0] +update ft_null_dml set right_text = 'reviverighttoken' where id = 1; +select id from ft_null_dml where match(left_text, right_text) against('reviverighttoken') order by id; +➤ id[4,32,0] 𝄀 +1 +delete from ft_null_dml where id = 1; +select id from ft_null_dml where match(left_text, right_text) against('reviverighttoken') order by id; +➤ id[4,32,0] +create table ft_null_async( +id int primary key, +left_text text, +right_text text, +fulltext ft_body(left_text, right_text) async +); +insert into ft_null_async(id, right_text) values (1, 'asyncrighttoken'); +select id from ft_null_async where match(left_text, right_text) against('asyncrighttoken') order by id; +➤ id[4,32,0] 𝄀 +1 drop database fulltext_update_consistency; diff --git a/test/distributed/cases/fulltext/fulltext_update_consistency.sql b/test/distributed/cases/fulltext/fulltext_update_consistency.sql index 0e8b02f0b999e..91911e2b8fdab 100644 --- a/test/distributed/cases/fulltext/fulltext_update_consistency.sql +++ b/test/distributed/cases/fulltext/fulltext_update_consistency.sql @@ -62,4 +62,75 @@ update ft_async_composite set tenant = 'b', id = 2 where tenant = 'a' and id = 1 -- @wait_expect(2, 30) select tenant, id from ft_async_composite where match(body) against('composite') order by tenant, id; +-- A NULL in one indexed content column skips only that column. Cover index +-- creation for every parser accepted by fulltext_index_tokenize. +create table ft_null_default(id int primary key, left_text text, right_text text); +insert into ft_null_default values + (1, null, 'defaultrighttoken'), + (2, 'defaultlefttoken', null), + (3, null, null); +create fulltext index ft_body on ft_null_default(left_text, right_text); +select id from ft_null_default where match(left_text, right_text) against('defaultrighttoken') order by id; +select id from ft_null_default where match(left_text, right_text) against('defaultlefttoken') order by id; + +create table ft_null_ngram(id int primary key, left_text text, right_text text); +insert into ft_null_ngram values (1, null, '風月無情'), (2, '神雕俠侶', null), (3, null, null); +create fulltext index ft_body on ft_null_ngram(left_text, right_text) with parser ngram; +select id from ft_null_ngram where match(left_text, right_text) against('風月無情') order by id; +select id from ft_null_ngram where match(left_text, right_text) against('神雕俠侶') order by id; + +create table ft_null_gojieba(id int primary key, left_text text, right_text text); +insert into ft_null_gojieba values (1, null, '清华大学'), (2, '北京', null), (3, null, null); +create fulltext index ft_body on ft_null_gojieba(left_text, right_text) with parser gojieba; +select id from ft_null_gojieba where match(left_text, right_text) against('清华大学') order by id; +select id from ft_null_gojieba where match(left_text, right_text) against('北京') order by id; + +create table ft_null_json(id int primary key, left_doc json, right_doc json); +insert into ft_null_json values + (1, null, '{"k":"jsonrighttoken"}'), + (2, '{"k":"jsonlefttoken"}', null), + (3, null, null); +create fulltext index ft_body on ft_null_json(left_doc, right_doc) with parser json; +select id from ft_null_json where match(left_doc, right_doc) against('jsonrighttoken') order by id; +select id from ft_null_json where match(left_doc, right_doc) against('jsonlefttoken') order by id; + +create table ft_null_json_value(id int primary key, left_doc json, right_doc json); +insert into ft_null_json_value values + (1, null, '{"k":"jsonvaluerighttoken"}'), + (2, '{"k":"jsonvaluelefttoken"}', null), + (3, null, null); +create fulltext index ft_body on ft_null_json_value(left_doc, right_doc) with parser json_value; +select id from ft_null_json_value where match(left_doc, right_doc) against('jsonvaluerighttoken') order by id; +select id from ft_null_json_value where match(left_doc, right_doc) against('jsonvaluelefttoken') order by id; + +-- Synchronous DML maintenance must add, replace, remove, and revive documents +-- when indexed columns move between NULL and non-NULL. +create table ft_null_dml( + id int primary key, + left_text text, + right_text text, + fulltext ft_body(left_text, right_text) +); +insert into ft_null_dml(id, right_text) values (1, 'insertrighttoken'); +select id from ft_null_dml where match(left_text, right_text) against('insertrighttoken') order by id; +update ft_null_dml set left_text = 'updatelefttoken', right_text = null where id = 1; +select id from ft_null_dml where match(left_text, right_text) against('insertrighttoken') order by id; +select id from ft_null_dml where match(left_text, right_text) against('updatelefttoken') order by id; +update ft_null_dml set left_text = null, right_text = null where id = 1; +select id from ft_null_dml where match(left_text, right_text) against('updatelefttoken') order by id; +update ft_null_dml set right_text = 'reviverighttoken' where id = 1; +select id from ft_null_dml where match(left_text, right_text) against('reviverighttoken') order by id; +delete from ft_null_dml where id = 1; +select id from ft_null_dml where match(left_text, right_text) against('reviverighttoken') order by id; + +create table ft_null_async( + id int primary key, + left_text text, + right_text text, + fulltext ft_body(left_text, right_text) async +); +insert into ft_null_async(id, right_text) values (1, 'asyncrighttoken'); +-- @wait_expect(2, 30) +select id from ft_null_async where match(left_text, right_text) against('asyncrighttoken') order by id; + drop database fulltext_update_consistency; diff --git a/test/distributed/cases/fulltext/gojieba.result b/test/distributed/cases/fulltext/gojieba.result index 8bfcbe64df0f2..2533e52351fa0 100644 --- a/test/distributed/cases/fulltext/gojieba.result +++ b/test/distributed/cases/fulltext/gojieba.result @@ -43,8 +43,8 @@ select * from src where match(body, title) against('版一、二冊' in natural id body title select *, match(body, title) against('遠東兒童中文' in natural language mode) as score from src; id body title score -4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 3.6247623 -5 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 遠東兒童中文 1.8123811 +4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 4.8855906 +5 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 遠東兒童中文 2.4427953 select *, match(body) against('遠東兒童中文' in natural language mode) as score from src; id body title score 4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 4.552894 @@ -90,8 +90,8 @@ select * from src where match(body, title) against('"blue is red"' in boolean mo id body title select src.*, match(body, title) against('blue') from src; id body title MATCH (body, title) AGAINST (blue) -2 sky is blue no limit 0.36247623 -3 blue is not red colorful 0.36247623 +2 sky is blue no limit 0.48855907 +3 blue is not red colorful 0.48855907 select count(*) from src where match(title, body) against('red'); count(*) 2 @@ -149,8 +149,8 @@ select * from src where match(body, title) against('版一、二冊' in natural id body title select *, match(body, title) against('遠東兒童中文' in natural language mode) as score from src; id body title score -4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 3.6247623 -5 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 遠東兒童中文 1.8123811 +4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 4.8855906 +5 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 遠東兒童中文 2.4427953 select *, match(body) against('遠東兒童中文' in natural language mode) as score from src; id body title score 4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 4.552894 @@ -183,8 +183,8 @@ id body title 3 blue is not red colorful select src.*, match(body, title) against('blue') from src; id body title MATCH (body, title) AGAINST (blue) -2 sky is blue no limit 0.36247623 -3 blue is not red colorful 0.36247623 +2 sky is blue no limit 0.48855907 +3 blue is not red colorful 0.48855907 select count(*) from src where match(title, body) against('red'); count(*) 2 @@ -271,8 +271,8 @@ select * from src where match(body, title) against('版一、二冊' in natural id body title select *, match(body, title) against('遠東兒童中文' in natural language mode) as score from src; id body title score -4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 3.6247623 -5 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 遠東兒童中文 1.8123811 +4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 4.8855906 +5 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 遠東兒童中文 2.4427953 select * from src where match(body, title) against('+red blue' in boolean mode); id body title 3 blue is not red colorful @@ -302,8 +302,8 @@ id body title 3 blue is not red colorful select src.*, match(body, title) against('blue') from src; id body title MATCH (body, title) AGAINST (blue) -2 sky is blue no limit 0.36247623 -3 blue is not red colorful 0.36247623 +2 sky is blue no limit 0.48855907 +3 blue is not red colorful 0.48855907 select count(*) from src where match(title, body) against('red'); count(*) 2 @@ -329,6 +329,7 @@ id body title insert into src (id, body) values (11, 'color is brown'); select * from src where match(body, title) against('brown'); id body title +11 color is brown null update src set title='a good title' where id=11; select * from src where match(body, title) against('brown'); id body title @@ -448,14 +449,14 @@ id body title 3 blue is not red colorful select *, match(body, title) against('is red' in natural language mode) as score from src; id body title score -0 color is red t1 2.5699196 +0 color is red t1 3.0634575 select * from src where match(body, title) against('教學指引'); id body title 6 各個單元主題內容涵蓋中華文化及生活應用的介紹。本套教材含課本、教學指引、生字卡、學生作業本與CD，中英對照，精美大字版。本系列有繁體字及簡體字兩種版本印行。 中文短篇小說 select *, match(body, title) against('遠東兒童中文' in natural language mode) as score from src; id body title score -4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 2.1168313 -5 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 遠東兒童中文 1.3810165 +4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 2.6046562 +5 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 遠東兒童中文 1.6553308 select * from src where match(body, title) against('+red blue' in boolean mode); id body title 3 blue is not red colorful From d7ee184ca9c8336854eb3397b23119c7094fde6e Mon Sep 17 00:00:00 2001 From: VioletQwQ-0 <1659648118@qq.com> Date: Mon, 3 Aug 2026 14:32:27 +0800 Subject: [PATCH 2/2] test: update fulltext null-column expectations --- test/distributed/cases/ddl/table_kind.result | 4 ++-- .../pessimistic_transaction/fulltext/fulltext_async.result | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test/distributed/cases/ddl/table_kind.result b/test/distributed/cases/ddl/table_kind.result index f899a4314e79c..cbd868517b755 100644 --- a/test/distributed/cases/ddl/table_kind.result +++ b/test/distributed/cases/ddl/table_kind.result @@ -327,7 +327,7 @@ select * from src2 where match(body, title) against('版一、二冊' in natural 4 ¦ 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 ¦ 遠東兒童中文 select *, match(body, title) against('遠東兒童中文' in natural language mode) as score from src2; ➤ id[-5,64,0] ¦ body[12,-1,0] ¦ title[12,0,0] ¦ score[7,5,0] 𝄀 -4 ¦ 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 ¦ 遠東兒童中文 ¦ 1.4499049 𝄀 -5 ¦ 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 ¦ 遠東兒童中文 ¦ 0.72495246 +4 ¦ 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 ¦ 遠東兒童中文 ¦ 1.9542363 𝄀 +5 ¦ 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 ¦ 遠東兒童中文 ¦ 0.97711813 drop table src2; DROP DATABASE IF EXISTS db1; diff --git a/test/distributed/cases/pessimistic_transaction/fulltext/fulltext_async.result b/test/distributed/cases/pessimistic_transaction/fulltext/fulltext_async.result index 04d6f2d4c3145..efd32ef869ba2 100644 --- a/test/distributed/cases/pessimistic_transaction/fulltext/fulltext_async.result +++ b/test/distributed/cases/pessimistic_transaction/fulltext/fulltext_async.result @@ -25,6 +25,7 @@ select * from src2 where match(body, title) against('red'); id1 id2 body title id0 0 red t1 id3 3 blue red t4 +id4 4 bright red null NULL show create table src2; Table Create Table src2 CREATE TABLE `src2` (\n `id1` varchar(65535) NOT NULL,\n `id2` bigint NOT NULL,\n `body` char(128) DEFAULT NULL,\n `title` text DEFAULT NULL,\n PRIMARY KEY (`id1`,`id2`),\n FULLTEXT `ftidx`(`body`,`title`) ASYNC\n)