Skip to content

fix: tokenize non-null fulltext columns independently #26609

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
XuPeng-SH merged 3 commits into from
Aug 3, 2026
Merged

fix: tokenize non-null fulltext columns independently #26609

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5d4b02a
fix fulltext tokenization with nullable columns
VioletQwQ-0 Aug 3, 2026
d7ee184
test: update fulltext null-column expectations
VioletQwQ-0 Aug 3, 2026
a38fe4f
Merge branch 'main' into codex/issue-26557-fulltext-null-columns
XuPeng-SH Aug 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 12 additions & 10 deletions pkg/sql/colexec/table_function/fulltext_tokenize.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,21 +135,16 @@ func (u *tokenizeState) start(tf *TableFunction, proc *process.Process, nthRow i
idVec := tf.ctr.argVecs[0]
id := vector.GetAny(idVec, nthRow, true)

isnull := false
for i := 1; i < vlen; i++ {
isnull = (isnull || tf.ctr.argVecs[i].IsNull(uint64(nthRow)))
}

if isnull {
return nil
}

switch u.param.Parser {
case "", "ngram", "default", "gojieba":

var content bytes.Buffer
hasContentColumn := false
for i := 1; i < vlen; i++ {
if i > 1 {
if tf.ctr.argVecs[i].IsNull(uint64(nthRow)) {
continue
}
if hasContentColumn {
content.WriteByte('\n')
}
data := tf.ctr.argVecs[i].GetStringAt(nthRow)
Expand All @@ -172,6 +167,7 @@ func (u *tokenizeState) start(tf *TableFunction, proc *process.Process, nthRow i
} else {
content.WriteString(data)
}
hasContentColumn = true
}

var tok tokenizer.Tokenizer
Expand Down Expand Up @@ -201,6 +197,9 @@ func (u *tokenizeState) start(tf *TableFunction, proc *process.Process, nthRow i
case "json":
joffset := int32(0)
for i := 1; i < vlen; i++ {
if tf.ctr.argVecs[i].IsNull(uint64(nthRow)) {
continue
}
c := tf.ctr.argVecs[i].GetRawBytesAt(nthRow)

var bj bytejson.ByteJson
Expand Down Expand Up @@ -238,6 +237,9 @@ func (u *tokenizeState) start(tf *TableFunction, proc *process.Process, nthRow i
case "json_value":
joffset := int32(0)
for i := 1; i < vlen; i++ {
if tf.ctr.argVecs[i].IsNull(uint64(nthRow)) {
continue
}
c := tf.ctr.argVecs[i].GetRawBytesAt(nthRow)

var bj bytejson.ByteJson
Expand Down
110 changes: 110 additions & 0 deletions pkg/sql/colexec/table_function/fulltext_tokenize_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,116 @@ func TestFullTextTokenizeCallJSONValue(t *testing.T) {
ut.arg.ctr.state.free(ut.arg, ut.proc, false, nil)
}

func TestFullTextTokenizeSkipsNullContentColumns(t *testing.T) {
testCases := []struct {
name string
param string
left string
right string
words []string
}{
{name: "ordinary", left: "lefttoken", right: "righttoken", words: []string{"lefttoken", "righttoken"}},
{name: "default", param: `{"parser":"default"}`, left: "lefttoken", right: "righttoken", words: []string{"lefttoken", "righttoken"}},
{name: "ngram", param: `{"parser":"ngram"}`, left: "lefttoken", right: "righttoken", words: []string{"lefttoken", "righttoken"}},
{name: "gojieba", param: `{"parser":"gojieba"}`, left: "北京", right: "清华大学", words: []string{"北京", "清华大学"}},
{name: "json", param: `{"parser":"json"}`, left: `{"k":"lefttoken"}`, right: `{"k":"righttoken"}`, words: []string{"lefttoken", "righttoken"}},
{name: "json_value", param: `{"parser":"json_value"}`, left: `{"k":"lefttoken"}`, right: `{"k":"righttoken"}`, words: []string{"lefttoken", "righttoken"}},
}

for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
ut := newFTTTestCase(t, mpool.MustNewZero(), fttdefaultAttrs, testCase.param)
ut.arg.Args = fullTextTokenizeThreeArgExprs()
inbat := makeNullableFullTextBatch(t, ut.proc, testCase.left, testCase.right)
defer inbat.Clean(ut.proc.Mp())

state := &tokenizeState{}
ut.arg.ctr.argVecs = inbat.Vecs
defer state.free(ut.arg, ut.proc, false, nil)

for row := 0; row < inbat.RowCount(); row++ {
require.NoError(t, state.start(ut.arg, ut.proc, row, nil))

if row == 2 {
require.Empty(t, state.doc.Words)
result, err := state.call(ut.arg, ut.proc)
require.NoError(t, err)
require.Equal(t, vm.CancelResult.Status, result.Status)
continue
}

expectedWords := testCase.words
if row < 2 {
expectedWords = []string{testCase.words[1-row]}
}
require.Len(t, state.doc.Words, len(expectedWords)+1)
for i, expectedWord := range expectedWords {
require.Equal(t, expectedWord, state.doc.Words[i].Word)
}
require.Equal(t, int32(0), state.doc.Words[0].Pos)
require.Equal(t, "__DocLen", state.doc.Words[len(expectedWords)].Word)
require.Equal(t, int32(len(expectedWords)), state.doc.Words[len(expectedWords)].Pos)
}
})
}
}

func TestFullTextTokenizeSkipsConstNullContentColumn(t *testing.T) {
ut := newFTTTestCase(t, mpool.MustNewZero(), fttdefaultAttrs, "")
ut.arg.Args = fullTextTokenizeThreeArgExprs()

idVec, err := vector.NewConstFixed(types.T_int32.ToType(), int32(1), 1, ut.proc.Mp())
require.NoError(t, err)
leftVec := vector.NewConstNull(types.T_varchar.ToType(), 1, ut.proc.Mp())
rightVec, err := vector.NewConstBytes(types.T_varchar.ToType(), []byte("righttoken"), 1, ut.proc.Mp())
require.NoError(t, err)
for _, vec := range []*vector.Vector{idVec, leftVec, rightVec} {
defer vec.Free(ut.proc.Mp())
}
ut.arg.ctr.argVecs = []*vector.Vector{idVec, leftVec, rightVec}

state := &tokenizeState{}
defer state.free(ut.arg, ut.proc, false, nil)
require.NoError(t, state.start(ut.arg, ut.proc, 0, nil))
require.Len(t, state.doc.Words, 2)
require.Equal(t, "righttoken", state.doc.Words[0].Word)
require.Equal(t, int32(0), state.doc.Words[0].Pos)
require.Equal(t, "__DocLen", state.doc.Words[1].Word)
}

func fullTextTokenizeThreeArgExprs() []*plan.Expr {
return []*plan.Expr{
{Typ: plan.Type{Id: int32(types.T_int32)}},
{Typ: plan.Type{Id: int32(types.T_varchar), Width: 128}},
{Typ: plan.Type{Id: int32(types.T_varchar), Width: 128}},
}
}

func makeNullableFullTextBatch(t *testing.T, proc *process.Process, left, right string) *batch.Batch {
t.Helper()
bat := batch.NewWithSize(3)
bat.Vecs[0] = vector.NewVec(types.T_int32.ToType())
bat.Vecs[1] = vector.NewVec(types.T_varchar.ToType())
bat.Vecs[2] = vector.NewVec(types.T_varchar.ToType())

rows := []struct {
leftNull bool
rightNull bool
}{
{leftNull: true},
{rightNull: true},
{leftNull: true, rightNull: true},
{},
}
for i, row := range rows {
require.NoError(t, vector.AppendFixed(bat.Vecs[0], int32(i+1), false, proc.Mp()))
require.NoError(t, vector.AppendBytes(bat.Vecs[1], []byte(left), row.leftNull, proc.Mp()))
require.NoError(t, vector.AppendBytes(bat.Vecs[2], []byte(right), row.rightNull, proc.Mp()))
}
bat.SetRowCount(len(rows))
return bat
}

// create const input exprs
func makeConstInputExprsFTT() []*plan.Expr {

Expand Down
4 changes: 2 additions & 2 deletions test/distributed/cases/ddl/table_kind.result
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -327,7 +327,7 @@ select * from src2 where match(body, title) against('版一、二冊' in natural
4 ¦ 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 ¦ 遠東兒童中文
select *, match(body, title) against('遠東兒童中文' in natural language mode) as score from src2;
➤ id[-5,64,0] ¦ body[12,-1,0] ¦ title[12,0,0] ¦ score[7,5,0] 𝄀
4 ¦ 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 ¦ 遠東兒童中文 ¦ 1.4499049 𝄀
5 ¦ 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 ¦ 遠東兒童中文 ¦ 0.72495246
4 ¦ 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 ¦ 遠東兒童中文 ¦ 1.9542363 𝄀
5 ¦ 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 ¦ 遠東兒童中文 ¦ 0.97711813
drop table src2;
DROP DATABASE IF EXISTS db1;
25 changes: 13 additions & 12 deletions test/distributed/cases/fulltext/fulltext.result
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,8 @@ id body title
4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文
select *, match(body, title) against('遠東兒童中文' in natural language mode) as score from src;
id body title score
4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 1.4499049
5 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 遠東兒童中文 0.72495246
4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 1.9542363
5 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 遠東兒童中文 0.97711813
select *, match(body) against('遠東兒童中文' in natural language mode) as score from src;
id body title score
4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 1.8211576
Expand Down Expand Up @@ -125,8 +125,8 @@ select * from src where match(body, title) against('"blue is red"' in boolean mo
id body title
select src.*, match(body, title) against('blue') from src;
id body title MATCH (body, title) AGAINST (blue)
2 sky is blue no limit 0.36247623
3 blue is not red colorful 0.36247623
2 sky is blue no limit 0.48855907
3 blue is not red colorful 0.48855907
select count(*) from src where match(title, body) against('red');
count(*)
2
Expand Down Expand Up @@ -185,8 +185,8 @@ id body title
4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文
select *, match(body, title) against('遠東兒童中文' in natural language mode) as score from src;
id body title score
4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 1.4499049
5 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 遠東兒童中文 0.72495246
4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 1.9542363
5 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 遠東兒童中文 0.97711813
select *, match(body) against('遠東兒童中文' in natural language mode) as score from src;
id body title score
4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 1.8211576
Expand Down Expand Up @@ -219,8 +219,8 @@ id body title
3 blue is not red colorful
select src.*, match(body, title) against('blue') from src;
id body title MATCH (body, title) AGAINST (blue)
2 sky is blue no limit 0.36247623
3 blue is not red colorful 0.36247623
2 sky is blue no limit 0.48855907
3 blue is not red colorful 0.48855907
select count(*) from src where match(title, body) against('red');
count(*)
2
Expand Down Expand Up @@ -366,8 +366,8 @@ id body title
4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文
select *, match(body, title) against('遠東兒童中文' in natural language mode) as score from src;
id body title score
4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 1.4499049
5 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 遠東兒童中文 0.72495246
4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 1.9542363
5 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 遠東兒童中文 0.97711813
select * from src where match(body, title) against('+red blue' in boolean mode);
id body title
3 blue is not red colorful
Expand Down Expand Up @@ -397,8 +397,8 @@ id body title
3 blue is not red colorful
select src.*, match(body, title) against('blue') from src;
id body title MATCH (body, title) AGAINST (blue)
2 sky is blue no limit 0.36247623
3 blue is not red colorful 0.36247623
2 sky is blue no limit 0.48855907
3 blue is not red colorful 0.48855907
select count(*) from src where match(title, body) against('red');
count(*)
2
Expand All @@ -424,6 +424,7 @@ id body title
insert into src (id, body) values (11, 'color is brown');
select * from src where match(body, title) against('brown');
id body title
11 color is brown null
update src set title='a good title' where id=11;
select * from src where match(body, title) against('brown');
id body title
Expand Down
31 changes: 16 additions & 15 deletions test/distributed/cases/fulltext/fulltext_bm25.result
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ id body title
3 blue is not red colorful
select *, match(body, title) against('is red' in natural language mode) as score from src;
id body title score
0 color is red t1 2.6239278
0 color is red t1 3.1378522
select * from src where match(body, title) against('教學指引');
id body title
6 各個單元主題內容涵蓋中華文化及生活應用的介紹。本套教材含課本、教學指引、生字卡、學生作業本與CD，中英對照，精美大字版。本系列有繁體字及簡體字兩種版本印行。 中文短篇小說
Expand All @@ -76,8 +76,8 @@ id body title
4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文
select *, match(body, title) against('遠東兒童中文' in natural language mode) as score from src;
id body title score
4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 0.82570016
5 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 遠東兒童中文 0.5491443
4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 1.0109811
5 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 遠東兒童中文 0.6560832
select *, match(body) against('遠東兒童中文' in natural language mode) as score from src;
id body title score
4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 1.2537473
Expand Down Expand Up @@ -123,8 +123,8 @@ select * from src where match(body, title) against('"blue is red"' in boolean mo
id body title
select src.*, match(body, title) against('blue') from src;
id body title MATCH (body, title) AGAINST (blue)
2 sky is blue no limit 0.56676567
3 blue is not red colorful 0.56676567
2 sky is blue no limit 0.7411132
3 blue is not red colorful 0.7411132
select count(*) from src where match(title, body) against('red');
count(*)
2
Expand Down Expand Up @@ -167,7 +167,7 @@ id body title
3 blue is not red colorful
select *, match(body, title) against('is red' in natural language mode) as score from src;
id body title score
0 color is red t1 2.6239278
0 color is red t1 3.1378522
select * from src where match(body, title) against('教學指引');
id body title
6 各個單元主題內容涵蓋中華文化及生活應用的介紹。本套教材含課本、教學指引、生字卡、學生作業本與CD，中英對照，精美大字版。本系列有繁體字及簡體字兩種版本印行。 中文短>篇小說
Expand All @@ -183,8 +183,8 @@ id body title
4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文
select *, match(body, title) against('遠東兒童中文' in natural language mode) as score from src;
id body title score
4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 0.82570016
5 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 遠東兒童中文 0.5491443
4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 1.0109811
5 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 遠東兒童中文 0.6560832
select *, match(body) against('遠東兒童中文' in natural language mode) as score from src;
id body title score
4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 1.2537473
Expand Down Expand Up @@ -217,8 +217,8 @@ id body title
3 blue is not red colorful
select src.*, match(body, title) against('blue') from src;
id body title MATCH (body, title) AGAINST (blue)
2 sky is blue no limit 0.56676567
3 blue is not red colorful 0.56676567
2 sky is blue no limit 0.7411132
3 blue is not red colorful 0.7411132
select count(*) from src where match(title, body) against('red');
count(*)
2
Expand Down Expand Up @@ -348,7 +348,7 @@ id body title
3 blue is not red colorful
select *, match(body, title) against('is red' in natural language mode) as score from src;
id body title score
0 color is red t1 2.6239278
0 color is red t1 3.1378522
select * from src where match(body, title) against('教學指引');
id body title
6 各個單元主題內容涵蓋中華文化及生活應用的介紹。本套教材含課本、教學指引、生字卡、學生作業本與CD，中英對照，精美大字版。本系列有繁體字及簡體字兩種版本印行。 中文短篇小說\n
Expand All @@ -364,8 +364,8 @@ id body title
4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文
select *, match(body, title) against('遠東兒童中文' in natural language mode) as score from src;
id body title score
4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 0.82570016
5 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 遠東兒童中文 0.5491443
4 遠東兒童中文是針對6到9歲的小朋友精心設計的中文學習教材，共三冊，目前已出版一、二冊。 遠東兒童中文 1.0109811
5 每冊均採用近百張全幅彩圖及照片，生動活潑、自然真實，加深兒童學習印象，洋溢學習樂趣。 遠東兒童中文 0.6560832
select * from src where match(body, title) against('+red blue' in boolean mode);
id body title
3 blue is not red colorful
Expand Down Expand Up @@ -395,8 +395,8 @@ id body title
3 blue is not red colorful
select src.*, match(body, title) against('blue') from src;
id body title MATCH (body, title) AGAINST (blue)
2 sky is blue no limit 0.56676567
3 blue is not red colorful 0.56676567
2 sky is blue no limit 0.7411132
3 blue is not red colorful 0.7411132
select count(*) from src where match(title, body) against('red');
count(*)
2
Expand All @@ -422,6 +422,7 @@ id body title
insert into src (id, body) values (11, 'color is brown');
select * from src where match(body, title) against('brown');
id body title
11 color is brown null
update src set title='a good title' where id=11;
select * from src where match(body, title) against('brown');
id body title
Expand Down
Loading
Loading