feat(query): Enhanced Inverted Index for VARIANT Type to precise matching Object within Arrays

Author: b41sh

Cargo.lock

@@ -661,9 +661,9 @@ recursive = { git = "https://github.com/datafuse-extras/recursive.git", rev = "1
 sled = { git = "https://github.com/datafuse-extras/sled", tag = "v0.34.7-datafuse.1" }
 state-machine-api = { git = "https://github.com/databendlabs/state-machine-api.git", tag = "v0.3.4" }
 sub-cache = { git = "https://github.com/databendlabs/sub-cache", tag = "v0.2.1" }
-tantivy = { git = "https://github.com/datafuse-extras/tantivy", rev = "9065a4d" }
-tantivy-common = { git = "https://github.com/datafuse-extras/tantivy", rev = "9065a4d", package = "tantivy-common" }
-tantivy-jieba = { git = "https://github.com/datafuse-extras/tantivy-jieba", rev = "ac27464" }
-tantivy-query-grammar = { git = "https://github.com/datafuse-extras/tantivy", rev = "9065a4d", package = "tantivy-query-grammar" }
+tantivy = { git = "https://github.com/b41sh/tantivy", rev = "edadb8a6bc167723219d60d11f509fa4882e93b7" }
+tantivy-common = { git = "https://github.com/b41sh/tantivy", rev = "edadb8a6bc167723219d60d11f509fa4882e93b7", package = "tantivy-common" }
+tantivy-jieba = { git = "https://github.com/b41sh/tantivy-jieba", rev = "4486c42aa2933fe28c4bcd40273193f59c4e3182" }
+tantivy-query-grammar = { git = "https://github.com/b41sh/tantivy", rev = "edadb8a6bc167723219d60d11f509fa4882e93b7", package = "tantivy-query-grammar" }
 watcher = { git = "https://github.com/databendlabs/watcher", tag = "v0.4.2" }
 xorfilter-rs = { git = "https://github.com/datafuse-extras/xorfilter", tag = "databend-alpha.4" }

Cargo.toml

@@ -18,6 +18,7 @@ use std::sync::Arc;
 use databend_common_catalog::plan::InvertedIndexInfo;
 use databend_common_catalog::plan::PushDownInfo;
 use databend_common_exception::Result;
+use databend_common_expression::types::DataType;
 use databend_common_expression::types::F32;
 use databend_storages_common_io::ReadSettings;
 use opendal::Operator;
@@ -95,6 +96,14 @@ impl InvertedIndexPruner {
                     need_position = true;
                 }
             });
+            for field_id in &field_ids {
+                let field = inverted_index_info.index_schema.field(*field_id as usize);
+                let data_type = field.data_type().remove_nullable();
+                if data_type == DataType::Variant {
+                    need_position = true;
+                    break;
+                }
+            }
 
             // whether need to generate score internl column
             let has_score = inverted_index_info.has_score;

src/query/storages/fuse/src/pruning/inverted_index_pruner.rs

@@ -500,6 +500,33 @@ query IT
 select * from t2 where query('body:test');
 ----
 
+
+statement ok
+CREATE TABLE t3 (id int, body variant, INVERTED INDEX idx (body))
+
+statement ok
+INSERT INTO t3 VALUES
+(1, '{"videoInfo":{"extraData":[{ "name": "codecA", "type": "mp4" },{ "name": "codecB", "type": "jpg" }]}}'),
+(2, '{"videoInfo":{"extraData":[{ "name": "codecA", "type": "jpg" },{ "name": "codecA", "type": "mp4" }]}}'),
+(3, '{"videoInfo":{"extraData":[{ "name": "codecA", "type": "jpg" },{ "name": "codecB", "type": "mp4" }]}}');
+
+query IT
+select * from t3 where query('body.videoInfo.extraData.name:codecA AND body.videoInfo.extraData.type:jpg');
+----
+2 {"videoInfo":{"extraData":[{"name":"codecA","type":"jpg"},{"name":"codecA","type":"mp4"}]}}
+3 {"videoInfo":{"extraData":[{"name":"codecA","type":"jpg"},{"name":"codecB","type":"mp4"}]}}
+
+query IT
+select * from t3 where query('body.videoInfo.extraData.name:codecA AND body.videoInfo.extraData.type:mp4');
+----
+1 {"videoInfo":{"extraData":[{"name":"codecA","type":"mp4"},{"name":"codecB","type":"jpg"}]}}
+2 {"videoInfo":{"extraData":[{"name":"codecA","type":"jpg"},{"name":"codecA","type":"mp4"}]}}
+
+query IT
+select * from t3 where query('body.videoInfo.extraData.name:codecB AND body.videoInfo.extraData.type:jpg');
+----
+1 {"videoInfo":{"extraData":[{"name":"codecA","type":"mp4"},{"name":"codecB","type":"jpg"}]}}
+
 statement ok
 CREATE TABLE t_native (id int, content string, INVERTED INDEX idx1 (content)) storage_format = 'native' row_per_page = 2;
 
@@ -528,3 +555,4 @@ use default
 statement ok
 drop database test_inverted_index
 
+