diff --git a/cmd/index_analyzer/main.go b/cmd/index_analyzer/main.go index b7422da4..9e3ab7cb 100644 --- a/cmd/index_analyzer/main.go +++ b/cmd/index_analyzer/main.go @@ -168,7 +168,7 @@ func analyzeIndex( break } block := token.TableBlock{} - block.Unpack(data) + block.Unpack(data, ver) tokenTableBlocks = append(tokenTableBlocks, block) } tokenTable := token.TableFromBlocks(tokenTableBlocks) diff --git a/config/frac_version.go b/config/frac_version.go index 73c3261a..1ed8f27a 100644 --- a/config/frac_version.go +++ b/config/frac_version.go @@ -21,6 +21,9 @@ const ( // BinaryDataV4 - delta bitpack encoded MIDs and LIDs BinaryDataV4 + + // BinaryDataV5 - token blocks have zone maps (eng letters presense, max token length) + BinaryDataV5 ) -const CurrentFracVersion = BinaryDataV4 +const CurrentFracVersion = BinaryDataV5 diff --git a/frac/fraction_concurrency_test.go b/frac/fraction_concurrency_test.go index dd85ae91..cbc0b76b 100644 --- a/frac/fraction_concurrency_test.go +++ b/frac/fraction_concurrency_test.go @@ -287,6 +287,17 @@ func generatesMessages(numMessages, bulkSize int) ([]*testDoc, [][]string, time. for i := 0; i < numMessages; i++ { service := services[rand.IntN(len(services))] message := messages[rand.IntN(len(messages))] + // populate message with various unique tokens like ids and hex numbers (matches real installation) + x := rand.IntN(20) + switch x { + case 1: + message = message + fmt.Sprintf(" %dms", rand.IntN(10000000)) + case 2: + message = message + fmt.Sprintf(" %dus", rand.IntN(10000000)) + default: + message = message + fmt.Sprintf(" %d", rand.IntN(10000000)) + } + level := rand.IntN(6) timestamp := fromTime.Add(time.Duration(i) * time.Millisecond) id := fmt.Sprintf("id-%d", i) diff --git a/frac/fraction_test.go b/frac/fraction_test.go index 180ec4e2..abe1f7b2 100644 --- a/frac/fraction_test.go +++ b/frac/fraction_test.go @@ -1454,6 +1454,15 @@ func (s *FractionTestSuite) TestSearchLargeFrac() { fromTime: fromTime, toTime: toTime, }, + { + name: "message:req*t OR message:f*ed", + query: "message:req*t OR message:f*ed", + filter: func(doc *testDoc) bool { + return strings.Contains(doc.message, "request") || strings.Contains(doc.message, "failed") + }, + fromTime: fromTime, + toTime: toTime, + }, { name: "message:*uest OR id:*1", query: "message:*uest OR id:*1", @@ -1911,11 +1920,11 @@ func (s *FractionTestSuite) TestFractionInfo() { s.Require().Equal(uint64(0), info.IndexOnDisk, "index on disk doesn't match") case *frac.Sealed: s.Require().Equal(uint64(0), info.MetaOnDisk, "meta on disk doesn't match. actual value") - s.Require().True(info.IndexOnDisk > uint64(1300) && info.IndexOnDisk < uint64(1450), + s.Require().True(info.IndexOnDisk > uint64(1300) && info.IndexOnDisk < uint64(1500), "index on disk doesn't match. actual value: %d", info.IndexOnDisk) case *frac.Remote: s.Require().Equal(uint64(0), info.MetaOnDisk, "meta on disk doesn't match. actual value") - s.Require().True(info.IndexOnDisk > uint64(1300) && info.IndexOnDisk < uint64(1450), + s.Require().True(info.IndexOnDisk > uint64(1300) && info.IndexOnDisk < uint64(1500), "index on disk doesn't match. actual value: %d", info.IndexOnDisk) default: s.Require().Fail("unsupported fraction type") diff --git a/frac/remote.go b/frac/remote.go index 9630b8ca..e45a5d47 100644 --- a/frac/remote.go +++ b/frac/remote.go @@ -192,7 +192,7 @@ func (f *Remote) createDataProvider(ctx context.Context) (*sealedDataProvider, e lidsTable: f.blocksData.LIDsTable, lidsLoader: lids.NewLoader(f.info.BinaryDataVer, lidReader, f.indexCache.LIDs), tokenBlockLoader: token.NewBlockLoader(f.BaseFileName, tokenReader, f.indexCache.Tokens), - tokenTableLoader: token.NewTableLoader(f.BaseFileName, f.IsLegacy, tokenReader, f.indexCache.TokenTable), + tokenTableLoader: token.NewTableLoader(f.BaseFileName, f.Info().BinaryDataVer, f.IsLegacy, tokenReader, f.indexCache.TokenTable), idsTable: &f.blocksData.IDsTable, idsProvider: seqids.NewProvider( diff --git a/frac/sealed.go b/frac/sealed.go index dcf91f3f..5b8efa2b 100644 --- a/frac/sealed.go +++ b/frac/sealed.go @@ -515,7 +515,7 @@ func (f *Sealed) createDataProvider(ctx context.Context) *sealedDataProvider { lidsTable: f.blocksData.LIDsTable, lidsLoader: lids.NewLoader(f.info.BinaryDataVer, lidReader, f.indexCache.LIDs), tokenBlockLoader: token.NewBlockLoader(f.BaseFileName, tokenReader, f.indexCache.Tokens), - tokenTableLoader: token.NewTableLoader(f.BaseFileName, f.IsLegacy, tokenReader, f.indexCache.TokenTable), + tokenTableLoader: token.NewTableLoader(f.BaseFileName, f.Info().BinaryDataVer, f.IsLegacy, tokenReader, f.indexCache.TokenTable), idsTable: &f.blocksData.IDsTable, idsProvider: seqids.NewProvider( diff --git a/frac/sealed/token/block_loader.go b/frac/sealed/token/block_loader.go index 77c73798..2ff18407 100644 --- a/frac/sealed/token/block_loader.go +++ b/frac/sealed/token/block_loader.go @@ -13,6 +13,7 @@ import ( "github.com/ozontech/seq-db/logger" "github.com/ozontech/seq-db/pattern" "github.com/ozontech/seq-db/storage" + "github.com/ozontech/seq-db/util" ) const sizeOfUint32 = uint32(unsafe.Sizeof(uint32(0))) @@ -62,6 +63,14 @@ func (b *Block) GetToken(index int) []byte { return b.Payload[offset : offset+l] } +func (b *Block) LettersBitset() util.LettersBitset { + var builder util.LetterBitsetBuilder + for i := 0; i < b.Len(); i++ { + builder.Add(b.GetToken(i)) + } + return builder.Build() +} + func (b *Block) contains(from, to int, needle []byte) ([]int, error) { indexes := make([]int, 0) for i := from; i <= to; i++ { diff --git a/frac/sealed/token/provider.go b/frac/sealed/token/provider.go index 58286484..d3428c51 100644 --- a/frac/sealed/token/provider.go +++ b/frac/sealed/token/provider.go @@ -5,6 +5,7 @@ import ( "sort" "github.com/ozontech/seq-db/pattern" + "github.com/ozontech/seq-db/util" ) type Provider struct { @@ -60,24 +61,45 @@ func (tp *Provider) GetToken(tid uint32) []byte { } func (tp *Provider) FindContains(needle []byte) ([]uint32, error) { - return tp.findInBlocks(tp.FirstTID(), tp.LastTID(), func(b *Block, firstIndex, lastIndex int) ([]int, error) { - return b.contains(firstIndex, lastIndex, needle) - }) + requiredLetters := util.NewLettersBitset(needle) + + return tp.findInBlocks( + tp.FirstTID(), + tp.LastTID(), + func(e *TableEntry) bool { + return e.Letters.IsNil() || e.Letters.ContainsAll(requiredLetters) + }, + func(b *Block, firstIndex, lastIndex int) ([]int, error) { + return b.contains(firstIndex, lastIndex, needle) + }) } func (tp *Provider) FindToken(searcher pattern.Searcher) ([]uint32, error) { - return tp.findInBlocks(searcher.FirstTID(), searcher.LastTID(), func(b *Block, firstIndex, lastIndex int) ([]int, error) { - return b.find(firstIndex, lastIndex, searcher) - }) + return tp.findInBlocks( + searcher.FirstTID(), + searcher.LastTID(), + func(e *TableEntry) bool { + return searcher.CheckEntry(e.Letters) + }, + func(b *Block, firstIndex, lastIndex int) ([]int, error) { + return b.find(firstIndex, lastIndex, searcher) + }) } -func (tp *Provider) findInBlocks(firstTID, lastTID uint32, search func(*Block, int, int) ([]int, error)) ([]uint32, error) { +func (tp *Provider) findInBlocks( + firstTID, + lastTID uint32, + entryFilter func(*TableEntry) bool, + search func(*Block, int, int) ([]int, error)) ([]uint32, error) { var tids []uint32 for _, entry := range tp.entries { if !entry.checkTIDsInBlock(firstTID, lastTID) { continue } + if !entryFilter(entry) { + continue + } block := tp.findBlock(entry.BlockIndex) firstIndex, lastIndex := entry.narrowIndexes(firstTID, lastTID) diff --git a/frac/sealed/token/table_entry.go b/frac/sealed/token/table_entry.go index 705f364b..4c34f7f0 100644 --- a/frac/sealed/token/table_entry.go +++ b/frac/sealed/token/table_entry.go @@ -1,5 +1,7 @@ package token +import "github.com/ozontech/seq-db/util" + // TableEntry is part of token.Table and points to a fragment of token.Block type TableEntry struct { StartIndex uint32 // offset from the beginning of the block to the first token pointed to by the TableEntry @@ -9,6 +11,8 @@ type TableEntry struct { MinVal string // only saved for the first entry in block MaxVal string + + Letters util.LettersBitset // case-insensitive English letters present in entry tokens } func (t *TableEntry) GetIndexInTokensBlock(tid uint32) int { diff --git a/frac/sealed/token/table_loader.go b/frac/sealed/token/table_loader.go index a3fd61c0..beb61d48 100644 --- a/frac/sealed/token/table_loader.go +++ b/frac/sealed/token/table_loader.go @@ -9,15 +9,18 @@ import ( "go.uber.org/zap" "github.com/ozontech/seq-db/cache" + "github.com/ozontech/seq-db/config" "github.com/ozontech/seq-db/logger" "github.com/ozontech/seq-db/packer" "github.com/ozontech/seq-db/storage" + "github.com/ozontech/seq-db/util" ) const CacheKeyTable = 1 type TableLoader struct { fracName string + fracVer config.BinaryDataVersion isLegacy bool reader *storage.IndexReader @@ -31,12 +34,14 @@ type TableLoader struct { func NewTableLoader( fracName string, + fracVer config.BinaryDataVersion, isLegacy bool, reader *storage.IndexReader, c *cache.Cache[Table], ) *TableLoader { return &TableLoader{ fracName: fracName, + fracVer: fracVer, isLegacy: isLegacy, reader: reader, cache: c, @@ -126,7 +131,7 @@ func (l *TableLoader) loadBlocksLegacy() ([]TableBlock, error) { } var tb TableBlock - tb.Unpack(blockData) + tb.Unpack(blockData, l.fracVer) blocks = append(blocks, tb) blockIndex += 1 @@ -149,7 +154,7 @@ func (l *TableLoader) loadBlocks() ([]TableBlock, error) { } var tb TableBlock - tb.Unpack(data) + tb.Unpack(data, l.fracVer) blocks = append(blocks, tb) } @@ -206,6 +211,8 @@ func (b TableBlock) packedSize() int { // MaxVal size += sizeOfUint32 size += len(entry.MaxVal) + // Letters + size += sizeOfUint32 } } return size @@ -231,12 +238,14 @@ func (b TableBlock) Pack(buf []byte) []byte { // MaxVal buf = binary.LittleEndian.AppendUint32(buf, uint32(len(entry.MaxVal))) buf = append(buf, entry.MaxVal...) + // Letters + buf = binary.LittleEndian.AppendUint32(buf, uint32(entry.Letters)) } } return buf } -func (b *TableBlock) Unpack(data []byte) { +func (b *TableBlock) Unpack(data []byte, fracVer config.BinaryDataVersion) { b.FieldsTables = make([]FieldTable, 0) unpacker := packer.NewBytesUnpacker(data) @@ -260,6 +269,11 @@ func (b *TableBlock) Unpack(data []byte) { e.MinVal = minVal } e.MaxVal = maxVal + if fracVer >= config.BinaryDataV5 { + e.Letters = util.LettersBitset(unpacker.GetUint32()) + } else { + e.Letters = util.NewLettersBitsetNil() + } ft.Entries[i] = e } b.FieldsTables = append(b.FieldsTables, ft) diff --git a/frac/sealed_source.go b/frac/sealed_source.go index d537c16f..96d86e67 100644 --- a/frac/sealed_source.go +++ b/frac/sealed_source.go @@ -42,7 +42,7 @@ func NewSealedSource(f *Sealed) *SealedSource { ), lidsLoader: lids.NewLoader(f.Info().BinaryDataVer, &f.lidReader, f.indexCache.LIDs), tokenBlockLoader: token.NewBlockLoader(f.BaseFileName, &f.tokenReader, f.indexCache.Tokens), - tokenTableLoader: token.NewTableLoader(f.BaseFileName, f.IsLegacy, &f.tokenReader, f.indexCache.TokenTable), + tokenTableLoader: token.NewTableLoader(f.BaseFileName, f.Info().BinaryDataVer, f.IsLegacy, &f.tokenReader, f.indexCache.TokenTable), } } diff --git a/indexwriter/blocks.go b/indexwriter/blocks.go index 7b2d6c0c..3ddcf419 100644 --- a/indexwriter/blocks.go +++ b/indexwriter/blocks.go @@ -159,6 +159,7 @@ func newTokenTableEntry( ValCount: lastIndex - firstIndex + 1, // Number of tokens in this entry MinVal: minVal, // Smallest token value in range MaxVal: maxVal, // Largest token value in range + Letters: block.payload.LettersBitset(), } } diff --git a/indexwriter/blocks_test.go b/indexwriter/blocks_test.go index 546d08a7..63a283ce 100644 --- a/indexwriter/blocks_test.go +++ b/indexwriter/blocks_test.go @@ -10,6 +10,7 @@ import ( "github.com/ozontech/seq-db/frac/sealed/lids" "github.com/ozontech/seq-db/frac/sealed/token" "github.com/ozontech/seq-db/seq" + "github.com/ozontech/seq-db/util" ) type mockSource struct { @@ -59,6 +60,8 @@ func (m *mockSource) ID() iter.Seq2[DocLocation, error] { } func TestBlocksBuilder_BuildTokenBlocks(t *testing.T) { + lettersFV := util.NewLettersBitsetFromArray([30]bool{5: true, 21: true, 26: true}) + src := mockSource{ tokens: [][]byte{ []byte("f1v1"), // 1 @@ -156,6 +159,7 @@ func TestBlocksBuilder_BuildTokenBlocks(t *testing.T) { ValCount: 2, MinVal: "f1v1", MaxVal: "f1v2", + Letters: lettersFV, }, }, }, { @@ -168,6 +172,7 @@ func TestBlocksBuilder_BuildTokenBlocks(t *testing.T) { ValCount: 1, MinVal: "f2v1", MaxVal: "f2v1", + Letters: lettersFV, }, { StartIndex: 0, StartTID: 4, @@ -175,6 +180,7 @@ func TestBlocksBuilder_BuildTokenBlocks(t *testing.T) { ValCount: 3, MinVal: "f2v2", MaxVal: "f2v4", + Letters: lettersFV, }, { StartIndex: 0, StartTID: 7, @@ -182,6 +188,7 @@ func TestBlocksBuilder_BuildTokenBlocks(t *testing.T) { ValCount: 1, MinVal: "f2v5", MaxVal: "f2v5", + Letters: lettersFV, }, }, }, { @@ -194,6 +201,7 @@ func TestBlocksBuilder_BuildTokenBlocks(t *testing.T) { ValCount: 2, MinVal: "f3v1", MaxVal: "f3v2", + Letters: lettersFV, }, }, }, { @@ -206,6 +214,7 @@ func TestBlocksBuilder_BuildTokenBlocks(t *testing.T) { ValCount: 3, MinVal: "f4v1", MaxVal: "f4v3", + Letters: lettersFV, }, }, }, { @@ -218,6 +227,7 @@ func TestBlocksBuilder_BuildTokenBlocks(t *testing.T) { ValCount: 1, MinVal: "f5v1", MaxVal: "f5v1", + Letters: lettersFV, }, }, }, { @@ -230,6 +240,7 @@ func TestBlocksBuilder_BuildTokenBlocks(t *testing.T) { ValCount: 1, MinVal: "f6v1", MaxVal: "f6v1", + Letters: lettersFV, }, }, }, diff --git a/pattern/pattern.go b/pattern/pattern.go index 21e253cb..504d3929 100644 --- a/pattern/pattern.go +++ b/pattern/pattern.go @@ -37,6 +37,10 @@ func (s *baseSearch) LastTID() uint32 { return uint32(s.last) } +func (s *baseSearch) CheckEntry(letters util.LettersBitset) bool { + return true +} + type literalSearch struct { baseSearch value []byte @@ -78,11 +82,12 @@ func (s *literalSearch) Check(val []byte) (bool, error) { type wildcardSearch struct { baseSearch - prefix []byte - suffix []byte - middle [][]byte - middleLen int - narrowed bool + prefix []byte + suffix []byte + middle [][]byte + middleLen int + narrowed bool + lettersBitset util.LettersBitset } func newWildcardSearch(base baseSearch, token *parser.Literal) *wildcardSearch { @@ -106,6 +111,16 @@ func newWildcardSearch(base baseSearch, token *parser.Literal) *wildcardSearch { s.middleLen += len(val) } } + + // compute required letters for block filtering + allBytes := make([]byte, 0, len(s.prefix)+len(s.suffix)+s.middleLen) + allBytes = append(allBytes, s.prefix...) + allBytes = append(allBytes, s.suffix...) + for _, m := range s.middle { + allBytes = append(allBytes, m...) + } + s.lettersBitset = util.NewLettersBitset(allBytes) + return s } @@ -171,6 +186,10 @@ func (s *wildcardSearch) Check(val []byte) (bool, error) { return s.checkPrefix(val) && s.checkSuffix(val) && s.checkMiddle(val), nil } +func (s *wildcardSearch) CheckEntry(letters util.LettersBitset) bool { + return letters.IsNil() || letters.ContainsAll(s.lettersBitset) +} + type rangeTextSearch struct { baseSearch token *parser.Range @@ -341,6 +360,7 @@ type Searcher interface { FirstTID() uint32 LastTID() uint32 Check(val []byte) (bool, error) + CheckEntry(letters util.LettersBitset) bool } func newSearcher(token parser.Token, tp tokenProvider) Searcher { diff --git a/util/letters_bitset.go b/util/letters_bitset.go new file mode 100644 index 00000000..c9ebc955 --- /dev/null +++ b/util/letters_bitset.go @@ -0,0 +1,68 @@ +package util + +// LettersBitset is a bitset of symbols present in token blocks. Allows to prune token blocks on wildcard search. +// It uses 26 bits for eng letters (case-insensitive) as well as two bits for first bytes of ru letters, and a single +// bit for everything else. +// +// The highest bit (bit 31) is reserved as the nil marker. +type LettersBitset uint32 + +const nilSet LettersBitset = 1 << 31 + +const ( + reservedByteUtf8Ru1 = 208 + reservedByteUtf8Ru2 = 209 +) + +type LetterBitsetBuilder [30]bool + +func (b *LetterBitsetBuilder) Add(token []byte) { + for _, c := range token { + switch { + case c >= 'a' && c <= 'z': + b[c-'a'] = true + case c >= 'A' && c <= 'Z': + b[c-'A'] = true + case c >= '0' && c <= '9': + b[26] = true + case c == reservedByteUtf8Ru1: + b[27] = true + case c == reservedByteUtf8Ru2: + b[28] = true + default: + b[29] = true + } + } +} + +func (b *LetterBitsetBuilder) Build() LettersBitset { + return NewLettersBitsetFromArray(*b) +} + +func NewLettersBitsetNil() LettersBitset { + return nilSet +} + +func NewLettersBitsetFromArray(letters [30]bool) LettersBitset { + var s LettersBitset + for i, has := range letters { + if has { + s |= 1 << i + } + } + return s +} + +func NewLettersBitset(data []byte) LettersBitset { + var builder LetterBitsetBuilder + builder.Add(data) + return builder.Build() +} + +func (s LettersBitset) IsNil() bool { + return s&nilSet != 0 +} + +func (s LettersBitset) ContainsAll(required LettersBitset) bool { + return s&required == required +} diff --git a/util/letters_bitset_test.go b/util/letters_bitset_test.go new file mode 100644 index 00000000..8bedcb4d --- /dev/null +++ b/util/letters_bitset_test.go @@ -0,0 +1,124 @@ +package util + +import ( + "testing" +) + +func TestLettersBitset_ContainsAll(t *testing.T) { + tests := []struct { + name string + set string + required string + expected bool + }{ + { + name: "empty set contains empty", + set: "", + required: "", + expected: true, + }, + { + name: "set contains all its characters", + set: "abcd", + required: "ab", + expected: true, + }, + { + name: "set does not contain missing character", + set: "abcd", + required: "xyz", + expected: false, + }, + { + name: "set contains subset", + set: "abcdef", + required: "bce", + expected: true, + }, + { + name: "set does not contain partial subset", + set: "abc", + required: "abcd", + expected: false, + }, + { + name: "case insensitive", + set: "ABC", + required: "abc", + expected: true, + }, + { + name: "case insensitive reverse", + set: "abc", + required: "ABC", + expected: true, + }, + { + name: "mixed content - contains digits", + set: "abc123", + required: "1", + expected: true, + }, + { + name: "mixed content - does not contain digit", + set: "abc", + required: "1", + expected: false, + }, + { + name: "mixed content - contains special", + set: "abc!@", + required: "!", + expected: true, + }, + { + name: "mixed content - does not contain special", + set: "abc", + required: "!", + expected: false, + }, + { + name: "russian characters", + set: "а", + required: string([]byte{208}), + expected: true, + }, + { + name: "russian not present", + set: "abc", + required: string([]byte{208}), + expected: false, + }, + { + name: "exact match", + set: "abc", + required: "abc", + expected: true, + }, + { + name: "single character", + set: "abc", + required: "b", + expected: true, + }, + { + name: "single character missing", + set: "abc", + required: "x", + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + set := NewLettersBitset([]byte(tt.set)) + required := NewLettersBitset([]byte(tt.required)) + result := set.ContainsAll(required) + + if result != tt.expected { + t.Errorf("set.ContainsAll(required) = %v, want %v (set=%q, required=%q)", + result, tt.expected, tt.set, tt.required) + } + }) + } +}