perf: implement letter based skip index for token blocks

cmd/index_analyzer/main.go

@@ -168,7 +168,7 @@ func analyzeIndex(
 			break
 		}
 		block := token.TableBlock{}
-		block.Unpack(data)
+		block.Unpack(data, ver)
 		tokenTableBlocks = append(tokenTableBlocks, block)
 	}
 	tokenTable := token.TableFromBlocks(tokenTableBlocks)

config/frac_version.go

@@ -21,6 +21,9 @@ const (
 
 	// BinaryDataV4 - delta bitpack encoded MIDs and LIDs
 	BinaryDataV4
+
+	// BinaryDataV5 - token blocks have zone maps (eng letters presense, max token length)
+	BinaryDataV5
 )
 
-const CurrentFracVersion = BinaryDataV4
+const CurrentFracVersion = BinaryDataV5

frac/fraction_concurrency_test.go

@@ -287,6 +287,17 @@
 	for i := 0; i < numMessages; i++ {
 		service := services[rand.IntN(len(services))]
 		message := messages[rand.IntN(len(messages))]
+		// populate message with various unique tokens like ids and hex numbers (matches real installation)
+		x := rand.IntN(20)
+		switch x {
+		case 1:
+			message = message + fmt.Sprintf(" %dms", rand.IntN(10000000))
+		case 2:
+			message = message + fmt.Sprintf(" %dus", rand.IntN(10000000))
+		default:
+			message = message + fmt.Sprintf(" %d", rand.IntN(10000000))
+		}
+
 		level := rand.IntN(6)
 		timestamp := fromTime.Add(time.Duration(i) * time.Millisecond)
 		id := fmt.Sprintf("id-%d", i)

frac/fraction_test.go

@@ -1454,6 +1454,15 @@ func (s *FractionTestSuite) TestSearchLargeFrac() {
 			fromTime: fromTime,
 			toTime:   toTime,
 		},
+		{
+			name:  "message:req*t OR message:f*ed",
+			query: "message:req*t OR message:f*ed",
+			filter: func(doc *testDoc) bool {
+				return strings.Contains(doc.message, "request") || strings.Contains(doc.message, "failed")
+			},
+			fromTime: fromTime,
+			toTime:   toTime,
+		},
 		{
 			name:  "message:*uest OR id:*1",
 			query: "message:*uest OR id:*1",
@@ -1911,11 +1920,11 @@ func (s *FractionTestSuite) TestFractionInfo() {
 		s.Require().Equal(uint64(0), info.IndexOnDisk, "index on disk doesn't match")
 	case *frac.Sealed:
 		s.Require().Equal(uint64(0), info.MetaOnDisk, "meta on disk doesn't match. actual value")
-		s.Require().True(info.IndexOnDisk > uint64(1300) && info.IndexOnDisk < uint64(1450),
+		s.Require().True(info.IndexOnDisk > uint64(1300) && info.IndexOnDisk < uint64(1500),
 			"index on disk doesn't match. actual value: %d", info.IndexOnDisk)
 	case *frac.Remote:
 		s.Require().Equal(uint64(0), info.MetaOnDisk, "meta on disk doesn't match. actual value")
-		s.Require().True(info.IndexOnDisk > uint64(1300) && info.IndexOnDisk < uint64(1450),
+		s.Require().True(info.IndexOnDisk > uint64(1300) && info.IndexOnDisk < uint64(1500),
 			"index on disk doesn't match. actual value: %d", info.IndexOnDisk)
 	default:
 		s.Require().Fail("unsupported fraction type")

frac/remote.go

@@ -192,7 +192,7 @@ func (f *Remote) createDataProvider(ctx context.Context) (*sealedDataProvider, e
 		lidsTable:        f.blocksData.LIDsTable,
 		lidsLoader:       lids.NewLoader(f.info.BinaryDataVer, lidReader, f.indexCache.LIDs),
 		tokenBlockLoader: token.NewBlockLoader(f.BaseFileName, tokenReader, f.indexCache.Tokens),
-		tokenTableLoader: token.NewTableLoader(f.BaseFileName, f.IsLegacy, tokenReader, f.indexCache.TokenTable),
+		tokenTableLoader: token.NewTableLoader(f.BaseFileName, f.Info().BinaryDataVer, f.IsLegacy, tokenReader, f.indexCache.TokenTable),
 
 		idsTable: &f.blocksData.IDsTable,
 		idsProvider: seqids.NewProvider(

frac/sealed.go

@@ -515,7 +515,7 @@ func (f *Sealed) createDataProvider(ctx context.Context) *sealedDataProvider {
 		lidsTable:        f.blocksData.LIDsTable,
 		lidsLoader:       lids.NewLoader(f.info.BinaryDataVer, lidReader, f.indexCache.LIDs),
 		tokenBlockLoader: token.NewBlockLoader(f.BaseFileName, tokenReader, f.indexCache.Tokens),
-		tokenTableLoader: token.NewTableLoader(f.BaseFileName, f.IsLegacy, tokenReader, f.indexCache.TokenTable),
+		tokenTableLoader: token.NewTableLoader(f.BaseFileName, f.Info().BinaryDataVer, f.IsLegacy, tokenReader, f.indexCache.TokenTable),
 
 		idsTable: &f.blocksData.IDsTable,
 		idsProvider: seqids.NewProvider(

frac/sealed/token/block_loader.go

@@ -13,6 +13,7 @@ import (
 	"github.com/ozontech/seq-db/logger"
 	"github.com/ozontech/seq-db/pattern"
 	"github.com/ozontech/seq-db/storage"
+	"github.com/ozontech/seq-db/util"
 )
 
 const sizeOfUint32 = uint32(unsafe.Sizeof(uint32(0)))
@@ -62,6 +63,14 @@ func (b *Block) GetToken(index int) []byte {
 	return b.Payload[offset : offset+l]
 }
 
+func (b *Block) LettersBitset() util.LettersBitset {
+	var builder util.LetterBitsetBuilder
+	for i := 0; i < b.Len(); i++ {
+		builder.Add(b.GetToken(i))
+	}
+	return builder.Build()
+}
+
 func (b *Block) contains(from, to int, needle []byte) ([]int, error) {
 	indexes := make([]int, 0)
 	for i := from; i <= to; i++ {

frac/sealed/token/provider.go

@@ -5,6 +5,7 @@ import (
 	"sort"
 
 	"github.com/ozontech/seq-db/pattern"
+	"github.com/ozontech/seq-db/util"
 )
 
 type Provider struct {
@@ -60,24 +61,45 @@ func (tp *Provider) GetToken(tid uint32) []byte {
 }
 
 func (tp *Provider) FindContains(needle []byte) ([]uint32, error) {
-	return tp.findInBlocks(tp.FirstTID(), tp.LastTID(), func(b *Block, firstIndex, lastIndex int) ([]int, error) {
-		return b.contains(firstIndex, lastIndex, needle)
-	})
+	requiredLetters := util.NewLettersBitset(needle)
+
+	return tp.findInBlocks(
+		tp.FirstTID(),
+		tp.LastTID(),
+		func(e *TableEntry) bool {
+			return e.Letters.IsNil() || e.Letters.ContainsAll(requiredLetters)
+		},
+		func(b *Block, firstIndex, lastIndex int) ([]int, error) {
+			return b.contains(firstIndex, lastIndex, needle)
+		})
 }
 
 func (tp *Provider) FindToken(searcher pattern.Searcher) ([]uint32, error) {
-	return tp.findInBlocks(searcher.FirstTID(), searcher.LastTID(), func(b *Block, firstIndex, lastIndex int) ([]int, error) {
-		return b.find(firstIndex, lastIndex, searcher)
-	})
+	return tp.findInBlocks(
+		searcher.FirstTID(),
+		searcher.LastTID(),
+		func(e *TableEntry) bool {
+			return searcher.CheckEntry(e.Letters)
+		},
+		func(b *Block, firstIndex, lastIndex int) ([]int, error) {
+			return b.find(firstIndex, lastIndex, searcher)
+		})
 }
 
-func (tp *Provider) findInBlocks(firstTID, lastTID uint32, search func(*Block, int, int) ([]int, error)) ([]uint32, error) {
+func (tp *Provider) findInBlocks(
+	firstTID,
+	lastTID uint32,
+	entryFilter func(*TableEntry) bool,
+	search func(*Block, int, int) ([]int, error)) ([]uint32, error) {
 	var tids []uint32
 
 	for _, entry := range tp.entries {
 		if !entry.checkTIDsInBlock(firstTID, lastTID) {
 			continue
 		}
+		if !entryFilter(entry) {
+			continue
+		}
 
 		block := tp.findBlock(entry.BlockIndex)
 		firstIndex, lastIndex := entry.narrowIndexes(firstTID, lastTID)

frac/sealed/token/table_entry.go

@@ -1,5 +1,7 @@
 package token
 
+import "github.com/ozontech/seq-db/util"
+
 // TableEntry is part of token.Table and points to a fragment of token.Block
 type TableEntry struct {
 	StartIndex uint32 // offset from the beginning of the block to the first token pointed to by the TableEntry
@@ -9,6 +11,8 @@ type TableEntry struct {
 
 	MinVal string // only saved for the first entry in block
 	MaxVal string
+
+	Letters util.LettersBitset // case-insensitive English letters present in entry tokens
 }
 
 func (t *TableEntry) GetIndexInTokensBlock(tid uint32) int {

frac/sealed/token/table_loader.go

@@ -9,15 +9,18 @@ import (
 	"go.uber.org/zap"
 
 	"github.com/ozontech/seq-db/cache"
+	"github.com/ozontech/seq-db/config"
 	"github.com/ozontech/seq-db/logger"
 	"github.com/ozontech/seq-db/packer"
 	"github.com/ozontech/seq-db/storage"
+	"github.com/ozontech/seq-db/util"
 )
 
 const CacheKeyTable = 1
 
 type TableLoader struct {
 	fracName string
+	fracVer  config.BinaryDataVersion
 	isLegacy bool
 
 	reader *storage.IndexReader
@@ -31,12 +34,14 @@ type TableLoader struct {
 
 func NewTableLoader(
 	fracName string,
+	fracVer config.BinaryDataVersion,
 	isLegacy bool,
 	reader *storage.IndexReader,
 	c *cache.Cache[Table],
 ) *TableLoader {
 	return &TableLoader{
 		fracName: fracName,
+		fracVer:  fracVer,
 		isLegacy: isLegacy,
 		reader:   reader,
 		cache:    c,
@@ -126,7 +131,7 @@ func (l *TableLoader) loadBlocksLegacy() ([]TableBlock, error) {
 		}
 
 		var tb TableBlock
-		tb.Unpack(blockData)
+		tb.Unpack(blockData, l.fracVer)
 
 		blocks = append(blocks, tb)
 		blockIndex += 1
@@ -149,7 +154,7 @@ func (l *TableLoader) loadBlocks() ([]TableBlock, error) {
 		}
 
 		var tb TableBlock
-		tb.Unpack(data)
+		tb.Unpack(data, l.fracVer)
 
 		blocks = append(blocks, tb)
 	}
@@ -206,6 +211,8 @@ func (b TableBlock) packedSize() int {
 			// MaxVal
 			size += sizeOfUint32
 			size += len(entry.MaxVal)
+			// Letters
+			size += sizeOfUint32
 		}
 	}
 	return size
@@ -231,12 +238,14 @@ func (b TableBlock) Pack(buf []byte) []byte {
 			// MaxVal
 			buf = binary.LittleEndian.AppendUint32(buf, uint32(len(entry.MaxVal)))
 			buf = append(buf, entry.MaxVal...)
+			// Letters
+			buf = binary.LittleEndian.AppendUint32(buf, uint32(entry.Letters))
 		}
 	}
 	return buf
 }
 
-func (b *TableBlock) Unpack(data []byte) {
+func (b *TableBlock) Unpack(data []byte, fracVer config.BinaryDataVersion) {
 	b.FieldsTables = make([]FieldTable, 0)
 	unpacker := packer.NewBytesUnpacker(data)
 
@@ -260,6 +269,11 @@ func (b *TableBlock) Unpack(data []byte) {
 				e.MinVal = minVal
 			}
 			e.MaxVal = maxVal
+			if fracVer >= config.BinaryDataV5 {
+				e.Letters = util.LettersBitset(unpacker.GetUint32())
+			} else {
+				e.Letters = util.NewLettersBitsetNil()
+			}
 			ft.Entries[i] = e
 		}
 		b.FieldsTables = append(b.FieldsTables, ft)

frac/sealed_source.go

@@ -42,7 +42,7 @@ func NewSealedSource(f *Sealed) *SealedSource {
 		),
 		lidsLoader:       lids.NewLoader(f.Info().BinaryDataVer, &f.lidReader, f.indexCache.LIDs),
 		tokenBlockLoader: token.NewBlockLoader(f.BaseFileName, &f.tokenReader, f.indexCache.Tokens),
-		tokenTableLoader: token.NewTableLoader(f.BaseFileName, f.IsLegacy, &f.tokenReader, f.indexCache.TokenTable),
+		tokenTableLoader: token.NewTableLoader(f.BaseFileName, f.Info().BinaryDataVer, f.IsLegacy, &f.tokenReader, f.indexCache.TokenTable),
 	}
 }
 

indexwriter/blocks.go

@@ -159,6 +159,7 @@ func newTokenTableEntry(
 		ValCount:   lastIndex - firstIndex + 1, // Number of tokens in this entry
 		MinVal:     minVal,                     // Smallest token value in range
 		MaxVal:     maxVal,                     // Largest token value in range
+		Letters:    block.payload.LettersBitset(),
 	}
 }
 

indexwriter/blocks_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/ozontech/seq-db/frac/sealed/lids"
 	"github.com/ozontech/seq-db/frac/sealed/token"
 	"github.com/ozontech/seq-db/seq"
+	"github.com/ozontech/seq-db/util"
 )
 
 type mockSource struct {
@@ -59,6 +60,8 @@ func (m *mockSource) ID() iter.Seq2[DocLocation, error] {
 }
 
 func TestBlocksBuilder_BuildTokenBlocks(t *testing.T) {
+	lettersFV := util.NewLettersBitsetFromArray([30]bool{5: true, 21: true, 26: true})
+
 	src := mockSource{
 		tokens: [][]byte{
 			[]byte("f1v1"), // 1
@@ -156,6 +159,7 @@ func TestBlocksBuilder_BuildTokenBlocks(t *testing.T) {
 						ValCount:   2,
 						MinVal:     "f1v1",
 						MaxVal:     "f1v2",
+						Letters:    lettersFV,
 					},
 				},
 			}, {
@@ -168,20 +172,23 @@ func TestBlocksBuilder_BuildTokenBlocks(t *testing.T) {
 						ValCount:   1,
 						MinVal:     "f2v1",
 						MaxVal:     "f2v1",
+						Letters:    lettersFV,
 					}, {
 						StartIndex: 0,
 						StartTID:   4,
 						BlockIndex: 1,
 						ValCount:   3,
 						MinVal:     "f2v2",
 						MaxVal:     "f2v4",
+						Letters:    lettersFV,
 					}, {
 						StartIndex: 0,
 						StartTID:   7,
 						BlockIndex: 2,
 						ValCount:   1,
 						MinVal:     "f2v5",
 						MaxVal:     "f2v5",
+						Letters:    lettersFV,
 					},
 				},
 			}, {
@@ -194,6 +201,7 @@ func TestBlocksBuilder_BuildTokenBlocks(t *testing.T) {
 						ValCount:   2,
 						MinVal:     "f3v1",
 						MaxVal:     "f3v2",
+						Letters:    lettersFV,
 					},
 				},
 			}, {
@@ -206,6 +214,7 @@ func TestBlocksBuilder_BuildTokenBlocks(t *testing.T) {
 						ValCount:   3,
 						MinVal:     "f4v1",
 						MaxVal:     "f4v3",
+						Letters:    lettersFV,
 					},
 				},
 			}, {
@@ -218,6 +227,7 @@ func TestBlocksBuilder_BuildTokenBlocks(t *testing.T) {
 						ValCount:   1,
 						MinVal:     "f5v1",
 						MaxVal:     "f5v1",
+						Letters:    lettersFV,
 					},
 				},
 			}, {
@@ -230,6 +240,7 @@ func TestBlocksBuilder_BuildTokenBlocks(t *testing.T) {
 						ValCount:   1,
 						MinVal:     "f6v1",
 						MaxVal:     "f6v1",
+						Letters:    lettersFV,
 					},
 				},
 			},