diff --git a/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/csharp.cs b/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/csharp.cs index 3afef59948..32fb26ba2c 100644 --- a/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/csharp.cs +++ b/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/csharp.cs @@ -15,7 +15,8 @@ await client.QueryAsync( Model = "qdrant/bm25", Options = { - ["language"] = "none", + ["stemmer"] = new Dictionary { ["type"] = "none" }, + ["stopwords"] = new Dictionary(), ["tokenizer"] = "multilingual", ["ascii_folding"] = true, }, diff --git a/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/generated/csharp.md b/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/generated/csharp.md index 234e1b7376..7b9023cc43 100644 --- a/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/generated/csharp.md +++ b/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/generated/csharp.md @@ -10,7 +10,8 @@ await client.QueryAsync( Model = "qdrant/bm25", Options = { - ["language"] = "none", + ["stemmer"] = new Dictionary { ["type"] = "none" }, + ["stopwords"] = new Dictionary(), ["tokenizer"] = "multilingual", ["ascii_folding"] = true, }, diff --git a/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/generated/go.md b/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/generated/go.md index ff2b35bca8..fc6d232014 100644 --- a/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/generated/go.md +++ b/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/generated/go.md @@ -5,7 +5,12 @@ client.Query(context.Background(), &qdrant.QueryPoints{ qdrant.NewVectorInputDocument(&qdrant.Document{ Model: "qdrant/bm25", Text: "Mieville", - Options: qdrant.NewValueMap(map[string]any{"language": "none", "tokenizer": "multilingual", "ascii_folding": true}), + Options: qdrant.NewValueMap(map[string]any{ + "stemmer": map[string]any{"type": "none"}, + "stopwords": map[string]any{}, + "tokenizer": "multilingual", + "ascii_folding": true, + }), }), ), Using: qdrant.PtrOf("author-bm25"), diff --git a/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/generated/java.md b/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/generated/java.md index 5a2da34ba9..bf623dc9ab 100644 --- a/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/generated/java.md +++ b/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/generated/java.md @@ -1,10 +1,12 @@ ```java import static io.qdrant.client.QueryFactory.nearest; +import static io.qdrant.client.ValueFactory.value; import static io.qdrant.client.WithPayloadSelectorFactory.enable; import io.qdrant.client.QdrantClient; import io.qdrant.client.QdrantGrpcClient; import io.qdrant.client.grpc.Points.*; +import java.util.*; QdrantClient client = @@ -14,7 +16,14 @@ client .setCollectionName("books") .setQuery( nearest( - Document.newBuilder().setText("Mieville").setModel("qdrant/bm25").build())) + Document.newBuilder() + .setText("Mieville") + .setModel("qdrant/bm25") + .putOptions("stemmer", value(Map.of("type", value("none")))) + .putOptions("stopwords", value(Map.of())) + .putOptions("tokenizer", value("multilingual")) + .putOptions("ascii_folding", value(true)) + .build())) .setUsing("author-bm25") .setLimit(10) .setWithPayload(enable(true)) diff --git a/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/generated/python.md b/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/generated/python.md index 985912ac19..6993c402e8 100644 --- a/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/generated/python.md +++ b/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/generated/python.md @@ -7,13 +7,17 @@ client = QdrantClient( cloud_inference=True, ) -# Note: these BM25 options are not supported by FastEmbed client.query_points( collection_name="books", query=models.Document( text="Mieville", model="qdrant/bm25", - options={"language": "none", "tokenizer": "multilingual", "ascii_folding": True}, + options=models.Bm25Config( + stemmer=models.DisabledStemmerParams(type=models.NoStemmer.NONE), + stopwords=models.StopwordsSet(), + tokenizer=models.TokenizerType.MULTILINGUAL, + ascii_folding=True, + ), ), using="author-bm25", limit=10, diff --git a/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/generated/rust.md b/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/generated/rust.md index 3e75d0163d..42eb504641 100644 --- a/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/generated/rust.md +++ b/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/generated/rust.md @@ -2,10 +2,18 @@ use std::collections::HashMap; use qdrant_client::Qdrant; -use qdrant_client::qdrant::{DocumentBuilder, Query, QueryPointsBuilder, Value}; +use qdrant_client::qdrant::{value::Kind, DocumentBuilder, Query, QueryPointsBuilder, Struct, Value}; let mut options = HashMap::new(); -options.insert("language".to_string(), Value::from("none")); +options.insert("stemmer".to_string(), Value::from(vec![("type", "none")])); +options.insert( + "stopwords".to_string(), + Value { + kind: Some(Kind::StructValue(Struct { + fields: HashMap::new(), + })), + }, +); options.insert("tokenizer".to_string(), Value::from("multilingual")); options.insert("ascii_folding".to_string(), Value::from(true)); diff --git a/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/generated/typescript.md b/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/generated/typescript.md index 9f32ae87ee..2bce73e81d 100644 --- a/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/generated/typescript.md +++ b/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/generated/typescript.md @@ -3,7 +3,7 @@ client.query("books", { query: { text: "Mieville", model: "qdrant/bm25", - options: { language: "none", tokenizer: "multilingual", ascii_folding: true }, + options: { stemmer: { type: "none" }, stopwords: {}, tokenizer: "multilingual", ascii_folding: true }, }, using: "author-bm25", limit: 10, diff --git a/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/go.go b/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/go.go index dd01b99ea8..4890c1119a 100644 --- a/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/go.go +++ b/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/go.go @@ -26,7 +26,12 @@ func Main() { qdrant.NewVectorInputDocument(&qdrant.Document{ Model: "qdrant/bm25", Text: "Mieville", - Options: qdrant.NewValueMap(map[string]any{"language": "none", "tokenizer": "multilingual", "ascii_folding": true}), + Options: qdrant.NewValueMap(map[string]any{ + "stemmer": map[string]any{"type": "none"}, + "stopwords": map[string]any{}, + "tokenizer": "multilingual", + "ascii_folding": true, + }), }), ), Using: qdrant.PtrOf("author-bm25"), diff --git a/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/http.md b/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/http.md index afdd467490..955d1be3c9 100644 --- a/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/http.md +++ b/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/http.md @@ -5,7 +5,8 @@ POST /collections/books/points/query "text": "Mieville", "model": "qdrant/bm25", "options": { - "language": "none", + "stemmer": {"type": "none"}, + "stopwords": {}, "tokenizer": "multilingual", "ascii_folding": true } diff --git a/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/java.java b/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/java.java index cbf6bf98de..52eb24d265 100644 --- a/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/java.java +++ b/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/java.java @@ -1,11 +1,13 @@ package com.example.snippets_amalgamation; import static io.qdrant.client.QueryFactory.nearest; +import static io.qdrant.client.ValueFactory.value; import static io.qdrant.client.WithPayloadSelectorFactory.enable; import io.qdrant.client.QdrantClient; import io.qdrant.client.QdrantGrpcClient; import io.qdrant.client.grpc.Points.*; +import java.util.*; public class Snippet { public static void run() throws Exception { @@ -18,7 +20,14 @@ public static void run() throws Exception { .setCollectionName("books") .setQuery( nearest( - Document.newBuilder().setText("Mieville").setModel("qdrant/bm25").build())) + Document.newBuilder() + .setText("Mieville") + .setModel("qdrant/bm25") + .putOptions("stemmer", value(Map.of("type", value("none")))) + .putOptions("stopwords", value(Map.of())) + .putOptions("tokenizer", value("multilingual")) + .putOptions("ascii_folding", value(true)) + .build())) .setUsing("author-bm25") .setLimit(10) .setWithPayload(enable(true)) diff --git a/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/python.py b/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/python.py index 4031f1479e..7f3036521c 100644 --- a/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/python.py +++ b/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/python.py @@ -6,13 +6,17 @@ cloud_inference=True, ) -# Note: these BM25 options are not supported by FastEmbed client.query_points( collection_name="books", query=models.Document( text="Mieville", model="qdrant/bm25", - options={"language": "none", "tokenizer": "multilingual", "ascii_folding": True}, + options=models.Bm25Config( + stemmer=models.DisabledStemmerParams(type=models.NoStemmer.NONE), + stopwords=models.StopwordsSet(), + tokenizer=models.TokenizerType.MULTILINGUAL, + ascii_folding=True, + ), ), using="author-bm25", limit=10, diff --git a/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/rust.rs b/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/rust.rs index 7bc9273288..8449b0ef18 100644 --- a/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/rust.rs +++ b/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/rust.rs @@ -1,13 +1,21 @@ use std::collections::HashMap; use qdrant_client::Qdrant; -use qdrant_client::qdrant::{DocumentBuilder, Query, QueryPointsBuilder, Value}; +use qdrant_client::qdrant::{value::Kind, DocumentBuilder, Query, QueryPointsBuilder, Struct, Value}; pub async fn main() -> anyhow::Result<()> { let client = Qdrant::from_url("http://localhost:6334").build()?; // @hide let mut options = HashMap::new(); - options.insert("language".to_string(), Value::from("none")); + options.insert("stemmer".to_string(), Value::from(vec![("type", "none")])); + options.insert( + "stopwords".to_string(), + Value { + kind: Some(Kind::StructValue(Struct { + fields: HashMap::new(), + })), + }, + ); options.insert("tokenizer".to_string(), Value::from("multilingual")); options.insert("ascii_folding".to_string(), Value::from(true)); diff --git a/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/typescript.ts b/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/typescript.ts index 254b803d09..ff0e9daeca 100644 --- a/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/typescript.ts +++ b/qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/typescript.ts @@ -6,7 +6,7 @@ client.query("books", { query: { text: "Mieville", model: "qdrant/bm25", - options: { language: "none", tokenizer: "multilingual", ascii_folding: true }, + options: { stemmer: { type: "none" }, stopwords: {}, tokenizer: "multilingual", ascii_folding: true }, }, using: "author-bm25", limit: 10, diff --git a/qdrant-landing/content/documentation/inference/_index.md b/qdrant-landing/content/documentation/inference/_index.md index 02820bb262..84167de36c 100644 --- a/qdrant-landing/content/documentation/inference/_index.md +++ b/qdrant-landing/content/documentation/inference/_index.md @@ -34,4 +34,4 @@ The right option depends on your deployment and what you need to embed. Use this | Already manage your own inference service | Client-side inference | | Self-host Qdrant | Client-side inference, for example using [FastEmbed](/documentation/fastembed/) | | Use Qdrant Cloud and want to use one of the supported embedding models | [Qdrant Cloud Inference](/documentation/inference/cloud-inference/) | -| Use Qdrant Cloud and want to use a model from OpenAI, Cohere, Jina AI, or OpenRouter | [Qdrant Cloud Inference with an external provider](/documentation/inference/external-inference-providers/) (requires API key) | \ No newline at end of file +| Use Qdrant Cloud and want to use a model from OpenAI, Cohere, Jina AI, or OpenRouter | [Qdrant Cloud Inference with an external provider](/documentation/inference/external-inference-providers/) (requires API key) | diff --git a/qdrant-landing/content/documentation/manage-data/indexing.md b/qdrant-landing/content/documentation/manage-data/indexing.md index 69971cc80a..410f2065ae 100644 --- a/qdrant-landing/content/documentation/manage-data/indexing.md +++ b/qdrant-landing/content/documentation/manage-data/indexing.md @@ -199,7 +199,7 @@ Available tokenizers are: * `word` (default) - splits the string into words, separated by spaces, punctuation marks, and special characters. * `whitespace` - splits the string into words, separated by spaces. * `prefix` - splits the string into words, separated by spaces, punctuation marks, and special characters, and then creates a prefix index for each word. For example: `hello` will be indexed as `h`, `he`, `hel`, `hell`, `hello`. -* `multilingual` - a special type of tokenizer based on multiple packages like [charabia](https://github.com/meilisearch/charabia) and [vaporetto](https://github.com/daac-tools/vaporetto) to deliver fast and accurate tokenization for a large variety of languages. It allows proper tokenization and lemmatization for multiple languages, including those with non-Latin alphabets and non-space delimiters. See the [charabia documentation](https://github.com/meilisearch/charabia) for a full list of supported languages and normalization options. Note: For the Japanese language, Qdrant relies on the `vaporetto` project, which has much less overhead compared to `charabia`, while maintaining comparable performance. +* `multilingual` - a special type of tokenizer based on multiple packages like [charabia](https://github.com/meilisearch/charabia) and [vaporetto](https://github.com/daac-tools/vaporetto) to deliver fast and accurate tokenization for a large variety of languages. It allows proper tokenization for multiple languages, including those with non-Latin alphabets and non-space delimiters. See the [charabia documentation](https://github.com/meilisearch/charabia) for a full list of supported languages and normalization options. Note: For the Japanese language, Qdrant relies on the `vaporetto` project, which has much less overhead compared to `charabia`, while maintaining comparable performance. ### Lowercasing diff --git a/qdrant-landing/content/documentation/search/text-search/full-text-search.md b/qdrant-landing/content/documentation/search/text-search/full-text-search.md index e3ef80cc96..9f44d38050 100644 --- a/qdrant-landing/content/documentation/search/text-search/full-text-search.md +++ b/qdrant-landing/content/documentation/search/text-search/full-text-search.md @@ -62,9 +62,9 @@ If you set any of the options discussed in this section, ensure that you apply t To configure stemming and stopword removal, use the following options: -- `language`: sets the language for stemming and stopword removal. Defaults to `english`. To disable stemming and stopword removal, set `language` to `none`. -- `stemmer`: defaults to stemming for `language` (if set), but can be configured independently. -- `stopwords`: defaults to a set of stopwords for `language` (if set) but can be configured independently. You can configure a specific `language` and/or configure an explicit set of stopwords that will be merged with the stopword set of the configured language. +- `language`: sets the language for stemming and stopword removal. Defaults to `english`. +- `stemmer`: defaults to stemming for `language` (if set), but can be configured independently. Set to `{"type": "none"}` to explicitly disable stemming. +- `stopwords`: defaults to a set of stopwords for `language` (if set) but can be configured independently. You can configure a specific `language` and/or configure an explicit set of stopwords that will be merged with the stopword set of the configured language. Configure an empty stopwords set to disable stopword removal. For example, to use Spanish stemming and stopwords during data ingestion, use: @@ -74,7 +74,7 @@ At query time, use the exact same parameters to ensure consistent text processin {{< code-snippet path="/documentation/headless/snippets/text-search/query-bm25-spanish/" >}} -To configure only a stemmer or a stopword set, rather than both, set `language` to `none` and specify the configuration for the desired stemmer or stopwords. +To use only stemming, set an empty stopword list. To use only stopword removal, disable stemming by setting `stemmer: {"type": "none"}`. #### ASCII Folding @@ -92,13 +92,20 @@ The tokenizer breaks down text into individual tokens (words). By default, the B #### Language-neutral Text Processing +*Available as of v1.19.0* + In some situations, you may want to disable language-specific processing altogether. For example, when searching for author names, that don't necessarily conform to the rules of a specific language. To disable language-specific processing, set the following options: -- `language`: set to `none` to disable language-specific stemming and stopword removal. -- `tokenizer`: set to `multilingual` for multilingual tokenization and lemmatization. +- `stemmer`: set to `{"type": "none"}` to explicitly disable stemming. +- `stopwords`: configure an empty stopwords set to disable stopword removal. +- `tokenizer`: set to `multilingual` for language-aware tokenization that automatically detects the script and applies the appropriate segmentation for each language. - Optionally, set `ascii_folding` to `true` to enable ASCII folding and ignore diacritics. + + {{< code-snippet path="/documentation/headless/snippets/text-search/query-bm25-language-neutral/" >}} ## SPLADE++ diff --git a/qdrant-landing/content/documentation/search/text-search/text-filtering.md b/qdrant-landing/content/documentation/search/text-search/text-filtering.md index 7e5d652b60..aea9bd02b1 100644 --- a/qdrant-landing/content/documentation/search/text-search/text-filtering.md +++ b/qdrant-landing/content/documentation/search/text-search/text-filtering.md @@ -73,8 +73,8 @@ The following text processing steps are applied to text strings: - The string is broken down into individual tokens (words) using a process called [tokenization](/documentation/manage-data/indexing/#tokenizers). By default, Qdrant uses the `word` tokenizer, which splits the string using word boundaries, discarding spaces, punctuation marks, and special characters. - By default, each word is then [converted to lowercase](/documentation/manage-data/indexing/#lowercasing). Lowercasing the tokens allows Qdrant to ignore capitalization, making full-text filters case-insensitive. - Optionally, Qdrant can remove diacritics (accents) from characters using a process called [ASCII folding](/documentation/manage-data/indexing/#ascii-folding). This ensures that diacritics are ignored. As a result, filtering for the word "cafe" matches "café". -- Optionally, tokens can be reduced to their root form using a [stemmer](/documentation/manage-data/indexing/#stemmer). This ensures that filtering for "running" also matches "run" and "ran". Because stemming is language-specific, if enabled, it must be configured for a specific language. -- Certain words like "the", "is", and "and" are very common in text and do not contribute much to the meaning of text. These words are called [stopwords](/documentation/manage-data/indexing/#stopwords) and can optionally be removed during indexing. Like stemming, stopword removal is language-specific. You can configure specific languages for stopword removal and/or provide a custom list of stopwords to remove. +- Optionally, tokens can be reduced to their root form using a [stemmer](/documentation/manage-data/indexing/#stemmer). This ensures that filtering for "running" also matches "run" and "ran". Stemming is disabled by default. Because it's language-specific, it must be configured for a specific language when enabled. +- Certain words like "the", "is", and "and" are very common in text and don't contribute much to the meaning of text. These words are called [stopwords](/documentation/manage-data/indexing/#stopwords) and can optionally be removed during indexing. Stopword removal is disabled by default. Like stemming, it's language-specific: you can configure specific languages for stopword removal and/or provide a custom list of stopwords to remove. - Optionally, you can enable [phrase matching](/documentation/manage-data/indexing/#phrase-search) to allow filtering for multiple words in the exact same order as they appear in the original text. These text processing steps can be configured when creating a [full-text index](/documentation/manage-data/indexing/#full-text-index). For example, to create a text index on the `title` field with ASCII folding enabled: