[1.19] Document language-neutral BM25 text processing

qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/csharp.cs

@@ -15,7 +15,8 @@ await client.QueryAsync(
                 Model = "qdrant/bm25",
                 Options =
                 {
-                    ["language"] = "none",
+                    ["stemmer"] = new Dictionary<string, Value> { ["type"] = "none" },
+                    ["stopwords"] = new Dictionary<string, Value>(),
                     ["tokenizer"] = "multilingual",
                     ["ascii_folding"] = true,
                 },

qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/generated/csharp.md

@@ -10,7 +10,8 @@ await client.QueryAsync(
         Model = "qdrant/bm25",
         Options =
         {
-            ["language"] = "none",
+            ["stemmer"] = new Dictionary<string, Value> { ["type"] = "none" },
+            ["stopwords"] = new Dictionary<string, Value>(),
             ["tokenizer"] = "multilingual",
             ["ascii_folding"] = true,
         },

qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/generated/go.md

@@ -5,7 +5,12 @@ client.Query(context.Background(), &qdrant.QueryPoints{
 		qdrant.NewVectorInputDocument(&qdrant.Document{
 			Model:   "qdrant/bm25",
 			Text:    "Mieville",
-			Options: qdrant.NewValueMap(map[string]any{"language": "none", "tokenizer": "multilingual", "ascii_folding": true}),
+			Options: qdrant.NewValueMap(map[string]any{
+			"stemmer":       map[string]any{"type": "none"},
+			"stopwords":     map[string]any{},
+			"tokenizer":     "multilingual",
+			"ascii_folding": true,
+		}),
 		}),
 	),
 	Using:       qdrant.PtrOf("author-bm25"),

qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/generated/java.md

@@ -1,10 +1,12 @@
 ```java
 import static io.qdrant.client.QueryFactory.nearest;
+import static io.qdrant.client.ValueFactory.value;
 import static io.qdrant.client.WithPayloadSelectorFactory.enable;
 
 import io.qdrant.client.QdrantClient;
 import io.qdrant.client.QdrantGrpcClient;
 import io.qdrant.client.grpc.Points.*;
+import java.util.*;
 
 QdrantClient client =
 
@@ -14,7 +16,14 @@ client
             .setCollectionName("books")
             .setQuery(
                 nearest(
-                    Document.newBuilder().setText("Mieville").setModel("qdrant/bm25").build()))
+                    Document.newBuilder()
+                        .setText("Mieville")
+                        .setModel("qdrant/bm25")
+                        .putOptions("stemmer", value(Map.of("type", value("none"))))
+                        .putOptions("stopwords", value(Map.of()))
+                        .putOptions("tokenizer", value("multilingual"))
+                        .putOptions("ascii_folding", value(true))
+                        .build()))
             .setUsing("author-bm25")
             .setLimit(10)
             .setWithPayload(enable(true))

qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/generated/python.md

@@ -7,13 +7,17 @@ client = QdrantClient(
     cloud_inference=True,
 )
 
-# Note: these BM25 options are not supported by FastEmbed
 client.query_points(
     collection_name="books",
     query=models.Document(
         text="Mieville",
         model="qdrant/bm25",
-        options={"language": "none", "tokenizer": "multilingual", "ascii_folding": True},
+        options=models.Bm25Config(
+            stemmer=models.DisabledStemmerParams(type=models.NoStemmer.NONE),
+            stopwords=models.StopwordsSet(),
+            tokenizer=models.TokenizerType.MULTILINGUAL,
+            ascii_folding=True,
+        ),
     ),
     using="author-bm25",
     limit=10,

qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/generated/rust.md

@@ -2,10 +2,18 @@
 use std::collections::HashMap;
 
 use qdrant_client::Qdrant;
-use qdrant_client::qdrant::{DocumentBuilder, Query, QueryPointsBuilder, Value};
+use qdrant_client::qdrant::{value::Kind, DocumentBuilder, Query, QueryPointsBuilder, Struct, Value};
 
 let mut options = HashMap::new();
-options.insert("language".to_string(), Value::from("none"));
+options.insert("stemmer".to_string(), Value::from(vec![("type", "none")]));
+options.insert(
+    "stopwords".to_string(),
+    Value {
+        kind: Some(Kind::StructValue(Struct {
+            fields: HashMap::new(),
+        })),
+    },
+);
 options.insert("tokenizer".to_string(), Value::from("multilingual"));
 options.insert("ascii_folding".to_string(), Value::from(true));
 

qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/generated/typescript.md

@@ -3,7 +3,7 @@ client.query("books", {
   query: {
     text: "Mieville",
     model: "qdrant/bm25",
-    options: { language: "none", tokenizer: "multilingual", ascii_folding: true },
+    options: { stemmer: { type: "none" }, stopwords: {}, tokenizer: "multilingual", ascii_folding: true },
   },
   using: "author-bm25",
   limit: 10,

qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/go.go

@@ -26,7 +26,12 @@ func Main() {
 			qdrant.NewVectorInputDocument(&qdrant.Document{
 				Model:   "qdrant/bm25",
 				Text:    "Mieville",
-				Options: qdrant.NewValueMap(map[string]any{"language": "none", "tokenizer": "multilingual", "ascii_folding": true}),
+				Options: qdrant.NewValueMap(map[string]any{
+				"stemmer":       map[string]any{"type": "none"},
+				"stopwords":     map[string]any{},
+				"tokenizer":     "multilingual",
+				"ascii_folding": true,
+			}),
 			}),
 		),
 		Using:       qdrant.PtrOf("author-bm25"),

qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/http.md

@@ -5,7 +5,8 @@ POST /collections/books/points/query
     "text": "Mieville",
     "model": "qdrant/bm25",
     "options": {
-      "language": "none",
+      "stemmer": {"type": "none"},
+      "stopwords": {},
       "tokenizer": "multilingual",
       "ascii_folding": true
     }

qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/java.java

@@ -1,11 +1,13 @@
 package com.example.snippets_amalgamation;
 
 import static io.qdrant.client.QueryFactory.nearest;
+import static io.qdrant.client.ValueFactory.value;
 import static io.qdrant.client.WithPayloadSelectorFactory.enable;
 
 import io.qdrant.client.QdrantClient;
 import io.qdrant.client.QdrantGrpcClient;
 import io.qdrant.client.grpc.Points.*;
+import java.util.*;
 
 public class Snippet {
   public static void run() throws Exception {
@@ -18,7 +20,14 @@ public static void run() throws Exception {
                 .setCollectionName("books")
                 .setQuery(
                     nearest(
-                        Document.newBuilder().setText("Mieville").setModel("qdrant/bm25").build()))
+                        Document.newBuilder()
+                            .setText("Mieville")
+                            .setModel("qdrant/bm25")
+                            .putOptions("stemmer", value(Map.of("type", value("none"))))
+                            .putOptions("stopwords", value(Map.of()))
+                            .putOptions("tokenizer", value("multilingual"))
+                            .putOptions("ascii_folding", value(true))
+                            .build()))
                 .setUsing("author-bm25")
                 .setLimit(10)
                 .setWithPayload(enable(true))

qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/python.py

@@ -6,13 +6,17 @@
     cloud_inference=True,
 )
 
-# Note: these BM25 options are not supported by FastEmbed
 client.query_points(
     collection_name="books",
     query=models.Document(
         text="Mieville",
         model="qdrant/bm25",
-        options={"language": "none", "tokenizer": "multilingual", "ascii_folding": True},
+        options=models.Bm25Config(
+            stemmer=models.DisabledStemmerParams(type=models.NoStemmer.NONE),
+            stopwords=models.StopwordsSet(),
+            tokenizer=models.TokenizerType.MULTILINGUAL,
+            ascii_folding=True,
+        ),
     ),
     using="author-bm25",
     limit=10,

qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/rust.rs

@@ -1,13 +1,21 @@
 use std::collections::HashMap;
 
 use qdrant_client::Qdrant;
-use qdrant_client::qdrant::{DocumentBuilder, Query, QueryPointsBuilder, Value};
+use qdrant_client::qdrant::{value::Kind, DocumentBuilder, Query, QueryPointsBuilder, Struct, Value};
 
 pub async fn main() -> anyhow::Result<()> {
     let client = Qdrant::from_url("http://localhost:6334").build()?; // @hide
 
     let mut options = HashMap::new();
-    options.insert("language".to_string(), Value::from("none"));
+    options.insert("stemmer".to_string(), Value::from(vec![("type", "none")]));
+    options.insert(
+        "stopwords".to_string(),
+        Value {
+            kind: Some(Kind::StructValue(Struct {
+                fields: HashMap::new(),
+            })),
+        },
+    );
     options.insert("tokenizer".to_string(), Value::from("multilingual"));
     options.insert("ascii_folding".to_string(), Value::from(true));
 

qdrant-landing/content/documentation/headless/snippets/text-search/query-bm25-language-neutral/typescript.ts

@@ -6,7 +6,7 @@ client.query("books", {
   query: {
     text: "Mieville",
     model: "qdrant/bm25",
-    options: { language: "none", tokenizer: "multilingual", ascii_folding: true },
+    options: { stemmer: { type: "none" }, stopwords: {}, tokenizer: "multilingual", ascii_folding: true },
   },
   using: "author-bm25",
   limit: 10,

qdrant-landing/content/documentation/inference/_index.md

@@ -34,4 +34,4 @@ The right option depends on your deployment and what you need to embed. Use this
 | Already manage your own inference service | Client-side inference |
 | Self-host Qdrant | Client-side inference, for example using [FastEmbed](/documentation/fastembed/) |
 | Use Qdrant Cloud and want to use one of the supported embedding models | [Qdrant Cloud Inference](/documentation/inference/cloud-inference/) |
-| Use Qdrant Cloud and want to use a model from OpenAI, Cohere, Jina AI, or OpenRouter | [Qdrant Cloud Inference with an external provider](/documentation/inference/external-inference-providers/) (requires API key) |
+| Use Qdrant Cloud and want to use a model from OpenAI, Cohere, Jina AI, or OpenRouter | [Qdrant Cloud Inference with an external provider](/documentation/inference/external-inference-providers/) (requires API key) |

qdrant-landing/content/documentation/manage-data/indexing.md

@@ -199,7 +199,7 @@ Available tokenizers are:
 * `word` (default) - splits the string into words, separated by spaces, punctuation marks, and special characters.
 * `whitespace` - splits the string into words, separated by spaces.
 * `prefix` - splits the string into words, separated by spaces, punctuation marks, and special characters, and then creates a prefix index for each word. For example: `hello` will be indexed as `h`, `he`, `hel`, `hell`, `hello`.
-* `multilingual` - a special type of tokenizer based on multiple packages like [charabia](https://github.com/meilisearch/charabia) and [vaporetto](https://github.com/daac-tools/vaporetto) to deliver fast and accurate tokenization for a large variety of languages. It allows proper tokenization and lemmatization for multiple languages, including those with non-Latin alphabets and non-space delimiters. See the [charabia documentation](https://github.com/meilisearch/charabia) for a full list of supported languages and normalization options. Note: For the Japanese language, Qdrant relies on the `vaporetto` project, which has much less overhead compared to `charabia`, while maintaining comparable performance.
+* `multilingual` - a special type of tokenizer based on multiple packages like [charabia](https://github.com/meilisearch/charabia) and [vaporetto](https://github.com/daac-tools/vaporetto) to deliver fast and accurate tokenization for a large variety of languages. It allows proper tokenization for multiple languages, including those with non-Latin alphabets and non-space delimiters. See the [charabia documentation](https://github.com/meilisearch/charabia) for a full list of supported languages and normalization options. Note: For the Japanese language, Qdrant relies on the `vaporetto` project, which has much less overhead compared to `charabia`, while maintaining comparable performance.
 
 ### Lowercasing
 

qdrant-landing/content/documentation/search/text-search/full-text-search.md

@@ -62,9 +62,9 @@ If you set any of the options discussed in this section, ensure that you apply t
 
 To configure stemming and stopword removal, use the following options:
 
-- `language`: sets the language for stemming and stopword removal. Defaults to `english`. To disable stemming and stopword removal, set `language` to `none`.
-- `stemmer`: defaults to stemming for `language` (if set), but can be configured independently.
-- `stopwords`: defaults to a set of stopwords for `language` (if set) but can be configured independently. You can configure a specific `language` and/or configure an explicit set of stopwords that will be merged with the stopword set of the configured language.
+- `language`: sets the language for stemming and stopword removal. Defaults to `english`.
+- `stemmer`: defaults to stemming for `language` (if set), but can be configured independently. Set to `{"type": "none"}` to explicitly disable stemming.
+- `stopwords`: defaults to a set of stopwords for `language` (if set) but can be configured independently. You can configure a specific `language` and/or configure an explicit set of stopwords that will be merged with the stopword set of the configured language. Configure an empty stopwords set to disable stopword removal.
 
 For example, to use Spanish stemming and stopwords during data ingestion, use:
 
@@ -74,7 +74,7 @@ At query time, use the exact same parameters to ensure consistent text processin
 
 {{< code-snippet path="/documentation/headless/snippets/text-search/query-bm25-spanish/" >}}
 
-To configure only a stemmer or a stopword set, rather than both, set `language` to `none` and specify the configuration for the desired stemmer or stopwords.
+To use only stemming, set an empty stopword list. To use only stopword removal, disable stemming by setting `stemmer: {"type": "none"}`.
 
 #### ASCII Folding
 
@@ -92,13 +92,20 @@ The tokenizer breaks down text into individual tokens (words). By default, the B
 
 #### Language-neutral Text Processing
 
+*Available as of v1.19.0*
+
 In some situations, you may want to disable language-specific processing altogether. For example, when searching for author names, that don't necessarily conform to the rules of a specific language.
 
 To disable language-specific processing, set the following options:
-- `language`: set to `none` to disable language-specific stemming and stopword removal.
-- `tokenizer`: set to `multilingual` for multilingual tokenization and lemmatization.
+- `stemmer`: set to `{"type": "none"}` to explicitly disable stemming.
+- `stopwords`: configure an empty stopwords set to disable stopword removal.
+- `tokenizer`: set to `multilingual` for language-aware tokenization that automatically detects the script and applies the appropriate segmentation for each language.
 - Optionally, set `ascii_folding` to `true` to enable ASCII folding and ignore diacritics.
 
+<aside role="status">
+Setting <code>language</code> to <code>"none"</code> to disable text processing is deprecated as of Qdrant 1.19. Instead, set <code>stemmer</code> to <code>{"type": "none"}</code> and configure an empty stopwords set.
+</aside>
+
 {{< code-snippet path="/documentation/headless/snippets/text-search/query-bm25-language-neutral/" >}}
 
 ## SPLADE++

qdrant-landing/content/documentation/search/text-search/text-filtering.md

@@ -73,8 +73,8 @@ The following text processing steps are applied to text strings:
 - The string is broken down into individual tokens (words) using a process called [tokenization](/documentation/manage-data/indexing/#tokenizers). By default, Qdrant uses the `word` tokenizer, which splits the string using word boundaries, discarding spaces, punctuation marks, and special characters.
 - By default, each word is then [converted to lowercase](/documentation/manage-data/indexing/#lowercasing). Lowercasing the tokens allows Qdrant to ignore capitalization, making full-text filters case-insensitive.
 - Optionally, Qdrant can remove diacritics (accents) from characters using a process called [ASCII folding](/documentation/manage-data/indexing/#ascii-folding). This ensures that diacritics are ignored. As a result, filtering for the word "cafe" matches "café".
-- Optionally, tokens can be reduced to their root form using a [stemmer](/documentation/manage-data/indexing/#stemmer). This ensures that filtering for "running" also matches "run" and "ran". Because stemming is language-specific, if enabled, it must be configured for a specific language.
-- Certain words like "the", "is", and "and" are very common in text and do not contribute much to the meaning of text. These words are called [stopwords](/documentation/manage-data/indexing/#stopwords) and can optionally be removed during indexing. Like stemming, stopword removal is language-specific. You can configure specific languages for stopword removal and/or provide a custom list of stopwords to remove.
+- Optionally, tokens can be reduced to their root form using a [stemmer](/documentation/manage-data/indexing/#stemmer). This ensures that filtering for "running" also matches "run" and "ran". Stemming is disabled by default. Because it's language-specific, it must be configured for a specific language when enabled.
+- Certain words like "the", "is", and "and" are very common in text and don't contribute much to the meaning of text. These words are called [stopwords](/documentation/manage-data/indexing/#stopwords) and can optionally be removed during indexing. Stopword removal is disabled by default. Like stemming, it's language-specific: you can configure specific languages for stopword removal and/or provide a custom list of stopwords to remove.
 - Optionally, you can enable [phrase matching](/documentation/manage-data/indexing/#phrase-search) to allow filtering for multiple words in the exact same order as they appear in the original text.
 
 These text processing steps can be configured when creating a [full-text index](/documentation/manage-data/indexing/#full-text-index). For example, to create a text index on the `title` field with ASCII folding enabled: