diff --git a/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-oracle/src/main/java/org/springframework/ai/vectorstore/oracle/autoconfigure/OracleVectorStoreAutoConfiguration.java b/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-oracle/src/main/java/org/springframework/ai/vectorstore/oracle/autoconfigure/OracleVectorStoreAutoConfiguration.java index fdca1aeaa2..abb14ff6d7 100644 --- a/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-oracle/src/main/java/org/springframework/ai/vectorstore/oracle/autoconfigure/OracleVectorStoreAutoConfiguration.java +++ b/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-oracle/src/main/java/org/springframework/ai/vectorstore/oracle/autoconfigure/OracleVectorStoreAutoConfiguration.java @@ -42,6 +42,7 @@ * @author Eddú Meléndez * @author Christian Tzolov * @author Soby Chacko + * @author Anders Swanson */ @AutoConfiguration @ConditionalOnClass({ OracleVectorStore.class, DataSource.class, JdbcTemplate.class }) @@ -69,6 +70,11 @@ public OracleVectorStore vectorStore(JdbcTemplate jdbcTemplate, EmbeddingModel e .distanceType(properties.getDistanceType()) .dimensions(properties.getDimensions()) .searchAccuracy(properties.getSearchAccuracy()) + .hnswNeighbors(properties.getHnswNeighbors()) + .hnswEfConstruction(properties.getHnswEfConstruction()) + .ivfNeighborPartitions(properties.getIvfNeighborPartitions()) + .ivfSamplePerPartition(properties.getIvfSamplePerPartition()) + .ivfMinVectorsPerPartition(properties.getIvfMinVectorsPerPartition()) .initializeSchema(properties.isInitializeSchema()) .removeExistingVectorStoreTable(properties.isRemoveExistingVectorStoreTable()) .forcedNormalization(properties.isForcedNormalization()) diff --git a/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-oracle/src/main/java/org/springframework/ai/vectorstore/oracle/autoconfigure/OracleVectorStoreProperties.java b/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-oracle/src/main/java/org/springframework/ai/vectorstore/oracle/autoconfigure/OracleVectorStoreProperties.java index dec3a98b05..cf25da5591 100644 --- a/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-oracle/src/main/java/org/springframework/ai/vectorstore/oracle/autoconfigure/OracleVectorStoreProperties.java +++ b/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-oracle/src/main/java/org/springframework/ai/vectorstore/oracle/autoconfigure/OracleVectorStoreProperties.java @@ -24,6 +24,7 @@ * Configuration properties for Oracle Vector Store. * * @author Loïc Lefèvre + * @author Anders Swanson */ @ConfigurationProperties(OracleVectorStoreProperties.CONFIG_PREFIX) public class OracleVectorStoreProperties extends CommonVectorStoreProperties { @@ -44,6 +45,16 @@ public class OracleVectorStoreProperties extends CommonVectorStoreProperties { private int searchAccuracy = OracleVectorStore.DEFAULT_SEARCH_ACCURACY; + private int hnswNeighbors = OracleVectorStore.DEFAULT_HNSW_NEIGHBORS; + + private int hnswEfConstruction = OracleVectorStore.DEFAULT_HNSW_EF_CONSTRUCTION; + + private int ivfNeighborPartitions = OracleVectorStore.DEFAULT_IVF_NEIGHBOR_PARTITIONS; + + private int ivfSamplePerPartition = OracleVectorStore.DEFAULT_IVF_SAMPLE_PER_PARTITION; + + private int ivfMinVectorsPerPartition = OracleVectorStore.DEFAULT_IVF_MIN_VECTORS_PER_PARTITION; + public String getTableName() { return this.tableName; } @@ -100,4 +111,94 @@ public void setSearchAccuracy(int searchAccuracy) { this.searchAccuracy = searchAccuracy; } + /** + * Returns the configured HNSW neighbors value. + * @return the configured HNSW neighbors value + * @since 2.0.0 + */ + public int getHnswNeighbors() { + return this.hnswNeighbors; + } + + /** + * Sets the HNSW neighbors value. + * @param hnswNeighbors the HNSW neighbors value + * @since 2.0.0 + */ + public void setHnswNeighbors(int hnswNeighbors) { + this.hnswNeighbors = hnswNeighbors; + } + + /** + * Returns the configured HNSW efConstruction value. + * @return the configured HNSW efConstruction value + * @since 2.0.0 + */ + public int getHnswEfConstruction() { + return this.hnswEfConstruction; + } + + /** + * Sets the HNSW efConstruction value. + * @param hnswEfConstruction the HNSW efConstruction value + * @since 2.0.0 + */ + public void setHnswEfConstruction(int hnswEfConstruction) { + this.hnswEfConstruction = hnswEfConstruction; + } + + /** + * Returns the configured IVF neighbor partitions value. + * @return the configured IVF neighbor partitions value + * @since 2.0.0 + */ + public int getIvfNeighborPartitions() { + return this.ivfNeighborPartitions; + } + + /** + * Sets the IVF neighbor partitions value. + * @param ivfNeighborPartitions the IVF neighbor partitions value + * @since 2.0.0 + */ + public void setIvfNeighborPartitions(int ivfNeighborPartitions) { + this.ivfNeighborPartitions = ivfNeighborPartitions; + } + + /** + * Returns the configured IVF sample per partition value. + * @return the configured IVF sample per partition value + * @since 2.0.0 + */ + public int getIvfSamplePerPartition() { + return this.ivfSamplePerPartition; + } + + /** + * Sets the IVF sample per partition value. + * @param ivfSamplePerPartition the IVF sample per partition value + * @since 2.0.0 + */ + public void setIvfSamplePerPartition(int ivfSamplePerPartition) { + this.ivfSamplePerPartition = ivfSamplePerPartition; + } + + /** + * Returns the configured IVF minimum vectors per partition value. + * @return the configured IVF minimum vectors per partition value + * @since 2.0.0 + */ + public int getIvfMinVectorsPerPartition() { + return this.ivfMinVectorsPerPartition; + } + + /** + * Sets the IVF minimum vectors per partition value. + * @param ivfMinVectorsPerPartition the IVF minimum vectors per partition value + * @since 2.0.0 + */ + public void setIvfMinVectorsPerPartition(int ivfMinVectorsPerPartition) { + this.ivfMinVectorsPerPartition = ivfMinVectorsPerPartition; + } + } diff --git a/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-oracle/src/test/java/org/springframework/ai/vectorstore/oracle/autoconfigure/OracleVectorStoreAutoConfigurationIT.java b/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-oracle/src/test/java/org/springframework/ai/vectorstore/oracle/autoconfigure/OracleVectorStoreAutoConfigurationIT.java index 946a156bbc..fb5e034f4b 100644 --- a/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-oracle/src/test/java/org/springframework/ai/vectorstore/oracle/autoconfigure/OracleVectorStoreAutoConfigurationIT.java +++ b/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-oracle/src/test/java/org/springframework/ai/vectorstore/oracle/autoconfigure/OracleVectorStoreAutoConfigurationIT.java @@ -51,14 +51,15 @@ * @author Christian Tzolov * @author Eddú Meléndez * @author Thomas Vitale + * @author Anders Swanson */ @Testcontainers public class OracleVectorStoreAutoConfigurationIT { @Container - static OracleContainer oracle23aiContainer = new OracleContainer("gvenzl/oracle-free:23-slim") - .withCopyFileToContainer(MountableFile.forClasspathResource("/oracle/initialize.sql"), - "/container-entrypoint-initdb.d/initialize.sql"); + static OracleContainer oracleContainer = new OracleContainer("gvenzl/oracle-free:23-slim").withCopyFileToContainer( + MountableFile.forClasspathResource("/oracle/initialize.sql"), + "/container-entrypoint-initdb.d/initialize.sql"); private final ApplicationContextRunner contextRunner = new ApplicationContextRunner() .withConfiguration(AutoConfigurations.of(OracleVectorStoreAutoConfiguration.class, @@ -68,9 +69,9 @@ public class OracleVectorStoreAutoConfigurationIT { "spring.ai.vectorstore.oracle.initialize-schema=true", "test.spring.ai.vectorstore.oracle.dimensions=384", // JdbcTemplate configuration - String.format("spring.datasource.url=%s", oracle23aiContainer.getJdbcUrl()), - String.format("spring.datasource.username=%s", oracle23aiContainer.getUsername()), - String.format("spring.datasource.password=%s", oracle23aiContainer.getPassword()), + String.format("spring.datasource.url=%s", oracleContainer.getJdbcUrl()), + String.format("spring.datasource.username=%s", oracleContainer.getUsername()), + String.format("spring.datasource.password=%s", oracleContainer.getPassword()), "spring.datasource.type=oracle.jdbc.pool.OracleDataSource"); List documents = List.of( @@ -152,6 +153,60 @@ public void autoConfigurationEnabledWhenTypeIsOracle() { }); } + @Test + public void customOracleIndexPropertiesAreApplied() { + this.contextRunner + .withPropertyValues("spring.ai.vectorstore.oracle.initialize-schema=false", + "spring.ai.vectorstore.oracle.index-type=HNSW", "spring.ai.vectorstore.oracle.hnsw-neighbors=64", + "spring.ai.vectorstore.oracle.hnsw-ef-construction=300", + "spring.ai.vectorstore.oracle.ivf-neighbor-partitions=32", + "spring.ai.vectorstore.oracle.ivf-sample-per-partition=16", + "spring.ai.vectorstore.oracle.ivf-min-vectors-per-partition=8") + .run(context -> { + OracleVectorStoreProperties properties = context.getBean(OracleVectorStoreProperties.class); + OracleVectorStore vectorStore = context.getBean(OracleVectorStore.class); + + assertThat(properties.getIndexType()).isEqualTo(OracleVectorStore.OracleVectorStoreIndexType.HNSW); + assertThat(properties.getHnswNeighbors()).isEqualTo(64); + assertThat(properties.getHnswEfConstruction()).isEqualTo(300); + assertThat(properties.getIvfNeighborPartitions()).isEqualTo(32); + assertThat(properties.getIvfSamplePerPartition()).isEqualTo(16); + assertThat(properties.getIvfMinVectorsPerPartition()).isEqualTo(8); + + assertThat(vectorStore.getIndexType()).isEqualTo(OracleVectorStore.OracleVectorStoreIndexType.HNSW); + assertThat(vectorStore.getHnswNeighbors()).isEqualTo(64); + assertThat(vectorStore.getHnswEfConstruction()).isEqualTo(300); + assertThat(vectorStore.getIvfNeighborPartitions()).isEqualTo(32); + assertThat(vectorStore.getIvfSamplePerPartition()).isEqualTo(16); + assertThat(vectorStore.getIvfMinVectorsPerPartition()).isEqualTo(8); + }); + } + + @Test + public void invalidHnswPropertiesFailAtStartup() { + this.contextRunner + .withPropertyValues("spring.ai.vectorstore.oracle.initialize-schema=false", + "spring.ai.vectorstore.oracle.index-type=HNSW", "spring.ai.vectorstore.oracle.hnsw-neighbors=0") + .run(context -> { + assertThat(context).hasFailed(); + assertThat(context.getStartupFailure()).hasRootCauseInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("HNSW neighbors must be greater than 0"); + }); + } + + @Test + public void invalidIvfPropertiesFailAtStartup() { + this.contextRunner + .withPropertyValues("spring.ai.vectorstore.oracle.initialize-schema=false", + "spring.ai.vectorstore.oracle.index-type=IVF", + "spring.ai.vectorstore.oracle.ivf-sample-per-partition=-2") + .run(context -> { + assertThat(context).hasFailed(); + assertThat(context.getStartupFailure()).hasRootCauseInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("IVF sample per partition must be greater than 0"); + }); + } + @Configuration(proxyBeanMethods = false) static class Config { diff --git a/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-oracle/src/test/java/org/springframework/ai/vectorstore/oracle/autoconfigure/OracleVectorStorePropertiesTests.java b/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-oracle/src/test/java/org/springframework/ai/vectorstore/oracle/autoconfigure/OracleVectorStorePropertiesTests.java index 74dba0bab9..4b4927f29f 100644 --- a/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-oracle/src/test/java/org/springframework/ai/vectorstore/oracle/autoconfigure/OracleVectorStorePropertiesTests.java +++ b/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-oracle/src/test/java/org/springframework/ai/vectorstore/oracle/autoconfigure/OracleVectorStorePropertiesTests.java @@ -26,6 +26,7 @@ /** * @author Christian Tzolov + * @author Anders Swanson */ public class OracleVectorStorePropertiesTests { @@ -35,6 +36,12 @@ public void defaultValues() { assertThat(props.getDimensions()).isEqualTo(OracleVectorStore.DEFAULT_DIMENSIONS); assertThat(props.getDistanceType()).isEqualTo(OracleVectorStoreDistanceType.COSINE); assertThat(props.getIndexType()).isEqualTo(OracleVectorStoreIndexType.IVF); + assertThat(props.getHnswNeighbors()).isEqualTo(OracleVectorStore.DEFAULT_HNSW_NEIGHBORS); + assertThat(props.getHnswEfConstruction()).isEqualTo(OracleVectorStore.DEFAULT_HNSW_EF_CONSTRUCTION); + assertThat(props.getIvfNeighborPartitions()).isEqualTo(OracleVectorStore.DEFAULT_IVF_NEIGHBOR_PARTITIONS); + assertThat(props.getIvfSamplePerPartition()).isEqualTo(OracleVectorStore.DEFAULT_IVF_SAMPLE_PER_PARTITION); + assertThat(props.getIvfMinVectorsPerPartition()) + .isEqualTo(OracleVectorStore.DEFAULT_IVF_MIN_VECTORS_PER_PARTITION); assertThat(props.isRemoveExistingVectorStoreTable()).isFalse(); } @@ -44,12 +51,22 @@ public void customValues() { props.setDimensions(1536); props.setDistanceType(OracleVectorStoreDistanceType.EUCLIDEAN); - props.setIndexType(OracleVectorStoreIndexType.IVF); + props.setIndexType(OracleVectorStoreIndexType.HNSW); + props.setHnswNeighbors(64); + props.setHnswEfConstruction(300); + props.setIvfNeighborPartitions(20); + props.setIvfSamplePerPartition(16); + props.setIvfMinVectorsPerPartition(8); props.setRemoveExistingVectorStoreTable(true); assertThat(props.getDimensions()).isEqualTo(1536); assertThat(props.getDistanceType()).isEqualTo(OracleVectorStoreDistanceType.EUCLIDEAN); - assertThat(props.getIndexType()).isEqualTo(OracleVectorStoreIndexType.IVF); + assertThat(props.getIndexType()).isEqualTo(OracleVectorStoreIndexType.HNSW); + assertThat(props.getHnswNeighbors()).isEqualTo(64); + assertThat(props.getHnswEfConstruction()).isEqualTo(300); + assertThat(props.getIvfNeighborPartitions()).isEqualTo(20); + assertThat(props.getIvfSamplePerPartition()).isEqualTo(16); + assertThat(props.getIvfMinVectorsPerPartition()).isEqualTo(8); assertThat(props.isRemoveExistingVectorStoreTable()).isTrue(); } diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat-memory.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat-memory.adoc index a1e87e4b20..a7abb5e1ad 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat-memory.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat-memory.adoc @@ -130,7 +130,7 @@ Spring AI supports multiple relational databases via a dialect abstraction. The - MySQL / MariaDB - SQL Server - HSQLDB -- Oracle Database +- Oracle AI Database The correct dialect can be auto-detected from the JDBC URL when using `JdbcChatMemoryRepositoryDialect.from(DataSource)`. You can extend support for other databases by implementing the `JdbcChatMemoryRepositoryDialect` interface. diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/vectordbs.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/vectordbs.adoc index d45e66d312..2861217f4c 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/vectordbs.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/vectordbs.adoc @@ -389,7 +389,7 @@ These are the available implementations of the `VectorStore` interface: * xref:api/vectordbs/mongodb.adoc[MongoDB Atlas Vector Store] - The https://www.mongodb.com/atlas/database[MongoDB Atlas] vector store. * xref:api/vectordbs/neo4j.adoc[Neo4j Vector Store] - The https://neo4j.com/[Neo4j] vector store. * xref:api/vectordbs/opensearch.adoc[OpenSearch Vector Store] - The https://opensearch.org/platform/search/vector-database.html[OpenSearch] vector store. -* xref:api/vectordbs/oracle.adoc[Oracle Vector Store] - The https://docs.oracle.com/en/database/oracle/oracle-database/23/vecse/overview-ai-vector-search.html[Oracle Database] vector store. +* xref:api/vectordbs/oracle.adoc[Oracle Vector Store] - The https://docs.oracle.com/en/database/oracle/oracle-database/26/vecse/overview-ai-vector-search.html[Oracle AI Database] vector store. * xref:api/vectordbs/pgvector.adoc[PgVector Store] - The https://github.com/pgvector/pgvector[PostgreSQL/PGVector] vector store. * xref:api/vectordbs/pinecone.adoc[Pinecone Vector Store] - https://www.pinecone.io/[Pinecone] vector store. * xref:api/vectordbs/qdrant.adoc[Qdrant Vector Store] - https://www.qdrant.tech/[Qdrant] vector store. diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/vectordbs/oracle.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/vectordbs/oracle.adoc index b889209f78..ad27f94bd3 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/vectordbs/oracle.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/vectordbs/oracle.adoc @@ -1,8 +1,8 @@ -= Oracle Database 23ai - AI Vector Search += Oracle AI Database - AI Vector Search -The link:https://docs.oracle.com/en/database/oracle/oracle-database/23/vecse/overview-ai-vector-search.html[AI Vector Search] capabilities of the Oracle Database 23ai (23.4+) are available as a Spring AI `VectorStore` to help you to store document embeddings and perform similarity searches. Of course, all other features are also available. +The link:https://docs.oracle.com/en/database/oracle/oracle-database/26/vecse/overview-ai-vector-search.html[AI Vector Search] capabilities of Oracle AI Database are available as a Spring AI `VectorStore` to help you to store document embeddings and perform similarity searches. Of course, all other features are also available. -TIP: The <> appendix shows how to start a database with a lightweight Docker container. +TIP: The <> appendix shows how to start a database with a lightweight Docker container. == Auto-Configuration @@ -35,6 +35,8 @@ If you need this vector store to initialize the schema for you then you'll need NOTE: this is a breaking change! In earlier versions of Spring AI, this schema initialization happened by default. +NOTE: HNSW index creation requires an Oracle AI Database release that supports HNSW vector indexes. The local Testcontainers example below uses `gvenzl/oracle-free:23-slim`. + The Vector Store, also requires an `EmbeddingModel` instance to calculate embeddings for the documents. You can pick one of the available xref:api/embeddings.adoc#available-implementations[EmbeddingModel Implementations]. @@ -109,7 +111,7 @@ You can use the following properties in your Spring Boot configuration to custom |=== |Property| Description | Default value -|`spring.ai.vectorstore.oracle.index-type`| Nearest neighbor search index type. Options are `NONE` - exact nearest neighbor search, `IVF` - Inverted Flat File index. It has faster build times and uses less memory than HNSW, but has lower query performance (in terms of speed-recall tradeoff). `HNSW` - creates a multilayer graph. It has slower build times and uses more memory than IVF, but has better query performance (in terms of speed-recall tradeoff). | NONE +|`spring.ai.vectorstore.oracle.index-type`| Nearest neighbor search index type. Options are `NONE` - exact nearest neighbor search, `IVF` - Inverted Flat File index. It has faster build times and uses less memory than HNSW, but has lower query performance (in terms of speed-recall tradeoff). `HNSW` - creates a multilayer graph. It has slower build times and uses more memory than IVF, but has better query performance (in terms of speed-recall tradeoff). If Oracle rejects the configured index DDL at startup, initialization fails and the application must be reconfigured. | IVF |`spring.ai.vectorstore.oracle.distance-type`| Search distance type among `COSINE` (default), `DOT`, `EUCLIDEAN`, `EUCLIDEAN_SQUARED`, and `MANHATTAN`. NOTE: If vectors are normalized, you can use `DOT` or `COSINE` for best performance.| COSINE @@ -122,6 +124,11 @@ NOTE: If vectors are normalized, you can use `DOT` or `COSINE` for best performa |`spring.ai.vectorstore.oracle.remove-existing-vector-store-table` | Drops the existing table on start up. | false |`spring.ai.vectorstore.oracle.initialize-schema` | Whether to initialize the required schema. | false |`spring.ai.vectorstore.oracle.search-accuracy` | Denote the requested accuracy target in the presence of index. Disabled by default. You need to provide an integer in the range [1,100] to override the default index accuracy (95). Using lower accuracy provides approximate similarity search trading off speed versus accuracy. | -1 (`DEFAULT_SEARCH_ACCURACY`) +|`spring.ai.vectorstore.oracle.hnsw-neighbors` | HNSW `NEIGHBORS` index build parameter. Used only when `index-type=HNSW`. | 40 +|`spring.ai.vectorstore.oracle.hnsw-ef-construction` | HNSW `EFCONSTRUCTION` index build parameter. Used only when `index-type=HNSW`. | 500 +|`spring.ai.vectorstore.oracle.ivf-neighbor-partitions` | IVF `NEIGHBOR PARTITIONS` index build parameter. Used only when `index-type=IVF`. | 10 +|`spring.ai.vectorstore.oracle.ivf-sample-per-partition` | Optional IVF `SAMPLE_PER_PARTITION` index build parameter. Omitted unless explicitly configured. | -1 +|`spring.ai.vectorstore.oracle.ivf-min-vectors-per-partition` | Optional IVF `MIN_VECTORS_PER_PARTITION` index build parameter. Omitted unless explicitly configured. | -1 |=== @@ -192,19 +199,34 @@ To configure the `OracleVectorStore` in your application, you can use the follow public VectorStore vectorStore(JdbcTemplate jdbcTemplate, EmbeddingModel embeddingModel) { return OracleVectorStore.builder(jdbcTemplate, embeddingModel) .tableName("my_vectors") - .indexType(OracleVectorStoreIndexType.IVF) + .indexType(OracleVectorStoreIndexType.HNSW) .distanceType(OracleVectorStoreDistanceType.COSINE) .dimensions(1536) .searchAccuracy(95) + .hnswNeighbors(64) + .hnswEfConstruction(300) .initializeSchema(true) .build(); } ---- -== Run Oracle Database 23ai locally +To customize IVF index creation instead, configure the IVF-specific builder settings: +[source,java] ---- -docker run --rm --name oracle23ai -p 1521:1521 -e APP_USER=mlops -e APP_USER_PASSWORD=mlops -e ORACLE_PASSWORD=mlops gvenzl/oracle-free:23-slim +OracleVectorStore.builder(jdbcTemplate, embeddingModel) + .indexType(OracleVectorStoreIndexType.IVF) + .ivfNeighborPartitions(32) + .ivfSamplePerPartition(16) + .ivfMinVectorsPerPartition(8) + .initializeSchema(true) + .build(); +---- + +== Run Oracle AI Database locally + +---- +docker run --rm --name oracle-ai-db -p 1521:1521 -e APP_USER=mlops -e APP_USER_PASSWORD=mlops -e ORACLE_PASSWORD=mlops gvenzl/oracle-free:23-slim ---- You can then connect to the database using: @@ -221,10 +243,8 @@ The Oracle Vector Store implementation provides access to the underlying native ---- OracleVectorStore vectorStore = context.getBean(OracleVectorStore.class); Optional nativeClient = vectorStore.getNativeClient(); - if (nativeClient.isPresent()) { - OracleConnection connection = nativeClient.get(); - // Use the native client for Oracle-specific operations + OracleConnection connection = nativeClient.get(); // Use the native client for Oracle-specific operations } ---- diff --git a/vector-stores/spring-ai-oracle-store/README.md b/vector-stores/spring-ai-oracle-store/README.md index 0484b1911d..3b18d4114e 100644 --- a/vector-stores/spring-ai-oracle-store/README.md +++ b/vector-stores/spring-ai-oracle-store/README.md @@ -1 +1 @@ -[Oracle AI Vector Search Documentation](https://docs.oracle.com/en/database/oracle/oracle-database/23/nfcoa/ai_vector_search.html) \ No newline at end of file +[Oracle AI Vector Search Documentation](https://docs.oracle.com/en/database/oracle/oracle-database/26/nfcoa/ai_vector_search.html) \ No newline at end of file diff --git a/vector-stores/spring-ai-oracle-store/pom.xml b/vector-stores/spring-ai-oracle-store/pom.xml index a7fd58deaf..0574101851 100644 --- a/vector-stores/spring-ai-oracle-store/pom.xml +++ b/vector-stores/spring-ai-oracle-store/pom.xml @@ -27,7 +27,7 @@ spring-ai-oracle-store jar Spring AI Vector Store - Oracle - AI Vector Search from Oracle Database 23ai+ as a Spring AI Vector Store + AI Vector Search from Oracle AI Database as a Spring AI Vector Store https://github.com/spring-projects/spring-ai diff --git a/vector-stores/spring-ai-oracle-store/src/main/java/org/springframework/ai/vectorstore/oracle/OracleVectorStore.java b/vector-stores/spring-ai-oracle-store/src/main/java/org/springframework/ai/vectorstore/oracle/OracleVectorStore.java index 84025bce13..0c9ad02746 100644 --- a/vector-stores/spring-ai-oracle-store/src/main/java/org/springframework/ai/vectorstore/oracle/OracleVectorStore.java +++ b/vector-stores/spring-ai-oracle-store/src/main/java/org/springframework/ai/vectorstore/oracle/OracleVectorStore.java @@ -58,12 +58,11 @@ /** *

- * Integration of Oracle database 23ai as a Vector Store. + * Integration of Oracle AI Database as a Vector Store. *

*

- * With the release 23ai (23.4), the Oracle database provides numerous features useful for - * artificial intelligence such as Vectors, Similarity search, Vector indexes, ONNX - * models... + * Oracle AI Database provides numerous features useful for artificial intelligence such + * as Vectors, Similarity search, Vector indexes, ONNX models... *

*

* This Spring AI Vector store supports the following features: @@ -72,7 +71,7 @@ *

  • Distance type for similarity search (note that similarity threshold can be used * only with distance type COSINE and DOT when ingested vectors are normalized, see * forcedNormalization)
    • - *
  • Vector indexes (use IVF as of 23.4)
    • + *
  • Vector indexes (IVF and, where supported by the connected database, HNSW)
    • *
  • Exact and Approximate similarity search
    • *
  • Filter expression as SQL/JSON Path expression evaluation
    • * @@ -81,6 +80,7 @@ * @author Christian Tzolov * @author Soby Chacko * @author Thomas Vitale + * @author Anders Swanson */ public class OracleVectorStore extends AbstractObservationVectorStore implements InitializingBean { @@ -96,6 +96,16 @@ public class OracleVectorStore extends AbstractObservationVectorStore implements public static final int DEFAULT_SEARCH_ACCURACY = -1; + public static final int DEFAULT_HNSW_NEIGHBORS = 40; + + public static final int DEFAULT_HNSW_EF_CONSTRUCTION = 500; + + public static final int DEFAULT_IVF_NEIGHBOR_PARTITIONS = 10; + + public static final int DEFAULT_IVF_SAMPLE_PER_PARTITION = -1; + + public static final int DEFAULT_IVF_MIN_VECTORS_PER_PARTITION = -1; + private static final Logger logger = LoggerFactory.getLogger(OracleVectorStore.class); private static final Map SIMILARITY_TYPE_MAPPING = Map @@ -137,6 +147,16 @@ public class OracleVectorStore extends AbstractObservationVectorStore implements private final int searchAccuracy; + private final int hnswNeighbors; + + private final int hnswEfConstruction; + + private final int ivfNeighborPartitions; + + private final int ivfSamplePerPartition; + + private final int ivfMinVectorsPerPartition; + private final OracleJsonFactory osonFactory = new OracleJsonFactory(); private final ByteArrayOutputStream out = new ByteArrayOutputStream(); @@ -157,6 +177,11 @@ protected OracleVectorStore(Builder builder) { this.distanceType = builder.distanceType; this.dimensions = builder.dimensions; this.searchAccuracy = builder.searchAccuracy; + this.hnswNeighbors = builder.hnswNeighbors; + this.hnswEfConstruction = builder.hnswEfConstruction; + this.ivfNeighborPartitions = builder.ivfNeighborPartitions; + this.ivfSamplePerPartition = builder.ivfSamplePerPartition; + this.ivfMinVectorsPerPartition = builder.ivfMinVectorsPerPartition; this.initializeSchema = builder.initializeSchema; this.removeExistingVectorStoreTable = builder.removeExistingVectorStoreTable; this.forcedNormalization = builder.forcedNormalization; @@ -492,33 +517,112 @@ embedding vector(%s,FLOAT64) annotations(Distance '%s') switch (this.indexType) { case IVF: - this.jdbcTemplate.execute(String.format(""" - create vector index if not exists vector_index_%s on %s (embedding) - organization neighbor partitions - distance %s - with target accuracy %d - parameters (type IVF, neighbor partitions 10)""", this.tableName, this.tableName, - this.distanceType.name(), - this.searchAccuracy == DEFAULT_SEARCH_ACCURACY ? 95 : this.searchAccuracy)); + this.jdbcTemplate.execute(createIvfIndexSql()); + break; + case HNSW: + this.jdbcTemplate.execute(createHnswIndexSql()); + break; + case NONE: break; - - /* - * TODO: Enable for 23.5 case HNSW: - * this.jdbcTemplate.execute(String.format(""" create vector index if not - * exists vector_index_%s on %s (embedding) organization inmemory neighbor - * graph distance %s with target accuracy %d parameters (type HNSW, - * neighbors 40, efconstruction 500)""", tableName, tableName, - * distanceType.name(), searchAccuracy == DEFAULT_SEARCH_ACCURACY ? 95 : - * searchAccuracy)); break; - */ } } } + private String createIvfIndexSql() { + return String.format(""" + create vector index if not exists vector_index_%s on %s (embedding) + organization neighbor partitions + distance %s + with target accuracy %d + parameters (%s)""", this.tableName, this.tableName, this.distanceType.name(), + getTargetAccuracy(), getIvfIndexParameters()); + } + + private String getIvfIndexParameters() { + List parameters = new ArrayList<>(); + parameters.add("type IVF"); + parameters.add("neighbor partitions " + this.ivfNeighborPartitions); + if (this.ivfSamplePerPartition != DEFAULT_IVF_SAMPLE_PER_PARTITION) { + parameters.add("sample_per_partition " + this.ivfSamplePerPartition); + } + if (this.ivfMinVectorsPerPartition != DEFAULT_IVF_MIN_VECTORS_PER_PARTITION) { + parameters.add("min_vectors_per_partition " + this.ivfMinVectorsPerPartition); + } + return String.join(", ", parameters); + } + + private String createHnswIndexSql() { + return String.format(""" + create vector index if not exists vector_index_%s on %s (embedding) + organization inmemory neighbor graph + distance %s + with target accuracy %d + parameters (type HNSW, neighbors %d, efconstruction %d)""", this.tableName, this.tableName, + this.distanceType.name(), getTargetAccuracy(), this.hnswNeighbors, this.hnswEfConstruction); + } + + private int getTargetAccuracy() { + return this.searchAccuracy == DEFAULT_SEARCH_ACCURACY ? 95 : this.searchAccuracy; + } + public String getTableName() { return this.tableName; } + /** + * Returns the configured vector index type. + * @return the configured index type + * @since 2.0.0 + */ + public OracleVectorStoreIndexType getIndexType() { + return this.indexType; + } + + /** + * Returns the configured HNSW neighbors value. + * @return the configured HNSW neighbors value + * @since 2.0.0 + */ + public int getHnswNeighbors() { + return this.hnswNeighbors; + } + + /** + * Returns the configured HNSW efConstruction value. + * @return the configured HNSW efConstruction value + * @since 2.0.0 + */ + public int getHnswEfConstruction() { + return this.hnswEfConstruction; + } + + /** + * Returns the configured IVF neighbor partitions value. + * @return the configured IVF neighbor partitions value + * @since 2.0.0 + */ + public int getIvfNeighborPartitions() { + return this.ivfNeighborPartitions; + } + + /** + * Returns the configured IVF sample per partition value. + * @return the configured IVF sample per partition value + * @since 2.0.0 + */ + public int getIvfSamplePerPartition() { + return this.ivfSamplePerPartition; + } + + /** + * Returns the configured IVF minimum vectors per partition value. + * @return the configured IVF minimum vectors per partition value + * @since 2.0.0 + */ + public int getIvfMinVectorsPerPartition() { + return this.ivfMinVectorsPerPartition; + } + @Override public VectorStoreObservationContext.Builder createObservationContextBuilder(String operationName) { return VectorStoreObservationContext.builder(VectorStoreProvider.ORACLE.value(), operationName) @@ -567,7 +671,7 @@ public enum OracleVectorStoreIndexType { * * * @see Oracle + * "https://docs.oracle.com/en/database/oracle/oracle-database/26/vecse/hierarchical-navigable-small-world-index-syntax-and-parameters.html">Oracle * Database documentation */ HNSW, @@ -581,7 +685,7 @@ public enum OracleVectorStoreIndexType { *

    * * * @see Oracle + * "https://docs.oracle.com/en/database/oracle/oracle-database/26/vecse/understand-inverted-file-flat-vector-indexes.html">Oracle * Database documentation */ IVF @@ -686,6 +790,16 @@ public static class Builder extends AbstractVectorStoreBuilder { private int searchAccuracy = DEFAULT_SEARCH_ACCURACY; + private int hnswNeighbors = DEFAULT_HNSW_NEIGHBORS; + + private int hnswEfConstruction = DEFAULT_HNSW_EF_CONSTRUCTION; + + private int ivfNeighborPartitions = DEFAULT_IVF_NEIGHBOR_PARTITIONS; + + private int ivfSamplePerPartition = DEFAULT_IVF_SAMPLE_PER_PARTITION; + + private int ivfMinVectorsPerPartition = DEFAULT_IVF_MIN_VECTORS_PER_PARTITION; + private boolean initializeSchema = false; private boolean removeExistingVectorStoreTable = false; @@ -769,6 +883,72 @@ public Builder searchAccuracy(int searchAccuracy) { return this; } + /** + * Sets the NEIGHBORS parameter for HNSW indexes. + * @param hnswNeighbors the maximum number of neighbors per node + * @return the builder instance + * @since 2.0.0 + */ + public Builder hnswNeighbors(int hnswNeighbors) { + Assert.isTrue(hnswNeighbors > 0, "HNSW neighbors must be greater than 0"); + this.hnswNeighbors = hnswNeighbors; + return this; + } + + /** + * Sets the EFCONSTRUCTION parameter for HNSW indexes. + * @param hnswEfConstruction the size of the candidate list during construction + * @return the builder instance + * @since 2.0.0 + */ + public Builder hnswEfConstruction(int hnswEfConstruction) { + Assert.isTrue(hnswEfConstruction > 0, "HNSW efConstruction must be greater than 0"); + this.hnswEfConstruction = hnswEfConstruction; + return this; + } + + /** + * Sets the NEIGHBOR PARTITIONS parameter for IVF indexes. + * @param ivfNeighborPartitions the number of IVF partitions + * @return the builder instance + * @since 2.0.0 + */ + public Builder ivfNeighborPartitions(int ivfNeighborPartitions) { + Assert.isTrue(ivfNeighborPartitions > 0, "IVF neighbor partitions must be greater than 0"); + this.ivfNeighborPartitions = ivfNeighborPartitions; + return this; + } + + /** + * Sets the SAMPLE_PER_PARTITION parameter for IVF indexes. + * @param ivfSamplePerPartition the optional sample count per partition, or -1 to + * omit it from DDL + * @return the builder instance + * @since 2.0.0 + */ + public Builder ivfSamplePerPartition(int ivfSamplePerPartition) { + if (ivfSamplePerPartition != DEFAULT_IVF_SAMPLE_PER_PARTITION) { + Assert.isTrue(ivfSamplePerPartition > 0, "IVF sample per partition must be greater than 0"); + } + this.ivfSamplePerPartition = ivfSamplePerPartition; + return this; + } + + /** + * Sets the MIN_VECTORS_PER_PARTITION parameter for IVF indexes. + * @param ivfMinVectorsPerPartition the optional minimum vector count per + * partition, or -1 to omit it from DDL + * @return the builder instance + * @since 2.0.0 + */ + public Builder ivfMinVectorsPerPartition(int ivfMinVectorsPerPartition) { + if (ivfMinVectorsPerPartition != DEFAULT_IVF_MIN_VECTORS_PER_PARTITION) { + Assert.isTrue(ivfMinVectorsPerPartition > 0, "IVF min vectors per partition must be greater than 0"); + } + this.ivfMinVectorsPerPartition = ivfMinVectorsPerPartition; + return this; + } + /** * Sets whether to initialize the database schema. * @param initializeSchema true to initialize schema, false otherwise diff --git a/vector-stores/spring-ai-oracle-store/src/main/java/org/springframework/ai/vectorstore/oracle/SqlJsonPathFilterExpressionConverter.java b/vector-stores/spring-ai-oracle-store/src/main/java/org/springframework/ai/vectorstore/oracle/SqlJsonPathFilterExpressionConverter.java index 52175e0493..5f6e3887a3 100644 --- a/vector-stores/spring-ai-oracle-store/src/main/java/org/springframework/ai/vectorstore/oracle/SqlJsonPathFilterExpressionConverter.java +++ b/vector-stores/spring-ai-oracle-store/src/main/java/org/springframework/ai/vectorstore/oracle/SqlJsonPathFilterExpressionConverter.java @@ -24,8 +24,9 @@ * Converts a {@link Filter} into a JSON Path expression. * * @author Loïc Lefèvre + * @author Anders Swanson * @see JSON + * "https://docs.oracle.com/en/database/oracle/oracle-database/26/adjsn/overview-sql-json-path-expressions.html">JSON * Path Documentation */ public class SqlJsonPathFilterExpressionConverter extends AbstractFilterExpressionConverter { diff --git a/vector-stores/spring-ai-oracle-store/src/test/java/org/springframework/ai/vectorstore/oracle/OracleImage.java b/vector-stores/spring-ai-oracle-store/src/test/java/org/springframework/ai/vectorstore/oracle/OracleImage.java index 2cdd78a3d8..00599e7849 100644 --- a/vector-stores/spring-ai-oracle-store/src/test/java/org/springframework/ai/vectorstore/oracle/OracleImage.java +++ b/vector-stores/spring-ai-oracle-store/src/test/java/org/springframework/ai/vectorstore/oracle/OracleImage.java @@ -20,9 +20,13 @@ /** * @author Thomas Vitale + * @author Anders Swanson */ public final class OracleImage { + /** + * The 23-slim tag resolves to the latest version of Oracle AI Database. + */ public static final DockerImageName DEFAULT_IMAGE = DockerImageName.parse("gvenzl/oracle-free:23-slim"); private OracleImage() { diff --git a/vector-stores/spring-ai-oracle-store/src/test/java/org/springframework/ai/vectorstore/oracle/OracleVectorStoreBuilderTests.java b/vector-stores/spring-ai-oracle-store/src/test/java/org/springframework/ai/vectorstore/oracle/OracleVectorStoreBuilderTests.java new file mode 100644 index 0000000000..a28be8eaaf --- /dev/null +++ b/vector-stores/spring-ai-oracle-store/src/test/java/org/springframework/ai/vectorstore/oracle/OracleVectorStoreBuilderTests.java @@ -0,0 +1,207 @@ +/* + * Copyright 2023-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.oracle; + +import java.lang.reflect.Method; +import java.util.List; + +import org.junit.jupiter.api.Test; + +import org.springframework.ai.document.Document; +import org.springframework.ai.embedding.Embedding; +import org.springframework.ai.embedding.EmbeddingModel; +import org.springframework.ai.embedding.EmbeddingRequest; +import org.springframework.ai.embedding.EmbeddingResponse; +import org.springframework.jdbc.core.JdbcTemplate; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** + * @author Anders Swanson + */ +class OracleVectorStoreBuilderTests { + + private final JdbcTemplate jdbcTemplate = new JdbcTemplate(); + + private final EmbeddingModel embeddingModel = new TestEmbeddingModel(); + + @Test + void shouldUseDefaultIndexConfiguration() { + OracleVectorStore vectorStore = OracleVectorStore.builder(this.jdbcTemplate, this.embeddingModel).build(); + + assertThat(vectorStore.getIndexType()).isEqualTo(OracleVectorStore.DEFAULT_INDEX_TYPE); + assertThat(vectorStore.getHnswNeighbors()).isEqualTo(OracleVectorStore.DEFAULT_HNSW_NEIGHBORS); + assertThat(vectorStore.getHnswEfConstruction()).isEqualTo(OracleVectorStore.DEFAULT_HNSW_EF_CONSTRUCTION); + assertThat(vectorStore.getIvfNeighborPartitions()).isEqualTo(OracleVectorStore.DEFAULT_IVF_NEIGHBOR_PARTITIONS); + assertThat(vectorStore.getIvfSamplePerPartition()) + .isEqualTo(OracleVectorStore.DEFAULT_IVF_SAMPLE_PER_PARTITION); + assertThat(vectorStore.getIvfMinVectorsPerPartition()) + .isEqualTo(OracleVectorStore.DEFAULT_IVF_MIN_VECTORS_PER_PARTITION); + } + + @Test + void shouldValidateHnswNeighbors() { + assertThatThrownBy( + () -> OracleVectorStore.builder(this.jdbcTemplate, this.embeddingModel).hnswNeighbors(0).build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("HNSW neighbors must be greater than 0"); + } + + @Test + void shouldValidateHnswEfConstruction() { + assertThatThrownBy( + () -> OracleVectorStore.builder(this.jdbcTemplate, this.embeddingModel).hnswEfConstruction(0).build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("HNSW efConstruction must be greater than 0"); + } + + @Test + void shouldValidateIvfNeighborPartitions() { + assertThatThrownBy(() -> OracleVectorStore.builder(this.jdbcTemplate, this.embeddingModel) + .ivfNeighborPartitions(0) + .build()).isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("IVF neighbor partitions must be greater than 0"); + } + + @Test + void shouldAllowIvfOptionalSentinelValues() { + OracleVectorStore vectorStore = OracleVectorStore.builder(this.jdbcTemplate, this.embeddingModel) + .ivfSamplePerPartition(OracleVectorStore.DEFAULT_IVF_SAMPLE_PER_PARTITION) + .ivfMinVectorsPerPartition(OracleVectorStore.DEFAULT_IVF_MIN_VECTORS_PER_PARTITION) + .build(); + + assertThat(vectorStore.getIvfSamplePerPartition()) + .isEqualTo(OracleVectorStore.DEFAULT_IVF_SAMPLE_PER_PARTITION); + assertThat(vectorStore.getIvfMinVectorsPerPartition()) + .isEqualTo(OracleVectorStore.DEFAULT_IVF_MIN_VECTORS_PER_PARTITION); + } + + @Test + void shouldGenerateHnswIndexSql() { + OracleVectorStore vectorStore = OracleVectorStore.builder(this.jdbcTemplate, this.embeddingModel) + .tableName("custom_vectors") + .indexType(OracleVectorStore.OracleVectorStoreIndexType.HNSW) + .distanceType(OracleVectorStore.OracleVectorStoreDistanceType.COSINE) + .hnswNeighbors(64) + .hnswEfConstruction(300) + .build(); + + String sql = invokePrivateSqlMethod(vectorStore, "createHnswIndexSql"); + + assertThat(sql).contains("type HNSW, neighbors 64, efconstruction 300"); + assertThat(sql).contains("with target accuracy 95"); + assertThat(sql).contains("organization inmemory neighbor graph"); + assertThat(sql).contains("vector_index_custom_vectors"); + } + + @Test + void shouldGenerateIvfIndexSqlWithOptionalParameters() { + OracleVectorStore vectorStore = OracleVectorStore.builder(this.jdbcTemplate, this.embeddingModel) + .tableName("custom_vectors") + .indexType(OracleVectorStore.OracleVectorStoreIndexType.IVF) + .distanceType(OracleVectorStore.OracleVectorStoreDistanceType.COSINE) + .ivfNeighborPartitions(32) + .ivfSamplePerPartition(16) + .ivfMinVectorsPerPartition(8) + .build(); + + String sql = invokePrivateSqlMethod(vectorStore, "createIvfIndexSql"); + + assertThat(sql) + .contains("type IVF, neighbor partitions 32, sample_per_partition 16, min_vectors_per_partition 8"); + assertThat(sql).contains("with target accuracy 95"); + assertThat(sql).contains("organization neighbor partitions"); + } + + @Test + void shouldGenerateIvfIndexSqlWithoutOptionalParameters() { + OracleVectorStore vectorStore = OracleVectorStore.builder(this.jdbcTemplate, this.embeddingModel) + .tableName("custom_vectors") + .indexType(OracleVectorStore.OracleVectorStoreIndexType.IVF) + .distanceType(OracleVectorStore.OracleVectorStoreDistanceType.COSINE) + .ivfNeighborPartitions(32) + .build(); + + String sql = invokePrivateSqlMethod(vectorStore, "createIvfIndexSql"); + + assertThat(sql).contains("type IVF, neighbor partitions 32"); + assertThat(sql).doesNotContain("sample_per_partition"); + assertThat(sql).doesNotContain("min_vectors_per_partition"); + } + + @Test + void shouldValidateIvfSamplePerPartition() { + assertThatThrownBy(() -> OracleVectorStore.builder(this.jdbcTemplate, this.embeddingModel) + .ivfSamplePerPartition(0) + .build()).isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("IVF sample per partition must be greater than 0"); + + assertThatThrownBy(() -> OracleVectorStore.builder(this.jdbcTemplate, this.embeddingModel) + .ivfSamplePerPartition(-2) + .build()).isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("IVF sample per partition must be greater than 0"); + } + + @Test + void shouldValidateIvfMinVectorsPerPartition() { + assertThatThrownBy(() -> OracleVectorStore.builder(this.jdbcTemplate, this.embeddingModel) + .ivfMinVectorsPerPartition(0) + .build()).isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("IVF min vectors per partition must be greater than 0"); + + assertThatThrownBy(() -> OracleVectorStore.builder(this.jdbcTemplate, this.embeddingModel) + .ivfMinVectorsPerPartition(-2) + .build()).isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("IVF min vectors per partition must be greater than 0"); + } + + private static String invokePrivateSqlMethod(OracleVectorStore vectorStore, String methodName) { + try { + Method method = OracleVectorStore.class.getDeclaredMethod(methodName); + method.setAccessible(true); + return (String) method.invoke(vectorStore); + } + catch (Exception ex) { + throw new RuntimeException(ex); + } + } + + private static final class TestEmbeddingModel implements EmbeddingModel { + + @Override + public EmbeddingResponse call(EmbeddingRequest request) { + List embeddings = request.getInstructions() + .stream() + .map(text -> new Embedding(new float[] { 1.0f }, 0)) + .toList(); + return new EmbeddingResponse(embeddings); + } + + @Override + public float[] embed(Document document) { + return new float[] { 1.0f }; + } + + @Override + public int dimensions() { + return 1; + } + + } + +} diff --git a/vector-stores/spring-ai-oracle-store/src/test/java/org/springframework/ai/vectorstore/oracle/OracleVectorStoreIT.java b/vector-stores/spring-ai-oracle-store/src/test/java/org/springframework/ai/vectorstore/oracle/OracleVectorStoreIT.java index 4b3c491833..8fa9ae5cd4 100644 --- a/vector-stores/spring-ai-oracle-store/src/test/java/org/springframework/ai/vectorstore/oracle/OracleVectorStoreIT.java +++ b/vector-stores/spring-ai-oracle-store/src/test/java/org/springframework/ai/vectorstore/oracle/OracleVectorStoreIT.java @@ -68,7 +68,7 @@ public class OracleVectorStoreIT extends BaseVectorStoreTests { @Container - static OracleContainer oracle23aiContainer = new OracleContainer(OracleImage.DEFAULT_IMAGE) + static OracleContainer oracleContainer = new OracleContainer(OracleImage.DEFAULT_IMAGE) .withCopyFileToContainer(MountableFile.forClasspathResource("/initialize.sql"), "/container-entrypoint-initdb.d/initialize.sql") .withStartupTimeout(Duration.ofMinutes(5)) @@ -82,12 +82,19 @@ public class OracleVectorStoreIT extends BaseVectorStoreTests { private final ApplicationContextRunner contextRunner = new ApplicationContextRunner() .withUserConfiguration(TestClient.class) - .withPropertyValues("test.spring.ai.vectorstore.oracle.distanceType=COSINE", + .withPropertyValues("test.spring.ai.vectorstore.oracle.indexType=IVF", + "test.spring.ai.vectorstore.oracle.distanceType=COSINE", "test.spring.ai.vectorstore.oracle.dimensions=384", + "test.spring.ai.vectorstore.oracle.hnswNeighbors=40", + "test.spring.ai.vectorstore.oracle.hnswEfConstruction=500", + "test.spring.ai.vectorstore.oracle.ivfNeighborPartitions=10", + "test.spring.ai.vectorstore.oracle.ivfSamplePerPartition=-1", + "test.spring.ai.vectorstore.oracle.ivfMinVectorsPerPartition=-1", + "test.spring.ai.vectorstore.oracle.initializeSchema=true", // JdbcTemplate configuration - String.format("app.datasource.url=%s", oracle23aiContainer.getJdbcUrl()), - String.format("app.datasource.username=%s", oracle23aiContainer.getUsername()), - String.format("app.datasource.password=%s", oracle23aiContainer.getPassword()), + String.format("app.datasource.url=%s", oracleContainer.getJdbcUrl()), + String.format("app.datasource.username=%s", oracleContainer.getUsername()), + String.format("app.datasource.password=%s", oracleContainer.getPassword()), "app.datasource.type=oracle.jdbc.pool.OracleDataSource"); public static String getText(final String uri) { @@ -104,6 +111,14 @@ private static void dropTable(ApplicationContext context, String tableName) { jdbcTemplate.execute("DROP TABLE IF EXISTS " + tableName + " PURGE"); } + private static boolean isIndexPresent(ApplicationContext context, String tableName) { + JdbcTemplate jdbcTemplate = context.getBean(JdbcTemplate.class); + String indexName = ("VECTOR_INDEX_" + tableName).toUpperCase(); + Integer count = jdbcTemplate.queryForObject("SELECT COUNT(*) FROM USER_INDEXES WHERE INDEX_NAME = ?", + Integer.class, indexName); + return count != null && count > 0; + } + private static boolean isSortedBySimilarity(final List documents) { final List scores = documents.stream().map(Document::getScore).toList(); @@ -127,7 +142,8 @@ private static boolean isSortedBySimilarity(final List documents) { @Override protected void executeTest(Consumer testFunction) { this.contextRunner - .withPropertyValues("test.spring.ai.vectorstore.oracle.distanceType=COSINE", + .withPropertyValues("test.spring.ai.vectorstore.oracle.indexType=IVF", + "test.spring.ai.vectorstore.oracle.distanceType=COSINE", "test.spring.ai.vectorstore.oracle.searchAccuracy=" + OracleVectorStore.DEFAULT_SEARCH_ACCURACY) .run(context -> { VectorStore vectorStore = context.getBean(VectorStore.class); @@ -135,10 +151,14 @@ protected void executeTest(Consumer testFunction) { }); } - @ParameterizedTest(name = "{0} : {displayName} ") - @ValueSource(strings = { "COSINE", "DOT", "EUCLIDEAN", "EUCLIDEAN_SQUARED", "MANHATTAN" }) - public void addAndSearch(String distanceType) { - this.contextRunner.withPropertyValues("test.spring.ai.vectorstore.oracle.distanceType=" + distanceType) + @ParameterizedTest(name = "{0} / {1} : {displayName} ") + @CsvSource({ "NONE,COSINE", "NONE,DOT", "NONE,EUCLIDEAN", "NONE,EUCLIDEAN_SQUARED", "NONE,MANHATTAN", "IVF,COSINE", + "IVF,DOT", "IVF,EUCLIDEAN", "IVF,EUCLIDEAN_SQUARED", "IVF,MANHATTAN", "HNSW,COSINE", "HNSW,DOT", + "HNSW,EUCLIDEAN", "HNSW,EUCLIDEAN_SQUARED", "HNSW,MANHATTAN" }) + public void addAndSearch(String indexType, String distanceType) { + this.contextRunner + .withPropertyValues("test.spring.ai.vectorstore.oracle.indexType=" + indexType, + "test.spring.ai.vectorstore.oracle.distanceType=" + distanceType) .withPropertyValues( "test.spring.ai.vectorstore.oracle.searchAccuracy=" + OracleVectorStore.DEFAULT_SEARCH_ACCURACY) .run(context -> { @@ -170,7 +190,9 @@ public void addAndSearch(String distanceType) { @CsvSource({ "COSINE,-1", "DOT,-1", "EUCLIDEAN,-1", "EUCLIDEAN_SQUARED,-1", "MANHATTAN,-1", "COSINE,75", "DOT,80", "EUCLIDEAN,60", "EUCLIDEAN_SQUARED,30", "MANHATTAN,42" }) public void searchWithFilters(String distanceType, int searchAccuracy) { - this.contextRunner.withPropertyValues("test.spring.ai.vectorstore.oracle.distanceType=" + distanceType) + this.contextRunner + .withPropertyValues("test.spring.ai.vectorstore.oracle.indexType=IVF", + "test.spring.ai.vectorstore.oracle.distanceType=" + distanceType) .withPropertyValues("test.spring.ai.vectorstore.oracle.searchAccuracy=" + searchAccuracy) .run(context -> { @@ -251,7 +273,9 @@ public void searchWithFilters(String distanceType, int searchAccuracy) { @ParameterizedTest(name = "{0} : {displayName} ") @ValueSource(strings = { "COSINE", "DOT", "EUCLIDEAN", "EUCLIDEAN_SQUARED", "MANHATTAN" }) public void documentUpdate(String distanceType) { - this.contextRunner.withPropertyValues("test.spring.ai.vectorstore.oracle.distanceType=" + distanceType) + this.contextRunner + .withPropertyValues("test.spring.ai.vectorstore.oracle.indexType=IVF", + "test.spring.ai.vectorstore.oracle.distanceType=" + distanceType) .withPropertyValues( "test.spring.ai.vectorstore.oracle.searchAccuracy=" + OracleVectorStore.DEFAULT_SEARCH_ACCURACY) .run(context -> { @@ -292,7 +316,9 @@ public void documentUpdate(String distanceType) { @ParameterizedTest(name = "{0} : {displayName} ") @ValueSource(strings = { "COSINE", "DOT" }) public void searchWithThreshold(String distanceType) { - this.contextRunner.withPropertyValues("test.spring.ai.vectorstore.oracle.distanceType=" + distanceType) + this.contextRunner + .withPropertyValues("test.spring.ai.vectorstore.oracle.indexType=IVF", + "test.spring.ai.vectorstore.oracle.distanceType=" + distanceType) .withPropertyValues( "test.spring.ai.vectorstore.oracle.searchAccuracy=" + OracleVectorStore.DEFAULT_SEARCH_ACCURACY) .run(context -> { @@ -330,7 +356,8 @@ public void searchWithThreshold(String distanceType) { @Test void deleteWithComplexFilterExpression() { this.contextRunner - .withPropertyValues("test.spring.ai.vectorstore.oracle.distanceType=COSINE", + .withPropertyValues("test.spring.ai.vectorstore.oracle.indexType=IVF", + "test.spring.ai.vectorstore.oracle.distanceType=COSINE", "test.spring.ai.vectorstore.oracle.searchAccuracy=" + OracleVectorStore.DEFAULT_SEARCH_ACCURACY) .run(context -> { VectorStore vectorStore = context.getBean(VectorStore.class); @@ -369,7 +396,8 @@ void deleteWithComplexFilterExpression() { @Test void getNativeClientTest() { this.contextRunner - .withPropertyValues("test.spring.ai.vectorstore.oracle.distanceType=COSINE", + .withPropertyValues("test.spring.ai.vectorstore.oracle.indexType=IVF", + "test.spring.ai.vectorstore.oracle.distanceType=COSINE", "test.spring.ai.vectorstore.oracle.searchAccuracy=" + OracleVectorStore.DEFAULT_SEARCH_ACCURACY) .run(context -> { OracleVectorStore vectorStore = context.getBean(OracleVectorStore.class); @@ -378,6 +406,87 @@ void getNativeClientTest() { }); } + @Test + void customIvfParametersAreApplied() { + this.contextRunner + .withPropertyValues("test.spring.ai.vectorstore.oracle.indexType=IVF", + "test.spring.ai.vectorstore.oracle.distanceType=COSINE", + "test.spring.ai.vectorstore.oracle.searchAccuracy=" + OracleVectorStore.DEFAULT_SEARCH_ACCURACY, + "test.spring.ai.vectorstore.oracle.initializeSchema=false", + "test.spring.ai.vectorstore.oracle.ivfNeighborPartitions=32", + "test.spring.ai.vectorstore.oracle.ivfSamplePerPartition=16", + "test.spring.ai.vectorstore.oracle.ivfMinVectorsPerPartition=8") + .run(context -> { + OracleVectorStore vectorStore = context.getBean(OracleVectorStore.class); + assertThat(vectorStore.getIndexType()).isEqualTo(OracleVectorStore.OracleVectorStoreIndexType.IVF); + assertThat(vectorStore.getIvfNeighborPartitions()).isEqualTo(32); + assertThat(vectorStore.getIvfSamplePerPartition()).isEqualTo(16); + assertThat(vectorStore.getIvfMinVectorsPerPartition()).isEqualTo(8); + }); + } + + @Test + void defaultIvfInitializationCreatesIndex() { + this.contextRunner + .withPropertyValues("test.spring.ai.vectorstore.oracle.indexType=IVF", + "test.spring.ai.vectorstore.oracle.distanceType=COSINE", + "test.spring.ai.vectorstore.oracle.searchAccuracy=" + OracleVectorStore.DEFAULT_SEARCH_ACCURACY, + "test.spring.ai.vectorstore.oracle.initializeSchema=true") + .run(context -> { + OracleVectorStore vectorStore = context.getBean(OracleVectorStore.class); + assertThat(isIndexPresent(context, vectorStore.getTableName())).isTrue(); + dropTable(context, vectorStore.getTableName()); + }); + } + + @Test + void hnswInitializationPath() { + this.contextRunner + .withPropertyValues("test.spring.ai.vectorstore.oracle.indexType=HNSW", + "test.spring.ai.vectorstore.oracle.distanceType=COSINE", + "test.spring.ai.vectorstore.oracle.searchAccuracy=" + OracleVectorStore.DEFAULT_SEARCH_ACCURACY, + "test.spring.ai.vectorstore.oracle.initializeSchema=true", + "test.spring.ai.vectorstore.oracle.hnswNeighbors=64", + "test.spring.ai.vectorstore.oracle.hnswEfConstruction=300") + .run(context -> { + assertThat(context.getStartupFailure()).isNull(); + OracleVectorStore vectorStore = context.getBean(OracleVectorStore.class); + assertThat(vectorStore.getIndexType()).isEqualTo(OracleVectorStore.OracleVectorStoreIndexType.HNSW); + assertThat(vectorStore.getHnswNeighbors()).isEqualTo(64); + assertThat(vectorStore.getHnswEfConstruction()).isEqualTo(300); + assertThat(isIndexPresent(context, vectorStore.getTableName())).isTrue(); + dropTable(context, vectorStore.getTableName()); + }); + } + + @Test + void invalidIvfSamplePerPartitionFailsAtStartup() { + this.contextRunner + .withPropertyValues("test.spring.ai.vectorstore.oracle.indexType=IVF", + "test.spring.ai.vectorstore.oracle.distanceType=COSINE", + "test.spring.ai.vectorstore.oracle.searchAccuracy=" + OracleVectorStore.DEFAULT_SEARCH_ACCURACY, + "test.spring.ai.vectorstore.oracle.ivfSamplePerPartition=-2") + .run(context -> { + assertThat(context).hasFailed(); + assertThat(context.getStartupFailure()).hasRootCauseInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("IVF sample per partition must be greater than 0"); + }); + } + + @Test + void invalidHnswNeighborsFailAtStartup() { + this.contextRunner + .withPropertyValues("test.spring.ai.vectorstore.oracle.indexType=HNSW", + "test.spring.ai.vectorstore.oracle.distanceType=COSINE", + "test.spring.ai.vectorstore.oracle.searchAccuracy=" + OracleVectorStore.DEFAULT_SEARCH_ACCURACY, + "test.spring.ai.vectorstore.oracle.hnswNeighbors=0") + .run(context -> { + assertThat(context).hasFailed(); + assertThat(context.getStartupFailure()).hasRootCauseInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("HNSW neighbors must be greater than 0"); + }); + } + @SpringBootConfiguration @EnableAutoConfiguration(exclude = DataSourceAutoConfiguration.class) public static class TestClient { @@ -385,18 +494,44 @@ public static class TestClient { @Value("${test.spring.ai.vectorstore.oracle.distanceType}") OracleVectorStore.OracleVectorStoreDistanceType distanceType; + @Value("${test.spring.ai.vectorstore.oracle.indexType}") + OracleVectorStore.OracleVectorStoreIndexType indexType; + @Value("${test.spring.ai.vectorstore.oracle.searchAccuracy}") int searchAccuracy; + @Value("${test.spring.ai.vectorstore.oracle.hnswNeighbors}") + int hnswNeighbors; + + @Value("${test.spring.ai.vectorstore.oracle.hnswEfConstruction}") + int hnswEfConstruction; + + @Value("${test.spring.ai.vectorstore.oracle.ivfNeighborPartitions}") + int ivfNeighborPartitions; + + @Value("${test.spring.ai.vectorstore.oracle.ivfSamplePerPartition}") + int ivfSamplePerPartition; + + @Value("${test.spring.ai.vectorstore.oracle.ivfMinVectorsPerPartition}") + int ivfMinVectorsPerPartition; + + @Value("${test.spring.ai.vectorstore.oracle.initializeSchema}") + boolean initializeSchema; + @Bean public VectorStore vectorStore(JdbcTemplate jdbcTemplate, EmbeddingModel embeddingModel) { return OracleVectorStore.builder(jdbcTemplate, embeddingModel) .tableName(OracleVectorStore.DEFAULT_TABLE_NAME) - .indexType(OracleVectorStore.OracleVectorStoreIndexType.IVF) + .indexType(this.indexType) .distanceType(this.distanceType) .dimensions(384) .searchAccuracy(this.searchAccuracy) - .initializeSchema(true) + .hnswNeighbors(this.hnswNeighbors) + .hnswEfConstruction(this.hnswEfConstruction) + .ivfNeighborPartitions(this.ivfNeighborPartitions) + .ivfSamplePerPartition(this.ivfSamplePerPartition) + .ivfMinVectorsPerPartition(this.ivfMinVectorsPerPartition) + .initializeSchema(this.initializeSchema) .removeExistingVectorStoreTable(true) .forcedNormalization(true) .build(); @@ -410,9 +545,9 @@ public JdbcTemplate myJdbcTemplate(DataSource dataSource) { @Bean public DataSourceProperties dataSourceProperties() { DataSourceProperties properties = new DataSourceProperties(); - properties.setUrl(oracle23aiContainer.getJdbcUrl()); - properties.setUsername(oracle23aiContainer.getUsername()); - properties.setPassword(oracle23aiContainer.getPassword()); + properties.setUrl(oracleContainer.getJdbcUrl()); + properties.setUsername(oracleContainer.getUsername()); + properties.setPassword(oracleContainer.getPassword()); return properties; } diff --git a/vector-stores/spring-ai-oracle-store/src/test/java/org/springframework/ai/vectorstore/oracle/OracleVectorStoreObservationIT.java b/vector-stores/spring-ai-oracle-store/src/test/java/org/springframework/ai/vectorstore/oracle/OracleVectorStoreObservationIT.java index d16b56b33b..43274083cf 100644 --- a/vector-stores/spring-ai-oracle-store/src/test/java/org/springframework/ai/vectorstore/oracle/OracleVectorStoreObservationIT.java +++ b/vector-stores/spring-ai-oracle-store/src/test/java/org/springframework/ai/vectorstore/oracle/OracleVectorStoreObservationIT.java @@ -63,12 +63,13 @@ * @author Christian Tzolov * @author Thomas Vitale * @author Eddú Meléndez + * @author Anders Swanson */ @Testcontainers public class OracleVectorStoreObservationIT { @Container - static OracleContainer oracle23aiContainer = new OracleContainer(OracleImage.DEFAULT_IMAGE) + static OracleContainer oracleContainer = new OracleContainer(OracleImage.DEFAULT_IMAGE) .withCopyFileToContainer(MountableFile.forClasspathResource("/initialize.sql"), "/container-entrypoint-initdb.d/initialize.sql") .withStartupTimeout(Duration.ofMinutes(5)) @@ -210,9 +211,9 @@ public JdbcTemplate myJdbcTemplate(DataSource dataSource) { @Bean public DataSourceProperties dataSourceProperties() { DataSourceProperties properties = new DataSourceProperties(); - properties.setUrl(oracle23aiContainer.getJdbcUrl()); - properties.setUsername(oracle23aiContainer.getUsername()); - properties.setPassword(oracle23aiContainer.getPassword()); + properties.setUrl(oracleContainer.getJdbcUrl()); + properties.setUsername(oracleContainer.getUsername()); + properties.setPassword(oracleContainer.getPassword()); return properties; }