diff --git a/core-web/apps/dotcms-ui/src/app/portlets/dot-apps/components/dot-ai-config-detail/dot-ai-config-detail.component.ts b/core-web/apps/dotcms-ui/src/app/portlets/dot-apps/components/dot-ai-config-detail/dot-ai-config-detail.component.ts index 0bf9ba4a4633..d61ff807ecd6 100644 --- a/core-web/apps/dotcms-ui/src/app/portlets/dot-apps/components/dot-ai-config-detail/dot-ai-config-detail.component.ts +++ b/core-web/apps/dotcms-ui/src/app/portlets/dot-apps/components/dot-ai-config-detail/dot-ai-config-detail.component.ts @@ -36,13 +36,13 @@ const EXAMPLE_CONFIG = { image: { provider: 'openai', apiKey: 'sk-...', - model: 'dall-e-3' + model: 'gpt-image-1' }, settings: { rolePrompt: 'You are dotCMSbot, an AI assistant to help content creators.', textPrompt: 'Use Descriptive writing style.', imagePrompt: 'Use 16:9 aspect ratio.', - imageSize: '1792x1024', + imageSize: '1024x1024', listenerIndexer: { default: 'blog,news,webPageContent' }, completionRolePrompt: 'You are a helpful assistant with a descriptive writing style.', completionTextPrompt: diff --git a/dotCMS/src/main/java/com/dotcms/ai/api/BulkEmbeddingsRunner.java b/dotCMS/src/main/java/com/dotcms/ai/api/BulkEmbeddingsRunner.java index 0d0a8528eb90..675d48caf963 100644 --- a/dotCMS/src/main/java/com/dotcms/ai/api/BulkEmbeddingsRunner.java +++ b/dotCMS/src/main/java/com/dotcms/ai/api/BulkEmbeddingsRunner.java @@ -2,6 +2,7 @@ import com.dotcms.ai.rest.forms.EmbeddingsForm; import com.dotcms.contenttype.model.field.Field; +import com.dotmarketing.beans.Host; import com.dotmarketing.business.APILocator; import com.dotmarketing.portlets.contentlet.model.Contentlet; import com.dotmarketing.util.Logger; @@ -47,12 +48,16 @@ public void run() { .stream() .filter(f -> embeddingsForm.fieldsAsList().contains(f.variable().toLowerCase())) .collect(Collectors.toList()); + final Host host = Try + .of(() -> APILocator.getHostAPI().find( + contentlet.getHost(), APILocator.systemUser(), false)) + .getOrElse(APILocator.systemHost()); // if a velocity template is passed in, use it. Otherwise, try the fields - if (!APILocator.getDotAIAPI().getEmbeddingsAPI().generateEmbeddingsForContent( + if (!APILocator.getDotAIAPI().getEmbeddingsAPI(host).generateEmbeddingsForContent( contentlet, embeddingsForm.velocityTemplate, embeddingsForm.indexName)) { - APILocator.getDotAIAPI().getEmbeddingsAPI().generateEmbeddingsForContent(contentlet, fields, embeddingsForm.indexName); + APILocator.getDotAIAPI().getEmbeddingsAPI(host).generateEmbeddingsForContent(contentlet, fields, embeddingsForm.indexName); } } catch (Exception e) { Logger.warn(this.getClass(), "unable to embed content:" + inode + " error:" + e.getMessage(), e); diff --git a/dotCMS/src/main/java/com/dotcms/ai/api/EmbeddingsAPI.java b/dotCMS/src/main/java/com/dotcms/ai/api/EmbeddingsAPI.java index db967d0df90d..0cd78b26db1b 100644 --- a/dotCMS/src/main/java/com/dotcms/ai/api/EmbeddingsAPI.java +++ b/dotCMS/src/main/java/com/dotcms/ai/api/EmbeddingsAPI.java @@ -122,6 +122,8 @@ public interface EmbeddingsAPI { */ Map> countEmbeddingsByIndex(); + boolean indexExists(String indexName); + /** * drops the dot_embeddings table */ diff --git a/dotCMS/src/main/java/com/dotcms/ai/api/EmbeddingsAPIImpl.java b/dotCMS/src/main/java/com/dotcms/ai/api/EmbeddingsAPIImpl.java index df063efee3d1..8b47e6cfb32e 100644 --- a/dotCMS/src/main/java/com/dotcms/ai/api/EmbeddingsAPIImpl.java +++ b/dotCMS/src/main/java/com/dotcms/ai/api/EmbeddingsAPIImpl.java @@ -371,6 +371,10 @@ public Tuple2> pullOrGenerateEmbeddings(final String conten @CloseDBIfOpened @Override + public boolean indexExists(final String indexName) { + return EmbeddingsFactory.impl.get().indexExists(indexName); + } + public boolean embeddingExists(final String inode, final String indexName, final String extractedText) { return EmbeddingsFactory.impl.get().embeddingExists(inode, indexName, extractedText); } diff --git a/dotCMS/src/main/java/com/dotcms/ai/api/OpenAITranslationService.java b/dotCMS/src/main/java/com/dotcms/ai/api/OpenAITranslationService.java index 7ead7cff09d0..05338ab3be48 100644 --- a/dotCMS/src/main/java/com/dotcms/ai/api/OpenAITranslationService.java +++ b/dotCMS/src/main/java/com/dotcms/ai/api/OpenAITranslationService.java @@ -1,5 +1,7 @@ package com.dotcms.ai.api; +import com.dotcms.ai.app.AppConfig; +import com.dotcms.ai.app.ConfigService; import com.dotcms.ai.util.AIUtil; import com.dotcms.ai.workflow.OpenAITranslationActionlet; import com.dotcms.contenttype.model.field.Field; @@ -9,6 +11,7 @@ import com.dotcms.translate.ServiceParameter; import com.dotcms.translate.TranslationException; import com.dotcms.translate.TranslationService; +import com.dotmarketing.beans.Host; import com.dotmarketing.business.APILocator; import com.dotmarketing.exception.DotRuntimeException; import com.dotmarketing.portlets.contentlet.model.Contentlet; @@ -134,8 +137,12 @@ public Contentlet translateContent(Contentlet contentlet, Language targetLanguag // Create AI request JSONObject promptJson = buildAIRequest(contentlet, promptData); + // Resolve host-specific AppConfig so the correct AI provider is used + final Host host = Try.of(() -> APILocator.getHostAPI().find(contentlet.getHost(), user, false)).getOrNull(); + final AppConfig appConfig = ConfigService.INSTANCE.config(host); + // Execute AI call and process response - JSONObject aiResponse = executeTranslation(promptJson); + JSONObject aiResponse = executeTranslation(promptJson, appConfig); if (aiResponse.isEmpty()) { return null; @@ -242,11 +249,11 @@ private JSONObject buildAIRequest(Contentlet contentlet, PromptData promptData) /** * Executes the translation request to the AI API. */ - private JSONObject executeTranslation(JSONObject promptJson) { + private JSONObject executeTranslation(final JSONObject promptJson, final AppConfig appConfig) { Logger.info(this.getClass(), "promptJson: " + promptJson.toString(2) + "\n\n"); final JSONObject openAIResponse = APILocator.getDotAIAPI() - .getCompletionsAPI() + .getCompletionsAPI(appConfig) .raw(promptJson, APILocator.systemUser().getUserId()); Logger.info(this.getClass(), "openAIResponse: " + openAIResponse.toString(2) + "\n\n"); diff --git a/dotCMS/src/main/java/com/dotcms/ai/client/langchain4j/LangChain4jAIClient.java b/dotCMS/src/main/java/com/dotcms/ai/client/langchain4j/LangChain4jAIClient.java index a774de9ddb2a..06fd2b541e50 100644 --- a/dotCMS/src/main/java/com/dotcms/ai/client/langchain4j/LangChain4jAIClient.java +++ b/dotCMS/src/main/java/com/dotcms/ai/client/langchain4j/LangChain4jAIClient.java @@ -22,7 +22,10 @@ import dev.langchain4j.data.image.Image; import dev.langchain4j.data.message.AiMessage; import dev.langchain4j.data.message.ChatMessage; +import dev.langchain4j.data.message.Content; +import dev.langchain4j.data.message.ImageContent; import dev.langchain4j.data.message.SystemMessage; +import dev.langchain4j.data.message.TextContent; import dev.langchain4j.data.message.UserMessage; import dev.langchain4j.data.segment.TextSegment; import dev.langchain4j.model.chat.ChatModel; @@ -379,18 +382,44 @@ static List toMessages(final JSONArray messagesArray) { for (int i = 0; i < messagesArray.length(); i++) { final JSONObject msg = messagesArray.getJSONObject(i); final String role = msg.optString(AiKeys.ROLE, AiKeys.USER).toLowerCase(); - final String content = msg.optString(AiKeys.CONTENT, ""); + final Object contentRaw = msg.opt(AiKeys.CONTENT); if ("system".equals(role)) { - messages.add(new SystemMessage(content)); + messages.add(new SystemMessage(contentRaw != null ? contentRaw.toString() : "")); } else if ("assistant".equals(role)) { - messages.add(new AiMessage(content)); + messages.add(new AiMessage(contentRaw != null ? contentRaw.toString() : "")); + } else if (contentRaw instanceof JSONArray) { + messages.add(toMultimodalUserMessage((JSONArray) contentRaw)); } else { - messages.add(new UserMessage(content)); + messages.add(new UserMessage(contentRaw != null ? contentRaw.toString() : "")); } } return messages; } + static UserMessage toMultimodalUserMessage(final JSONArray contentParts) { + final List parts = new ArrayList<>(); + for (int i = 0; i < contentParts.length(); i++) { + final JSONObject part = contentParts.getJSONObject(i); + parts.add("image_url".equals(part.optString("type", "text")) + ? toImageContent(part) + : TextContent.from(part.optString("text", ""))); + } + return UserMessage.from(parts); + } + + private static Content toImageContent(final JSONObject part) { + final JSONObject imageUrlObj = part.optJSONObject("image_url"); + final String url = imageUrlObj != null ? imageUrlObj.optString("url", "") : part.optString("image_url", ""); + if (url.startsWith("data:")) { + final int semicolon = url.indexOf(';'); + final int comma = url.indexOf(','); + if (semicolon > 0 && comma > semicolon) { + return ImageContent.from(url.substring(comma + 1), url.substring(5, semicolon)); + } + } + return ImageContent.from(url); + } + static String toChatResponseJson(final ChatResponse response) { final JSONObject message = new JSONObject(); message.put(AiKeys.ROLE, "assistant"); diff --git a/dotCMS/src/main/java/com/dotcms/ai/db/EmbeddingsFactory.java b/dotCMS/src/main/java/com/dotcms/ai/db/EmbeddingsFactory.java index 2b0911520e16..8d9525f47fcb 100644 --- a/dotCMS/src/main/java/com/dotcms/ai/db/EmbeddingsFactory.java +++ b/dotCMS/src/main/java/com/dotcms/ai/db/EmbeddingsFactory.java @@ -183,6 +183,16 @@ public Tuple3> searchExistingEmbeddings(final Strin * @param extractedText the text to check * @return true if embeddings exist, false otherwise */ + public boolean indexExists(final String indexName) { + try (final Connection conn = getPGVectorConnection(); + final PreparedStatement statement = conn.prepareStatement(EmbeddingsSQL.INDEX_EXISTS)) { + statement.setObject(1, indexName); + return statement.executeQuery().next(); + } catch (SQLException e) { + throw new DotRuntimeException(e); + } + } + public boolean embeddingExists(final String inode, final String indexName, final String extractedText) { try (final Connection conn = getPGVectorConnection(); final PreparedStatement statement = diff --git a/dotCMS/src/main/java/com/dotcms/ai/db/EmbeddingsSQL.java b/dotCMS/src/main/java/com/dotcms/ai/db/EmbeddingsSQL.java index 3e94bfe5f5f4..9ec68cb5b39d 100644 --- a/dotCMS/src/main/java/com/dotcms/ai/db/EmbeddingsSQL.java +++ b/dotCMS/src/main/java/com/dotcms/ai/db/EmbeddingsSQL.java @@ -125,6 +125,9 @@ class EmbeddingsSQL { " index_name " + ") data;"; + static final String INDEX_EXISTS = + "SELECT 1 FROM dot_embeddings WHERE index_name = ? LIMIT 1"; + private EmbeddingsSQL() { } diff --git a/dotCMS/src/main/java/com/dotcms/ai/listener/EmbeddingContentListener.java b/dotCMS/src/main/java/com/dotcms/ai/listener/EmbeddingContentListener.java index 135310ddbf5b..57f8a55c738c 100644 --- a/dotCMS/src/main/java/com/dotcms/ai/listener/EmbeddingContentListener.java +++ b/dotCMS/src/main/java/com/dotcms/ai/listener/EmbeddingContentListener.java @@ -1,5 +1,6 @@ package com.dotcms.ai.listener; +import com.dotcms.ai.api.EmbeddingsAPI; import com.dotcms.ai.app.AppConfig; import com.dotcms.ai.app.ConfigService; import com.dotcms.ai.db.EmbeddingsDTO; @@ -78,12 +79,14 @@ public void onDeleted(final ContentletDeletedEvent contentletDeleted deleteFromIndexes(contentlet); } - private AppConfig getAppConfig(final String hostId) { - final Host host = Try + private Host resolveHost(final String hostId) { + return Try .of(() -> APILocator.getHostAPI().find(hostId, APILocator.systemUser(), false)) .getOrElse(APILocator.systemHost()); + } - final AppConfig appConfig = ConfigService.INSTANCE.config(host); + private AppConfig getAppConfig(final String hostId) { + final AppConfig appConfig = ConfigService.INSTANCE.config(resolveHost(hostId)); if (!appConfig.isEnabled()) { appConfig.debugLogger( getClass(), @@ -121,17 +124,18 @@ private void addToIndexesIfNeeded(final Contentlet contentlet) { } try { + final Host host = resolveHost(contentlet.getHost()); final JSONObject config = getConfigJson(contentlet.getHost()); for (final Entry entry : (Set>) config.entrySet()) { final String indexName = entry.getKey(); + final EmbeddingsAPI embeddingsAPI = APILocator.getDotAIAPI().getEmbeddingsAPI(host); final Map> typesAndFields = - APILocator.getDotAIAPI().getEmbeddingsAPI().parseTypesAndFields((String) entry.getValue()); + embeddingsAPI.parseTypesAndFields((String) entry.getValue()); typesAndFields.entrySet() .stream() .filter(typeFields -> contentType.equalsIgnoreCase(typeFields.getKey())) - .forEach(e -> APILocator.getDotAIAPI().getEmbeddingsAPI() - .generateEmbeddingsForContent( + .forEach(e -> embeddingsAPI.generateEmbeddingsForContent( contentlet, e.getValue(), indexName)); @@ -149,6 +153,7 @@ private void addToIndexesIfNeeded(final Contentlet contentlet) { @WrapInTransaction private void deleteFromIndexes(final Contentlet contentlet) { try { + final Host host = resolveHost(contentlet.getHost()); getConfigJson(contentlet.getHost()); final EmbeddingsDTO dto = new EmbeddingsDTO.Builder() @@ -156,7 +161,7 @@ private void deleteFromIndexes(final Contentlet contentlet) { .withLanguage(contentlet.getLanguageId()) .withIndexName(ALL_INDICES) .build(); - APILocator.getDotAIAPI().getEmbeddingsAPI().deleteEmbedding(dto); + APILocator.getDotAIAPI().getEmbeddingsAPI(host).deleteEmbedding(dto); } catch (final Exception e) { Logger.error(getClass(), "Error deleting content from embeddings index: " + e.getMessage(), e); throw new DotRuntimeException("Error deleting content from embeddings index: " + e.getMessage(), e); diff --git a/dotCMS/src/main/java/com/dotcms/ai/rest/SearchResource.java b/dotCMS/src/main/java/com/dotcms/ai/rest/SearchResource.java index 20b49430db17..f2777a1b5644 100644 --- a/dotCMS/src/main/java/com/dotcms/ai/rest/SearchResource.java +++ b/dotCMS/src/main/java/com/dotcms/ai/rest/SearchResource.java @@ -126,10 +126,18 @@ public final Response searchByPost(@Context final HttpServletRequest request, .init() .getUser(); + final Host host = WebAPILocator.getHostWebAPI().getCurrentHostNoThrow(request); + if (!APILocator.getDotAIAPI().getEmbeddingsAPI(host).indexExists(form.indexName)) { + return Response.status(Response.Status.NOT_FOUND) + .entity(Map.of(AiKeys.ERROR, "Index '" + form.indexName + "' not found")) + .type(MediaType.APPLICATION_JSON) + .build(); + } + final EmbeddingsDTO searcher = EmbeddingsDTO.from(form).withUser(user).build(); return Response.ok( - APILocator.getDotAIAPI().getEmbeddingsAPI().searchForContent(searcher).toString(), + APILocator.getDotAIAPI().getEmbeddingsAPI(host).searchForContent(searcher).toString(), MediaType.APPLICATION_JSON).build(); } diff --git a/dotCMS/src/main/java/com/dotcms/ai/workflow/OpenAIAutoTagRunner.java b/dotCMS/src/main/java/com/dotcms/ai/workflow/OpenAIAutoTagRunner.java index 202497022932..8616c7353558 100644 --- a/dotCMS/src/main/java/com/dotcms/ai/workflow/OpenAIAutoTagRunner.java +++ b/dotCMS/src/main/java/com/dotcms/ai/workflow/OpenAIAutoTagRunner.java @@ -1,7 +1,10 @@ package com.dotcms.ai.workflow; +import com.dotcms.ai.app.AppConfig; +import com.dotcms.ai.app.ConfigService; import com.dotcms.ai.util.ContentToStringUtil; import com.dotcms.ai.util.VelocityContextFactory; +import com.dotmarketing.beans.Host; import com.dotcms.api.system.event.message.MessageSeverity; import com.dotcms.api.system.event.message.MessageType; import com.dotcms.api.system.event.message.SystemMessageEventUtil; @@ -135,7 +138,9 @@ private String openAIRequest(final Contentlet workingContentlet, final String co final String parsedSystemPrompt = VelocityUtil.eval(systemPrompt, ctx); final String parsedContentPrompt = VelocityUtil.eval(contentToTag, ctx); - final JSONObject openAIResponse = APILocator.getDotAIAPI().getCompletionsAPI() + final Host host = Try.of(() -> APILocator.getHostAPI().find(workingContentlet.getHost(), user, false)).getOrNull(); + final AppConfig appConfig = ConfigService.INSTANCE.config(host); + final JSONObject openAIResponse = APILocator.getDotAIAPI().getCompletionsAPI(appConfig) .prompt(parsedSystemPrompt, parsedContentPrompt, null, 0f, 2000, user.getUserId()); return openAIResponse.getJSONArray("choices").getJSONObject(0).getJSONObject("message").getString("content"); diff --git a/dotCMS/src/main/java/com/dotcms/ai/workflow/OpenAIContentPromptRunner.java b/dotCMS/src/main/java/com/dotcms/ai/workflow/OpenAIContentPromptRunner.java index a5f6f3d8f138..1a0cd6ce6d53 100644 --- a/dotCMS/src/main/java/com/dotcms/ai/workflow/OpenAIContentPromptRunner.java +++ b/dotCMS/src/main/java/com/dotcms/ai/workflow/OpenAIContentPromptRunner.java @@ -1,7 +1,10 @@ package com.dotcms.ai.workflow; +import com.dotcms.ai.app.AppConfig; +import com.dotcms.ai.app.ConfigService; import com.dotcms.ai.util.ContentToStringUtil; import com.dotcms.ai.util.VelocityContextFactory; +import com.dotmarketing.beans.Host; import com.dotcms.api.system.event.Payload; import com.dotcms.api.system.event.SystemEventType; import com.dotcms.api.system.event.message.MessageSeverity; @@ -130,8 +133,10 @@ private String openAIRequest(final Contentlet workingContentlet) throws Exceptio final Context ctx = VelocityContextFactory.getMockContext(workingContentlet, user); final String parsedPrompt = VelocityUtil.eval(prompt, ctx); + final Host host = Try.of(() -> APILocator.getHostAPI().find(workingContentlet.getHost(), user, false)).getOrNull(); + final AppConfig appConfig = ConfigService.INSTANCE.config(host); final JSONObject openAIResponse = APILocator.getDotAIAPI() - .getCompletionsAPI() + .getCompletionsAPI(appConfig) .raw(buildRequest(parsedPrompt), user.getUserId()); try { diff --git a/dotCMS/src/main/webapp/WEB-INF/jsp/dotai/render.jsp b/dotCMS/src/main/webapp/WEB-INF/jsp/dotai/render.jsp index d9b6ae7d6462..559637e06850 100644 --- a/dotCMS/src/main/webapp/WEB-INF/jsp/dotai/render.jsp +++ b/dotCMS/src/main/webapp/WEB-INF/jsp/dotai/render.jsp @@ -59,8 +59,8 @@ Content index to search: - + @@ -98,7 +98,7 @@ - @@ -356,20 +356,7 @@ placeholder="Image prompt"> - - - Size: - - - - - +
diff --git a/dotCMS/src/main/webapp/html/portlet/ext/dotai/dotai.js b/dotCMS/src/main/webapp/html/portlet/ext/dotai/dotai.js index decab1a0afe1..d69414b5a31a 100644 --- a/dotCMS/src/main/webapp/html/portlet/ext/dotai/dotai.js +++ b/dotCMS/src/main/webapp/html/portlet/ext/dotai/dotai.js @@ -127,26 +127,28 @@ const writeIndexesToDropdowns = async () => { const writeModelToDropdown = async () => { const modelName = document.getElementById("modelName"); - let options = modelName.getElementsByTagName('option'); + modelName.innerHTML = ''; - for (i = 1; i < options.length; i++) { - indexName.removeChild(options[i]); + let models = []; + try { + const providerConfig = JSON.parse(dotAiState.config.providerConfig); + const chatModel = providerConfig?.chat?.model ?? ''; + models = chatModel.split(',').map(m => m.trim()).filter(m => m.length > 0); + } catch (e) { + console.error('[DotAI] writeModelToDropdown: providerConfig missing or not valid JSON', e); } - for (i = 0; i < dotAiState.config.availableModels.length; i++) { - if (dotAiState.config.availableModels[i].type !== 'TEXT') { - continue; - } - - const newOption = document.createElement("option"); - newOption.value = dotAiState.config.availableModels[i].name; - newOption.text = `${dotAiState.config.availableModels[i].name}` - if (dotAiState.config.availableModels[i].current) { - newOption.selected = true; - newOption.text = `${dotAiState.config.availableModels[i].name} (default)` - } - modelName.appendChild(newOption); + if (models.length === 0) { + console.warn('[DotAI] writeModelToDropdown: models is empty, dropdown will not be populated'); } + + models.forEach((model, index) => { + const opt = document.createElement("option"); + opt.value = model; + opt.text = index === 0 ? `${model} (default)` : model; + if (index === 0) opt.selected = true; + modelName.appendChild(opt); + }); }; @@ -566,14 +568,14 @@ const doImageJsonDebounced = async () => { const temp = json.response; const width = formData.size.split("x")[0]; const height = formData.size.split("x")[1]; - const jsonString=JSON.stringify(json, 2); + const jsonString = JSON.stringify(json, null, 2); const rewrittenPrompt = json.revised_prompt; const imageTemplate =` -
+
- +

 
-
+
OpenAI Prompt (Rewritten):
${rewrittenPrompt}
-
- JSON Response:
${jsonString} +
+ JSON Response: + 
${jsonString}
@@ -646,6 +649,9 @@ const clearSaveMessage =() =>{ const doSearchChatJson = () => { + if (!document.getElementById("chatForm").reportValidity()) { + return; + } document.getElementById("submitChat").style.display = "none"; document.getElementById("loaderChat").style.display = "block"; setTimeout(function () { diff --git a/dotCMS/src/test/java/com/dotcms/ai/client/langchain4j/LangChain4jAIClientTest.java b/dotCMS/src/test/java/com/dotcms/ai/client/langchain4j/LangChain4jAIClientTest.java index 4059b39dfd4e..e35552923a74 100644 --- a/dotCMS/src/test/java/com/dotcms/ai/client/langchain4j/LangChain4jAIClientTest.java +++ b/dotCMS/src/test/java/com/dotcms/ai/client/langchain4j/LangChain4jAIClientTest.java @@ -12,7 +12,10 @@ import dev.langchain4j.data.image.Image; import dev.langchain4j.data.message.AiMessage; import dev.langchain4j.data.message.ChatMessage; +import dev.langchain4j.data.message.Content; +import dev.langchain4j.data.message.ImageContent; import dev.langchain4j.data.message.SystemMessage; +import dev.langchain4j.data.message.TextContent; import dev.langchain4j.data.message.UserMessage; import dev.langchain4j.model.chat.response.ChatResponse; import org.junit.Test; @@ -99,6 +102,122 @@ public void test_toMessages_multipleRoles_preservesOrder() { assertTrue(messages.get(2) instanceof AiMessage); } + @Test + public void test_toMessages_multimodalContentArray_producesUserMessageWithImageAndText() { + final JSONObject imagePart = new JSONObject(); + imagePart.put("type", "image_url"); + final JSONObject imageUrl = new JSONObject(); + imageUrl.put("url", "data:image/webp;base64,ABC123"); + imagePart.put("image_url", imageUrl); + + final JSONObject textPart = new JSONObject(); + textPart.put("type", "text"); + textPart.put("text", "Describe this image"); + + final JSONArray contentArray = new JSONArray(); + contentArray.put(textPart); + contentArray.put(imagePart); + + final JSONObject msg = new JSONObject(); + msg.put(AiKeys.ROLE, "user"); + msg.put(AiKeys.CONTENT, contentArray); + + final JSONArray messages = new JSONArray(); + messages.put(msg); + + final List result = LangChain4jAIClient.toMessages(messages); + + assertEquals(1, result.size()); + assertTrue(result.get(0) instanceof UserMessage); + final UserMessage userMessage = (UserMessage) result.get(0); + assertEquals(2, userMessage.contents().size()); + assertTrue(userMessage.contents().get(0) instanceof TextContent); + assertTrue(userMessage.contents().get(1) instanceof ImageContent); + } + + @Test + public void test_toMessages_contentArrayNotTreatedAsString() { + // Regression: before the fix, JSONArray.optString() would serialize the array + // to its toString() representation and pass it as a plain text message, + // so the model would never see the actual image data. + final JSONObject imagePart = new JSONObject(); + imagePart.put("type", "image_url"); + final JSONObject imageUrl = new JSONObject(); + imageUrl.put("url", "data:image/png;base64,iVBORw0KGgo="); + imagePart.put("image_url", imageUrl); + + final JSONArray contentArray = new JSONArray(); + contentArray.put(imagePart); + + final JSONObject msg = new JSONObject(); + msg.put(AiKeys.ROLE, "user"); + msg.put(AiKeys.CONTENT, contentArray); + + final JSONArray messages = new JSONArray(); + messages.put(msg); + + final List result = LangChain4jAIClient.toMessages(messages); + + final UserMessage userMessage = (UserMessage) result.get(0); + // Must have structured content, not a plain text message + final List contents = userMessage.contents(); + assertEquals(1, contents.size()); + assertTrue(contents.get(0) instanceof ImageContent); + } + + @Test + public void test_toMultimodalUserMessage_dataUri_extractsMimeTypeAndBase64() { + final JSONObject imagePart = new JSONObject(); + imagePart.put("type", "image_url"); + final JSONObject imageUrl = new JSONObject(); + imageUrl.put("url", "data:image/webp;base64,ABC123=="); + imagePart.put("image_url", imageUrl); + + final JSONArray contentArray = new JSONArray(); + contentArray.put(imagePart); + + final UserMessage msg = LangChain4jAIClient.toMultimodalUserMessage(contentArray); + + assertEquals(1, msg.contents().size()); + final ImageContent imageContent = (ImageContent) msg.contents().get(0); + assertEquals("ABC123==", imageContent.image().base64Data()); + assertEquals("image/webp", imageContent.image().mimeType()); + } + + @Test + public void test_toMultimodalUserMessage_plainUrl_producesImageContentWithUrl() { + final JSONObject imagePart = new JSONObject(); + imagePart.put("type", "image_url"); + final JSONObject imageUrl = new JSONObject(); + imageUrl.put("url", "https://example.com/photo.jpg"); + imagePart.put("image_url", imageUrl); + + final JSONArray contentArray = new JSONArray(); + contentArray.put(imagePart); + + final UserMessage msg = LangChain4jAIClient.toMultimodalUserMessage(contentArray); + + assertEquals(1, msg.contents().size()); + final ImageContent imageContent = (ImageContent) msg.contents().get(0); + assertEquals("https://example.com/photo.jpg", imageContent.image().url().toString()); + } + + @Test + public void test_toMultimodalUserMessage_textType_producesTextContent() { + final JSONObject textPart = new JSONObject(); + textPart.put("type", "text"); + textPart.put("text", "What is in this image?"); + + final JSONArray contentArray = new JSONArray(); + contentArray.put(textPart); + + final UserMessage msg = LangChain4jAIClient.toMultimodalUserMessage(contentArray); + + assertEquals(1, msg.contents().size()); + assertTrue(msg.contents().get(0) instanceof TextContent); + assertEquals("What is in this image?", ((TextContent) msg.contents().get(0)).text()); + } + @Test public void test_toChatResponseJson_correctStructure() { final AiMessage aiMessage = new AiMessage("Test response content"); diff --git a/dotCMS/src/test/java/com/dotcms/ai/workflow/OpenAIAutoTagRunnerTest.java b/dotCMS/src/test/java/com/dotcms/ai/workflow/OpenAIAutoTagRunnerTest.java new file mode 100644 index 000000000000..030101031aa3 --- /dev/null +++ b/dotCMS/src/test/java/com/dotcms/ai/workflow/OpenAIAutoTagRunnerTest.java @@ -0,0 +1,41 @@ +package com.dotcms.ai.workflow; + +import com.dotmarketing.portlets.contentlet.model.Contentlet; +import com.liferay.portal.model.User; +import org.junit.Test; + +import static org.junit.Assert.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class OpenAIAutoTagRunnerTest { + + @Test + public void test_constructor_missingIdentifier_throws() { + final Contentlet contentlet = mock(Contentlet.class); + when(contentlet.getIdentifier()).thenReturn(null); + + assertThrows( + IllegalArgumentException.class, + () -> new OpenAIAutoTagRunner(contentlet, mock(User.class), true, false)); + } + + @Test + public void test_constructor_emptyIdentifier_throws() { + final Contentlet contentlet = mock(Contentlet.class); + when(contentlet.getIdentifier()).thenReturn(" "); + + assertThrows( + IllegalArgumentException.class, + () -> new OpenAIAutoTagRunner(contentlet, mock(User.class), true, false)); + } + + @Test + public void test_constructor_validIdentifier_succeeds() { + final Contentlet contentlet = mock(Contentlet.class); + when(contentlet.getIdentifier()).thenReturn("abc-123"); + + new OpenAIAutoTagRunner(contentlet, mock(User.class), true, false); + } + +} diff --git a/dotCMS/src/test/java/com/dotcms/ai/workflow/OpenAIContentPromptRunnerTest.java b/dotCMS/src/test/java/com/dotcms/ai/workflow/OpenAIContentPromptRunnerTest.java new file mode 100644 index 000000000000..f936f36c366f --- /dev/null +++ b/dotCMS/src/test/java/com/dotcms/ai/workflow/OpenAIContentPromptRunnerTest.java @@ -0,0 +1,41 @@ +package com.dotcms.ai.workflow; + +import com.dotmarketing.portlets.contentlet.model.Contentlet; +import com.liferay.portal.model.User; +import org.junit.Test; + +import static org.junit.Assert.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class OpenAIContentPromptRunnerTest { + + @Test + public void test_constructor_missingIdentifier_throws() { + final Contentlet contentlet = mock(Contentlet.class); + when(contentlet.getIdentifier()).thenReturn(null); + + assertThrows( + IllegalArgumentException.class, + () -> new OpenAIContentPromptRunner(contentlet, mock(User.class), "prompt", true, "field")); + } + + @Test + public void test_constructor_emptyIdentifier_throws() { + final Contentlet contentlet = mock(Contentlet.class); + when(contentlet.getIdentifier()).thenReturn(" "); + + assertThrows( + IllegalArgumentException.class, + () -> new OpenAIContentPromptRunner(contentlet, mock(User.class), "prompt", true, "field")); + } + + @Test + public void test_constructor_validIdentifier_succeeds() { + final Contentlet contentlet = mock(Contentlet.class); + when(contentlet.getIdentifier()).thenReturn("abc-123"); + + new OpenAIContentPromptRunner(contentlet, mock(User.class), "prompt", true, "field"); + } + +} diff --git a/dotcms-integration/src/test/java/com/dotcms/ai/api/BulkEmbeddingsRunnerTest.java b/dotcms-integration/src/test/java/com/dotcms/ai/api/BulkEmbeddingsRunnerTest.java new file mode 100644 index 000000000000..95ad92e2e9d1 --- /dev/null +++ b/dotcms-integration/src/test/java/com/dotcms/ai/api/BulkEmbeddingsRunnerTest.java @@ -0,0 +1,126 @@ +package com.dotcms.ai.api; + +import com.dotcms.ai.AiTest; +import com.dotcms.contenttype.model.type.ContentType; +import com.dotcms.datagen.SiteDataGen; +import com.dotcms.datagen.TestDataUtils; +import com.dotcms.ai.rest.forms.EmbeddingsForm; +import com.dotcms.util.IntegrationTestInitService; +import com.dotcms.util.network.IPUtils; +import com.dotmarketing.beans.Host; +import com.dotmarketing.business.APILocator; +import com.dotmarketing.exception.DotDataException; +import com.dotmarketing.exception.DotSecurityException; +import com.dotmarketing.portlets.contentlet.model.Contentlet; +import com.dotmarketing.portlets.languagesmanager.business.LanguageAPI; +import com.github.tomakehurst.wiremock.WireMockServer; +import com.liferay.portal.model.User; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.List; + +import static com.dotmarketing.util.ThreadUtils.sleep; +import static org.junit.Assert.assertTrue; + +/** + * Integration tests for {@link BulkEmbeddingsRunner}. + * + *

The setup intentionally configures AI secrets only on the site host, not on System Host. + * This validates that the runner resolves the host per-contentlet and uses the correct + * site config rather than falling back to System Host. + */ +public class BulkEmbeddingsRunnerTest { + + private static final int MAX_ATTEMPTS = 30; + + private static User user; + private static Host host; + private static LanguageAPI languageApi; + private static WireMockServer wireMockServer; + private static ContentType blogContentType; + private static Contentlet contentlet; + + @BeforeClass + public static void beforeClass() throws Exception { + IntegrationTestInitService.getInstance().init(); + IPUtils.disabledIpPrivateSubnet(true); + user = APILocator.getUserAPI().getSystemUser(); + languageApi = APILocator.getLanguageAPI(); + host = new SiteDataGen().nextPersisted(); + wireMockServer = AiTest.prepareWireMock(); + // Configure AI only on the site host — intentionally NOT on System Host + AiTest.aiAppSecretsWithProviderConfig(host, AiTest.providerConfigJson(AiTest.PORT, AiTest.MODEL)); + } + + @AfterClass + public static void afterClass() throws Exception { + wireMockServer.stop(); + IPUtils.disabledIpPrivateSubnet(false); + if (contentlet != null) { + try { + APILocator.getContentletAPI().archive(contentlet, user, false); + APILocator.getContentletAPI().delete(contentlet, user, false); + } catch (DotDataException | DotSecurityException e) { + // ignore + } + } + if (blogContentType != null) { + try { + APILocator.getContentTypeAPI(user).delete(blogContentType); + } catch (DotDataException | DotSecurityException e) { + // ignore + } + } + AiTest.removeAiAppSecrets(host); + } + + /** + * Given a contentlet published on a site that has AI configured, + * and System Host does NOT have AI configured, + * When BulkEmbeddingsRunner processes that contentlet, + * Then embeddings should be generated using the site config. + */ + @Test + public void test_run_generatesEmbeddings_withSiteOnlyConfig() throws Exception { + DotAIAPIFacadeImpl.setDefaultEmbeddingsAPIProvider( + new DotAIAPIFacadeImpl.DefaultEmbeddingsAPIProvider()); + + blogContentType = TestDataUtils.getBlogLikeContentType("blog", host); + final String text = "BulkEmbeddingsRunner should resolve the host from the contentlet " + + "and use the site config rather than falling back to System Host."; + contentlet = TestDataUtils.withEmbeddings( + true, + host, + languageApi.getDefaultLanguage().getId(), + blogContentType.id(), + text); + APILocator.getContentletAPI().publish(contentlet, user, false); + + final EmbeddingsForm form = new EmbeddingsForm.Builder() + .indexName("default") + .build(); + new BulkEmbeddingsRunner(List.of(contentlet.getInode()), form).run(); + + assertTrue( + "Expected embeddings after BulkEmbeddingsRunner.run() with site-only AI config", + waitForEmbeddings(contentlet, text)); + } + + private static boolean waitForEmbeddings(final Contentlet contentlet, final String text) { + int count = 0; + boolean exists = APILocator.getDotAIAPI().getEmbeddingsAPI(host) + .embeddingExists(contentlet.getInode(), "default", text); + while (!exists) { + if (count++ > MAX_ATTEMPTS) { + break; + } + sleep(500); + exists = APILocator.getDotAIAPI().getEmbeddingsAPI(host) + .embeddingExists(contentlet.getInode(), "default", text); + } + return exists; + } + +} diff --git a/dotcms-integration/src/test/java/com/dotcms/ai/listener/EmbeddingContentListenerTest.java b/dotcms-integration/src/test/java/com/dotcms/ai/listener/EmbeddingContentListenerTest.java index 68028c0a13f4..91464e3519ab 100644 --- a/dotcms-integration/src/test/java/com/dotcms/ai/listener/EmbeddingContentListenerTest.java +++ b/dotcms-integration/src/test/java/com/dotcms/ai/listener/EmbeddingContentListenerTest.java @@ -81,6 +81,49 @@ public static void afterClass() throws Exception { removeDotAISecrets(); } + /** + * Given a contentlet published on a site that has AI configured, + * and System Host does NOT have AI configured, + * When the contentlet is published, + * Then the EmbeddingContentListener should resolve the site config from the contentlet's host + * and generate embeddings successfully. + */ + @Test + public void test_onPublish_withSiteOnlyConfig() throws Exception { + DotAIAPIFacadeImpl.setDefaultEmbeddingsAPIProvider( + new DotAIAPIFacadeImpl.DefaultEmbeddingsAPIProvider()); + + final Host siteOnlyHost = new SiteDataGen().nextPersisted(); + AiTest.aiAppSecretsWithProviderConfig(siteOnlyHost, AiTest.providerConfigJson(AiTest.PORT, AiTest.MODEL)); + ContentType ct = null; + Contentlet siteContentlet = null; + + try { + // Temporarily remove System Host config so only the site has AI configured + AiTest.removeAiAppSecrets(APILocator.systemHost()); + + ct = TestDataUtils.getBlogLikeContentType("blog", siteOnlyHost); + contentTypes.add(ct); + final String text = "EmbeddingContentListener should resolve the host from the contentlet " + + "and use the site config rather than falling back to System Host."; + siteContentlet = TestDataUtils.withEmbeddings( + true, + siteOnlyHost, + languageApi.getDefaultLanguage().getId(), + ct.id(), + text); + contentlets.add(siteContentlet); + contentletApi.publish(siteContentlet, user, false); + + assertTrue(waitForEmbeddings(siteContentlet, text, true)); + } finally { + // Restore System Host config for subsequent tests + AiTest.aiAppSecretsWithProviderConfig( + APILocator.systemHost(), AiTest.providerConfigJson(AiTest.PORT, AiTest.MODEL)); + AiTest.removeAiAppSecrets(siteOnlyHost); + } + } + /** * Given a ContentType and a Contentlet of that type with some text, * When the Contentlet is published,