diff --git a/AGENTS.md b/AGENTS.md index 60c2397..2978507 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,4 +16,6 @@ **Don't write boilerplate that can be generated.** Use Lombok (`@Data`, `@Builder`, `@RequiredArgsConstructor`), MapStruct for mappers, and IDE generation for equals/hashCode. If it's mechanical, don't type it by hand. **For records with many fields, prefer `@Builder` over positional `new` — the named setter style improves readability at every call site.** A 6-argument `new Xxx(a, b, c, d, e, f)` forces readers to count positions; `.name(a).type(b)...` does not. +**Prefer imports over fully-qualified names for JDK and framework classes.** Import the class at the top and use its short name in code — `Objects::nonNull`, not `java.util.Objects::nonNull`. The exception is when a project class shares the same simple name as a JDK or framework class you need to use. In that case, qualify the JDK/framework class with its full package path everywhere — even if the conflicting project class isn't imported in the current file. For example, use `javax.sql.DataSource` fully-qualified because `io.github.malonetalk.entity.Datasource` exists in the project and the two names are easily confused. + **Write code for the next reader, not for the compiler.** The compiler can parse anything. A human shouldn't have to. Choose names that reveal intent, structure code in small logical steps, and prefer clarity over cleverness. If a line makes you pause — it will make someone else pause too. diff --git a/data-agent-backend/pom.xml b/data-agent-backend/pom.xml index b72d477..d297164 100644 --- a/data-agent-backend/pom.xml +++ b/data-agent-backend/pom.xml @@ -72,7 +72,7 @@ 1.0.11 3.4.0 1.28.0 - 4.0.0 + 4.0.1 2.1.1 1.18.42 1.6.3 diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/agent/datasource/SchemaReader.java b/data-agent-backend/src/main/java/io/github/malonetalk/agent/datasource/SchemaReader.java index 91ab643..7503361 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/agent/datasource/SchemaReader.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/agent/datasource/SchemaReader.java @@ -25,8 +25,12 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Collection; import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Set; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -63,6 +67,36 @@ public List getTableSchema(Datasource datasource, String tab } } + public Map> getTableColumnNames( + Datasource datasource, Collection tableNames) { + if (tableNames == null || tableNames.isEmpty()) { + return Map.of(); + } + Map> columnNamesByTable = new LinkedHashMap<>(); + for (String tableName : tableNames) { + columnNamesByTable.putIfAbsent(tableName, new LinkedHashSet<>()); + } + + javax.sql.DataSource ds = dynamicDataSourceManager.getOrCreateDataSource(datasource); + + try (Connection conn = ds.getConnection()) { + DatabaseMetaData metaData = conn.getMetaData(); + try (ResultSet rs = + metaData.getColumns(conn.getCatalog(), conn.getSchema(), "%", null)) { + while (rs.next()) { + Set columnNames = columnNamesByTable.get(rs.getString("TABLE_NAME")); + if (columnNames != null) { + columnNames.add(rs.getString("COLUMN_NAME")); + } + } + } + return columnNamesByTable; + } catch (SQLException e) { + log.error("Failed to read column names for tables: {}", e.getMessage(), e); + throw new SchemaReadException("Failed to read column names: " + e.getMessage(), e); + } + } + private List getTables(Connection conn) throws SQLException { List tables = new ArrayList<>(); DatabaseMetaData metaData = conn.getMetaData(); diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/agent/tools/GetDomainsTool.java b/data-agent-backend/src/main/java/io/github/malonetalk/agent/tools/GetDomainsTool.java index c53fcf0..792c359 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/agent/tools/GetDomainsTool.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/agent/tools/GetDomainsTool.java @@ -18,7 +18,7 @@ package io.github.malonetalk.agent.tools; import io.agentscope.core.tool.Tool; -import io.github.malonetalk.service.DomainService; +import io.github.malonetalk.service.semantic.DomainService; import java.util.List; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/controller/DomainController.java b/data-agent-backend/src/main/java/io/github/malonetalk/controller/DomainController.java index 3f905c1..f228756 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/controller/DomainController.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/controller/DomainController.java @@ -23,7 +23,7 @@ import io.github.malonetalk.dto.DomainUpdateRequest; import io.github.malonetalk.dto.pagination.PageResponse; import io.github.malonetalk.entity.DomainInfo; -import io.github.malonetalk.service.DomainService; +import io.github.malonetalk.service.semantic.DomainService; import jakarta.validation.Valid; import lombok.RequiredArgsConstructor; import org.springframework.web.bind.annotation.DeleteMapping; diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/controller/TableRelationWorkspaceController.java b/data-agent-backend/src/main/java/io/github/malonetalk/controller/TableRelationWorkspaceController.java new file mode 100644 index 0000000..c2827a7 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/controller/TableRelationWorkspaceController.java @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.controller; + +import io.github.malonetalk.common.Result; +import io.github.malonetalk.dto.semantic.RelationWorkspacePageQuery; +import io.github.malonetalk.dto.semantic.RelationWorkspaceResponse; +import io.github.malonetalk.service.semantic.relation.RelationSemanticService; +import jakarta.validation.Valid; +import lombok.RequiredArgsConstructor; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping("/api/semantic/tables/relations/workspace") +@RequiredArgsConstructor +public class TableRelationWorkspaceController { + + private final RelationSemanticService relationSemanticService; + + @GetMapping + public Result getWorkspace(@Valid RelationWorkspacePageQuery query) { + return Result.success(relationSemanticService.getRelationWorkspace(query)); + } +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/controller/TableSemanticSyncController.java b/data-agent-backend/src/main/java/io/github/malonetalk/controller/TableSemanticSyncController.java new file mode 100644 index 0000000..e155adb --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/controller/TableSemanticSyncController.java @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.controller; + +import io.github.malonetalk.common.Result; +import io.github.malonetalk.dto.pagination.PageResponse; +import io.github.malonetalk.dto.semantic.PhysicalTableCandidatePageQuery; +import io.github.malonetalk.dto.semantic.PhysicalTableCandidateResponse; +import io.github.malonetalk.dto.semantic.RefreshPhysicalStatusRequest; +import io.github.malonetalk.dto.semantic.SyncTableSemanticsRequest; +import io.github.malonetalk.dto.semantic.SyncTableSemanticsResponse; +import io.github.malonetalk.service.semantic.sync.SemanticSyncService; +import jakarta.validation.Valid; +import lombok.RequiredArgsConstructor; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping("/api/semantic/tables/sync") +@RequiredArgsConstructor +public class TableSemanticSyncController { + + private final SemanticSyncService semanticSyncService; + + @GetMapping("/candidates") + public Result> getPhysicalTableCandidates( + @Valid PhysicalTableCandidatePageQuery query) { + return Result.success(semanticSyncService.getPhysicalTableCandidates(query)); + } + + @PostMapping + public Result syncTables( + @Valid @RequestBody SyncTableSemanticsRequest request) { + return Result.success(semanticSyncService.syncTables(request)); + } + + @PostMapping("/physical-status") + public Result refreshPhysicalStatus( + @Valid @RequestBody RefreshPhysicalStatusRequest request) { + return Result.success(semanticSyncService.refreshPhysicalStatus(request)); + } +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/convertor/PromptConverter.java b/data-agent-backend/src/main/java/io/github/malonetalk/convertor/PromptConverter.java index ec340e0..d9b1364 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/convertor/PromptConverter.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/convertor/PromptConverter.java @@ -25,11 +25,11 @@ import io.github.malonetalk.dto.prompt.TableRelationPromptResponse; import io.github.malonetalk.entity.ColumnInfo; import io.github.malonetalk.entity.TableInfo; +import io.github.malonetalk.service.semantic.SemanticAvailabilityHelper; +import io.github.malonetalk.service.semantic.enums.UsageLevelEnum; import io.github.malonetalk.utils.SemanticUtils; import java.util.List; -import java.util.Locale; import java.util.Map; -import java.util.Optional; /** 物理层/语义层 → Agent Prompt DTO 的统一转换器,集中管理所有面向 LLM 的 DTO 映射逻辑。 */ public final class PromptConverter { @@ -40,16 +40,19 @@ private PromptConverter() {} public static ColumnPromptResponse mapColumnPrompt( PhysicalColumnInfo physicalColumn, Map semanticByName) { ColumnInfo semanticColumn = - semanticByName.get(physicalColumn.columnName().toLowerCase(Locale.ROOT)); - if (semanticColumn != null && !Boolean.TRUE.equals(semanticColumn.getIsVisible())) { + semanticByName.get( + SemanticUtils.normalizeObjectName( + physicalColumn.columnName(), + "Missing physical column name for prompt conversion.")); + if (!SemanticAvailabilityHelper.isColumnAvailable( + semanticColumn, UsageLevelEnum.AI_PROMPT)) { return null; } String description = - Optional.ofNullable(semanticColumn) - .map(ColumnInfo::getColumnDescription) - .map(SemanticUtils::trimToNull) - .orElseGet(() -> SemanticUtils.trimToNull(physicalColumn.remarks())); + SemanticUtils.firstNonBlank( + semanticColumn == null ? null : semanticColumn.getColumnDescription(), + physicalColumn.remarks()); StringBuilder typeBuilder = new StringBuilder(physicalColumn.typeName()); if (physicalColumn.columnSize() > 0) { @@ -72,8 +75,11 @@ public static TablePromptResponse mapTablePrompt( Map semanticByName, List resolvedRelations) { TableInfo semanticTable = - semanticByName.get(physicalTable.tableName().toLowerCase(Locale.ROOT)); - if (semanticTable != null && !Boolean.TRUE.equals(semanticTable.getIsVisible())) { + semanticByName.get( + SemanticUtils.normalizeObjectName( + physicalTable.tableName(), + "Missing physical table name for prompt conversion.")); + if (!SemanticAvailabilityHelper.isTableAvailable(semanticTable, UsageLevelEnum.AI_PROMPT)) { return null; } @@ -93,9 +99,8 @@ private static String resolveDomain(TableInfo semanticTable) { private static String resolveDescription( PhysicalTableInfo physicalTable, TableInfo semanticTable) { - return Optional.ofNullable(semanticTable) - .map(TableInfo::getTableDescription) - .map(SemanticUtils::trimToNull) - .orElseGet(() -> SemanticUtils.trimToNull(physicalTable.remarks())); + return SemanticUtils.firstNonBlank( + semanticTable == null ? null : semanticTable.getTableDescription(), + physicalTable.remarks()); } } diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/convertor/SemanticConverter.java b/data-agent-backend/src/main/java/io/github/malonetalk/convertor/SemanticConverter.java index d2192d6..8cf96d8 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/convertor/SemanticConverter.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/convertor/SemanticConverter.java @@ -20,11 +20,15 @@ import io.github.malonetalk.common.SemanticConstants; import io.github.malonetalk.dto.semantic.ColumnSemanticResponse; import io.github.malonetalk.dto.semantic.LogicalTableRelationResponse; +import io.github.malonetalk.dto.semantic.RelationWorkspaceColumnResponse; +import io.github.malonetalk.dto.semantic.RelationWorkspaceTableResponse; import io.github.malonetalk.dto.semantic.TableSemanticResponse; import io.github.malonetalk.entity.ColumnInfo; import io.github.malonetalk.entity.LogicalTableRelation; import io.github.malonetalk.entity.TableInfo; import io.github.malonetalk.enums.LogicalTableRelationType; +import io.github.malonetalk.service.semantic.SemanticAvailabilityHelper; +import io.github.malonetalk.service.semantic.enums.UsageLevelEnum; import io.github.malonetalk.service.semantic.relation.LogicalTableRelationHelper; import io.github.malonetalk.utils.SemanticUtils; import java.util.List; @@ -39,25 +43,41 @@ public class SemanticConverter { public TableSemanticResponse toResponse(TableInfo tableInfo) { Boolean isVisible = tableInfo.getIsVisible(); + boolean hasPhysicalTable = SemanticAvailabilityHelper.hasPhysicalTable(tableInfo); return TableSemanticResponse.builder() .id(tableInfo.getId()) .tableName(tableInfo.getTableName()) .domain(SemanticUtils.normalizeDomain(tableInfo.getDomain())) + .physicalTableDescription( + SemanticUtils.trimToNull(tableInfo.getPhysicalTableDescription())) .tableDescription(SemanticUtils.trimToNull(tableInfo.getTableDescription())) .isVisible(isVisible) - .hasPhysicalTable(true) + .hasPhysicalTable(hasPhysicalTable) + .invalidReason( + SemanticAvailabilityHelper.tableInvalidReason( + tableInfo, UsageLevelEnum.USER_OPERATION)) .updateTime(tableInfo.getUpdateTime()) .build(); } public ColumnSemanticResponse toResponse(ColumnInfo columnInfo) { + boolean hasPhysicalColumn = SemanticAvailabilityHelper.hasPhysicalColumn(columnInfo); return ColumnSemanticResponse.builder() .id(columnInfo.getId()) .columnName(columnInfo.getColumnName()) + .physicalColumnDescription( + SemanticUtils.trimToNull(columnInfo.getPhysicalColumnDescription())) .columnDescription(columnInfo.getColumnDescription()) + .typeName(SemanticUtils.trimToNull(columnInfo.getTypeName())) + .primaryKey(columnInfo.getPrimaryKey()) .isVisible(columnInfo.getIsVisible()) - .hasPhysicalColumn(true) - .effective(Boolean.TRUE.equals(columnInfo.getIsVisible())) + .hasPhysicalColumn(hasPhysicalColumn) + .effective( + SemanticAvailabilityHelper.isColumnAvailable( + columnInfo, UsageLevelEnum.USER_OPERATION)) + .invalidReason( + SemanticAvailabilityHelper.columnInvalidReason( + columnInfo, UsageLevelEnum.USER_OPERATION)) .updateTime(columnInfo.getUpdateTime()) .build(); } @@ -91,4 +111,41 @@ public LogicalTableRelationResponse toResponse(LogicalTableRelation relation) { .updateTime(relation.getUpdateTime()) .build(); } + + public RelationWorkspaceTableResponse toWorkspaceTable( + TableInfo tableInfo, List columns) { + return RelationWorkspaceTableResponse.builder() + .tableName(tableInfo.getTableName()) + .domain(SemanticUtils.normalizeDomain(tableInfo.getDomain())) + .description( + SemanticUtils.firstNonBlank( + tableInfo.getTableDescription(), + tableInfo.getPhysicalTableDescription())) + .operable( + SemanticAvailabilityHelper.isTableAvailable( + tableInfo, UsageLevelEnum.USER_OPERATION)) + .invalidReason( + SemanticAvailabilityHelper.tableInvalidReason( + tableInfo, UsageLevelEnum.USER_OPERATION)) + .columns(columns.stream().map(this::toWorkspaceColumn).toList()) + .build(); + } + + public RelationWorkspaceColumnResponse toWorkspaceColumn(ColumnInfo columnInfo) { + return RelationWorkspaceColumnResponse.builder() + .columnName(columnInfo.getColumnName()) + .description( + SemanticUtils.firstNonBlank( + columnInfo.getColumnDescription(), + columnInfo.getPhysicalColumnDescription())) + .typeName(SemanticUtils.trimToNull(columnInfo.getTypeName())) + .primaryKey(columnInfo.getPrimaryKey()) + .operable( + SemanticAvailabilityHelper.isColumnAvailable( + columnInfo, UsageLevelEnum.USER_OPERATION)) + .invalidReason( + SemanticAvailabilityHelper.columnInvalidReason( + columnInfo, UsageLevelEnum.USER_OPERATION)) + .build(); + } } diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/ColumnSemanticResponse.java b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/ColumnSemanticResponse.java index 55ffc35..e1db962 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/ColumnSemanticResponse.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/ColumnSemanticResponse.java @@ -30,6 +30,11 @@ public record ColumnSemanticResponse( Boolean primaryKey, Boolean isVisible, Boolean hasPhysicalColumn, + // 最终可用性(推导值,非存储字段):effective = isVisible && hasPhysicalColumn。 + // 与 isVisible 含义不同——isVisible 只表示用户是否手动隐藏该列; + // effective 还需物理列依然存在,二者任一为假即为 false。 + // 它由 SemanticConverter 调 SemanticAvailabilityHelper.isColumnAvailable(USER_OPERATION) 计算, + // 前端据此渲染“有效/无效”徽标。 Boolean effective, String invalidReason, LocalDateTime updateTime) {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/PhysicalTableCandidatePageQuery.java b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/PhysicalTableCandidatePageQuery.java new file mode 100644 index 0000000..f172c96 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/PhysicalTableCandidatePageQuery.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.dto.semantic; + +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.Pattern; + +public record PhysicalTableCandidatePageQuery( + @NotNull(message = "datasourceId 不能为空") Integer datasourceId, + @Min(value = 1, message = "page 不能小于 1") Integer page, + @Min(value = 1, message = "pageSize 不能小于 1") Integer pageSize, + String keyword, + @Pattern(regexp = "^(?i)(asc|desc)$", message = "sortOrder must be asc or desc.") + String sortOrder) {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/PhysicalTableCandidateResponse.java b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/PhysicalTableCandidateResponse.java new file mode 100644 index 0000000..9c1acb0 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/PhysicalTableCandidateResponse.java @@ -0,0 +1,21 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.dto.semantic; + +public record PhysicalTableCandidateResponse( + String tableName, String physicalTableDescription, Boolean synced) {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RefreshPhysicalStatusRequest.java b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RefreshPhysicalStatusRequest.java new file mode 100644 index 0000000..65627be --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RefreshPhysicalStatusRequest.java @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.dto.semantic; + +import jakarta.validation.constraints.NotNull; + +public record RefreshPhysicalStatusRequest( + @NotNull(message = "datasourceId 不能为空") Integer datasourceId) {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RelationWorkspaceColumnResponse.java b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RelationWorkspaceColumnResponse.java new file mode 100644 index 0000000..bbd6a35 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RelationWorkspaceColumnResponse.java @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.dto.semantic; + +import lombok.Builder; + +@Builder +public record RelationWorkspaceColumnResponse( + String columnName, + String description, + String typeName, + Boolean primaryKey, + boolean operable, + String invalidReason) {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RelationWorkspacePageQuery.java b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RelationWorkspacePageQuery.java new file mode 100644 index 0000000..f4a8fa0 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RelationWorkspacePageQuery.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.dto.semantic; + +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.Pattern; + +public record RelationWorkspacePageQuery( + @NotNull @Min(1) Integer datasourceId, + @Min(1) Integer page, + @Min(1) Integer pageSize, + String keyword, + @Pattern(regexp = "^(?i)(asc|desc)$", message = "sortOrder must be asc or desc.") + String sortOrder) {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RelationWorkspaceResponse.java b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RelationWorkspaceResponse.java new file mode 100644 index 0000000..071c6fc --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RelationWorkspaceResponse.java @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.dto.semantic; + +import io.github.malonetalk.dto.pagination.PageResponse; +import java.util.List; + +public record RelationWorkspaceResponse( + PageResponse nodes, + List relations) {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RelationWorkspaceTableResponse.java b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RelationWorkspaceTableResponse.java new file mode 100644 index 0000000..8d8bc49 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RelationWorkspaceTableResponse.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.dto.semantic; + +import java.util.List; +import lombok.Builder; + +@Builder +public record RelationWorkspaceTableResponse( + String tableName, + String domain, + String description, + boolean operable, + String invalidReason, + List columns) {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/SyncTableResult.java b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/SyncTableResult.java new file mode 100644 index 0000000..e1c0849 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/SyncTableResult.java @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.dto.semantic; + +import lombok.Builder; + +@Builder +public record SyncTableResult( + String tableName, + boolean physicalTableFound, + boolean tableAdded, + boolean tableReactivated, + boolean tableUpdated, + // 本次同步是否对这张表执行了"标缺失"动作(语义层无记录则无动作,为 false) + boolean tableMarkedAsMissing, + int addedColumns, + int reactivatedColumns, + int updatedColumns, + int missingColumnsMarked, + String message) {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/SyncTableSemanticsRequest.java b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/SyncTableSemanticsRequest.java new file mode 100644 index 0000000..5098f01 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/SyncTableSemanticsRequest.java @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.dto.semantic; + +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; +import java.util.List; + +public record SyncTableSemanticsRequest( + @NotNull(message = "datasourceId 不能为空") Integer datasourceId, + @NotEmpty(message = "tableNames 不能为空") List tableNames) {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/SyncTableSemanticsResponse.java b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/SyncTableSemanticsResponse.java new file mode 100644 index 0000000..deff25e --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/SyncTableSemanticsResponse.java @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.dto.semantic; + +import java.util.List; + +public record SyncTableSemanticsResponse( + int addedTables, + int reactivatedTables, + int updatedTables, + int missingTablesMarked, + int addedColumns, + int reactivatedColumns, + int updatedColumns, + int missingColumnsMarked, + List results) {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/entity/ColumnInfo.java b/data-agent-backend/src/main/java/io/github/malonetalk/entity/ColumnInfo.java index 45bb532..e3df009 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/entity/ColumnInfo.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/entity/ColumnInfo.java @@ -27,8 +27,12 @@ public class ColumnInfo { private Integer datasourceId; private String tableName; private String columnName; + private String physicalColumnDescription; + private String typeName; + private Boolean primaryKey; private String columnDescription; private Boolean isVisible; + private Boolean physicalStatus; private LocalDateTime createTime; private LocalDateTime updateTime; } diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/entity/TableInfo.java b/data-agent-backend/src/main/java/io/github/malonetalk/entity/TableInfo.java index cb2c16f..653e429 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/entity/TableInfo.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/entity/TableInfo.java @@ -25,10 +25,12 @@ public class TableInfo { private Integer id; private String tableName; + private String physicalTableDescription; private String tableDescription; private String domain; private Integer datasourceId; private Boolean isVisible; + private Boolean physicalStatus; private LocalDateTime createTime; private LocalDateTime updateTime; } diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/mapper/ColumnSemanticInfoMapper.java b/data-agent-backend/src/main/java/io/github/malonetalk/mapper/ColumnSemanticInfoMapper.java index 470e761..b4ab0f6 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/mapper/ColumnSemanticInfoMapper.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/mapper/ColumnSemanticInfoMapper.java @@ -19,6 +19,7 @@ import io.github.malonetalk.dto.semantic.ColumnSemanticPageQuery; import io.github.malonetalk.entity.ColumnInfo; +import java.time.LocalDateTime; import java.util.List; import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.annotations.Param; @@ -31,6 +32,10 @@ public interface ColumnSemanticInfoMapper { List selectByDatasourceIdAndTableName( @Param("datasourceId") Integer datasourceId, @Param("tableName") String tableName); + List selectByDatasourceIdAndTableNames( + @Param("datasourceId") Integer datasourceId, + @Param("tableNames") List tableNames); + List selectPageByDatasourceIdAndTableName( @Param("query") ColumnSemanticPageQuery query, @Param("sortDescending") boolean sortDescending); @@ -42,7 +47,16 @@ ColumnInfo selectByDatasourceIdAndTableNameAndColumnName( int insert(ColumnInfo columnInfo); - int update(ColumnInfo columnInfo); + int batchUpsertPhysicalCache(@Param("columns") List columns); + + int updateSemanticFields(ColumnInfo columnInfo); + + int updatePhysicalCacheFields(ColumnInfo columnInfo); + + int markPhysicalMissingByIds( + @Param("datasourceId") Integer datasourceId, + @Param("ids") List ids, + @Param("now") LocalDateTime now); int deleteByDatasourceId(@Param("datasourceId") Integer datasourceId); diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/mapper/LogicalTableRelationMapper.java b/data-agent-backend/src/main/java/io/github/malonetalk/mapper/LogicalTableRelationMapper.java index f68b5d2..e915ad3 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/mapper/LogicalTableRelationMapper.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/mapper/LogicalTableRelationMapper.java @@ -35,6 +35,10 @@ List selectByDatasourceIdAndSourceTable( @Param("datasourceId") Integer datasourceId, @Param("sourceTableName") String sourceTableName); + List selectByDatasourceIdAndSourceTables( + @Param("datasourceId") Integer datasourceId, + @Param("sourceTableNames") List sourceTableNames); + List selectPageByDatasourceIdAndSourceTable( @Param("query") RelationSemanticPageQuery query, @Param("sortDescending") boolean sortDescending); diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/mapper/TableInfoMapper.java b/data-agent-backend/src/main/java/io/github/malonetalk/mapper/TableInfoMapper.java index 92bfee3..3b70caf 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/mapper/TableInfoMapper.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/mapper/TableInfoMapper.java @@ -19,6 +19,7 @@ import io.github.malonetalk.dto.semantic.TableSemanticPageQuery; import io.github.malonetalk.entity.TableInfo; +import java.time.LocalDateTime; import java.util.List; import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.annotations.Param; @@ -28,7 +29,9 @@ public interface TableInfoMapper { int insert(TableInfo tableInfo); - int update(TableInfo tableInfo); + int updateSemanticFields(TableInfo tableInfo); + + int updatePhysicalCacheFields(TableInfo tableInfo); int deleteByDatasourceIdAndIds( @Param("datasourceId") Integer datasourceId, @Param("ids") List ids); @@ -42,6 +45,17 @@ List selectPageByDatasourceId( TableInfo selectByDatasourceIdAndTableName( @Param("datasourceId") Integer datasourceId, @Param("tableName") String tableName); + List selectByDatasourceIdAndTableNames( + @Param("datasourceId") Integer datasourceId, + @Param("tableNames") List tableNames); + + int batchUpsertPhysicalCache(@Param("tables") List tables); + + int markPhysicalMissingByDatasourceIdAndTableNames( + @Param("datasourceId") Integer datasourceId, + @Param("tableNames") List tableNames, + @Param("now") LocalDateTime now); + List selectByDatasourceIdAndDomains( @Param("datasourceId") Integer datasourceId, @Param("domains") List domains); diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/DomainService.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/DomainService.java similarity index 96% rename from data-agent-backend/src/main/java/io/github/malonetalk/service/DomainService.java rename to data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/DomainService.java index cfb6117..fdfd11f 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/service/DomainService.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/DomainService.java @@ -15,7 +15,7 @@ * along with this program. If not, see . * limitations under the License. */ -package io.github.malonetalk.service; +package io.github.malonetalk.service.semantic; import io.github.malonetalk.dto.DomainCreateRequest; import io.github.malonetalk.dto.DomainPageQuery; diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/DomainServiceImpl.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/DomainServiceImpl.java similarity index 94% rename from data-agent-backend/src/main/java/io/github/malonetalk/service/DomainServiceImpl.java rename to data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/DomainServiceImpl.java index 59579e2..fabaa39 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/service/DomainServiceImpl.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/DomainServiceImpl.java @@ -15,7 +15,7 @@ * along with this program. If not, see . * limitations under the License. */ -package io.github.malonetalk.service; +package io.github.malonetalk.service.semantic; import com.github.pagehelper.Page; import com.github.pagehelper.PageHelper; @@ -30,7 +30,6 @@ import io.github.malonetalk.utils.SemanticUtils; import java.time.LocalDateTime; import java.util.List; -import java.util.Locale; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -73,7 +72,8 @@ public DomainInfo findById(Integer id) { @Override public DomainInfo create(DomainCreateRequest request) { String normalizedName = - SemanticUtils.trimToNotBlank(request.name(), "领域名称").toLowerCase(Locale.ROOT); + SemanticUtils.normalizeObjectName( + request.name(), "Missing domain name for domain creation."); DomainInfo existing = domainInfoMapper.selectByName(normalizedName); if (existing != null) { throw new IllegalArgumentException("领域名称已存在: " + normalizedName); @@ -97,7 +97,8 @@ public DomainInfo update(Integer id, DomainUpdateRequest request) { throw new IllegalArgumentException("领域不存在: id=" + id); } String normalizedName = - SemanticUtils.trimToNotBlank(request.name(), "领域名称").toLowerCase(Locale.ROOT); + SemanticUtils.normalizeObjectName( + request.name(), "Missing domain name for domain update."); DomainInfo nameConflict = domainInfoMapper.selectByName(normalizedName); if (nameConflict != null && !nameConflict.getId().equals(id)) { throw new IllegalArgumentException("领域名称已存在: " + normalizedName); diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/SemanticAvailabilityHelper.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/SemanticAvailabilityHelper.java new file mode 100644 index 0000000..3349012 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/SemanticAvailabilityHelper.java @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.service.semantic; + +import io.github.malonetalk.entity.ColumnInfo; +import io.github.malonetalk.entity.TableInfo; +import io.github.malonetalk.service.semantic.enums.ColumnInvalidReasonEnum; +import io.github.malonetalk.service.semantic.enums.TableInvalidReasonEnum; +import io.github.malonetalk.service.semantic.enums.UsageLevelEnum; +import java.util.Objects; + +// TODO 不应该抛出NPE +public final class SemanticAvailabilityHelper { + + private SemanticAvailabilityHelper() {} + + public static boolean isTableAvailable(TableInfo tableInfo, UsageLevelEnum usageLevel) { + Objects.requireNonNull(tableInfo, "tableInfo should not be null"); + if (usageLevel == UsageLevelEnum.FRONTEND_DISPLAY) { + return true; + } + return Boolean.TRUE.equals(tableInfo.getIsVisible()) && hasPhysicalTable(tableInfo); + } + + public static boolean isColumnAvailable(ColumnInfo columnInfo, UsageLevelEnum usageLevel) { + Objects.requireNonNull(columnInfo, "columnInfo should not be null"); + if (usageLevel == UsageLevelEnum.FRONTEND_DISPLAY) { + return true; + } + return Boolean.TRUE.equals(columnInfo.getIsVisible()) && hasPhysicalColumn(columnInfo); + } + + public static boolean hasPhysicalTable(TableInfo tableInfo) { + Objects.requireNonNull(tableInfo, "tableInfo should not be null"); + return !Boolean.FALSE.equals(tableInfo.getPhysicalStatus()); + } + + public static boolean hasPhysicalColumn(ColumnInfo columnInfo) { + Objects.requireNonNull(columnInfo, "columnInfo should not be null"); + return !Boolean.FALSE.equals(columnInfo.getPhysicalStatus()); + } + + public static String tableInvalidReason(TableInfo tableInfo, UsageLevelEnum usageLevel) { + if (isTableAvailable(tableInfo, usageLevel)) { + return null; + } + if (!hasPhysicalTable(tableInfo)) { + return TableInvalidReasonEnum.PHYSICAL_TABLE_NOT_FOUND.getReason(); + } + if (!Boolean.TRUE.equals(tableInfo.getIsVisible())) { + return TableInvalidReasonEnum.TABLE_HIDDEN.getReason(); + } + return TableInvalidReasonEnum.TABLE_UNAVAILABLE.getReason(); + } + + public static String columnInvalidReason(ColumnInfo columnInfo, UsageLevelEnum usageLevel) { + if (isColumnAvailable(columnInfo, usageLevel)) { + return null; + } + if (!hasPhysicalColumn(columnInfo)) { + return ColumnInvalidReasonEnum.PHYSICAL_COLUMN_NOT_FOUND.getReason(); + } + if (!Boolean.TRUE.equals(columnInfo.getIsVisible())) { + return ColumnInvalidReasonEnum.COLUMN_HIDDEN.getReason(); + } + return ColumnInvalidReasonEnum.COLUMN_UNAVAILABLE.getReason(); + } + + public static String unavailableMessage(String fieldName, String objectName, String reason) { + String fallbackReason = reason == null || reason.isBlank() ? "不可用" : reason; + return fieldName + " " + objectName + " is unavailable: " + fallbackReason; + } +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/SemanticMergeService.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/SemanticMergeService.java index ae9b8d3..87a2c96 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/SemanticMergeService.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/SemanticMergeService.java @@ -32,6 +32,7 @@ import io.github.malonetalk.mapper.ColumnSemanticInfoMapper; import io.github.malonetalk.mapper.LogicalTableRelationMapper; import io.github.malonetalk.mapper.TableInfoMapper; +import io.github.malonetalk.service.semantic.enums.UsageLevelEnum; import io.github.malonetalk.service.semantic.relation.LogicalTableRelationHelper; import io.github.malonetalk.utils.SemanticUtils; import java.util.ArrayList; @@ -39,7 +40,6 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -85,7 +85,9 @@ public List listVisibleTablesByDomains( } public List getTableSchema(Datasource datasource, String tableName) { - String normalizedTableName = SemanticUtils.trimToNotBlank(tableName, "tableName"); + String normalizedTableName = + SemanticUtils.requireTrimmed( + tableName, "Missing tableName for merged table schema lookup."); List physicalColumns = schemaReader.getTableSchema(datasource, normalizedTableName); @@ -97,6 +99,10 @@ public List getTableSchema(Datasource datasource, String t TableInfo semanticTable = tableInfoMapper.selectByDatasourceIdAndTableName( datasource.getId(), normalizedTableName); + if (semanticTable != null && !SemanticAvailabilityHelper.hasPhysicalTable(semanticTable)) { + throw new IllegalArgumentException( + "Table " + normalizedTableName + " does not exist physically."); + } if (semanticTable != null && !Boolean.TRUE.equals(semanticTable.getIsVisible())) { throw new IllegalArgumentException("Table " + normalizedTableName + " is hidden."); } @@ -209,7 +215,11 @@ private Map buildSemanticColumnIndex( for (ColumnInfo column : columnSemanticInfoMapper.selectByDatasourceIdAndTableName( datasourceId, tableName)) { - result.put(column.getColumnName().toLowerCase(Locale.ROOT), column); + result.put( + SemanticUtils.normalizeObjectName( + column.getColumnName(), + "Missing columnName while building semantic column index."), + column); } return result; } @@ -249,7 +259,11 @@ private record TableNameIndex(Map index) { private static TableNameIndex of(List tables) { Map map = new HashMap<>(); for (TableInfo table : tables) { - map.put(table.getTableName().toLowerCase(Locale.ROOT), table); + map.put( + SemanticUtils.normalizeObjectName( + table.getTableName(), + "Missing tableName while building semantic table index."), + table); } return new TableNameIndex(map); } @@ -259,12 +273,19 @@ private Map asMap() { } private TableInfo get(String tableName) { - return index.get(tableName.toLowerCase(Locale.ROOT)); + return index.get( + SemanticUtils.normalizeObjectName( + tableName, "Missing tableName while reading semantic table index.")); } private boolean isHidden(String tableName) { TableInfo tableInfo = get(tableName); - return tableInfo != null && !Boolean.TRUE.equals(tableInfo.getIsVisible()); + if (tableInfo == null) { + // 表不在已加载的语义索引里:当作“未隐藏”保留,避免误丢合法关系 + return false; + } + return !SemanticAvailabilityHelper.isTableAvailable( + tableInfo, UsageLevelEnum.AI_PROMPT); } } @@ -274,22 +295,42 @@ private static TableColumnIndex of(List columns) { Map> map = new HashMap<>(); for (ColumnInfo column : columns) { map.computeIfAbsent( - column.getTableName().toLowerCase(Locale.ROOT), + SemanticUtils.normalizeObjectName( + column.getTableName(), + "Missing tableName while building semantic column index."), key -> new HashMap<>()) - .put(column.getColumnName().toLowerCase(Locale.ROOT), column); + .put( + SemanticUtils.normalizeObjectName( + column.getColumnName(), + "Missing columnName while building semantic column index."), + column); } return new TableColumnIndex(map); } private ColumnInfo get(String tableName, String columnName) { - Map columns = index.get(tableName.toLowerCase(Locale.ROOT)); - return columns == null ? null : columns.get(columnName.toLowerCase(Locale.ROOT)); + Map columns = + index.get( + SemanticUtils.normalizeObjectName( + tableName, + "Missing tableName while reading semantic column index.")); + return columns == null + ? null + : columns.get( + SemanticUtils.normalizeObjectName( + columnName, + "Missing columnName while reading semantic column index.")); } private boolean hasHiddenColumn(String tableName, List columnNames) { for (String columnName : columnNames) { ColumnInfo columnInfo = get(tableName, columnName); - if (columnInfo != null && !Boolean.TRUE.equals(columnInfo.getIsVisible())) { + if (columnInfo == null) { + // 列不在已加载的语义索引里:当作“未隐藏”保留 + return false; + } + if (!SemanticAvailabilityHelper.isColumnAvailable( + columnInfo, UsageLevelEnum.AI_PROMPT)) { return true; } } @@ -303,7 +344,9 @@ private static RelationSourceIndex of(List relations) { Map> map = new HashMap<>(); for (LogicalTableRelation relation : relations) { map.computeIfAbsent( - relation.getSourceTableName().toLowerCase(Locale.ROOT), + SemanticUtils.normalizeObjectName( + relation.getSourceTableName(), + "Missing sourceTableName while building relation index."), key -> new ArrayList<>()) .add(relation); } @@ -312,7 +355,10 @@ private static RelationSourceIndex of(List relations) { private List get(String sourceTableName) { return index.getOrDefault( - sourceTableName.toLowerCase(Locale.ROOT), Collections.emptyList()); + SemanticUtils.normalizeObjectName( + sourceTableName, + "Missing sourceTableName while reading relation index."), + Collections.emptyList()); } } } diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/column/ColumnSemanticServiceImpl.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/column/ColumnSemanticServiceImpl.java index 87e723e..3caff36 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/column/ColumnSemanticServiceImpl.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/column/ColumnSemanticServiceImpl.java @@ -33,7 +33,6 @@ import io.github.malonetalk.utils.SemanticUtils; import java.time.LocalDateTime; import java.util.List; -import java.util.Locale; import java.util.Set; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; @@ -51,7 +50,9 @@ public class ColumnSemanticServiceImpl implements ColumnSemanticService { @Override public PageResponse getColumnPage(ColumnSemanticPageQuery query) { SemanticUtils.requireDatasourceId(query.datasourceId()); - String normalizedTableName = SemanticUtils.trimToNotBlank(query.tableName(), "tableName"); + String normalizedTableName = + SemanticUtils.requireTrimmed( + query.tableName(), "Missing tableName for column semantic page query."); int pageNumber = PageResponse.resolvePage(query.page()); int pageSize = PageResponse.resolvePageSize(query.pageSize()); Datasource datasource = datasourceService.findById(query.datasourceId()); @@ -80,10 +81,11 @@ public PageResponse getColumnPage(ColumnSemanticPageQuer public void updateColumnSemantic(String tableName, ColumnSemanticUpdateRequest request) { requireDatasource(request.datasourceId()); String normalizedTableName = - SemanticUtils.trimToNotBlank(tableName, "tableName").toLowerCase(Locale.ROOT); + SemanticUtils.normalizeObjectName( + tableName, "Missing tableName for column semantic update."); String normalizedColumnName = - SemanticUtils.trimToNotBlank(request.columnName(), "columnName") - .toLowerCase(Locale.ROOT); + SemanticUtils.normalizeObjectName( + request.columnName(), "Missing columnName for column semantic update."); ColumnInfo existing = columnSemanticInfoMapper.selectByDatasourceIdAndTableNameAndColumnName( request.datasourceId(), normalizedTableName, normalizedColumnName); @@ -94,6 +96,7 @@ public void updateColumnSemantic(String tableName, ColumnSemanticUpdateRequest r columnInfo.setColumnName(normalizedColumnName); columnInfo.setColumnDescription(SemanticUtils.trimToNull(request.columnDescription())); columnInfo.setIsVisible(request.isVisible()); + columnInfo.setPhysicalStatus(Boolean.FALSE); columnInfo.setCreateTime(LocalDateTime.now()); columnInfo.setUpdateTime(LocalDateTime.now()); columnSemanticInfoMapper.insert(columnInfo); @@ -104,7 +107,7 @@ public void updateColumnSemantic(String tableName, ColumnSemanticUpdateRequest r existing.setColumnDescription(SemanticUtils.trimToNull(request.columnDescription())); existing.setIsVisible(request.isVisible()); existing.setUpdateTime(LocalDateTime.now()); - columnSemanticInfoMapper.update(existing); + columnSemanticInfoMapper.updateSemanticFields(existing); } @Override @@ -131,7 +134,9 @@ public void resetColumnSemantic(Integer datasourceId, String tableName, String c public int resetColumnSemantics( Integer datasourceId, String tableName, List columnNames) { requireDatasource(datasourceId); - String normalizedTableName = SemanticUtils.trimToNotBlank(tableName, "tableName"); + String normalizedTableName = + SemanticUtils.requireTrimmed( + tableName, "Missing tableName for batch column semantic reset."); if (columnNames == null || columnNames.isEmpty()) { return 0; } @@ -139,8 +144,10 @@ public int resetColumnSemantics( columnNames.stream() .map( columnName -> - SemanticUtils.trimToNotBlank(columnName, "columnName") - .toLowerCase(Locale.ROOT)) + SemanticUtils.normalizeObjectName( + columnName, + "Missing columnName for batch column semantic" + + " reset.")) .collect(Collectors.toCollection(java.util.LinkedHashSet::new)); List matchedIds = columnSemanticInfoMapper @@ -149,10 +156,10 @@ public int resetColumnSemantics( .filter( column -> normalizedColumnNames.contains( - SemanticUtils.trimToNotBlank( - column.getColumnName(), - "columnName") - .toLowerCase(Locale.ROOT))) + SemanticUtils.normalizeObjectName( + column.getColumnName(), + "Missing columnName while matching column" + + " semantic reset."))) .map(ColumnInfo::getId) .distinct() .toList(); diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/enums/ColumnInvalidReasonEnum.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/enums/ColumnInvalidReasonEnum.java new file mode 100644 index 0000000..a7c1fea --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/enums/ColumnInvalidReasonEnum.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.service.semantic.enums; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +@Getter +@AllArgsConstructor +public enum ColumnInvalidReasonEnum { + PHYSICAL_COLUMN_NOT_FOUND("物理列不存在"), + COLUMN_HIDDEN("列已隐藏"), + COLUMN_UNAVAILABLE("列不可用"); + private final String reason; +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/enums/TableInvalidReasonEnum.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/enums/TableInvalidReasonEnum.java new file mode 100644 index 0000000..2c9a61e --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/enums/TableInvalidReasonEnum.java @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.service.semantic.enums; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +@Getter +@AllArgsConstructor +public enum TableInvalidReasonEnum { + PHYSICAL_TABLE_NOT_FOUND("物理表不存在"), + TABLE_HIDDEN("表已隐藏"), + TABLE_UNAVAILABLE("表不可用"); + + private final String reason; +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/enums/UsageLevelEnum.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/enums/UsageLevelEnum.java new file mode 100644 index 0000000..83e5fdd --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/enums/UsageLevelEnum.java @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.service.semantic.enums; + +public enum UsageLevelEnum { + AI_PROMPT, + FRONTEND_DISPLAY, + USER_OPERATION +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/LogicalTableRelationHelper.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/LogicalTableRelationHelper.java index edd9d45..ca86777 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/LogicalTableRelationHelper.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/LogicalTableRelationHelper.java @@ -27,7 +27,6 @@ import io.github.malonetalk.utils.SemanticUtils; import java.util.LinkedHashSet; import java.util.List; -import java.util.Locale; import java.util.Set; import org.springframework.stereotype.Component; @@ -42,8 +41,8 @@ public LogicalTableRelationHelper(ObjectMapper objectMapper) { this.objectMapper = objectMapper; } - public String normalizeTableName(String tableName, String label) { - return SemanticUtils.trimToNotBlank(tableName, label).toLowerCase(Locale.ROOT); + public String normalizeTableName(String tableName, String missingMessage) { + return SemanticUtils.normalizeObjectName(tableName, missingMessage); } public List normalizeColumnNames(List columnNames, String fieldName) { @@ -57,7 +56,10 @@ public List normalizeColumnNames(List columnNames, String fieldN throw new IllegalArgumentException(fieldName + " contains a blank column name."); } String normalizedColumnName = columnName.trim(); - String uniqueKey = normalizedColumnName.toLowerCase(Locale.ROOT); + String uniqueKey = + SemanticUtils.normalizeObjectName( + normalizedColumnName, + "Missing columnName while normalizing logical relation columns."); if (!uniqueKeys.add(uniqueKey)) { throw new IllegalArgumentException( fieldName + " contains duplicate column: " + normalizedColumnName); @@ -69,7 +71,11 @@ public List normalizeColumnNames(List columnNames, String fieldN public String buildColumnSignature(List columnNames) { return normalizeColumnNames(columnNames, "columnNames").stream() - .map(columnName -> columnName.toLowerCase(Locale.ROOT)) + .map( + columnName -> + SemanticUtils.normalizeObjectName( + columnName, + "Missing columnName while building column signature.")) .reduce((left, right) -> left + RELATION_KEY_SEPARATOR + right) .orElse(""); } @@ -79,13 +85,13 @@ public String buildRelationKey( List sourceColumnNames, String targetTableName, List targetColumnNames) { - return SemanticUtils.trimToNotBlank(sourceTableName, "sourceTableName") - .toLowerCase(Locale.ROOT) + return SemanticUtils.normalizeObjectName( + sourceTableName, "Missing sourceTableName for logical relation key.") + RELATION_TABLE_COLUMN_SEPARATOR + buildColumnSignature(sourceColumnNames) + RELATION_GROUP_SEPARATOR - + SemanticUtils.trimToNotBlank(targetTableName, "targetTableName") - .toLowerCase(Locale.ROOT) + + SemanticUtils.normalizeObjectName( + targetTableName, "Missing targetTableName for logical relation key.") + RELATION_TABLE_COLUMN_SEPARATOR + buildColumnSignature(targetColumnNames); } diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/RelationSemanticService.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/RelationSemanticService.java index bafecff..676b74c 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/RelationSemanticService.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/RelationSemanticService.java @@ -21,6 +21,8 @@ import io.github.malonetalk.dto.semantic.BindLogicalTableRelationRequest; import io.github.malonetalk.dto.semantic.LogicalTableRelationResponse; import io.github.malonetalk.dto.semantic.RelationSemanticPageQuery; +import io.github.malonetalk.dto.semantic.RelationWorkspacePageQuery; +import io.github.malonetalk.dto.semantic.RelationWorkspaceResponse; import io.github.malonetalk.dto.semantic.UpdateLogicalTableRelationEnabledRequest; import io.github.malonetalk.dto.semantic.UpdateLogicalTableRelationRequest; import java.util.List; @@ -29,6 +31,8 @@ public interface RelationSemanticService { PageResponse getRelationPage(RelationSemanticPageQuery query); + RelationWorkspaceResponse getRelationWorkspace(RelationWorkspacePageQuery query); + LogicalTableRelationResponse createRelationSemantic( String tableName, BindLogicalTableRelationRequest request); diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/RelationSemanticServiceImpl.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/RelationSemanticServiceImpl.java index 02da81c..98eb412 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/RelationSemanticServiceImpl.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/RelationSemanticServiceImpl.java @@ -25,15 +25,28 @@ import io.github.malonetalk.dto.semantic.BindLogicalTableRelationRequest; import io.github.malonetalk.dto.semantic.LogicalTableRelationResponse; import io.github.malonetalk.dto.semantic.RelationSemanticPageQuery; +import io.github.malonetalk.dto.semantic.RelationWorkspacePageQuery; +import io.github.malonetalk.dto.semantic.RelationWorkspaceResponse; +import io.github.malonetalk.dto.semantic.RelationWorkspaceTableResponse; +import io.github.malonetalk.dto.semantic.TableSemanticPageQuery; import io.github.malonetalk.dto.semantic.UpdateLogicalTableRelationEnabledRequest; import io.github.malonetalk.dto.semantic.UpdateLogicalTableRelationRequest; +import io.github.malonetalk.entity.ColumnInfo; import io.github.malonetalk.entity.LogicalTableRelation; +import io.github.malonetalk.entity.TableInfo; +import io.github.malonetalk.mapper.ColumnSemanticInfoMapper; import io.github.malonetalk.mapper.LogicalTableRelationMapper; +import io.github.malonetalk.mapper.TableInfoMapper; import io.github.malonetalk.service.DatasourceService; +import io.github.malonetalk.service.semantic.SemanticAvailabilityHelper; +import io.github.malonetalk.service.semantic.enums.UsageLevelEnum; import io.github.malonetalk.utils.SemanticUtils; import java.time.LocalDateTime; +import java.util.LinkedHashMap; import java.util.List; -import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -43,6 +56,8 @@ public class RelationSemanticServiceImpl implements RelationSemanticService { private final DatasourceService datasourceService; + private final TableInfoMapper tableInfoMapper; + private final ColumnSemanticInfoMapper columnSemanticInfoMapper; private final LogicalTableRelationMapper logicalTableRelationMapper; private final LogicalTableRelationHelper logicalTableRelationHelper; private final SemanticConverter semanticConverter; @@ -52,7 +67,8 @@ public PageResponse getRelationPage( RelationSemanticPageQuery query) { requireDatasource(query.datasourceId()); String normalizedTableName = - logicalTableRelationHelper.normalizeTableName(query.tableName(), "tableName"); + logicalTableRelationHelper.normalizeTableName( + query.tableName(), "Missing tableName for logical relation page query."); int pageNumber = PageResponse.resolvePage(query.page()); int pageSize = PageResponse.resolvePageSize(query.pageSize()); boolean sortDescending = SemanticUtils.isDescendingSort(query.sortOrder()); @@ -77,6 +93,85 @@ public PageResponse getRelationPage( return PageResponse.of(items, page.getTotal(), pageNumber, pageSize); } + @Override + public RelationWorkspaceResponse getRelationWorkspace(RelationWorkspacePageQuery query) { + requireDatasource(query.datasourceId()); + int pageNumber = PageResponse.resolvePage(query.page()); + int pageSize = PageResponse.resolvePageSize(query.pageSize()); + boolean sortDescending = SemanticUtils.isDescendingSort(query.sortOrder()); + + PageHelper.startPage(pageNumber, pageSize); + Page page = + (Page) + tableInfoMapper.selectPageByDatasourceId( + new TableSemanticPageQuery( + query.datasourceId(), + pageNumber, + pageSize, + SemanticUtils.trimToNull(query.keyword()), + query.sortOrder()), + sortDescending); + if (page.isEmpty()) { + return new RelationWorkspaceResponse( + PageResponse.empty(pageNumber, pageSize), List.of()); + } + + List tableNames = page.stream().map(TableInfo::getTableName).distinct().toList(); + Set currentPageTableNames = + tableNames.stream() + .map( + tableName -> + SemanticUtils.normalizeObjectName( + tableName, + "Missing tableName while building relation" + + " workspace page.")) + .collect(Collectors.toSet()); + Map> columnsByTableName = + columnSemanticInfoMapper + .selectByDatasourceIdAndTableNames(query.datasourceId(), tableNames) + .stream() + .collect( + Collectors.groupingBy( + column -> + SemanticUtils.normalizeObjectName( + column.getTableName(), + "Missing tableName while grouping relation" + + " workspace columns."), + LinkedHashMap::new, + Collectors.toList())); + List nodes = + page.stream() + .map( + table -> + semanticConverter.toWorkspaceTable( + table, + columnsByTableName.getOrDefault( + SemanticUtils.normalizeObjectName( + table.getTableName(), + "Missing tableName while mapping" + + " relation workspace" + + " table."), + List.of()))) + .toList(); + List relations = + logicalTableRelationMapper + .selectByDatasourceIdAndSourceTables(query.datasourceId(), tableNames) + .stream() + .filter( + relation -> + currentPageTableNames.contains( + SemanticUtils.normalizeObjectName( + relation.getTargetTableName(), + "Missing targetTableName while filtering" + + " relation workspace" + + " relations."))) + .map(semanticConverter::toResponse) + .toList(); + + return new RelationWorkspaceResponse( + PageResponse.of(nodes, page.getTotal(), pageNumber, pageSize), relations); + } + @Override @Transactional public LogicalTableRelationResponse createRelationSemantic( @@ -120,6 +215,9 @@ public boolean updateRelationSemanticEnabled( } LogicalTableRelation relation = requireRelation(request.datasourceId(), tableName, request.relationId()); + if (Boolean.TRUE.equals(request.enabled())) { + ensureRelationEndpointsOperable(relation); + } return logicalTableRelationMapper.updateEnabled( request.relationId(), request.datasourceId(), @@ -146,7 +244,8 @@ public int deleteRelationSemantics( Integer datasourceId, String tableName, List relationIds) { requireDatasource(datasourceId); String normalizedTableName = - logicalTableRelationHelper.normalizeTableName(tableName, "tableName"); + logicalTableRelationHelper.normalizeTableName( + tableName, "Missing tableName for batch logical relation delete."); if (relationIds == null || relationIds.isEmpty()) { return 0; } @@ -187,6 +286,7 @@ private LogicalTableRelation buildRelation( request.targetTableName(), request.description(), request.enabled()); + ensureRelationEndpointsOperable(relation); relation.setCreateTime(LocalDateTime.now()); relation.setUpdateTime(LocalDateTime.now()); return relation; @@ -204,6 +304,71 @@ private void applyRelationUpdate( request.targetTableName(), request.description(), request.enabled()); + ensureRelationEndpointsOperable(relation); + } + + private void ensureRelationEndpointsOperable(LogicalTableRelation relation) { + ensureTableOperable( + relation.getDatasourceId(), relation.getSourceTableName(), "sourceTable"); + ensureTableOperable( + relation.getDatasourceId(), relation.getTargetTableName(), "targetTable"); + List sourceColumns = + logicalTableRelationHelper.fromJson( + relation.getSourceColumnNamesJson(), "sourceColumnNames"); + List targetColumns = + logicalTableRelationHelper.fromJson( + relation.getTargetColumnNamesJson(), "targetColumnNames"); + ensureColumnsOperable( + relation.getDatasourceId(), + relation.getSourceTableName(), + sourceColumns, + "sourceColumnNames"); + ensureColumnsOperable( + relation.getDatasourceId(), + relation.getTargetTableName(), + targetColumns, + "targetColumnNames"); + } + + private void ensureTableOperable(Integer datasourceId, String tableName, String fieldName) { + TableInfo tableInfo = + tableInfoMapper.selectByDatasourceIdAndTableName(datasourceId, tableName); + if (tableInfo == null) { + throw new IllegalArgumentException( + fieldName + " " + tableName + " semantic metadata does not exist."); + } + if (SemanticAvailabilityHelper.isTableAvailable(tableInfo, UsageLevelEnum.USER_OPERATION)) { + return; + } + throw new IllegalArgumentException( + SemanticAvailabilityHelper.unavailableMessage( + fieldName, + tableName, + SemanticAvailabilityHelper.tableInvalidReason( + tableInfo, UsageLevelEnum.USER_OPERATION))); + } + + private void ensureColumnsOperable( + Integer datasourceId, String tableName, List columnNames, String fieldName) { + for (String columnName : columnNames) { + ColumnInfo columnInfo = + columnSemanticInfoMapper.selectByDatasourceIdAndTableNameAndColumnName( + datasourceId, tableName, columnName); + if (columnInfo == null) { + throw new IllegalArgumentException( + fieldName + " " + columnName + " semantic metadata does not exist."); + } + if (SemanticAvailabilityHelper.isColumnAvailable( + columnInfo, UsageLevelEnum.USER_OPERATION)) { + continue; + } + throw new IllegalArgumentException( + SemanticAvailabilityHelper.unavailableMessage( + fieldName, + columnName, + SemanticAvailabilityHelper.columnInvalidReason( + columnInfo, UsageLevelEnum.USER_OPERATION))); + } } private void populateRelationFields( @@ -215,7 +380,8 @@ private void populateRelationFields( String description, Boolean enabled) { relation.setSourceTableName( - logicalTableRelationHelper.normalizeTableName(tableName, "tableName")); + logicalTableRelationHelper.normalizeTableName( + tableName, "Missing source tableName for logical relation save.")); List normalizedSourceColumns = logicalTableRelationHelper.normalizeColumnNames( sourceColumnNames, "sourceColumnNames"); @@ -224,7 +390,8 @@ private void populateRelationFields( relation.setSourceColumnSignature( logicalTableRelationHelper.buildColumnSignature(normalizedSourceColumns)); relation.setTargetTableName( - logicalTableRelationHelper.normalizeTableName(targetTableName, "targetTableName")); + logicalTableRelationHelper.normalizeTableName( + targetTableName, "Missing targetTableName for logical relation save.")); List normalizedTargetColumns = logicalTableRelationHelper.normalizeColumnNames( targetColumnNames, "targetColumnNames"); @@ -265,8 +432,9 @@ private LogicalTableRelation requireRelation( || !datasourceId.equals(relation.getDatasourceId()) || !relation.getSourceTableName() .equals( - SemanticUtils.trimToNotBlank(tableName, "tableName") - .toLowerCase(Locale.ROOT))) { + SemanticUtils.normalizeObjectName( + tableName, + "Missing tableName for logical relation lookup."))) { throw new IllegalArgumentException("Logical relation does not exist."); } return relation; diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncApplyService.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncApplyService.java new file mode 100644 index 0000000..1a075c0 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncApplyService.java @@ -0,0 +1,352 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.service.semantic.sync; + +import io.github.malonetalk.common.SemanticConstants; +import io.github.malonetalk.dto.semantic.SyncTableResult; +import io.github.malonetalk.entity.ColumnInfo; +import io.github.malonetalk.entity.TableInfo; +import io.github.malonetalk.mapper.ColumnSemanticInfoMapper; +import io.github.malonetalk.mapper.TableInfoMapper; +import io.github.malonetalk.utils.SemanticUtils; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +@Service +@RequiredArgsConstructor +public class SemanticSyncApplyService { + + private final TableInfoMapper tableInfoMapper; + private final ColumnSemanticInfoMapper columnSemanticInfoMapper; + + @Transactional + public List applyTableSync( + Integer datasourceId, List presentTables, List missingTables) { + Set tableNames = new LinkedHashSet<>(); + presentTables.forEach(table -> tableNames.add(table.tableName())); + tableNames.addAll(missingTables); + if (tableNames.isEmpty()) { + return List.of(); + } + LocalDateTime now = LocalDateTime.now(); + List selectedTableNames = List.copyOf(tableNames); + + Map existingTableIndex = + loadSemanticTableIndex(datasourceId, selectedTableNames); + Map> columnsByTableName = + loadSemanticColumnsByTable(datasourceId, selectedTableNames); + + // 表和列分别执行一次批量 upsert,不按表或列逐条写库。 + batchUpsertPresentSchema(datasourceId, presentTables); + + // 基于写入前快照生成统计结果,同时收集需要批量标记缺失的列 ID。 + List missingTableNames = new ArrayList<>(); + List missingColumnIds = new ArrayList<>(); + List results = new ArrayList<>(); + for (TableSyncSource table : presentTables) { + results.add( + buildPresentTableResult( + table, + existingTableIndex.get(table.tableName()), + columnsByTableName.getOrDefault(table.tableName(), List.of()), + missingColumnIds)); + } + for (String missingTable : missingTables) { + SyncTableResult result = + buildMissingTableResult( + missingTable, + existingTableIndex.get(missingTable), + columnsByTableName.getOrDefault(missingTable, List.of()), + missingColumnIds); + results.add(result); + if (result.tableMarkedAsMissing()) { + missingTableNames.add(missingTable); + } + } + + markMissingTables(datasourceId, missingTableNames, now); + markMissingColumns(datasourceId, missingColumnIds, now); + return results; + } + + @Transactional + public List refreshPhysicalStatus( + Integer datasourceId, + Set physicalTableNames, + Map> physicalColumnNamesByTable) { + // 全量对比语义缓存与物理结构,在内存中收集差异后统一批量更新。 + List semanticTables = tableInfoMapper.selectByDatasourceId(datasourceId); + if (semanticTables.isEmpty()) { + return List.of(); + } + List tableNames = + semanticTables.stream().map(TableInfo::getTableName).distinct().toList(); + Map> columnsByTableName = + loadSemanticColumnsByTable(datasourceId, tableNames); + List tableNamesToMarkMissing = new ArrayList<>(); + List columnIdsToMarkMissing = new ArrayList<>(); + List results = new ArrayList<>(); + + for (TableInfo tableInfo : semanticTables) { + String normalizedTableName = + SemanticUtils.normalizeObjectName( + tableInfo.getTableName(), "Missing semantic tableName."); + List columns = + columnsByTableName.getOrDefault(normalizedTableName, List.of()); + if (!physicalTableNames.contains(normalizedTableName)) { + SyncTableResult result = + buildMissingTableResult( + normalizedTableName, tableInfo, columns, columnIdsToMarkMissing); + results.add(result); + if (result.tableMarkedAsMissing()) { + tableNamesToMarkMissing.add(normalizedTableName); + } + continue; + } + + int missingColumnsMarked = + collectMissingColumnIds( + columns, + physicalColumnNamesByTable.getOrDefault(normalizedTableName, Set.of()), + columnIdsToMarkMissing); + if (missingColumnsMarked > 0) { + results.add( + SyncTableResult.builder() + .tableName(tableInfo.getTableName()) + .physicalTableFound(true) + .missingColumnsMarked(missingColumnsMarked) + .message("Physical column status refreshed") + .build()); + } + } + + LocalDateTime now = LocalDateTime.now(); + markMissingTables(datasourceId, tableNamesToMarkMissing, now); + markMissingColumns(datasourceId, columnIdsToMarkMissing, now); + return results; + } + + private void batchUpsertPresentSchema( + Integer datasourceId, List presentTables) { + if (presentTables.isEmpty()) { + return; + } + + tableInfoMapper.batchUpsertPhysicalCache( + presentTables.stream() + .map(table -> buildPhysicalTableInfo(datasourceId, table)) + .toList()); + List physicalColumns = new ArrayList<>(); + for (TableSyncSource table : presentTables) { + for (ColumnSyncSource column : table.columns()) { + physicalColumns.add( + buildPhysicalColumnInfo(datasourceId, table.tableName(), column)); + } + } + if (!physicalColumns.isEmpty()) { + columnSemanticInfoMapper.batchUpsertPhysicalCache(physicalColumns); + } + } + + private void markMissingTables( + Integer datasourceId, List tableNames, LocalDateTime now) { + if (!tableNames.isEmpty()) { + tableInfoMapper.markPhysicalMissingByDatasourceIdAndTableNames( + datasourceId, tableNames, now); + } + } + + private void markMissingColumns( + Integer datasourceId, List missingColumnIds, LocalDateTime now) { + if (!missingColumnIds.isEmpty()) { + columnSemanticInfoMapper.markPhysicalMissingByIds(datasourceId, missingColumnIds, now); + } + } + + private SyncTableResult buildPresentTableResult( + TableSyncSource table, + TableInfo existingTable, + List existingColumns, + List missingColumnIds) { + // 使用写入前快照区分新增、恢复和更新,批量 upsert 后无需再次查询数据库。 + Map existingColumnIndex = loadColumnIndex(existingColumns); + Set physicalColumnNames = + table.columns().stream() + .map(ColumnSyncSource::columnName) + .collect(Collectors.toSet()); + int addedColumns = 0; + int reactivatedColumns = 0; + int updatedColumns = 0; + for (ColumnSyncSource column : table.columns()) { + ColumnInfo existingColumn = existingColumnIndex.get(column.columnName()); + if (existingColumn == null) { + addedColumns++; + continue; + } + if (Boolean.FALSE.equals(existingColumn.getPhysicalStatus())) { + reactivatedColumns++; + } + if (!Objects.equals(existingColumn.getPhysicalColumnDescription(), column.description()) + || !Objects.equals(existingColumn.getTypeName(), column.typeName()) + || !Objects.equals(existingColumn.getPrimaryKey(), column.primaryKey())) { + updatedColumns++; + } + } + int missingColumnsMarked = + collectMissingColumnIds(existingColumns, physicalColumnNames, missingColumnIds); + return SyncTableResult.builder() + .tableName(table.displayName()) + .physicalTableFound(true) + .tableAdded(existingTable == null) + .tableReactivated( + existingTable != null + && Boolean.FALSE.equals(existingTable.getPhysicalStatus())) + .tableUpdated( + existingTable != null + && !Objects.equals( + existingTable.getPhysicalTableDescription(), + table.description())) + .addedColumns(addedColumns) + .reactivatedColumns(reactivatedColumns) + .updatedColumns(updatedColumns) + .missingColumnsMarked(missingColumnsMarked) + .message("同步完成") + .build(); + } + + private SyncTableResult buildMissingTableResult( + String tableName, + TableInfo existingTable, + List existingColumns, + List missingColumnIds) { + int missingColumnsMarked = + collectMissingColumnIds(existingColumns, Set.of(), missingColumnIds); + return SyncTableResult.builder() + .tableName(existingTable == null ? tableName : existingTable.getTableName()) + .physicalTableFound(false) + .tableMarkedAsMissing( + existingTable != null + && !Boolean.FALSE.equals(existingTable.getPhysicalStatus())) + .missingColumnsMarked(missingColumnsMarked) + .message("物理表不存在") + .build(); + } + + private Map loadSemanticTableIndex( + Integer datasourceId, List tableNames) { + Map result = new LinkedHashMap<>(); + for (TableInfo table : + tableInfoMapper.selectByDatasourceIdAndTableNames(datasourceId, tableNames)) { + String tableName = + SemanticUtils.normalizeObjectName( + table.getTableName(), "Missing semantic tableName."); + result.putIfAbsent(tableName, table); + } + return result; + } + + private Map> loadSemanticColumnsByTable( + Integer datasourceId, List tableNames) { + Map> result = new LinkedHashMap<>(); + for (ColumnInfo column : + columnSemanticInfoMapper.selectByDatasourceIdAndTableNames( + datasourceId, tableNames)) { + String tableName = + SemanticUtils.normalizeObjectName( + column.getTableName(), "Missing semantic tableName."); + result.computeIfAbsent(tableName, _key -> new ArrayList<>()).add(column); + } + return result; + } + + private Map loadColumnIndex(List columns) { + Map index = new LinkedHashMap<>(); + for (ColumnInfo column : columns) { + index.put( + SemanticUtils.normalizeObjectName( + column.getColumnName(), "Missing semantic columnName."), + column); + } + return index; + } + + private int collectMissingColumnIds( + List semanticColumns, + Set physicalColumnNames, + List missingColumnIds) { + // 收集缺失列 ID 供后续一次性更新,同时返回当前表的缺失列数量。 + int missingColumnsMarked = 0; + for (ColumnInfo column : semanticColumns) { + String columnName = + SemanticUtils.normalizeObjectName( + column.getColumnName(), "Missing semantic columnName."); + if (physicalColumnNames.contains(columnName) + || Boolean.FALSE.equals(column.getPhysicalStatus()) + || column.getId() == null) { + continue; + } + missingColumnIds.add(column.getId()); + missingColumnsMarked++; + } + return missingColumnsMarked; + } + + private TableInfo buildPhysicalTableInfo(Integer datasourceId, TableSyncSource table) { + TableInfo tableInfo = new TableInfo(); + tableInfo.setDatasourceId(datasourceId); + tableInfo.setTableName(table.tableName()); + tableInfo.setPhysicalTableDescription(table.description()); + tableInfo.setDomain(SemanticConstants.DEFAULT_DOMAIN); + tableInfo.setIsVisible(Boolean.TRUE); + tableInfo.setPhysicalStatus(Boolean.TRUE); + return tableInfo; + } + + private ColumnInfo buildPhysicalColumnInfo( + Integer datasourceId, String tableName, ColumnSyncSource column) { + ColumnInfo columnInfo = new ColumnInfo(); + columnInfo.setDatasourceId(datasourceId); + columnInfo.setTableName(tableName); + columnInfo.setColumnName(column.columnName()); + columnInfo.setPhysicalColumnDescription(column.description()); + columnInfo.setTypeName(column.typeName()); + columnInfo.setPrimaryKey(column.primaryKey()); + columnInfo.setIsVisible(Boolean.TRUE); + columnInfo.setPhysicalStatus(Boolean.TRUE); + return columnInfo; + } + + public record TableSyncSource( + String tableName, + String displayName, + String description, + List columns) {} + + public record ColumnSyncSource( + String columnName, String description, String typeName, Boolean primaryKey) {} +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncResultService.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncResultService.java new file mode 100644 index 0000000..543f088 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncResultService.java @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.service.semantic.sync; + +import io.github.malonetalk.dto.semantic.SyncTableResult; +import io.github.malonetalk.dto.semantic.SyncTableSemanticsResponse; +import java.util.List; +import org.springframework.stereotype.Service; + +@Service +public class SemanticSyncResultService { + + public SyncTableSemanticsResponse summarize(List results) { + int addedTables = 0; + int reactivatedTables = 0; + int updatedTables = 0; + int missingTablesMarked = 0; + int addedColumns = 0; + int reactivatedColumns = 0; + int updatedColumns = 0; + int missingColumnsMarked = 0; + + for (SyncTableResult result : results) { + if (result.tableAdded()) { + addedTables++; + } + if (result.tableReactivated()) { + reactivatedTables++; + } + if (result.tableUpdated()) { + updatedTables++; + } + if (result.tableMarkedAsMissing()) { + missingTablesMarked++; + } + addedColumns += result.addedColumns(); + reactivatedColumns += result.reactivatedColumns(); + updatedColumns += result.updatedColumns(); + missingColumnsMarked += result.missingColumnsMarked(); + } + + return new SyncTableSemanticsResponse( + addedTables, + reactivatedTables, + updatedTables, + missingTablesMarked, + addedColumns, + reactivatedColumns, + updatedColumns, + missingColumnsMarked, + results); + } +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncService.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncService.java new file mode 100644 index 0000000..dbd57aa --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncService.java @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.service.semantic.sync; + +import io.github.malonetalk.dto.pagination.PageResponse; +import io.github.malonetalk.dto.semantic.PhysicalTableCandidatePageQuery; +import io.github.malonetalk.dto.semantic.PhysicalTableCandidateResponse; +import io.github.malonetalk.dto.semantic.RefreshPhysicalStatusRequest; +import io.github.malonetalk.dto.semantic.SyncTableSemanticsRequest; +import io.github.malonetalk.dto.semantic.SyncTableSemanticsResponse; + +public interface SemanticSyncService { + + PageResponse getPhysicalTableCandidates( + PhysicalTableCandidatePageQuery query); + + SyncTableSemanticsResponse syncTables(SyncTableSemanticsRequest request); + + SyncTableSemanticsResponse refreshPhysicalStatus(RefreshPhysicalStatusRequest request); +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncServiceImpl.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncServiceImpl.java new file mode 100644 index 0000000..d216d29 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncServiceImpl.java @@ -0,0 +1,219 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.service.semantic.sync; + +import io.github.malonetalk.agent.datasource.SchemaReader; +import io.github.malonetalk.dto.datasource.PhysicalColumnInfo; +import io.github.malonetalk.dto.datasource.PhysicalTableInfo; +import io.github.malonetalk.dto.pagination.PageResponse; +import io.github.malonetalk.dto.semantic.PhysicalTableCandidatePageQuery; +import io.github.malonetalk.dto.semantic.PhysicalTableCandidateResponse; +import io.github.malonetalk.dto.semantic.RefreshPhysicalStatusRequest; +import io.github.malonetalk.dto.semantic.SyncTableResult; +import io.github.malonetalk.dto.semantic.SyncTableSemanticsRequest; +import io.github.malonetalk.dto.semantic.SyncTableSemanticsResponse; +import io.github.malonetalk.entity.Datasource; +import io.github.malonetalk.mapper.TableInfoMapper; +import io.github.malonetalk.service.DatasourceService; +import io.github.malonetalk.service.semantic.sync.SemanticSyncApplyService.ColumnSyncSource; +import io.github.malonetalk.service.semantic.sync.SemanticSyncApplyService.TableSyncSource; +import io.github.malonetalk.utils.SemanticUtils; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +@Service +@RequiredArgsConstructor +public class SemanticSyncServiceImpl implements SemanticSyncService { + + private final DatasourceService datasourceService; + private final TableInfoMapper tableInfoMapper; + private final SchemaReader schemaReader; + private final SemanticSyncApplyService semanticSyncApplyService; + private final SemanticSyncResultService semanticSyncResultService; + + @Override + public PageResponse getPhysicalTableCandidates( + PhysicalTableCandidatePageQuery query) { + Datasource datasource = requireDatasource(query.datasourceId()); + int pageNumber = PageResponse.resolvePage(query.page()); + int pageSize = PageResponse.resolvePageSize(query.pageSize()); + String keyword = SemanticUtils.trimToNull(query.keyword()); + Comparator comparator = + Comparator.comparing(PhysicalTableInfo::tableName, String.CASE_INSENSITIVE_ORDER); + if (SemanticUtils.isDescendingSort(query.sortOrder())) { + comparator = comparator.reversed(); + } + // SchemaReader 一次性读取物理表,筛选、排序和分页统一在内存中完成。 + List filteredTables = + schemaReader.getTables(datasource).stream() + .filter( + table -> + keyword == null + || SemanticUtils.containsIgnoreCase( + table.tableName(), keyword) + || SemanticUtils.containsIgnoreCase( + table.remarks(), keyword)) + .sorted(comparator) + .toList(); + int fromIndex = Math.min((pageNumber - 1) * pageSize, filteredTables.size()); + int toIndex = Math.min(fromIndex + pageSize, filteredTables.size()); + List pageTables = filteredTables.subList(fromIndex, toIndex); + Set syncedTableNames = loadSyncedTableNames(query.datasourceId(), pageTables); + List responses = + pageTables.stream() + .map( + table -> + new PhysicalTableCandidateResponse( + table.tableName(), + SemanticUtils.trimToNull(table.remarks()), + syncedTableNames.contains( + SemanticUtils.normalizeObjectName( + table.tableName(), + "Missing physical tableName.")))) + .toList(); + return PageResponse.of(responses, filteredTables.size(), pageNumber, pageSize); + } + + @Override + public SyncTableSemanticsResponse syncTables(SyncTableSemanticsRequest request) { + Datasource datasource = requireDatasource(request.datasourceId()); + Map physicalTables = loadPhysicalTableIndex(datasource); + Set selectedTableNames = new LinkedHashSet<>(); + for (String tableName : request.tableNames()) { + selectedTableNames.add( + SemanticUtils.normalizeObjectName(tableName, "Missing physical tableName.")); + } + + List presentTables = new ArrayList<>(); + List missingTableNames = new ArrayList<>(); + // 先读取并整理全部物理结构,再交给事务方法批量写库,避免事务中访问外部数据源。 + for (String normalizedTableName : selectedTableNames) { + PhysicalTableInfo physicalTable = physicalTables.get(normalizedTableName); + if (physicalTable == null) { + missingTableNames.add(normalizedTableName); + continue; + } + presentTables.add(readTableSyncSource(datasource, physicalTable, normalizedTableName)); + } + + List results = + semanticSyncApplyService.applyTableSync( + request.datasourceId(), presentTables, missingTableNames); + return semanticSyncResultService.summarize(results); + } + + @Override + public SyncTableSemanticsResponse refreshPhysicalStatus(RefreshPhysicalStatusRequest request) { + Datasource datasource = requireDatasource(request.datasourceId()); + Map physicalTables = loadPhysicalTableIndex(datasource); + Map> physicalColumnNamesByTable = + loadPhysicalColumnNamesByTable(datasource, physicalTables); + List results = + semanticSyncApplyService.refreshPhysicalStatus( + request.datasourceId(), + physicalTables.keySet(), + physicalColumnNamesByTable); + return semanticSyncResultService.summarize(results); + } + + private Map loadPhysicalTableIndex(Datasource datasource) { + Map result = new LinkedHashMap<>(); + for (PhysicalTableInfo table : schemaReader.getTables(datasource)) { + String tableName = + SemanticUtils.normalizeObjectName( + table.tableName(), "Missing physical tableName."); + result.putIfAbsent(tableName, table); + } + return result; + } + + private Set loadSyncedTableNames( + Integer datasourceId, List pageTables) { + // 只查询当前页表的同步状态,避免加载数据源下全部语义表。 + if (pageTables.isEmpty()) { + return Set.of(); + } + List tableNames = + pageTables.stream().map(PhysicalTableInfo::tableName).distinct().toList(); + return tableInfoMapper.selectByDatasourceIdAndTableNames(datasourceId, tableNames).stream() + .map( + table -> + SemanticUtils.normalizeObjectName( + table.getTableName(), "Missing physical tableName.")) + .collect(Collectors.toSet()); + } + + private TableSyncSource readTableSyncSource( + Datasource datasource, PhysicalTableInfo physicalTable, String normalizedTableName) { + // 在开启写事务前读取物理列快照,避免 JDBC 元数据读取延长事务时间。 + List columns = new ArrayList<>(); + for (PhysicalColumnInfo column : + schemaReader.getTableSchema(datasource, physicalTable.tableName())) { + columns.add( + new ColumnSyncSource( + SemanticUtils.normalizeObjectName( + column.columnName(), "Missing physical columnName."), + SemanticUtils.trimToNull(column.remarks()), + SemanticUtils.trimToNull(column.typeName()), + column.primaryKey())); + } + return new TableSyncSource( + normalizedTableName, + physicalTable.tableName(), + SemanticUtils.trimToNull(physicalTable.remarks()), + columns); + } + + private Map> loadPhysicalColumnNamesByTable( + Datasource datasource, Map physicalTables) { + List tableNames = + physicalTables.values().stream().map(PhysicalTableInfo::tableName).toList(); + Map> result = new LinkedHashMap<>(); + for (Map.Entry> entry : + schemaReader.getTableColumnNames(datasource, tableNames).entrySet()) { + String tableName = + SemanticUtils.normalizeObjectName( + entry.getKey(), "Missing physical tableName."); + Set columnNames = + result.computeIfAbsent(tableName, _key -> new LinkedHashSet<>()); + for (String columnName : entry.getValue()) { + columnNames.add( + SemanticUtils.normalizeObjectName( + columnName, "Missing physical columnName.")); + } + } + return result; + } + + private Datasource requireDatasource(Integer datasourceId) { + SemanticUtils.requireDatasourceId(datasourceId); + Datasource datasource = datasourceService.findById(datasourceId); + if (datasource == null) { + throw new IllegalArgumentException("Datasource does not exist: " + datasourceId); + } + return datasource; + } +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/table/TableSemanticServiceImpl.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/table/TableSemanticServiceImpl.java index 13c9875..4d4e806 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/table/TableSemanticServiceImpl.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/table/TableSemanticServiceImpl.java @@ -33,7 +33,6 @@ import io.github.malonetalk.utils.SemanticUtils; import java.time.LocalDateTime; import java.util.List; -import java.util.Locale; import java.util.Set; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; @@ -141,8 +140,8 @@ public List listMergedTablesByDomains( public void updateTableSemantic(TableSemanticUpdateRequest request) { requireDatasource(request.datasourceId()); String normalizedTableName = - SemanticUtils.trimToNotBlank(request.tableName(), "tableName") - .toLowerCase(Locale.ROOT); + SemanticUtils.normalizeObjectName( + request.tableName(), "Missing tableName for table semantic update."); TableInfo existing = tableInfoMapper.selectByDatasourceIdAndTableName( request.datasourceId(), normalizedTableName); @@ -153,6 +152,7 @@ public void updateTableSemantic(TableSemanticUpdateRequest request) { tableInfo.setTableDescription(SemanticUtils.trimToNull(request.tableDescription())); tableInfo.setDomain(SemanticUtils.normalizeDomain(request.domain())); tableInfo.setIsVisible(request.isVisible()); + tableInfo.setPhysicalStatus(Boolean.FALSE); tableInfo.setCreateTime(LocalDateTime.now()); tableInfo.setUpdateTime(LocalDateTime.now()); tableInfoMapper.insert(tableInfo); @@ -163,13 +163,15 @@ public void updateTableSemantic(TableSemanticUpdateRequest request) { existing.setDomain(SemanticUtils.normalizeDomain(request.domain())); existing.setIsVisible(request.isVisible()); existing.setUpdateTime(LocalDateTime.now()); - tableInfoMapper.update(existing); + tableInfoMapper.updateSemanticFields(existing); } @Override public void resetTableSemantic(Integer datasourceId, String tableName) { requireDatasource(datasourceId); - String normalizedTableName = SemanticUtils.trimToNotBlank(tableName, "tableName"); + String normalizedTableName = + SemanticUtils.requireTrimmed( + tableName, "Missing tableName for table semantic reset."); TableInfo existing = tableInfoMapper.selectByDatasourceIdAndTableName(datasourceId, normalizedTableName); if (existing == null) { @@ -188,17 +190,20 @@ public int resetTableSemantics(Integer datasourceId, List tableNames) { tableNames.stream() .map( name -> - SemanticUtils.trimToNotBlank(name, "tableName") - .toLowerCase(Locale.ROOT)) + SemanticUtils.normalizeObjectName( + name, + "Missing tableName for batch table semantic" + + " reset.")) .collect(Collectors.toCollection(java.util.LinkedHashSet::new)); List matchedIds = tableInfoMapper.selectByDatasourceId(datasourceId).stream() .filter( table -> normalizedNames.contains( - SemanticUtils.trimToNotBlank( - table.getTableName(), "tableName") - .toLowerCase(Locale.ROOT))) + SemanticUtils.normalizeObjectName( + table.getTableName(), + "Missing tableName while matching table" + + " semantic reset."))) .map(TableInfo::getId) .distinct() .toList(); diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/utils/SemanticUtils.java b/data-agent-backend/src/main/java/io/github/malonetalk/utils/SemanticUtils.java index 1784ad4..d041e0f 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/utils/SemanticUtils.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/utils/SemanticUtils.java @@ -59,13 +59,26 @@ public static String trimToNull(String value) { return value.trim(); } - public static String trimToNotBlank(String value, String label) { + public static String requireTrimmed(String value, String missingMessage) { if (value == null || value.isBlank()) { - throw new IllegalArgumentException(label + " cannot be blank."); + throw new IllegalArgumentException(missingMessage); } return value.trim(); } + public static String firstNonBlank(String... values) { + if (values == null) { + return null; + } + for (String value : values) { + String normalized = trimToNull(value); + if (normalized != null) { + return normalized; + } + } + return null; + } + public static String normalizeDomain(String domain) { String normalized = trimToNull(domain); return normalized == null @@ -73,6 +86,20 @@ public static String normalizeDomain(String domain) { : normalized.toLowerCase(Locale.ROOT); } + public static String normalizeObjectName(String value, String missingMessage) { + return requireTrimmed(value, missingMessage).toLowerCase(Locale.ROOT); + } + + public static boolean containsIgnoreCase(String value, String keyword) { + String normalizedValue = trimToNull(value); + String normalizedKeyword = trimToNull(keyword); + return normalizedValue != null + && normalizedKeyword != null + && normalizedValue + .toLowerCase(Locale.ROOT) + .contains(normalizedKeyword.toLowerCase(Locale.ROOT)); + } + public static String formatTableSchema(String tableName, List columns) { StringBuilder sb = new StringBuilder(); sb.append(String.format("Schema of table %s:%n", tableName)); diff --git a/data-agent-backend/src/main/resources/application.properties b/data-agent-backend/src/main/resources/application.properties index 50ae6d6..02d19fb 100644 --- a/data-agent-backend/src/main/resources/application.properties +++ b/data-agent-backend/src/main/resources/application.properties @@ -16,7 +16,7 @@ spring.datasource.hikari.connection-timeout=20000 # MyBatis Configuration mybatis.mapper-locations=classpath:mapper/*.xml -mybatis.type-aliases-package=com.malone.entity +mybatis.type-aliases-package=io.github.malonetalk.entity mybatis.configuration.map-underscore-to-camel-case=true mybatis.configuration.log-impl=org.apache.ibatis.logging.stdout.StdOutImpl diff --git a/data-agent-backend/src/main/resources/mapper/ColumnSemanticInfoMapper.xml b/data-agent-backend/src/main/resources/mapper/ColumnSemanticInfoMapper.xml index eb3cf09..87a2302 100644 --- a/data-agent-backend/src/main/resources/mapper/ColumnSemanticInfoMapper.xml +++ b/data-agent-backend/src/main/resources/mapper/ColumnSemanticInfoMapper.xml @@ -7,8 +7,12 @@ + + + + @@ -26,6 +30,16 @@ ORDER BY column_name ASC, id ASC + + + + + + INSERT INTO table_info ( - table_name, table_description, domain, datasource_id, is_visible, + table_name, physical_table_description, table_description, domain, datasource_id, + is_visible, physical_status, create_time, update_time ) VALUES ( - #{tableName}, #{tableDescription}, #{domain}, #{datasourceId}, #{isVisible}, - #{createTime}, #{updateTime} + #{tableName}, #{physicalTableDescription}, #{tableDescription}, #{domain}, + #{datasourceId}, COALESCE(#{isVisible}, 1), + COALESCE(#{physicalStatus}, 1), + COALESCE(#{createTime}, NOW()), COALESCE(#{updateTime}, NOW()) ) - + + INSERT INTO table_info ( + table_name, physical_table_description, table_description, domain, datasource_id, + is_visible, physical_status, create_time, update_time + ) VALUES + + ( + #{table.tableName}, #{table.physicalTableDescription}, #{table.tableDescription}, + #{table.domain}, #{table.datasourceId}, COALESCE(#{table.isVisible}, 1), + COALESCE(#{table.physicalStatus}, 1), + COALESCE(#{table.createTime}, NOW()), COALESCE(#{table.updateTime}, NOW()) + ) + + ON DUPLICATE KEY UPDATE + physical_table_description = VALUES(physical_table_description), + physical_status = VALUES(physical_status), + update_time = VALUES(update_time) + + + UPDATE table_info table_name = #{tableName}, table_description = #{tableDescription}, domain = #{domain}, - datasource_id = #{datasourceId}, is_visible = #{isVisible}, update_time = #{updateTime}, WHERE id = #{id} + + UPDATE table_info + + physical_table_description = #{physicalTableDescription}, + physical_status = #{physicalStatus}, + update_time = #{updateTime}, + + WHERE id = #{id} + + + + UPDATE table_info + SET physical_status = 0, + update_time = #{now} + WHERE datasource_id = #{datasourceId} + AND LOWER(table_name) IN + + LOWER(#{tableName}) + + AND (physical_status IS NULL OR physical_status <> 0) + +