From c9875a60b563fb1f0809f26d2b19a618fc6556b4 Mon Sep 17 00:00:00 2001 From: lance Date: Sat, 30 May 2026 15:00:06 +0800 Subject: [PATCH 1/3] Add batch delete option to Delete transform Signed-off-by: lance --- .../pipeline/transforms/delete/Delete.java | 44 ++-- .../transforms/delete/DeleteDialog.java | 45 +++- .../transforms/delete/DeleteKeyField.java | 44 +--- .../transforms/delete/DeleteLookupField.java | 50 +--- .../transforms/delete/DeleteMeta.java | 67 ++---- .../delete/messages/messages_en_US.properties | 4 +- .../delete/messages/messages_zh_CN.properties | 4 +- .../transforms/delete/DeleteDataTest.java | 48 ++++ .../transforms/delete/DeleteKeyFieldTest.java | 97 ++++++++ .../delete/DeleteLookupFieldTest.java | 102 ++++++++ .../delete/DeleteMetaInjectionTest.java | 34 +++ .../transforms/delete/DeleteMetaTest.java | 81 ++++--- .../transforms/delete/DeleteSqlTest.java | 177 ++++++++++++++ .../transforms/delete/DeleteTest.java | 218 ++++++++++++++++++ 14 files changed, 843 insertions(+), 172 deletions(-) create mode 100644 plugins/transforms/delete/src/test/java/org/apache/hop/pipeline/transforms/delete/DeleteDataTest.java create mode 100644 plugins/transforms/delete/src/test/java/org/apache/hop/pipeline/transforms/delete/DeleteKeyFieldTest.java create mode 100644 plugins/transforms/delete/src/test/java/org/apache/hop/pipeline/transforms/delete/DeleteLookupFieldTest.java create mode 100644 plugins/transforms/delete/src/test/java/org/apache/hop/pipeline/transforms/delete/DeleteMetaInjectionTest.java create mode 100644 plugins/transforms/delete/src/test/java/org/apache/hop/pipeline/transforms/delete/DeleteSqlTest.java create mode 100644 plugins/transforms/delete/src/test/java/org/apache/hop/pipeline/transforms/delete/DeleteTest.java diff --git a/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/Delete.java b/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/Delete.java index 87cd36d8a6d..3b3a42181db 100644 --- a/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/Delete.java +++ b/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/Delete.java @@ -36,7 +36,6 @@ /** Delete data in a database table. */ public class Delete extends BaseTransform { - private static final Class PKG = DeleteMeta.class; public Delete( @@ -75,10 +74,11 @@ private synchronized void deleteValues(IRowMeta rowMeta, Object[] row) throws Ho PKG, "Delete.Log.SetValuesForDelete", data.deleteParameterRowMeta.getString(deleteRow), - rowMeta.getString(row))); + rowMeta.getString(row), + meta.isUseBatchUpdate())); } - data.db.insertRow(data.prepStatementDelete); + data.db.insertRow(data.prepStatementDelete, meta.isUseBatchUpdate(), true); incrementLinesUpdated(); } @@ -86,10 +86,10 @@ private synchronized void deleteValues(IRowMeta rowMeta, Object[] row) throws Ho public boolean processRow() throws HopException { boolean sendToErrorRow = false; String errorMessage = null; - - Object[] r = getRow(); // Get row from input rowset & set row busy! - if (r == null) { // no more input to be expected... - + // Get row from input rowset & set row busy! + Object[] r = getRow(); + // no more input to be expected... + if (r == null) { setOutputDone(); return false; } @@ -151,24 +151,24 @@ public boolean processRow() throws HopException { } try { - deleteValues(getInputRowMeta(), r); // add new values to the row in rowset[0]. - putRow( - data.outputRowMeta, r); // output the same rows of data, but with a copy of the metadata + // add new values to the row in rowset[0]. + deleteValues(getInputRowMeta(), r); + // output the same rows of data, but with a copy of the metadata + putRow(data.outputRowMeta, r); if (checkFeedback(getLinesRead()) && isBasic()) { logBasic(BaseMessages.getString(PKG, "Delete.Log.LineNumber") + getLinesRead()); } } catch (HopException e) { - if (getTransformMeta().isDoingErrorHandling()) { sendToErrorRow = true; errorMessage = e.toString(); } else { - logError(BaseMessages.getString(PKG, "Delete.Log.ErrorInTransform") + e.getMessage()); setErrors(1); stopAll(); - setOutputDone(); // signal end to receiver(s) + // signal end to receiver(s) + setOutputDone(); return false; } @@ -225,7 +225,6 @@ public void prepareDelete(IRowMeta rowMeta) throws HopDatabaseException { @Override public boolean init() { if (super.init()) { - if (Utils.isEmpty(meta.getConnection())) { logError(BaseMessages.getString(PKG, "Delete.Init.ConnectionMissing", getTransformName())); return false; @@ -238,16 +237,13 @@ public boolean init() { } data.db = new Database(this, variables, databaseMeta); - try { data.db.connect(); - if (isDetailed()) { logDetailed(BaseMessages.getString(PKG, "Delete.Log.ConnectedToDB")); } data.db.setCommit(meta.getCommitSize(this)); - return true; } catch (HopException ke) { logError(BaseMessages.getString(PKG, "Delete.Log.ErrorOccurred") + ke.getMessage()); @@ -274,18 +270,26 @@ private void commitBatch(boolean dispose) { try { if (!data.db.isAutoCommit()) { if (getErrors() == 0) { - data.db.commit(); + if (dispose) { + data.db.emptyAndCommit(data.prepStatementDelete, meta.isUseBatchUpdate()); + data.prepStatementDelete = null; + } else { + data.db.commit(); + } } else { data.db.rollback(); } } - if (dispose) data.db.closeUpdate(); + if (dispose && data.prepStatementDelete != null) { + data.db.closePreparedStatement(data.prepStatementDelete); + data.prepStatementDelete = null; + } } catch (HopDatabaseException e) { logError( BaseMessages.getString(PKG, "Delete.Log.UnableToCommitUpdateConnection") + data.db + "] :" - + e.toString()); + + e); setErrors(1); } finally { if (dispose) { diff --git a/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/DeleteDialog.java b/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/DeleteDialog.java index be331621d51..3ee219c1706 100644 --- a/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/DeleteDialog.java +++ b/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/DeleteDialog.java @@ -69,6 +69,8 @@ public class DeleteDialog extends BaseTransformDialog { private TextVar wCommit; + private Button wBatch; + private final DeleteMeta input; private final List inputFields = new ArrayList<>(); @@ -183,12 +185,37 @@ public void widgetSelected(SelectionEvent e) { fdCommit.right = new FormAttachment(100, 0); wCommit.setLayoutData(fdCommit); + // Batch delete + Label wlBatch = new Label(shell, SWT.RIGHT); + wlBatch.setText(BaseMessages.getString(PKG, "DeleteDialog.Batch.Label")); + PropsUi.setLook(wlBatch); + FormData fdlBatch = new FormData(); + fdlBatch.left = new FormAttachment(0, 0); + fdlBatch.top = new FormAttachment(wCommit, margin); + fdlBatch.right = new FormAttachment(middle, -margin); + wlBatch.setLayoutData(fdlBatch); + wBatch = new Button(shell, SWT.CHECK); + PropsUi.setLook(wBatch); + FormData fdBatch = new FormData(); + fdBatch.left = new FormAttachment(middle, 0); + fdBatch.top = new FormAttachment(wlBatch, 0, SWT.CENTER); + fdBatch.right = new FormAttachment(100, 0); + wBatch.setLayoutData(fdBatch); + wBatch.addSelectionListener( + new SelectionAdapter() { + @Override + public void widgetSelected(SelectionEvent arg0) { + setFlags(); + input.setChanged(); + } + }); + Label wlKey = new Label(shell, SWT.NONE); wlKey.setText(BaseMessages.getString(PKG, "DeleteDialog.Key.Label")); PropsUi.setLook(wlKey); FormData fdlKey = new FormData(); fdlKey.left = new FormAttachment(0, 0); - fdlKey.top = new FormAttachment(wCommit, margin); + fdlKey.top = new FormAttachment(wBatch, margin); wlKey.setLayoutData(fdlKey); int nrKeyCols = 4; @@ -288,6 +315,7 @@ public void widgetSelected(SelectionEvent e) { getData(); setTableFieldCombo(); + setFlags(); focusTransformName(); BaseDialog.defaultShellHandling(shell, c -> ok(), c -> cancel()); @@ -309,6 +337,7 @@ public void getData() { } wCommit.setText(input.getCommitSizeVar()); + wBatch.setSelection(input.isUseBatchUpdate()); List keyFields = input.getLookup().getFields(); @@ -352,6 +381,19 @@ private void cancel() { dispose(); } + public void setFlags() { + DatabaseMeta databaseMeta = pipelineMeta.findDatabase(wConnection.getText(), variables); + boolean hasErrorHandling = pipelineMeta.findTransform(transformName).isDoingErrorHandling(); + + boolean enableBatch = wBatch.getSelection(); + enableBatch = + enableBatch + && !(databaseMeta != null + && databaseMeta.supportsErrorHandlingOnBatchUpdates() + && hasErrorHandling); + wBatch.setSelection(enableBatch); + } + private void setTableFieldCombo() { Runnable fieldLoader = () -> { @@ -409,6 +451,7 @@ private void getInfo(DeleteMeta inf) { int nrkeys = wKey.nrNonEmpty(); inf.setCommitSize(wCommit.getText()); + inf.setUseBatchUpdate(wBatch.getSelection()); if (log.isDebug()) { logDebug(BaseMessages.getString(PKG, "DeleteDialog.Log.FoundKeys", String.valueOf(nrkeys))); diff --git a/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/DeleteKeyField.java b/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/DeleteKeyField.java index ef6afd302b5..cd0543b1fad 100644 --- a/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/DeleteKeyField.java +++ b/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/DeleteKeyField.java @@ -18,9 +18,13 @@ package org.apache.hop.pipeline.transforms.delete; import java.util.Objects; +import lombok.Getter; +import lombok.Setter; import org.apache.hop.metadata.api.HopMetadataProperty; import org.apache.hop.metadata.api.HopMetadataPropertyType; +@Getter +@Setter public class DeleteKeyField { /** which field in input stream to compare with? */ @@ -65,42 +69,14 @@ public DeleteKeyField(DeleteKeyField f) { this.keyStream2 = f.keyStream2; } - public String getKeyStream() { - return keyStream; - } - - public void setKeyStream(String keyStream) { - this.keyStream = keyStream; - } - - public String getKeyLookup() { - return keyLookup; - } - - public void setKeyLookup(String keyLookup) { - this.keyLookup = keyLookup; - } - - public String getKeyCondition() { - return keyCondition; - } - - public void setKeyCondition(String keyCondition) { - this.keyCondition = keyCondition; - } - - public String getKeyStream2() { - return keyStream2; - } - - public void setKeyStream2(String keyStream2) { - this.keyStream2 = keyStream2; - } - @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } DeleteKeyField that = (DeleteKeyField) o; return keyStream.equals(that.keyStream) && keyLookup.equals(that.keyLookup) diff --git a/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/DeleteLookupField.java b/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/DeleteLookupField.java index 52db89b1450..4d1d5cec56e 100644 --- a/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/DeleteLookupField.java +++ b/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/DeleteLookupField.java @@ -20,9 +20,13 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; +import lombok.Getter; +import lombok.Setter; import org.apache.hop.metadata.api.HopMetadataProperty; import org.apache.hop.metadata.api.HopMetadataPropertyType; +@Getter +@Setter public class DeleteLookupField { @HopMetadataProperty( @@ -65,48 +69,14 @@ public DeleteLookupField(String schemaName, String tableName, List getFields() { - return fields; - } - - /** - * @param fields The fields to set - */ - public void setFields(List fields) { - this.fields = fields; - } - @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } DeleteLookupField that = (DeleteLookupField) o; return fields.equals(that.fields) && Objects.equals(schemaName, that.schemaName) diff --git a/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/DeleteMeta.java b/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/DeleteMeta.java index 60ffd46ddd1..abeefd633a3 100644 --- a/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/DeleteMeta.java +++ b/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/DeleteMeta.java @@ -18,6 +18,8 @@ package org.apache.hop.pipeline.transforms.delete; import java.util.List; +import lombok.Getter; +import lombok.Setter; import org.apache.commons.lang3.StringUtils; import org.apache.hop.core.CheckResult; import org.apache.hop.core.Const; @@ -46,6 +48,8 @@ * This class takes care of deleting values in a table using a certain condition and values for * input. */ +@Getter +@Setter @Transform( id = "Delete", image = "delete.svg", @@ -73,20 +77,19 @@ public class DeleteMeta extends BaseTransformMeta { @HopMetadataProperty(key = "commit", injectionKeyDescription = "DeleteMeta.Injection.CommitSize") private String commitSize; + /** Flag to indicate the use of batch deletes, disabled by default for backward compatibility */ + @HopMetadataProperty( + key = "use_batch", + injectionKeyDescription = "DeleteMeta.Injection.UseBatchUpdate", + injectionKey = "BATCH_UPDATE") + private boolean useBatchUpdate; + public DeleteMeta() { super(); lookup = new DeleteLookupField(); // allocate BaseTransformMeta } - public String getConnection() { - return connection; - } - - public void setConnection(String connection) { - this.connection = connection; - } - /** * @return Returns the commitSize. */ @@ -94,18 +97,6 @@ public String getCommitSizeVar() { return commitSize; } - public DeleteLookupField getLookup() { - return lookup; - } - - public void setLookup(DeleteLookupField lookup) { - this.lookup = lookup; - } - - public String getCommitSize() { - return commitSize; - } - /** * @param vs - variable variables to be used for searching variable value usually "this" for a * calling transform @@ -117,17 +108,11 @@ public int getCommitSize(IVariables vs) { return Integer.parseInt(vs.resolve(commitSize)); } - /** - * @param commitSize The commitSize to set. - */ - public void setCommitSize(String commitSize) { - this.commitSize = commitSize; - } - public DeleteMeta(DeleteMeta obj) { this.connection = obj.connection; this.commitSize = obj.commitSize; + this.useBatchUpdate = obj.useBatchUpdate; this.lookup = new DeleteLookupField(obj.lookup); } @@ -150,8 +135,7 @@ public void getFields( IRowMeta[] info, TransformMeta nextTransform, IVariables variables, - IHopMetadataProvider metadataProvider) - throws HopTransformException { + IHopMetadataProvider metadataProvider) { // Default: nothing changes to rowMeta } @@ -350,8 +334,8 @@ public SqlStatement getSqlStatements( DatabaseMeta databaseMeta = pipelineMeta.findDatabase(connection, variables); - SqlStatement retval = - new SqlStatement(transformMeta.getName(), databaseMeta, null); // default: nothing to do! + // default: nothing to do! + SqlStatement ret = new SqlStatement(transformMeta.getName(), databaseMeta, null); if (databaseMeta != null) { if (prev != null && !prev.isEmpty()) { @@ -374,20 +358,19 @@ public SqlStatement getSqlStatements( idxFields[i] = keyFields.get(i).getKeyLookup(); } } else { - retval.setError( - BaseMessages.getString(PKG, "DeleteMeta.CheckResult.KeyFieldsRequired")); + ret.setError(BaseMessages.getString(PKG, "DeleteMeta.CheckResult.KeyFieldsRequired")); } // Key lookup dimensions... if (idxFields != null && idxFields.length > 0 && !db.checkIndexExists(schemaTable, idxFields)) { - String indexname = "idx_" + lookup.getTableName() + "_lookup"; + String indexName = "idx_" + lookup.getTableName() + "_lookup"; crIndex = db.getCreateIndexStatement( lookup.getSchemaName(), lookup.getTableName(), - indexname, + indexName, idxFields, false, false, @@ -397,27 +380,27 @@ public SqlStatement getSqlStatements( String sql = crTable + crIndex; if (sql.isEmpty()) { - retval.setSql(null); + ret.setSql(null); } else { - retval.setSql(sql); + ret.setSql(sql); } } catch (HopException e) { - retval.setError( + ret.setError( BaseMessages.getString(PKG, "DeleteMeta.Returnvalue.ErrorOccurred") + e.getMessage()); } } else { - retval.setError( + ret.setError( BaseMessages.getString(PKG, "DeleteMeta.Returnvalue.NoTableDefinedOnConnection")); } } else { - retval.setError(BaseMessages.getString(PKG, "DeleteMeta.Returnvalue.NoReceivingAnyFields")); + ret.setError(BaseMessages.getString(PKG, "DeleteMeta.Returnvalue.NoReceivingAnyFields")); } } else { - retval.setError(BaseMessages.getString(PKG, "DeleteMeta.Returnvalue.NoConnectionDefined")); + ret.setError(BaseMessages.getString(PKG, "DeleteMeta.Returnvalue.NoConnectionDefined")); } - return retval; + return ret; } @Override diff --git a/plugins/transforms/delete/src/main/resources/org/apache/hop/pipeline/transforms/delete/messages/messages_en_US.properties b/plugins/transforms/delete/src/main/resources/org/apache/hop/pipeline/transforms/delete/messages/messages_en_US.properties index 284bef099dd..40af50f72ac 100644 --- a/plugins/transforms/delete/src/main/resources/org/apache/hop/pipeline/transforms/delete/messages/messages_en_US.properties +++ b/plugins/transforms/delete/src/main/resources/org/apache/hop/pipeline/transforms/delete/messages/messages_en_US.properties @@ -24,7 +24,7 @@ Delete.Log.ErrorInTransform=Error in transform, asking everyone to stop because Delete.Log.ErrorOccurred=An error occurred, processing will be stopped\: Delete.Log.FieldInfo=Field [{0}] has nr. Delete.Log.LineNumber=linenr -Delete.Log.SetValuesForDelete=Values set for delete\: {0}, input row\: {1} +Delete.Log.SetValuesForDelete=Values set for delete\: {0}, input row\: {1}, use batch: {2} Delete.Log.UnableToCommitUpdateConnection=Unable to commit Update connection [ Delete.Name=Delete DeleteDialog.AvailableSchemas.Message=Please select a schema name @@ -76,7 +76,9 @@ DeleteMeta.Injection.Field.KeyLookup=The table field DeleteMeta.Injection.Field.KeyStream=The stream field 1 DeleteMeta.Injection.Field.KeyStream2=The stream field 2 DeleteMeta.Injection.SchemaName=The name of the schema to use +DeleteDialog.Batch.Label=Use batch deletes DeleteMeta.Injection.TableName=The name of the table to use +DeleteMeta.Injection.UseBatchUpdate=Set this flag to perform batch deletes DeleteMeta.Keyword=delete DeleteMeta.Returnvalue.ErrorOccurred=An error occurred\: DeleteMeta.Returnvalue.NoConnectionDefined=There is no connection defined in this transform. diff --git a/plugins/transforms/delete/src/main/resources/org/apache/hop/pipeline/transforms/delete/messages/messages_zh_CN.properties b/plugins/transforms/delete/src/main/resources/org/apache/hop/pipeline/transforms/delete/messages/messages_zh_CN.properties index 233cf116823..fc2f4add69d 100644 --- a/plugins/transforms/delete/src/main/resources/org/apache/hop/pipeline/transforms/delete/messages/messages_zh_CN.properties +++ b/plugins/transforms/delete/src/main/resources/org/apache/hop/pipeline/transforms/delete/messages/messages_zh_CN.properties @@ -26,7 +26,7 @@ Delete.Log.ErrorInTransform=Error in transform, asking everyone to stop because Delete.Log.ErrorOccurred=An error occurred, processing will be stopped\: Delete.Log.FieldInfo=Field [{0}] has nr. Delete.Log.LineNumber=linenr -Delete.Log.SetValuesForDelete=Values set for delete\: {0}, input row\: {1} +Delete.Log.SetValuesForDelete=Values set for delete\: {0}, input row\: {1}, use batch: {2} Delete.Log.UnableToCommitUpdateConnection=Unable to commit Update connection [ Delete.Name=\u5220\u9664 DeleteDialog.AvailableSchemas.Message=\u8BF7\u9009\u62E9\u4E00\u4E2A Schema \u540D\u79F0 @@ -37,6 +37,7 @@ DeleteDialog.ColumnInfo.StreamField1=\u6D41\u91CC\u7684\u5B57\u6BB51 DeleteDialog.ColumnInfo.StreamField2=\u6D41\u91CC\u7684\u5B57\u6BB52 DeleteDialog.ColumnInfo.TableField=\u8868\u5B57\u6BB5 DeleteDialog.Commit.Label=\u63D0\u4EA4\u8BB0\u5F55\u6570\u91CF +DeleteDialog.Batch.Label=\u4F7F\u7528\u6279\u91CF\u5220\u9664 DeleteDialog.ErrorGettingSchemas=\u83B7\u53D6 Schema \u65F6\u51FA\u9519 DeleteDialog.FailedToGetFields.DialogMessage=\u65E0\u6CD5\u4ECE\u524D\u7F6E\u901A\u9053\u91CC\u83B7\u53D6\u5B57\u6BB5, \u56E0\u4E3A\u4E00\u4E2A\u9519\u8BEF DeleteDialog.FailedToGetFields.DialogTitle=\u83B7\u53D6\u5B57\u6BB5\u5931\u8D25 @@ -79,6 +80,7 @@ DeleteMeta.Injection.Field.KeyStream2=\u6D41\u5B57\u6BB5 2 DeleteMeta.Injection.Fields=\u5B57\u6BB5 DeleteMeta.Injection.SchemaName=\u8981\u4F7F\u7528\u7684\u6A21\u5F0F\u540D\u79F0 DeleteMeta.Injection.TableName=\u8981\u4F7F\u7528\u7684\u8868\u540D\u79F0 +DeleteMeta.Injection.UseBatchUpdate=\u8BBE\u7F6E\u6B64\u6807\u5FD7\u4EE5\u6267\u884C\u6279\u91CF\u5220\u9664 DeleteMeta.Keyword=delete DeleteMeta.Returnvalue.ErrorOccurred=An error occurred\: DeleteMeta.Returnvalue.NoConnectionDefined=There is no connection defined in this transform. diff --git a/plugins/transforms/delete/src/test/java/org/apache/hop/pipeline/transforms/delete/DeleteDataTest.java b/plugins/transforms/delete/src/test/java/org/apache/hop/pipeline/transforms/delete/DeleteDataTest.java new file mode 100644 index 00000000000..0063a54bb1c --- /dev/null +++ b/plugins/transforms/delete/src/test/java/org/apache/hop/pipeline/transforms/delete/DeleteDataTest.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.delete; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +import org.junit.jupiter.api.Test; + +/** Unit test for {@link DeleteData} */ +class DeleteDataTest { + + @Test + void testDefaultConstructor() { + DeleteData data = new DeleteData(); + assertNull(data.db); + assertNull(data.keynrs); + assertNull(data.keynrs2); + assertNull(data.outputRowMeta); + assertNull(data.schemaTable); + assertNull(data.deleteParameterRowMeta); + assertNull(data.prepStatementDelete); + } + + @Test + void testFieldsCanBeAssigned() { + DeleteData data = new DeleteData(); + data.schemaTable = "public.customers"; + assertNotNull(data.schemaTable); + assertEquals("public.customers", data.schemaTable); + } +} diff --git a/plugins/transforms/delete/src/test/java/org/apache/hop/pipeline/transforms/delete/DeleteKeyFieldTest.java b/plugins/transforms/delete/src/test/java/org/apache/hop/pipeline/transforms/delete/DeleteKeyFieldTest.java new file mode 100644 index 00000000000..03209a830dd --- /dev/null +++ b/plugins/transforms/delete/src/test/java/org/apache/hop/pipeline/transforms/delete/DeleteKeyFieldTest.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.delete; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import org.junit.jupiter.api.Test; + +/** Unit test for {@link DeleteKeyField} */ +class DeleteKeyFieldTest { + + @Test + void testDefaultConstructor() { + DeleteKeyField field = new DeleteKeyField(); + assertNotNull(field); + } + + @Test + void testParameterizedConstructorAndGetters() { + DeleteKeyField field = new DeleteKeyField("id", "=", "streamId", "streamId2"); + assertEquals("id", field.getKeyLookup()); + assertEquals("=", field.getKeyCondition()); + assertEquals("streamId", field.getKeyStream()); + assertEquals("streamId2", field.getKeyStream2()); + } + + @Test + void testCopyConstructor() { + DeleteKeyField original = new DeleteKeyField("name", "LIKE", "streamName", null); + DeleteKeyField copy = new DeleteKeyField(original); + assertEquals(original, copy); + assertEquals(original.hashCode(), copy.hashCode()); + } + + @Test + void testSetters() { + DeleteKeyField field = new DeleteKeyField(); + field.setKeyLookup("col"); + field.setKeyCondition(">="); + field.setKeyStream("s1"); + field.setKeyStream2("s2"); + assertEquals("col", field.getKeyLookup()); + assertEquals(">=", field.getKeyCondition()); + assertEquals("s1", field.getKeyStream()); + assertEquals("s2", field.getKeyStream2()); + } + + @Test + void testEqualsAndHashCodeWithNullStream2() { + DeleteKeyField a = new DeleteKeyField("id", "=", "streamId", null); + DeleteKeyField b = new DeleteKeyField("id", "=", "streamId", null); + DeleteKeyField c = new DeleteKeyField("id", "<>", "streamId", null); + + assertEquals(a, b); + assertEquals(a.hashCode(), b.hashCode()); + assertNotEquals(a, c); + } + + @Test + void testEqualsAndHashCodeWithBetweenCondition() { + DeleteKeyField a = new DeleteKeyField("age", "BETWEEN", "minAge", "maxAge"); + DeleteKeyField b = new DeleteKeyField("age", "BETWEEN", "minAge", "maxAge"); + DeleteKeyField c = new DeleteKeyField("age", "BETWEEN", "minAge", "otherMax"); + + assertEquals(a, b); + assertNotEquals(a, c); + } + + @Test + void testEqualsSameInstance() { + DeleteKeyField field = new DeleteKeyField("id", "=", "streamId", null); + assertEquals(field, field); + } + + @Test + void testNotEqualsDifferentClass() { + DeleteKeyField field = new DeleteKeyField("id", "=", "streamId", null); + assertNotEquals("not a DeleteKeyField", field); + } +} diff --git a/plugins/transforms/delete/src/test/java/org/apache/hop/pipeline/transforms/delete/DeleteLookupFieldTest.java b/plugins/transforms/delete/src/test/java/org/apache/hop/pipeline/transforms/delete/DeleteLookupFieldTest.java new file mode 100644 index 00000000000..0c528eb8f5b --- /dev/null +++ b/plugins/transforms/delete/src/test/java/org/apache/hop/pipeline/transforms/delete/DeleteLookupFieldTest.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.delete; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Arrays; +import java.util.Collections; +import org.junit.jupiter.api.Test; + +/** Unit test for {@link DeleteKeyField} */ +class DeleteLookupFieldTest { + + @Test + void testDefaultConstructorInitializesFieldsList() { + DeleteLookupField lookup = new DeleteLookupField(); + assertNotNull(lookup.getFields()); + assertTrue(lookup.getFields().isEmpty()); + } + + @Test + void testParameterizedConstructor() { + DeleteKeyField key = new DeleteKeyField("id", "=", "streamId", null); + DeleteLookupField lookup = + new DeleteLookupField("public", "customers", Collections.singletonList(key)); + + assertEquals("public", lookup.getSchemaName()); + assertEquals("customers", lookup.getTableName()); + assertEquals(1, lookup.getFields().size()); + assertEquals(key, lookup.getFields().getFirst()); + } + + @Test + void testCopyConstructorDeepCopiesFields() { + DeleteKeyField key = new DeleteKeyField("id", "=", "streamId", null); + DeleteLookupField original = + new DeleteLookupField("dbo", "orders", Collections.singletonList(key)); + + DeleteLookupField copy = new DeleteLookupField(original); + assertEquals(original, copy); + assertEquals(original.hashCode(), copy.hashCode()); + + copy.getFields().getFirst().setKeyLookup("changed"); + assertNotEquals( + original.getFields().getFirst().getKeyLookup(), copy.getFields().getFirst().getKeyLookup()); + } + + @Test + void testSetters() { + DeleteLookupField lookup = new DeleteLookupField(); + lookup.setSchemaName("schema1"); + lookup.setTableName("table1"); + lookup.setFields( + Arrays.asList( + new DeleteKeyField("a", "=", "b", null), new DeleteKeyField("c", "<>", "d", null))); + + assertEquals("schema1", lookup.getSchemaName()); + assertEquals("table1", lookup.getTableName()); + assertEquals(2, lookup.getFields().size()); + } + + @Test + void testEqualsAndHashCode() { + DeleteKeyField key = new DeleteKeyField("id", "=", "streamId", null); + DeleteLookupField a = + new DeleteLookupField("public", "customers", Collections.singletonList(key)); + DeleteLookupField b = + new DeleteLookupField( + "public", "customers", Collections.singletonList(new DeleteKeyField(key))); + DeleteLookupField c = + new DeleteLookupField( + "public", "orders", Collections.singletonList(new DeleteKeyField(key))); + + assertEquals(a, b); + assertEquals(a.hashCode(), b.hashCode()); + assertNotEquals(a, c); + } + + @Test + void testEqualsSameInstance() { + DeleteLookupField lookup = new DeleteLookupField(); + assertEquals(lookup, lookup); + } +} diff --git a/plugins/transforms/delete/src/test/java/org/apache/hop/pipeline/transforms/delete/DeleteMetaInjectionTest.java b/plugins/transforms/delete/src/test/java/org/apache/hop/pipeline/transforms/delete/DeleteMetaInjectionTest.java new file mode 100644 index 00000000000..5bc8545ecf7 --- /dev/null +++ b/plugins/transforms/delete/src/test/java/org/apache/hop/pipeline/transforms/delete/DeleteMetaInjectionTest.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.delete; + +import org.apache.hop.core.injection.BaseMetadataInjectionTestJunit5; +import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** DeleteMeta inject */ +class DeleteMetaInjectionTest extends BaseMetadataInjectionTestJunit5 { + @RegisterExtension + static RestoreHopEngineEnvironmentExtension env = new RestoreHopEngineEnvironmentExtension(); + + @BeforeEach + void setup() throws Exception { + setup(new DeleteMeta()); + } +} diff --git a/plugins/transforms/delete/src/test/java/org/apache/hop/pipeline/transforms/delete/DeleteMetaTest.java b/plugins/transforms/delete/src/test/java/org/apache/hop/pipeline/transforms/delete/DeleteMetaTest.java index b572d6b8ad9..9c958f4eab9 100644 --- a/plugins/transforms/delete/src/test/java/org/apache/hop/pipeline/transforms/delete/DeleteMetaTest.java +++ b/plugins/transforms/delete/src/test/java/org/apache/hop/pipeline/transforms/delete/DeleteMetaTest.java @@ -18,14 +18,14 @@ package org.apache.hop.pipeline.transforms.delete; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; -import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Random; import java.util.UUID; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.hop.core.HopEnvironment; @@ -49,9 +49,10 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; +/** Unit test for {@link DeleteMeta} */ class DeleteMetaTest implements IInitializer { - LoadSaveTester loadSaveTester; - Class testMetaClass = DeleteMeta.class; + private LoadSaveTester loadSaveTester; + private final Class testMetaClass = DeleteMeta.class; @RegisterExtension static RestoreHopEngineEnvironmentExtension env = new RestoreHopEngineEnvironmentExtension(); @@ -59,7 +60,7 @@ class DeleteMetaTest implements IInitializer { @BeforeEach void setUpLoadSave() throws Exception { PluginRegistry.init(); - List attributes = Arrays.asList("commit", "connection", "lookup"); + List attributes = Arrays.asList("commit", "connection", "lookup", "use_batch"); Map getterMap = new HashMap<>() { @@ -67,6 +68,7 @@ void setUpLoadSave() throws Exception { put("commit", "getCommitSize"); put("connection", "getConnection"); put("lookup", "getLookup"); + put("use_batch", "isUseBatchUpdate"); } }; Map setterMap = @@ -75,6 +77,7 @@ void setUpLoadSave() throws Exception { put("commit", "setCommitSize"); put("connection", "setConnection"); put("lookup", "setLookup"); + put("use_batch", "setUseBatchUpdate"); } }; @@ -142,9 +145,7 @@ void testSerialization() throws HopException { loadSaveTester.testSerialization(); } - private TransformMeta transformMeta; private Delete del; - private DeleteData data; private DeleteMeta meta; @BeforeAll @@ -158,12 +159,12 @@ void setUp() { pipelineMeta.setName("delete1"); meta = new DeleteMeta(); - data = new DeleteData(); + DeleteData data = new DeleteData(); PluginRegistry plugReg = PluginRegistry.getInstance(); String deletePid = plugReg.getPluginId(TransformPluginType.class, meta); - transformMeta = new TransformMeta(deletePid, "delete", meta); + TransformMeta transformMeta = new TransformMeta(deletePid, "delete", meta); Pipeline pipeline = new LocalPipelineEngine(pipelineMeta); Map vars = new HashMap<>(); @@ -193,36 +194,50 @@ void testCommitCountMissedVar() { meta.getCommitSize(del); fail(); } catch (Exception ex) { + // ignore ex } } - public class DeleteLookupKeyFieldInputFieldLoadSaveValidator - implements IFieldLoadSaveValidator { - final Random rand = new Random(); + @Test + void testUseBatchUpdateDefaultIsFalse() { + assertFalse(meta.isUseBatchUpdate()); + } - @Override - public DeleteLookupField getTestObject() { - return new DeleteLookupField( - UUID.randomUUID().toString(), UUID.randomUUID().toString(), new ArrayList<>()); - } + @Test + void testUseBatchUpdateSetterAndGetter() { + meta.setUseBatchUpdate(true); + assertTrue(meta.isUseBatchUpdate()); + } - @Override - public boolean validateTestObject(DeleteLookupField testObject, Object actual) { - if (!(actual instanceof DeleteLookupField)) { - return false; - } - DeleteLookupField another = (DeleteLookupField) actual; - return new EqualsBuilder() - .append(testObject.getSchemaName(), another.getSchemaName()) - .append(testObject.getTableName(), another.getTableName()) - .append(testObject.getFields(), another.getFields()) - .isEquals(); - } + @Test + void testCloneIncludesUseBatchUpdate() { + meta.setUseBatchUpdate(true); + DeleteMeta cloned = (DeleteMeta) meta.clone(); + assertTrue(cloned.isUseBatchUpdate()); + } + + @Test + void testCopyConstructorIncludesUseBatchUpdate() { + meta.setUseBatchUpdate(true); + DeleteMeta copied = new DeleteMeta(meta); + assertTrue(copied.isUseBatchUpdate()); } - public class DeleteKeyFieldInputFieldLoadSaveValidator + @Test + void testSetDefaultValues() { + DeleteMeta defaults = new DeleteMeta(); + defaults.setDefault(); + assertEquals("100", defaults.getCommitSize()); + assertFalse(defaults.isUseBatchUpdate()); + } + + @Test + void testSupportsErrorHandling() { + assertTrue(meta.supportsErrorHandling()); + } + + public static class DeleteKeyFieldInputFieldLoadSaveValidator implements IFieldLoadSaveValidator { - final Random rand = new Random(); @Override public DeleteKeyField getTestObject() { @@ -235,10 +250,10 @@ public DeleteKeyField getTestObject() { @Override public boolean validateTestObject(DeleteKeyField testObject, Object actual) { - if (!(actual instanceof DeleteKeyField)) { + if (!(actual instanceof DeleteKeyField another)) { return false; } - DeleteKeyField another = (DeleteKeyField) actual; + return new EqualsBuilder() .append(testObject.getKeyLookup(), another.getKeyLookup()) .append(testObject.getKeyCondition(), another.getKeyCondition()) diff --git a/plugins/transforms/delete/src/test/java/org/apache/hop/pipeline/transforms/delete/DeleteSqlTest.java b/plugins/transforms/delete/src/test/java/org/apache/hop/pipeline/transforms/delete/DeleteSqlTest.java new file mode 100644 index 00000000000..6fba2b08ff1 --- /dev/null +++ b/plugins/transforms/delete/src/test/java/org/apache/hop/pipeline/transforms/delete/DeleteSqlTest.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.delete; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import org.apache.hop.core.database.Database; +import org.apache.hop.core.database.DatabaseMeta; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.core.row.value.ValueMetaInteger; +import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension; +import org.apache.hop.pipeline.Pipeline; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.engines.local.LocalPipelineEngine; +import org.apache.hop.pipeline.transform.TransformMeta; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; + +/** Unit test for {@link DatabaseMeta} */ +@ExtendWith(RestoreHopEngineEnvironmentExtension.class) +class DeleteSqlTest { + private DeleteMeta meta; + private DeleteData data; + private Delete deleteTransform; + + @BeforeEach + void setUp() { + DatabaseMeta databaseMeta = mock(DatabaseMeta.class); + when(databaseMeta.getQuotedSchemaTableCombination(any(), eq("public"), eq("customers"))) + .thenReturn("\"public\".\"customers\""); + when(databaseMeta.quoteField(anyString())) + .thenAnswer(invocation -> "\"" + invocation.getArgument(0) + "\""); + when(databaseMeta.stripCR(anyString())).thenAnswer(invocation -> invocation.getArgument(0)); + + PipelineMeta pipelineMeta = spy(new PipelineMeta()); + pipelineMeta.setName("delete-sql-test"); + doReturn(databaseMeta).when(pipelineMeta).findDatabase(anyString(), any()); + + meta = new DeleteMeta(); + meta.setConnection("unit-test-db"); + meta.getLookup().setSchemaName("public"); + meta.getLookup().setTableName("customers"); + + data = new DeleteData(); + TransformMeta transformMeta = new TransformMeta("delete", meta); + pipelineMeta.addTransform(transformMeta); + + Pipeline pipeline = new LocalPipelineEngine(pipelineMeta); + deleteTransform = new Delete(transformMeta, meta, data, 0, pipelineMeta, pipeline); + data.schemaTable = "\"public\".\"customers\""; + } + + @Test + void testPrepareDeleteWithEqualsCondition() throws Exception { + meta.getLookup().getFields().add(new DeleteKeyField("id", "=", "streamId", null)); + + IRowMeta rowMeta = new RowMeta(); + rowMeta.addValueMeta(new ValueMetaInteger("streamId")); + + prepareDeleteAndCaptureSql(rowMeta); + + assertEquals(1, data.deleteParameterRowMeta.size()); + assertTrue(capturedSql().contains("DELETE FROM")); + assertTrue(capturedSql().contains("WHERE")); + assertTrue(capturedSql().contains("\"id\" = ?")); + } + + @Test + void testPrepareDeleteWithBetweenCondition() throws Exception { + meta.getLookup().getFields().add(new DeleteKeyField("age", "BETWEEN", "minAge", "maxAge")); + + IRowMeta rowMeta = new RowMeta(); + rowMeta.addValueMeta(new ValueMetaInteger("minAge")); + rowMeta.addValueMeta(new ValueMetaInteger("maxAge")); + + prepareDeleteAndCaptureSql(rowMeta); + + assertEquals(2, data.deleteParameterRowMeta.size()); + assertTrue(capturedSql().contains("\"age\" BETWEEN ? AND ?")); + } + + @Test + void testPrepareDeleteWithIsNullCondition() throws Exception { + meta.getLookup().getFields().add(new DeleteKeyField("deleted_at", "IS NULL", "", null)); + + IRowMeta rowMeta = new RowMeta(); + + prepareDeleteAndCaptureSql(rowMeta); + + assertEquals(0, data.deleteParameterRowMeta.size()); + assertTrue(capturedSql().contains("\"deleted_at\" IS NULL")); + } + + @Test + void testPrepareDeleteWithIsNotNullCondition() throws Exception { + meta.getLookup().getFields().add(new DeleteKeyField("updated_at", "IS NOT NULL", "", null)); + + IRowMeta rowMeta = new RowMeta(); + + prepareDeleteAndCaptureSql(rowMeta); + + assertEquals(0, data.deleteParameterRowMeta.size()); + assertTrue(capturedSql().contains("\"updated_at\" IS NOT NULL")); + } + + @Test + void testPrepareDeleteWithMultipleKeyConditions() throws Exception { + meta.getLookup().getFields().add(new DeleteKeyField("id", "=", "streamId", null)); + meta.getLookup().getFields().add(new DeleteKeyField("name", "<>", "streamName", null)); + meta.getLookup().getFields().add(new DeleteKeyField("status", "IS NULL", "", null)); + + IRowMeta rowMeta = new RowMeta(); + rowMeta.addValueMeta(new ValueMetaInteger("streamId")); + rowMeta.addValueMeta(new ValueMetaString("streamName")); + + prepareDeleteAndCaptureSql(rowMeta); + + assertEquals(2, data.deleteParameterRowMeta.size()); + String sql = capturedSql(); + assertTrue(sql.contains("\"id\" = ?")); + assertTrue(sql.contains("AND")); + assertTrue(sql.contains("\"name\" <> ?")); + assertTrue(sql.contains("\"status\" IS NULL")); + } + + private String capturedSqlValue; + + private void prepareDeleteAndCaptureSql(IRowMeta rowMeta) throws Exception { + Database db = mock(Database.class); + Connection connection = mock(Connection.class); + PreparedStatement preparedStatement = mock(PreparedStatement.class); + + when(db.getConnection()).thenReturn(connection); + when(connection.prepareStatement(anyString())).thenReturn(preparedStatement); + + data.db = db; + deleteTransform.prepareDelete(rowMeta); + + ArgumentCaptor sqlCaptor = ArgumentCaptor.forClass(String.class); + verify(connection).prepareStatement(sqlCaptor.capture()); + capturedSqlValue = sqlCaptor.getValue(); + } + + private String capturedSql() { + return capturedSqlValue; + } +} diff --git a/plugins/transforms/delete/src/test/java/org/apache/hop/pipeline/transforms/delete/DeleteTest.java b/plugins/transforms/delete/src/test/java/org/apache/hop/pipeline/transforms/delete/DeleteTest.java new file mode 100644 index 00000000000..0400d470376 --- /dev/null +++ b/plugins/transforms/delete/src/test/java/org/apache/hop/pipeline/transforms/delete/DeleteTest.java @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.delete; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.sql.PreparedStatement; +import org.apache.hop.core.HopEnvironment; +import org.apache.hop.core.database.Database; +import org.apache.hop.core.database.DatabaseMeta; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.logging.ILoggingObject; +import org.apache.hop.core.plugins.PluginRegistry; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.core.row.value.ValueMetaInteger; +import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension; +import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider; +import org.apache.hop.pipeline.Pipeline; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.engines.local.LocalPipelineEngine; +import org.apache.hop.pipeline.transform.TransformMeta; +import org.apache.hop.pipeline.transforms.mock.TransformMockHelper; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; + +/** Unit test for {@link Delete} */ +@MockitoSettings(strictness = Strictness.LENIENT) +class DeleteTest { + + @RegisterExtension + static RestoreHopEngineEnvironmentExtension env = new RestoreHopEngineEnvironmentExtension(); + + private TransformMockHelper mockHelper; + private DeleteMeta meta; + private DeleteData data; + private PipelineMeta pipelineMeta; + + @BeforeAll + static void initEnvironment() throws Exception { + HopEnvironment.init(); + PluginRegistry.init(); + } + + @BeforeEach + void setUp() { + mockHelper = new TransformMockHelper<>("DeleteTest", DeleteMeta.class, DeleteData.class); + when(mockHelper.logChannelFactory.create(any(), any(ILoggingObject.class))) + .thenReturn(mockHelper.iLogChannel); + when(mockHelper.pipeline.isRunning()).thenReturn(true); + when(mockHelper.transformMeta.isPartitioned()).thenReturn(false); + + pipelineMeta = spy(new PipelineMeta()); + pipelineMeta.setName("delete-test"); + + DatabaseMeta databaseMeta = mock(DatabaseMeta.class); + when(databaseMeta.getQuotedSchemaTableCombination(any(), anyString(), anyString())) + .thenReturn("\"customers\""); + doReturn(databaseMeta).when(pipelineMeta).findDatabase(anyString(), any()); + + meta = new DeleteMeta(); + meta.setConnection("unit-test-db"); + meta.getLookup().setTableName("customers"); + meta.getLookup().getFields().add(new DeleteKeyField("id", "=", "streamId", null)); + + data = new DeleteData(); + } + + @AfterEach + void tearDown() { + mockHelper.cleanUp(); + } + + private Delete newSpyTransform() { + TransformMeta transformMeta = new TransformMeta("delete", meta); + pipelineMeta.addTransform(transformMeta); + Pipeline pipeline = new LocalPipelineEngine(pipelineMeta); + Delete delete = spy(new Delete(transformMeta, meta, data, 0, pipelineMeta, pipeline)); + delete.setMetadataProvider(new MemoryMetadataProvider()); + return delete; + } + + @Test + void testInitFailsWhenConnectionMissing() { + meta.setConnection(null); + Delete delete = newSpyTransform(); + assertFalse(delete.init()); + } + + @Test + void testInitFailsWhenConnectionNotFound() { + meta.setConnection("missing-connection"); + Delete delete = newSpyTransform(); + assertFalse(delete.init()); + } + + @Test + void testProcessRowUsesBatchWhenEnabled() throws HopException { + meta.setUseBatchUpdate(true); + + Database db = mock(Database.class); + PreparedStatement preparedStatement = mock(PreparedStatement.class); + data.db = db; + + Delete delete = newSpyTransform(); + + IRowMeta inputRowMeta = new RowMeta(); + inputRowMeta.addValueMeta(new ValueMetaInteger("streamId")); + doReturn(new Object[] {1L}).doReturn(null).when(delete).getRow(); + doReturn(inputRowMeta).when(delete).getInputRowMeta(); + doAnswer( + inv -> { + data.prepStatementDelete = preparedStatement; + data.deleteParameterRowMeta = new RowMeta(); + data.deleteParameterRowMeta.addValueMeta(new ValueMetaInteger("streamId")); + return null; + }) + .when(delete) + .prepareDelete(any()); + + assertTrue(delete.processRow()); + verify(db).insertRow(preparedStatement, true, true); + assertFalse(delete.processRow()); + } + + @Test + void testProcessRowDoesNotUseBatchWhenDisabled() throws HopException { + meta.setUseBatchUpdate(false); + + Database db = mock(Database.class); + PreparedStatement preparedStatement = mock(PreparedStatement.class); + data.db = db; + + Delete delete = newSpyTransform(); + + IRowMeta inputRowMeta = new RowMeta(); + inputRowMeta.addValueMeta(new ValueMetaInteger("streamId")); + doReturn(new Object[] {1L}).doReturn(null).when(delete).getRow(); + doReturn(inputRowMeta).when(delete).getInputRowMeta(); + doAnswer( + inv -> { + data.prepStatementDelete = preparedStatement; + data.deleteParameterRowMeta = new RowMeta(); + data.deleteParameterRowMeta.addValueMeta(new ValueMetaInteger("streamId")); + return null; + }) + .when(delete) + .prepareDelete(any()); + + assertTrue(delete.processRow()); + verify(db).insertRow(preparedStatement, false, true); + } + + @Test + void testDisposeCallsEmptyAndCommitWhenBatchEnabled() throws Exception { + meta.setUseBatchUpdate(true); + + Database db = mock(Database.class); + PreparedStatement preparedStatement = mock(PreparedStatement.class); + when(db.isAutoCommit()).thenReturn(false); + + data.db = db; + data.prepStatementDelete = preparedStatement; + + Delete delete = newSpyTransform(); + delete.dispose(); + + verify(db).emptyAndCommit(preparedStatement, true); + verify(db).disconnect(); + } + + @Test + void testDisposeCallsEmptyAndCommitWhenBatchDisabled() throws Exception { + meta.setUseBatchUpdate(false); + + Database db = mock(Database.class); + PreparedStatement preparedStatement = mock(PreparedStatement.class); + when(db.isAutoCommit()).thenReturn(false); + + data.db = db; + data.prepStatementDelete = preparedStatement; + + Delete delete = newSpyTransform(); + delete.dispose(); + + verify(db).emptyAndCommit(preparedStatement, false); + verify(db).disconnect(); + } +} From c704688421ffc532882414cab7d0b1146c48f539 Mon Sep 17 00:00:00 2001 From: lance Date: Sat, 30 May 2026 15:10:39 +0800 Subject: [PATCH 2/3] Add batch delete option to Delete transform Signed-off-by: lance --- .../hop/pipeline/transforms/delete/DeleteDialog.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/DeleteDialog.java b/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/DeleteDialog.java index 3ee219c1706..1ac068c40f4 100644 --- a/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/DeleteDialog.java +++ b/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/DeleteDialog.java @@ -381,6 +381,13 @@ private void cancel() { dispose(); } + /** + * Updates dialog control states based on the current configuration. + * + *

Batch deletes are disabled when this transform uses error handling and the selected database + * does not support batch updates together with error handling (for example mysql and + * look-likes). + */ public void setFlags() { DatabaseMeta databaseMeta = pipelineMeta.findDatabase(wConnection.getText(), variables); boolean hasErrorHandling = pipelineMeta.findTransform(transformName).isDoingErrorHandling(); From 6008534aabbe5c4514b1cd719da509e2111e4ff8 Mon Sep 17 00:00:00 2001 From: lance Date: Sun, 31 May 2026 19:31:34 +0800 Subject: [PATCH 3/3] Add batch delete option to Delete transform Signed-off-by: lance --- .../apache/hop/pipeline/transforms/delete/DeleteDialog.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/DeleteDialog.java b/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/DeleteDialog.java index 1ac068c40f4..cc0c4427826 100644 --- a/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/DeleteDialog.java +++ b/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/DeleteDialog.java @@ -385,8 +385,7 @@ private void cancel() { * Updates dialog control states based on the current configuration. * *

Batch deletes are disabled when this transform uses error handling and the selected database - * does not support batch updates together with error handling (for example mysql and - * look-likes). + * does not support batch updates together with error handling (for example mysql and look-likes). */ public void setFlags() { DatabaseMeta databaseMeta = pipelineMeta.findDatabase(wConnection.getText(), variables);