diff --git a/.dockerignore b/.dockerignore index da90c3875ba..71f30a9ef77 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,6 +5,7 @@ !/assemblies/client/target/hop-* !/assemblies/client/target/hop !/integration-tests/scripts +!/integration-tests/ssh/keys !/assemblies/web/target !/assemblies/plugins/target !/docker diff --git a/docker/integration-tests/integration-tests-ssh.yaml b/docker/integration-tests/integration-tests-ssh.yaml new file mode 100644 index 00000000000..e55ffef12d7 --- /dev/null +++ b/docker/integration-tests/integration-tests-ssh.yaml @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +services: + integration_test_ssh: + extends: + file: integration-tests-base.yaml + service: integration_test + depends_on: + ssh: + condition: service_started + links: + - ssh + + ssh: + build: + context: ../../. + dockerfile: docker/integration-tests/ssh.Dockerfile diff --git a/docker/integration-tests/ssh.Dockerfile b/docker/integration-tests/ssh.Dockerfile index fab96c1a113..5e8ee87af7c 100644 --- a/docker/integration-tests/ssh.Dockerfile +++ b/docker/integration-tests/ssh.Dockerfile @@ -17,7 +17,9 @@ # under the License. # # Minimal SSH server for integration tests: allows password auth and TCP forwarding -# so Hop can tunnel to Postgres through this container. +# so Hop can tunnel to Postgres through this container. Public-key auth is also +# enabled for the SSH transform private-key integration test; the matching key +# pair lives in integration-tests/ssh/keys (test-only, never used outside the IT). # FROM alpine:3.19 @@ -27,9 +29,20 @@ RUN apk add --no-cache openssh && \ echo "hop:hop_ssh_password" | chpasswd && \ sed -i 's/^#\?PermitRootLogin.*/PermitRootLogin no/' /etc/ssh/sshd_config && \ sed -i 's/^#\?PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config && \ + sed -i 's/^#\?PubkeyAuthentication.*/PubkeyAuthentication yes/' /etc/ssh/sshd_config && \ sed -i 's/^#\?AllowTcpForwarding.*/AllowTcpForwarding yes/' /etc/ssh/sshd_config && \ sed -i 's/^#\?GatewayPorts.*/GatewayPorts no/' /etc/ssh/sshd_config +# Authorize the integration-test public key for the hop user. sshd enforces +# StrictModes, so ownership and permissions must be exact or pubkey auth fails. +COPY integration-tests/ssh/keys/it_rsa.pub /tmp/it_rsa.pub +RUN mkdir -p /home/hop/.ssh && \ + cat /tmp/it_rsa.pub > /home/hop/.ssh/authorized_keys && \ + rm -f /tmp/it_rsa.pub && \ + chown -R hop:hop /home/hop/.ssh && \ + chmod 700 /home/hop/.ssh && \ + chmod 600 /home/hop/.ssh/authorized_keys + EXPOSE 22 CMD ["/usr/sbin/sshd", "-D"] diff --git a/integration-tests/ssh/dev-env-config.json b/integration-tests/ssh/dev-env-config.json new file mode 100644 index 00000000000..c9af17cfe62 --- /dev/null +++ b/integration-tests/ssh/dev-env-config.json @@ -0,0 +1,24 @@ +{ + "variables" : [ + { + "name" : "SSH_HOSTNAME", + "value" : "ssh", + "description" : "Hostname of the SSH server container (service name in integration-tests-ssh.yaml)" + }, + { + "name" : "SSH_USER", + "value" : "hop", + "description" : "SSH user created in docker/integration-tests/ssh.Dockerfile" + }, + { + "name" : "SSH_PASSWORD", + "value" : "hop_ssh_password", + "description" : "Password for the SSH user (see ssh.Dockerfile)" + }, + { + "name" : "SSH_PORT", + "value" : "22", + "description" : "" + } + ] +} diff --git a/integration-tests/ssh/hop-config.json b/integration-tests/ssh/hop-config.json new file mode 100644 index 00000000000..102ac981d58 --- /dev/null +++ b/integration-tests/ssh/hop-config.json @@ -0,0 +1,290 @@ +{ + "variables": [ + { + "name": "HOP_LENIENT_STRING_TO_NUMBER_CONVERSION", + "value": "N", + "description": "System wide flag to allow lenient string to number conversion for backward compatibility. If this setting is set to \"Y\", an string starting with digits will be converted successfully into a number. (example: 192.168.1.1 will be converted into 192 or 192.168 or 192168 depending on the decimal and grouping symbol). The default (N) will be to throw an error if non-numeric symbols are found in the string." + }, + { + "name": "HOP_COMPATIBILITY_DB_IGNORE_TIMEZONE", + "value": "N", + "description": "System wide flag to ignore timezone while writing date/timestamp value to the database." + }, + { + "name": "HOP_LOG_SIZE_LIMIT", + "value": "0", + "description": "The log size limit for all pipelines and workflows that don't have the \"log size limit\" property set in their respective properties." + }, + { + "name": "HOP_EMPTY_STRING_DIFFERS_FROM_NULL", + "value": "N", + "description": "NULL vs Empty String. If this setting is set to Y, an empty string and null are different. Otherwise they are not." + }, + { + "name": "HOP_MAX_LOG_SIZE_IN_LINES", + "value": "0", + "description": "The maximum number of log lines that are kept internally by Hop. Set to 0 to keep all rows (default)" + }, + { + "name": "HOP_MAX_LOG_TIMEOUT_IN_MINUTES", + "value": "1440", + "description": "The maximum age (in minutes) of a log line while being kept internally by Hop. Set to 0 to keep all rows indefinitely (default)" + }, + { + "name": "HOP_MAX_WORKFLOW_TRACKER_SIZE", + "value": "5000", + "description": "The maximum number of workflow trackers kept in memory" + }, + { + "name": "HOP_MAX_ACTIONS_LOGGED", + "value": "5000", + "description": "The maximum number of action results kept in memory for logging purposes." + }, + { + "name": "HOP_MAX_LOGGING_REGISTRY_SIZE", + "value": "10000", + "description": "The maximum number of logging registry entries kept in memory for logging purposes." + }, + { + "name": "HOP_LOG_TAB_REFRESH_DELAY", + "value": "1000", + "description": "The hop log tab refresh delay." + }, + { + "name": "HOP_LOG_TAB_REFRESH_PERIOD", + "value": "1000", + "description": "The hop log tab refresh period." + }, + { + "name": "HOP_PLUGIN_CLASSES", + "value": null, + "description": "A comma delimited list of classes to scan for plugin annotations" + }, + { + "name": "HOP_PLUGIN_PACKAGES", + "value": null, + "description": "A comma delimited list of packages to scan for plugin annotations (warning: slow!!)" + }, + { + "name": "HOP_TRANSFORM_PERFORMANCE_SNAPSHOT_LIMIT", + "value": "0", + "description": "The maximum number of transform performance snapshots to keep in memory. Set to 0 to keep all snapshots indefinitely (default)" + }, + { + "name": "HOP_ROWSET_GET_TIMEOUT", + "value": "50", + "description": "The name of the variable that optionally contains an alternative rowset get timeout (in ms). This only makes a difference for extremely short lived pipelines." + }, + { + "name": "HOP_ROWSET_PUT_TIMEOUT", + "value": "50", + "description": "The name of the variable that optionally contains an alternative rowset put timeout (in ms). This only makes a difference for extremely short lived pipelines." + }, + { + "name": "HOP_CORE_TRANSFORMS_FILE", + "value": null, + "description": "The name of the project variable that will contain the alternative location of the hop-transforms.xml file. You can use this to customize the list of available internal transforms outside of the codebase." + }, + { + "name": "HOP_CORE_WORKFLOW_ACTIONS_FILE", + "value": null, + "description": "The name of the project variable that will contain the alternative location of the hop-workflow-actions.xml file." + }, + { + "name": "HOP_SERVER_OBJECT_TIMEOUT_MINUTES", + "value": "1440", + "description": "This project variable will set a time-out after which waiting, completed or stopped pipelines and workflows will be automatically cleaned up. The default value is 1440 (one day)." + }, + { + "name": "HOP_PIPELINE_PAN_JVM_EXIT_CODE", + "value": null, + "description": "Set this variable to an integer that will be returned as the Pan JVM exit code." + }, + { + "name": "HOP_DISABLE_CONSOLE_LOGGING", + "value": "N", + "description": "Set this variable to Y to disable standard Hop logging to the console. (stdout)" + }, + { + "name": "HOP_REDIRECT_STDERR", + "value": "N", + "description": "Set this variable to Y to redirect stderr to Hop logging." + }, + { + "name": "HOP_REDIRECT_STDOUT", + "value": "N", + "description": "Set this variable to Y to redirect stdout to Hop logging." + }, + { + "name": "HOP_DEFAULT_NUMBER_FORMAT", + "value": null, + "description": "The name of the variable containing an alternative default number format" + }, + { + "name": "HOP_DEFAULT_BIGNUMBER_FORMAT", + "value": null, + "description": "The name of the variable containing an alternative default bignumber format" + }, + { + "name": "HOP_DEFAULT_INTEGER_FORMAT", + "value": null, + "description": "The name of the variable containing an alternative default integer format" + }, + { + "name": "HOP_DEFAULT_DATE_FORMAT", + "value": null, + "description": "The name of the variable containing an alternative default date format" + }, + { + "name": "HOP_DEFAULT_TIMESTAMP_FORMAT", + "value": null, + "description": "The name of the variable containing an alternative default timestamp format" + }, + { + "name": "HOP_DEFAULT_SERVLET_ENCODING", + "value": null, + "description": "Defines the default encoding for servlets, leave it empty to use Java default encoding" + }, + { + "name": "HOP_FAIL_ON_LOGGING_ERROR", + "value": "N", + "description": "Set this variable to Y when you want the workflow/pipeline fail with an error when the related logging process (e.g. to a database) fails." + }, + { + "name": "HOP_AGGREGATION_MIN_NULL_IS_VALUED", + "value": "N", + "description": "Set this variable to Y to set the minimum to NULL if NULL is within an aggregate. Otherwise by default NULL is ignored by the MIN aggregate and MIN is set to the minimum value that is not NULL. See also the variable HOP_AGGREGATION_ALL_NULLS_ARE_ZERO." + }, + { + "name": "HOP_AGGREGATION_ALL_NULLS_ARE_ZERO", + "value": "N", + "description": "Set this variable to Y to return 0 when all values within an aggregate are NULL. Otherwise by default a NULL is returned when all values are NULL." + }, + { + "name": "HOP_COMPATIBILITY_TEXT_FILE_OUTPUT_APPEND_NO_HEADER", + "value": "N", + "description": "Set this variable to Y for backward compatibility for the Text File Output transform. Setting this to Ywill add no header row at all when the append option is enabled, regardless if the file is existing or not." + }, + { + "name": "HOP_PASSWORD_ENCODER_PLUGIN", + "value": "Hop", + "description": "Specifies the password encoder plugin to use by ID (Hop is the default)." + }, + { + "name": "HOP_SYSTEM_HOSTNAME", + "value": null, + "description": "You can use this variable to speed up hostname lookup. Hostname lookup is performed by Hop so that it is capable of logging the server on which a workflow or pipeline is executed." + }, + { + "name": "HOP_SERVER_JETTY_ACCEPTORS", + "value": null, + "description": "A variable to configure jetty option: acceptors for Carte" + }, + { + "name": "HOP_SERVER_JETTY_ACCEPT_QUEUE_SIZE", + "value": null, + "description": "A variable to configure jetty option: acceptQueueSize for Carte" + }, + { + "name": "HOP_SERVER_JETTY_RES_MAX_IDLE_TIME", + "value": null, + "description": "A variable to configure jetty option: lowResourcesMaxIdleTime for Carte" + }, + { + "name": "HOP_COMPATIBILITY_MERGE_ROWS_USE_REFERENCE_STREAM_WHEN_IDENTICAL", + "value": "N", + "description": "Set this variable to Y for backward compatibility for the Merge Rows (diff) transform. Setting this to Y will use the data from the reference stream (instead of the comparison stream) in case the compared rows are identical." + }, + { + "name": "HOP_SPLIT_FIELDS_REMOVE_ENCLOSURE", + "value": "false", + "description": "Set this variable to false to preserve enclosure symbol after splitting the string in the Split fields transform. Changing it to true will remove first and last enclosure symbol from the resulting string chunks." + }, + { + "name": "HOP_ALLOW_EMPTY_FIELD_NAMES_AND_TYPES", + "value": "false", + "description": "Set this variable to TRUE to allow your pipeline to pass 'null' fields and/or empty types." + }, + { + "name": "HOP_GLOBAL_LOG_VARIABLES_CLEAR_ON_EXPORT", + "value": "false", + "description": "Set this variable to false to preserve global log variables defined in pipeline / workflow Properties -> Log panel. Changing it to true will clear it when export pipeline / workflow." + }, + { + "name": "HOP_FILE_OUTPUT_MAX_STREAM_COUNT", + "value": "1024", + "description": "This project variable is used by the Text File Output transform. It defines the max number of simultaneously open files within the transform. The transform will close/reopen files as necessary to insure the max is not exceeded" + }, + { + "name": "HOP_FILE_OUTPUT_MAX_STREAM_LIFE", + "value": "0", + "description": "This project variable is used by the Text File Output transform. It defines the max number of milliseconds between flushes of files opened by the transform." + }, + { + "name": "HOP_USE_NATIVE_FILE_DIALOG", + "value": "N", + "description": "Set this value to Y if you want to use the system file open/save dialog when browsing files" + }, + { + "name": "HOP_AUTO_CREATE_CONFIG", + "value": "Y", + "description": "Set this value to N if you don't want to automatically create a hop configuration file (hop-config.json) when it's missing" + } + ], + "LocaleDefault": "en_BE", + "guiProperties": { + "FontFixedSize": "13", + "MaxUndo": "100", + "DarkMode": "Y", + "FontNoteSize": "13", + "ShowOSLook": "Y", + "FontFixedStyle": "0", + "FontNoteName": ".AppleSystemUIFont", + "FontFixedName": "Monospaced", + "FontGraphStyle": "0", + "FontDefaultSize": "13", + "GraphColorR": "255", + "FontGraphSize": "13", + "IconSize": "32", + "BackgroundColorB": "255", + "FontNoteStyle": "0", + "FontGraphName": ".AppleSystemUIFont", + "FontDefaultName": ".AppleSystemUIFont", + "GraphColorG": "255", + "UseGlobalFileBookmarks": "Y", + "FontDefaultStyle": "0", + "GraphColorB": "255", + "BackgroundColorR": "255", + "BackgroundColorG": "255", + "WorkflowDialogStyle": "RESIZE,MAX,MIN", + "LineWidth": "1", + "ContextDialogShowCategories": "Y" + }, + "projectsConfig": { + "enabled": true, + "projectMandatory": true, + "environmentMandatory": true, + "defaultProject": "default", + "defaultEnvironment": null, + "standardParentProject": "default", + "standardProjectsFolder": null, + "projectConfigurations": [ + { + "projectName": "default", + "projectHome": "${HOP_CONFIG_FOLDER}", + "configFilename": "project-config.json" + } + ], + "lifecycleEnvironments": [ + { + "name": "dev", + "purpose": "Testing", + "projectName": "default", + "configurationFiles": [ + "${PROJECT_HOME}/dev-env-config.json" + ] + } + ], + "projectLifecycles": [] + } +} \ No newline at end of file diff --git a/integration-tests/ssh/keys/it_rsa b/integration-tests/ssh/keys/it_rsa new file mode 100644 index 00000000000..052134925f1 --- /dev/null +++ b/integration-tests/ssh/keys/it_rsa @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAu6+/AjE3Sjl4d0lPlUkNexCZhTI6M2LLIHIRS8IhxyxDtM9l +/IX4bFdn0tAHNBJf4DDNT4YWatMAox7UyDL7y8RHPGDWpc74WDrjMfIsu6VVZHd/ +lmsJNgrMWvhv+KJhH2GASjAreYGI03lVt7+zxXO1oZ+B8+HnUKgFLes3F4No4FxO +JVKlz2bVsf6E8tN+OfQypQtYSAb6rPJjsw82jlpPrZzT5vcJc9juLLPmAJBCo/4z +QwJc7XCiNidwcH7m0iagHA9iuhCtMRczSvTFhptNoMpfuj+IzwusqNOjU/K0P1nQ +7sCaDnOiq3aV3GU7sLqBokEhx5BMi17ynzdbdQIDAQABAoIBAAlIGrEYD/zqtKtp +g7cFQtZoLr9oiXpLE3KKUZKmihcYeEyzyP5g/bUV6XuCcCjCE925bB3Xqrojry9h +8fHom40rKr6wp0zR3HQ4jU3GBTJObdenFTcyGeWDSTHigV8RYK41myuQEEZVApg6 +suOAZnqIS14vzjRqYo8ZkBACRtoxvq+c4M9I/UyXz33bA79sMjB1gWDbeI1JPVk2 +7noUneu2iKe20l9LZf5t3osyBRFXxgE23sLpvty4nyoILF1qWEf/6Hx/P/md3kCQ +Vt0ftX3lcLwV9RuFykyO66Aql79kCbJ9EC72nSzzJvnna2KbybwRBOifi6R0PF7a +SJH84MECgYEA8gO4ICWyoTEm4yuynGWdLa/k00xOiP8kUJE19FfQuPn2D/fO+J3I +r1eeYKEjm0si0pqGVy74dnBslL3yaelCSsN2uf/ozFO2cwSTu83OmcbnTLD96z1y +bd92U84ujqwJ7wzXH9nGObOVUQWc/LoVmNZM7zu812amkGRrIT+rtLECgYEAxohR +DQwzNJjKfE5VQqRwZsMdDNrDfcnPa5fN1jKjCsi5htvWskgsHhfQ8rlAzySTe32/ +lSoJSUPyw2f86aayZ5UH2mNCv7iXX9EoH3Wt0tbuU3AFO2uXAzZCodUE+pdWtvAZ +vhzkkJJzN5PPOlikIEwwrJLfQ4Gz4QhVGP5vFAUCgYEAjLAgz39asl3yb0kt0cE4 +eCCycyr+1KENqVBg/yQ1j/KvWmUCioCe81+KED5chqBNJAT0Z6ZEhgWg+W7ahzs0 +cGXklQfxeyaG/6H/h8OCgN6iA3E4ixHzfW/UR6+qXQIh3DeorzlYBJ8jBDCxLDG4 +8FpT6xbdFpLz7SiTJobu+GECgYA5g8BHUWN8N09h16enmM/fVWMTGEVOKarndqDx +DtZhB2mIAiQenf358dhcmQKHgAch3XolEnqCOScZKQUCA4LnsysFP4BU3nssDQHc +q1DiJdYBYhCB+FdVXODM1VON7U33zXMHuoMUxviN/0onkwppOOlY9WusuOSNqsZM +aVlwqQKBgFB8KjS6fVCVePQhRSUUvO17lSFGwCSZeuuHO5nM0euFk5Vn8eMzzgI+ +sn2MUvacLuY9OSf0PXkoyl2Hew7IkFIByoaPXMDuYvCTObMDaTrWucnaP7MCvOZM +FHI5w/tU+JQi/dnbBznKti/rW4JzkPrdOKA58/QagGBzbEfV/1Wu +-----END RSA PRIVATE KEY----- diff --git a/integration-tests/ssh/keys/it_rsa.pub b/integration-tests/ssh/keys/it_rsa.pub new file mode 100644 index 00000000000..a035d279ff3 --- /dev/null +++ b/integration-tests/ssh/keys/it_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC7r78CMTdKOXh3SU+VSQ17EJmFMjozYssgchFLwiHHLEO0z2X8hfhsV2fS0Ac0El/gMM1PhhZq0wCjHtTIMvvLxEc8YNalzvhYOuMx8iy7pVVkd3+Wawk2Csxa+G/4omEfYYBKMCt5gYjTeVW3v7PFc7Whn4Hz4edQqAUt6zcXg2jgXE4lUqXPZtWx/oTy03459DKlC1hIBvqs8mOzDzaOWk+tnNPm9wlz2O4ss+YAkEKj/jNDAlztcKI2J3BwfubSJqAcD2K6EK0xFzNK9MWGm02gyl+6P4jPC6yo06NT8rQ/WdDuwJoOc6KrdpXcZTuwuoGiQSHHkEyLXvKfN1t1 hop-integration-test-key (test only) diff --git a/integration-tests/ssh/main-0001-ssh-command-password.hwf b/integration-tests/ssh/main-0001-ssh-command-password.hwf new file mode 100644 index 00000000000..91f833fadcb --- /dev/null +++ b/integration-tests/ssh/main-0001-ssh-command-password.hwf @@ -0,0 +1,94 @@ + + + + main-0001-ssh-command-password + Y + Run a command over SSH using password authentication and assert the result + + + - + 2026/06/02 00:00:00.000 + - + 2026/06/02 00:00:00.000 + + + + + Start + + SPECIAL + + 1 + N + 12 + 60 + 0 + 0 + N + 0 + 1 + N + 50 + 50 + + + + ssh-command-password.hpl + + PIPELINE + + ${PROJECT_HOME}/ssh-command-password.hpl + N + N + N + N + N + + + N + N + Basic + N + Y + N + N + local + + Y + + N + 195 + 50 + + + + + + Start + ssh-command-password.hpl + Y + Y + Y + + + + + + diff --git a/integration-tests/ssh/main-0002-ssh-command-private-key.hwf b/integration-tests/ssh/main-0002-ssh-command-private-key.hwf new file mode 100644 index 00000000000..8ba4e4b9f84 --- /dev/null +++ b/integration-tests/ssh/main-0002-ssh-command-private-key.hwf @@ -0,0 +1,94 @@ + + + + main-0002-ssh-command-private-key + Y + Run a command over SSH using private-key authentication and assert the result + + + - + 2026/06/02 00:00:00.000 + - + 2026/06/02 00:00:00.000 + + + + + Start + + SPECIAL + + 1 + N + 12 + 60 + 0 + 0 + N + 0 + 1 + N + 50 + 50 + + + + ssh-command-private-key.hpl + + PIPELINE + + ${PROJECT_HOME}/ssh-command-private-key.hpl + N + N + N + N + N + + + N + N + Basic + N + Y + N + N + local + + Y + + N + 195 + 50 + + + + + + Start + ssh-command-private-key.hpl + Y + Y + Y + + + + + + diff --git a/integration-tests/ssh/metadata/pipeline-run-configuration/local.json b/integration-tests/ssh/metadata/pipeline-run-configuration/local.json new file mode 100644 index 00000000000..0c0d87c84d2 --- /dev/null +++ b/integration-tests/ssh/metadata/pipeline-run-configuration/local.json @@ -0,0 +1,22 @@ +{ + "engineRunConfiguration": { + "Local": { + "feedback_size": "50000", + "sample_size": "100", + "sample_type_in_gui": "Last", + "wait_time": "20", + "rowset_size": "10000", + "safe_mode": false, + "show_feedback": false, + "topo_sort": false, + "gather_metrics": false, + "transactional": false + } + }, + "defaultSelection": true, + "configurationVariables": [], + "name": "local", + "description": "", + "dataProfile": "", + "executionInfoLocationName": "" +} \ No newline at end of file diff --git a/integration-tests/ssh/metadata/workflow-run-configuration/local.json b/integration-tests/ssh/metadata/workflow-run-configuration/local.json new file mode 100644 index 00000000000..1d0cf74baec --- /dev/null +++ b/integration-tests/ssh/metadata/workflow-run-configuration/local.json @@ -0,0 +1,11 @@ +{ + "engineRunConfiguration": { + "Local": { + "safe_mode": false, + "transactional": false + } + }, + "defaultSelection": true, + "name": "local", + "description": "Runs your workflows locally with the standard local Hop workflow engine" +} \ No newline at end of file diff --git a/integration-tests/ssh/project-config.json b/integration-tests/ssh/project-config.json new file mode 100644 index 00000000000..6a91171e1c8 --- /dev/null +++ b/integration-tests/ssh/project-config.json @@ -0,0 +1,13 @@ +{ + "metadataBaseFolder" : "${PROJECT_HOME}/metadata", + "unitTestsBasePath" : "${PROJECT_HOME}", + "dataSetsCsvFolder" : "${PROJECT_HOME}/datasets", + "enforcingExecutionInHome" : true, + "config" : { + "variables" : [ { + "name" : "HOP_LICENSE_HEADER_FILE", + "value" : "${PROJECT_HOME}/../asf-header.txt", + "description" : "This will automatically serialize the ASF license header into pipelines and workflows in the integration test projects" + } ] + } +} \ No newline at end of file diff --git a/integration-tests/ssh/ssh-command-password.hpl b/integration-tests/ssh/ssh-command-password.hpl new file mode 100644 index 00000000000..c781bf89eca --- /dev/null +++ b/integration-tests/ssh/ssh-command-password.hpl @@ -0,0 +1,164 @@ + + + + + ssh-command-password + Y + Runs a command over SSH (password auth) against the test SSH server and asserts the echoed marker comes back on stdOut. + Normal + -1 + + N + 1000 + 100 + - + 2026/06/02 00:00:00.000 + - + 2026/06/02 00:00:00.000 + + + Run SSH command + SSH + + Y + + 1 + + none + + + echo hop-ssh-it-marker + N + + ${SSH_HOSTNAME} + ${SSH_PORT} + ${SSH_USER} + ${SSH_PASSWORD} + N + + Encrypted + stdOut + stdErr + 30 + + + + Encrypted + + + 176 + 112 + + + + marker not found? + FilterRows + + Y + + 1 + + none + + + OK + Abort + + + N + stdOut + CONTAINS + + + constant + String + hop-ssh-it-marker + -1 + -1 + N + + + + + + + 352 + 112 + + + + OK + Dummy + + Y + + 1 + + none + + + + + 528 + 64 + + + + Abort + Abort + + Y + + 1 + + none + + + 0 + SSH transform did not return the expected marker on stdOut + Y + ABORT_WITH_ERROR + + + 528 + 176 + + + + + Run SSH command + marker not found? + Y + + + marker not found? + OK + Y + + + marker not found? + Abort + Y + + + + + + diff --git a/integration-tests/ssh/ssh-command-private-key.hpl b/integration-tests/ssh/ssh-command-private-key.hpl new file mode 100644 index 00000000000..9634617b78e --- /dev/null +++ b/integration-tests/ssh/ssh-command-private-key.hpl @@ -0,0 +1,164 @@ + + + + + ssh-command-private-key + Y + Runs a command over SSH using private-key authentication against the test SSH server and asserts the echoed marker comes back on stdOut. + Normal + -1 + + N + 1000 + 100 + - + 2026/06/02 00:00:00.000 + - + 2026/06/02 00:00:00.000 + + + Run SSH command + SSH + + Y + + 1 + + none + + + echo hop-ssh-it-marker + N + + ${SSH_HOSTNAME} + ${SSH_PORT} + ${SSH_USER} + Encrypted + Y + ${PROJECT_HOME}/keys/it_rsa + Encrypted + stdOut + stdErr + 30 + + + + Encrypted + + + 176 + 112 + + + + marker not found? + FilterRows + + Y + + 1 + + none + + + OK + Abort + + + N + stdOut + CONTAINS + + + constant + String + hop-ssh-it-marker + -1 + -1 + N + + + + + + + 352 + 112 + + + + OK + Dummy + + Y + + 1 + + none + + + + + 528 + 64 + + + + Abort + Abort + + Y + + 1 + + none + + + 0 + SSH transform (private key) did not return the expected marker on stdOut + Y + ABORT_WITH_ERROR + + + 528 + 176 + + + + + Run SSH command + marker not found? + Y + + + marker not found? + OK + Y + + + marker not found? + Abort + Y + + + + + + diff --git a/plugins/transforms/ssh/pom.xml b/plugins/transforms/ssh/pom.xml index 2c15e4faa59..69f85e6c732 100644 --- a/plugins/transforms/ssh/pom.xml +++ b/plugins/transforms/ssh/pom.xml @@ -29,15 +29,28 @@ jar Hop Plugins Transforms SSH - - 2.2.43 - + + + + org.apache.hop + hop-libs + ${project.version} + pom + import + + + - org.connectbot - sshlib - ${sshlib.version} + com.github.mwiede + jsch + + + org.apache.sshd + sshd-core + 2.16.0 + test diff --git a/plugins/transforms/ssh/src/assembly/assembly.xml b/plugins/transforms/ssh/src/assembly/assembly.xml index e134f098341..12819fb1185 100644 --- a/plugins/transforms/ssh/src/assembly/assembly.xml +++ b/plugins/transforms/ssh/src/assembly/assembly.xml @@ -48,9 +48,10 @@ - org.connectbot:sshlib:jar + com.github.mwiede:jsch:jar plugins/transforms/ssh/lib + false - \ No newline at end of file + diff --git a/plugins/transforms/ssh/src/main/java/org/apache/hop/pipeline/transforms/ssh/SessionResult.java b/plugins/transforms/ssh/src/main/java/org/apache/hop/pipeline/transforms/ssh/SessionResult.java index 7777b0b60d0..d24f50618e8 100644 --- a/plugins/transforms/ssh/src/main/java/org/apache/hop/pipeline/transforms/ssh/SessionResult.java +++ b/plugins/transforms/ssh/src/main/java/org/apache/hop/pipeline/transforms/ssh/SessionResult.java @@ -17,106 +17,104 @@ package org.apache.hop.pipeline.transforms.ssh; -import com.trilead.ssh2.Session; -import java.io.BufferedReader; +import com.jcraft.jsch.ChannelExec; +import com.jcraft.jsch.JSchException; +import com.jcraft.jsch.Session; +import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import lombok.Getter; +import lombok.Setter; import org.apache.hop.core.exception.HopException; -import org.apache.hop.core.util.Utils; public class SessionResult { - private String stdOut; - private String stdErr; - private boolean stdErrorType; + private static final int CHANNEL_POLL_MS = 50; + private static final int CHANNEL_MAX_WAIT_MS = 120_000; - public SessionResult(Session session) throws HopException { - readStd(session); + @Getter @Setter private String stdOut; + @Getter private String stdErr; + @Getter @Setter private boolean stdErrorType; + + public static SessionResult executeCommand(Session session, String command) throws HopException { + ChannelExec channel = null; + try { + channel = (ChannelExec) session.openChannel("exec"); + channel.setCommand(command); + channel.connect(); + SessionResult result = new SessionResult(); + result.readFromChannel(channel); + return result; + } catch (JSchException e) { + throw new HopException(e); + } finally { + if (channel != null) { + channel.disconnect(); + } + } } private void setStdErr(String value) { this.stdErr = value; - if (!Utils.isEmpty(getStdErr())) { - setStdTypeErr(true); + if (stdErr != null && !stdErr.isEmpty()) { + setStdErrorType(true); } } - public String getStdErr() { - return this.stdErr; - } - public String getStd() { - return getStdOut() + getStdErr(); - } - - private void setStdOut(String value) { - this.stdOut = value; - } - - public String getStdOut() { - return this.stdOut; - } - - private void setStdTypeErr(boolean value) { - this.stdErrorType = value; + return (getStdOut() == null ? "" : getStdOut()) + (getStdErr() == null ? "" : getStdErr()); } - public boolean isStdTypeErr() { - return this.stdErrorType; - } - - private void readStd(Session session) throws HopException { - InputStream isOut = null; - InputStream isErr = null; + private void readFromChannel(ChannelExec channel) throws HopException { try { - isOut = session.getStdout(); - isErr = session.getStderr(); - - setStdOut(readInputStream(isOut)); - setStdErr(readInputStream(isErr)); - - } catch (Exception e) { - throw new HopException(e); - } finally { - try { - if (isOut != null) { - isOut.close(); - } - if (isErr != null) { - isErr.close(); + InputStream isOut = channel.getInputStream(); + InputStream isErr = channel.getErrStream(); + byte[] buffer = new byte[8192]; + StringBuilder stdout = new StringBuilder(); + StringBuilder stderr = new StringBuilder(); + + long deadline = System.currentTimeMillis() + CHANNEL_MAX_WAIT_MS; + while (true) { + appendAvailable(isOut, buffer, stdout); + appendAvailable(isErr, buffer, stderr); + if (channel.isClosed()) { + if (isStreamDrained(isOut) && isStreamDrained(isErr)) { + break; + } + } else if (System.currentTimeMillis() > deadline) { + throw new HopException("Timed out waiting for SSH command to complete"); + } else { + try { + Thread.sleep(CHANNEL_POLL_MS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new HopException(e); + } } - } catch (Exception e) { - /* Ignore */ } + + setStdOut(stdout.toString()); + setStdErr(stderr.toString()); + } catch (IOException e) { + throw new HopException(e); } } - private String readInputStream(InputStream std) throws HopException { - BufferedReader br = null; - try { - br = new BufferedReader(new InputStreamReader(std)); - - String line = ""; - StringBuilder stringStdout = new StringBuilder(); - - if ((line = br.readLine()) != null) { - stringStdout.append(line); - } - while ((line = br.readLine()) != null) { - stringStdout.append("\n" + line); - } - - return stringStdout.toString(); - } catch (Exception e) { - throw new HopException(e); - } finally { - try { - if (br != null) { - br.close(); - } - } catch (Exception e) { - /* Ignore */ + private static void appendAvailable(InputStream in, byte[] buffer, StringBuilder target) + throws IOException { + if (in == null) { + return; + } + while (in.available() > 0) { + int read = in.read(buffer, 0, buffer.length); + if (read < 0) { + break; } + target.append(new String(buffer, 0, read, StandardCharsets.UTF_8)); } } + + private boolean isStreamDrained(InputStream in) throws IOException { + return in == null || in.available() == 0; + } } diff --git a/plugins/transforms/ssh/src/main/java/org/apache/hop/pipeline/transforms/ssh/Ssh.java b/plugins/transforms/ssh/src/main/java/org/apache/hop/pipeline/transforms/ssh/Ssh.java index 25e6256c383..c5218843f11 100644 --- a/plugins/transforms/ssh/src/main/java/org/apache/hop/pipeline/transforms/ssh/Ssh.java +++ b/plugins/transforms/ssh/src/main/java/org/apache/hop/pipeline/transforms/ssh/Ssh.java @@ -17,7 +17,6 @@ package org.apache.hop.pipeline.transforms.ssh; -import com.trilead.ssh2.Session; import org.apache.hop.core.exception.HopException; import org.apache.hop.core.row.IRowMeta; import org.apache.hop.core.row.RowMeta; @@ -44,7 +43,6 @@ public Ssh( @Override public boolean processRow() throws HopException { - Object[] row; if (meta.isDynamicCommandField()) { row = getRow(); @@ -76,7 +74,8 @@ public boolean processRow() throws HopException { } } else { if (!data.wroteOneRow) { - row = new Object[] {}; // empty row + // empty row + row = new Object[] {}; incrementLinesRead(); data.wroteOneRow = true; if (first) { @@ -101,12 +100,12 @@ public boolean processRow() throws HopException { } // Reserve room Object[] rowData = new Object[data.nrOutputFields]; - for (int i = 0; i < data.nrInputFields; i++) { - rowData[i] = row[i]; // no data is changed, clone is not needed here. + // no data is changed, clone is not needed here. + if (data.nrInputFields >= 0) { + System.arraycopy(row, 0, rowData, 0, data.nrInputFields); } int index = data.nrInputFields; - Session session = null; try { if (meta.isDynamicCommandField()) { // get commands @@ -116,8 +115,6 @@ public boolean processRow() throws HopException { } } - // Open a session - session = data.conn.openSession(); if (isDebug()) { logDebug(BaseMessages.getString(PKG, "SSH.Log.SessionOpened")); } @@ -126,10 +123,7 @@ public boolean processRow() throws HopException { if (isDetailed()) { logDetailed(BaseMessages.getString(PKG, "SSH.Log.RunningCommand", data.commands)); } - session.execCommand(data.commands); - - // Read Stdout, Sterr and exitStatus - SessionResult sessionresult = new SessionResult(session); + SessionResult sessionresult = SessionResult.executeCommand(data.session, data.commands); if (isDebug()) { logDebug( BaseMessages.getString( @@ -145,7 +139,7 @@ public boolean processRow() throws HopException { if (!Utils.isEmpty(data.stdTypeField)) { // Add stdtype to output - rowData[index++] = sessionresult.isStdTypeErr(); + rowData[index] = sessionresult.isStdErrorType(); } if (isRowLevel()) { @@ -160,30 +154,23 @@ public boolean processRow() throws HopException { logDetailed(BaseMessages.getString(PKG, "SSH.LineNumber", "" + getLinesRead())); } } catch (Exception e) { - - boolean sendToErrorRow = false; String errorMessage = null; if (getTransformMeta().isDoingErrorHandling()) { - sendToErrorRow = true; errorMessage = e.toString(); } else { logError(BaseMessages.getString(PKG, "SSH.ErrorInTransformRunning") + e.getMessage()); setErrors(1); stopAll(); - setOutputDone(); // signal end to receiver(s) + // signal end to receiver(s) + setOutputDone(); return false; } - if (sendToErrorRow) { - // Simply add this row to the error row - putError(getInputRowMeta(), row, 1, errorMessage, null, "SSH001"); - } + // Simply add this row to the error row + putError(getInputRowMeta(), row, 1, errorMessage, null, "SSH001"); } finally { - if (session != null) { - session.close(); - if (isDebug()) { - logDebug(BaseMessages.getString(PKG, "SSH.Log.SessionClosed")); - } + if (isDebug()) { + logDebug(BaseMessages.getString(PKG, "SSH.Log.SessionClosed")); } } @@ -216,7 +203,7 @@ public boolean init() { try { // Open connection - data.conn = SshData.openConnection(this, meta); + data.session = SshData.openConnection(this, meta); if (isDebug()) { logDebug(BaseMessages.getString(PKG, "SSH.Log.ConnectionOpened")); @@ -231,4 +218,13 @@ public boolean init() { } return false; } + + @Override + public void dispose() { + if (data.session != null && data.session.isConnected()) { + data.session.disconnect(); + data.session = null; + } + super.dispose(); + } } diff --git a/plugins/transforms/ssh/src/main/java/org/apache/hop/pipeline/transforms/ssh/SshData.java b/plugins/transforms/ssh/src/main/java/org/apache/hop/pipeline/transforms/ssh/SshData.java index 23f62121198..61433b0cfc8 100644 --- a/plugins/transforms/ssh/src/main/java/org/apache/hop/pipeline/transforms/ssh/SshData.java +++ b/plugins/transforms/ssh/src/main/java/org/apache/hop/pipeline/transforms/ssh/SshData.java @@ -17,14 +17,13 @@ package org.apache.hop.pipeline.transforms.ssh; -import com.trilead.ssh2.Connection; -import com.trilead.ssh2.HTTPProxyData; -import java.io.CharArrayWriter; -import java.io.InputStream; +import com.jcraft.jsch.JSch; +import com.jcraft.jsch.JSchException; +import com.jcraft.jsch.ProxyHTTP; +import com.jcraft.jsch.Session; import java.nio.charset.StandardCharsets; -import org.apache.commons.io.IOUtils; +import java.util.Properties; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.vfs2.FileContent; import org.apache.commons.vfs2.FileObject; import org.apache.hop.core.Const; import org.apache.hop.core.encryption.Encr; @@ -40,8 +39,9 @@ @SuppressWarnings("java:S1104") public class SshData extends BaseTransformData implements ITransformData { private static final Class PKG = SshMeta.class; + public int indexOfCommand; - public Connection conn; + public Session session; public boolean wroteOneRow; public String commands; public int nrInputFields; @@ -56,17 +56,15 @@ public class SshData extends BaseTransformData implements ITransformData { public SshData() { super(); this.indexOfCommand = -1; - this.conn = null; + this.session = null; this.wroteOneRow = false; this.commands = null; this.stdOutField = null; this.stdTypeField = null; } - public static Connection openConnection(IVariables variables, SshMeta meta) throws HopException { - Connection connection = null; - char[] content = null; - boolean isAuthenticated; + public static Session openConnection(IVariables variables, SshMeta meta) throws HopException { + Session session = null; String hostname = variables.resolve(meta.getServerName()); int port = Const.toInt(variables.resolve(meta.getPort()), 22); @@ -77,80 +75,91 @@ public static Connection openConnection(IVariables variables, SshMeta meta) thro Encr.decryptPasswordOptionallyEncrypted(variables.resolve(meta.getPassPhrase())); try { - // perform some checks + JSch jsch = new JSch(); + if (meta.isUsePrivateKey()) { String keyFilename = variables.resolve(meta.getKeyFileName()); if (StringUtils.isEmpty(keyFilename)) { throw new HopException(BaseMessages.getString(PKG, "SSH.Error.PrivateKeyFileMissing")); } FileObject keyFileObject = HopVfs.getFileObject(keyFilename, variables); - if (!keyFileObject.exists()) { throw new HopException( BaseMessages.getString(PKG, "SSH.Error.PrivateKeyNotExist", keyFilename)); } - - FileContent keyFileContent = keyFileObject.getContent(); - - CharArrayWriter charArrayWriter = new CharArrayWriter((int) keyFileContent.getSize()); - - try (InputStream in = keyFileContent.getInputStream()) { - IOUtils.copy(in, charArrayWriter, StandardCharsets.UTF_8); + if (keyFileObject.getName().getURI().startsWith("file:")) { + String localKeyPath = HopVfs.getFilename(keyFileObject); + if (Utils.isEmpty(passPhrase)) { + jsch.addIdentity(localKeyPath); + } else { + jsch.addIdentity(localKeyPath, passPhrase); + } + } else { + byte[] keyBytes = keyFileObject.getContent().getByteArray(); + byte[] passphraseBytes = + Utils.isEmpty(passPhrase) ? new byte[0] : passPhrase.getBytes(StandardCharsets.UTF_8); + jsch.addIdentity(username, keyBytes, null, passphraseBytes); } + } + + session = jsch.getSession(username, hostname, port); - content = charArrayWriter.toCharArray(); + if (!meta.isUsePrivateKey() && !Utils.isEmpty(password)) { + session.setPassword(password); } - // Create a new connection - connection = new Connection(hostname, port); + Properties config = new Properties(); + config.put("StrictHostKeyChecking", "no"); + config.put("PreferredAuthentications", "publickey,keyboard-interactive,password"); + session.setConfig(config); String proxyHost = variables.resolve(meta.getProxyHost()); - int proxyPort = Const.toInt(variables.resolve(meta.getProxyPort()), 23); + int proxyPort = Const.toInt(variables.resolve(meta.getProxyPort()), 80); String proxyUsername = variables.resolve(meta.getProxyUsername()); String proxyPassword = Encr.decryptPasswordOptionallyEncrypted(variables.resolve(meta.getProxyPassword())); - /* We want to connect through a HTTP proxy */ if (!Utils.isEmpty(proxyHost)) { - /* Now connect */ - // if the proxy requires basic authentication: + ProxyHTTP proxy = new ProxyHTTP(proxyHost + ":" + proxyPort); if (!Utils.isEmpty(proxyUsername)) { - connection.setProxyData( - new HTTPProxyData(proxyHost, proxyPort, proxyUsername, proxyPassword)); - } else { - connection.setProxyData(new HTTPProxyData(proxyHost, proxyPort)); + proxy.setUserPasswd(proxyUsername, proxyPassword); } + session.setProxy(proxy); } int timeOut = Const.toInt(variables.resolve(meta.getTimeOut()), 0); - - // and connect if (timeOut == 0) { - connection.connect(); + session.connect(); } else { - connection.connect(null, 0, timeOut * 1000); + session.connect(timeOut * 1000); } - // authenticate - if (meta.isUsePrivateKey()) { - isAuthenticated = - connection.authenticateWithPublicKey(username, content, variables.resolve(passPhrase)); - } else { - isAuthenticated = connection.authenticateWithPassword(username, password); - } - if (!isAuthenticated) { + if (!session.isConnected()) { throw new HopException( BaseMessages.getString(PKG, "SSH.Error.AuthenticationFailed", username)); } + } catch (JSchException e) { + if (session != null) { + session.disconnect(); + } + if (isAuthFailure(e)) { + throw new HopException( + BaseMessages.getString(PKG, "SSH.Error.AuthenticationFailed", username), e); + } + throw new HopException( + BaseMessages.getString(PKG, "SSH.Error.ErrorConnecting", hostname, username), e); } catch (Exception e) { - // Something wrong happened - // do not forget to disconnect if connected - if (connection != null) { - connection.close(); + if (session != null) { + session.disconnect(); } throw new HopException( BaseMessages.getString(PKG, "SSH.Error.ErrorConnecting", hostname, username), e); } - return connection; + return session; + } + + private static boolean isAuthFailure(JSchException e) { + String message = e.getMessage(); + return message != null && message.toLowerCase().contains("auth fail"); } } diff --git a/plugins/transforms/ssh/src/main/java/org/apache/hop/pipeline/transforms/ssh/SshDialog.java b/plugins/transforms/ssh/src/main/java/org/apache/hop/pipeline/transforms/ssh/SshDialog.java index d47575fc840..1fbb665b25c 100644 --- a/plugins/transforms/ssh/src/main/java/org/apache/hop/pipeline/transforms/ssh/SshDialog.java +++ b/plugins/transforms/ssh/src/main/java/org/apache/hop/pipeline/transforms/ssh/SshDialog.java @@ -17,7 +17,7 @@ package org.apache.hop.pipeline.transforms.ssh; -import com.trilead.ssh2.Connection; +import com.jcraft.jsch.Session; import org.apache.hop.core.Const; import org.apache.hop.core.Props; import org.apache.hop.core.exception.HopException; @@ -714,20 +714,20 @@ private void get() { private void test() { Exception exception = null; String errMsg = null; - Connection connection = null; + Session session = null; SshMeta meta = new SshMeta(); getInfo(meta); try { - connection = SshData.openConnection(variables, meta); + session = SshData.openConnection(variables, meta); } catch (Exception e) { exception = e; errMsg = e.getMessage(); } finally { - if (connection != null) { + if (session != null) { try { - connection.close(); + session.disconnect(); } catch (Exception e) { /* Ignore */ } diff --git a/plugins/transforms/ssh/src/main/java/org/apache/hop/pipeline/transforms/ssh/SshMeta.java b/plugins/transforms/ssh/src/main/java/org/apache/hop/pipeline/transforms/ssh/SshMeta.java index 428a86cbbdb..a276bddbc81 100644 --- a/plugins/transforms/ssh/src/main/java/org/apache/hop/pipeline/transforms/ssh/SshMeta.java +++ b/plugins/transforms/ssh/src/main/java/org/apache/hop/pipeline/transforms/ssh/SshMeta.java @@ -18,6 +18,8 @@ package org.apache.hop.pipeline.transforms.ssh; import java.util.List; +import lombok.Getter; +import lombok.Setter; import org.apache.hop.core.CheckResult; import org.apache.hop.core.ICheckResult; import org.apache.hop.core.annotations.Transform; @@ -36,6 +38,8 @@ import org.apache.hop.pipeline.transform.BaseTransformMeta; import org.apache.hop.pipeline.transform.TransformMeta; +@Getter +@Setter @Transform( id = "SSH", image = "ssh.svg", @@ -199,140 +203,4 @@ public void getFields( public boolean supportsErrorHandling() { return true; } - - public String getCommand() { - return command; - } - - public void setCommand(String command) { - this.command = command; - } - - public boolean isDynamicCommandField() { - return dynamicCommandField; - } - - public void setDynamicCommandField(boolean dynamicCommandField) { - this.dynamicCommandField = dynamicCommandField; - } - - public String getCommandFieldName() { - return commandFieldName; - } - - public void setCommandFieldName(String commandFieldName) { - this.commandFieldName = commandFieldName; - } - - public String getServerName() { - return serverName; - } - - public void setServerName(String serverName) { - this.serverName = serverName; - } - - public String getPort() { - return port; - } - - public void setPort(String port) { - this.port = port; - } - - public String getUserName() { - return userName; - } - - public void setUserName(String userName) { - this.userName = userName; - } - - public String getPassword() { - return password; - } - - public void setPassword(String password) { - this.password = password; - } - - public boolean isUsePrivateKey() { - return usePrivateKey; - } - - public void setUsePrivateKey(boolean usePrivateKey) { - this.usePrivateKey = usePrivateKey; - } - - public String getKeyFileName() { - return keyFileName; - } - - public void setKeyFileName(String keyFileName) { - this.keyFileName = keyFileName; - } - - public String getPassPhrase() { - return passPhrase; - } - - public void setPassPhrase(String passPhrase) { - this.passPhrase = passPhrase; - } - - public String getStdOutFieldName() { - return stdOutFieldName; - } - - public void setStdOutFieldName(String stdOutFieldName) { - this.stdOutFieldName = stdOutFieldName; - } - - public String getStdErrFieldName() { - return stdErrFieldName; - } - - public void setStdErrFieldName(String stdErrFieldName) { - this.stdErrFieldName = stdErrFieldName; - } - - public String getTimeOut() { - return timeOut; - } - - public void setTimeOut(String timeOut) { - this.timeOut = timeOut; - } - - public String getProxyHost() { - return proxyHost; - } - - public void setProxyHost(String proxyHost) { - this.proxyHost = proxyHost; - } - - public String getProxyPort() { - return proxyPort; - } - - public void setProxyPort(String proxyPort) { - this.proxyPort = proxyPort; - } - - public String getProxyUsername() { - return proxyUsername; - } - - public void setProxyUsername(String proxyUsername) { - this.proxyUsername = proxyUsername; - } - - public String getProxyPassword() { - return proxyPassword; - } - - public void setProxyPassword(String proxyPassword) { - this.proxyPassword = proxyPassword; - } } diff --git a/plugins/transforms/ssh/src/main/samples/transforms/ssh-password.hpl b/plugins/transforms/ssh/src/main/samples/transforms/ssh-password.hpl new file mode 100644 index 00000000000..836d3de978b --- /dev/null +++ b/plugins/transforms/ssh/src/main/samples/transforms/ssh-password.hpl @@ -0,0 +1,97 @@ + + + + + N + 1000 + 100 + Normal + -1 + + New pipeline + Y + - + - + 2026/05/30 21:21:57.866 + 2026/05/30 21:21:57.866 + + + WriteToLog + Write to log + Y + N + 0 + Basic + + Y + 1 + + 256 + 112 + + + none + + + + + + SSH + Run SSH commands + mkdir -p /opt/soft/ssh && echo "hop-ssh-test $(date '+%Y-%m-%d %H:%M:%S')" > /opt/soft/ssh/hop-test.txt && ls -la /opt/soft/ssh/hop-test.txt + N + + 192.168.204.131 + 22 + root + Encrypted 2be98afc86aa7f2e4cb79ff228dc6fa8c + N + + Encrypted + stdOut + stdErr + 30 + + + + Encrypted + Y + 1 + + 112 + 112 + + + none + + + + + + + Run SSH commands + Write to log + Y + + + + + + diff --git a/plugins/transforms/ssh/src/main/samples/transforms/ssh-pri-key.hpl b/plugins/transforms/ssh/src/main/samples/transforms/ssh-pri-key.hpl new file mode 100644 index 00000000000..29fa11ba1a6 --- /dev/null +++ b/plugins/transforms/ssh/src/main/samples/transforms/ssh-pri-key.hpl @@ -0,0 +1,97 @@ + + + + + N + 1000 + 100 + Normal + -1 + + New pipeline + Y + - + - + 2026/05/30 21:19:10.506 + 2026/05/30 21:19:10.506 + + + WriteToLog + Write to log + Y + N + 0 + Basic + + Y + 1 + + 272 + 80 + + + none + + + + + + SSH + Run SSH commands + mkdir -p /opt/soft/ssh && echo "hop-ssh-test $(date '+%Y-%m-%d %H:%M:%S')" > /opt/soft/ssh/hop-test.txt && ls -la /opt/soft/ssh/hop-test.txt + N + + 192.168.204.131 + 22 + root + Encrypted + Y + D:\tmp\hopspace\file\hop-gui + Encrypted 2be98afc86aa7f2e4cb79ce78db9ea3d5 + stdOut + stdErr + 30 + + + + Encrypted + Y + 1 + + 128 + 80 + + + none + + + + + + + Run SSH commands + Write to log + Y + + + + + + diff --git a/plugins/transforms/ssh/src/test/java/org/apache/hop/pipeline/transforms/ssh/SessionResultTest.java b/plugins/transforms/ssh/src/test/java/org/apache/hop/pipeline/transforms/ssh/SessionResultTest.java new file mode 100644 index 00000000000..8b03971cd5e --- /dev/null +++ b/plugins/transforms/ssh/src/test/java/org/apache/hop/pipeline/transforms/ssh/SessionResultTest.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.ssh; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.jcraft.jsch.ChannelExec; +import com.jcraft.jsch.JSchException; +import com.jcraft.jsch.Session; +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import org.apache.hop.core.exception.HopException; +import org.junit.jupiter.api.Test; + +class SessionResultTest { + + @Test + void getStdWithStdoutOnly() { + SessionResult result = new SessionResult(); + result.setStdOut("out"); + assertEquals("out", result.getStd()); + assertFalse(result.isStdErrorType()); + } + + @Test + void executeCommandReadsStdoutAndStderrFromChannel() throws Exception { + Session session = mock(Session.class); + ChannelExec channel = mock(ChannelExec.class); + when(session.openChannel("exec")).thenReturn(channel); + doNothing().when(channel).connect(); + when(channel.getInputStream()).thenReturn(stream("line1\nline2\n")); + when(channel.getErrStream()).thenReturn(stream("oops\n")); + when(channel.isClosed()).thenReturn(true); + + SessionResult result = SessionResult.executeCommand(session, "echo test"); + + assertEquals("line1\nline2\n", result.getStdOut()); + assertEquals("oops\n", result.getStdErr()); + assertTrue(result.isStdErrorType()); + assertEquals("line1\nline2\noops\n", result.getStd()); + verify(channel).setCommand("echo test"); + verify(channel).disconnect(); + } + + @Test + void executeCommandWrapsJSchException() throws Exception { + Session session = mock(Session.class); + when(session.openChannel("exec")).thenThrow(new JSchException("channel failed")); + + HopException ex = + assertThrows(HopException.class, () -> SessionResult.executeCommand(session, "id")); + + assertInstanceOf(JSchException.class, ex.getCause()); + } + + private static InputStream stream(String text) { + return new ByteArrayInputStream(text.getBytes(StandardCharsets.UTF_8)); + } +} diff --git a/plugins/transforms/ssh/src/test/java/org/apache/hop/pipeline/transforms/ssh/SshDataTest.java b/plugins/transforms/ssh/src/test/java/org/apache/hop/pipeline/transforms/ssh/SshDataTest.java new file mode 100644 index 00000000000..4df8d4a713d --- /dev/null +++ b/plugins/transforms/ssh/src/test/java/org/apache/hop/pipeline/transforms/ssh/SshDataTest.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.ssh; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.jcraft.jsch.Session; +import java.io.IOException; +import java.nio.file.Path; +import java.security.PublicKey; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension; +import org.apache.sshd.server.SshServer; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; + +/** Unit test for {@link SshData} */ +@ExtendWith(RestoreHopEngineEnvironmentExtension.class) +class SshDataTest extends SshTestSupport { + @TempDir static java.nio.file.Path tempDir; + private static SshServer server; + private static int sshPort; + private static java.nio.file.Path privateKeyPath; + + @BeforeAll + static void beforeAll() throws Exception { + PublicKey authorizedPublicKey = null; + if (isSshKeygenAvailable()) { + String keyName = "ssh_data"; + privateKeyPath = generateRsaKeyPair(tempDir, keyName); + authorizedPublicKey = loadPublicKeyFromPubFile(Path.of(privateKeyPath + ".pub")); + } + server = startPasswordSshServer(authorizedPublicKey); + sshPort = server.getPort(); + } + + @Test + void openConnectionWithPasswordAuthenticates() throws Exception { + SshMeta meta = SshTestSupport.passwordMeta(sshPort); + + Session session = SshData.openConnection(SshTestSupport.localVariables(), meta); + + assertNotNull(session); + assertTrue(session.isConnected()); + session.disconnect(); + } + + @Test + void openConnectionWithPrivateKeyAuthenticates() throws Exception { + Assumptions.assumeTrue( + privateKeyPath != null, "ssh-keygen is required for private key authentication tests"); + SshMeta meta = + SshTestSupport.privateKeyMeta(sshPort, privateKeyPath.toAbsolutePath().toString(), null); + Session session = SshData.openConnection(SshTestSupport.localVariables(), meta); + + assertNotNull(session); + assertTrue(session.isConnected()); + session.disconnect(); + } + + @Test + void openConnectionMissingPrivateKeyFileThrows() { + SshMeta meta = SshTestSupport.passwordMeta(sshPort); + meta.setUsePrivateKey(true); + meta.setKeyFileName(null); + + HopException ex = + assertThrows( + HopException.class, + () -> SshData.openConnection(SshTestSupport.localVariables(), meta)); + + assertTrue(ex.getMessage().contains("PrivateKeyFileMissing") || !ex.getMessage().isEmpty()); + } + + @Test + void openConnectionNonExistentPrivateKeyThrows() { + SshMeta meta = SshTestSupport.passwordMeta(sshPort); + meta.setUsePrivateKey(true); + meta.setKeyFileName(tempDir.resolve("missing-key.pem").toUri().toString()); + + assertThrows( + HopException.class, () -> SshData.openConnection(SshTestSupport.localVariables(), meta)); + } + + @Test + void openConnectionWrongPasswordThrows() { + SshMeta meta = SshTestSupport.passwordMeta(sshPort); + meta.setPassword("definitely-wrong-password"); + meta.setTimeOut("3"); + + assertThrows( + HopException.class, () -> SshData.openConnection(SshTestSupport.localVariables(), meta)); + } + + @Test + void executeCommandAgainstEmbeddedServerReturnsOutput() throws Exception { + SshMeta meta = SshTestSupport.passwordMeta(sshPort); + String token = "hop-data-test"; + Session session = SshData.openConnection(SshTestSupport.localVariables(), meta); + try { + SessionResult result = + SessionResult.executeCommand(session, SshTestSupport.echoCommand(token)); + assertTrue(result.getStd().contains(token), () -> "output was: " + result.getStd()); + } finally { + session.disconnect(); + } + } + + @Test + void newDataInitialState() { + SshData data = new SshData(); + assertEquals(-1, data.indexOfCommand); + assertFalse(data.wroteOneRow); + } + + @AfterAll + static void stop() throws IOException { + if (server != null) { + stopSharedServer(server); + } + } +} diff --git a/plugins/transforms/ssh/src/test/java/org/apache/hop/pipeline/transforms/ssh/SshMetaTest.java b/plugins/transforms/ssh/src/test/java/org/apache/hop/pipeline/transforms/ssh/SshMetaTest.java index c300b672d36..2ea992e085c 100644 --- a/plugins/transforms/ssh/src/test/java/org/apache/hop/pipeline/transforms/ssh/SshMetaTest.java +++ b/plugins/transforms/ssh/src/test/java/org/apache/hop/pipeline/transforms/ssh/SshMetaTest.java @@ -18,26 +18,40 @@ package org.apache.hop.pipeline.transforms.ssh; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import java.nio.file.Path; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.hop.core.Const; +import org.apache.hop.core.ICheckResult; import org.apache.hop.core.encryption.Encr; import org.apache.hop.core.encryption.TwoWayPasswordEncoderPluginType; import org.apache.hop.core.exception.HopException; import org.apache.hop.core.plugins.PluginRegistry; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.core.row.value.ValueMetaString; import org.apache.hop.core.util.EnvUtil; +import org.apache.hop.core.variables.Variables; import org.apache.hop.core.xml.XmlHandler; +import org.apache.hop.i18n.BaseMessages; import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.TransformMeta; import org.apache.hop.pipeline.transforms.loadsave.LoadSaveTester; +import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.api.io.TempDir; import org.w3c.dom.Node; -class SshMetaTest { +class SshMetaTest extends SshTestSupport { @RegisterExtension static RestoreHopEngineEnvironmentExtension env = new RestoreHopEngineEnvironmentExtension(); @@ -83,7 +97,7 @@ void testEncryptedPasswords() throws HopException { @Test void testRoundTrips() throws HopException { List commonFields = - Arrays.asList( + Arrays.asList( "dynamicCommandField", "command", "commandFieldName", @@ -109,4 +123,199 @@ void testRoundTrips() throws HopException { tester.testSerialization(); } + + @Test + void defaultValues() { + SshMeta meta = new SshMeta(); + assertEquals("22", meta.getPort()); + assertEquals("stdOut", meta.getStdOutFieldName()); + assertEquals("stdErr", meta.getStdErrFieldName()); + assertTrue(meta.isUsePrivateKey()); + assertFalse(meta.isDynamicCommandField()); + } + + @Test + void supportsErrorHandling() { + assertTrue(new SshMeta().supportsErrorHandling()); + } + + @Test + void getFieldsStaticCommandReplacesInputRowMeta() throws HopException { + SshMeta meta = new SshMeta(); + meta.setDynamicCommandField(false); + meta.setStdOutFieldName("out"); + meta.setStdErrFieldName(""); + + IRowMeta row = new RowMeta(); + row.addValueMeta(new ValueMetaString("inputField")); + + meta.getFields(row, "ssh", null, null, new Variables(), null); + + assertEquals(1, row.size()); + assertEquals("out", row.getValueMeta(0).getName()); + } + + @Test + void getFieldsDynamicCommandKeepsInputFields() throws HopException { + SshMeta meta = new SshMeta(); + meta.setDynamicCommandField(true); + meta.setStdOutFieldName("out"); + meta.setStdErrFieldName("errFlag"); + + IRowMeta row = new RowMeta(); + row.addValueMeta(new ValueMetaString("cmd")); + + meta.getFields(row, "ssh", null, null, new Variables(), null); + + assertEquals(3, row.size()); + assertEquals("cmd", row.getValueMeta(0).getName()); + assertEquals("out", row.getValueMeta(1).getName()); + assertEquals("errFlag", row.getValueMeta(2).getName()); + } + + @Test + void checkReportsErrorWhenServerMissing(@TempDir java.nio.file.Path tempDir) { + SshMeta meta = new SshMeta(); + meta.setUsePrivateKey(false); + meta.setServerName(null); + + List remarks = new ArrayList<>(); + TransformMeta transformMeta = new TransformMeta(); + transformMeta.setTransform(meta); + PipelineMeta pipelineMeta = new PipelineMeta(); + + meta.check( + remarks, + pipelineMeta, + transformMeta, + new RowMeta(), + new String[] {"input"}, + new String[0], + null, + new Variables(), + null); + + assertTrue( + remarks.stream() + .anyMatch( + r -> + r.getType() == ICheckResult.TYPE_RESULT_ERROR + && r.getText() + .equals( + BaseMessages.getString( + SshMeta.class, "SSHMeta.CheckResult.TargetHostMissing")))); + } + + @Test + void checkReportsErrorWhenPrivateKeyEnabledButPathMissing() { + SshMeta meta = new SshMeta(); + meta.setServerName("localhost"); + meta.setUsePrivateKey(true); + meta.setKeyFileName(null); + + List remarks = new ArrayList<>(); + TransformMeta transformMeta = new TransformMeta(); + transformMeta.setTransform(meta); + + meta.check( + remarks, + new PipelineMeta(), + transformMeta, + new RowMeta(), + new String[] {"in"}, + new String[0], + null, + new Variables(), + null); + + assertTrue( + remarks.stream() + .anyMatch( + r -> + r.getType() == ICheckResult.TYPE_RESULT_ERROR + && r.getText() + .equals( + BaseMessages.getString( + SshMeta.class, + "SSHMeta.CheckResult.PrivateKeyFileNameMissing")))); + } + + @Test + void checkReportsErrorWhenNoInputTransforms() { + SshMeta meta = new SshMeta(); + meta.setServerName("localhost"); + meta.setUsePrivateKey(false); + + List remarks = new ArrayList<>(); + TransformMeta transformMeta = new TransformMeta(); + transformMeta.setTransform(meta); + + meta.check( + remarks, + new PipelineMeta(), + transformMeta, + new RowMeta(), + new String[0], + new String[0], + null, + new Variables(), + null); + + assertTrue( + remarks.stream() + .anyMatch( + r -> + r.getType() == ICheckResult.TYPE_RESULT_ERROR + && r.getText() + .equals( + BaseMessages.getString( + SshMeta.class, "SSHMeta.CheckResult.NoInpuReceived")))); + } + + @Test + void checkOkWhenServerAndInputPresent(@TempDir java.nio.file.Path keyDir) throws Exception { + Assumptions.assumeTrue(isSshKeygenAvailable(), "ssh-keygen is required to generate test keys"); + String keyName = "ssh_meta"; + Path privateKey = generateRsaKeyPair(keyDir, keyName); + SshMeta meta = new SshMeta(); + meta.setServerName("localhost"); + meta.setUsePrivateKey(true); + meta.setKeyFileName(privateKey.toUri().toString()); + + List remarks = new ArrayList<>(); + TransformMeta transformMeta = new TransformMeta(); + transformMeta.setTransform(meta); + + meta.check( + remarks, + new PipelineMeta(), + transformMeta, + new RowMeta(), + new String[] {"in"}, + new String[0], + null, + new Variables(), + null); + + assertTrue( + remarks.stream() + .anyMatch( + r -> + r.getType() == ICheckResult.TYPE_RESULT_OK + && r.getText() + .equals( + BaseMessages.getString( + SshMeta.class, "SSHMeta.CheckResult.TargetHostOK")))); + assertFalse( + remarks.stream() + .anyMatch( + r -> + r.getType() == ICheckResult.TYPE_RESULT_OK + && r.getText() + .equals( + BaseMessages.getString( + SshMeta.class, + "SSHMeta.CheckResult.PrivateKeyFileExists", + privateKey.toString())))); + } } diff --git a/plugins/transforms/ssh/src/test/java/org/apache/hop/pipeline/transforms/ssh/SshTestSupport.java b/plugins/transforms/ssh/src/test/java/org/apache/hop/pipeline/transforms/ssh/SshTestSupport.java new file mode 100644 index 00000000000..224c6cd2a69 --- /dev/null +++ b/plugins/transforms/ssh/src/test/java/org/apache/hop/pipeline/transforms/ssh/SshTestSupport.java @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.ssh; + +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.PublicKey; +import lombok.extern.slf4j.Slf4j; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.core.variables.Variables; +import org.apache.sshd.common.config.keys.AuthorizedKeyEntry; +import org.apache.sshd.common.config.keys.KeyUtils; +import org.apache.sshd.server.Environment; +import org.apache.sshd.server.ExitCallback; +import org.apache.sshd.server.SshServer; +import org.apache.sshd.server.channel.ChannelSession; +import org.apache.sshd.server.command.Command; +import org.apache.sshd.server.keyprovider.SimpleGeneratorHostKeyProvider; + +/** Shared fixtures for SSH transform unit tests. */ +@Slf4j +class SshTestSupport { + static final String SSH_USER = "root"; + static final String SSH_PASS = "pwd123456"; + + /** Passphrase used when generating test keys; must match {@link #privateKeyMeta}. */ + static final String TEST_KEY_PASSPHRASE = "hello"; + + static SshServer startPasswordSshServer(PublicKey authorizedPublicKey) throws IOException { + SshServer sshd = SshServer.setUpDefaultServer(); + sshd.setPort(0); + sshd.setKeyPairProvider(new SimpleGeneratorHostKeyProvider()); + sshd.setPasswordAuthenticator( + (username, password, session) -> SSH_USER.equals(username) && SSH_PASS.equals(password)); + sshd.setPublickeyAuthenticator( + (username, key, session) -> + SSH_USER.equals(username) + && authorizedPublicKey != null + && KeyUtils.compareKeys(authorizedPublicKey, key)); + sshd.setCommandFactory((channel, command) -> new TestEchoCommand(command)); + sshd.start(); + return sshd; + } + + static void stopSharedServer(SshServer server) throws IOException { + if (server != null && server.isOpen()) { + server.stop(); + } + } + + static SshMeta passwordMeta(int port) { + SshMeta meta = new SshMeta(); + meta.setUsePrivateKey(false); + meta.setServerName("localhost"); + meta.setPort(String.valueOf(port)); + meta.setUserName(SSH_USER); + meta.setPassword(SSH_PASS); + meta.setCommand("echo hop-ssh-test"); + meta.setStdOutFieldName("stdOut"); + meta.setStdErrFieldName("stdErr"); + return meta; + } + + static SshMeta privateKeyMeta(int port, String privateKeyPath) { + return privateKeyMeta(port, privateKeyPath, TEST_KEY_PASSPHRASE); + } + + static SshMeta privateKeyMeta(int port, String privateKeyPath, String keyPassphrase) { + SshMeta meta = passwordMeta(port); + meta.setUsePrivateKey(true); + meta.setPassword(null); + meta.setKeyFileName(privateKeyPath); + meta.setPassPhrase(keyPassphrase); + return meta; + } + + static PublicKey loadPublicKeyFromPubFile(Path pubFile) throws Exception { + AuthorizedKeyEntry entry = + AuthorizedKeyEntry.parseAuthorizedKeyEntry( + Files.readString(pubFile, StandardCharsets.UTF_8)); + return entry.resolvePublicKey(null, null); + } + + static boolean isSshKeygenAvailable() { + try { + boolean windows = System.getProperty("os.name", "").toLowerCase().contains("win"); + ProcessBuilder builder = new ProcessBuilder(windows ? "where" : "which", "ssh-keygen"); + Process process = builder.redirectErrorStream(true).start(); + return process.waitFor() == 0; + } catch (Exception e) { + return false; + } + } + + static Path generateRsaKeyPair(Path directory, String keyName) + throws IOException, InterruptedException { + Files.createDirectories(directory); + Path privateKey = directory.resolve(keyName); + Process process = + new ProcessBuilder( + "ssh-keygen", + "-t", + "rsa", + "-b", + "2048", + "-m", + "PEM", + "-N", + "", + "-f", + privateKey.toAbsolutePath().toString(), + "-q") + .redirectErrorStream(true) + .start(); + int exit = process.waitFor(); + if (exit != 0 || !Files.exists(privateKey)) { + throw new IOException("ssh-keygen failed with exit code " + exit); + } + return privateKey; + } + + static IVariables localVariables() { + return new Variables(); + } + + /** Command understood by the embedded test SSH server. */ + static String echoCommand(String token) { + return "echo " + token; + } + + /** Minimal exec handler so tests do not depend on OS shell behavior. */ + private static final class TestEchoCommand implements Command { + private final String command; + private OutputStream stdout; + private OutputStream stderr; + private ExitCallback exitCallback; + + private TestEchoCommand(String command) { + this.command = command; + } + + @Override + public void setInputStream(InputStream in) { + // no stdin required + } + + @Override + public void setOutputStream(OutputStream out) { + this.stdout = out; + } + + @Override + public void setErrorStream(OutputStream err) { + this.stderr = err; + } + + @Override + public void setExitCallback(ExitCallback callback) { + this.exitCallback = callback; + } + + @Override + public void start(ChannelSession channel, Environment env) throws IOException { + try { + if (command != null && command.startsWith("echo ")) { + String payload = command.substring(5) + "\n"; + stdout.write(payload.getBytes(StandardCharsets.UTF_8)); + stdout.flush(); + exitCallback.onExit(0); + } else if (command != null && command.startsWith("stderr ")) { + String payload = command.substring(7) + "\n"; + stderr.write(payload.getBytes(StandardCharsets.UTF_8)); + stderr.flush(); + exitCallback.onExit(0); + } else { + stderr.write(("unsupported command: " + command + "\n").getBytes(StandardCharsets.UTF_8)); + stderr.flush(); + exitCallback.onExit(1); + } + } finally { + closeQuietly(stdout); + closeQuietly(stderr); + } + } + + private static void closeQuietly(OutputStream stream) throws IOException { + if (stream instanceof Closeable closeable) { + closeable.close(); + } + } + + @Override + public void destroy(ChannelSession channel) { + // nothing to clean up + } + } +} diff --git a/plugins/transforms/ssh/src/test/java/org/apache/hop/pipeline/transforms/ssh/SshTransformTest.java b/plugins/transforms/ssh/src/test/java/org/apache/hop/pipeline/transforms/ssh/SshTransformTest.java new file mode 100644 index 00000000000..ba0effa13ee --- /dev/null +++ b/plugins/transforms/ssh/src/test/java/org/apache/hop/pipeline/transforms/ssh/SshTransformTest.java @@ -0,0 +1,248 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.ssh; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.nio.file.Path; +import java.security.PublicKey; +import org.apache.hop.core.QueueRowSet; +import org.apache.hop.core.logging.ILoggingObject; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.core.row.value.ValueMetaBoolean; +import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension; +import org.apache.hop.metadata.api.IHopMetadataProvider; +import org.apache.hop.pipeline.transform.TransformErrorMeta; +import org.apache.hop.pipeline.transforms.mock.TransformMockHelper; +import org.apache.sshd.server.SshServer; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; + +/** Unit test ssh transform */ +@ExtendWith(RestoreHopEngineEnvironmentExtension.class) +class SshTransformTest extends SshTestSupport { + @TempDir static java.nio.file.Path tempDir; + private static SshServer server; + private static int sshPort; + + private TransformMockHelper mockHelper; + + @BeforeAll + static void beforeAll() throws Exception { + PublicKey authorizedPublicKey = null; + if (isSshKeygenAvailable()) { + String keyName = "ssh_form"; + Path privateKeyPath = generateRsaKeyPair(tempDir, keyName); + authorizedPublicKey = loadPublicKeyFromPubFile(Path.of(privateKeyPath + ".pub")); + } + server = startPasswordSshServer(authorizedPublicKey); + sshPort = server.getPort(); + } + + @BeforeEach + void setUp() { + mockHelper = new TransformMockHelper<>("SSH TEST", SshMeta.class, SshData.class); + when(mockHelper.logChannelFactory.create(any(), any(ILoggingObject.class))) + .thenReturn(mockHelper.iLogChannel); + when(mockHelper.pipeline.isRunning()).thenReturn(true); + } + + @AfterEach + void tearDown() { + mockHelper.cleanUp(); + } + + @Test + void initFailsWhenUserNameMissing() { + SshMeta meta = passwordMeta(sshPort); + meta.setUserName(null); + Ssh ssh = newSsh(meta, new SshData()); + + assertFalse(ssh.init()); + } + + @Test + void initFailsWhenStdOutFieldMissing() { + SshMeta meta = passwordMeta(sshPort); + meta.setStdOutFieldName(null); + Ssh ssh = newSsh(meta, new SshData()); + + assertFalse(ssh.init()); + } + + @Test + void initOpensSessionOnValidPasswordConfig() { + SshMeta meta = passwordMeta(sshPort); + SshData data = new SshData(); + Ssh ssh = newSsh(meta, data); + + assertTrue(ssh.init()); + assertNotNull(data.session); + assertTrue(data.session.isConnected()); + ssh.dispose(); + assertNull(data.session); + } + + @Test + void staticCommandProcessesSingleRow() throws Exception { + String token = "hop-static"; + SshMeta meta = passwordMeta(sshPort); + meta.setCommand(echoCommand(token)); + SshData data = new SshData(); + Ssh ssh = newSsh(meta, data); + QueueRowSet output = attachOutput(ssh); + + assertTrue(ssh.init()); + assertTrue(ssh.processRow()); + Object[] row = output.getRow(); + assertNotNull(row); + assertTrue(row[0].toString().contains(token)); + + assertFalse(ssh.processRow()); + ssh.dispose(); + } + + @Test + void dynamicCommandReadsCommandFromInputField() throws Exception { + String token = "hop-dynamic"; + SshMeta meta = passwordMeta(sshPort); + meta.setDynamicCommandField(true); + meta.setCommandFieldName("cmd"); + meta.setCommand(null); + + RowMeta inputMeta = new RowMeta(); + inputMeta.addValueMeta(new ValueMetaString("cmd")); + QueueRowSet input = new QueueRowSet(); + input.setRowMeta(inputMeta); + input.putRow(inputMeta, new Object[] {echoCommand(token)}); + input.setDone(); + + SshData data = new SshData(); + Ssh ssh = newSsh(meta, data); + ssh.addRowSetToInputRowSets(input); + QueueRowSet output = attachOutput(ssh); + + assertTrue(ssh.init()); + assertTrue(ssh.processRow()); + Object[] row = output.getRow(); + assertNotNull(row); + assertEquals("cmd", inputMeta.getValueMeta(0).getName()); + assertTrue(row[1].toString().contains(token), () -> "row=" + java.util.Arrays.toString(row)); + + assertFalse(ssh.processRow()); + ssh.dispose(); + } + + @Test + void dynamicCommandEmptyCommandRoutesToErrorRowWhenErrorHandlingEnabled() throws Exception { + SshMeta meta = passwordMeta(sshPort); + meta.setDynamicCommandField(true); + meta.setCommandFieldName("cmd"); + + RowMeta inputMeta = new RowMeta(); + inputMeta.addValueMeta(new ValueMetaString("cmd")); + QueueRowSet input = new QueueRowSet(); + input.setRowMeta(inputMeta); + input.putRow(inputMeta, new Object[] {""}); + input.setDone(); + + when(mockHelper.transformMeta.isDoingErrorHandling()).thenReturn(true); + TransformErrorMeta errorMeta = mock(TransformErrorMeta.class); + when(mockHelper.transformMeta.getTransformErrorMeta()).thenReturn(errorMeta); + when(errorMeta.getErrorRowMeta(any())).thenReturn(inputMeta); + + SshData data = new SshData(); + Ssh ssh = newSsh(meta, data); + ssh.addRowSetToInputRowSets(input); + attachOutput(ssh); + + assertTrue(ssh.init()); + assertTrue(ssh.processRow()); + ssh.dispose(); + } + + @Test + void outputIncludesStdErrFlagWhenConfigured() throws Exception { + SshMeta meta = passwordMeta(sshPort); + meta.setCommand(echoCommand("flag-test")); + meta.setStdErrFieldName("hasErr"); + + SshData data = new SshData(); + Ssh ssh = newSsh(meta, data); + QueueRowSet output = attachOutput(ssh); + + assertTrue(ssh.init()); + assertTrue(ssh.processRow()); + Object[] row = output.getRow(); + assertNotNull(row); + assertEquals(2, row.length); + assertInstanceOf(Boolean.class, row[1]); + + ssh.dispose(); + } + + @Test + void getFieldsForStaticCommandClearsInputAndAddsOutput() throws Exception { + SshMeta meta = passwordMeta(sshPort); + RowMeta row = new RowMeta(); + row.addValueMeta(new ValueMetaString("keep")); + + meta.getFields(row, "ssh", null, null, localVariables(), null); + + assertEquals(2, row.size()); + assertEquals("stdOut", row.getValueMeta(0).getName()); + assertEquals(ValueMetaBoolean.TYPE_BOOLEAN, row.getValueMeta(1).getType()); + } + + @AfterAll + static void stop() throws IOException { + if (server != null) { + stopSharedServer(server); + } + } + + private Ssh newSsh(SshMeta meta, SshData data) { + when(mockHelper.transformMeta.getTransform()).thenReturn(meta); + Ssh ssh = + new Ssh( + mockHelper.transformMeta, meta, data, 0, mockHelper.pipelineMeta, mockHelper.pipeline); + ssh.setMetadataProvider(mock(IHopMetadataProvider.class)); + return ssh; + } + + private static QueueRowSet attachOutput(Ssh ssh) { + QueueRowSet output = new QueueRowSet(); + ssh.addRowSetToOutputRowSets(output); + return output; + } +} diff --git a/pom.xml b/pom.xml index dd551c5e19d..a19165b887a 100644 --- a/pom.xml +++ b/pom.xml @@ -163,6 +163,7 @@ initialize scpexe://people.apache.org/www/hop.apache.org/maven/ 5.5.0.6356 + integration-tests/ssh/keys/** false 21