diff --git a/pom.xml b/pom.xml index 3481e5e26..d6a939418 100644 --- a/pom.xml +++ b/pom.xml @@ -77,7 +77,7 @@ 1.6 2.16.1 1.4 - 3.0 + 3.1 2.6 3.5 3.6.1 diff --git a/wrangler-api/src/main/java/io/cdap/wrangler/api/DirectiveConfig.java b/wrangler-api/src/main/java/io/cdap/wrangler/api/DirectiveConfig.java index d139594cc..2355271db 100644 --- a/wrangler-api/src/main/java/io/cdap/wrangler/api/DirectiveConfig.java +++ b/wrangler-api/src/main/java/io/cdap/wrangler/api/DirectiveConfig.java @@ -21,6 +21,7 @@ import com.google.gson.JsonObject; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -28,22 +29,30 @@ import java.util.Set; /** - * This class {@link DirectiveConfig} defines the configuration for the Wrangler. + * This class {@link DirectiveConfig} defines the configuration for the + * Wrangler. * It specifies the directive exclusions -- meaning directives that should * not be accessible to the users and as well as directive aliases. * * { - * "exclusions" : [ - * "parse-as-csv", - * "parse-as-excel", - * "set", - * "invoke-http" - * ], - * "aliases" : { - * "json-parser" : "parse-as-json", - * "js-parser" : "parse-as-json" - * } - * } + * "exclusions" : [ + * "parse-as-csv", + * "parse-as-excel", + * "set", + * "invoke-http" + * ], + * "aliases" : { + * "json-parser" : "parse-as-json", + * "js-parser" : "parse-as-json" + * } + * "jexlInclusions" : [ + * { + * "className": "com.xyz.JsonParser", + * "methods": ["parse"], + * "properties": ["offset"] + * } + * ] + * } */ @Deprecated public final class DirectiveConfig { @@ -54,6 +63,19 @@ public final class DirectiveConfig { // RecipeParser to be aliased. private final Map aliases = new HashMap<>(); + /** + * The JEXL inclusions rules. + */ + private final List jexlInclusions = new ArrayList<>(); + + /** + * Gets the list of JEXL inclusions. + * + * @return the list of JEXL inclusions + */ + public List getJexlInclusions() { + return Collections.unmodifiableList(jexlInclusions); + } /** * Checks if a directive is aliased. @@ -110,6 +132,7 @@ public JsonElement toJson() { JsonObject object = new JsonObject(); object.add("exclusions", gson.toJsonTree(exclusions)); object.add("aliases", gson.toJsonTree(aliases)); + object.add("jexlInclusions", gson.toJsonTree(jexlInclusions)); return object; } } diff --git a/wrangler-api/src/main/java/io/cdap/wrangler/api/DirectiveContext.java b/wrangler-api/src/main/java/io/cdap/wrangler/api/DirectiveContext.java index 78df981d6..24b359235 100644 --- a/wrangler-api/src/main/java/io/cdap/wrangler/api/DirectiveContext.java +++ b/wrangler-api/src/main/java/io/cdap/wrangler/api/DirectiveContext.java @@ -16,9 +16,29 @@ package io.cdap.wrangler.api; +import javax.annotation.Nullable; + /** * {@link DirectiveContext} provides the context object to the processing of * directives. */ public interface DirectiveContext extends DirectiveEnforcer, DirectiveAlias { + /** + * Gets the DirectiveConfig. + * + * @return the DirectiveConfig + */ + @Nullable + default DirectiveConfig getConfig() { + return null; + } + + /** + * Checks if secure JEXL feature is enabled. + * + * @return true if enabled + */ + default boolean isSecureJexlFeatureEnabled() { + return false; + } } diff --git a/wrangler-api/src/main/java/io/cdap/wrangler/api/JexlInclusion.java b/wrangler-api/src/main/java/io/cdap/wrangler/api/JexlInclusion.java new file mode 100644 index 000000000..7381f8782 --- /dev/null +++ b/wrangler-api/src/main/java/io/cdap/wrangler/api/JexlInclusion.java @@ -0,0 +1,115 @@ +/* + * Copyright © 2024 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.wrangler.api; + +import io.cdap.wrangler.api.annotations.PublicEvolving; +import java.io.Serializable; +import java.util.Collections; +import java.util.List; +import javax.annotation.Nullable; + +/** + * Defines custom class, method, and property inclusion rules. + */ +@PublicEvolving +public final class JexlInclusion implements Serializable { + private static final long serialVersionUID = 1L; + + /** + * The name of the class to include. + */ + private final String className; + + /** + * The list of allowed methods for the class. + */ + private final List methods; + + /** + * The list of allowed properties for the class. + */ + private final List properties; + + /** + * Default constructor required for Gson serialization/deserialization. + */ + private JexlInclusion() { + this.className = null; + this.methods = Collections.emptyList(); + this.properties = Collections.emptyList(); + } + + /** + * Constructs a JexlInclusion. + * + * @param className the name of the class + * @param methods the list of allowed methods + * @param properties the list of allowed properties + */ + public JexlInclusion(final String className, + @Nullable final List methods, + @Nullable final List properties) { + this.className = className; + this.methods = (methods == null) ? Collections.emptyList() : methods; + this.properties = (properties == null) ? Collections.emptyList() : properties; + } + + /** + * Gets the class name. + * + * @return the class name + */ + public String getClassName() { + return className; + } + + /** + * Gets the list of allowed methods. + * + * @return the allowed methods + */ + public List getMethods() { + return methods; + } + + /** + * Gets the list of allowed properties. + * + * @return the allowed properties + */ + public List getProperties() { + return properties; + } + + /** + * Checks if all methods are allowed. + * + * @return true if all methods are allowed + */ + public boolean isAllMethods() { + return methods.isEmpty() || methods.contains("*"); + } + + /** + * Checks if all properties are allowed. + * + * @return true if all properties are allowed + */ + public boolean isAllProperties() { + return properties.isEmpty() || properties.contains("*"); + } +} diff --git a/wrangler-core/pom.xml b/wrangler-core/pom.xml index 788938a6c..0542479f5 100644 --- a/wrangler-core/pom.xml +++ b/wrangler-core/pom.xml @@ -60,6 +60,11 @@ io.cdap.cdap cdap-api ${cdap.version} + + + io.cdap.cdap + cdap-features + ${cdap.version} provided diff --git a/wrangler-core/src/main/java/io/cdap/directives/aggregates/IncrementTransientVariable.java b/wrangler-core/src/main/java/io/cdap/directives/aggregates/IncrementTransientVariable.java index fbe89793d..3d21adc5c 100644 --- a/wrangler-core/src/main/java/io/cdap/directives/aggregates/IncrementTransientVariable.java +++ b/wrangler-core/src/main/java/io/cdap/directives/aggregates/IncrementTransientVariable.java @@ -38,6 +38,7 @@ import io.cdap.wrangler.expression.ELContext; import io.cdap.wrangler.expression.ELException; import io.cdap.wrangler.expression.ELResult; +import io.cdap.wrangler.utils.JexlHelper; import java.util.List; @@ -54,7 +55,7 @@ public class IncrementTransientVariable implements Directive { public static final String NAME = "increment-variable"; private String variable; private long incrementBy; - private EL el; + private EL el; @Override public UsageDefinition define() { @@ -69,9 +70,10 @@ public UsageDefinition define() { public void initialize(Arguments args) throws DirectiveParseException { this.variable = ((Identifier) args.value("variable")).value(); this.incrementBy = ((Numeric) args.value("value")).value().longValue(); - String expression = ((Expression) args.value("condition")).value(); + Expression expression = args.value("condition"); try { - el = EL.compile(expression); + this.el = EL.compile(expression.value(), JexlHelper.getJexlInclusions(args), + JexlHelper.isSecureJexlFeatureEnabled(args)); } catch (ELException e) { throw new DirectiveParseException(NAME, e.getMessage(), e); } diff --git a/wrangler-core/src/main/java/io/cdap/directives/aggregates/SetTransientVariable.java b/wrangler-core/src/main/java/io/cdap/directives/aggregates/SetTransientVariable.java index becf12cc6..a74789d4f 100644 --- a/wrangler-core/src/main/java/io/cdap/directives/aggregates/SetTransientVariable.java +++ b/wrangler-core/src/main/java/io/cdap/directives/aggregates/SetTransientVariable.java @@ -37,6 +37,7 @@ import io.cdap.wrangler.expression.ELContext; import io.cdap.wrangler.expression.ELException; import io.cdap.wrangler.expression.ELResult; +import io.cdap.wrangler.utils.JexlHelper; import java.util.List; @@ -56,7 +57,7 @@ public class SetTransientVariable implements Directive { public static final String NAME = "set-variable"; private EL el; private String variable; - + @Override public UsageDefinition define() { UsageDefinition.Builder builder = UsageDefinition.builder(NAME); @@ -68,9 +69,10 @@ public UsageDefinition define() { @Override public void initialize(Arguments args) throws DirectiveParseException { this.variable = ((Identifier) args.value("variable")).value(); - String expression = ((Expression) args.value("condition")).value(); + Expression expression = args.value("condition"); try { - el = EL.compile(expression); + this.el = EL.compile(expression.value(), JexlHelper.getJexlInclusions(args), + JexlHelper.isSecureJexlFeatureEnabled(args)); } catch (ELException e) { throw new DirectiveParseException(NAME, e.getMessage(), e); } diff --git a/wrangler-core/src/main/java/io/cdap/directives/row/Fail.java b/wrangler-core/src/main/java/io/cdap/directives/row/Fail.java index 880ffc63c..1cd8b3430 100644 --- a/wrangler-core/src/main/java/io/cdap/directives/row/Fail.java +++ b/wrangler-core/src/main/java/io/cdap/directives/row/Fail.java @@ -37,6 +37,7 @@ import io.cdap.wrangler.expression.ELContext; import io.cdap.wrangler.expression.ELException; import io.cdap.wrangler.expression.ELResult; +import io.cdap.wrangler.utils.JexlHelper; import java.util.List; @@ -53,7 +54,7 @@ public class Fail implements Directive, Lineage { public static final String NAME = "fail"; private String condition; private EL el; - + @Override public UsageDefinition define() { UsageDefinition.Builder builder = UsageDefinition.builder(NAME); @@ -70,7 +71,8 @@ public void initialize(Arguments args) throws DirectiveParseException { } condition = expression.value(); try { - el = EL.compile(condition); + this.el = EL.compile(expression.value(), JexlHelper.getJexlInclusions(args), + JexlHelper.isSecureJexlFeatureEnabled(args)); } catch (ELException e) { throw new DirectiveParseException(NAME, e.getMessage(), e); } diff --git a/wrangler-core/src/main/java/io/cdap/directives/row/RecordConditionFilter.java b/wrangler-core/src/main/java/io/cdap/directives/row/RecordConditionFilter.java index b3eb4adb2..b79bb8f82 100644 --- a/wrangler-core/src/main/java/io/cdap/directives/row/RecordConditionFilter.java +++ b/wrangler-core/src/main/java/io/cdap/directives/row/RecordConditionFilter.java @@ -38,6 +38,7 @@ import io.cdap.wrangler.expression.EL; import io.cdap.wrangler.expression.ELContext; import io.cdap.wrangler.expression.ELException; +import io.cdap.wrangler.utils.JexlHelper; import java.util.ArrayList; import java.util.List; @@ -61,7 +62,7 @@ public class RecordConditionFilter implements Directive, Lineage { public static final String NAME = "filter-row"; private EL el; private boolean isTrue; - + @Override public UsageDefinition define() { UsageDefinition.Builder builder = UsageDefinition.builder(NAME); @@ -76,9 +77,10 @@ public void initialize(Arguments args) throws DirectiveParseException { if (args.contains("type")) { isTrue = ((Bool) args.value("type")).value(); } - String condition = ((Expression) args.value("condition")).value(); + Expression expression = args.value("condition"); try { - el = EL.compile(condition); + this.el = EL.compile(expression.value(), JexlHelper.getJexlInclusions(args), + JexlHelper.isSecureJexlFeatureEnabled(args)); } catch (ELException e) { throw new DirectiveParseException(NAME, e.getMessage(), e); } diff --git a/wrangler-core/src/main/java/io/cdap/directives/row/SendToError.java b/wrangler-core/src/main/java/io/cdap/directives/row/SendToError.java index 7cbd4bf82..b4f19bdf8 100644 --- a/wrangler-core/src/main/java/io/cdap/directives/row/SendToError.java +++ b/wrangler-core/src/main/java/io/cdap/directives/row/SendToError.java @@ -41,6 +41,7 @@ import io.cdap.wrangler.expression.ELContext; import io.cdap.wrangler.expression.ELException; import io.cdap.wrangler.expression.ELResult; +import io.cdap.wrangler.utils.JexlHelper; import java.util.ArrayList; import java.util.List; @@ -66,7 +67,7 @@ public class SendToError implements Directive, Lineage { private String condition; private String metric = null; private String message = null; - + @Override public UsageDefinition define() { UsageDefinition.Builder builder = UsageDefinition.builder(NAME); @@ -78,9 +79,10 @@ public UsageDefinition define() { @Override public void initialize(Arguments args) throws DirectiveParseException { - condition = ((Expression) args.value("condition")).value(); + Expression expression = args.value("condition"); + condition = expression.value(); try { - el = EL.compile(condition); + this.el = EL.compile(condition, JexlHelper.getJexlInclusions(args), JexlHelper.isSecureJexlFeatureEnabled(args)); } catch (ELException e) { throw new DirectiveParseException( NAME, String.format(" Invalid condition '%s'.", condition) diff --git a/wrangler-core/src/main/java/io/cdap/directives/row/SendToErrorAndContinue.java b/wrangler-core/src/main/java/io/cdap/directives/row/SendToErrorAndContinue.java index 20922d1e8..67ac68a43 100644 --- a/wrangler-core/src/main/java/io/cdap/directives/row/SendToErrorAndContinue.java +++ b/wrangler-core/src/main/java/io/cdap/directives/row/SendToErrorAndContinue.java @@ -42,6 +42,7 @@ import io.cdap.wrangler.expression.ELContext; import io.cdap.wrangler.expression.ELException; import io.cdap.wrangler.expression.ELResult; +import io.cdap.wrangler.utils.JexlHelper; import java.util.ArrayList; import java.util.List; @@ -67,7 +68,7 @@ public class SendToErrorAndContinue implements Directive, Lineage { private String condition; private String metric = null; private String message = null; - + @Override public UsageDefinition define() { UsageDefinition.Builder builder = UsageDefinition.builder(NAME); @@ -79,19 +80,20 @@ public UsageDefinition define() { @Override public void initialize(Arguments args) throws DirectiveParseException { - condition = ((Expression) args.value("condition")).value(); - try { - el = EL.compile(condition); - } catch (ELException e) { - throw new DirectiveParseException( - NAME, String.format("Invalid condition '%s'.", condition), e); - } + Expression expression = args.value("condition"); + condition = expression.value(); if (args.contains("metric")) { metric = ((Identifier) args.value("metric")).value(); } if (args.contains("message")) { message = ((Text) args.value("message")).value(); } + try { + this.el = EL.compile(expression.value(), JexlHelper.getJexlInclusions(args), + JexlHelper.isSecureJexlFeatureEnabled(args)); + } catch (ELException e) { + throw new DirectiveParseException(NAME, e.getMessage(), e); + } } @Override diff --git a/wrangler-core/src/main/java/io/cdap/directives/transformation/ColumnExpression.java b/wrangler-core/src/main/java/io/cdap/directives/transformation/ColumnExpression.java index 25c9c895b..cf069c84c 100644 --- a/wrangler-core/src/main/java/io/cdap/directives/transformation/ColumnExpression.java +++ b/wrangler-core/src/main/java/io/cdap/directives/transformation/ColumnExpression.java @@ -39,6 +39,7 @@ import io.cdap.wrangler.expression.ELContext; import io.cdap.wrangler.expression.ELException; import io.cdap.wrangler.expression.ELResult; +import io.cdap.wrangler.utils.JexlHelper; import java.util.List; @@ -64,10 +65,9 @@ public class ColumnExpression implements Directive, Lineage { public static final String NAME = "set-column"; // Column to which the result of experience is applied to. private String column; - // The actual expression - private String expression; + private String expressionValue; private EL el; - + @Override public UsageDefinition define() { UsageDefinition.Builder builder = UsageDefinition.builder(NAME); @@ -79,9 +79,11 @@ public UsageDefinition define() { @Override public void initialize(Arguments args) throws DirectiveParseException { this.column = ((ColumnName) args.value("column")).value(); - this.expression = ((Expression) args.value("expression")).value(); + Expression expression = args.value("expression"); + this.expressionValue = expression.value(); try { - el = EL.compile(expression); + this.el = EL.compile(expression.value(), JexlHelper.getJexlInclusions(args), + JexlHelper.isSecureJexlFeatureEnabled(args)); } catch (ELException e) { throw new DirectiveParseException(NAME, e.getMessage(), e); } @@ -118,7 +120,7 @@ public List execute(List rows, ExecutorContext context) throws Directi @Override public Mutation lineage() { Mutation.Builder builder = Mutation.builder() - .readable("Mapped result of expression '%s' to column '%s'", expression, column); + .readable("Mapped result of expression '%s' to column '%s'", expressionValue, column); builder.relation(Many.of(el.variables()), column); el.variables().forEach(col -> { if (!col.equals(column)) { diff --git a/wrangler-core/src/main/java/io/cdap/wrangler/expression/EL.java b/wrangler-core/src/main/java/io/cdap/wrangler/expression/EL.java index 28baad940..4c9e76061 100644 --- a/wrangler-core/src/main/java/io/cdap/wrangler/expression/EL.java +++ b/wrangler-core/src/main/java/io/cdap/wrangler/expression/EL.java @@ -28,6 +28,7 @@ import io.cdap.functions.JsonFunctions; import io.cdap.functions.Logical; import io.cdap.functions.NumberFunctions; +import io.cdap.wrangler.api.JexlInclusion; import io.cdap.wrangler.utils.ArithmeticOperations; import io.cdap.wrangler.utils.DecimalTransform; import org.apache.commons.jexl3.JexlBuilder; @@ -35,6 +36,7 @@ import org.apache.commons.jexl3.JexlException; import org.apache.commons.jexl3.JexlInfo; import org.apache.commons.jexl3.JexlScript; +import org.apache.commons.jexl3.introspection.JexlSandbox; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang3.StringEscapeUtils; import org.apache.commons.logging.Log; @@ -46,6 +48,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import javax.annotation.Nullable; /** * This class EL is a Expression Language Handler. @@ -65,56 +68,187 @@ public static boolean isUsed() { } /** - * Same as calling {@link #compile(ELRegistration, String)} using {@link DefaultFunctions}. + * Same as calling {@link #compile(ELRegistration, String)} using + * {@link DefaultFunctions}. + * Note: This defaults to allowlist disabled. */ public static EL compile(String expression) throws ELException { - return compile(new DefaultFunctions(), expression); + return compile(new DefaultFunctions(), expression, Collections.emptyList(), false); } /** - * Compiles the given expressions and return an {@link EL} for script execution. + * Compiles with specific JEXL allowlist settings. + */ + /** + * Compiles with specific JEXL allowlist settings. + * + * @param expression the JEXL expression + * @param inclusions the JEXL inclusions + * @param allowlistEnabled whether to enforce the allowlist + * @return the compiled EL + * @throws ELException if failed to compile + */ + public static EL compile(final String expression, + @Nullable final List inclusions, + final boolean allowlistEnabled) throws ELException { + return compile(new DefaultFunctions(), expression, inclusions, + allowlistEnabled); + } + + /** + * Compiles the expression and returns a executable expression. * - * @param registration extra objects available for the script to use - * @param expression the JEXL expresion - * @return an {@link EL} instance - * @throws ELException if failed to compile the expression + * @param registration to be registered with the JEXL context. + * @param expression to be compiled. + * @param inclusions a list of additional functions/classes to allowlist + * @param allowlistEnabled whether to enforce the allowlist + * @return a compiled {@link EL} object + * @throws ELException if failed to compile */ - public static EL compile(ELRegistration registration, String expression) throws ELException { + public static EL compile(final ELRegistration registration, + final String expression, + @Nullable final List inclusions, + final boolean allowlistEnabled) + throws ELException { used = true; + JexlSandbox sandbox = createSandbox(inclusions, allowlistEnabled); JexlEngine engine = new JexlBuilder() - .namespaces(registration.functions()) - .silent(false) - .cache(1024) - .strict(true) - .logger(new NullLogger()) - .create(); + .sandbox(sandbox) + .namespaces(registration.functions()) + .silent(false) + .cache(1024) + .strict(true) + .logger(new NullLogger()) + .create(); try { - Set variables = new HashSet<>(); JexlScript script = engine.createScript(expression); - Set> varSet = script.getVariables(); - for (List vars : varSet) { - variables.add(Joiner.on(".").join(vars)); - } - + Set variables = extractVariables(script); return new EL(script, variables); - } catch (JexlException e) { - // JexlException.getMessage() uses 'io.cdap.wrangler.expression.EL' class name in the error message. - // So instead use info object to get information about error message and create custom error message. - JexlInfo info = e.getInfo(); - throw new ELException( - String.format("Error encountered while compiling '%s' at line '%d' and column '%d'. " + - "Make sure a valid jexl transformation is provided.", - // here the detail can be null since there are multiple subclasses which extends this - // JexlException, not all of them has this detail information - info.getDetail() == null ? expression : info.getDetail(), info.getLine(), info.getColumn()), e); } catch (Exception e) { - throw new ELException(e); + throw handleCompilationException(e, expression); } + } + /** + * Extracts variables from the script. + * + * @param script the script + * @return the variables + */ + private static Set extractVariables(final JexlScript script) { + Set variables = new HashSet<>(); + for (List vars : script.getVariables()) { + variables.add(Joiner.on(".").join(vars)); + } + return variables; } - private EL(JexlScript script, Set variables) { + /** + * Handles compilation exceptions. + * + * @param ex the exception + * @param expression the expression + * @return the ELException + */ + private static ELException handleCompilationException(final Exception ex, + final String expression) { + if (ex instanceof JexlException) { + JexlException jexlEx = (JexlException) ex; + JexlInfo info = jexlEx.getInfo(); + String detail = info.getDetail() == null ? expression : info.getDetail().toString(); + String errorMessage = jexlEx.getMessage(); + if (errorMessage != null && (errorMessage.contains("unsolvable function/method") + || errorMessage.contains("unsolvable property"))) { + return new ELException( + String.format("Security violation: Access to JEXL component '%s' is not " + + "permitted by wrangler. Hence, this JEXL expression '%s' can't be resolved.", + detail, expression), + jexlEx); + } + return new ELException( + String.format("Error encountered while compiling '%s' at line '%d' " + + "and column '%d'. Make sure a valid jexl " + + "transformation is provided.", + detail, info.getLine(), info.getColumn()), + jexlEx); + } + return new ELException(ex); + } + + /** + * Creates a JEXL sandbox. + * + * @param inclusions the inclusions + * @param allowlistEnabled true if allowlist is enabled + * @return the sandbox + */ + public static JexlSandbox createSandbox( + @Nullable final List inclusions, + final boolean allowlistEnabled) { + if (!allowlistEnabled && (inclusions == null || inclusions.isEmpty())) { + return null; + } + + JexlSandbox sandbox = new JexlSandbox(false); + for (Class allowedClass : JexlAllowedClasses.DEFAULT_ALLOWED_CLASSES) { + sandbox.white(allowedClass.getName()); + } + + if (inclusions != null) { + for (JexlInclusion inclusion : inclusions) { + applyInclusionRule(sandbox, inclusion); + } + } + + return sandbox; + } + + /** + * Applies an inclusion rule to the sandbox. + * + * @param sandbox the sandbox + * @param rule the rule + */ + private static void applyInclusionRule(final JexlSandbox sandbox, + final JexlInclusion rule) { + if (rule.getClassName() == null || rule.getClassName().trim().isEmpty()) { + return; + } + String className = rule.getClassName().trim(); + if (rule.isAllMethods() && rule.isAllProperties()) { + sandbox.white(className); + return; + } + + JexlSandbox.Permissions perm = sandbox.permissions(className, + rule.isAllProperties(), + rule.isAllProperties(), + rule.isAllMethods()); + if (!rule.isAllMethods() && rule.getMethods() != null) { + rule.getMethods().stream() + .map(String::trim) + .filter(m -> !m.isEmpty()) + .forEach(m -> perm.execute(m)); + } + if (!rule.isAllProperties() && rule.getProperties() != null) { + rule.getProperties().stream() + .map(String::trim) + .filter(p -> !p.isEmpty()) + .forEach(p -> { + perm.read(p); + perm.write(p); + }); + } + } + + /** + * Constructor for EL. + * + * @param script the script + * @param variables the variables + */ + private EL(final JexlScript script, final Set variables) { this.script = script; this.variables = Collections.unmodifiableSet(variables); } @@ -137,28 +271,42 @@ public ELResult execute(ELContext context) throws ELException { } Object value = script.execute(context); return new ELResult(value); - } catch (JexlException e) { - // JexlException.getMessage() uses 'io.cdap.wrangler.expression.EL' class name in the error message. - // So instead use info object to get information about error message and create custom error message. - JexlInfo info = e.getInfo(); - throw new ELException( - String.format("Error encountered while executing '%s', at line '%d' and column '%d'. " + - "Make sure a valid jexl transformation is provided.", - // here the detail can be null since there are multiple subclasses which extends this - // JexlException, not all of them has this detail information - info.getDetail() == null ? script.getSourceText() : info.getDetail(), - info.getLine(), info.getColumn()), e); - } catch (NumberFormatException e) { - throw new ELException("Type mismatch. Change type of constant " + - "or convert to right data type using conversion functions available. Reason : " - + e.getMessage(), e); } catch (Exception e) { - if (e.getCause() != null) { - throw new ELException(e.getCause().getMessage(), e); - } else { - throw new ELException(e); + throw handleExecutionException(e); + } + } + + private ELException handleExecutionException(Exception e) { + if (e instanceof JexlException) { + JexlException jexlEx = (JexlException) e; + JexlInfo info = jexlEx.getInfo(); + String detail = (info == null || info.getDetail() == null) ? script.getSourceText() : info.getDetail().toString(); + int line = info == null ? 0 : info.getLine(); + int column = info == null ? 0 : info.getColumn(); + + String errorMessage = jexlEx.getMessage(); + if (errorMessage != null && (errorMessage.contains("unsolvable function/method") + || errorMessage.contains("unsolvable property"))) { + return new ELException( + String.format("Security violation: Access to JEXL component '%s' is not " + + "permitted by wrangler. JEXL expression '%s' can't be resolved.", + detail, script.getSourceText()), + jexlEx); } + return new ELException( + String.format("Error encountered while executing '%s', at line '%d' and column '%d'. " + + "Make sure a valid jexl transformation is provided.", + detail, line, column), + jexlEx); } + + if (e instanceof NumberFormatException) { + return new ELException("Type mismatch. Change type of constant " + + "or convert to right data type using conversion functions available. Reason : " + + e.getMessage(), e); + } + + return new ELException(e); } /** diff --git a/wrangler-core/src/main/java/io/cdap/wrangler/expression/JexlAllowedClasses.java b/wrangler-core/src/main/java/io/cdap/wrangler/expression/JexlAllowedClasses.java new file mode 100644 index 000000000..0b64dd6ae --- /dev/null +++ b/wrangler-core/src/main/java/io/cdap/wrangler/expression/JexlAllowedClasses.java @@ -0,0 +1,107 @@ +/* + * Copyright © 2024 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.wrangler.expression; + +import com.google.common.base.Strings; +import io.cdap.cdap.api.common.Bytes; +import io.cdap.functions.DateAndTime; +import io.cdap.functions.Dates; + +import io.cdap.functions.Logical; +import io.cdap.functions.NumberFunctions; +import io.cdap.wrangler.api.Pair; +import io.cdap.wrangler.api.Row; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringEscapeUtils; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.MathContext; +import java.math.RoundingMode; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.Period; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.Calendar; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.StringJoiner; +import java.util.TimeZone; +import java.util.TreeSet; +import java.util.UUID; + +/** + * Defines the set of classes allowed in the JEXL Sandbox by default. + */ +public final class JexlAllowedClasses { + + /** + * The list of default allowed classes. + */ + public static final List> DEFAULT_ALLOWED_CLASSES = + Collections.unmodifiableList(Arrays.asList( + // Primitives and Numeric Types + Boolean.class, Byte.class, Character.class, Double.class, Float.class, + Integer.class, Long.class, Short.class, Number.class, Math.class, + BigDecimal.class, BigInteger.class, MathContext.class, RoundingMode.class, + + // Strings & Text Processing + String.class, StringBuilder.class, StringBuffer.class, StringJoiner.class, + CharSequence.class, StringUtils.class, + org.apache.commons.lang3.StringUtils.class, + Strings.class, StringEscapeUtils.class, + + // Date & Time + LocalDate.class, LocalDateTime.class, LocalTime.class, ZonedDateTime.class, + Instant.class, Duration.class, Period.class, ZoneId.class, ZoneOffset.class, + DateTimeFormatter.class, Date.class, Calendar.class, TimeZone.class, + DateAndTime.class, Dates.class, + + // Collections & Core Utilities + Arrays.class, Collections.class, Objects.class, UUID.class, + Base64.class, Base64.Encoder.class, Base64.Decoder.class, + List.class, ArrayList.class, LinkedList.class, + Map.class, HashMap.class, LinkedHashMap.class, Map.Entry.class, + Set.class, HashSet.class, LinkedHashSet.class, TreeSet.class, + Collection.class, Iterable.class, Iterator.class, Comparable.class, + + // CDAP & Wrangler Expression Language Functions and Context + Bytes.class, NumberFunctions.class, Logical.class, Row.class, Pair.class)); + + private JexlAllowedClasses() { + } +} diff --git a/wrangler-core/src/main/java/io/cdap/wrangler/parser/ConfigDirectiveContext.java b/wrangler-core/src/main/java/io/cdap/wrangler/parser/ConfigDirectiveContext.java index 20d100077..e8f4226f4 100644 --- a/wrangler-core/src/main/java/io/cdap/wrangler/parser/ConfigDirectiveContext.java +++ b/wrangler-core/src/main/java/io/cdap/wrangler/parser/ConfigDirectiveContext.java @@ -31,6 +31,11 @@ public ConfigDirectiveContext(DirectiveConfig config) { this.config = config; } + @Override + public DirectiveConfig getConfig() { + return config; + } + /** * Checks if the directive is aliased. * diff --git a/wrangler-core/src/main/java/io/cdap/wrangler/parser/GrammarBasedParser.java b/wrangler-core/src/main/java/io/cdap/wrangler/parser/GrammarBasedParser.java index 21ac03ca1..5b5827fa8 100644 --- a/wrangler-core/src/main/java/io/cdap/wrangler/parser/GrammarBasedParser.java +++ b/wrangler-core/src/main/java/io/cdap/wrangler/parser/GrammarBasedParser.java @@ -84,7 +84,7 @@ public List parse() throws RecipeException { try { Directive directive = info.instance(); UsageDefinition definition = directive.define(); - Arguments arguments = new MapArguments(definition, tokenGroup); + Arguments arguments = new MapArguments(definition, tokenGroup, context); directive.initialize(arguments); result.add(directive); diff --git a/wrangler-core/src/main/java/io/cdap/wrangler/parser/MapArguments.java b/wrangler-core/src/main/java/io/cdap/wrangler/parser/MapArguments.java index 8b8c429cf..8c6507da4 100644 --- a/wrangler-core/src/main/java/io/cdap/wrangler/parser/MapArguments.java +++ b/wrangler-core/src/main/java/io/cdap/wrangler/parser/MapArguments.java @@ -19,6 +19,8 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import io.cdap.wrangler.api.Arguments; +import io.cdap.wrangler.api.DirectiveConfig; +import io.cdap.wrangler.api.DirectiveContext; import io.cdap.wrangler.api.DirectiveParseException; import io.cdap.wrangler.api.LazyNumber; import io.cdap.wrangler.api.TokenGroup; @@ -26,6 +28,7 @@ import io.cdap.wrangler.api.parser.BoolList; import io.cdap.wrangler.api.parser.ColumnName; import io.cdap.wrangler.api.parser.ColumnNameList; +import io.cdap.wrangler.api.parser.Expression; import io.cdap.wrangler.api.parser.Numeric; import io.cdap.wrangler.api.parser.NumericList; import io.cdap.wrangler.api.parser.Text; @@ -40,6 +43,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import javax.annotation.Nullable; /** * Class description here. @@ -49,12 +53,22 @@ public class MapArguments implements Arguments { private final int lineno; private final int columnno; private final String source; + private final DirectiveContext context; public MapArguments(UsageDefinition definition, TokenGroup group) throws DirectiveParseException { + this(definition, group, null); + } + + public MapArguments(UsageDefinition definition, TokenGroup group, + @Nullable DirectiveContext context) throws DirectiveParseException { this.tokens = new HashMap<>(); this.lineno = group.getSourceInfo().getLineNumber(); this.columnno = group.getSourceInfo().getColumnNumber(); this.source = group.getSourceInfo().getSource(); + this.context = context; + + DirectiveConfig config = context != null ? context.getConfig() : null; + int required = definition.getTokens().size() - definition.getOptionalTokensCount(); if ((required > group.size() - 1) || ((group.size() - 1) > definition.getTokens().size())) { throw new DirectiveParseException( @@ -234,4 +248,9 @@ public JsonElement toJson() { object.add("arguments", arguments); return object; } + + public DirectiveContext getDirectiveContext() { + return context; + } + } diff --git a/wrangler-core/src/main/java/io/cdap/wrangler/utils/JexlHelper.java b/wrangler-core/src/main/java/io/cdap/wrangler/utils/JexlHelper.java new file mode 100644 index 000000000..32167151e --- /dev/null +++ b/wrangler-core/src/main/java/io/cdap/wrangler/utils/JexlHelper.java @@ -0,0 +1,65 @@ +/* + * Copyright © 2017-2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.wrangler.utils; + +import io.cdap.wrangler.api.Arguments; +import io.cdap.wrangler.api.DirectiveContext; +import io.cdap.wrangler.api.JexlInclusion; +import io.cdap.wrangler.parser.MapArguments; + +import java.util.Collections; +import java.util.List; + +/** + * Helper class for JEXL related operations. + */ +public final class JexlHelper { + + private JexlHelper() { + // Utility class + } + + /** + * Checks if secure JEXL feature is enabled. + * + * @param args the arguments + * @return true if enabled + */ + public static boolean isSecureJexlFeatureEnabled(final Arguments args) { + if (args instanceof MapArguments) { + DirectiveContext context = ((MapArguments) args).getDirectiveContext(); + return context != null && context.isSecureJexlFeatureEnabled(); + } + return false; + } + + /** + * Gets the list of JEXL inclusions. + * + * @param args the arguments + * @return the list of JEXL inclusions + */ + public static List getJexlInclusions(final Arguments args) { + if (args instanceof MapArguments) { + DirectiveContext context = ((MapArguments) args).getDirectiveContext(); + if (context != null && context.getConfig() != null) { + return context.getConfig().getJexlInclusions(); + } + } + return Collections.emptyList(); + } +} diff --git a/wrangler-service/src/main/java/io/cdap/wrangler/service/directive/AbstractDirectiveHandler.java b/wrangler-service/src/main/java/io/cdap/wrangler/service/directive/AbstractDirectiveHandler.java index 4debd5eee..3fc7ddd0a 100644 --- a/wrangler-service/src/main/java/io/cdap/wrangler/service/directive/AbstractDirectiveHandler.java +++ b/wrangler-service/src/main/java/io/cdap/wrangler/service/directive/AbstractDirectiveHandler.java @@ -25,6 +25,7 @@ import io.cdap.directives.aggregates.DefaultTransientStore; import io.cdap.wrangler.api.CompileException; import io.cdap.wrangler.api.DirectiveConfig; +import io.cdap.wrangler.api.DirectiveContext; import io.cdap.wrangler.api.DirectiveParseException; import io.cdap.wrangler.api.ErrorRecordBase; import io.cdap.wrangler.api.ExecutorContext; @@ -132,15 +133,24 @@ protected List executeDirectives( // Parse and call grammar visitor DirectiveConfig config = getDirectiveConfig(); + DirectiveContext directiveContext = new ConfigDirectiveContext(config) { + @Override + public boolean isSecureJexlFeatureEnabled() { + return false; + // TODO: Hardcoded false for now, fetch value from feature flag once available + // in CDAP. + // return Feature.WRANGLER_JEXL_ALLOWLIST.isEnabled(getContext()); + } + }; try { - GrammarWalker walker = new GrammarWalker(new RecipeCompiler(), new ConfigDirectiveContext(config)); + GrammarWalker walker = new GrammarWalker(new RecipeCompiler(), directiveContext); walker.walk(recipe, grammarVisitor); } catch (CompileException e) { throw new BadRequestException(e.getMessage(), e); } RecipeParser parser = new GrammarBasedParser(namespace, recipe, composite, - new ConfigDirectiveContext(config)); + directiveContext); try (RecipePipelineExecutor executor = new RecipePipelineExecutor(parser, new ServicePipelineContext( namespace, ExecutorContext.Environment.SERVICE, diff --git a/wrangler-service/src/main/java/io/cdap/wrangler/service/directive/RemoteExecutionTask.java b/wrangler-service/src/main/java/io/cdap/wrangler/service/directive/RemoteExecutionTask.java index 65a0ebb12..dfab5b9db 100644 --- a/wrangler-service/src/main/java/io/cdap/wrangler/service/directive/RemoteExecutionTask.java +++ b/wrangler-service/src/main/java/io/cdap/wrangler/service/directive/RemoteExecutionTask.java @@ -28,6 +28,7 @@ import io.cdap.wrangler.api.CompileException; import io.cdap.wrangler.api.Directive; import io.cdap.wrangler.api.DirectiveConfig; +import io.cdap.wrangler.api.DirectiveContext; import io.cdap.wrangler.api.DirectiveLoadException; import io.cdap.wrangler.api.DirectiveParseException; import io.cdap.wrangler.api.ErrorRecordBase; @@ -83,7 +84,16 @@ public void run(RunnableTaskContext runnableTaskContext) throws Exception { try (UserDirectiveRegistry userDirectiveRegistry = new UserDirectiveRegistry(systemAppContext)) { List directives = new ArrayList<>(); DirectiveConfig config = directiveRequest.getDirectiveConfig(); - GrammarWalker walker = new GrammarWalker(new RecipeCompiler(), new ConfigDirectiveContext(config)); + DirectiveContext directiveContext = new ConfigDirectiveContext(config) { + @Override + public boolean isSecureJexlFeatureEnabled() { + return false; + // TODO: Hardcoded false for now, fetch value from feature flag once available + // in CDAP. + // return Feature.WRANGLER_JEXL_ALLOWLIST.isEnabled(systemAppContext); + } + }; + GrammarWalker walker = new GrammarWalker(new RecipeCompiler(), directiveContext); walker.walk(directiveRequest.getRecipe(), (command, tokenGroup) -> { DirectiveInfo info; DirectiveClass directiveClass = systemDirectives.get(command); @@ -101,7 +111,7 @@ public void run(RunnableTaskContext runnableTaskContext) throws Exception { Directive directive = info.instance(); UsageDefinition definition = directive.define(); - Arguments arguments = new MapArguments(definition, tokenGroup); + Arguments arguments = new MapArguments(definition, tokenGroup, directiveContext); directive.initialize(arguments); directives.add(directive); }); diff --git a/wrangler-transform/src/main/java/io/cdap/wrangler/Wrangler.java b/wrangler-transform/src/main/java/io/cdap/wrangler/Wrangler.java index b2d03c46a..8f15545ee 100644 --- a/wrangler-transform/src/main/java/io/cdap/wrangler/Wrangler.java +++ b/wrangler-transform/src/main/java/io/cdap/wrangler/Wrangler.java @@ -624,7 +624,15 @@ private RecipeParser getRecipeParser(StageContext context) { } DirectiveConfig directiveConfig = getSystemDirectiveConfigFromRuntimeArgs(context); - DirectiveContext directiveContext = new ConfigDirectiveContext(directiveConfig); + DirectiveContext directiveContext = new ConfigDirectiveContext(directiveConfig) { + @Override + public boolean isSecureJexlFeatureEnabled() { + return false; + // TODO: Hardcoded false for now, fetch value from feature flag once available + // in CDAP. + // return context.isFeatureEnabled("WRANGLER_JEXL_ALLOWLIST"); + } + }; try { return new GrammarBasedParser(context.getNamespace(), new MigrateToV2(directives).migrate(),