diff --git a/be/src/core/data_type/data_type_array.cpp b/be/src/core/data_type/data_type_array.cpp index a923ee47734245..fe854075ad1601 100644 --- a/be/src/core/data_type/data_type_array.cpp +++ b/be/src/core/data_type/data_type_array.cpp @@ -48,7 +48,13 @@ namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } -DataTypeArray::DataTypeArray(const DataTypePtr& nested_) : nested {nested_} {} +DataTypeArray::DataTypeArray(const DataTypePtr& nested_) { + DataTypePtr nullable_nested = make_nullable(nested_); + auto nested_type = std::dynamic_pointer_cast(nullable_nested); + DORIS_CHECK(nested_type != nullptr); + nested = std::move(nested_type); + nested_as_base = nested; +} MutableColumnPtr DataTypeArray::create_column() const { return ColumnArray::create(nested->create_column(), ColumnArray::ColumnOffsets::create()); @@ -72,9 +78,10 @@ bool DataTypeArray::equals(const IDataType& rhs) const { // here we should remove nullable, otherwise here always be 1 size_t DataTypeArray::get_number_of_dimensions() const { - const DataTypeArray* nested_array = - typeid_cast(remove_nullable(nested).get()); - if (!nested_array) return 1; + auto* nested_array = typeid_cast(remove_nullable(nested).get()); + if (!nested_array) { + return 1; + } return 1 + nested_array ->get_number_of_dimensions(); /// Every modern C++ compiler optimizes tail recursion. @@ -133,7 +140,7 @@ const char* DataTypeArray::deserialize(const char* buf, MutableColumnPtr* column void DataTypeArray::to_pb_column_meta(PColumnMeta* col_meta) const { IDataType::to_pb_column_meta(col_meta); - auto children = col_meta->add_children(); + auto* children = col_meta->add_children(); get_nested_type()->to_pb_column_meta(children); } diff --git a/be/src/core/data_type/data_type_array.h b/be/src/core/data_type/data_type_array.h index d17743cd958489..18154d9e730950 100644 --- a/be/src/core/data_type/data_type_array.h +++ b/be/src/core/data_type/data_type_array.h @@ -29,6 +29,7 @@ #include "common/status.h" #include "core/data_type/data_type.h" +#include "core/data_type/data_type_nullable.h" #include "core/data_type/define_primitive_type.h" #include "core/data_type_serde/data_type_array_serde.h" #include "core/data_type_serde/data_type_serde.h" @@ -47,7 +48,8 @@ namespace doris { class DataTypeArray final : public IDataType { private: /// The type of array elements. - DataTypePtr nested; + DataTypeNullablePtr nested; + DataTypePtr nested_as_base; public: static constexpr PrimitiveType PType = TYPE_ARRAY; @@ -79,7 +81,8 @@ class DataTypeArray final : public IDataType { bool equals(const IDataType& rhs) const override; - const DataTypePtr& get_nested_type() const { return nested; } + const DataTypePtr& get_nested_type() const { return nested_as_base; } + const DataTypeNullablePtr& get_nullable_nested_type() const { return nested; } /// 1 for plain array, 2 for array of arrays and so on. size_t get_number_of_dimensions() const; @@ -99,7 +102,7 @@ class DataTypeArray final : public IDataType { void to_protobuf(PTypeDesc* ptype, PTypeNode* node, PScalarType* scalar_type) const override { node->set_type(TTypeNodeType::ARRAY); node->set_contains_null(nested->is_nullable()); - nested->to_protobuf(ptype); + get_nested_type()->to_protobuf(ptype); } #ifdef BE_TEST @@ -107,7 +110,7 @@ class DataTypeArray final : public IDataType { node.type = TTypeNodeType::ARRAY; node.__isset.contains_nulls = true; node.contains_nulls.push_back(nested->is_nullable()); - nested->to_thrift(thrift_type); + get_nested_type()->to_thrift(thrift_type); } #endif }; diff --git a/be/src/core/data_type/data_type_map.cpp b/be/src/core/data_type/data_type_map.cpp index 3a83848d82e96d..74e38c623d1d1e 100644 --- a/be/src/core/data_type/data_type_map.cpp +++ b/be/src/core/data_type/data_type_map.cpp @@ -40,8 +40,16 @@ namespace doris { DataTypeMap::DataTypeMap(const DataTypePtr& key_type_, const DataTypePtr& value_type_) { - key_type = key_type_; - value_type = value_type_; + auto nullable_key_type = + std::dynamic_pointer_cast(make_nullable(key_type_)); + auto nullable_value_type = + std::dynamic_pointer_cast(make_nullable(value_type_)); + DORIS_CHECK(nullable_key_type != nullptr); + DORIS_CHECK(nullable_value_type != nullptr); + key_type = std::move(nullable_key_type); + value_type = std::move(nullable_value_type); + key_type_as_base = key_type; + value_type_as_base = value_type; } Field DataTypeMap::get_default() const { @@ -68,8 +76,8 @@ Status DataTypeMap::check_column(const IColumn& column) const { void DataTypeMap::to_pb_column_meta(PColumnMeta* col_meta) const { IDataType::to_pb_column_meta(col_meta); - auto key_children = col_meta->add_children(); - auto value_children = col_meta->add_children(); + auto* key_children = col_meta->add_children(); + auto* value_children = col_meta->add_children(); key_type->to_pb_column_meta(key_children); value_type->to_pb_column_meta(value_children); } @@ -79,7 +87,7 @@ bool DataTypeMap::equals(const IDataType& rhs) const { return false; } - const DataTypeMap& rhs_map = static_cast(rhs); + const auto& rhs_map = static_cast(rhs); if (!key_type->equals(*rhs_map.key_type)) { return false; diff --git a/be/src/core/data_type/data_type_map.h b/be/src/core/data_type/data_type_map.h index 2224eb0adcb5e9..3be88b10ae738d 100644 --- a/be/src/core/data_type/data_type_map.h +++ b/be/src/core/data_type/data_type_map.h @@ -29,6 +29,7 @@ #include "common/status.h" #include "core/data_type/data_type.h" +#include "core/data_type/data_type_nullable.h" #include "core/data_type/define_primitive_type.h" #include "core/data_type_serde/data_type_map_serde.h" #include "core/data_type_serde/data_type_serde.h" @@ -47,8 +48,10 @@ namespace doris { */ class DataTypeMap final : public IDataType { private: - DataTypePtr key_type; - DataTypePtr value_type; + DataTypeNullablePtr key_type; + DataTypeNullablePtr value_type; + DataTypePtr key_type_as_base; + DataTypePtr value_type_as_base; public: static constexpr bool is_parametric = true; @@ -74,8 +77,10 @@ class DataTypeMap final : public IDataType { } bool equals(const IDataType& rhs) const override; - const DataTypePtr& get_key_type() const { return key_type; } - const DataTypePtr& get_value_type() const { return value_type; } + const DataTypePtr& get_key_type() const { return key_type_as_base; } + const DataTypePtr& get_value_type() const { return value_type_as_base; } + const DataTypeNullablePtr& get_nullable_key_type() const { return key_type; } + const DataTypeNullablePtr& get_nullable_value_type() const { return value_type; } int64_t get_uncompressed_serialized_bytes(const IColumn& column, int be_exec_version) const override; @@ -93,16 +98,16 @@ class DataTypeMap final : public IDataType { node->set_type(TTypeNodeType::MAP); node->add_contains_nulls(key_type->is_nullable()); node->add_contains_nulls(value_type->is_nullable()); - key_type->to_protobuf(ptype); - value_type->to_protobuf(ptype); + get_key_type()->to_protobuf(ptype); + get_value_type()->to_protobuf(ptype); } #ifdef BE_TEST void to_thrift(TTypeDesc& thrift_type, TTypeNode& node) const override { node.type = TTypeNodeType::MAP; node.contains_nulls.push_back(key_type->is_nullable()); node.contains_nulls.push_back(value_type->is_nullable()); - key_type->to_thrift(thrift_type); - value_type->to_thrift(thrift_type); + get_key_type()->to_thrift(thrift_type); + get_value_type()->to_thrift(thrift_type); } #endif }; diff --git a/be/src/core/data_type/primitive_type.h b/be/src/core/data_type/primitive_type.h index 4fefb70860b16f..57abb3b09a9edb 100644 --- a/be/src/core/data_type/primitive_type.h +++ b/be/src/core/data_type/primitive_type.h @@ -92,10 +92,12 @@ class DataTypeHLL; class DataTypeJsonb; class DataTypeArray; class DataTypeMap; +class DataTypeNullable; class DataTypeVariant; class DataTypeStruct; class DataTypeBitMap; class DataTypeQuantileState; +using DataTypeNullablePtr = std::shared_ptr; template class ColumnVector; using ColumnUInt8 = ColumnVector; diff --git a/be/src/core/data_type_serde/data_type_array_serde.cpp b/be/src/core/data_type_serde/data_type_array_serde.cpp index effc8bffc41df6..e4a91084c722dc 100644 --- a/be/src/core/data_type_serde/data_type_array_serde.cpp +++ b/be/src/core/data_type_serde/data_type_array_serde.cpp @@ -84,7 +84,7 @@ Status DataTypeArraySerDe::deserialize_one_cell_from_json(IColumn& column, Slice auto& array_column = assert_cast(column); auto& offsets = array_column.get_offsets(); IColumn& nested_column = array_column.get_data(); - DCHECK(nested_column.is_nullable()); + DORIS_CHECK(nested_column.is_nullable()); if (slice[0] != '[') { return Status::InvalidArgument("Array does not start with '[' character, found '{}'", slice[0]); @@ -162,7 +162,7 @@ Status DataTypeArraySerDe::deserialize_one_cell_from_hive_text( auto& array_column = assert_cast(column); auto& offsets = array_column.get_offsets(); IColumn& nested_column = array_column.get_data(); - DCHECK(nested_column.is_nullable()); + DORIS_CHECK(nested_column.is_nullable()); char collection_delimiter = options.get_collection_delimiter(hive_text_complex_type_delimiter_level); diff --git a/be/src/core/data_type_serde/data_type_array_serde.h b/be/src/core/data_type_serde/data_type_array_serde.h index 2fcb7f3c9c4a7e..05366962d1ef8c 100644 --- a/be/src/core/data_type_serde/data_type_array_serde.h +++ b/be/src/core/data_type_serde/data_type_array_serde.h @@ -39,7 +39,9 @@ class IDataType; class DataTypeArraySerDe : public DataTypeSerDe { public: DataTypeArraySerDe(DataTypeSerDeSPtr _nested_serde, int nesting_level = 1) - : DataTypeSerDe(nesting_level), nested_serde(std::move(_nested_serde)) {} + : DataTypeSerDe(nesting_level), nested_serde(std::move(_nested_serde)) { + DORIS_CHECK(nested_serde != nullptr); + } std::string get_name() const override { return "Array(" + nested_serde->get_name() + ")"; } diff --git a/be/src/core/data_type_serde/data_type_map_serde.cpp b/be/src/core/data_type_serde/data_type_map_serde.cpp index 00f18d0b699120..1903d0afc14436 100644 --- a/be/src/core/data_type_serde/data_type_map_serde.cpp +++ b/be/src/core/data_type_serde/data_type_map_serde.cpp @@ -77,8 +77,8 @@ Status DataTypeMapSerDe::deserialize_one_cell_from_hive_text( auto& offsets = array_column.get_offsets(); IColumn& nested_key_column = array_column.get_keys(); IColumn& nested_val_column = array_column.get_values(); - DCHECK(nested_key_column.is_nullable()); - DCHECK(nested_val_column.is_nullable()); + DORIS_CHECK(nested_key_column.is_nullable()); + DORIS_CHECK(nested_val_column.is_nullable()); char collection_delimiter = options.get_collection_delimiter(hive_text_complex_type_delimiter_level); @@ -193,8 +193,8 @@ Status DataTypeMapSerDe::deserialize_one_cell_from_json(IColumn& column, Slice& auto& offsets = array_column.get_offsets(); IColumn& nested_key_column = array_column.get_keys(); IColumn& nested_val_column = array_column.get_values(); - DCHECK(nested_key_column.is_nullable()); - DCHECK(nested_val_column.is_nullable()); + DORIS_CHECK(nested_key_column.is_nullable()); + DORIS_CHECK(nested_val_column.is_nullable()); if (slice[0] != '{') { std::stringstream ss; ss << slice[0] << '\''; diff --git a/be/src/core/data_type_serde/data_type_map_serde.h b/be/src/core/data_type_serde/data_type_map_serde.h index 72ac28721b926a..5978efb29d324c 100644 --- a/be/src/core/data_type_serde/data_type_map_serde.h +++ b/be/src/core/data_type_serde/data_type_map_serde.h @@ -21,6 +21,7 @@ #include #include +#include #include "common/status.h" #include "core/data_type_serde/data_type_serde.h" @@ -33,9 +34,14 @@ class Arena; class DataTypeMapSerDe : public DataTypeSerDe { public: - DataTypeMapSerDe(const DataTypeSerDeSPtr& _key_serde, const DataTypeSerDeSPtr& _value_serde, + DataTypeMapSerDe(DataTypeSerDeSPtr _key_serde, DataTypeSerDeSPtr _value_serde, int nesting_level = 1) - : DataTypeSerDe(nesting_level), key_serde(_key_serde), value_serde(_value_serde) {} + : DataTypeSerDe(nesting_level), + key_serde(std::move(_key_serde)), + value_serde(std::move(_value_serde)) { + DORIS_CHECK(key_serde != nullptr); + DORIS_CHECK(value_serde != nullptr); + } std::string get_name() const override { return "Map(" + key_serde->get_name() + ", " + value_serde->get_name() + ")"; @@ -108,9 +114,7 @@ class DataTypeMapSerDe : public DataTypeSerDe { Status serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, JsonbWriter& writer) const override; - virtual DataTypeSerDeSPtrs get_nested_serdes() const override { - return {key_serde, value_serde}; - } + DataTypeSerDeSPtrs get_nested_serdes() const override { return {key_serde, value_serde}; } void to_string(const IColumn& column, size_t row_num, BufferWritable& bw, const FormatOptions& options) const override; diff --git a/be/src/exprs/function/cast/cast_to_array.h b/be/src/exprs/function/cast/cast_to_array.h index 0b1648d47dce68..c1d96c25f35e55 100644 --- a/be/src/exprs/function/cast/cast_to_array.h +++ b/be/src/exprs/function/cast/cast_to_array.h @@ -51,7 +51,7 @@ WrapperType create_array_wrapper(FunctionContext* context, const DataTypePtr& fr "CAST AS Array can only be performed between same-dimensional array types"); } - const DataTypePtr& to_nested_type = to_type.get_nested_type(); + DataTypePtr to_nested_type = to_type.get_nested_type(); /// Prepare nested type conversion const auto nested_function = diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index db82e639971085..1386dd1e8700f5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -5259,7 +5259,7 @@ public DataType visitComplexDataType(ComplexDataTypeContext ctx) { return ParserUtils.withOrigin(ctx, () -> { switch (ctx.complex.getType()) { case DorisParser.ARRAY: - return ArrayType.of(typedVisit(ctx.dataType(0)), true); + return ArrayType.of(typedVisit(ctx.dataType(0))); case DorisParser.MAP: return MapType.of(typedVisit(ctx.dataType(0)), typedVisit(ctx.dataType(1))); case DorisParser.STRUCT: diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java index 0a34b8af2962e9..11716ac78525f1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java @@ -629,13 +629,13 @@ public static List getResultExpression(DataType type, PValues resultCon private static Pair convertToNereidsType(List typeNodes, int start) { PScalarType pScalarType = typeNodes.get(start).getScalarType(); - boolean containsNull = typeNodes.get(start).getContainsNull(); TPrimitiveType tPrimitiveType = TPrimitiveType.findByValue(pScalarType.getType()); DataType type; int parsedNodes; if (tPrimitiveType == TPrimitiveType.ARRAY) { Pair itemType = convertToNereidsType(typeNodes, start + 1); - type = ArrayType.of(itemType.key(), containsNull); + // Array elements are always nullable + type = ArrayType.of(itemType.key()); parsedNodes = 1 + itemType.value(); } else if (tPrimitiveType == TPrimitiveType.MAP) { Pair keyType = convertToNereidsType(typeNodes, start + 1); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java index 3c368f0f4b3cce..69d5994cdecab3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java @@ -503,8 +503,7 @@ public DataType pruneCastType(DataTypeAccessTree origin, DataTypeAccessTree cast children.values().iterator().next().pruneCastType( origin.children.values().iterator().next(), cast.children.values().iterator().next() - ), - ((ArrayType) cast.type).containsNull() + ) ); } else if (type instanceof MapType) { return MapType.of( @@ -717,7 +716,7 @@ private DataType pruneDataType(DataType dataType, List> n } return new StructType(newFields); } else if (dataType instanceof ArrayType) { - return ArrayType.of(newChildrenTypes.get(0).second, ((ArrayType) dataType).containsNull()); + return ArrayType.of(newChildrenTypes.get(0).second); } else if (dataType instanceof MapType) { return MapType.of(newChildrenTypes.get(0).second, newChildrenTypes.get(1).second); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ArrayItemReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ArrayItemReference.java index b763d6741ab49c..bfbb184ec30191 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ArrayItemReference.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ArrayItemReference.java @@ -77,7 +77,8 @@ public List getQualifier() { @Override public boolean nullable() { - return ((ArrayType) (this.children.get(0).getDataType())).containsNull(); + // Array elements are always nullable + return true; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/AggCombinerFunctionBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/AggCombinerFunctionBuilder.java index fc4cca7ead57a7..6c14ac576feb32 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/AggCombinerFunctionBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/AggCombinerFunctionBuilder.java @@ -96,7 +96,8 @@ private AggregateFunction buildForEach(String nestedName, List "foreach must be input array type: '" + nestedName); } DataType itemType = ((ArrayType) arrayType).getItemType(); - return new SlotReference("mocked", itemType, (((ArrayType) arrayType).containsNull())); + // Array elements are always nullable + return new SlotReference("mocked", itemType, true); }).collect(Collectors.toList()); return (AggregateFunction) nestedBuilder.build(nestedName, forEachargs).first; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignatureHelper.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignatureHelper.java index c8629fb7b84d2d..0d0abe92789834 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignatureHelper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignatureHelper.java @@ -483,7 +483,7 @@ public static FunctionSignature ensureNestedNullableOfArray(FunctionSignature si return signature; } ArrayType arrayType = (ArrayType) signature.returnType; - return signature.withReturnType(ArrayType.of(arrayType.getItemType(), true)); + return signature.withReturnType(ArrayType.of(arrayType.getItemType())); } // for time type with precision(now are DateTimeV2Type and TimeV2Type), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/combinator/ForEachCombinator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/combinator/ForEachCombinator.java index be52d59b62ca48..3c5fce06159930 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/combinator/ForEachCombinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/combinator/ForEachCombinator.java @@ -119,7 +119,7 @@ public R accept(ExpressionVisitor visitor, C context) { @Override public DataType getDataType() { - return ArrayType.of(nested.getDataType(), true); + return ArrayType.of(nested.getDataType()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayMap.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayMap.java index 1a1a246cea6d8e..1ef0e996d0d6ef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayMap.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayMap.java @@ -68,7 +68,7 @@ public ArrayMap withChildren(List children) { public DataType getDataType() { Preconditions.checkArgument(children.get(0) instanceof Lambda, "The first arg of array_map must be lambda"); - return ArrayType.of(((Lambda) children.get(0)).getRetType(), true); + return ArrayType.of(((Lambda) children.get(0)).getRetType()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArraySort.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArraySort.java index 2e8b738237ab04..67d579efeaf465 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArraySort.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArraySort.java @@ -90,11 +90,11 @@ public DataType getDataType() { ArrayItemReference argRef = lambda.getLambdaArguments().get(0); Expression arrayExpr = argRef.getArrayExpression(); ArrayType arrayType = (ArrayType) arrayExpr.getDataType(); - return ArrayType.of(arrayType.getItemType(), true); + return ArrayType.of(arrayType.getItemType()); } else if (children.get(0).getDataType() instanceof ArrayType) { Expression arrayExpr = children.get(0); ArrayType arrayType = (ArrayType) arrayExpr.getDataType(); - return ArrayType.of(arrayType.getItemType(), true); + return ArrayType.of(arrayType.getItemType()); } else { throw new AnalysisException("The first arg of array_sort must be lambda or array"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/ArrayType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/ArrayType.java index 6d4ec539ff306e..084756303b64fc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/ArrayType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/ArrayType.java @@ -24,48 +24,43 @@ /** * Array type in Nereids. + * + *

Note: Array elements are always nullable in Doris. The NOT NULL constraint on array elements + * (e.g., ARRAY<INT NOT NULL>) is not supported. This simplification aligns with the actual + * behavior where both FE planning and BE execution always treat array elements as nullable.

*/ public class ArrayType extends DataType implements ComplexDataType, NestedColumnPrunable { - public static final ArrayType SYSTEM_DEFAULT = new ArrayType(NullType.INSTANCE, true); + public static final ArrayType SYSTEM_DEFAULT = new ArrayType(NullType.INSTANCE); public static final int WIDTH = 64; private final DataType itemType; - private final boolean containsNull; - private ArrayType(DataType itemType, boolean containsNull) { + private ArrayType(DataType itemType) { this.itemType = Objects.requireNonNull(itemType, "itemType can not be null"); - this.containsNull = containsNull; } public static ArrayType of(DataType itemType) { - return of(itemType, true); - } - - public static ArrayType of(DataType itemType, boolean containsNull) { if (itemType.equals(NullType.INSTANCE)) { return SYSTEM_DEFAULT; } - return new ArrayType(itemType, containsNull); + return new ArrayType(itemType); } @Override public DataType conversion() { - return new ArrayType(itemType.conversion(), containsNull); + return new ArrayType(itemType.conversion()); } public DataType getItemType() { return itemType; } - public boolean containsNull() { - return containsNull; - } - @Override public Type toCatalogDataType() { - return new org.apache.doris.catalog.ArrayType(itemType.toCatalogDataType(), containsNull); + // Catalog ArrayType defaults containsNull to true via single-arg constructor + return new org.apache.doris.catalog.ArrayType(itemType.toCatalogDataType()); } @Override @@ -85,8 +80,7 @@ public boolean equals(Object o) { return false; } ArrayType arrayType = (ArrayType) o; - return Objects.equals(itemType, arrayType.itemType) - && Objects.equals(containsNull, arrayType.containsNull); + return Objects.equals(itemType, arrayType.itemType); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java index 61041674db29bd..9d7bb0f29a0a06 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java @@ -476,7 +476,7 @@ public static DataType fromCatalogType(Type type) { return MapType.of(fromCatalogType(mapType.getKeyType()), fromCatalogType(mapType.getValueType())); } else if (type.isArrayType()) { org.apache.doris.catalog.ArrayType arrayType = (org.apache.doris.catalog.ArrayType) type; - return ArrayType.of(fromCatalogType(arrayType.getItemType()), arrayType.getContainsNull()); + return ArrayType.of(fromCatalogType(arrayType.getItemType())); } else if (type.isVariantType()) { // In the past, variant metadata used the ScalarType type. // Now, we use VariantType, which inherits from ScalarType, as the new metadata storage. @@ -803,7 +803,7 @@ public List getAllPromotions() { return arrayType.getItemType() .getAllPromotions() .stream() - .map(promotionType -> ArrayType.of(promotionType, arrayType.containsNull())) + .map(promotionType -> ArrayType.of(promotionType)) .collect(ImmutableList.toImmutableList()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java index 052f63237145ee..0dc70934c6d45c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java @@ -385,8 +385,7 @@ public static DataType replaceTimesWithTargetPrecision(DataType dataType, int ta public static DataType replaceSpecifiedType(DataType dataType, Class specifiedType, DataType newType) { if (dataType instanceof ArrayType) { - return ArrayType.of(replaceSpecifiedType(((ArrayType) dataType).getItemType(), specifiedType, newType), - ((ArrayType) dataType).containsNull()); + return ArrayType.of(replaceSpecifiedType(((ArrayType) dataType).getItemType(), specifiedType, newType)); } else if (dataType instanceof MapType) { return MapType.of(replaceSpecifiedType(((MapType) dataType).getKeyType(), specifiedType, newType), replaceSpecifiedType(((MapType) dataType).getValueType(), specifiedType, newType)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java index fe3e2b0bd0a2fa..d3a41f1523525d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java @@ -35,9 +35,9 @@ public void testArrayOfArrayExactMatch() { ArrayType a3 = new ArrayType(new ArrayType(Type.BIGINT, true), true); Assert.assertFalse(Type.matchExactType(a1, a3, false)); - // containsNull differs -> matchesType fails + // containsNull is always true now, so a4 is equivalent to a1 ArrayType a4 = new ArrayType(new ArrayType(Type.INT, true), false); - Assert.assertFalse(Type.matchExactType(a1, a4, false)); + Assert.assertTrue(Type.matchExactType(a1, a4, false)); // array nested decimal test ArrayType a5 = new ArrayType(new ArrayType(ScalarType.createDecimalV3Type(8, 2), true), true); @@ -63,7 +63,7 @@ public void testMapWithNestedValueExactMatch() { Assert.assertFalse(Type.matchExactType(m1, m3, false)); Assert.assertFalse(Type.matchExactType(m1, m3, true)); - // key/value containsNull differs -> doesn't matter for matching + // key/value containsNull is always true now, so m4 is equivalent to m1 MapType m4 = new MapType(Type.INT, arrayOfD, false, true); Assert.assertTrue(Type.matchExactType(m1, m4, false)); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/LiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/LiteralTest.java index fcb64ff0bface9..4964414e6b3db5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/LiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/LiteralTest.java @@ -64,7 +64,7 @@ public void testGetResultExpressionArrayInt() { for (int i = 0; i < elementsArray.length; ++i) { elementsArray[i] = i; } - DataType arrayType = ArrayType.of(IntegerType.INSTANCE, true); + DataType arrayType = ArrayType.of(IntegerType.INSTANCE); PGenericType.Builder childTypeBuilder = PGenericType.newBuilder(); childTypeBuilder.setId(TypeId.INT32); PGenericType.Builder typeBuilder = PGenericType.newBuilder(); @@ -94,8 +94,8 @@ public void testGetResultExpressionArrayArrayInt() { for (int i = 0; i < elementsArray.length; ++i) { elementsArray[i] = i; } - DataType nestedArrayType = ArrayType.of(IntegerType.INSTANCE, true); - DataType outArrayType = ArrayType.of(nestedArrayType, true); + DataType nestedArrayType = ArrayType.of(IntegerType.INSTANCE); + DataType outArrayType = ArrayType.of(nestedArrayType); PGenericType.Builder childTypeBuilder = PGenericType.newBuilder(); childTypeBuilder.setId(TypeId.INT32); PGenericType.Builder typeBuilder = PGenericType.newBuilder(); @@ -134,8 +134,8 @@ public void testGetResultExpressionArrayArrayInt2() { for (int i = 0; i < elementsArray.length; ++i) { elementsArray[i] = i; } - DataType nestedArrayType = ArrayType.of(IntegerType.INSTANCE, true); - DataType outArrayType = ArrayType.of(nestedArrayType, true); + DataType nestedArrayType = ArrayType.of(IntegerType.INSTANCE); + DataType outArrayType = ArrayType.of(nestedArrayType); PGenericType.Builder childTypeBuilder = PGenericType.newBuilder(); childTypeBuilder.setId(TypeId.INT32); PGenericType.Builder typeBuilder = PGenericType.newBuilder(); @@ -182,7 +182,7 @@ public void testGetResultExpressionArrayNull() { elementsArray[i] = i; nullMap[i] = (i % 2 == 1); } - DataType arrayType = ArrayType.of(IntegerType.INSTANCE, true); + DataType arrayType = ArrayType.of(IntegerType.INSTANCE); PGenericType.Builder childTypeBuilder = PGenericType.newBuilder(); childTypeBuilder.setId(TypeId.INT32); PGenericType.Builder typeBuilder = PGenericType.newBuilder(); diff --git a/fe/fe-type/src/main/java/org/apache/doris/catalog/ArrayType.java b/fe/fe-type/src/main/java/org/apache/doris/catalog/ArrayType.java index fb53269d6309de..3c66f6dd24abb9 100644 --- a/fe/fe-type/src/main/java/org/apache/doris/catalog/ArrayType.java +++ b/fe/fe-type/src/main/java/org/apache/doris/catalog/ArrayType.java @@ -31,6 +31,11 @@ /** * Describes an ARRAY type. + * + *

Array elements are always nullable in Doris. The {@code containsNull} field is retained + * only for backward compatibility with serialized metadata (Gson). It is always treated as + * {@code true} at runtime. The two-argument constructor is kept but deprecated; prefer the + * single-argument constructor {@link #ArrayType(Type)} which defaults to nullable elements.

*/ public class ArrayType extends Type { @@ -39,6 +44,8 @@ public class ArrayType extends Type { @SerializedName(value = "itemType") private final Type itemType; + // Retained for Gson deserialization compatibility with older metadata. + // Always treated as true at runtime — array elements are always nullable. @SerializedName(value = "containsNull") private final boolean containsNull; @@ -48,20 +55,31 @@ public ArrayType() { } public ArrayType(Type itemType) { - this(itemType, true); + this.itemType = itemType; + this.containsNull = true; } + /** + * @deprecated Array elements are always nullable. Use {@link #ArrayType(Type)} instead. + * This constructor is retained only for call-site compatibility during transition. + */ + @Deprecated public ArrayType(Type itemType, boolean containsNull) { this.itemType = itemType; - this.containsNull = containsNull; + // Ignore the parameter — always true + this.containsNull = true; } public Type getItemType() { return itemType; } + /** + * Always returns {@code true}. Array elements are always nullable in Doris. + */ public boolean getContainsNull() { - return containsNull; + // Always true — array elements are always nullable + return true; } @Override @@ -83,10 +101,7 @@ public boolean matchesType(Type t) { return false; } - if (((ArrayType) t).getContainsNull() != getContainsNull()) { - return false; - } - + // containsNull is always true, no need to compare return itemType.matchesType(((ArrayType) t).itemType); } @@ -94,24 +109,27 @@ public static ArrayType create() { return new ArrayType(); } + public static ArrayType create(Type type) { + return new ArrayType(type); + } + + /** + * @deprecated Array elements are always nullable. Use {@link #create(Type)} instead. + */ + @Deprecated public static ArrayType create(Type type, boolean containsNull) { - return new ArrayType(type, containsNull); + return new ArrayType(type); } @Override public String toSql(int depth) { - StringBuilder typeStr = new StringBuilder(); - typeStr.append("array<").append(itemType.toSql(depth + 1)); - if (!containsNull) { - typeStr.append(" not null"); - } - typeStr.append(">"); - return typeStr.toString(); + // Array elements are always nullable, no "not null" suffix needed + return "array<" + itemType.toSql(depth + 1) + ">"; } @Override public int hashCode() { - return Objects.hash(itemType, containsNull); + return Objects.hash(itemType); } @Override @@ -120,7 +138,7 @@ public boolean equals(Object other) { return false; } ArrayType otherArrayType = (ArrayType) other; - return otherArrayType.itemType.equals(itemType) && otherArrayType.containsNull == containsNull; + return otherArrayType.itemType.equals(itemType); } @Override @@ -129,8 +147,9 @@ public void toThrift(TTypeDesc container) { container.types.add(node); Preconditions.checkNotNull(itemType); node.setType(TTypeNodeType.ARRAY); - node.setContainsNull(containsNull); - node.setContainsNulls(Lists.newArrayList(containsNull)); + // Array elements are always nullable + node.setContainsNull(true); + node.setContainsNulls(Lists.newArrayList(true)); itemType.toThrift(container); } diff --git a/fe/fe-type/src/main/java/org/apache/doris/catalog/MapType.java b/fe/fe-type/src/main/java/org/apache/doris/catalog/MapType.java index f321fe52a5db24..36b610834b0368 100644 --- a/fe/fe-type/src/main/java/org/apache/doris/catalog/MapType.java +++ b/fe/fe-type/src/main/java/org/apache/doris/catalog/MapType.java @@ -31,20 +31,26 @@ /** * Describes a MAP type. MAP types have a scalar key and an arbitrarily-typed value. + * + *

Map keys and values are always nullable in Doris. The {@code isKeyContainsNull} and + * {@code isValueContainsNull} fields are retained only for backward compatibility with + * serialized metadata (Gson). They are always treated as {@code true} at runtime.

*/ public class MapType extends Type { @SerializedName(value = "keyType") private final Type keyType; + // Retained for Gson deserialization compatibility. Always treated as true. @SerializedName(value = "isKeyContainsNull") - private final boolean isKeyContainsNull; // Now always true + private final boolean isKeyContainsNull; @SerializedName(value = "valueType") private final Type valueType; + // Retained for Gson deserialization compatibility. Always treated as true. @SerializedName(value = "isValueContainsNull") - private final boolean isValueContainsNull; // Now always true + private final boolean isValueContainsNull; public MapType() { this.keyType = NULL; @@ -62,13 +68,19 @@ public MapType(Type keyType, Type valueType) { this.isValueContainsNull = true; } + /** + * @deprecated Map keys and values are always nullable. Use {@link #MapType(Type, Type)} instead. + * This constructor is retained only for call-site compatibility during transition. + */ + @Deprecated public MapType(Type keyType, Type valueType, boolean keyContainsNull, boolean valueContainsNull) { Preconditions.checkNotNull(keyType); Preconditions.checkNotNull(valueType); this.keyType = keyType; - this.isKeyContainsNull = keyContainsNull; + // Ignore the parameters — always true + this.isKeyContainsNull = true; this.valueType = valueType; - this.isValueContainsNull = valueContainsNull; + this.isValueContainsNull = true; } @Override @@ -80,12 +92,18 @@ public Type getKeyType() { return keyType; } + /** + * Always returns {@code true}. Map keys are always nullable in Doris. + */ public Boolean getIsKeyContainsNull() { - return isKeyContainsNull; + return true; } + /** + * Always returns {@code true}. Map values are always nullable in Doris. + */ public Boolean getIsValueContainsNull() { - return isValueContainsNull; + return true; } public Type getValueType() { @@ -125,13 +143,7 @@ public boolean matchesType(Type t) { return false; } - if (((MapType) t).getIsKeyContainsNull() != getIsKeyContainsNull()) { - return false; - } - if (((MapType) t).getIsValueContainsNull() != getIsValueContainsNull()) { - return false; - } - + // containsNull is always true, no need to compare return keyType.matchesType(((MapType) t).keyType) && (valueType.matchesType(((MapType) t).valueType)); } @@ -177,7 +189,8 @@ public void toThrift(TTypeDesc container) { Preconditions.checkNotNull(keyType); Preconditions.checkNotNull(valueType); node.setType(TTypeNodeType.MAP); - node.setContainsNulls(Lists.newArrayList(isKeyContainsNull, isValueContainsNull)); + // Map keys and values are always nullable + node.setContainsNulls(Lists.newArrayList(true, true)); keyType.toThrift(container); valueType.toThrift(container); }