Skip to content

Commit 453b7d4

Browse files
MryangeMryange
committed
upd
1 parent c494892 commit 453b7d4

23 files changed

Lines changed: 116 additions & 79 deletions

File tree

‎be/src/core/data_type/data_type_array.cpp‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,13 @@ namespace ErrorCodes {
4848
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
4949
}
5050

51-
DataTypeArray::DataTypeArray(const DataTypePtr& nested_) : nested {nested_} {}
51+
DataTypeArray::DataTypeArray(const DataTypePtr& nested_) {
52+
DataTypePtr nullable_nested = make_nullable(nested_);
53+
auto nested_type = std::dynamic_pointer_cast<const DataTypeNullable>(nullable_nested);
54+
DORIS_CHECK(nested_type != nullptr);
55+
nested = std::move(nested_type);
56+
nested_as_base = nested;
57+
}
5258

5359
MutableColumnPtr DataTypeArray::create_column() const {
5460
return ColumnArray::create(nested->create_column(), ColumnArray::ColumnOffsets::create());
@@ -72,9 +78,10 @@ bool DataTypeArray::equals(const IDataType& rhs) const {
7278

7379
// here we should remove nullable, otherwise here always be 1
7480
size_t DataTypeArray::get_number_of_dimensions() const {
75-
const DataTypeArray* nested_array =
76-
typeid_cast<const DataTypeArray*>(remove_nullable(nested).get());
77-
if (!nested_array) return 1;
81+
auto* nested_array = typeid_cast<const DataTypeArray*>(remove_nullable(nested).get());
82+
if (!nested_array) {
83+
return 1;
84+
}
7885
return 1 +
7986
nested_array
8087
->get_number_of_dimensions(); /// Every modern C++ compiler optimizes tail recursion.
@@ -133,7 +140,7 @@ const char* DataTypeArray::deserialize(const char* buf, MutableColumnPtr* column
133140

134141
void DataTypeArray::to_pb_column_meta(PColumnMeta* col_meta) const {
135142
IDataType::to_pb_column_meta(col_meta);
136-
auto children = col_meta->add_children();
143+
auto* children = col_meta->add_children();
137144
get_nested_type()->to_pb_column_meta(children);
138145
}
139146

‎be/src/core/data_type/data_type_array.h‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929

3030
#include "common/status.h"
3131
#include "core/data_type/data_type.h"
32+
#include "core/data_type/data_type_nullable.h"
3233
#include "core/data_type/define_primitive_type.h"
3334
#include "core/data_type_serde/data_type_array_serde.h"
3435
#include "core/data_type_serde/data_type_serde.h"
@@ -47,7 +48,8 @@ namespace doris {
4748
class DataTypeArray final : public IDataType {
4849
private:
4950
/// The type of array elements.
50-
DataTypePtr nested;
51+
DataTypeNullablePtr nested;
52+
DataTypePtr nested_as_base;
5153

5254
public:
5355
static constexpr PrimitiveType PType = TYPE_ARRAY;
@@ -79,7 +81,8 @@ class DataTypeArray final : public IDataType {
7981

8082
bool equals(const IDataType& rhs) const override;
8183

82-
const DataTypePtr& get_nested_type() const { return nested; }
84+
const DataTypePtr& get_nested_type() const { return nested_as_base; }
85+
const DataTypeNullablePtr& get_nullable_nested_type() const { return nested; }
8386

8487
/// 1 for plain array, 2 for array of arrays and so on.
8588
size_t get_number_of_dimensions() const;
@@ -99,15 +102,15 @@ class DataTypeArray final : public IDataType {
99102
void to_protobuf(PTypeDesc* ptype, PTypeNode* node, PScalarType* scalar_type) const override {
100103
node->set_type(TTypeNodeType::ARRAY);
101104
node->set_contains_null(nested->is_nullable());
102-
nested->to_protobuf(ptype);
105+
get_nested_type()->to_protobuf(ptype);
103106
}
104107

105108
#ifdef BE_TEST
106109
void to_thrift(TTypeDesc& thrift_type, TTypeNode& node) const override {
107110
node.type = TTypeNodeType::ARRAY;
108111
node.__isset.contains_nulls = true;
109112
node.contains_nulls.push_back(nested->is_nullable());
110-
nested->to_thrift(thrift_type);
113+
get_nested_type()->to_thrift(thrift_type);
111114
}
112115
#endif
113116
};

‎be/src/core/data_type/primitive_type.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,10 +92,12 @@ class DataTypeHLL;
9292
class DataTypeJsonb;
9393
class DataTypeArray;
9494
class DataTypeMap;
95+
class DataTypeNullable;
9596
class DataTypeVariant;
9697
class DataTypeStruct;
9798
class DataTypeBitMap;
9899
class DataTypeQuantileState;
100+
using DataTypeNullablePtr = std::shared_ptr<const DataTypeNullable>;
99101
template <PrimitiveType T>
100102
class ColumnVector;
101103
using ColumnUInt8 = ColumnVector<TYPE_BOOLEAN>;

‎be/src/core/data_type_serde/data_type_array_serde.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ Status DataTypeArraySerDe::deserialize_one_cell_from_json(IColumn& column, Slice
8484
auto& array_column = assert_cast<ColumnArray&>(column);
8585
auto& offsets = array_column.get_offsets();
8686
IColumn& nested_column = array_column.get_data();
87-
DCHECK(nested_column.is_nullable());
87+
DORIS_CHECK(nested_column.is_nullable());
8888
if (slice[0] != '[') {
8989
return Status::InvalidArgument("Array does not start with '[' character, found '{}'",
9090
slice[0]);
@@ -162,7 +162,7 @@ Status DataTypeArraySerDe::deserialize_one_cell_from_hive_text(
162162
auto& array_column = assert_cast<ColumnArray&>(column);
163163
auto& offsets = array_column.get_offsets();
164164
IColumn& nested_column = array_column.get_data();
165-
DCHECK(nested_column.is_nullable());
165+
DORIS_CHECK(nested_column.is_nullable());
166166

167167
char collection_delimiter =
168168
options.get_collection_delimiter(hive_text_complex_type_delimiter_level);

‎be/src/core/data_type_serde/data_type_array_serde.h‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
#include <utility>
2626

2727
#include "common/status.h"
28+
#include "core/data_type_serde/data_type_nullable_serde.h"
2829
#include "core/data_type_serde/data_type_serde.h"
2930

3031
namespace doris {
@@ -39,7 +40,12 @@ class IDataType;
3940
class DataTypeArraySerDe : public DataTypeSerDe {
4041
public:
4142
DataTypeArraySerDe(DataTypeSerDeSPtr _nested_serde, int nesting_level = 1)
42-
: DataTypeSerDe(nesting_level), nested_serde(std::move(_nested_serde)) {}
43+
: DataTypeSerDe(nesting_level) {
44+
auto nullable_serde =
45+
std::dynamic_pointer_cast<DataTypeNullableSerDe>(std::move(_nested_serde));
46+
DORIS_CHECK(nullable_serde != nullptr);
47+
nested_serde = std::move(nullable_serde);
48+
}
4349

4450
std::string get_name() const override { return "Array(" + nested_serde->get_name() + ")"; }
4551

@@ -133,6 +139,6 @@ class DataTypeArraySerDe : public DataTypeSerDe {
133139
template <bool is_strict_mode>
134140
Status _from_string(StringRef& str, IColumn& column, const FormatOptions& options) const;
135141

136-
DataTypeSerDeSPtr nested_serde;
142+
DataTypeNullableSerDeSPtr nested_serde;
137143
};
138144
} // namespace doris

‎be/src/core/data_type_serde/data_type_nullable_serde.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,4 +129,7 @@ class DataTypeNullableSerDe : public DataTypeSerDe {
129129
private:
130130
DataTypeSerDeSPtr nested_serde;
131131
};
132+
133+
using DataTypeNullableSerDeSPtr = std::shared_ptr<DataTypeNullableSerDe>;
134+
132135
} // namespace doris

‎be/src/exprs/function/cast/cast_to_array.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ WrapperType create_array_wrapper(FunctionContext* context, const DataTypePtr& fr
5151
"CAST AS Array can only be performed between same-dimensional array types");
5252
}
5353

54-
const DataTypePtr& to_nested_type = to_type.get_nested_type();
54+
DataTypePtr to_nested_type = to_type.get_nested_type();
5555

5656
/// Prepare nested type conversion
5757
const auto nested_function =

‎be/test/core/data_type_serde/data_type_serde_string_test.cpp‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
#include "agent/be_exec_version_manager.h"
3232
#include "core/assert_cast.h"
3333
#include "core/column/column.h"
34-
#include "core/column/column_array.h"
34+
#include "core/column/column_array.h"\n #include "core/column/column_nullable.h"
3535
#include "core/data_type/common_data_type_serder_test.h"
3636
#include "core/data_type/common_data_type_test.h"
3737
#include "core/data_type/data_type.h"
@@ -308,9 +308,12 @@ TEST_F(DataTypeStringSerDeTest, ArrowMemNotAlignedNestedArr) {
308308
EXPECT_EQ(values_address % 4, 1);
309309

310310
// 5.Test read_column_from_arrow
311-
auto ser_col = ColumnArray::create(ColumnString::create(), ColumnOffset64::create());
311+
auto ser_col = ColumnArray::create(
312+
ColumnNullable::create(ColumnString::create(), ColumnUInt8::create()),
313+
ColumnOffset64::create());
312314
cctz::time_zone tz;
313-
auto serde_list = std::make_shared<DataTypeArraySerDe>(serde_str);
315+
auto serde_nullable_str = std::make_shared<DataTypeNullableSerDe>(serde_str);
316+
auto serde_list = std::make_shared<DataTypeArraySerDe>(serde_nullable_str);
314317
auto st = serde_list->read_column_from_arrow(*ser_col, arr.get(), 0, 1, tz);
315318
EXPECT_TRUE(st.ok());
316319
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5259,7 +5259,7 @@ public DataType visitComplexDataType(ComplexDataTypeContext ctx) {
52595259
return ParserUtils.withOrigin(ctx, () -> {
52605260
switch (ctx.complex.getType()) {
52615261
case DorisParser.ARRAY:
5262-
return ArrayType.of(typedVisit(ctx.dataType(0)), true);
5262+
return ArrayType.of(typedVisit(ctx.dataType(0)));
52635263
case DorisParser.MAP:
52645264
return MapType.of(typedVisit(ctx.dataType(0)), typedVisit(ctx.dataType(1)));
52655265
case DorisParser.STRUCT:

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -629,13 +629,13 @@ public static List<Literal> getResultExpression(DataType type, PValues resultCon
629629

630630
private static Pair<DataType, Integer> convertToNereidsType(List<PTypeNode> typeNodes, int start) {
631631
PScalarType pScalarType = typeNodes.get(start).getScalarType();
632-
boolean containsNull = typeNodes.get(start).getContainsNull();
633632
TPrimitiveType tPrimitiveType = TPrimitiveType.findByValue(pScalarType.getType());
634633
DataType type;
635634
int parsedNodes;
636635
if (tPrimitiveType == TPrimitiveType.ARRAY) {
637636
Pair<DataType, Integer> itemType = convertToNereidsType(typeNodes, start + 1);
638-
type = ArrayType.of(itemType.key(), containsNull);
637+
// Array elements are always nullable
638+
type = ArrayType.of(itemType.key());
639639
parsedNodes = 1 + itemType.value();
640640
} else if (tPrimitiveType == TPrimitiveType.MAP) {
641641
Pair<DataType, Integer> keyType = convertToNereidsType(typeNodes, start + 1);

0 commit comments

Comments
 (0)