[WIP] Variant shredded storage

extension/parquet/include/reader/variant/variant_binary_decoder.hpp

@@ -2,7 +2,8 @@
 
 #include "duckdb/common/types/string_type.hpp"
 #include "duckdb/common/types/value.hpp"
-#include "reader/variant/variant_value.hpp"
+#include "duckdb/common/types/variant_value.hpp"
+#include "yyjson.hpp"
 
 using namespace duckdb_yyjson;
 

extension/parquet/include/reader/variant/variant_shredded_conversion.hpp

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "reader/variant/variant_value.hpp"
+#include "duckdb/common/types/variant_value.hpp"
 #include "reader/variant/variant_binary_decoder.hpp"
 
 namespace duckdb {

extension/parquet/reader/variant/CMakeLists.txt

@@ -1,6 +1,5 @@
-add_library_unity(
-  duckdb_parquet_reader_variant OBJECT variant_binary_decoder.cpp
-  variant_value.cpp variant_shredded_conversion.cpp)
+add_library_unity(duckdb_parquet_reader_variant OBJECT
+                  variant_binary_decoder.cpp variant_shredded_conversion.cpp)
 
 set(PARQUET_EXTENSION_FILES
     ${PARQUET_EXTENSION_FILES} $<TARGET_OBJECTS:duckdb_parquet_reader_variant>

src/catalog/catalog_entry/duck_table_entry.cpp

@@ -53,9 +53,6 @@ DuckTableEntry::DuckTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, Bou
 	// create the physical storage
 	vector<ColumnDefinition> column_defs;
 	for (auto &col_def : columns.Physical()) {
-		if (TypeVisitor::Contains(col_def.Type(), LogicalTypeId::VARIANT)) {
-			throw NotImplementedException("A table cannot be created from a VARIANT column yet");
-		}
 		column_defs.push_back(col_def.Copy());
 	}
 	storage = make_shared_ptr<DataTable>(catalog.GetAttached(), StorageManager::Get(catalog).GetTableIOManager(&info),

src/common/enum_util.cpp

@@ -95,6 +95,7 @@
 #include "duckdb/common/types/row/tuple_data_states.hpp"
 #include "duckdb/common/types/timestamp.hpp"
 #include "duckdb/common/types/variant.hpp"
+#include "duckdb/common/types/variant_value.hpp"
 #include "duckdb/common/types/vector.hpp"
 #include "duckdb/common/types/vector_buffer.hpp"
 #include "duckdb/execution/index/art/art.hpp"
@@ -4318,19 +4319,20 @@ const StringUtil::EnumStringLiteral *GetStatisticsTypeValues() {
 		{ static_cast<uint32_t>(StatisticsType::STRUCT_STATS), "STRUCT_STATS" },
 		{ static_cast<uint32_t>(StatisticsType::BASE_STATS), "BASE_STATS" },
 		{ static_cast<uint32_t>(StatisticsType::ARRAY_STATS), "ARRAY_STATS" },
-		{ static_cast<uint32_t>(StatisticsType::GEOMETRY_STATS), "GEOMETRY_STATS" }
+		{ static_cast<uint32_t>(StatisticsType::GEOMETRY_STATS), "GEOMETRY_STATS" },
+		{ static_cast<uint32_t>(StatisticsType::VARIANT_STATS), "VARIANT_STATS" }
 	};
 	return values;
 }
 
 template<>
 const char* EnumUtil::ToChars<StatisticsType>(StatisticsType value) {
-	return StringUtil::EnumToString(GetStatisticsTypeValues(), 7, "StatisticsType", static_cast<uint32_t>(value));
+	return StringUtil::EnumToString(GetStatisticsTypeValues(), 8, "StatisticsType", static_cast<uint32_t>(value));
 }
 
 template<>
 StatisticsType EnumUtil::FromString<StatisticsType>(const char *value) {
-	return static_cast<StatisticsType>(StringUtil::StringToEnum(GetStatisticsTypeValues(), 7, "StatisticsType", value));
+	return static_cast<StatisticsType>(StringUtil::StringToEnum(GetStatisticsTypeValues(), 8, "StatisticsType", value));
 }
 
 const StringUtil::EnumStringLiteral *GetStatsInfoValues() {
@@ -4920,6 +4922,26 @@ VariantLogicalType EnumUtil::FromString<VariantLogicalType>(const char *value) {
 	return static_cast<VariantLogicalType>(StringUtil::StringToEnum(GetVariantLogicalTypeValues(), 35, "VariantLogicalType", value));
 }
 
+const StringUtil::EnumStringLiteral *GetVariantValueTypeValues() {
+	static constexpr StringUtil::EnumStringLiteral values[] {
+		{ static_cast<uint32_t>(VariantValueType::PRIMITIVE), "PRIMITIVE" },
+		{ static_cast<uint32_t>(VariantValueType::OBJECT), "OBJECT" },
+		{ static_cast<uint32_t>(VariantValueType::ARRAY), "ARRAY" },
+		{ static_cast<uint32_t>(VariantValueType::MISSING), "MISSING" }
+	};
+	return values;
+}
+
+template<>
+const char* EnumUtil::ToChars<VariantValueType>(VariantValueType value) {
+	return StringUtil::EnumToString(GetVariantValueTypeValues(), 4, "VariantValueType", static_cast<uint32_t>(value));
+}
+
+template<>
+VariantValueType EnumUtil::FromString<VariantValueType>(const char *value) {
+	return static_cast<VariantValueType>(StringUtil::StringToEnum(GetVariantValueTypeValues(), 4, "VariantValueType", value));
+}
+
 const StringUtil::EnumStringLiteral *GetVectorAuxiliaryDataTypeValues() {
 	static constexpr StringUtil::EnumStringLiteral values[] {
 		{ static_cast<uint32_t>(VectorAuxiliaryDataType::ARROW_AUXILIARY), "ARROW_AUXILIARY" }

src/common/types/CMakeLists.txt

@@ -1,5 +1,6 @@
 add_subdirectory(column)
 add_subdirectory(row)
+add_subdirectory(variant)
 
 if(${EXIT_TIME_DESTRUCTORS_WARNING})
   set(CMAKE_CXX_FLAGS_DEBUG

src/common/types/variant/CMakeLists.txt

@@ -0,0 +1,5 @@
+add_library_unity(duckdb_common_variant OBJECT variant.cpp variant_value.cpp
+                  variant_value_convert.cpp)
+set(ALL_OBJECT_FILES
+    ${ALL_OBJECT_FILES} $<TARGET_OBJECTS:duckdb_common_variant>
+    PARENT_SCOPE)

src/common/types/variant/variant.cpp

@@ -0,0 +1,76 @@
+#include "duckdb/common/types/variant.hpp"
+#include "duckdb/common/types/vector.hpp"
+
+namespace duckdb {
+
+VariantVectorData::VariantVectorData(Vector &variant)
+    : variant(variant), keys_index_validity(FlatVector::Validity(VariantVector::GetChildrenKeysIndex(variant))),
+      keys(VariantVector::GetKeys(variant)) {
+	blob_data = FlatVector::GetData<string_t>(VariantVector::GetData(variant));
+	type_ids_data = FlatVector::GetData<uint8_t>(VariantVector::GetValuesTypeId(variant));
+	byte_offset_data = FlatVector::GetData<uint32_t>(VariantVector::GetValuesByteOffset(variant));
+	keys_index_data = FlatVector::GetData<uint32_t>(VariantVector::GetChildrenKeysIndex(variant));
+	values_index_data = FlatVector::GetData<uint32_t>(VariantVector::GetChildrenValuesIndex(variant));
+	values_data = FlatVector::GetData<list_entry_t>(VariantVector::GetValues(variant));
+	children_data = FlatVector::GetData<list_entry_t>(VariantVector::GetChildren(variant));
+	keys_data = FlatVector::GetData<list_entry_t>(keys);
+}
+
+UnifiedVariantVectorData::UnifiedVariantVectorData(const RecursiveUnifiedVectorFormat &variant)
+    : variant(variant), keys(UnifiedVariantVector::GetKeys(variant)),
+      keys_entry(UnifiedVariantVector::GetKeysEntry(variant)), children(UnifiedVariantVector::GetChildren(variant)),
+      keys_index(UnifiedVariantVector::GetChildrenKeysIndex(variant)),
+      values_index(UnifiedVariantVector::GetChildrenValuesIndex(variant)),
+      values(UnifiedVariantVector::GetValues(variant)), type_id(UnifiedVariantVector::GetValuesTypeId(variant)),
+      byte_offset(UnifiedVariantVector::GetValuesByteOffset(variant)), data(UnifiedVariantVector::GetData(variant)),
+      keys_index_validity(keys_index.validity) {
+	blob_data = data.GetData<string_t>();
+	type_id_data = type_id.GetData<uint8_t>();
+	byte_offset_data = byte_offset.GetData<uint32_t>();
+	keys_index_data = keys_index.GetData<uint32_t>();
+	values_index_data = values_index.GetData<uint32_t>();
+	values_data = values.GetData<list_entry_t>();
+	children_data = children.GetData<list_entry_t>();
+	keys_data = keys.GetData<list_entry_t>();
+	keys_entry_data = keys_entry.GetData<string_t>();
+}
+
+bool UnifiedVariantVectorData::RowIsValid(idx_t row) const {
+	return variant.unified.validity.RowIsValid(variant.unified.sel->get_index(row));
+}
+bool UnifiedVariantVectorData::KeysIndexIsValid(idx_t row, idx_t index) const {
+	auto list_entry = GetChildrenListEntry(row);
+	return keys_index_validity.RowIsValid(keys_index.sel->get_index(list_entry.offset + index));
+}
+
+list_entry_t UnifiedVariantVectorData::GetChildrenListEntry(idx_t row) const {
+	return children_data[children.sel->get_index(row)];
+}
+list_entry_t UnifiedVariantVectorData::GetValuesListEntry(idx_t row) const {
+	return values_data[values.sel->get_index(row)];
+}
+const string_t &UnifiedVariantVectorData::GetKey(idx_t row, idx_t index) const {
+	auto list_entry = keys_data[keys.sel->get_index(row)];
+	return keys_entry_data[keys_entry.sel->get_index(list_entry.offset + index)];
+}
+uint32_t UnifiedVariantVectorData::GetKeysIndex(idx_t row, idx_t child_index) const {
+	auto list_entry = GetChildrenListEntry(row);
+	return keys_index_data[keys_index.sel->get_index(list_entry.offset + child_index)];
+}
+uint32_t UnifiedVariantVectorData::GetValuesIndex(idx_t row, idx_t child_index) const {
+	auto list_entry = GetChildrenListEntry(row);
+	return values_index_data[values_index.sel->get_index(list_entry.offset + child_index)];
+}
+VariantLogicalType UnifiedVariantVectorData::GetTypeId(idx_t row, idx_t value_index) const {
+	auto list_entry = values_data[values.sel->get_index(row)];
+	return static_cast<VariantLogicalType>(type_id_data[type_id.sel->get_index(list_entry.offset + value_index)]);
+}
+uint32_t UnifiedVariantVectorData::GetByteOffset(idx_t row, idx_t value_index) const {
+	auto list_entry = values_data[values.sel->get_index(row)];
+	return byte_offset_data[byte_offset.sel->get_index(list_entry.offset + value_index)];
+}
+const string_t &UnifiedVariantVectorData::GetData(idx_t row) const {
+	return blob_data[data.sel->get_index(row)];
+}
+
+} // namespace duckdb

extension/parquet/reader/variant/variant_value.cpp → src/common/types/variant/variant_value.cpp

@@ -1,4 +1,6 @@
-#include "reader/variant/variant_value.hpp"
+#include "duckdb/common/types/variant_value.hpp"
+#include "yyjson.hpp"
+
 #include "duckdb/common/serializer/varint.hpp"
 #include "duckdb/common/enum_util.hpp"
 #include "duckdb/common/types/time.hpp"
@@ -14,6 +16,8 @@
 #include "duckdb/common/helper.hpp"
 #include "duckdb/function/cast/variant/to_variant_fwd.hpp"
 
+using namespace duckdb_yyjson;
+
 namespace duckdb {
 
 void VariantValue::AddChild(const string &key, VariantValue &&val) {

src/common/types/variant/variant_value_convert.cpp

@@ -0,0 +1,55 @@
+#include "duckdb/function/variant/variant_value_convert.hpp"
+
+namespace duckdb {
+
+template <>
+Value ValueConverter::VisitInteger<int8_t>(int8_t val) {
+	return Value::TINYINT(val);
+}
+
+template <>
+Value ValueConverter::VisitInteger<int16_t>(int16_t val) {
+	return Value::SMALLINT(val);
+}
+
+template <>
+Value ValueConverter::VisitInteger<int32_t>(int32_t val) {
+	return Value::INTEGER(val);
+}
+
+template <>
+Value ValueConverter::VisitInteger<int64_t>(int64_t val) {
+	return Value::BIGINT(val);
+}
+
+template <>
+Value ValueConverter::VisitInteger<hugeint_t>(hugeint_t val) {
+	return Value::HUGEINT(val);
+}
+
+template <>
+Value ValueConverter::VisitInteger<uint8_t>(uint8_t val) {
+	return Value::UTINYINT(val);
+}
+
+template <>
+Value ValueConverter::VisitInteger<uint16_t>(uint16_t val) {
+	return Value::USMALLINT(val);
+}
+
+template <>
+Value ValueConverter::VisitInteger<uint32_t>(uint32_t val) {
+	return Value::UINTEGER(val);
+}
+
+template <>
+Value ValueConverter::VisitInteger<uint64_t>(uint64_t val) {
+	return Value::UBIGINT(val);
+}
+
+template <>
+Value ValueConverter::VisitInteger<uhugeint_t>(uhugeint_t val) {
+	return Value::UHUGEINT(val);
+}
+
+} // namespace duckdb

src/function/CMakeLists.txt

@@ -4,6 +4,7 @@ add_subdirectory(pragma)
 add_subdirectory(scalar)
 add_subdirectory(table)
 add_subdirectory(window)
+add_subdirectory(variant)
 
 if(GENERATE_EXTENSION_ENTRIES)
   add_definitions(-DGENERATE_EXTENSION_ENTRIES)