Skip to content

Commit fe4bb8f

Browse files
huan233uschuan233usc
committed
feat(schema): represent, serialize and validate v3 column default values
First of a multi-part split of column default value support (#730) — the schema foundation the read and evolution paths build on. Purely additive; no read/write behavior change on its own. - SchemaField carries `initial-default` / `write-default` (immutable std::shared_ptr<const Literal>) with copy-preserving WithInitialDefault / WithWriteDefault modifiers; getters return optional<reference_wrapper>. - JSON serde reads/writes `initial-default` / `write-default` via the existing single-value serialization. - Schema::Validate rejects default values below format v3 and validates they are non-null primitive literals matching the field type. - Generic schema projection maps a column missing from a data file with an initial-default to FieldProjection::Kind::kDefault. Read-path application (Parquet/Avro) and schema evolution follow in separate PRs. See #731 for the full end-to-end proof-of-concept.
1 parent c0c6b01 commit fe4bb8f

9 files changed

Lines changed: 492 additions & 10 deletions

File tree

‎src/iceberg/json_serde.cc‎

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@
2727
#include <nlohmann/json.hpp>
2828

2929
#include "iceberg/constants.h"
30+
#include "iceberg/expression/json_serde_internal.h"
31+
#include "iceberg/expression/literal.h"
3032
#include "iceberg/json_serde_internal.h"
3133
#include "iceberg/name_mapping.h"
3234
#include "iceberg/partition_field.h"
@@ -298,6 +300,15 @@ nlohmann::json ToJson(const SchemaField& field) {
298300
if (!field.doc().empty()) {
299301
json[kDoc] = field.doc();
300302
}
303+
// Defaults are validated to be primitive literals matching the field type, so
304+
// single-value serialization cannot fail here.
305+
if (field.initial_default().has_value()) {
306+
ICEBERG_ASSIGN_OR_THROW(json[kInitialDefault],
307+
ToJson(field.initial_default()->get()));
308+
}
309+
if (field.write_default().has_value()) {
310+
ICEBERG_ASSIGN_OR_THROW(json[kWriteDefault], ToJson(field.write_default()->get()));
311+
}
301312
return json;
302313
}
303314

@@ -310,7 +321,6 @@ nlohmann::json ToJson(const Type& type) {
310321
nlohmann::json fields_json = nlohmann::json::array();
311322
for (const auto& field : struct_type.fields()) {
312323
fields_json.push_back(ToJson(field));
313-
// TODO(gangwu): add default values
314324
}
315325
json[kFields] = fields_json;
316326
return json;
@@ -552,9 +562,23 @@ Result<std::unique_ptr<SchemaField>> FieldFromJson(const nlohmann::json& json) {
552562
ICEBERG_ASSIGN_OR_RAISE(auto name, GetJsonValue<std::string>(json, kName));
553563
ICEBERG_ASSIGN_OR_RAISE(auto required, GetJsonValue<bool>(json, kRequired));
554564
ICEBERG_ASSIGN_OR_RAISE(auto doc, GetJsonValueOrDefault<std::string>(json, kDoc));
555-
556-
return std::make_unique<SchemaField>(field_id, std::move(name), std::move(type),
557-
!required, doc);
565+
ICEBERG_ASSIGN_OR_RAISE(std::optional<nlohmann::json> initial_default_json,
566+
GetJsonValueOptional<nlohmann::json>(json, kInitialDefault));
567+
ICEBERG_ASSIGN_OR_RAISE(std::optional<nlohmann::json> write_default_json,
568+
GetJsonValueOptional<nlohmann::json>(json, kWriteDefault));
569+
570+
SchemaField field(field_id, std::move(name), std::move(type), !required, doc);
571+
if (initial_default_json.has_value()) {
572+
ICEBERG_ASSIGN_OR_RAISE(Literal literal,
573+
LiteralFromJson(*initial_default_json, field.type().get()));
574+
field = std::move(field).WithInitialDefault(std::move(literal));
575+
}
576+
if (write_default_json.has_value()) {
577+
ICEBERG_ASSIGN_OR_RAISE(Literal literal,
578+
LiteralFromJson(*write_default_json, field.type().get()));
579+
field = std::move(field).WithWriteDefault(std::move(literal));
580+
}
581+
return std::make_unique<SchemaField>(std::move(field));
558582
}
559583

560584
Result<std::unique_ptr<Schema>> SchemaFromJson(const nlohmann::json& json) {

‎src/iceberg/schema.cc‎

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -116,9 +116,15 @@ std::shared_ptr<Type> ReassignTypeIds(const std::shared_ptr<Type>& type,
116116
SchemaField ReassignField(const SchemaField& field, int32_t new_id,
117117
const Schema::GetId& get_id, Schema::IdMap& ids_to_reassigned,
118118
Schema::IdMap& ids_to_original) {
119-
return {new_id, std::string(field.name()),
120-
ReassignTypeIds(field.type(), get_id, ids_to_reassigned, ids_to_original),
121-
field.optional(), std::string(field.doc())};
119+
SchemaField reassigned{
120+
new_id, std::string(field.name()),
121+
ReassignTypeIds(field.type(), get_id, ids_to_reassigned, ids_to_original),
122+
field.optional(), std::string(field.doc())};
123+
// Reassigning IDs must preserve the field's default values, which the constructor
124+
// does not carry over.
125+
return std::move(reassigned)
126+
.WithInitialDefault(field.initial_default())
127+
.WithWriteDefault(field.write_default());
122128
}
123129

124130
std::vector<SchemaField> ReassignIds(std::vector<SchemaField> fields,
@@ -447,7 +453,21 @@ Status Schema::Validate(int32_t format_version) const {
447453
}
448454
}
449455

450-
// TODO(GuoTao.yu): Check default values when they are supported
456+
// Only the initial-default is gated on format version: it changes how existing
457+
// data files are read (rows written before the column existed materialize this
458+
// value), so it requires the v3 reader contract. A write-default only affects
459+
// values written going forward and does not reinterpret existing data.
460+
if (field.initial_default().has_value() &&
461+
format_version < TableMetadata::kMinFormatVersionDefaultValues) {
462+
return InvalidSchema(
463+
"Invalid initial default for {}: non-null default ({}) is not supported "
464+
"until v{}",
465+
field.name(), field.initial_default()->get(),
466+
TableMetadata::kMinFormatVersionDefaultValues);
467+
}
468+
if (field.initial_default().has_value() || field.write_default().has_value()) {
469+
ICEBERG_RETURN_UNEXPECTED(field.Validate());
470+
}
451471
}
452472

453473
return {};

‎src/iceberg/schema_field.cc‎

Lines changed: 105 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,12 @@
2121

2222
#include <format>
2323
#include <string_view>
24+
#include <utility>
2425

26+
#include "iceberg/expression/literal.h"
2527
#include "iceberg/type.h"
2628
#include "iceberg/util/formatter.h" // IWYU pragma: keep
29+
#include "iceberg/util/macros.h"
2730

2831
namespace iceberg {
2932

@@ -55,13 +58,100 @@ bool SchemaField::optional() const { return optional_; }
5558

5659
std::string_view SchemaField::doc() const { return doc_; }
5760

61+
std::optional<std::reference_wrapper<const Literal>> SchemaField::initial_default()
62+
const {
63+
if (initial_default_ == nullptr) {
64+
return std::nullopt;
65+
}
66+
return std::cref(*initial_default_);
67+
}
68+
69+
std::optional<std::reference_wrapper<const Literal>> SchemaField::write_default() const {
70+
if (write_default_ == nullptr) {
71+
return std::nullopt;
72+
}
73+
return std::cref(*write_default_);
74+
}
75+
76+
SchemaField SchemaField::WithInitialDefault(Literal initial_default) && {
77+
initial_default_ = std::make_shared<const Literal>(std::move(initial_default));
78+
return std::move(*this);
79+
}
80+
81+
SchemaField SchemaField::WithInitialDefault(Literal initial_default) const& {
82+
return SchemaField(*this).WithInitialDefault(std::move(initial_default));
83+
}
84+
85+
SchemaField SchemaField::WithInitialDefault(
86+
std::optional<std::reference_wrapper<const Literal>> initial_default) && {
87+
initial_default_ = initial_default.has_value()
88+
? std::make_shared<const Literal>(initial_default->get())
89+
: nullptr;
90+
return std::move(*this);
91+
}
92+
93+
SchemaField SchemaField::WithInitialDefault(
94+
std::optional<std::reference_wrapper<const Literal>> initial_default) const& {
95+
return SchemaField(*this).WithInitialDefault(initial_default);
96+
}
97+
98+
SchemaField SchemaField::WithWriteDefault(Literal write_default) && {
99+
write_default_ = std::make_shared<const Literal>(std::move(write_default));
100+
return std::move(*this);
101+
}
102+
103+
SchemaField SchemaField::WithWriteDefault(Literal write_default) const& {
104+
return SchemaField(*this).WithWriteDefault(std::move(write_default));
105+
}
106+
107+
SchemaField SchemaField::WithWriteDefault(
108+
std::optional<std::reference_wrapper<const Literal>> write_default) && {
109+
write_default_ = write_default.has_value()
110+
? std::make_shared<const Literal>(write_default->get())
111+
: nullptr;
112+
return std::move(*this);
113+
}
114+
115+
SchemaField SchemaField::WithWriteDefault(
116+
std::optional<std::reference_wrapper<const Literal>> write_default) const& {
117+
return SchemaField(*this).WithWriteDefault(write_default);
118+
}
119+
120+
namespace {
121+
122+
Status ValidateDefault(const SchemaField& field, const Literal& value,
123+
std::string_view kind) {
124+
if (value.IsNull() || value.IsAboveMax() || value.IsBelowMin()) {
125+
return InvalidSchema("Invalid {} value for {}: must be a non-null value", kind,
126+
field.name());
127+
}
128+
if (field.type() == nullptr || !field.type()->is_primitive()) {
129+
return InvalidSchema("Invalid {} value for {}: {} (must be null)", kind, field.name(),
130+
value);
131+
}
132+
if (*value.type() != *field.type()) {
133+
return InvalidSchema("{} of field {} has type {} but expected {}", kind, field.name(),
134+
*value.type(), *field.type());
135+
}
136+
return {};
137+
}
138+
139+
} // namespace
140+
58141
Status SchemaField::Validate() const {
59142
if (name_.empty()) [[unlikely]] {
60143
return InvalidSchema("SchemaField cannot have empty name");
61144
}
62145
if (type_ == nullptr) [[unlikely]] {
63146
return InvalidSchema("SchemaField cannot have null type");
64147
}
148+
if (initial_default_ != nullptr) {
149+
ICEBERG_RETURN_UNEXPECTED(
150+
ValidateDefault(*this, *initial_default_, "initial-default"));
151+
}
152+
if (write_default_ != nullptr) {
153+
ICEBERG_RETURN_UNEXPECTED(ValidateDefault(*this, *write_default_, "write-default"));
154+
}
65155
return {};
66156
}
67157

@@ -72,9 +162,23 @@ std::string SchemaField::ToString() const {
72162
return result;
73163
}
74164

165+
namespace {
166+
167+
bool DefaultEquals(const std::shared_ptr<const Literal>& lhs,
168+
const std::shared_ptr<const Literal>& rhs) {
169+
if (lhs == nullptr || rhs == nullptr) {
170+
return lhs == rhs;
171+
}
172+
return *lhs == *rhs;
173+
}
174+
175+
} // namespace
176+
75177
bool SchemaField::Equals(const SchemaField& other) const {
76178
return field_id_ == other.field_id_ && name_ == other.name_ && *type_ == *other.type_ &&
77-
optional_ == other.optional_;
179+
optional_ == other.optional_ &&
180+
DefaultEquals(initial_default_, other.initial_default_) &&
181+
DefaultEquals(write_default_, other.write_default_);
78182
}
79183

80184
} // namespace iceberg

‎src/iceberg/schema_field.h‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,9 @@
2424
/// type (e.g. a struct).
2525

2626
#include <cstdint>
27+
#include <functional>
2728
#include <memory>
29+
#include <optional>
2830
#include <string>
2931
#include <string_view>
3032

@@ -71,6 +73,22 @@ class ICEBERG_EXPORT SchemaField : public iceberg::util::Formattable {
7173
/// \brief Get the field documentation.
7274
std::string_view doc() const;
7375

76+
/// \brief Get the default value for this field used when reading rows written
77+
/// before the field existed (v3 `initial-default`). Empty if absent.
78+
///
79+
/// The returned reference is a non-owning view into a value owned by this field;
80+
/// it remains valid for the lifetime of this SchemaField.
81+
[[nodiscard]] std::optional<std::reference_wrapper<const Literal>> initial_default()
82+
const;
83+
84+
/// \brief Get the default value for this field used when a writer does not
85+
/// supply a value (v3 `write-default`). Empty if absent.
86+
///
87+
/// The returned reference is a non-owning view into a value owned by this field;
88+
/// it remains valid for the lifetime of this SchemaField.
89+
[[nodiscard]] std::optional<std::reference_wrapper<const Literal>> write_default()
90+
const;
91+
7492
[[nodiscard]] std::string ToString() const override;
7593

7694
Status Validate() const;
@@ -91,6 +109,40 @@ class ICEBERG_EXPORT SchemaField : public iceberg::util::Formattable {
91109
return copy;
92110
}
93111

112+
/// \brief Return a copy of this field with the given `initial-default` value.
113+
///
114+
/// The returned field takes ownership of the value. The rvalue overload sets the
115+
/// value in place and moves this field into the result, so chained builders on a
116+
/// temporary do not copy the field.
117+
[[nodiscard]] SchemaField WithInitialDefault(Literal initial_default) const&;
118+
[[nodiscard]] SchemaField WithInitialDefault(Literal initial_default) &&;
119+
120+
/// \brief Return a copy of this field with the `initial-default` value copied from
121+
/// `initial_default`, or without one if it is empty.
122+
///
123+
/// The referenced value is copied; no reference to it is retained.
124+
[[nodiscard]] SchemaField WithInitialDefault(
125+
std::optional<std::reference_wrapper<const Literal>> initial_default) const&;
126+
[[nodiscard]] SchemaField WithInitialDefault(
127+
std::optional<std::reference_wrapper<const Literal>> initial_default) &&;
128+
129+
/// \brief Return a copy of this field with the given `write-default` value.
130+
///
131+
/// The returned field takes ownership of the value. The rvalue overload sets the
132+
/// value in place and moves this field into the result, so chained builders on a
133+
/// temporary do not copy the field.
134+
[[nodiscard]] SchemaField WithWriteDefault(Literal write_default) const&;
135+
[[nodiscard]] SchemaField WithWriteDefault(Literal write_default) &&;
136+
137+
/// \brief Return a copy of this field with the `write-default` value copied from
138+
/// `write_default`, or without one if it is empty.
139+
///
140+
/// The referenced value is copied; no reference to it is retained.
141+
[[nodiscard]] SchemaField WithWriteDefault(
142+
std::optional<std::reference_wrapper<const Literal>> write_default) const&;
143+
[[nodiscard]] SchemaField WithWriteDefault(
144+
std::optional<std::reference_wrapper<const Literal>> write_default) &&;
145+
94146
private:
95147
/// \brief Compare two fields for equality.
96148
[[nodiscard]] bool Equals(const SchemaField& other) const;
@@ -100,6 +152,11 @@ class ICEBERG_EXPORT SchemaField : public iceberg::util::Formattable {
100152
std::shared_ptr<Type> type_;
101153
bool optional_;
102154
std::string doc_;
155+
// Default values are owned by this field and never mutated after being set; copies
156+
// of the field share the same payload (reference-counted) instead of deep-copying,
157+
// like `type_` above. Sharing is unobservable because the payload is immutable.
158+
std::shared_ptr<const Literal> initial_default_;
159+
std::shared_ptr<const Literal> write_default_;
103160
};
104161

105162
} // namespace iceberg

‎src/iceberg/schema_util.cc‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -172,10 +172,14 @@ Result<FieldProjection> ProjectNested(const Type& expected_type, const Type& sou
172172
iter->second.local_index, prune_source));
173173
} else if (MetadataColumns::IsMetadataColumn(field_id)) {
174174
child_projection.kind = FieldProjection::Kind::kMetadata;
175+
} else if (expected_field.initial_default().has_value()) {
176+
// Rows written before the field existed assume its `initial-default` value.
177+
child_projection.kind = FieldProjection::Kind::kDefault;
178+
child_projection.from = expected_field.initial_default()->get();
175179
} else if (expected_field.optional()) {
176180
child_projection.kind = FieldProjection::Kind::kNull;
177181
} else {
178-
// TODO(gangwu): support default value for v3 and constant value
182+
// TODO(gangwu): support constant value
179183
return InvalidSchema("Missing required field: {}", expected_field.ToString());
180184
}
181185
result.children.emplace_back(std::move(child_projection));

0 commit comments

Comments
 (0)