From c8524bbda748c759695966a977e4bcd454bd5307 Mon Sep 17 00:00:00 2001 From: Rui Mo Date: Thu, 24 Feb 2022 16:20:51 +0800 Subject: [PATCH] update Substrait --- cpp/src/compute/substrait_utils.h | 3 +- .../substrait/proto/substrait/algebra.proto | 702 ++++++++++++++++++ .../proto/substrait/expression.proto | 372 ---------- .../substrait/proto/substrait/plan.proto | 2 +- .../substrait/proto/substrait/relations.proto | 238 ------ 5 files changed, 704 insertions(+), 613 deletions(-) create mode 100644 jvm/src/main/resources/substrait/proto/substrait/algebra.proto delete mode 100644 jvm/src/main/resources/substrait/proto/substrait/expression.proto delete mode 100644 jvm/src/main/resources/substrait/proto/substrait/relations.proto diff --git a/cpp/src/compute/substrait_utils.h b/cpp/src/compute/substrait_utils.h index 3f6fa7bc9a1..eca4ee32fab 100644 --- a/cpp/src/compute/substrait_utils.h +++ b/cpp/src/compute/substrait_utils.h @@ -19,13 +19,12 @@ #include +#include "substrait/algebra.pb.h" #include "substrait/capabilities.pb.h" -#include "substrait/expression.pb.h" #include "substrait/extensions/extensions.pb.h" #include "substrait/function.pb.h" #include "substrait/parameterized_types.pb.h" #include "substrait/plan.pb.h" -#include "substrait/relations.pb.h" #include "substrait/type.pb.h" #include "substrait/type_expressions.pb.h" #include "utils/result_iterator.h" diff --git a/jvm/src/main/resources/substrait/proto/substrait/algebra.proto b/jvm/src/main/resources/substrait/proto/substrait/algebra.proto new file mode 100644 index 00000000000..03816e21f56 --- /dev/null +++ b/jvm/src/main/resources/substrait/proto/substrait/algebra.proto @@ -0,0 +1,702 @@ +syntax = "proto3"; + +package substrait; + +import "substrait/type.proto"; +import "substrait/extensions/extensions.proto"; +import "google/protobuf/any.proto"; + +option java_multiple_files = true; +option java_package = "io.substrait.proto"; +option csharp_namespace = "Substrait.Protobuf"; + +message RelCommon { + + oneof emit_kind { + Direct direct = 1; + Emit emit = 2; + } + + Hint hint = 3; + substrait.extensions.AdvancedExtension advanced_extension = 4; + + message Direct {} + message Emit { repeated int32 output_mapping = 1; } + + // Changes to the operation that can influence efficiency/performance but + // should not impact correctness. + message Hint { + Stats stats = 1; + RuntimeConstraint constraint = 2; + substrait.extensions.AdvancedExtension advanced_extension = 10; + + message Stats { + double row_count = 1; + double record_size = 2; + substrait.extensions.AdvancedExtension advanced_extension = 10; + } + + message RuntimeConstraint { + // TODO: nodes, cpu threads/%, memory, iops, etc. + + substrait.extensions.AdvancedExtension advanced_extension = 10; + } + } +} + +message ReadRel { + RelCommon common = 1; + NamedStruct base_schema = 2; + Expression filter = 3; + Expression.MaskExpression projection = 4; + substrait.extensions.AdvancedExtension advanced_extension = 10; + + oneof read_type { + VirtualTable virtual_table = 5; + LocalFiles local_files = 6; + NamedTable named_table = 7; + ExtensionTable extension_table = 8; + } + + message NamedTable { + repeated string names = 1; + substrait.extensions.AdvancedExtension advanced_extension = 10; + } + + // a table composed of literals. + message VirtualTable { repeated Expression.Literal.Struct values = 1; } + + // a stub type that can be used to extend/introduce new table types outside + // the specification. + message ExtensionTable { google.protobuf.Any detail = 1; } + + message LocalFiles { + + repeated FileOrFiles items = 1; + substrait.extensions.AdvancedExtension advanced_extension = 10; + + // Many files consist of indivisible chunks (e.g. parquet row groups + // or CSV rows). If a slice partially selects an indivisible chunk + // then the consumer should employ some rule to decide which slice to + // include the chunk in (e.g. include it in the slice that contains + // the midpoint of the chunk) + message FileOrFiles { + oneof path_type { + string uri_path = 1; + string uri_path_glob = 2; + string uri_file = 3; + string uri_folder = 4; + } + + FileFormat format = 5; + + // the index of the partition this item belongs to + uint64 partition_index = 6; + + // the start position in byte to read from this item + uint64 start = 7; + + // the length in byte to read from this item + uint64 length = 8; + + enum FileFormat { + FILE_FORMAT_UNSPECIFIED = 0; + FILE_FORMAT_PARQUET = 1; + } + } + } +} + +message ProjectRel { + RelCommon common = 1; + Rel input = 2; + repeated Expression expressions = 3; + substrait.extensions.AdvancedExtension advanced_extension = 10; +} + +message JoinRel { + RelCommon common = 1; + Rel left = 2; + Rel right = 3; + Expression expression = 4; + Expression post_join_filter = 5; + + JoinType type = 6; + + enum JoinType { + JOIN_TYPE_UNSPECIFIED = 0; + JOIN_TYPE_INNER = 1; + JOIN_TYPE_OUTER = 2; + JOIN_TYPE_LEFT = 3; + JOIN_TYPE_RIGHT = 4; + JOIN_TYPE_SEMI = 5; + JOIN_TYPE_ANTI = 6; + JOIN_TYPE_SINGLE = 7; + } + + substrait.extensions.AdvancedExtension advanced_extension = 10; +} + +message CrossRel { + RelCommon common = 1; + Rel left = 2; + Rel right = 3; + + substrait.extensions.AdvancedExtension advanced_extension = 10; +} + +message FetchRel { + RelCommon common = 1; + Rel input = 2; + int64 offset = 3; + int64 count = 4; + substrait.extensions.AdvancedExtension advanced_extension = 10; +} + +message AggregateRel { + RelCommon common = 1; + Rel input = 2; + repeated Grouping groupings = 3; + repeated Measure measures = 4; + + substrait.extensions.AdvancedExtension advanced_extension = 10; + + message Grouping { repeated Expression grouping_expressions = 1; } + + message Measure { + AggregateFunction measure = 1; + + // An optional boolean expression that acts to filter which records are + // included in the measure. True means include this record for calculation + // within the measure. + Expression filter = 2; + } +} + +message SortRel { + RelCommon common = 1; + Rel input = 2; + repeated SortField sorts = 3; + substrait.extensions.AdvancedExtension advanced_extension = 10; +} + +message FilterRel { + RelCommon common = 1; + Rel input = 2; + Expression condition = 3; + substrait.extensions.AdvancedExtension advanced_extension = 10; +} + +message SetRel { + RelCommon common = 1; + repeated Rel inputs = 2; + SetOp op = 3; + substrait.extensions.AdvancedExtension advanced_extension = 10; + + enum SetOp { + SET_OP_UNSPECIFIED = 0; + SET_OP_MINUS_PRIMARY = 1; + SET_OP_MINUS_MULTISET = 2; + SET_OP_INTERSECTION_PRIMARY = 3; + SET_OP_INTERSECTION_MULTISET = 4; + SET_OP_UNION_DISTINCT = 5; + SET_OP_UNION_ALL = 6; + } +} + +// Stub to support extension with a single input +message ExtensionSingleRel { + RelCommon common = 1; + Rel input = 2; + google.protobuf.Any detail = 3; +} + +// Stub to support extension with a zero inputs +message ExtensionLeafRel { + RelCommon common = 1; + google.protobuf.Any detail = 2; +} + +// Stub to support extension with multiple inputs +message ExtensionMultiRel { + RelCommon common = 1; + repeated Rel inputs = 2; + google.protobuf.Any detail = 3; +} + +// A relation with output field names. +// +// This is for use at the root of a `Rel` tree. +message RelRoot { + // A relation + Rel input = 1; + // Field names in depth-first order + repeated string names = 2; +} + +message Rel { + oneof rel_type { + ReadRel read = 1; + FilterRel filter = 2; + FetchRel fetch = 3; + AggregateRel aggregate = 4; + SortRel sort = 5; + JoinRel join = 6; + ProjectRel project = 7; + SetRel set = 8; + ExtensionSingleRel extension_single = 9; + ExtensionMultiRel extension_multi = 10; + ExtensionLeafRel extension_leaf = 11; + CrossRel cross = 12; + } +} + +message Expression { + oneof rex_type { + Literal literal = 1; + FieldReference selection = 2; + ScalarFunction scalar_function = 3; + WindowFunction window_function = 5; + IfThen if_then = 6; + SwitchExpression switch_expression = 7; + SingularOrList singular_or_list = 8; + MultiOrList multi_or_list = 9; + Enum enum = 10; + Cast cast = 11; + Subquery subquery = 12; + } + + message Enum { + oneof enum_kind { + string specified = 1; + Empty unspecified = 2; + } + + message Empty {} + } + + message Literal { + oneof literal_type { + bool boolean = 1; + int32 i8 = 2; + int32 i16 = 3; + int32 i32 = 5; + int64 i64 = 7; + float fp32 = 10; + double fp64 = 11; + string string = 12; + bytes binary = 13; + // Timestamp in units of microseconds since the UNIX epoch. + int64 timestamp = 14; + // Date in units of days since the UNIX epoch. + int32 date = 16; + // Time in units of microseconds past midnight + int64 time = 17; + IntervalYearToMonth interval_year_to_month = 19; + IntervalDayToSecond interval_day_to_second = 20; + string fixed_char = 21; + VarChar var_char = 22; + bytes fixed_binary = 23; + Decimal decimal = 24; + Struct struct = 25; + Map map = 26; + // Timestamp in units of microseconds since the UNIX epoch. + int64 timestamp_tz = 27; + bytes uuid = 28; + Type null = 29; // a typed null literal + List list = 30; + Type.List empty_list = 31; + Type.Map empty_map = 32; + } + + // whether the literal type should be treated as a nullable type. Applies to + // all members of union other than the Typed null (which should directly + // declare nullability). + bool nullable = 50; + + message VarChar { + string value = 1; + uint32 length = 2; + } + + message Decimal { + // little-endian twos-complement integer representation of complete value + // (ignoring precision) Always 16 bytes in length + bytes value = 1; + // The maximum number of digits allowed in the value. + // the maximum precision is 38. + int32 precision = 2; + // declared scale of decimal literal + int32 scale = 3; + } + + message Map { + message KeyValue { + Literal key = 1; + Literal value = 2; + } + + repeated KeyValue key_values = 1; + } + + message IntervalYearToMonth { + int32 years = 1; + int32 months = 2; + } + + message IntervalDayToSecond { + int32 days = 1; + int32 seconds = 2; + } + + message Struct { + // A possibly heterogeneously typed list of literals + repeated Literal fields = 1; + } + + message List { + // A homogeneously typed list of literals + repeated Literal values = 1; + } + } + + message ScalarFunction { + // points to a function_anchor defined in this plan + uint32 function_reference = 1; + repeated Expression args = 2; + Type output_type = 3; + } + + message WindowFunction { + // points to a function_anchor defined in this plan + uint32 function_reference = 1; + repeated Expression partitions = 2; + repeated SortField sorts = 3; + Bound upper_bound = 4; + Bound lower_bound = 5; + AggregationPhase phase = 6; + Type output_type = 7; + repeated Expression args = 8; + + message Bound { + + message Preceding { int64 offset = 1; } + + message Following { int64 offset = 1; } + + message CurrentRow {} + + message Unbounded {} + + oneof kind { + Preceding preceding = 1; + Following following = 2; + CurrentRow current_row = 3; + Unbounded unbounded = 4; + } + } + } + + message IfThen { + + repeated IfClause ifs = 1; + Expression else = 2; + + message IfClause { + Expression if = 1; + Expression then = 2; + } + } + + message Cast { + Type type = 1; + Expression input = 2; + } + + message SwitchExpression { + repeated IfValue ifs = 1; + Expression else = 2; + + message IfValue { + Literal if = 1; + Expression then = 2; + } + } + + message SingularOrList { + Expression value = 1; + repeated Expression options = 2; + } + + message MultiOrList { + repeated Expression value = 1; + repeated Record options = 2; + + message Record { repeated Expression fields = 1; } + } + + message EmbeddedFunction { + repeated Expression arguments = 1; + Type output_type = 2; + oneof kind { + PythonPickleFunction python_pickle_function = 3; + WebAssemblyFunction web_assembly_function = 4; + } + + message PythonPickleFunction { + bytes function = 1; + repeated string prerequisite = 2; + } + + message WebAssemblyFunction { + bytes script = 1; + repeated string prerequisite = 2; + } + } + + // A way to reference the inner property of a complex record. Can reference + // either a map key by literal, a struct field by the ordinal position of + // the desired field or a particular element in an array. Supports + // expressions that would roughly translate to something similar to: + // a.b[2].c['my_map_key'].x where a,b,c and x are struct field references + // (ordinalized in the internal representation here), [2] is a list offset + // and ['my_map_key'] is a reference into a map field. + message ReferenceSegment { + + oneof reference_type { + MapKey map_key = 1; + StructField struct_field = 2; + ListElement list_element = 3; + } + + message MapKey { + // literal based reference to specific possible value in map. + Literal map_key = 1; + + // Optional child segment + ReferenceSegment child = 2; + } + + message StructField { + // zero-indexed ordinal position of field in struct + int32 field = 1; + + // Optional child segment + ReferenceSegment child = 2; + } + + message ListElement { + // zero-indexed ordinal position of element in list + int32 offset = 1; + + // Optional child segment + ReferenceSegment child = 2; + } + } + + // A reference that takes an existing subtype and selectively removes fields + // from it. For example, one might initially have an inner struct with 100 + // fields but a a particular operation only needs to interact with only 2 of + // those 100 fields. In this situation, one would use a mask expression to + // eliminate the 98 fields that are not relevant to the rest of the operation + // pipeline. + // + // Note that this does not fundamentally alter the structure of data beyond + // the elimination of unecessary elements. + message MaskExpression { + + StructSelect select = 1; + bool maintain_singular_struct = 2; + + message Select { + oneof type { + StructSelect struct = 1; + ListSelect list = 2; + MapSelect map = 3; + } + } + + message StructSelect { repeated StructItem struct_items = 1; } + + message StructItem { + int32 field = 1; + Select child = 2; + } + + message ListSelect { + + repeated ListSelectItem selection = 1; + Select child = 2; + + message ListSelectItem { + oneof type { + ListElement item = 1; + ListSlice slice = 2; + } + + message ListElement { int32 field = 1; } + + message ListSlice { + int32 start = 1; + int32 end = 2; + } + } + } + + message MapSelect { + oneof select { + MapKey key = 1; + MapKeyExpression expression = 2; + } + + Select child = 3; + + message MapKey { string map_key = 1; } + + message MapKeyExpression { string map_key_expression = 1; } + } + } + + // A reference to an inner part of a complex object. Can reference reference a + // single element or a masked version of elements + message FieldReference { + + // Whether this is composed of a single element reference or a masked + // element subtree + oneof reference_type { + ReferenceSegment direct_reference = 1; + MaskExpression masked_reference = 2; + } + + // Whether this reference has an origin of a root struct or is based on the + // ouput of an expression. When this is a RootReference and direct_reference + // above is used, the direct_reference must be of a type StructField. + oneof root_type { + Expression expression = 3; + RootReference root_reference = 4; + OuterReference outer_reference = 5; + } + + // Singleton that expresses this FieldReference is rooted off the root + // incoming record type + message RootReference {} + + // A root reference for the outer relation's subquery + message OuterReference { + // number of subquery boundaries to traverse up for this field's reference + // + // This value must be >= 1 + uint32 steps_out = 1; + } + } + + // Subquery relation expression + message Subquery { + oneof subquery_type { + // Scalar subquery + Scalar scalar = 1; + // x IN y predicate + InPredicate in_predicate = 2; + // EXISTS/UNIQUE predicate + SetPredicate set_predicate = 3; + // ANY/ALL predicate + SetComparison set_comparison = 4; + } + + // A subquery with one row and one column. This is often an aggregate + // though not required to be. + message Scalar { Rel input = 1; } + + // Predicate checking that the left expression is contained in the right + // subquery + // + // Examples: + // + // x IN (SELECT * FROM t) + // (x, y) IN (SELECT a, b FROM t) + message InPredicate { + repeated Expression needles = 1; + Rel haystack = 2; + } + + // A predicate over a set of rows in the form of a subquery + // EXISTS and UNIQUE are common SQL forms of this operation. + message SetPredicate { + enum PredicateOp { + PREDICATE_OP_UNSPECIFIED = 0; + PREDICATE_OP_EXISTS = 1; + PREDICATE_OP_UNIQUE = 2; + } + // TODO: should allow expressions + PredicateOp predicate_op = 1; + Rel tuples = 2; + } + + // A subquery comparison using ANY or ALL. + // Examples: + // + // SELECT * + // FROM t1 + // WHERE x < ANY(SELECT y from t2) + message SetComparison { + enum ComparisonOp { + COMPARISON_OP_UNSPECIFIED = 0; + COMPARISON_OP_EQ = 1; + COMPARISON_OP_NE = 2; + COMPARISON_OP_LT = 3; + COMPARISON_OP_GT = 4; + COMPARISON_OP_LE = 5; + COMPARISON_OP_GE = 6; + } + + enum ReductionOp { + REDUCTION_OP_UNSPECIFIED = 0; + REDUCTION_OP_ANY = 1; + REDUCTION_OP_ALL = 2; + } + + // ANY or ALL + ReductionOp reduction_op = 1; + // A comparison operator + ComparisonOp comparison_op = 2; + // left side of the expression + Expression left = 3; + // right side of the expression + Rel right = 4; + } + } +} + +message SortField { + Expression expr = 1; + + oneof sort_kind { + SortDirection direction = 2; + uint32 comparison_function_reference = 3; + } + enum SortDirection { + SORT_DIRECTION_UNSPECIFIED = 0; + SORT_DIRECTION_ASC_NULLS_FIRST = 1; + SORT_DIRECTION_ASC_NULLS_LAST = 2; + SORT_DIRECTION_DESC_NULLS_FIRST = 3; + SORT_DIRECTION_DESC_NULLS_LAST = 4; + SORT_DIRECTION_CLUSTERED = 5; + } +} + +enum AggregationPhase { + AGGREGATION_PHASE_UNSPECIFIED = 0; + AGGREGATION_PHASE_INITIAL_TO_INTERMEDIATE = 1; + AGGREGATION_PHASE_INTERMEDIATE_TO_INTERMEDIATE = 2; + AGGREGATION_PHASE_INITIAL_TO_RESULT = 3; + AGGREGATION_PHASE_INTERMEDIATE_TO_RESULT = 4; +} + +message AggregateFunction { + // points to a function_anchor defined in this plan + uint32 function_reference = 1; + repeated Expression args = 2; + repeated SortField sorts = 3; + AggregationPhase phase = 4; + Type output_type = 5; +} diff --git a/jvm/src/main/resources/substrait/proto/substrait/expression.proto b/jvm/src/main/resources/substrait/proto/substrait/expression.proto deleted file mode 100644 index 02693a3addb..00000000000 --- a/jvm/src/main/resources/substrait/proto/substrait/expression.proto +++ /dev/null @@ -1,372 +0,0 @@ -syntax = "proto3"; - -package substrait; - -import "substrait/type.proto"; - -option java_multiple_files = true; -option java_package = "io.substrait.proto"; -option csharp_namespace = "Substrait.Protobuf"; - -message Expression { - oneof rex_type { - Literal literal = 1; - FieldReference selection = 2; - ScalarFunction scalar_function = 3; - WindowFunction window_function = 5; - IfThen if_then = 6; - SwitchExpression switch_expression = 7; - SingularOrList singular_or_list = 8; - MultiOrList multi_or_list = 9; - Enum enum = 10; - Cast cast = 11; - } - - message Enum { - oneof enum_kind { - string specified = 1; - Empty unspecified = 2; - } - - message Empty {} - } - - message Literal { - oneof literal_type { - bool boolean = 1; - int32 i8 = 2; - int32 i16 = 3; - int32 i32 = 5; - int64 i64 = 7; - float fp32 = 10; - double fp64 = 11; - string string = 12; - bytes binary = 13; - // Timestamp in units of microseconds since the UNIX epoch. - int64 timestamp = 14; - // Date in units of days since the UNIX epoch. - int32 date = 16; - // Time in units of microseconds past midnight - int64 time = 17; - IntervalYearToMonth interval_year_to_month = 19; - IntervalDayToSecond interval_day_to_second = 20; - string fixed_char = 21; - VarChar var_char = 22; - bytes fixed_binary = 23; - Decimal decimal = 24; - Struct struct = 25; - Map map = 26; - // Timestamp in units of microseconds since the UNIX epoch. - int64 timestamp_tz = 27; - bytes uuid = 28; - Type null = 29; // a typed null literal - List list = 30; - Type.List empty_list = 31; - Type.Map empty_map = 32; - } - - // whether the literal type should be treated as a nullable type. Applies to - // all members of union other than the Typed null (which should directly - // declare nullability). - bool nullable = 50; - - message VarChar { - string value = 1; - uint32 length = 2; - } - - message Decimal { - // little-endian twos-complement integer representation of complete value - // (ignoring precision) Always 16 bytes in length - bytes value = 1; - // The maximum number of digits allowed in the value. - // the maximum precision is 38. - int32 precision = 2; - // declared scale of decimal literal - int32 scale = 3; - } - - message Map { - message KeyValue { - Literal key = 1; - Literal value = 2; - } - - repeated KeyValue key_values = 1; - } - - message IntervalYearToMonth { - int32 years = 1; - int32 months = 2; - } - - message IntervalDayToSecond { - int32 days = 1; - int32 seconds = 2; - } - - message Struct { - // A possibly heterogeneously typed list of literals - repeated Literal fields = 1; - } - - message List { - // A homogeneously typed list of literals - repeated Literal values = 1; - } - } - - message ScalarFunction { - // points to a function_anchor defined in this plan - uint32 function_reference = 1; - repeated Expression args = 2; - Type output_type = 3; - } - - message WindowFunction { - // points to a function_anchor defined in this plan - uint32 function_reference = 1; - repeated Expression partitions = 2; - repeated SortField sorts = 3; - Bound upper_bound = 4; - Bound lower_bound = 5; - AggregationPhase phase = 6; - Type output_type = 7; - repeated Expression args = 8; - - message Bound { - - message Preceding { int64 offset = 1; } - - message Following { int64 offset = 1; } - - message CurrentRow {} - - message Unbounded {} - - oneof kind { - Preceding preceding = 1; - Following following = 2; - CurrentRow current_row = 3; - Unbounded unbounded = 4; - } - } - } - - message IfThen { - - repeated IfClause ifs = 1; - Expression else = 2; - - message IfClause { - Expression if = 1; - Expression then = 2; - } - } - - message Cast { - Type type = 1; - Expression input = 2; - } - - message SwitchExpression { - repeated IfValue ifs = 1; - Expression else = 2; - - message IfValue { - Literal if = 1; - Expression then = 2; - } - } - - message SingularOrList { - Expression value = 1; - repeated Expression options = 2; - } - - message MultiOrList { - repeated Expression value = 1; - repeated Record options = 2; - - message Record { repeated Expression fields = 1; } - } - - message EmbeddedFunction { - repeated Expression arguments = 1; - Type output_type = 2; - oneof kind { - PythonPickleFunction python_pickle_function = 3; - WebAssemblyFunction web_assembly_function = 4; - } - - message PythonPickleFunction { - bytes function = 1; - repeated string prerequisite = 2; - } - - message WebAssemblyFunction { - bytes script = 1; - repeated string prerequisite = 2; - } - } - - // A way to reference the inner property of a complex record. Can reference - // either a map key by literal, a struct field by the ordinal position of - // the desired field or a particular element in an array. Supports - // expressions that would roughly translate to something similar to: - // a.b[2].c['my_map_key'].x where a,b,c and x are struct field references - // (ordinalized in the internal representation here), [2] is a list offset - // and ['my_map_key'] is a reference into a map field. - message ReferenceSegment { - - oneof reference_type { - MapKey map_key = 1; - StructField struct_field = 2; - ListElement list_element = 3; - } - - message MapKey { - // literal based reference to specific possible value in map. - Literal map_key = 1; - - // Optional child segment - ReferenceSegment child = 2; - } - - message StructField { - // zero-indexed ordinal position of field in struct - int32 field = 1; - - // Optional child segment - ReferenceSegment child = 2; - } - - message ListElement { - // zero-indexed ordinal position of element in list - int32 offset = 1; - - // Optional child segment - ReferenceSegment child = 2; - } - - } - - // A reference that takes an existing subtype and selectively removes fields from - // it. For example, one might initially have an inner struct with 100 fields but a - // a particular operation only needs to interact with only 2 of those 100 fields. - // In this situation, one would use a mask expression to eliminate the 98 fields that - // are not relevant to the rest of the operation pipeline. - // - // Note that this does not fundamentally alter the structure of data beyond the - // elimination of unecessary elements. - message MaskExpression { - - StructSelect select = 1; - bool maintain_singular_struct = 2; - - message Select { - oneof type { - StructSelect struct = 1; - ListSelect list = 2; - MapSelect map = 3; - } - } - - message StructSelect { repeated StructItem struct_items = 1; } - - message StructItem { - int32 field = 1; - Select child = 2; - } - - message ListSelect { - - repeated ListSelectItem selection = 1; - Select child = 2; - - message ListSelectItem { - oneof type { - ListElement item = 1; - ListSlice slice = 2; - } - - message ListElement { int32 field = 1; } - - message ListSlice { - int32 start = 1; - int32 end = 2; - } - } - } - - message MapSelect { - oneof select { - MapKey key = 1; - MapKeyExpression expression = 2; - } - - Select child = 3; - - message MapKey { string map_key = 1; } - - message MapKeyExpression { string map_key_expression = 1; } - } - } - - // A reference to an inner part of a complex object. Can reference reference a single - // element or a masked version of elements - message FieldReference { - - // Whether this is composed of a single element reference or a masked element subtree - oneof reference_type { - ReferenceSegment direct_reference = 1; - MaskExpression masked_reference = 2; - } - - // Whether this reference has an origin of a root struct or is based on the ouput - // of an expression. When this is a RootReference and direct_reference above is used, - // the direct_reference must be of a type StructField. - oneof root_type { - Expression expression = 3; - RootReference root_reference = 4; - } - - // Singleton that expresses this FieldReference is rooted off the root incoming record type - message RootReference {} - - } -} - -message SortField { - Expression expr = 1; - - oneof sort_kind { - SortDirection direction = 2; - uint32 comparison_function_reference = 3; - } - enum SortDirection { - SORT_DIRECTION_UNSPECIFIED = 0; - SORT_DIRECTION_ASC_NULLS_FIRST = 1; - SORT_DIRECTION_ASC_NULLS_LAST = 2; - SORT_DIRECTION_DESC_NULLS_FIRST = 3; - SORT_DIRECTION_DESC_NULLS_LAST = 4; - SORT_DIRECTION_CLUSTERED = 5; - } -} - -enum AggregationPhase { - AGGREGATION_PHASE_UNSPECIFIED = 0; - AGGREGATION_PHASE_INITIAL_TO_INTERMEDIATE = 1; - AGGREGATION_PHASE_INTERMEDIATE_TO_INTERMEDIATE = 2; - AGGREGATION_PHASE_INITIAL_TO_RESULT = 3; - AGGREGATION_PHASE_INTERMEDIATE_TO_RESULT = 4; -} - -message AggregateFunction { - // points to a function_anchor defined in this plan - uint32 function_reference = 1; - repeated Expression args = 2; - repeated SortField sorts = 3; - AggregationPhase phase = 4; - Type output_type = 5; -} diff --git a/jvm/src/main/resources/substrait/proto/substrait/plan.proto b/jvm/src/main/resources/substrait/proto/substrait/plan.proto index 95978bdf972..050dc65e35e 100644 --- a/jvm/src/main/resources/substrait/proto/substrait/plan.proto +++ b/jvm/src/main/resources/substrait/proto/substrait/plan.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package substrait; -import "substrait/relations.proto"; +import "substrait/algebra.proto"; import "substrait/extensions/extensions.proto"; option java_multiple_files = true; diff --git a/jvm/src/main/resources/substrait/proto/substrait/relations.proto b/jvm/src/main/resources/substrait/proto/substrait/relations.proto deleted file mode 100644 index d0395e42d91..00000000000 --- a/jvm/src/main/resources/substrait/proto/substrait/relations.proto +++ /dev/null @@ -1,238 +0,0 @@ -syntax = "proto3"; - -package substrait; - -import "substrait/type.proto"; -import "substrait/expression.proto"; -import "substrait/extensions/extensions.proto"; -import "google/protobuf/any.proto"; - -option java_multiple_files = true; -option java_package = "io.substrait.proto"; -option csharp_namespace = "Substrait.Protobuf"; - -message RelCommon { - - oneof emit_kind { - Direct direct = 1; - Emit emit = 2; - } - - Hint hint = 3; - substrait.extensions.AdvancedExtension advanced_extension = 4; - - message Direct {} - message Emit { repeated int32 output_mapping = 1; } - - // Changes to the operation that can influence efficiency/performance but - // should not impact correctness. - message Hint { - Stats stats = 1; - RuntimeConstraint constraint = 2; - substrait.extensions.AdvancedExtension advanced_extension = 10; - - message Stats { - double row_count = 1; - double record_size = 2; - substrait.extensions.AdvancedExtension advanced_extension = 10; - } - - message RuntimeConstraint { - // TODO: nodes, cpu threads/%, memory, iops, etc. - - substrait.extensions.AdvancedExtension advanced_extension = 10; - } - } -} - -message ReadRel { - RelCommon common = 1; - NamedStruct base_schema = 2; - Expression filter = 3; - Expression.MaskExpression projection = 4; - substrait.extensions.AdvancedExtension advanced_extension = 10; - - oneof read_type { - VirtualTable virtual_table = 5; - LocalFiles local_files = 6; - NamedTable named_table = 7; - ExtensionTable extension_table = 8; - } - - message NamedTable { - repeated string names = 1; - substrait.extensions.AdvancedExtension advanced_extension = 10; - } - - // a table composed of literals. - message VirtualTable { repeated Expression.Literal.Struct values = 1; } - - // a stub type that can be used to extend/introduce new table types outside - // the specification. - message ExtensionTable { google.protobuf.Any detail = 1; } - - message LocalFiles { - - repeated FileOrFiles items = 1; - substrait.extensions.AdvancedExtension advanced_extension = 10; - - message FileOrFiles { - oneof path_type { - string uri_path = 1; - string uri_path_glob = 2; - string uri_file = 3; - string uri_folder = 4; - } - - FileFormat format = 5; - - // the index of the partition this item belongs to - uint64 partition_index = 6; - - // the start position in byte to read from this item - uint64 start = 7; - - // the length in byte to read from this item - uint64 length = 8; - - enum FileFormat { - FILE_FORMAT_UNSPECIFIED = 0; - FILE_FORMAT_PARQUET = 1; - } - } - } -} - -message ProjectRel { - RelCommon common = 1; - Rel input = 2; - repeated Expression expressions = 3; - substrait.extensions.AdvancedExtension advanced_extension = 10; -} - -message JoinRel { - RelCommon common = 1; - Rel left = 2; - Rel right = 3; - Expression expression = 4; - Expression post_join_filter = 5; - - JoinType type = 6; - - enum JoinType { - JOIN_TYPE_UNSPECIFIED = 0; - JOIN_TYPE_INNER = 1; - JOIN_TYPE_OUTER = 2; - JOIN_TYPE_LEFT = 3; - JOIN_TYPE_RIGHT = 4; - JOIN_TYPE_SEMI = 5; - JOIN_TYPE_ANTI = 6; - } - - substrait.extensions.AdvancedExtension advanced_extension = 10; -} - -message FetchRel { - RelCommon common = 1; - Rel input = 2; - int64 offset = 3; - int64 count = 4; - substrait.extensions.AdvancedExtension advanced_extension = 10; -} - -message AggregateRel { - RelCommon common = 1; - Rel input = 2; - repeated Grouping groupings = 3; - repeated Measure measures = 4; - - substrait.extensions.AdvancedExtension advanced_extension = 10; - - message Grouping { repeated Expression grouping_expressions = 1; } - - message Measure { - AggregateFunction measure = 1; - - // An optional boolean expression that acts to filter which records are - // included in the measure. True means include this record for calculation - // within the measure. - Expression filter = 2; - } -} - -message SortRel { - RelCommon common = 1; - Rel input = 2; - repeated SortField sorts = 3; - substrait.extensions.AdvancedExtension advanced_extension = 10; -} - -message FilterRel { - RelCommon common = 1; - Rel input = 2; - Expression condition = 3; - substrait.extensions.AdvancedExtension advanced_extension = 10; -} - -message SetRel { - RelCommon common = 1; - repeated Rel inputs = 2; - SetOp op = 3; - substrait.extensions.AdvancedExtension advanced_extension = 10; - - enum SetOp { - SET_OP_UNSPECIFIED = 0; - SET_OP_MINUS_PRIMARY = 1; - SET_OP_MINUS_MULTISET = 2; - SET_OP_INTERSECTION_PRIMARY = 3; - SET_OP_INTERSECTION_MULTISET = 4; - SET_OP_UNION_DISTINCT = 5; - SET_OP_UNION_ALL = 6; - } -} - -// Stub to support extension with a single input -message ExtensionSingleRel { - RelCommon common = 1; - Rel input = 2; - google.protobuf.Any detail = 3; -} - -// Stub to support extension with a zero inputs -message ExtensionLeafRel { - RelCommon common = 1; - google.protobuf.Any detail = 2; -} - -// Stub to support extension with multiple inputs -message ExtensionMultiRel { - RelCommon common = 1; - repeated Rel inputs = 2; - google.protobuf.Any detail = 3; -} - -// A relation with output field names. -// -// This is for use at the root of a `Rel` tree. -message RelRoot { - // A relation - Rel input = 1; - // Field names in depth-first order - repeated string names = 2; -} - -message Rel { - oneof rel_type { - ReadRel read = 1; - FilterRel filter = 2; - FetchRel fetch = 3; - AggregateRel aggregate = 4; - SortRel sort = 5; - JoinRel join = 6; - ProjectRel project = 7; - SetRel set = 8; - ExtensionSingleRel extension_single = 9; - ExtensionMultiRel extension_multi = 10; - ExtensionLeafRel extension_leaf = 11; - } -}