Skip to content

Commit 7cb2537

Browse files
Enmkianton-ru
Enmk
authored and
ianton-ru
committed
Merge pull request #1103 from Altinity/feature/antalya-25.8/timezone_for_iceberg_timestamptz
Timezone for iceberg timestamptz
1 parent e3029b7 commit 7cb2537

22 files changed

+221
-70
lines changed

‎src/Core/Settings.cpp‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7315,6 +7315,16 @@ Allows creation of [QBit](../../sql-reference/data-types/qbit.md) data type.
73157315
)", BETA, allow_experimental_qbit_type) \
73167316
DECLARE(UInt64, archive_adaptive_buffer_max_size_bytes, 8 * DBMS_DEFAULT_BUFFER_SIZE, R"(
73177317
Limits the maximum size of the adaptive buffer used when writing to archive files (for example, tar archives)", 0) \
7318+
DECLARE(Timezone, iceberg_timezone_for_timestamptz, "UTC", R"(
7319+
Timezone for Iceberg timestamptz field.
7320+
7321+
Possible values:
7322+
7323+
- Any valid timezone, e.g. `Europe/Berlin`, `UTC` or `Zulu`
7324+
- `` (empty value) - use session timezone
7325+
7326+
Default value is `UTC`.
7327+
)", 0) \
73187328
\
73197329
/* ####################################################### */ \
73207330
/* ########### START OF EXPERIMENTAL FEATURES ############ */ \

‎src/Core/SettingsChangesHistory.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -232,6 +232,8 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory()
232232
{"object_storage_max_nodes", 0, 0, "New setting"},
233233
{"allow_retries_in_cluster_requests", false, false, "New setting"},
234234
{"object_storage_remote_initiator", false, false, "New setting."},
235+
{"allow_experimental_export_merge_tree_part", false, true, "Turned ON by default for Antalya."},
236+
{"iceberg_timezone_for_timestamptz", "UTC", "UTC", "New setting."}
235237
});
236238
addSettingsChanges(settings_changes_history, "25.8",
237239
{

‎src/Databases/DataLake/Common.cpp‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -61,14 +61,14 @@ std::vector<String> splitTypeArguments(const String & type_str)
6161
return args;
6262
}
6363

64-
DB::DataTypePtr getType(const String & type_name, bool nullable, const String & prefix)
64+
DB::DataTypePtr getType(const String & type_name, bool nullable, DB::ContextPtr context, const String & prefix)
6565
{
6666
String name = trim(type_name);
6767

6868
if (name.starts_with("array<") && name.ends_with(">"))
6969
{
7070
String inner = name.substr(6, name.size() - 7);
71-
return std::make_shared<DB::DataTypeArray>(getType(inner, nullable));
71+
return std::make_shared<DB::DataTypeArray>(getType(inner, nullable, context));
7272
}
7373

7474
if (name.starts_with("map<") && name.ends_with(">"))
@@ -79,7 +79,7 @@ DB::DataTypePtr getType(const String & type_name, bool nullable, const String &
7979
if (args.size() != 2)
8080
throw DB::Exception(DB::ErrorCodes::DATALAKE_DATABASE_ERROR, "Invalid data type {}", type_name);
8181

82-
return std::make_shared<DB::DataTypeMap>(getType(args[0], false), getType(args[1], nullable));
82+
return std::make_shared<DB::DataTypeMap>(getType(args[0], false, context), getType(args[1], nullable, context));
8383
}
8484

8585
if (name.starts_with("struct<") && name.ends_with(">"))
@@ -101,13 +101,13 @@ DB::DataTypePtr getType(const String & type_name, bool nullable, const String &
101101
String full_field_name = prefix.empty() ? field_name : prefix + "." + field_name;
102102

103103
field_names.push_back(full_field_name);
104-
field_types.push_back(getType(field_type, nullable, full_field_name));
104+
field_types.push_back(getType(field_type, nullable, context, full_field_name));
105105
}
106106
return std::make_shared<DB::DataTypeTuple>(field_types, field_names);
107107
}
108108

109-
return nullable ? DB::makeNullable(DB::Iceberg::IcebergSchemaProcessor::getSimpleType(name))
110-
: DB::Iceberg::IcebergSchemaProcessor::getSimpleType(name);
109+
return nullable ? DB::makeNullable(DB::Iceberg::IcebergSchemaProcessor::getSimpleType(name, context))
110+
: DB::Iceberg::IcebergSchemaProcessor::getSimpleType(name, context);
111111
}
112112

113113
std::pair<std::string, std::string> parseTableName(const std::string & name)

‎src/Databases/DataLake/Common.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
#include <Core/NamesAndTypes.h>
44
#include <Core/Types.h>
5+
#include <Interpreters/Context_fwd.h>
56

67
namespace DataLake
78
{
@@ -10,7 +11,7 @@ String trim(const String & str);
1011

1112
std::vector<String> splitTypeArguments(const String & type_str);
1213

13-
DB::DataTypePtr getType(const String & type_name, bool nullable, const String & prefix = "");
14+
DB::DataTypePtr getType(const String & type_name, bool nullable, DB::ContextPtr context, const String & prefix = "");
1415

1516
/// Parse a string, containing at least one dot, into a two substrings:
1617
/// A.B.C.D.E -> A.B.C.D and E, where

‎src/Databases/DataLake/DatabaseDataLake.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -450,7 +450,7 @@ StoragePtr DatabaseDataLake::tryGetTableImpl(const String & name, ContextPtr con
450450

451451
auto [namespace_name, table_name] = DataLake::parseTableName(name);
452452

453-
if (!catalog->tryGetTableMetadata(namespace_name, table_name, table_metadata))
453+
if (!catalog->tryGetTableMetadata(namespace_name, table_name, context_, table_metadata))
454454
return nullptr;
455455
if (ignore_if_not_iceberg && !table_metadata.isDefaultReadableTable())
456456
return nullptr;
@@ -807,15 +807,15 @@ ASTPtr DatabaseDataLake::getCreateDatabaseQueryImpl() const
807807

808808
ASTPtr DatabaseDataLake::getCreateTableQueryImpl(
809809
const String & name,
810-
ContextPtr /* context_ */,
810+
ContextPtr context_,
811811
bool throw_on_error) const
812812
{
813813
auto catalog = getCatalog();
814814
auto table_metadata = DataLake::TableMetadata().withLocation().withSchema();
815815

816816
const auto [namespace_name, table_name] = DataLake::parseTableName(name);
817817

818-
if (!catalog->tryGetTableMetadata(namespace_name, table_name, table_metadata))
818+
if (!catalog->tryGetTableMetadata(namespace_name, table_name, context_, table_metadata))
819819
{
820820
if (throw_on_error)
821821
throw Exception(ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY, "Table `{}` doesn't exist", name);

‎src/Databases/DataLake/GlueCatalog.cpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -292,6 +292,7 @@ bool GlueCatalog::existsTable(const std::string & database_name, const std::stri
292292
bool GlueCatalog::tryGetTableMetadata(
293293
const std::string & database_name,
294294
const std::string & table_name,
295+
DB::ContextPtr /* context_ */,
295296
TableMetadata & result) const
296297
{
297298
Aws::Glue::Model::GetTableRequest request;
@@ -386,7 +387,7 @@ bool GlueCatalog::tryGetTableMetadata(
386387
column_type = "timestamptz";
387388
}
388389

389-
schema.push_back({column.GetName(), getType(column_type, can_be_nullable)});
390+
schema.push_back({column.GetName(), getType(column_type, can_be_nullable, getContext())});
390391
}
391392
result.setSchema(schema);
392393
}
@@ -408,9 +409,10 @@ bool GlueCatalog::tryGetTableMetadata(
408409
void GlueCatalog::getTableMetadata(
409410
const std::string & database_name,
410411
const std::string & table_name,
412+
DB::ContextPtr context_,
411413
TableMetadata & result) const
412414
{
413-
if (!tryGetTableMetadata(database_name, table_name, result))
415+
if (!tryGetTableMetadata(database_name, table_name, context_, result))
414416
{
415417
throw DB::Exception(
416418
DB::ErrorCodes::DATALAKE_DATABASE_ERROR,

‎src/Databases/DataLake/GlueCatalog.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,11 +41,13 @@ class GlueCatalog final : public ICatalog, private DB::WithContext
4141
void getTableMetadata(
4242
const std::string & database_name,
4343
const std::string & table_name,
44+
DB::ContextPtr context_,
4445
TableMetadata & result) const override;
4546

4647
bool tryGetTableMetadata(
4748
const std::string & database_name,
4849
const std::string & table_name,
50+
DB::ContextPtr context_,
4951
TableMetadata & result) const override;
5052

5153
std::optional<StorageType> getStorageType() const override

‎src/Databases/DataLake/HiveCatalog.cpp‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -121,13 +121,21 @@ bool HiveCatalog::existsTable(const std::string & namespace_name, const std::str
121121
return true;
122122
}
123123

124-
void HiveCatalog::getTableMetadata(const std::string & namespace_name, const std::string & table_name, TableMetadata & result) const
124+
void HiveCatalog::getTableMetadata(
125+
const std::string & namespace_name,
126+
const std::string & table_name,
127+
DB::ContextPtr context_,
128+
TableMetadata & result) const
125129
{
126-
if (!tryGetTableMetadata(namespace_name, table_name, result))
130+
if (!tryGetTableMetadata(namespace_name, table_name, context_, result))
127131
throw DB::Exception(DB::ErrorCodes::DATALAKE_DATABASE_ERROR, "No response from iceberg catalog");
128132
}
129133

130-
bool HiveCatalog::tryGetTableMetadata(const std::string & namespace_name, const std::string & table_name, TableMetadata & result) const
134+
bool HiveCatalog::tryGetTableMetadata(
135+
const std::string & namespace_name,
136+
const std::string & table_name,
137+
DB::ContextPtr context_,
138+
TableMetadata & result) const
131139
{
132140
Apache::Hadoop::Hive::Table table;
133141

@@ -155,7 +163,7 @@ bool HiveCatalog::tryGetTableMetadata(const std::string & namespace_name, const
155163
auto columns = table.sd.cols;
156164
for (const auto & column : columns)
157165
{
158-
schema.push_back({column.name, getType(column.type, true)});
166+
schema.push_back({column.name, getType(column.type, true, context_)});
159167
}
160168
result.setSchema(schema);
161169
}

‎src/Databases/DataLake/HiveCatalog.h‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,17 @@ class HiveCatalog final : public ICatalog, private DB::WithContext
3838

3939
bool existsTable(const std::string & namespace_name, const std::string & table_name) const override;
4040

41-
void getTableMetadata(const std::string & namespace_name, const std::string & table_name, TableMetadata & result) const override;
42-
43-
bool tryGetTableMetadata(const std::string & namespace_name, const std::string & table_name, TableMetadata & result) const override;
41+
void getTableMetadata(
42+
const std::string & namespace_name,
43+
const std::string & table_name,
44+
DB::ContextPtr context_,
45+
TableMetadata & result) const override;
46+
47+
bool tryGetTableMetadata(
48+
const std::string & namespace_name,
49+
const std::string & table_name,
50+
DB::ContextPtr context_,
51+
TableMetadata & result) const override;
4452

4553
std::optional<StorageType> getStorageType() const override;
4654

‎src/Databases/DataLake/ICatalog.h‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,14 @@
88
#include <Databases/DataLake/DatabaseDataLakeStorageType.h>
99
#include <Poco/JSON/Object.h>
1010

11+
namespace DB
12+
{
13+
14+
class Context;
15+
using ContextPtr = std::shared_ptr<const Context>;
16+
17+
}
18+
1119
namespace DataLake
1220
{
1321

@@ -153,13 +161,15 @@ class ICatalog
153161
virtual void getTableMetadata(
154162
const std::string & namespace_name,
155163
const std::string & table_name,
164+
DB::ContextPtr context,
156165
TableMetadata & result) const = 0;
157166

158167
/// Get table metadata in the given namespace.
159168
/// Return `false` if table does not exist, `true` otherwise.
160169
virtual bool tryGetTableMetadata(
161170
const std::string & namespace_name,
162171
const std::string & table_name,
172+
DB::ContextPtr context,
163173
TableMetadata & result) const = 0;
164174

165175
/// Get storage type, where Iceberg tables' data is stored.

0 commit comments

Comments
 (0)