Skip to content

Commit 95921d5

Browse files
leonardBangleonardBang
authored
[cdc-common] Introduce "pipeline.local-time-zone" config option which help handle time zone well (#2797)
This closes #2797.
1 parent e435004 commit 95921d5

12 files changed

Lines changed: 188 additions & 44 deletions

File tree

‎flink-cdc-cli/src/test/java/com/ververica/cdc/cli/parser/YamlPipelineDefinitionParserTest.java‎

Lines changed: 74 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,9 @@
3131
import java.util.Arrays;
3232
import java.util.Collections;
3333

34+
import static com.ververica.cdc.common.pipeline.PipelineOptions.PIPELINE_LOCAL_TIME_ZONE;
3435
import static org.assertj.core.api.Assertions.assertThat;
36+
import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy;
3537

3638
/** Unit test for {@link YamlPipelineDefinitionParser}. */
3739
class YamlPipelineDefinitionParserTest {
@@ -64,11 +66,81 @@ void testMinimizedDefinition() throws Exception {
6466
void testOverridingGlobalConfig() throws Exception {
6567
URL resource = Resources.getResource("definitions/pipeline-definition-full.yaml");
6668
YamlPipelineDefinitionParser parser = new YamlPipelineDefinitionParser();
67-
ImmutableMap.<String, String>builder().put("parallelism", "1").put("foo", "bar");
68-
PipelineDef pipelineDef = parser.parse(Paths.get(resource.toURI()), new Configuration());
69+
PipelineDef pipelineDef =
70+
parser.parse(
71+
Paths.get(resource.toURI()),
72+
Configuration.fromMap(
73+
ImmutableMap.<String, String>builder()
74+
.put("parallelism", "1")
75+
.put("foo", "bar")
76+
.build()));
6977
assertThat(pipelineDef).isEqualTo(fullDefWithGlobalConf);
7078
}
7179

80+
@Test
81+
void testEvaluateDefaultLocalTimeZone() throws Exception {
82+
URL resource = Resources.getResource("definitions/pipeline-definition-minimized.yaml");
83+
YamlPipelineDefinitionParser parser = new YamlPipelineDefinitionParser();
84+
PipelineDef pipelineDef = parser.parse(Paths.get(resource.toURI()), new Configuration());
85+
assertThat(pipelineDef.getConfig().get(PIPELINE_LOCAL_TIME_ZONE))
86+
.isNotEqualTo(PIPELINE_LOCAL_TIME_ZONE.defaultValue());
87+
}
88+
89+
@Test
90+
void testValidTimeZone() throws Exception {
91+
URL resource = Resources.getResource("definitions/pipeline-definition-minimized.yaml");
92+
YamlPipelineDefinitionParser parser = new YamlPipelineDefinitionParser();
93+
PipelineDef pipelineDef =
94+
parser.parse(
95+
Paths.get(resource.toURI()),
96+
Configuration.fromMap(
97+
ImmutableMap.<String, String>builder()
98+
.put(PIPELINE_LOCAL_TIME_ZONE.key(), "Asia/Shanghai")
99+
.build()));
100+
assertThat(pipelineDef.getConfig().get(PIPELINE_LOCAL_TIME_ZONE))
101+
.isEqualTo("Asia/Shanghai");
102+
103+
pipelineDef =
104+
parser.parse(
105+
Paths.get(resource.toURI()),
106+
Configuration.fromMap(
107+
ImmutableMap.<String, String>builder()
108+
.put(PIPELINE_LOCAL_TIME_ZONE.key(), "GMT+08:00")
109+
.build()));
110+
assertThat(pipelineDef.getConfig().get(PIPELINE_LOCAL_TIME_ZONE)).isEqualTo("GMT+08:00");
111+
112+
pipelineDef =
113+
parser.parse(
114+
Paths.get(resource.toURI()),
115+
Configuration.fromMap(
116+
ImmutableMap.<String, String>builder()
117+
.put(PIPELINE_LOCAL_TIME_ZONE.key(), "UTC")
118+
.build()));
119+
assertThat(pipelineDef.getConfig().get(PIPELINE_LOCAL_TIME_ZONE)).isEqualTo("UTC");
120+
}
121+
122+
@Test
123+
void testInvalidTimeZone() throws Exception {
124+
URL resource = Resources.getResource("definitions/pipeline-definition-minimized.yaml");
125+
YamlPipelineDefinitionParser parser = new YamlPipelineDefinitionParser();
126+
assertThatThrownBy(
127+
() ->
128+
parser.parse(
129+
Paths.get(resource.toURI()),
130+
Configuration.fromMap(
131+
ImmutableMap.<String, String>builder()
132+
.put(
133+
PIPELINE_LOCAL_TIME_ZONE.key(),
134+
"invalid time zone")
135+
.build())))
136+
.isInstanceOf(IllegalArgumentException.class)
137+
.hasMessageContaining(
138+
"Invalid time zone. The valid value should be a Time Zone Database ID"
139+
+ " such as 'America/Los_Angeles' to include daylight saving time. "
140+
+ "Fixed offsets are supported using 'GMT-08:00' or 'GMT+08:00'. "
141+
+ "Or use 'UTC' without time zone and daylight saving time.");
142+
}
143+
72144
private final PipelineDef fullDef =
73145
new PipelineDef(
74146
new SourceDef(

‎flink-cdc-common/src/main/java/com/ververica/cdc/common/factories/Factory.java‎

Lines changed: 9 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,6 @@
2020
import com.ververica.cdc.common.configuration.ConfigOption;
2121
import com.ververica.cdc.common.configuration.Configuration;
2222

23-
import java.util.Collections;
24-
import java.util.Map;
2523
import java.util.Set;
2624

2725
/**
@@ -63,23 +61,21 @@ public interface Factory {
6361
@PublicEvolving
6462
interface Context {
6563

66-
/** Gives the configuration of the current session. */
67-
Configuration getConfiguration();
68-
6964
/**
70-
* Returns the class loader of the current session.
65+
* Returns the factory options used to create the object instances.
7166
*
72-
* <p>The class loader is in particular useful for discovering factories.
67+
* @return options of the current session.
7368
*/
74-
ClassLoader getClassLoader();
69+
Configuration getFactoryConfiguration();
70+
71+
/** Returns the configuration of current pipeline. */
72+
Configuration getPipelineConfiguration();
7573

7674
/**
77-
* Returns the options of the current session.
75+
* Returns the class loader of the current session.
7876
*
79-
* @return options of the current session.
77+
* <p>The class loader is in particular useful for discovering factories.
8078
*/
81-
default Map<String, String> getEnrichmentOptions() {
82-
return Collections.emptyMap();
83-
}
79+
ClassLoader getClassLoader();
8480
}
8581
}

‎flink-cdc-common/src/main/java/com/ververica/cdc/common/factories/FactoryHelper.java‎

Lines changed: 12 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -19,41 +19,39 @@
1919
import com.ververica.cdc.common.annotation.PublicEvolving;
2020
import com.ververica.cdc.common.configuration.Configuration;
2121

22-
import java.util.Map;
23-
2422
/** A helper for working with {@link Factory}. */
2523
@PublicEvolving
2624
public class FactoryHelper {
2725

2826
/** Default implementation of {@link Factory.Context}. */
2927
public static class DefaultContext implements Factory.Context {
3028

31-
private final Map<String, String> enrichmentOptions;
29+
private final Configuration factoryConfiguration;
3230
private final ClassLoader classLoader;
33-
private final Configuration configuration;
31+
private final Configuration pipelineConfiguration;
3432

3533
public DefaultContext(
36-
Map<String, String> enrichmentOptions,
37-
Configuration configuration,
34+
Configuration factoryConfiguration,
35+
Configuration pipelineConfiguration,
3836
ClassLoader classLoader) {
39-
this.enrichmentOptions = enrichmentOptions;
40-
this.configuration = configuration;
37+
this.factoryConfiguration = factoryConfiguration;
38+
this.pipelineConfiguration = pipelineConfiguration;
4139
this.classLoader = classLoader;
4240
}
4341

4442
@Override
45-
public Configuration getConfiguration() {
46-
return configuration;
43+
public Configuration getFactoryConfiguration() {
44+
return factoryConfiguration;
4745
}
4846

4947
@Override
50-
public ClassLoader getClassLoader() {
51-
return classLoader;
48+
public Configuration getPipelineConfiguration() {
49+
return pipelineConfiguration;
5250
}
5351

5452
@Override
55-
public Map<String, String> getEnrichmentOptions() {
56-
return enrichmentOptions;
53+
public ClassLoader getClassLoader() {
54+
return classLoader;
5755
}
5856
}
5957
}

‎flink-cdc-common/src/main/java/com/ververica/cdc/common/pipeline/PipelineOptions.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,26 @@ public class PipelineOptions {
5757
"EXCEPTION: Throw an exception to terminate the sync pipeline.")))
5858
.build());
5959

60+
public static final ConfigOption<String> PIPELINE_LOCAL_TIME_ZONE =
61+
ConfigOptions.key("pipeline.local-time-zone")
62+
.stringType()
63+
// "systemDefault" is a special value to decide whether to use
64+
// ZoneId.systemDefault() in
65+
// PipelineOptions.getLocalTimeZone()
66+
.defaultValue("systemDefault")
67+
.withDescription(
68+
Description.builder()
69+
.text(
70+
"The local time zone defines current session time zone id. ")
71+
.linebreak()
72+
.text(
73+
"It is used when converting to/from <code>TIMESTAMP WITH LOCAL TIME ZONE</code>. "
74+
+ "Internally, timestamps with local time zone are always represented in the UTC time zone. "
75+
+ "However, when converting to data types that don't include a time zone (e.g. TIMESTAMP, STRING), "
76+
+ "the session time zone is used during conversion. The input of option is either a full name "
77+
+ "such as \"America/Los_Angeles\", or a custom timezone id such as \"GMT-08:00\".")
78+
.build());
79+
6080
public static final ConfigOption<String> SCHEMA_OPERATOR_UID =
6181
ConfigOptions.key("pipeline.schema.operator.uid")
6282
.stringType()

‎flink-cdc-composer/src/main/java/com/ververica/cdc/composer/definition/PipelineDef.java‎

Lines changed: 53 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,16 @@
1616

1717
package com.ververica.cdc.composer.definition;
1818

19+
import com.ververica.cdc.common.annotation.VisibleForTesting;
1920
import com.ververica.cdc.common.configuration.Configuration;
21+
import com.ververica.cdc.common.types.LocalZonedTimestampType;
2022

23+
import java.time.ZoneId;
2124
import java.util.List;
2225
import java.util.Objects;
26+
import java.util.TimeZone;
27+
28+
import static com.ververica.cdc.common.pipeline.PipelineOptions.PIPELINE_LOCAL_TIME_ZONE;
2329

2430
/**
2531
* Definition of a pipeline.
@@ -58,7 +64,7 @@ public PipelineDef(
5864
this.sink = sink;
5965
this.routes = routes;
6066
this.transforms = transforms;
61-
this.config = config;
67+
this.config = evaluatePipelineTimeZone(config);
6268
}
6369

6470
public SourceDef getSource() {
@@ -117,4 +123,50 @@ public boolean equals(Object o) {
117123
public int hashCode() {
118124
return Objects.hash(source, sink, routes, transforms, config);
119125
}
126+
127+
// ------------------------------------------------------------------------
128+
// Utilities
129+
// ------------------------------------------------------------------------
130+
131+
/**
132+
* Returns the current session time zone id. It is used when converting to/from {@code TIMESTAMP
133+
* WITH LOCAL TIME ZONE}.
134+
*
135+
* @see LocalZonedTimestampType
136+
*/
137+
@VisibleForTesting
138+
private static Configuration evaluatePipelineTimeZone(Configuration configuration) {
139+
final String zone = configuration.get(PIPELINE_LOCAL_TIME_ZONE);
140+
ZoneId zoneId;
141+
if (PIPELINE_LOCAL_TIME_ZONE.defaultValue().equals(zone)) {
142+
zoneId = ZoneId.systemDefault();
143+
} else {
144+
validateTimeZone(zone);
145+
zoneId = ZoneId.of(zone);
146+
}
147+
configuration.set(PIPELINE_LOCAL_TIME_ZONE, zoneId.toString());
148+
return configuration;
149+
}
150+
151+
/**
152+
* Validates a time zone is valid or not.
153+
*
154+
* @param zone given time zone
155+
*/
156+
private static void validateTimeZone(String zone) {
157+
boolean isValid;
158+
try {
159+
isValid = TimeZone.getTimeZone(zone).toZoneId().equals(ZoneId.of(zone));
160+
} catch (Exception ignore) {
161+
isValid = false;
162+
}
163+
164+
if (!isValid) {
165+
throw new IllegalArgumentException(
166+
"Invalid time zone. The valid value should be a Time Zone Database ID "
167+
+ "such as 'America/Los_Angeles' to include daylight saving time. "
168+
+ "Fixed offsets are supported using 'GMT-08:00' or 'GMT+08:00'. "
169+
+ "Or use 'UTC' without time zone and daylight saving time.");
170+
}
171+
}
120172
}

‎flink-cdc-composer/src/main/java/com/ververica/cdc/composer/flink/FlinkPipelineComposer.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
2121

2222
import com.ververica.cdc.common.annotation.Internal;
23+
import com.ververica.cdc.common.configuration.Configuration;
2324
import com.ververica.cdc.common.event.Event;
2425
import com.ververica.cdc.common.factories.DataSinkFactory;
2526
import com.ververica.cdc.common.factories.FactoryHelper;
@@ -80,14 +81,14 @@ public PipelineExecution compose(PipelineDef pipelineDef) {
8081
// Source
8182
DataSourceTranslator sourceTranslator = new DataSourceTranslator();
8283
DataStream<Event> stream =
83-
sourceTranslator.translate(pipelineDef.getSource(), env, parallelism);
84+
sourceTranslator.translate(pipelineDef.getSource(), env, pipelineDef.getConfig());
8485

8586
// Route
8687
RouteTranslator routeTranslator = new RouteTranslator();
8788
stream = routeTranslator.translate(stream, pipelineDef.getRoute());
8889

8990
// Create sink in advance as schema operator requires MetadataApplier
90-
DataSink dataSink = createDataSink(pipelineDef.getSink());
91+
DataSink dataSink = createDataSink(pipelineDef.getSink(), pipelineDef.getConfig());
9192

9293
// Schema operator
9394
SchemaOperatorTranslator schemaOperatorTranslator =
@@ -118,7 +119,7 @@ public PipelineExecution compose(PipelineDef pipelineDef) {
118119
env, pipelineDef.getConfig().get(PipelineOptions.PIPELINE_NAME), isBlocking);
119120
}
120121

121-
private DataSink createDataSink(SinkDef sinkDef) {
122+
private DataSink createDataSink(SinkDef sinkDef, Configuration pipelineConfig) {
122123
// Search the data sink factory
123124
DataSinkFactory sinkFactory =
124125
FactoryDiscoveryUtils.getFactoryByIdentifier(
@@ -131,8 +132,8 @@ private DataSink createDataSink(SinkDef sinkDef) {
131132
// Create data sink
132133
return sinkFactory.createDataSink(
133134
new FactoryHelper.DefaultContext(
134-
sinkDef.getConfig().toMap(),
135135
sinkDef.getConfig(),
136+
pipelineConfig,
136137
Thread.currentThread().getContextClassLoader()));
137138
}
138139

‎flink-cdc-composer/src/main/java/com/ververica/cdc/composer/flink/translator/DataSourceTranslator.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,11 @@
2121
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
2222

2323
import com.ververica.cdc.common.annotation.Internal;
24+
import com.ververica.cdc.common.configuration.Configuration;
2425
import com.ververica.cdc.common.event.Event;
2526
import com.ververica.cdc.common.factories.DataSourceFactory;
2627
import com.ververica.cdc.common.factories.FactoryHelper;
28+
import com.ververica.cdc.common.pipeline.PipelineOptions;
2729
import com.ververica.cdc.common.source.DataSource;
2830
import com.ververica.cdc.common.source.EventSourceProvider;
2931
import com.ververica.cdc.common.source.FlinkSourceFunctionProvider;
@@ -39,8 +41,9 @@
3941
*/
4042
@Internal
4143
public class DataSourceTranslator {
44+
4245
public DataStreamSource<Event> translate(
43-
SourceDef sourceDef, StreamExecutionEnvironment env, int sourceParallelism) {
46+
SourceDef sourceDef, StreamExecutionEnvironment env, Configuration pipelineConfig) {
4447
// Search the data source factory
4548
DataSourceFactory sourceFactory =
4649
FactoryDiscoveryUtils.getFactoryByIdentifier(
@@ -50,15 +53,16 @@ public DataStreamSource<Event> translate(
5053
DataSource dataSource =
5154
sourceFactory.createDataSource(
5255
new FactoryHelper.DefaultContext(
53-
sourceDef.getConfig().toMap(),
5456
sourceDef.getConfig(),
57+
pipelineConfig,
5558
Thread.currentThread().getContextClassLoader()));
5659

5760
// Add source JAR to environment
5861
FactoryDiscoveryUtils.getJarPathByIdentifier(sourceDef.getType(), DataSourceFactory.class)
5962
.ifPresent(jar -> FlinkEnvironmentUtils.addJar(env, jar));
6063

6164
// Get source provider
65+
final int sourceParallelism = pipelineConfig.get(PipelineOptions.GLOBAL_PARALLELISM);
6266
EventSourceProvider eventSourceProvider = dataSource.getEventSourceProvider();
6367
if (eventSourceProvider instanceof FlinkSourceProvider) {
6468
// Source

‎flink-cdc-composer/src/test/java/com/ververica/cdc/composer/flink/FlinkPipelineComposerTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,8 @@ public void testCreateDataSinkFromSinkDef() {
4949
DataSink dataSink =
5050
sinkFactory.createDataSink(
5151
new FactoryHelper.DefaultContext(
52-
sinkDef.getConfig().toMap(),
5352
sinkDef.getConfig(),
53+
new Configuration(),
5454
Thread.currentThread().getContextClassLoader()));
5555

5656
Assert.assertTrue(dataSink instanceof DataSinkFactory1.TestDataSink);

‎flink-cdc-composer/src/test/java/com/ververica/cdc/composer/flink/translator/DataSourceTranslatorTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,8 @@ public void testCreateDataSourceFromSourceDef() {
4949
DataSource dataSource =
5050
sourceFactory.createDataSource(
5151
new FactoryHelper.DefaultContext(
52-
sourceDef.getConfig().toMap(),
5352
sourceDef.getConfig(),
53+
new Configuration(),
5454
Thread.currentThread().getContextClassLoader()));
5555

5656
Assert.assertTrue(dataSource instanceof DataSourceFactory1.TestDataSource);

0 commit comments

Comments
 (0)