You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
|
|
|
|
|
hive 是分区表，需要在transform指定分区字段,例(y_birth)，另外hive_site_path和hdfs_site_path 需要和metastore_uri保持一致，即hive客户端均为同一个。
|
|
|
|
|
|
|
|
|
|
|
|
```hocon
|
|
|
|
|
|
env {
|
|
|
|
|
|
parallelism = 1
|
|
|
|
|
|
job.mode = "STREAMING"
|
|
|
|
|
|
}
|
|
|
|
|
|
source {
|
|
|
|
|
|
Kafka {
|
|
|
|
|
|
schema = {
|
|
|
|
|
|
fields {
|
|
|
|
|
|
name = "string"
|
|
|
|
|
|
age = "int"
|
|
|
|
|
|
y_birth = "int"
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
topic = "ST_DATA"
|
|
|
|
|
|
bootstrap.servers = "172.31.51.241:9092,172.31.51.242:9092,172.31.51.243:9092"
|
|
|
|
|
|
kafka.config = {
|
|
|
|
|
|
client.id = client_1
|
|
|
|
|
|
auto.offset.reset = "latest"
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
transform {
|
|
|
|
|
|
FieldMapper {
|
|
|
|
|
|
field_mapper = {
|
|
|
|
|
|
name = name
|
|
|
|
|
|
age = age
|
|
|
|
|
|
y_birth = y_birth
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
sink {
|
|
|
|
|
|
Hive {
|
|
|
|
|
|
metastore_uri = "thrift://bg1:9083"
|
|
|
|
|
|
table_name = "ods.zr_test_table"
|
|
|
|
|
|
hive_site_path = "/opt/datasophon/hive-3.1.3/conf/hive-site.xml"
|
|
|
|
|
|
hdfs_site_path="/opt/datasophon/hadoop/etc/hadoop/hdfs-site.xml"
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
```
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
hive 表建表语句
|
|
|
|
|
|
![[Pasted image 20250415152322.png]]
|
