diff --git a/Pasted image 20250429144840.png b/Pasted image 20250429144840.png new file mode 100644 index 0000000..7060084 Binary files /dev/null and b/Pasted image 20250429144840.png differ diff --git a/Pasted image 20250429144900.png b/Pasted image 20250429144900.png new file mode 100644 index 0000000..bc3259e Binary files /dev/null and b/Pasted image 20250429144900.png differ diff --git a/Pasted image 20250429145007.png b/Pasted image 20250429145007.png new file mode 100644 index 0000000..3725b69 Binary files /dev/null and b/Pasted image 20250429145007.png differ diff --git a/work常用/中间件/seatunnel/Kafka-To-Doris.md b/work常用/中间件/seatunnel/Kafka-To-Doris.md new file mode 100644 index 0000000..608b612 --- /dev/null +++ b/work常用/中间件/seatunnel/Kafka-To-Doris.md @@ -0,0 +1,97 @@ +原始json + +[![](file:///Users/old-tom/Documents/youduqt/74236399-103695-邹瑞/image/temp/feb6c40e-d820-454c-ad3a-361609534f4f.png)](/Users/old-tom/Documents/youduqt/74236399-103695-邹瑞/image/temp/feb6c40e-d820-454c-ad3a-361609534f4f.png) + +示例配置: +```text +env { + # 任务名称 + job.name = "zr-test-doris" + # 任务并行度 + parallelism = "1" + # 任务模式 + job.mode = "STREAMING" + # 检查点时间 + checkpoint.interval = "10000" +} +source { + Kafka { + # 结果输出表 + plugin_output = "source_result" + # kafka 连接地址 + bootstrap.servers = "172.100.40.152:9092,172.100.40.153:9092,172.100.40.154:9092" + # topic + topic = "ZR_TEST_3" + # 消费者配置 + kafka.config = { + client.id = client_1 + auto.offset.reset = "latest" + } + # kafka字段映射 + schema = { + fields { + data { + c_string = "string" + c_integer = "int" + c_date = "string" + } + } + } + } +} +transform { + Sql { + plugin_input = "source_result" + plugin_output = "transform_result" + query = "select data.c_string as c_string, data.c_integer as c_integer, data.c_date as c_date from source_result" + } +} +sink { + Doris { + # 数据来源表 + plugin_input = "transform_result" + # be连接地址 + fenodes = "172.100.40.152:8030" + # 用户名 + username = "root" + # 密码 + password = "maxvision@123" + # 数据库地址 + database = "ods" + # 表名 + table = "ods_json_path_test" + # 输出任务前缀 + sink.label-prefix = "244a7e723ca84de1b25debc626abe64c" + # 是否开启两端提交 + sink.enable-2pc = "true" + # 是否开启删除策略 + sink.enable-delete = "true" + doris.config { + # 数据格式化类型 + format = "json" + # 读取每行JSON + read_json_by_line = "true" + } + } +} +``` + +关键点: +需要按照json解析出这个格式 +```text + # kafka字段映射 + schema = { + fields { + data { + c_string = "string" + c_integer = "int" + c_date = "string" + } + } + } +``` + +transform中的sql字段名为schame.xxx +```sql +select data.c_string as c_string, data.c_integer as c_integer, data.c_date as c_date from source_result +``` \ No newline at end of file diff --git a/公司环境/武汉数据中台开发环境.md b/公司环境/武汉数据中台开发环境.md index 1947710..643bb77 100644 --- a/公司环境/武汉数据中台开发环境.md +++ b/公司环境/武汉数据中台开发环境.md @@ -36,4 +36,14 @@ mysql : 172.100.40.152 bg3 172.100.40.153 bg4 172.100.40.154 bg5 -172.100.40.155 bg6 \ No newline at end of file +172.100.40.155 bg6 + +seatunnel : bg2 、bg3 master节点 +bg 4~6 worker节点 + +```shell +# 主节点启动 +cd /opt/datasophon/seatunnel/bin && ./seatunnel-cluster.sh -d -r master +# 工作节点启动 +cd /opt/datasophon/seatunnel/bin && ./seatunnel-cluster.sh -d -r worker +``` \ No newline at end of file diff --git a/日常学习/大模型/MCP.md b/日常学习/大模型/MCP/MCP 基本原理.md similarity index 100% rename from 日常学习/大模型/MCP.md rename to 日常学习/大模型/MCP/MCP 基本原理.md diff --git a/日常学习/大模型/MCP/MCP 规范新版本特性全景解析与落地实践.md b/日常学习/大模型/MCP/MCP 规范新版本特性全景解析与落地实践.md new file mode 100644 index 0000000..daedaba --- /dev/null +++ b/日常学习/大模型/MCP/MCP 规范新版本特性全景解析与落地实践.md @@ -0,0 +1 @@ +https://my.oschina.net/u/3874284/blog/18334124 \ No newline at end of file diff --git a/日常学习/大模型/function calling.md b/日常学习/大模型/MCP/function calling.md similarity index 100% rename from 日常学习/大模型/function calling.md rename to 日常学习/大模型/MCP/function calling.md diff --git a/问题排查/踩坑记录/mysql 非严格模式.md b/问题排查/踩坑记录/mysql 非严格模式.md new file mode 100644 index 0000000..bbb0f34 --- /dev/null +++ b/问题排查/踩坑记录/mysql 非严格模式.md @@ -0,0 +1,10 @@ + +mysql 非严格模式下对字段长度不做校验,例: +显示长度为30 +![[Pasted image 20250429144840.png]] + +实际长度为36: +![[Pasted image 20250429144900.png]] + +做ETL时,对端表的长度不能设置的与非严格模式下的mysql 相同,不然会出现长度问题 +![[Pasted image 20250429145007.png]] \ No newline at end of file