|
|
#!/usr/bin/env python
|
|
|
# -*- coding: utf-8 -*-
|
|
|
# @Time : 2023/4/8 9:05
|
|
|
# @Author : old tom
|
|
|
# @File : metadata_reader.py
|
|
|
# @Project : futool-tiny-datahub
|
|
|
# @Desc : 元数据读取
|
|
|
|
|
|
import abc
|
|
|
from configparser import ConfigParser
|
|
|
|
|
|
from datahub.datasource.constant import ds_conf_param
|
|
|
from datahub.datasource.datasource_manage import DataSource, DataSourceManage
|
|
|
from datahub.metadata.metadatadao.metadata_dao import MetadataDao
|
|
|
from common.fudb.connectors.connector_factory import ConnFactory
|
|
|
from sqllineage.runner import LineageRunner
|
|
|
|
|
|
reader_conf = ConfigParser()
|
|
|
reader_conf.read('./reader_conf.ini')
|
|
|
|
|
|
|
|
|
class AbsMetadataReader(metaclass=abc.ABCMeta):
|
|
|
"""
|
|
|
抽象元数据读取器
|
|
|
"""
|
|
|
|
|
|
def __init__(self, datasource):
|
|
|
self.datasource = datasource
|
|
|
|
|
|
@abc.abstractmethod
|
|
|
def query_tables(self):
|
|
|
"""
|
|
|
查询当前连接下所有表
|
|
|
:return:
|
|
|
"""
|
|
|
pass
|
|
|
|
|
|
@abc.abstractmethod
|
|
|
def query_views(self):
|
|
|
"""
|
|
|
查询当前连接下所有视图
|
|
|
:return:
|
|
|
"""
|
|
|
pass
|
|
|
|
|
|
@abc.abstractmethod
|
|
|
def query_procedure(self):
|
|
|
"""
|
|
|
查询当前连接下所有存储过程
|
|
|
:return:
|
|
|
"""
|
|
|
pass
|
|
|
|
|
|
@abc.abstractmethod
|
|
|
def query_table_fields(self, table):
|
|
|
"""
|
|
|
查询字段
|
|
|
:param table:
|
|
|
:return:
|
|
|
"""
|
|
|
pass
|
|
|
|
|
|
|
|
|
class MetadataReader(AbsMetadataReader):
|
|
|
def __init__(self, datasource: DataSource):
|
|
|
super().__init__(datasource)
|
|
|
self.db_type = datasource.connector.db_type
|
|
|
self.dao = MetadataDao(datasource.connector)
|
|
|
|
|
|
def query_tables(self):
|
|
|
return self.dao.query_all_tables(reader_conf[self.db_type]['tables'])
|
|
|
|
|
|
def query_views(self):
|
|
|
return self.dao.query_all_views(reader_conf[self.db_type]['views'])
|
|
|
|
|
|
def query_procedure(self):
|
|
|
return self.dao.query_all_procedure(reader_conf[self.db_type]['procedure'])
|
|
|
|
|
|
def query_table_fields(self, table):
|
|
|
return self.dao.query_table_field(reader_conf[self.db_type]['table_field'].replace('#$#', table))
|
|
|
|
|
|
def query_metadata_detail(self, obj_name, obj_type):
|
|
|
"""
|
|
|
查询元数据明细
|
|
|
例：obj_type 为view或procedure 则查询对应创建语句
|
|
|
为table 则查询中文注释
|
|
|
:param obj_name: 对象名称
|
|
|
:param obj_type: 对象类型
|
|
|
:return:
|
|
|
"""
|
|
|
return {
|
|
|
'table': '',
|
|
|
'view': self.dao.query_view_detail(obj_name,
|
|
|
reader_conf[self.db_type][obj_type + '_detail'].replace('#$#',
|
|
|
obj_name)),
|
|
|
'procedure': self.dao.query_procedure_detail(
|
|
|
reader_conf[self.db_type][obj_type + '_detail'].replace('#$#', obj_name))
|
|
|
}[obj_type]
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
local_ds = ds_conf_param('postgresql', 'postgres', 'root@123', 'localhost', 5432, 'postgres')
|
|
|
dsm = DataSourceManage(ConnFactory(local_ds))
|
|
|
ds = dsm.get('834164a2d62de959c0261e6239dd1e55')
|
|
|
mtr = MetadataReader(ds)
|
|
|
# print(mtr.query_tables())
|
|
|
# print(mtr.query_views())
|
|
|
# print(mtr.query_procedure())
|
|
|
# sql = mtr.query_metadata_detail('DW__FINANCE_SURVEY_MONTH_TJ_VW', 'view')
|
|
|
# runner = LineageRunner(sql, dialect='oracle')
|
|
|
# 画图
|
|
|
# runner.draw(dialect='oracle')
|
|
|
# print(runner)
|
|
|
# runner.draw(dialect='oracle')
|
|
|
print(mtr.query_table_fields('ODS_CONTRACT_PAY_INFO'))
|
