#!/usr/bin/env python # -*- coding: utf-8 -*- # @Time : 2023/4/8 9:05 # @Author : old tom # @File : metadata_reader.py # @Project : futool-tiny-datahub # @Desc : 元数据读取 import abc from configparser import ConfigParser from datahub.datasource.constant import ds_conf_param from datahub.datasource.datasource_manage import DataSource, DataSourceManage from datahub.metadata.metadatadao.metadata_dao import MetadataDao from common.fudb.connectors.connector_factory import ConnFactory from sqllineage.runner import LineageRunner reader_conf = ConfigParser() reader_conf.read('./reader_conf.ini') class AbsMetadataReader(metaclass=abc.ABCMeta): """ 抽象元数据读取器 """ def __init__(self, datasource): self.datasource = datasource @abc.abstractmethod def query_tables(self): """ 查询当前连接下所有表 :return: """ pass @abc.abstractmethod def query_views(self): """ 查询当前连接下所有视图 :return: """ pass @abc.abstractmethod def query_procedure(self): """ 查询当前连接下所有存储过程 :return: """ pass @abc.abstractmethod def query_table_fields(self, table): """ 查询字段 :param table: :return: """ pass class MetadataReader(AbsMetadataReader): def __init__(self, datasource: DataSource): super().__init__(datasource) self.db_type = datasource.connector.db_type self.dao = MetadataDao(datasource.connector) def query_tables(self): return self.dao.query_all_tables(reader_conf[self.db_type]['tables']) def query_views(self): return self.dao.query_all_views(reader_conf[self.db_type]['views']) def query_procedure(self): return self.dao.query_all_procedure(reader_conf[self.db_type]['procedure']) def query_table_fields(self, table): return self.dao.query_table_field(reader_conf[self.db_type]['table_field'].replace('#$#', table)) def query_metadata_detail(self, obj_name, obj_type): """ 查询元数据明细 例:obj_type 为view或procedure 则查询对应创建语句 为table 则查询中文注释 :param obj_name: 对象名称 :param obj_type: 对象类型 :return: """ return { 'table': '', 'view': self.dao.query_view_detail(obj_name, reader_conf[self.db_type][obj_type + '_detail'].replace('#$#', obj_name)), 'procedure': self.dao.query_procedure_detail( reader_conf[self.db_type][obj_type + '_detail'].replace('#$#', obj_name)) }[obj_type] if __name__ == '__main__': local_ds = ds_conf_param('postgresql', 'postgres', 'root@123', 'localhost', 5432, 'postgres') dsm = DataSourceManage(ConnFactory(local_ds)) ds = dsm.get('834164a2d62de959c0261e6239dd1e55') mtr = MetadataReader(ds) # print(mtr.query_tables()) # print(mtr.query_views()) # print(mtr.query_procedure()) # sql = mtr.query_metadata_detail('DW__FINANCE_SURVEY_MONTH_TJ_VW', 'view') # runner = LineageRunner(sql, dialect='oracle') # 画图 # runner.draw(dialect='oracle') # print(runner) # runner.draw(dialect='oracle') print(mtr.query_table_fields('ODS_CONTRACT_PAY_INFO'))