commit
5bb28d2ebb
@ -0,0 +1,162 @@
|
||||
# ---> Python
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.pyc
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py,cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
cover/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
.pybuilder/
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
# For a library or package, you might want to ignore these files since the code is
|
||||
# intended to run in multiple environments; otherwise, check them in:
|
||||
# .python-version
|
||||
|
||||
# pipenv
|
||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||
# install all needed dependencies.
|
||||
#Pipfile.lock
|
||||
|
||||
# poetry
|
||||
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||
# commonly ignored for libraries.
|
||||
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
||||
#poetry.lock
|
||||
|
||||
# pdm
|
||||
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
||||
#pdm.lock
|
||||
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
||||
# in version control.
|
||||
# https://pdm.fming.dev/#use-with-ide
|
||||
.pdm.toml
|
||||
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
# pytype static type analyzer
|
||||
.pytype/
|
||||
|
||||
# Cython debug symbols
|
||||
cython_debug/
|
||||
|
||||
# PyCharm
|
||||
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
||||
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
||||
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||
.idea/
|
||||
@ -0,0 +1,12 @@
|
||||
[[source]]
|
||||
url = "https://pypi.org/simple"
|
||||
verify_ssl = true
|
||||
name = "pypi"
|
||||
|
||||
[packages]
|
||||
sqlalchemy = "*"
|
||||
|
||||
[dev-packages]
|
||||
|
||||
[requires]
|
||||
python_version = "3.8"
|
||||
@ -0,0 +1,142 @@
|
||||
{
|
||||
"_meta": {
|
||||
"hash": {
|
||||
"sha256": "b97e95210411fba046eb9e76c0be6c64ebad1ae4e815c86badba93f4590a3efe"
|
||||
},
|
||||
"pipfile-spec": 6,
|
||||
"requires": {
|
||||
"python_version": "3.8"
|
||||
},
|
||||
"sources": [
|
||||
{
|
||||
"name": "pypi",
|
||||
"url": "https://pypi.org/simple",
|
||||
"verify_ssl": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"default": {
|
||||
"greenlet": {
|
||||
"hashes": [
|
||||
"sha256:03a8f4f3430c3b3ff8d10a2a86028c660355ab637cee9333d63d66b56f09d52a",
|
||||
"sha256:0bf60faf0bc2468089bdc5edd10555bab6e85152191df713e2ab1fcc86382b5a",
|
||||
"sha256:18a7f18b82b52ee85322d7a7874e676f34ab319b9f8cce5de06067384aa8ff43",
|
||||
"sha256:18e98fb3de7dba1c0a852731c3070cf022d14f0d68b4c87a19cc1016f3bb8b33",
|
||||
"sha256:1a819eef4b0e0b96bb0d98d797bef17dc1b4a10e8d7446be32d1da33e095dbb8",
|
||||
"sha256:26fbfce90728d82bc9e6c38ea4d038cba20b7faf8a0ca53a9c07b67318d46088",
|
||||
"sha256:2780572ec463d44c1d3ae850239508dbeb9fed38e294c68d19a24d925d9223ca",
|
||||
"sha256:283737e0da3f08bd637b5ad058507e578dd462db259f7f6e4c5c365ba4ee9343",
|
||||
"sha256:2d4686f195e32d36b4d7cf2d166857dbd0ee9f3d20ae349b6bf8afc8485b3645",
|
||||
"sha256:2dd11f291565a81d71dab10b7033395b7a3a5456e637cf997a6f33ebdf06f8db",
|
||||
"sha256:30bcf80dda7f15ac77ba5af2b961bdd9dbc77fd4ac6105cee85b0d0a5fcf74df",
|
||||
"sha256:32e5b64b148966d9cccc2c8d35a671409e45f195864560829f395a54226408d3",
|
||||
"sha256:36abbf031e1c0f79dd5d596bfaf8e921c41df2bdf54ee1eed921ce1f52999a86",
|
||||
"sha256:3a06ad5312349fec0ab944664b01d26f8d1f05009566339ac6f63f56589bc1a2",
|
||||
"sha256:3a51c9751078733d88e013587b108f1b7a1fb106d402fb390740f002b6f6551a",
|
||||
"sha256:3c9b12575734155d0c09d6c3e10dbd81665d5c18e1a7c6597df72fd05990c8cf",
|
||||
"sha256:3f6ea9bd35eb450837a3d80e77b517ea5bc56b4647f5502cd28de13675ee12f7",
|
||||
"sha256:4b58adb399c4d61d912c4c331984d60eb66565175cdf4a34792cd9600f21b394",
|
||||
"sha256:4d2e11331fc0c02b6e84b0d28ece3a36e0548ee1a1ce9ddde03752d9b79bba40",
|
||||
"sha256:5454276c07d27a740c5892f4907c86327b632127dd9abec42ee62e12427ff7e3",
|
||||
"sha256:561091a7be172ab497a3527602d467e2b3fbe75f9e783d8b8ce403fa414f71a6",
|
||||
"sha256:6c3acb79b0bfd4fe733dff8bc62695283b57949ebcca05ae5c129eb606ff2d74",
|
||||
"sha256:703f18f3fda276b9a916f0934d2fb6d989bf0b4fb5a64825260eb9bfd52d78f0",
|
||||
"sha256:7492e2b7bd7c9b9916388d9df23fa49d9b88ac0640db0a5b4ecc2b653bf451e3",
|
||||
"sha256:76ae285c8104046b3a7f06b42f29c7b73f77683df18c49ab5af7983994c2dd91",
|
||||
"sha256:7cafd1208fdbe93b67c7086876f061f660cfddc44f404279c1585bbf3cdc64c5",
|
||||
"sha256:7efde645ca1cc441d6dc4b48c0f7101e8d86b54c8530141b09fd31cef5149ec9",
|
||||
"sha256:88d9ab96491d38a5ab7c56dd7a3cc37d83336ecc564e4e8816dbed12e5aaefc8",
|
||||
"sha256:8eab883b3b2a38cc1e050819ef06a7e6344d4a990d24d45bc6f2cf959045a45b",
|
||||
"sha256:910841381caba4f744a44bf81bfd573c94e10b3045ee00de0cbf436fe50673a6",
|
||||
"sha256:9190f09060ea4debddd24665d6804b995a9c122ef5917ab26e1566dcc712ceeb",
|
||||
"sha256:937e9020b514ceedb9c830c55d5c9872abc90f4b5862f89c0887033ae33c6f73",
|
||||
"sha256:94c817e84245513926588caf1152e3b559ff794d505555211ca041f032abbb6b",
|
||||
"sha256:971ce5e14dc5e73715755d0ca2975ac88cfdaefcaab078a284fea6cfabf866df",
|
||||
"sha256:9d14b83fab60d5e8abe587d51c75b252bcc21683f24699ada8fb275d7712f5a9",
|
||||
"sha256:9f35ec95538f50292f6d8f2c9c9f8a3c6540bbfec21c9e5b4b751e0a7c20864f",
|
||||
"sha256:a1846f1b999e78e13837c93c778dcfc3365902cfb8d1bdb7dd73ead37059f0d0",
|
||||
"sha256:acd2162a36d3de67ee896c43effcd5ee3de247eb00354db411feb025aa319857",
|
||||
"sha256:b0ef99cdbe2b682b9ccbb964743a6aca37905fda5e0452e5ee239b1654d37f2a",
|
||||
"sha256:b80f600eddddce72320dbbc8e3784d16bd3fb7b517e82476d8da921f27d4b249",
|
||||
"sha256:b864ba53912b6c3ab6bcb2beb19f19edd01a6bfcbdfe1f37ddd1778abfe75a30",
|
||||
"sha256:b9ec052b06a0524f0e35bd8790686a1da006bd911dd1ef7d50b77bfbad74e292",
|
||||
"sha256:ba2956617f1c42598a308a84c6cf021a90ff3862eddafd20c3333d50f0edb45b",
|
||||
"sha256:bdfea8c661e80d3c1c99ad7c3ff74e6e87184895bbaca6ee8cc61209f8b9b85d",
|
||||
"sha256:be4ed120b52ae4d974aa40215fcdfde9194d63541c7ded40ee12eb4dda57b76b",
|
||||
"sha256:c4302695ad8027363e96311df24ee28978162cdcdd2006476c43970b384a244c",
|
||||
"sha256:c48f54ef8e05f04d6eff74b8233f6063cb1ed960243eacc474ee73a2ea8573ca",
|
||||
"sha256:c9c59a2120b55788e800d82dfa99b9e156ff8f2227f07c5e3012a45a399620b7",
|
||||
"sha256:cd021c754b162c0fb55ad5d6b9d960db667faad0fa2ff25bb6e1301b0b6e6a75",
|
||||
"sha256:d27ec7509b9c18b6d73f2f5ede2622441de812e7b1a80bbd446cb0633bd3d5ae",
|
||||
"sha256:d5508f0b173e6aa47273bdc0a0b5ba055b59662ba7c7ee5119528f466585526b",
|
||||
"sha256:d75209eed723105f9596807495d58d10b3470fa6732dd6756595e89925ce2470",
|
||||
"sha256:db1a39669102a1d8d12b57de2bb7e2ec9066a6f2b3da35ae511ff93b01b5d564",
|
||||
"sha256:dbfcfc0218093a19c252ca8eb9aee3d29cfdcb586df21049b9d777fd32c14fd9",
|
||||
"sha256:e0f72c9ddb8cd28532185f54cc1453f2c16fb417a08b53a855c4e6a418edd099",
|
||||
"sha256:e7c8dc13af7db097bed64a051d2dd49e9f0af495c26995c00a9ee842690d34c0",
|
||||
"sha256:ea9872c80c132f4663822dd2a08d404073a5a9b5ba6155bea72fb2a79d1093b5",
|
||||
"sha256:eff4eb9b7eb3e4d0cae3d28c283dc16d9bed6b193c2e1ace3ed86ce48ea8df19",
|
||||
"sha256:f82d4d717d8ef19188687aa32b8363e96062911e63ba22a0cff7802a8e58e5f1",
|
||||
"sha256:fc3a569657468b6f3fb60587e48356fe512c1754ca05a564f11366ac9e306526"
|
||||
],
|
||||
"markers": "platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))",
|
||||
"version": "==2.0.2"
|
||||
},
|
||||
"sqlalchemy": {
|
||||
"hashes": [
|
||||
"sha256:07950fc82f844a2de67ddb4e535f29b65652b4d95e8b847823ce66a6d540a41d",
|
||||
"sha256:0a865b5ec4ba24f57c33b633b728e43fde77b968911a6046443f581b25d29dd9",
|
||||
"sha256:0b49f1f71d7a44329a43d3edd38cc5ee4c058dfef4487498393d16172007954b",
|
||||
"sha256:13f984a190d249769a050634b248aef8991acc035e849d02b634ea006c028fa8",
|
||||
"sha256:1b69666e25cc03c602d9d3d460e1281810109e6546739187044fc256c67941ef",
|
||||
"sha256:1d06e119cf79a3d80ab069f064a07152eb9ba541d084bdaee728d8a6f03fd03d",
|
||||
"sha256:246712af9fc761d6c13f4f065470982e175d902e77aa4218c9cb9fc9ff565a0c",
|
||||
"sha256:34eb96c1de91d8f31e988302243357bef3f7785e1b728c7d4b98bd0c117dafeb",
|
||||
"sha256:4c3020afb144572c7bfcba9d7cce57ad42bff6e6115dffcfe2d4ae6d444a214f",
|
||||
"sha256:4f759eccb66e6d495fb622eb7f4ac146ae674d829942ec18b7f5a35ddf029597",
|
||||
"sha256:68ed381bc340b4a3d373dbfec1a8b971f6350139590c4ca3cb722fdb50035777",
|
||||
"sha256:6b72dccc5864ea95c93e0a9c4e397708917fb450f96737b4a8395d009f90b868",
|
||||
"sha256:6e84ab63d25d8564d7a8c05dc080659931a459ee27f6ed1cf4c91f292d184038",
|
||||
"sha256:734805708632e3965c2c40081f9a59263c29ffa27cba9b02d4d92dfd57ba869f",
|
||||
"sha256:78612edf4ba50d407d0eb3a64e9ec76e6efc2b5d9a5c63415d53e540266a230a",
|
||||
"sha256:7e472e9627882f2d75b87ff91c5a2bc45b31a226efc7cc0a054a94fffef85862",
|
||||
"sha256:865392a50a721445156809c1a6d6ab6437be70c1c2599f591a8849ed95d3c693",
|
||||
"sha256:8d118e233f416d713aac715e2c1101e17f91e696ff315fc9efbc75b70d11e740",
|
||||
"sha256:8d3ece5960b3e821e43a4927cc851b6e84a431976d3ffe02aadb96519044807e",
|
||||
"sha256:93c78d42c14aa9a9e0866eacd5b48df40a50d0e2790ee377af7910d224afddcf",
|
||||
"sha256:95719215e3ec7337b9f57c3c2eda0e6a7619be194a5166c07c1e599f6afc20fa",
|
||||
"sha256:9838bd247ee42eb74193d865e48dd62eb50e45e3fdceb0fdef3351133ee53dcf",
|
||||
"sha256:aa5c270ece17c0c0e0a38f2530c16b20ea05d8b794e46c79171a86b93b758891",
|
||||
"sha256:ac6a0311fb21a99855953f84c43fcff4bdca27a2ffcc4f4d806b26b54b5cddc9",
|
||||
"sha256:ad5363a1c65fde7b7466769d4261126d07d872fc2e816487ae6cec93da604b6b",
|
||||
"sha256:b3e5864eba71a3718236a120547e52c8da2ccb57cc96cecd0480106a0c799c92",
|
||||
"sha256:bbda1da8d541904ba262825a833c9f619e93cb3fd1156be0a5e43cd54d588dcd",
|
||||
"sha256:c6e27189ff9aebfb2c02fd252c629ea58657e7a5ff1a321b7fc9c2bf6dc0b5f3",
|
||||
"sha256:c8239ce63a90007bce479adf5460d48c1adae4b933d8e39a4eafecfc084e503c",
|
||||
"sha256:d209594e68bec103ad5243ecac1b40bf5770c9ebf482df7abf175748a34f4853",
|
||||
"sha256:d5327f54a9c39e7871fc532639616f3777304364a0bb9b89d6033ad34ef6c5f8",
|
||||
"sha256:db4bd1c4792da753f914ff0b688086b9a8fd78bb9bc5ae8b6d2e65f176b81eb9",
|
||||
"sha256:e4780be0f19e5894c17f75fc8de2fe1ae233ab37827125239ceb593c6f6bd1e2",
|
||||
"sha256:e4a019f723b6c1e6b3781be00fb9e0844bc6156f9951c836ff60787cc3938d76",
|
||||
"sha256:e62c4e762d6fd2901692a093f208a6a6575b930e9458ad58c2a7f080dd6132da",
|
||||
"sha256:e730603cae5747bc6d6dece98b45a57d647ed553c8d5ecef602697b1c1501cf2",
|
||||
"sha256:ebc4eeb1737a5a9bdb0c24f4c982319fa6edd23cdee27180978c29cbb026f2bd",
|
||||
"sha256:ee2946042cc7851842d7a086a92b9b7b494cbe8c3e7e4627e27bc912d3a7655e",
|
||||
"sha256:f005245e1cb9b8ca53df73ee85e029ac43155e062405015e49ec6187a2e3fb44",
|
||||
"sha256:f49c5d3c070a72ecb96df703966c9678dda0d4cb2e2736f88d15f5e1203b4159",
|
||||
"sha256:f61ab84956dc628c8dfe9d105b6aec38afb96adae3e5e7da6085b583ff6ea789"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==2.0.9"
|
||||
},
|
||||
"typing-extensions": {
|
||||
"hashes": [
|
||||
"sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb",
|
||||
"sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4"
|
||||
],
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==4.5.0"
|
||||
}
|
||||
},
|
||||
"develop": {}
|
||||
}
|
||||
@ -0,0 +1,7 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Time : 2023/4/4 16:00
|
||||
# @Author : old tom
|
||||
# @File : __init__.py.py
|
||||
# @Project : futool-tiny-datahub
|
||||
# @Desc : 通用模块
|
||||
@ -0,0 +1,7 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Time : 2023/4/4 16:00
|
||||
# @Author : old tom
|
||||
# @File : __init__.py.py
|
||||
# @Project : futool-db
|
||||
# @Desc : 数据库模块
|
||||
@ -0,0 +1,7 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Time : 2023/4/4 16:02
|
||||
# @Author : old tom
|
||||
# @File : __init__.py.py
|
||||
# @Project : futool-db
|
||||
# @Desc : 数据库连接器
|
||||
@ -0,0 +1,28 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Time : 2023/4/9 0:02
|
||||
# @Author : old tom
|
||||
# @File : connector_cache.py
|
||||
# @Project : futool-tiny-datahub
|
||||
# @Desc :
|
||||
|
||||
class ConnectorCacheManage(object):
|
||||
"""
|
||||
连接器缓存管理
|
||||
todo 缓存可能会出现V被GC回收,但是K还存在的问题,需要加入轮询机制处理
|
||||
"""
|
||||
# 缓存容器
|
||||
CONNECTOR_CACHE = {}
|
||||
|
||||
def exist(self, connector_id):
|
||||
return connector_id in self.CONNECTOR_CACHE.keys()
|
||||
|
||||
def get(self, connector_id):
|
||||
return self.CONNECTOR_CACHE[connector_id]
|
||||
|
||||
def set(self, connector_id, connector):
|
||||
self.CONNECTOR_CACHE[connector_id] = connector
|
||||
|
||||
|
||||
# 模块导入实现单例模式
|
||||
connector_cache = ConnectorCacheManage()
|
||||
@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Time : 2023/4/4 18:33
|
||||
# @Author : old tom
|
||||
# @File : connector_factory.py
|
||||
# @Project : futool-db
|
||||
# @Desc : 连接器工厂
|
||||
import hashlib
|
||||
|
||||
from common.fudb.connectors.dialect.dialect_connector import OracleConnector, PostgresqlConnector
|
||||
from datahub.datasource.constant import ds_conf_param
|
||||
from urllib.parse import quote_plus as urlquote
|
||||
from common.fudb.connectors.connector_cache import connector_cache
|
||||
|
||||
|
||||
class ConnFactory(object):
|
||||
"""
|
||||
数据库连接器工厂
|
||||
"""
|
||||
CONNECTION_CONTAINER = {
|
||||
'oracle': OracleConnector,
|
||||
'postgresql': PostgresqlConnector
|
||||
}
|
||||
|
||||
def __init__(self, conf: ds_conf_param):
|
||||
self.db_type = conf.db_type
|
||||
# 生成连接ID
|
||||
self.connector_id = self._gen_connector_id(conf.db_type, conf.user, conf.password, conf.host, conf.port,
|
||||
conf.database)
|
||||
# 尝试从缓存获取
|
||||
if connector_cache.exist(self.connector_id):
|
||||
self.connector = connector_cache.get(self.connector_id)
|
||||
else:
|
||||
# urlquote 用于处理密码中的特殊字符例如@
|
||||
self.connector = self.CONNECTION_CONTAINER[self.db_type](conf.user, urlquote(conf.password), conf.host,
|
||||
conf.port, conf.database)
|
||||
connector_cache.set(self.connector_id, self.connector)
|
||||
|
||||
def get_conn(self):
|
||||
"""
|
||||
获取连接
|
||||
:return:
|
||||
"""
|
||||
return self.connector.get_conn()
|
||||
|
||||
@staticmethod
|
||||
def _gen_connector_id(db_type, user, password, host, port, database):
|
||||
# 保证相同输入下MD5计算结果一致
|
||||
md5 = hashlib.md5()
|
||||
md5.update(
|
||||
(db_type + user + password + host + str(port) + password + database).encode(encoding='utf-8'))
|
||||
return md5.hexdigest()
|
||||
@ -0,0 +1,7 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Time : 2023/4/4 18:54
|
||||
# @Author : old tom
|
||||
# @File : __init__.py.py
|
||||
# @Project : futool-db
|
||||
# @Desc :
|
||||
@ -0,0 +1,25 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Time : 2023/4/4 16:04
|
||||
# @Author : old tom
|
||||
# @File : abs_connector.py
|
||||
# @Project : futool-db
|
||||
# @Desc : 抽象层
|
||||
|
||||
import abc
|
||||
from sqlalchemy import create_engine
|
||||
|
||||
|
||||
class CommonConnector(metaclass=abc.ABCMeta):
|
||||
|
||||
def __init__(self, db_conf: str):
|
||||
# 初始化
|
||||
self.engine = create_engine(db_conf, pool_size=15, pool_recycle=3600)
|
||||
|
||||
@abc.abstractmethod
|
||||
def get_conn(self):
|
||||
"""
|
||||
获取连接
|
||||
:return:
|
||||
"""
|
||||
pass
|
||||
@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Time : 2023/4/4 18:55
|
||||
# @Author : old tom
|
||||
# @File : dialect_connector.py
|
||||
# @Project : futool-db
|
||||
# @Desc :
|
||||
from common.fudb.connectors.dialect.abs_connector import CommonConnector
|
||||
|
||||
|
||||
class OracleConnector(CommonConnector):
|
||||
"""
|
||||
oracle 连接器,目前仅支持sever_name方式,SID方式待开发
|
||||
"""
|
||||
|
||||
DSN = 'oracle+cx_oracle://{0}:{1}@{2}:{3}/?service_name={4}'
|
||||
|
||||
def __init__(self, user, password, host, port=1521, server_name='orcl'):
|
||||
super().__init__(self.DSN.format(user, password, host, port, server_name))
|
||||
|
||||
def get_conn(self):
|
||||
return self.engine.connect()
|
||||
|
||||
|
||||
class PostgresqlConnector(CommonConnector):
|
||||
"""
|
||||
pg连接器
|
||||
"""
|
||||
PG_DIALECT = 'postgresql+psycopg2://{0}:{1}@{2}:{3}/{4}'
|
||||
|
||||
def __init__(self, user, password, host, port=5432, database='postgres'):
|
||||
super().__init__(self.PG_DIALECT.format(user, password, host, port, database))
|
||||
|
||||
def get_conn(self):
|
||||
return self.engine.connect()
|
||||
@ -0,0 +1,7 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Time : 2023/4/4 16:03
|
||||
# @Author : old tom
|
||||
# @File : __init__.py.py
|
||||
# @Project : futool-db
|
||||
# @Desc : 操作数据库API
|
||||
@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Time : 2023/4/5 11:26
|
||||
# @Author : old tom
|
||||
# @File : fu_collection.py
|
||||
# @Project : futool-db
|
||||
# @Desc : 集合类工具
|
||||
|
||||
def split_coll(data: [], part_size=5):
|
||||
"""
|
||||
分割集合
|
||||
:param data:
|
||||
:param part_size:
|
||||
:return:
|
||||
"""
|
||||
rt = []
|
||||
if len(data) <= part_size:
|
||||
rt.append(data)
|
||||
else:
|
||||
rt.append(data[0:part_size])
|
||||
for j, d in enumerate(data):
|
||||
if j > 0 and j % part_size == 0:
|
||||
rt.append(data[j:j + part_size])
|
||||
return rt
|
||||
@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Time : 2023/4/8 10:12
|
||||
# @Author : old tom
|
||||
# @File : fu_dao.py
|
||||
# @Project : futool-tiny-datahub
|
||||
# @Desc : 通用dao层
|
||||
from collections import namedtuple
|
||||
|
||||
from common.fudb.connectors.connector_factory import ConnFactory
|
||||
from common.fudb.dbapis.fu_db_api import select_all, select_one, batch_insert, count, execute_update
|
||||
|
||||
|
||||
class BaseDao(object):
|
||||
def __init__(self, connector: ConnFactory):
|
||||
self.connector = connector
|
||||
|
||||
def query_all(self, sql):
|
||||
return select_all(self.connector.get_conn(), sql)
|
||||
|
||||
def query_one(self, sql):
|
||||
return select_one(self.connector.get_conn(), sql)
|
||||
|
||||
def count(self, table_name):
|
||||
return count(self.connector.get_conn(), table_name)
|
||||
|
||||
def execute_update(self, sql):
|
||||
return execute_update(self.connector.get_conn(), sql)
|
||||
|
||||
def batch_insert(self, db_type, sql_tpl, data, batch_size):
|
||||
return batch_insert(self.connector.get_conn(), db_type, sql_tpl, data, batch_size)
|
||||
|
||||
def dynamic_update_by_param(self, table_name, condition, param: dict):
|
||||
"""
|
||||
动态更新语句
|
||||
:param condition:
|
||||
:param table_name: 表名
|
||||
:param param: 命名元组参数
|
||||
:return:
|
||||
"""
|
||||
sql = f'update {table_name} set '
|
||||
for k, v in param.items():
|
||||
sql += f'{k}=' + (f"'{v}'" if isinstance(v, str) else f'{v}') + ","
|
||||
sql = sql[0: -1]
|
||||
sql += f' where {condition}'
|
||||
return self.execute_update(sql)
|
||||
@ -0,0 +1,38 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Time : 2023/4/8 11:05
|
||||
# @Author : old tom
|
||||
# @File : log_conf.py
|
||||
# @Project : futool-tiny-datahub
|
||||
# @Desc : 日志配置
|
||||
|
||||
import sys
|
||||
from loguru import logger
|
||||
|
||||
# 日志输出路径
|
||||
LOG_PATH = '../logout/info_log.txt'
|
||||
|
||||
|
||||
class Logger(object):
|
||||
def __init__(self):
|
||||
self.logger = logger
|
||||
self.logger.remove()
|
||||
self.logger.add(sys.stdout,
|
||||
format="<green>{time:YYYY-MM-DD HH:mm:ss}</green> | " # 颜色>时间
|
||||
"{process.name} | " # 进程名
|
||||
"{thread.name} | " # 进程名
|
||||
"<cyan>{module}</cyan>.<cyan>{function}</cyan>" # 模块名.方法名
|
||||
":<cyan>{line}</cyan> | " # 行号
|
||||
"<level>{level}</level>: " # 等级
|
||||
"<level>{message}</level>", # 日志内容
|
||||
)
|
||||
# 输出到文件的格式,注释下面的add',则关闭日志写入
|
||||
self.logger.add(LOG_PATH, level='DEBUG',
|
||||
format='{time:YYYYMMDD HH:mm:ss} - ' # 时间
|
||||
"{process.name} | " # 进程名
|
||||
"{thread.name} | " # 进程名
|
||||
'{module}.{function}:{line} - {level} -{message}', # 模块名.方法名:行号
|
||||
rotation="10 MB")
|
||||
|
||||
def get_logger(self):
|
||||
return self.logger
|
||||
@ -0,0 +1,7 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Time : 2023/4/6 9:31
|
||||
# @Author : old tom
|
||||
# @File : __init__.py.py
|
||||
# @Project : futool-tiny-datahub
|
||||
# @Desc :
|
||||
@ -0,0 +1,7 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Time : 2023/4/8 9:03
|
||||
# @Author : old tom
|
||||
# @File : __init__.py.py
|
||||
# @Project : futool-tiny-datahub
|
||||
# @Desc : 数据源管理
|
||||
@ -0,0 +1,16 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Time : 2023/4/8 13:19
|
||||
# @Author : old tom
|
||||
# @File : constant.py
|
||||
# @Project : futool-tiny-datahub
|
||||
# @Desc :
|
||||
from collections import namedtuple
|
||||
|
||||
# 数据库类型
|
||||
DB_TYPE = ['oracle', 'mysql', 'postgresql', 'sqlserver']
|
||||
# 需要执行select 1 from dual的数据库
|
||||
DUAL_DB_TYPE = ['oracle', 'mysql']
|
||||
# 命名元组处理传参
|
||||
# 数据库配置
|
||||
ds_conf_param = namedtuple('datasource_conf', ['db_type', 'user', 'password', 'host', 'port', 'database'])
|
||||
@ -0,0 +1,118 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Time : 2023/4/8 9:25
|
||||
# @Author : old tom
|
||||
# @File : datasource_manage.py
|
||||
# @Project : futool-tiny-datahub
|
||||
# @Desc : 元数据管理
|
||||
|
||||
from common.fudb.connectors.connector_factory import ConnFactory
|
||||
from common.fudb.dbapis.fu_db_api import select_one
|
||||
from datahub.datasource.dsdao.ds_dao import DataSourceDao
|
||||
from common.log_conf import Logger
|
||||
from datahub.datasource.constant import ds_conf_param, DUAL_DB_TYPE
|
||||
|
||||
# 日志
|
||||
logger = Logger().get_logger()
|
||||
|
||||
|
||||
class DataSource(object):
|
||||
def __init__(self, source_id, conf: ds_conf_param):
|
||||
self.source_id = source_id
|
||||
self.conf = conf
|
||||
self.connector = ConnFactory(conf)
|
||||
|
||||
def get_connector(self):
|
||||
return self.connector
|
||||
|
||||
|
||||
class DataSourceManage(object):
|
||||
"""
|
||||
数据源管理
|
||||
"""
|
||||
|
||||
def __init__(self, local_db):
|
||||
self.dao = DataSourceDao(local_db)
|
||||
|
||||
def add(self, conf: ds_conf_param):
|
||||
# 初始化连接器
|
||||
connector = ConnFactory(conf)
|
||||
if self.dao.exist_by_source(connector.connector_id):
|
||||
# 数据源已存在
|
||||
return False, 'datasource all ready exists'
|
||||
# 初始化校验器
|
||||
checker = DataSourceChecker(connector)
|
||||
result, msg = checker.check()
|
||||
if result:
|
||||
# 入库
|
||||
return self.dao.add_datasource(connector.connector_id, conf), ''
|
||||
else:
|
||||
# 返回错误信息
|
||||
return result, f'check failed,{msg}'
|
||||
|
||||
def remove(self, source_id):
|
||||
self.dao.remove_datasource(source_id)
|
||||
|
||||
def check(self, source_id):
|
||||
pass
|
||||
|
||||
def deactivate(self, source_id):
|
||||
"""
|
||||
停用数据源
|
||||
:param source_id:
|
||||
:return:
|
||||
"""
|
||||
return self.dao.deactivate_datasource(source_id)
|
||||
|
||||
def edit(self, source_id, param_dict):
|
||||
"""
|
||||
编辑数据源
|
||||
:return:
|
||||
"""
|
||||
return self.dao.edit_datasource_conf(source_id, param_dict)
|
||||
|
||||
def get(self, source_id) -> DataSource:
|
||||
"""
|
||||
获取数据源
|
||||
:param source_id:
|
||||
:return:
|
||||
"""
|
||||
conf = self.dao.query_datasource_conf(source_id)
|
||||
# 转为命名元组
|
||||
conf_tuple = ds_conf_param._make(conf)
|
||||
return DataSource(source_id, conf_tuple)
|
||||
|
||||
|
||||
class DataSourceChecker(object):
|
||||
"""
|
||||
数据源检测
|
||||
"""
|
||||
|
||||
def __init__(self, connector: ConnFactory):
|
||||
self.connector = connector
|
||||
|
||||
def check(self):
|
||||
"""
|
||||
未知数据源连接测试
|
||||
:return:
|
||||
"""
|
||||
try:
|
||||
# 获取连接
|
||||
conn = self.connector.get_conn()
|
||||
except Exception as e:
|
||||
return False, f'cannot get connection,e={e}'
|
||||
try:
|
||||
# 测试select 1
|
||||
return int(select_one(conn, 'select 1 ' + (
|
||||
'from dual' if self.connector.db_type in DUAL_DB_TYPE else ''))) > 0, 'success'
|
||||
except Exception as e:
|
||||
return False, f'cannot execute "select 1",e={e}'
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
local_ds = ds_conf_param('postgresql', 'postgres', 'root@123', 'localhost', 5432, 'postgres')
|
||||
factory_1 = ConnFactory(local_ds)
|
||||
factory_2 = ConnFactory(local_ds)
|
||||
print(factory_1.connector_id, factory_2.connector_id)
|
||||
print(factory_1 is factory_2)
|
||||
print(factory_2.connector is factory_1.connector)
|
||||
@ -0,0 +1,8 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Time : 2023/4/8 11:04
|
||||
# @Author : old tom
|
||||
# @File : datasource_web_api.py
|
||||
# @Project : futool-tiny-datahub
|
||||
# @Desc : http接口
|
||||
|
||||
@ -0,0 +1,7 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Time : 2023/4/8 10:11
|
||||
# @Author : old tom
|
||||
# @File : __init__.py.py
|
||||
# @Project : futool-tiny-datahub
|
||||
# @Desc :
|
||||
@ -0,0 +1,7 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Time : 2023/4/6 9:32
|
||||
# @Author : old tom
|
||||
# @File : __init__.py.py
|
||||
# @Project : futool-tiny-datahub
|
||||
# @Desc : 数据关系上图
|
||||
@ -0,0 +1,7 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Time : 2023/4/6 9:32
|
||||
# @Author : old tom
|
||||
# @File : __init__.py.py
|
||||
# @Project : futool-tiny-datahub
|
||||
# @Desc : 元数据管理
|
||||
@ -0,0 +1,88 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Time : 2023/4/8 9:05
|
||||
# @Author : old tom
|
||||
# @File : metadata_reader.py
|
||||
# @Project : futool-tiny-datahub
|
||||
# @Desc : 元数据读取
|
||||
|
||||
import abc
|
||||
from configparser import ConfigParser
|
||||
|
||||
from datahub.datasource.constant import ds_conf_param
|
||||
from datahub.datasource.datasource_manage import DataSource, DataSourceManage
|
||||
from datahub.metadata.metadatadao.metadata_dao import MetadataDao
|
||||
from common.fudb.connectors.connector_factory import ConnFactory
|
||||
|
||||
reader_conf = ConfigParser()
|
||||
reader_conf.read('./reader_conf.ini')
|
||||
|
||||
|
||||
class AbsMetadataReader(metaclass=abc.ABCMeta):
|
||||
"""
|
||||
抽象元数据读取器
|
||||
"""
|
||||
|
||||
def __init__(self, datasource):
|
||||
self.datasource = datasource
|
||||
|
||||
@abc.abstractmethod
|
||||
def query_tables(self):
|
||||
"""
|
||||
查询当前连接下所有表
|
||||
:return:
|
||||
"""
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
def query_views(self):
|
||||
"""
|
||||
查询当前连接下所有视图
|
||||
:return:
|
||||
"""
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
def query_procedure(self):
|
||||
"""
|
||||
查询当前连接下所有存储过程
|
||||
:return:
|
||||
"""
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
def query_table_fields(self, table):
|
||||
"""
|
||||
查询字段
|
||||
:param table:
|
||||
:return:
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class MetadataReader(AbsMetadataReader):
|
||||
def __init__(self, datasource: DataSource):
|
||||
super().__init__(datasource)
|
||||
self.db_type = datasource.connector.db_type
|
||||
self.dao = MetadataDao(datasource.connector)
|
||||
|
||||
def query_tables(self):
|
||||
return self.dao.query_all_tables(reader_conf[self.db_type]['tables'])
|
||||
|
||||
def query_views(self):
|
||||
return self.dao.query_all_views(reader_conf[self.db_type]['views'])
|
||||
|
||||
def query_procedure(self):
|
||||
pass
|
||||
|
||||
def query_table_fields(self, table):
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
local_ds = ds_conf_param('postgresql', 'postgres', 'root@123', 'localhost', 5432, 'postgres')
|
||||
dsm = DataSourceManage(ConnFactory(local_ds))
|
||||
ds = dsm.get('db143d11741a9575fdea92ed2b39dc53')
|
||||
mtr = MetadataReader(ds)
|
||||
print(mtr.query_tables())
|
||||
print(mtr.query_views())
|
||||
@ -0,0 +1,7 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Time : 2023/4/9 8:27
|
||||
# @Author : old tom
|
||||
# @File : __init__.py.py
|
||||
# @Project : futool-tiny-datahub
|
||||
# @Desc :
|
||||
@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Time : 2023/4/9 8:27
|
||||
# @Author : old tom
|
||||
# @File : metadata_dao.py
|
||||
# @Project : futool-tiny-datahub
|
||||
# @Desc :
|
||||
|
||||
from common.fudb.connectors.connector_factory import ConnFactory
|
||||
from common.fudb.dbapis.fu_dao import BaseDao
|
||||
|
||||
|
||||
class MetadataDao(BaseDao):
|
||||
def __init__(self, connector: ConnFactory):
|
||||
super().__init__(connector)
|
||||
|
||||
def query_all_tables(self, sql):
|
||||
"""
|
||||
查询所有表
|
||||
:param sql:
|
||||
:return:
|
||||
"""
|
||||
return [t[0] for t in self.query_all(sql)]
|
||||
|
||||
def query_all_views(self, sql):
|
||||
"""
|
||||
查询所有视图
|
||||
:param sql:
|
||||
:return:
|
||||
"""
|
||||
return [v[0] for v in self.query_all(sql)]
|
||||
@ -0,0 +1,7 @@
|
||||
[oracle]
|
||||
tables = select distinct table_name from user_tab_comments where table_type='TABLE'
|
||||
views = select distinct table_name from user_tab_comments where table_type='VIEW'
|
||||
|
||||
[postgresql]
|
||||
tables = SELECT distinct table_name FROM information_schema.tables WHERE table_schema = 'public' ORDER BY table_name
|
||||
views = select distinct table_name from information_schema.views WHERE table_schema = 'public' ORDER BY table_name
|
||||
@ -0,0 +1,7 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Time : 2023/4/6 9:33
|
||||
# @Author : old tom
|
||||
# @File : __init__.py.py
|
||||
# @Project : futool-tiny-datahub
|
||||
# @Desc : 表、字段依赖关系
|
||||
@ -0,0 +1,15 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Time : 2023/4/9 9:18
|
||||
# @Author : old tom
|
||||
# @File : relation_analyze.py
|
||||
# @Project : futool-tiny-datahub
|
||||
# @Desc :
|
||||
|
||||
class MetadataRelationAnalyzer(object):
|
||||
"""
|
||||
元数据关系分析
|
||||
"""
|
||||
|
||||
def __init__(self, source_id):
|
||||
pass
|
||||
@ -0,0 +1,3 @@
|
||||
20230408 11:11:37 - MainProcess | MainThread | test_log.test:16 - INFO -fefefe
|
||||
20230408 11:11:37 - MainProcess | MainThread | test_log.test:17 - ERROR -223232
|
||||
20230408 11:11:37 - MainProcess | MainThread | test_log.test:18 - WARNING -999
|
||||
Loading…
Reference in new issue