diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..c398b0d --- /dev/null +++ b/Pipfile @@ -0,0 +1,11 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] + +[dev-packages] + +[requires] +python_version = "3.10" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..eb6410c --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,20 @@ +{ + "_meta": { + "hash": { + "sha256": "fedbd2ab7afd84cf16f128af0619749267b62277b4cb6989ef16d4bef6e4eef2" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.10" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": {}, + "develop": {} +} diff --git a/db/__init__.py b/db/__init__.py new file mode 100644 index 0000000..0957f37 --- /dev/null +++ b/db/__init__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2023/7/31 16:21 +# @Author : old-tom +# @File : __init__.py +# @Project : jtxy_data_transfer +# @Desc : diff --git a/db/config/__init__.py b/db/config/__init__.py new file mode 100644 index 0000000..b2970a6 --- /dev/null +++ b/db/config/__init__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2023/6/22 19:58 +# @Author : old tom +# @File : __init__.py.py +# @Project : futool-db-lite +# @Desc : diff --git a/db/config/db_config.py b/db/config/db_config.py new file mode 100644 index 0000000..09208db --- /dev/null +++ b/db/config/db_config.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2023/6/22 19:58 +# @Author : old tom +# @File : db_config.py +# @Project : futool-db-lite +# @Desc : +import os +from pydantic import BaseModel +import toml + +# 默认配置文件名 +DEFAULT_CONF_NAME = 'db-conf.toml' +path = os.path.dirname(__file__) +path = os.path.dirname(path) +# 默认配置文件位置 +DEFAULT_CONF_PATH = os.path.join(path, DEFAULT_CONF_NAME) +# 配置项名称 +DB_CONF_ITEM_NAME = ['dialect', 'host', 'port', 'user', 'passwd', 'database'] + + +class DbConfigNotFoundError(Exception): + """ + 配置不存在异常 + """ + + def __init__(self, msg): + Exception.__init__(self, msg) + + +class DbConf(BaseModel): + dialect: str + host: str + port: int + user: str + passwd: str + database: str + pool_size: int + pool_recycle: int + show_sql: bool = False + + +class DbConfigLoader(object): + """ + 数据库连接加载 + """ + + def __init__(self, db_name, conf_path): + if not os.path.isfile(conf_path): + raise DbConfigNotFoundError(f'数据配置文件{DEFAULT_CONF_NAME}不存在') + self.db_conf = self.load(db_name, conf_path) + + @staticmethod + def load(dbname, conf_path) -> DbConf: + """ + 校验并加载配置 + :return: + """ + conf = toml.load(conf_path) + return DbConf(**conf[dbname]) diff --git a/db/db-conf.toml b/db/db-conf.toml new file mode 100644 index 0000000..fb5760e --- /dev/null +++ b/db/db-conf.toml @@ -0,0 +1,19 @@ +[jtt_crm] +# 方言 +dialect = 'oracle' +# ip +host = '172.16.1.36' +# 端口 +port = 1521 +# 用户名 +user = 'jtt_crm' +# 密码 +passwd = 'jtt_crm' +# 数据库 +database = 'dbcenter' +# 连接池大小 +pool_size = 15 +# 连接超时回收(秒) +pool_recycle = 3600 +# 打印SQL +show_sql = true \ No newline at end of file diff --git a/db/executor/__init__.py b/db/executor/__init__.py new file mode 100644 index 0000000..c30d6d1 --- /dev/null +++ b/db/executor/__init__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2023/6/22 21:55 +# @Author : old tom +# @File : __init__.py.py +# @Project : futool-db-lite +# @Desc : diff --git a/db/executor/sql_executor.py b/db/executor/sql_executor.py new file mode 100644 index 0000000..302d79b --- /dev/null +++ b/db/executor/sql_executor.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2023/6/22 21:55 +# @Author : old tom +# @File : sql_executor.py +# @Project : futool-db-lite +# @Desc : 获取游标,执行SQL +from sqlalchemy import text, CursorResult +from db.transaction.connect_transaction import Transaction + + +class SQLExecutorError(Exception): + def __init__(self, msg): + Exception.__init__(self, msg) + + +class SQLExecutor(object): + def __init__(self, tx: Transaction): + self._tx = tx + self._conn = tx.get_connection() + + @staticmethod + def _format_sql(sql): + return text(sql) + + def query(self, sql) -> CursorResult: + try: + return self._conn.execute(self._format_sql(sql)) + except Exception as e: + raise SQLExecutorError(msg=f'{e}') + + def execute_update(self, sql) -> int: + try: + self._tx.begin_transaction() + rt = self._conn.execute(self._format_sql(sql)) + self._tx.commit() + return rt.rowcount + except Exception as e: + self._tx.rollback() + raise SQLExecutorError(msg=f'{e}') + + def get_connection(self): + return self._conn + + def get_transaction(self): + return self._tx diff --git a/db/session/__init__.py b/db/session/__init__.py new file mode 100644 index 0000000..5e3576c --- /dev/null +++ b/db/session/__init__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2023/6/22 19:52 +# @Author : old tom +# @File : __init__.py.py +# @Project : futool-db-lite +# @Desc : diff --git a/db/session/engine/__init__.py b/db/session/engine/__init__.py new file mode 100644 index 0000000..9a4d012 --- /dev/null +++ b/db/session/engine/__init__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2023/6/22 19:53 +# @Author : old tom +# @File : __init__.py.py +# @Project : futool-db-lite +# @Desc : diff --git a/db/session/engine/dbengine.py b/db/session/engine/dbengine.py new file mode 100644 index 0000000..0d5f567 --- /dev/null +++ b/db/session/engine/dbengine.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2023/6/22 19:54 +# @Author : old tom +# @File : dbengine.py +# @Project : futool-db-lite +# @Desc : +from urllib.parse import quote_plus as urlquote +from sqlalchemy import create_engine +from db.config.db_config import DbConfigLoader, DEFAULT_CONF_PATH +from db.session.session import SqlsessionFactory +from util.futool_lang import str_md5 + + +class DatabaseEngineError(Exception): + def __init__(self, msg): + Exception.__init__(self, msg) + + +class ObjContainer(object): + # DatabaseEngine容器 + engines = {} + # SqlsessionFactory容器 + factory = {} + + def has_obj(self, attr, k): + return k in getattr(self, attr) + + def get_obj(self, attr, k): + return getattr(self, attr)[k] + + def put_obj(self, attr, k, v): + getattr(self, attr)[k] = v + + def remove_obj(self, attr, k): + del getattr(self, attr)[k] + + def clean(self): + self.engines = {} + self.factory = {} + + +container = ObjContainer() + + +class DatabaseEngine(object): + """ + 数据库连接 + dialect+driver://username:password@host:port/database + mysql: mysql+pymysql://scott:tiger@localhost/foo + sqlserver: mssql+pymssql://scott:tiger@hostname:port/dbname + """ + DB_URL = { + 'postgresql': 'postgresql+psycopg2://{0}:{1}@{2}:{3}/{4}', + # 使用thin客户端,不需要oracle client + 'oracle': 'oracle+oracledb://{0}:{1}@{2}:{3}/?service_name={4}' + } + # 连接池容器名称,具体看ObjContainer对象 + attr = 'engines' + + def __init__(self, db_name, conf_path=DEFAULT_CONF_PATH): + """ + pool_size:连接池大小 + pool_recycle:连接回收(秒) + pool_pre_ping:测试连接,执行select 1 + 参考 https://docs.sqlalchemy.org/en/20/core/pooling.html#connection-pool-configuration + """ + loader = DbConfigLoader(db_name, conf_path) + # 数据库配置 + conf = loader.db_conf + if conf.dialect not in self.DB_URL: + raise DatabaseEngineError(msg='不支持的数据库类型') + # urlquote 处理密码中的特殊字符 + url = self.DB_URL[conf.dialect].format(conf.user, urlquote(conf.passwd), conf.host, conf.port, conf.database) + # URL生成唯一ID + self.engine_id = str_md5(url) + # 根据数据源ID判断是否新建连接池 + if container.has_obj(self.attr, self.engine_id): + self.engine = container.get_obj(self.attr, self.engine_id) + else: + engine = create_engine(url, pool_size=conf.pool_size, pool_recycle=conf.pool_recycle, pool_pre_ping=True, + echo=conf.show_sql) + container.put_obj(self.attr, self.engine_id, engine) + self.engine = engine + + def create_session_factory(self): + """ + 创建session工厂 + 相同数据库URL下全局只有一个SqlsessionFactory,除非有其他数据源 + 根据engine_id判断是否新建对象 + :return: + """ + attr = 'factory' + if container.has_obj(attr, self.engine_id): + return container.get_obj(attr, self.engine_id) + else: + factory = SqlsessionFactory(self.engine) + container.put_obj(attr, self.engine_id, factory) + return factory diff --git a/db/session/session.py b/db/session/session.py new file mode 100644 index 0000000..f20abad --- /dev/null +++ b/db/session/session.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2023/6/22 19:53 +# @Author : old tom +# @File : session.py +# @Project : futool-db-lite +# @Desc : +import threading +from sqlalchemy import Engine +from db.executor.sql_executor import SQLExecutor +from db.transaction.connect_transaction import TransactionFactory + + +class CreateSessionError(Exception): + def __init__(self, msg): + Exception.__init__(self, msg) + + +class SqlSessionCache(object): + """ + 利用thread-local 实现线程隔离及单个线程获取到的session一致 + """ + + def __init__(self, create_session_func): + self.create_session_func = create_session_func + self.cache = threading.local() + + def __call__(self): + try: + if self.has_session(): + return self.cache.value + else: + val = self.cache.value = self.create_session_func() + return val + except Exception as e: + raise CreateSessionError(msg=f'从cache获取session异常,e={e}') + + def put_session(self, session): + self.cache.value = session + + def has_session(self) -> bool: + return hasattr(self.cache, "value") + + def remove_session(self): + try: + del self.cache.value + except AttributeError: + pass + + +class SqlsessionFactory(object): + """ + 工厂模式创建sqlSession + """ + + def __init__(self, engine: Engine): + self._engine = engine + self.cache = SqlSessionCache(self.create_session) + + def open_session(self): + """ + 从缓存获取,如果没有会自动调用create_session创建 + """ + return self.cache() + + def create_session(self): + try: + conn = self._engine.connect() + tx_factory = TransactionFactory(conn) + tx = tx_factory.create_transaction() + executor = SQLExecutor(tx) + return Sqlsession(executor, self.cache) + except Exception as e: + raise CreateSessionError(msg=f'创建sqlSession异常,e={e}') + + +class Sqlsession(object): + def __init__(self, executor: SQLExecutor, session_cache: SqlSessionCache): + self.executor = executor + self.session_cache = session_cache + + def select_one(self, sql): + return self.executor.query(sql).fetchone() + + def select_many(self, sql, size): + return self.executor.query(sql).fetchmany(size) + + def select_all(self, sql): + return self.executor.query(sql).fetchall() + + def insert(self, sql): + return self.executor.execute_update(sql) + + def insert_batch(self, sql): + """ + todo implement + """ + pass + + def update(self, sql): + return self.executor.execute_update(sql) + + def delete(self, sql): + return self.executor.execute_update(sql) + + def begin_transaction(self): + """ + 开启事务 + :return: + """ + self.executor.get_transaction().begin_transaction() + + def commit(self): + """ + 提交 + :return: + """ + self.executor.get_transaction().commit() + + def rollback(self): + """ + 回滚 + :return: + """ + self.executor.get_transaction().rollback() + + def close(self): + """ + 关闭连接 + :return: + """ + self.executor.get_connection().close() + self.session_cache.remove_session() diff --git a/db/transaction/__init__.py b/db/transaction/__init__.py new file mode 100644 index 0000000..4d306ae --- /dev/null +++ b/db/transaction/__init__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2023/6/22 21:35 +# @Author : old tom +# @File : __init__.py.py +# @Project : futool-db-lite +# @Desc : diff --git a/db/transaction/connect_transaction.py b/db/transaction/connect_transaction.py new file mode 100644 index 0000000..3b52560 --- /dev/null +++ b/db/transaction/connect_transaction.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2023/6/22 21:37 +# @Author : old tom +# @File : connect_transaction.py +# @Project : futool-db-lite +# @Desc : +from sqlalchemy import Connection + + +class TransactionFactory(object): + + def __init__(self, conn: Connection): + self.conn = conn + + def create_transaction(self): + return Transaction(self.conn) + + +class Transaction(object): + def __init__(self, conn: Connection): + self.conn = conn + + def begin_transaction(self): + self.conn.begin() + + def commit(self): + self.conn.commit() + + def rollback(self): + self.conn.rollback() + + def get_connection(self): + return self.conn diff --git a/main.py b/main.py new file mode 100644 index 0000000..7ba3223 --- /dev/null +++ b/main.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2023/6/22 21:55 +# @Author : old tom +# @File : main.py +# @Project : futool-db-lite +# @Desc : 数据库测试 +from db.session.engine.dbengine import DatabaseEngine + +if __name__ == '__main__': + db_engine = DatabaseEngine('jtt_crm') + session_factory = db_engine.create_session_factory() + sess = session_factory.open_session() + print(sess.select_all('select * from JTXY_CREDIT_TASK')) + sess.close() \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1b582c4 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +annotated-types +cffi +cryptography +greenlet +oracledb +psycopg2-binary +pycparser +pydantic +pydantic_core +SQLAlchemy +toml +typing_extensions diff --git a/transfer/__init__.py b/transfer/__init__.py new file mode 100644 index 0000000..649a0f4 --- /dev/null +++ b/transfer/__init__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2023/7/31 16:25 +# @Author : old-tom +# @File : __init__.py +# @Project : jtxy_data_transfer +# @Desc : diff --git a/transfer/glsl/__init__.py b/transfer/glsl/__init__.py new file mode 100644 index 0000000..0cc7902 --- /dev/null +++ b/transfer/glsl/__init__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2023/7/31 16:26 +# @Author : old-tom +# @File : __init__.py +# @Project : jtxy_data_transfer +# @Desc : diff --git a/transfer/glsl/glsl_transfer.py b/transfer/glsl/glsl_transfer.py new file mode 100644 index 0000000..d197912 --- /dev/null +++ b/transfer/glsl/glsl_transfer.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2023/7/31 16:30 +# @Author : old-tom +# @File : glsl_transfer +# @Project : jtxy_data_transfer +# @Desc : +from transfer.transfer import BaseTransfer + + +class GLSLDataTransfer(BaseTransfer): + """ + 信用公路水路数据迁移 + """ + pass diff --git a/transfer/slyz/__init__.py b/transfer/slyz/__init__.py new file mode 100644 index 0000000..0cc7902 --- /dev/null +++ b/transfer/slyz/__init__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2023/7/31 16:26 +# @Author : old-tom +# @File : __init__.py +# @Project : jtxy_data_transfer +# @Desc : diff --git a/transfer/transfer.py b/transfer/transfer.py new file mode 100644 index 0000000..5f8348f --- /dev/null +++ b/transfer/transfer.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2023/7/31 16:28 +# @Author : old-tom +# @File : transfer +# @Project : jtxy_data_transfer +# @Desc : + +class BaseTransfer(object): + def remove_user_from_idm(self): + """ + 从IDM中移除用户 + """ + pass + + def register_to_idm(self): + """ + 注册用户 + """ + pass diff --git a/util/__init__.py b/util/__init__.py new file mode 100644 index 0000000..0957f37 --- /dev/null +++ b/util/__init__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2023/7/31 16:21 +# @Author : old-tom +# @File : __init__.py +# @Project : jtxy_data_transfer +# @Desc : diff --git a/util/futool_lang.py b/util/futool_lang.py new file mode 100644 index 0000000..f8e481d --- /dev/null +++ b/util/futool_lang.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2023/6/23 17:30 +# @Author : old tom +# @File : futool_lang.py +# @Project : futool-db-0.1 +# @Desc : +import hashlib + + +def str_md5(content): + """ + 字符串转MD5 + """ + md5 = hashlib.md5() + md5.update(content.encode(encoding='utf-8')) + return md5.hexdigest()