first commit

Pipfile

@@ -0,0 +1,11 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+
+[dev-packages]
+
+[requires]
+python_version = "3.10"

Pipfile.lock

@@ -0,0 +1,20 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "fedbd2ab7afd84cf16f128af0619749267b62277b4cb6989ef16d4bef6e4eef2"
+        },
+        "pipfile-spec": 6,
+        "requires": {
+            "python_version": "3.10"
+        },
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {},
+    "develop": {}
+}

db/__init__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Time    : 2023/7/31 16:21
+# @Author  : old-tom
+# @File    : __init__.py
+# @Project : jtxy_data_transfer
+# @Desc    :

db/config/__init__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Time    : 2023/6/22 19:58
+# @Author  : old tom
+# @File    : __init__.py.py
+# @Project : futool-db-lite
+# @Desc    :

db/config/db_config.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Time    : 2023/6/22 19:58
+# @Author  : old tom
+# @File    : db_config.py
+# @Project : futool-db-lite
+# @Desc    :
+import os
+from pydantic import BaseModel
+import toml
+
+# 默认配置文件名
+DEFAULT_CONF_NAME = 'db-conf.toml'
+path = os.path.dirname(__file__)
+path = os.path.dirname(path)
+# 默认配置文件位置
+DEFAULT_CONF_PATH = os.path.join(path, DEFAULT_CONF_NAME)
+# 配置项名称
+DB_CONF_ITEM_NAME = ['dialect', 'host', 'port', 'user', 'passwd', 'database']
+
+
+class DbConfigNotFoundError(Exception):
+    """
+    配置不存在异常
+    """
+
+    def __init__(self, msg):
+        Exception.__init__(self, msg)
+
+
+class DbConf(BaseModel):
+    dialect: str
+    host: str
+    port: int
+    user: str
+    passwd: str
+    database: str
+    pool_size: int
+    pool_recycle: int
+    show_sql: bool = False
+
+
+class DbConfigLoader(object):
+    """
+    数据库连接加载
+    """
+
+    def __init__(self, db_name, conf_path):
+        if not os.path.isfile(conf_path):
+            raise DbConfigNotFoundError(f'数据配置文件{DEFAULT_CONF_NAME}不存在')
+        self.db_conf = self.load(db_name, conf_path)
+
+    @staticmethod
+    def load(dbname, conf_path) -> DbConf:
+        """
+        校验并加载配置
+        :return:
+        """
+        conf = toml.load(conf_path)
+        return DbConf(**conf[dbname])

db/db-conf.toml

@@ -0,0 +1,19 @@
+[jtt_crm]
+# 方言
+dialect = 'oracle'
+# ip
+host = '172.16.1.36'
+# 端口
+port = 1521
+# 用户名
+user = 'jtt_crm'
+# 密码
+passwd = 'jtt_crm'
+# 数据库
+database = 'dbcenter'
+# 连接池大小
+pool_size = 15
+# 连接超时回收(秒)
+pool_recycle = 3600
+# 打印SQL
+show_sql = true

db/executor/__init__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Time    : 2023/6/22 21:55
+# @Author  : old tom
+# @File    : __init__.py.py
+# @Project : futool-db-lite
+# @Desc    :

db/executor/sql_executor.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Time    : 2023/6/22 21:55
+# @Author  : old tom
+# @File    : sql_executor.py
+# @Project : futool-db-lite
+# @Desc    : 获取游标,执行SQL
+from sqlalchemy import text, CursorResult
+from db.transaction.connect_transaction import Transaction
+
+
+class SQLExecutorError(Exception):
+    def __init__(self, msg):
+        Exception.__init__(self, msg)
+
+
+class SQLExecutor(object):
+    def __init__(self, tx: Transaction):
+        self._tx = tx
+        self._conn = tx.get_connection()
+
+    @staticmethod
+    def _format_sql(sql):
+        return text(sql)
+
+    def query(self, sql) -> CursorResult:
+        try:
+            return self._conn.execute(self._format_sql(sql))
+        except Exception as e:
+            raise SQLExecutorError(msg=f'{e}')
+
+    def execute_update(self, sql) -> int:
+        try:
+            self._tx.begin_transaction()
+            rt = self._conn.execute(self._format_sql(sql))
+            self._tx.commit()
+            return rt.rowcount
+        except Exception as e:
+            self._tx.rollback()
+            raise SQLExecutorError(msg=f'{e}')
+
+    def get_connection(self):
+        return self._conn
+
+    def get_transaction(self):
+        return self._tx

db/session/__init__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Time    : 2023/6/22 19:52
+# @Author  : old tom
+# @File    : __init__.py.py
+# @Project : futool-db-lite
+# @Desc    :

db/session/engine/__init__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Time    : 2023/6/22 19:53
+# @Author  : old tom
+# @File    : __init__.py.py
+# @Project : futool-db-lite
+# @Desc    :

db/session/engine/dbengine.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Time    : 2023/6/22 19:54
+# @Author  : old tom
+# @File    : dbengine.py
+# @Project : futool-db-lite
+# @Desc    :
+from urllib.parse import quote_plus as urlquote
+from sqlalchemy import create_engine
+from db.config.db_config import DbConfigLoader, DEFAULT_CONF_PATH
+from db.session.session import SqlsessionFactory
+from util.futool_lang import str_md5
+
+
+class DatabaseEngineError(Exception):
+    def __init__(self, msg):
+        Exception.__init__(self, msg)
+
+
+class ObjContainer(object):
+    # DatabaseEngine容器
+    engines = {}
+    # SqlsessionFactory容器
+    factory = {}
+
+    def has_obj(self, attr, k):
+        return k in getattr(self, attr)
+
+    def get_obj(self, attr, k):
+        return getattr(self, attr)[k]
+
+    def put_obj(self, attr, k, v):
+        getattr(self, attr)[k] = v
+
+    def remove_obj(self, attr, k):
+        del getattr(self, attr)[k]
+
+    def clean(self):
+        self.engines = {}
+        self.factory = {}
+
+
+container = ObjContainer()
+
+
+class DatabaseEngine(object):
+    """
+    数据库连接
+    dialect+driver://username:password@host:port/database
+    mysql: mysql+pymysql://scott:tiger@localhost/foo
+    sqlserver: mssql+pymssql://scott:tiger@hostname:port/dbname
+    """
+    DB_URL = {
+        'postgresql': 'postgresql+psycopg2://{0}:{1}@{2}:{3}/{4}',
+        # 使用thin客户端,不需要oracle client
+        'oracle': 'oracle+oracledb://{0}:{1}@{2}:{3}/?service_name={4}'
+    }
+    # 连接池容器名称,具体看ObjContainer对象
+    attr = 'engines'
+
+    def __init__(self, db_name, conf_path=DEFAULT_CONF_PATH):
+        """
+        pool_size:连接池大小
+        pool_recycle:连接回收（秒）
+        pool_pre_ping:测试连接，执行select 1
+        参考 https://docs.sqlalchemy.org/en/20/core/pooling.html#connection-pool-configuration
+        """
+        loader = DbConfigLoader(db_name, conf_path)
+        # 数据库配置
+        conf = loader.db_conf
+        if conf.dialect not in self.DB_URL:
+            raise DatabaseEngineError(msg='不支持的数据库类型')
+        # urlquote 处理密码中的特殊字符
+        url = self.DB_URL[conf.dialect].format(conf.user, urlquote(conf.passwd), conf.host, conf.port, conf.database)
+        # URL生成唯一ID
+        self.engine_id = str_md5(url)
+        # 根据数据源ID判断是否新建连接池
+        if container.has_obj(self.attr, self.engine_id):
+            self.engine = container.get_obj(self.attr, self.engine_id)
+        else:
+            engine = create_engine(url, pool_size=conf.pool_size, pool_recycle=conf.pool_recycle, pool_pre_ping=True,
+                                   echo=conf.show_sql)
+            container.put_obj(self.attr, self.engine_id, engine)
+            self.engine = engine
+
+    def create_session_factory(self):
+        """
+        创建session工厂
+        相同数据库URL下全局只有一个SqlsessionFactory,除非有其他数据源
+        根据engine_id判断是否新建对象
+        :return:
+        """
+        attr = 'factory'
+        if container.has_obj(attr, self.engine_id):
+            return container.get_obj(attr, self.engine_id)
+        else:
+            factory = SqlsessionFactory(self.engine)
+            container.put_obj(attr, self.engine_id, factory)
+        return factory

db/session/session.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Time    : 2023/6/22 19:53
+# @Author  : old tom
+# @File    : session.py
+# @Project : futool-db-lite
+# @Desc    :
+import threading
+from sqlalchemy import Engine
+from db.executor.sql_executor import SQLExecutor
+from db.transaction.connect_transaction import TransactionFactory
+
+
+class CreateSessionError(Exception):
+    def __init__(self, msg):
+        Exception.__init__(self, msg)
+
+
+class SqlSessionCache(object):
+    """
+    利用thread-local 实现线程隔离及单个线程获取到的session一致
+    """
+
+    def __init__(self, create_session_func):
+        self.create_session_func = create_session_func
+        self.cache = threading.local()
+
+    def __call__(self):
+        try:
+            if self.has_session():
+                return self.cache.value
+            else:
+                val = self.cache.value = self.create_session_func()
+                return val
+        except Exception as e:
+            raise CreateSessionError(msg=f'从cache获取session异常,e={e}')
+
+    def put_session(self, session):
+        self.cache.value = session
+
+    def has_session(self) -> bool:
+        return hasattr(self.cache, "value")
+
+    def remove_session(self):
+        try:
+            del self.cache.value
+        except AttributeError:
+            pass
+
+
+class SqlsessionFactory(object):
+    """
+    工厂模式创建sqlSession
+    """
+
+    def __init__(self, engine: Engine):
+        self._engine = engine
+        self.cache = SqlSessionCache(self.create_session)
+
+    def open_session(self):
+        """
+        从缓存获取,如果没有会自动调用create_session创建
+        """
+        return self.cache()
+
+    def create_session(self):
+        try:
+            conn = self._engine.connect()
+            tx_factory = TransactionFactory(conn)
+            tx = tx_factory.create_transaction()
+            executor = SQLExecutor(tx)
+            return Sqlsession(executor, self.cache)
+        except Exception as e:
+            raise CreateSessionError(msg=f'创建sqlSession异常,e={e}')
+
+
+class Sqlsession(object):
+    def __init__(self, executor: SQLExecutor, session_cache: SqlSessionCache):
+        self.executor = executor
+        self.session_cache = session_cache
+
+    def select_one(self, sql):
+        return self.executor.query(sql).fetchone()
+
+    def select_many(self, sql, size):
+        return self.executor.query(sql).fetchmany(size)
+
+    def select_all(self, sql):
+        return self.executor.query(sql).fetchall()
+
+    def insert(self, sql):
+        return self.executor.execute_update(sql)
+
+    def insert_batch(self, sql):
+        """
+        todo implement
+        """
+        pass
+
+    def update(self, sql):
+        return self.executor.execute_update(sql)
+
+    def delete(self, sql):
+        return self.executor.execute_update(sql)
+
+    def begin_transaction(self):
+        """
+        开启事务
+        :return:
+        """
+        self.executor.get_transaction().begin_transaction()
+
+    def commit(self):
+        """
+        提交
+        :return:
+        """
+        self.executor.get_transaction().commit()
+
+    def rollback(self):
+        """
+        回滚
+        :return:
+        """
+        self.executor.get_transaction().rollback()
+
+    def close(self):
+        """
+        关闭连接
+        :return:
+        """
+        self.executor.get_connection().close()
+        self.session_cache.remove_session()

db/transaction/__init__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Time    : 2023/6/22 21:35
+# @Author  : old tom
+# @File    : __init__.py.py
+# @Project : futool-db-lite
+# @Desc    :

db/transaction/connect_transaction.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Time    : 2023/6/22 21:37
+# @Author  : old tom
+# @File    : connect_transaction.py
+# @Project : futool-db-lite
+# @Desc    :
+from sqlalchemy import Connection
+
+
+class TransactionFactory(object):
+
+    def __init__(self, conn: Connection):
+        self.conn = conn
+
+    def create_transaction(self):
+        return Transaction(self.conn)
+
+
+class Transaction(object):
+    def __init__(self, conn: Connection):
+        self.conn = conn
+
+    def begin_transaction(self):
+        self.conn.begin()
+
+    def commit(self):
+        self.conn.commit()
+
+    def rollback(self):
+        self.conn.rollback()
+
+    def get_connection(self):
+        return self.conn

main.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Time    : 2023/6/22 21:55
+# @Author  : old tom
+# @File    : main.py
+# @Project : futool-db-lite
+# @Desc    : 数据库测试
+from db.session.engine.dbengine import DatabaseEngine
+
+if __name__ == '__main__':
+    db_engine = DatabaseEngine('jtt_crm')
+    session_factory = db_engine.create_session_factory()
+    sess = session_factory.open_session()
+    print(sess.select_all('select * from JTXY_CREDIT_TASK'))
+    sess.close()

requirements.txt

@@ -0,0 +1,12 @@
+annotated-types
+cffi
+cryptography
+greenlet
+oracledb
+psycopg2-binary
+pycparser
+pydantic
+pydantic_core
+SQLAlchemy
+toml
+typing_extensions

transfer/__init__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Time    : 2023/7/31 16:25
+# @Author  : old-tom
+# @File    : __init__.py
+# @Project : jtxy_data_transfer
+# @Desc    :

transfer/glsl/__init__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Time    : 2023/7/31 16:26
+# @Author  : old-tom
+# @File    : __init__.py
+# @Project : jtxy_data_transfer
+# @Desc    :

transfer/glsl/glsl_transfer.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Time    : 2023/7/31 16:30
+# @Author  : old-tom
+# @File    : glsl_transfer
+# @Project : jtxy_data_transfer
+# @Desc    :
+from transfer.transfer import BaseTransfer
+
+
+class GLSLDataTransfer(BaseTransfer):
+    """
+    信用公路水路数据迁移
+    """
+    pass

transfer/slyz/__init__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Time    : 2023/7/31 16:26
+# @Author  : old-tom
+# @File    : __init__.py
+# @Project : jtxy_data_transfer
+# @Desc    :

transfer/transfer.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Time    : 2023/7/31 16:28
+# @Author  : old-tom
+# @File    : transfer
+# @Project : jtxy_data_transfer
+# @Desc    :
+
+class BaseTransfer(object):
+    def remove_user_from_idm(self):
+        """
+        从IDM中移除用户
+        """
+        pass
+
+    def register_to_idm(self):
+        """
+        注册用户
+        """
+        pass

util/__init__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Time    : 2023/7/31 16:21
+# @Author  : old-tom
+# @File    : __init__.py
+# @Project : jtxy_data_transfer
+# @Desc    :

util/futool_lang.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Time    : 2023/6/23 17:30
+# @Author  : old tom
+# @File    : futool_lang.py
+# @Project : futool-db-0.1
+# @Desc    :
+import hashlib
+
+
+def str_md5(content):
+    """
+    字符串转MD5
+    """
+    md5 = hashlib.md5()
+    md5.update(content.encode(encoding='utf-8'))
+    return md5.hexdigest()