Compare commits

...

No commits in common. 'main' and 'master' have entirely different histories.
main ... master

2
.gitignore vendored

@ -1,4 +1,4 @@
# ---> Python
### Python template
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]

8
.idea/.gitignore vendored

@ -0,0 +1,8 @@
# Default ignored files
/shelf/
/workspace.xml
# Editor-based HTTP Client requests
/httpRequests/
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml

@ -0,0 +1 @@
futool-db-lite

@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="jdk" jdkName="Pipenv (futool-db-lite)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="PyDocumentationSettings">
<option name="format" value="PLAIN" />
<option name="myDocStringFormat" value="Plain" />
</component>
</module>

@ -0,0 +1,22 @@
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
<inspection_tool class="PyPep8NamingInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
<option name="ignoredErrors">
<list>
<option value="N813" />
</list>
</option>
</inspection_tool>
<inspection_tool class="PyUnresolvedReferencesInspection" enabled="true" level="WARNING" enabled_by_default="true">
<option name="ignoredIdentifiers">
<list>
<option value="object.transform" />
<option value="object.cluster_centers_" />
<option value="object.labels_" />
<option value="nodes" />
</list>
</option>
</inspection_tool>
</profile>
</component>

@ -0,0 +1,6 @@
<component name="InspectionProjectProfileManager">
<settings>
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
</settings>
</component>

@ -0,0 +1,4 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Pipenv (futool-db-lite)" project-jdk-type="Python SDK" />
</project>

@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/../futool-db-lite/.idea/futool-db-lite.iml" filepath="$PROJECT_DIR$/../futool-db-lite/.idea/futool-db-lite.iml" />
</modules>
</component>
</project>

@ -1,9 +0,0 @@
MIT License
Copyright (c) <year> <copyright holders>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

@ -0,0 +1,12 @@
[[source]]
url = "https://pypi.org/simple"
verify_ssl = true
name = "pypi"
[packages]
[dev-packages]
[requires]
python_version = "3.10"
python_full_version = "3.10.11"

21
Pipfile.lock generated

@ -0,0 +1,21 @@
{
"_meta": {
"hash": {
"sha256": "34c67823d0895516cdc008b7a1e9cfbb2ac3da7fcb4b6c292c9e6fcbecc606b7"
},
"pipfile-spec": 6,
"requires": {
"python_full_version": "3.10.11",
"python_version": "3.10"
},
"sources": [
{
"name": "pypi",
"url": "https://pypi.org/simple",
"verify_ssl": true
}
]
},
"default": {},
"develop": {}
}

@ -1,3 +0,0 @@
# futool-db-lite
futool-db-v1.1版本

@ -0,0 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2023/6/22 19:58
# @Author : old tom
# @File : __init__.py.py
# @Project : futool-db-lite
# @Desc :

@ -0,0 +1,60 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2023/6/22 19:58
# @Author : old tom
# @File : db_config.py
# @Project : futool-db-lite
# @Desc :
import os
from pydantic import BaseModel
import toml
# 默认配置文件名
DEFAULT_CONF_NAME = 'db-conf.toml'
path = os.path.dirname(__file__)
path = os.path.dirname(path)
# 默认配置文件位置
DEFAULT_CONF_PATH = os.path.join(path, DEFAULT_CONF_NAME)
# 配置项名称
DB_CONF_ITEM_NAME = ['dialect', 'host', 'port', 'user', 'passwd', 'database']
class DbConfigNotFoundError(Exception):
"""
配置不存在异常
"""
def __init__(self, msg):
Exception.__init__(self, msg)
class DbConf(BaseModel):
dialect: str
host: str
port: int
user: str
passwd: str
database: str
pool_size: int
pool_recycle: int
show_sql: bool = False
class DbConfigLoader(object):
"""
数据库连接加载
"""
def __init__(self, db_name, conf_path):
if not os.path.isfile(conf_path):
raise DbConfigNotFoundError(f'数据配置文件{DEFAULT_CONF_NAME}不存在')
self.db_conf = self.load(db_name, conf_path)
@staticmethod
def load(dbname, conf_path) -> DbConf:
"""
校验并加载配置
:return:
"""
conf = toml.load(conf_path)
return DbConf(**conf[dbname])

@ -0,0 +1,39 @@
[jtt_crm]
# 方言
dialect = 'oracle'
# ip
host = '172.16.1.36'
# 端口
port = 1521
# 用户名
user = 'jtt_crm'
# 密码
passwd = 'jtt_crm'
# 数据库
database = 'dbcenter'
# 连接池大小
pool_size = 15
# 连接超时回收(秒)
pool_recycle = 3600
# 打印SQL
show_sql = false
[credit]
# 方言
dialect = 'sqlserver'
# ip
host = '172.16.1.2'
# 端口
port = 1433
# 用户名
user = 'sa'
# 密码
passwd = 'atdPriME@Gy9#'
# 数据库
database = 'credit'
# 连接池大小
pool_size = 15
# 连接超时回收(秒)
pool_recycle = 3600
# 打印SQL
show_sql = true

@ -0,0 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2023/6/22 21:55
# @Author : old tom
# @File : __init__.py.py
# @Project : futool-db-lite
# @Desc :

@ -0,0 +1,75 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2023/6/22 21:55
# @Author : old tom
# @File : sql_executor.py
# @Project : futool-db-lite
# @Desc : 获取游标,执行SQL
from sqlalchemy import text, CursorResult
from transaction.connect_transaction import Transaction
class SQLExecutorError(Exception):
def __init__(self, msg):
Exception.__init__(self, msg)
class SQLExecutor(object):
def __init__(self, tx: Transaction):
self._tx = tx
self._conn = tx.get_connection()
@staticmethod
def _format_sql(sql):
return text(sql)
def query(self, sql) -> CursorResult:
"""
关于auto begin官方文档给出的解释 https://docs.sqlalchemy.org/en/20/core/connections.html#commit-as-you-go
auto begin会导致出现 can't call begin() here unless rollback() or commit() is called first.
"""
try:
return self._conn.execute(self._format_sql(sql))
except Exception as e:
raise SQLExecutorError(msg=f'{e}')
finally:
self._conn.commit()
def query_by_stream(self, sql, stream_size) -> CursorResult:
"""
流式查询
:param sql: sql
:param stream_size: 每次返回数据量
:return:
"""
try:
return self._conn.execution_options(yield_per=stream_size).execute(self._format_sql(sql))
except Exception as e:
raise SQLExecutorError(msg=f'{e}')
finally:
self._conn.commit()
def execute_update(self, sql) -> int:
try:
rt = self._conn.execute(self._format_sql(sql))
return rt.rowcount
except Exception as e:
raise SQLExecutorError(msg=f'{e}')
def execute_batch(self, sql_template, data, batch_size=1000) -> int:
"""
批量执行
conn.exec_driver_sql(
"INSERT INTO table (id, value) VALUES (:id, :value)",
[{"id":1, "value":"v1"}, {"id":2, "value":"v2"}]
)
:return:
"""
return self._conn.exec_driver_sql(sql_template, data,
execution_options={'insertmanyvalues_page_size': batch_size})
def get_connection(self):
return self._conn
def get_transaction(self):
return self._tx

@ -0,0 +1,31 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2023/6/22 21:55
# @Author : old tom
# @File : main.py
# @Project : futool-db-lite
# @Desc : 数据库测试
from session.engine.dbengine import DatabaseEngine
if __name__ == '__main__':
db_engine = DatabaseEngine('credit')
session_factory = db_engine.create_session_factory()
sess = session_factory.open_session()
print(sess.select_all('select * from hat_area'))
# import pyodbc
# # Some other example server values are
# # server = 'localhost\sqlexpress' # for a named instance
# # server = 'myserver,port' # to specify an alternate port
# server = '172.16.1.2'
# database = 'credit'
# username = 'sa'
# password = 'atdPriME@Gy9#'
# # ENCRYPT defaults to yes starting in ODBC Driver 18. It's good to always specify ENCRYPT=yes on the client side to avoid MITM attacks.
# cnxn = pyodbc.connect(DRIVER='ODBC Driver 18 for SQL Server', SERVER=server, DATABASE=database, UID=username, PWD=password)
# cursor = cnxn.cursor()
# cursor.execute("SELECT TOP 10 * FROM ICSTOCKBILL")
# row = cursor.fetchone()
# while row:
# print(row)
# row = cursor.fetchone()

@ -0,0 +1,55 @@
# 一、在m1 mac下使用pyodbc (大坑需要修改tsl协议版本不建议使用)
参考官方文档 https://github.com/mkleehammer/pyodbc/wiki
```shell
# 1.安装odbc
brew update
brew install unixodbc
# 2.安装sqlserver linux驱动
brew tap microsoft/mssql-release https://github.com/Microsoft/homebrew-mssql-release
brew update
HOMEBREW_ACCEPT_EULA=Y brew install msodbcsql18 mssql-tools18
# 3.安装pyodbc
pip install --no-binary :all: pyodbc
```
```shell
# 查看配置文件路径
odbcinst -j
# 输出
unixODBC 2.3.12
DRIVERS............: /opt/homebrew/etc/odbcinst.ini
SYSTEM DATA SOURCES: /opt/homebrew/etc/odbc.ini
FILE DATA SOURCES..: /opt/homebrew/etc/ODBCDataSources
USER DATA SOURCES..: /Users/old-tom/.odbc.ini
SQLULEN Size.......: 8
SQLLEN Size........: 8
SQLSETPOSIROW Size.: 8
# 修改odbcinst.ini
[ODBC Driver 18 for SQL Server]
Description=Microsoft ODBC Driver 18 for SQL Server
Driver=/opt/homebrew/lib/libmsodbcsql.18.dylib
```
# 二、在m1 mac下使用pymssql 连接sqlserver
参考官方文档 https://pymssql.readthedocs.io/en/stable/freetds.html
```shell
# 1.安装freetds
brew install freetds
# 2.修改配置
vim /opt/homebrew/etc/freetds.conf
在[global]下添加
port = 1443
tds version = 7.0
# 3.测试连接
tsql -H 172.16.1.2 -p 1433 -U sa -P atdPriME@Gy9# -D credit
# 返回以下情况说明连接成功
locale is "zh_CN.UTF-8"
locale charset is "UTF-8"
using default charset "UTF-8"
Setting credit as default database in login packet
```
freetds连接成功后安装pymssql
```shell
pip install pymssql
```

@ -0,0 +1,13 @@
annotated-types==0.5.0
cffi==1.15.1
cryptography==41.0.2
greenlet==2.0.2
oracledb==1.3.1
psycopg2-binary==2.9.6
pycparser==2.21
pydantic==2.0.2
pydantic_core==2.1.2
pymssql==2.2.8
SQLAlchemy==2.0.7
toml==0.10.2
typing_extensions==4.7.1

@ -0,0 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2023/6/22 19:52
# @Author : old tom
# @File : __init__.py.py
# @Project : futool-db-lite
# @Desc :

@ -0,0 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2023/6/22 19:53
# @Author : old tom
# @File : __init__.py.py
# @Project : futool-db-lite
# @Desc :

@ -0,0 +1,101 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2023/6/22 19:54
# @Author : old tom
# @File : dbengine.py
# @Project : futool-db-lite
# @Desc :
from urllib.parse import quote_plus as urlquote
from sqlalchemy import create_engine
from config.db_config import DbConfigLoader, DEFAULT_CONF_PATH
from session.session import SqlsessionFactory
from util.futool_lang import str_md5
class DatabaseEngineError(Exception):
def __init__(self, msg):
Exception.__init__(self, msg)
class ObjContainer(object):
# DatabaseEngine容器
engines = {}
# SqlsessionFactory容器
factory = {}
def has_obj(self, attr, k):
return k in getattr(self, attr)
def get_obj(self, attr, k):
return getattr(self, attr)[k]
def put_obj(self, attr, k, v):
getattr(self, attr)[k] = v
def remove_obj(self, attr, k):
del getattr(self, attr)[k]
def clean(self):
self.engines = {}
self.factory = {}
container = ObjContainer()
class DatabaseEngine(object):
"""
数据库连接
dialect+driver://username:password@host:port/database
mysql: mysql+pymysql://scott:tiger@localhost/foo
sqlserver: mssql+pymssql://scott:tiger@hostname:port/dbname
"""
DB_URL = {
'postgresql': 'postgresql+psycopg2://{0}:{1}@{2}:{3}/{4}',
# 使用thin客户端,不需要oracle client
'oracle': 'oracle+oracledb://{0}:{1}@{2}:{3}/?service_name={4}',
# pyodbc 不支持m1这里使用pymssql
'sqlserver': 'mssql+pymssql://{0}:{1}@{2}/{4}?charset=CP936'
}
# 连接池容器名称,具体看ObjContainer对象
attr = 'engines'
def __init__(self, db_name, conf_path=DEFAULT_CONF_PATH):
"""
pool_size:连接池大小
pool_recycle:连接回收
pool_pre_ping:测试连接执行select 1
参考 https://docs.sqlalchemy.org/en/20/core/pooling.html#connection-pool-configuration
"""
loader = DbConfigLoader(db_name, conf_path)
# 数据库配置
conf = loader.db_conf
if conf.dialect not in self.DB_URL:
raise DatabaseEngineError(msg='不支持的数据库类型')
# urlquote 处理密码中的特殊字符
url = self.DB_URL[conf.dialect].format(conf.user, urlquote(conf.passwd), conf.host, conf.port, conf.database)
# URL生成唯一ID
self.engine_id = str_md5(url)
# 根据数据源ID判断是否新建连接池
if container.has_obj(self.attr, self.engine_id):
self.engine = container.get_obj(self.attr, self.engine_id)
else:
engine = create_engine(url, pool_size=conf.pool_size, pool_recycle=conf.pool_recycle, pool_pre_ping=True,
echo=conf.show_sql)
container.put_obj(self.attr, self.engine_id, engine)
self.engine = engine
def create_session_factory(self):
"""
创建session工厂
相同数据库URL下全局只有一个SqlsessionFactory,除非有其他数据源
根据engine_id判断是否新建对象
:return:
"""
attr = 'factory'
if container.has_obj(attr, self.engine_id):
return container.get_obj(attr, self.engine_id)
else:
factory = SqlsessionFactory(self.engine)
container.put_obj(attr, self.engine_id, factory)
return factory

@ -0,0 +1,167 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2023/6/22 19:53
# @Author : old tom
# @File : session.py
# @Project : futool-db-lite
# @Desc :
import threading
import sqlalchemy
from sqlalchemy import Engine
from executor.sql_executor import SQLExecutor
from transaction.connect_transaction import TransactionFactory
class CreateSessionError(Exception):
def __init__(self, msg):
Exception.__init__(self, msg)
class SqlSessionCache(object):
"""
利用thread-local 实现线程隔离及单个线程获取到的session一致
"""
def __init__(self, create_session_func):
self.create_session_func = create_session_func
self.cache = threading.local()
def __call__(self):
try:
if self.has_session():
return self.cache.value
else:
val = self.cache.value = self.create_session_func()
return val
except Exception as e:
raise CreateSessionError(msg=f'从cache获取session异常,e={e}')
def put_session(self, session):
self.cache.value = session
def has_session(self) -> bool:
return hasattr(self.cache, "value")
def remove_session(self):
try:
del self.cache.value
except AttributeError:
pass
class Sqlsession(object):
def __init__(self, executor: SQLExecutor, session_cache: SqlSessionCache):
self.executor = executor
self.session_cache = session_cache
def select_one(self, sql):
return self.executor.query(sql).fetchone()
def select_many(self, sql, size):
return self.executor.query(sql).fetchmany(size)
def select_all(self, sql):
return self.executor.query(sql).fetchall()
def select_by_stream(self, sql, stream_size):
"""
流式查询,循环取数据
:param sql:
:param stream_size: 每次返回量
for partition in result.partitions():
# partition is an iterable that will be at most stream_size items
for row in partition:
print(f"{row}")
:return:
"""
return self.executor.query_by_stream(sql, stream_size)
def insert(self, sql):
return self._execute_with_tx(sql)
def insert_batch(self, sql_template, data, batch_size=1000):
"""
批量插入实现
:param data: [{"id":1, "value":"v1"}, {"id":2, "value":"v2"}] 或者 [(1,'v1'),(2,'v2')]
:param sql_template: "INSERT INTO table (id, value) VALUES (:id, :value)"
:param batch_size: 批量插入size
:return:
"""
if data and isinstance(data, list) and len(data) > 0:
# 特殊处理SQLAlchemy row对象
if isinstance(data[0], sqlalchemy.engine.row.Row):
# 转为元组
data = [tuple(x) for x in data]
return self.executor.execute_batch(sql_template, data, batch_size)
def update(self, sql):
return self._execute_with_tx(sql)
def delete(self, sql):
return self._execute_with_tx(sql)
def _execute_with_tx(self, sql):
try:
self.begin_transaction()
rt = self.executor.execute_update(sql)
self.commit()
return rt
except Exception as e:
self.rollback()
print(e)
def begin_transaction(self):
"""
开启事务
:return:
"""
self.executor.get_transaction().begin_transaction()
def commit(self):
"""
提交
:return:
"""
self.executor.get_transaction().commit()
def rollback(self):
"""
回滚
:return:
"""
self.executor.get_transaction().rollback()
def close(self):
"""
关闭连接
:return:
"""
self.executor.get_connection().close()
self.session_cache.remove_session()
class SqlsessionFactory(object):
"""
工厂模式创建sqlSession
"""
def __init__(self, engine: Engine):
self._engine = engine
self.cache = SqlSessionCache(self.create_session)
def open_session(self) -> Sqlsession:
"""
从缓存获取,如果没有会自动调用create_session创建
"""
return self.cache()
def create_session(self) -> Sqlsession:
try:
conn = self._engine.connect()
tx_factory = TransactionFactory(conn)
tx = tx_factory.create_transaction()
executor = SQLExecutor(tx)
return Sqlsession(executor, self.cache)
except Exception as e:
raise CreateSessionError(msg=f'创建sqlSession异常,e={e}')

@ -0,0 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2023/6/22 21:35
# @Author : old tom
# @File : __init__.py.py
# @Project : futool-db-lite
# @Desc :

@ -0,0 +1,34 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2023/6/22 21:37
# @Author : old tom
# @File : connect_transaction.py
# @Project : futool-db-lite
# @Desc :
from sqlalchemy import Connection
class TransactionFactory(object):
def __init__(self, conn: Connection):
self.conn = conn
def create_transaction(self):
return Transaction(self.conn)
class Transaction(object):
def __init__(self, conn: Connection):
self.conn = conn
def begin_transaction(self):
self.conn.begin()
def commit(self):
self.conn.commit()
def rollback(self):
self.conn.rollback()
def get_connection(self):
return self.conn

@ -0,0 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2023/6/23 17:29
# @Author : old tom
# @File : __init__.py.py
# @Project : futool-db-0.1
# @Desc :

@ -0,0 +1,17 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2023/6/23 17:30
# @Author : old tom
# @File : futool_lang.py
# @Project : futool-db-0.1
# @Desc :
import hashlib
def str_md5(content):
"""
字符串转MD5
"""
md5 = hashlib.md5()
md5.update(content.encode(encoding='utf-8'))
return md5.hexdigest()
Loading…
Cancel
Save