You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
85 lines
2.5 KiB
85 lines
2.5 KiB
#!/usr/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
# @Time : 2023/4/11 19:08
|
|
# @Author : old tom
|
|
# @File : scan_task.py
|
|
# @Project : futool-tiny-datahub
|
|
# @Desc : 扫描任务,通过调度扫描数据源-->获取数据源-->读取元数据并同步
|
|
from datahub.local_db_conf import local_conn
|
|
from datahub.metadata.metadat_scan import MetadataScanner
|
|
from datahub.scheduletask.scandao.scan_task_dao import ScanTaskDao
|
|
from datahub.scheduletask.task_executor import ScheduleExecutor
|
|
from datahub.scheduletask.schedule import CronExpTrigger
|
|
from datahub.log_conf import log
|
|
|
|
|
|
class ScanTaskManage(object):
|
|
def __init__(self):
|
|
self.dao = ScanTaskDao(local_conn)
|
|
|
|
def add_task(self, source_id, cron='0 0 0 1/1 * ?'):
|
|
"""
|
|
添加任务
|
|
:param source_id: 数据源ID
|
|
:param cron: cron表达式,默认一天一次
|
|
:return:
|
|
"""
|
|
if self.dao.exist_by_id(source_id):
|
|
return False, f'[{source_id}] all ready exist'
|
|
return self.dao.add_task(source_id, cron), 'add success'
|
|
|
|
def switch_task(self, source_id, enable='N'):
|
|
"""
|
|
开关任务
|
|
:param source_id: 源ID
|
|
:param enable: 是否开启 Y开|N关
|
|
:return:
|
|
"""
|
|
return self.dao.switch_task(source_id, enable)
|
|
|
|
def edit_cron(self, source_id, cron):
|
|
"""
|
|
修改任务CRON表达式
|
|
:param source_id: 源ID
|
|
:param cron: cron表达式
|
|
:return:
|
|
"""
|
|
return self.dao.edit_task_cron(source_id, cron)
|
|
|
|
def query_task(self, enable=None):
|
|
"""
|
|
任务查询
|
|
:param enable:
|
|
:return:
|
|
"""
|
|
return self.dao.query_all_task(enable)
|
|
|
|
def query_task_by_id(self, source_id):
|
|
"""
|
|
根据ID 查任务
|
|
:return:
|
|
"""
|
|
return self.dao.query_task_by_id(source_id)
|
|
|
|
|
|
class ScanTaskRunner(object):
|
|
"""
|
|
扫描任务执行器
|
|
"""
|
|
|
|
def __init__(self):
|
|
self.dao = ScanTaskDao(local_conn)
|
|
self.executor = ScheduleExecutor()
|
|
self.scanner = MetadataScanner()
|
|
|
|
def run(self):
|
|
"""
|
|
获取扫描配置提交到执行器
|
|
:return:
|
|
"""
|
|
enable_task = self.dao.query_all_task('Y')
|
|
if enable_task:
|
|
for task in enable_task:
|
|
self.executor.submit(task[0], CronExpTrigger.parse_crontab(task[1]), self.scanner.scan_metadata)
|
|
log.info(f'datasource scan task [{task[0]}] submit success')
|