diff --git a/myapp/__init__.py b/myapp/__init__.py index 87185213a..db52a805c 100644 --- a/myapp/__init__.py +++ b/myapp/__init__.py @@ -1,14 +1,12 @@ # 避免多进程同时启动对系统cpu负载过高 -import time,random # time.sleep(random.randint(1,10)) -from flask import g from copy import deepcopy import json import logging from logging.handlers import TimedRotatingFileHandler -from flask import redirect, g, flash, request, session, abort +from flask import g, abort import os -from flask import render_template,redirect +from flask import render_template from flask import Flask, redirect from flask_appbuilder import AppBuilder, IndexView, SQLA from flask_appbuilder.baseviews import expose @@ -19,11 +17,9 @@ from werkzeug.middleware.proxy_fix import ProxyFix import wtforms_json -from myapp import config from myapp.security import MyappSecurityManager from myapp.utils.core import pessimistic_connection_handling, setup_cache from myapp.utils.log import DBEventLogger -import pysnooper wtforms_json.init() # 在这个文件里面只创建app，不要做view层面的事情。 @@ -273,7 +269,7 @@ def check_login(): return if not g.user or not g.user.get_id(): - redirect_url = appbuilder.get_url_for_login # +"?login_url="+request.url + appbuilder.get_url_for_login # +"?login_url="+request.url # return redirect(redirect_url) abort(401) @@ -292,7 +288,7 @@ def myapp_after_request(resp): - except Exception as e: + except Exception: # print(e) resp.set_cookie('myapp_username', 'myapp') # resp.delete_cookie('id') @@ -327,9 +323,6 @@ def page_not_found(e): app.logger.handlers = gunicorn_logger.handlers app.logger.setLevel(gunicorn_logger.level) -# 引入视图 -from myapp import views - # def can_access(menuitem): # print(menuitem.name,menuitem.label) diff --git a/myapp/check_tables.py b/myapp/check_tables.py index 7c34986f2..f698340bd 100755 --- a/myapp/check_tables.py +++ b/myapp/check_tables.py @@ -1,6 +1,5 @@ #!/usr/bin/env python import os -import pysnooper # @pysnooper.snoop() def check_tables(): SQLALCHEMY_DATABASE_URI = os.getenv('MYSQL_SERVICE','') diff --git a/myapp/cli.py b/myapp/cli.py index 95ac7c652..5676d48ea 100755 --- a/myapp/cli.py +++ b/myapp/cli.py @@ -1,24 +1,14 @@ #!/usr/bin/env python from datetime import datetime -import logging -from subprocess import Popen -from sys import stdout - -import click -from colorama import Fore, Style -from flask import g import json from myapp import app, appbuilder, db, security_manager -from myapp.models.model_notebook import Notebook from myapp.models.model_team import Project,Project_User from myapp.models.model_job import Repository,Images,Job_Template,Pipeline,Task from myapp.models.model_dataset import Dataset from myapp.models.model_serving import Service,InferenceService from myapp.models.model_train_model import Training_Model -import csv import uuid conf = app.config -import requests def create_app(script_info=None): return app @@ -27,8 +17,6 @@ def create_app(script_info=None): def make_shell_context(): return dict(app=app, db=db) -import pysnooper - # https://dormousehole.readthedocs.io/en/latest/cli.html @app.cli.command('init') # @pysnooper.snoop() diff --git a/myapp/create_db.py b/myapp/create_db.py index 4982d7f66..87df495c2 100755 --- a/myapp/create_db.py +++ b/myapp/create_db.py @@ -1,6 +1,5 @@ #!/usr/bin/env python import os -import pysnooper # @pysnooper.snoop() def init_db(): SQLALCHEMY_DATABASE_URI = os.getenv('MYSQL_SERVICE','') diff --git a/myapp/forms.py b/myapp/forms.py index 393537443..cfdeb6cc0 100644 --- a/myapp/forms.py +++ b/myapp/forms.py @@ -1,14 +1,12 @@ """Contains the logic to create cohesive forms on the explore view""" -from flask_appbuilder.fieldwidgets import BS3TextFieldWidget from wtforms import Field -from flask_appbuilder.fieldwidgets import BS3TextFieldWidget,BS3PasswordFieldWidget,DatePickerWidget,DateTimePickerWidget,Select2ManyWidget,Select2Widget -from wtforms import widgets +from flask_appbuilder.fieldwidgets import BS3TextFieldWidget from myapp import app conf = app.config -from wtforms.validators import DataRequired, Length, NumberRange, Optional,Regexp,ValidationError +from wtforms.validators import ValidationError # from myapp.models.base import MyappModelBase # model_base=MyappModelBase() # @@ -90,16 +88,9 @@ def filter_not_empty_values(value): -import pysnooper,datetime,time,json +import json from wtforms.widgets.core import HTMLString,html_params - -try: - from html import escape -except ImportError: - from cgi import escape -from wtforms.compat import text_type, iteritems - - +from wtforms.compat import text_type class MyCodeArea(object): def __init__(self, code=''): diff --git a/myapp/jinja_context.py b/myapp/jinja_context.py index 7f0e8d014..9d114fed7 100644 --- a/myapp/jinja_context.py +++ b/myapp/jinja_context.py @@ -4,7 +4,7 @@ import json import random import time -from typing import Any, List, Optional, Tuple +from typing import Any, List, Optional import uuid from dateutil.relativedelta import relativedelta diff --git a/myapp/models/__init__.py b/myapp/models/__init__.py index 5421a5f2f..8b1378917 100644 --- a/myapp/models/__init__.py +++ b/myapp/models/__init__.py @@ -1,4 +1 @@ -from . import log -from . import model_job -from . import user_attributes # noqa diff --git a/myapp/models/base.py b/myapp/models/base.py index cef55025d..56a2e0c87 100644 --- a/myapp/models/base.py +++ b/myapp/models/base.py @@ -1,4 +1,3 @@ -from flask_appbuilder import Model from flask_babel import lazy_gettext as _ import re from myapp.utils import core diff --git a/myapp/models/helpers.py b/myapp/models/helpers.py index f40f84543..6655a3ac6 100644 --- a/myapp/models/helpers.py +++ b/myapp/models/helpers.py @@ -14,7 +14,6 @@ from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.orm.exc import MultipleResultsFound import yaml -import pysnooper from myapp.utils.core import QueryStatus diff --git a/myapp/models/log.py b/myapp/models/log.py index b939a884a..613cefe9a 100755 --- a/myapp/models/log.py +++ b/myapp/models/log.py @@ -3,27 +3,17 @@ from datetime import datetime from flask_appbuilder import Model from sqlalchemy import ( - Boolean, Column, - create_engine, DateTime, ForeignKey, Integer, - MetaData, String, - Table, Text, ) -from sqlalchemy.orm import relationship, sessionmaker, subqueryload -from myapp import ( - app, - appbuilder, - conf, - db -) +from sqlalchemy.orm import relationship from myapp.models.base import MyappModelBase -from myapp import app, db, is_feature_enabled, security_manager +from myapp import app from myapp.security import MyUser diff --git a/myapp/models/model_aihub.py b/myapp/models/model_aihub.py index 22bb32a0d..8cd2b8011 100644 --- a/myapp/models/model_aihub.py +++ b/myapp/models/model_aihub.py @@ -1,27 +1,13 @@ from flask_appbuilder import Model -from sqlalchemy import ( - Boolean, - Column, - create_engine, - DateTime, - ForeignKey, - Integer, - MetaData, - String, - Table, - Text, - Enum, -) +from sqlalchemy import Text -from myapp import app,db +from myapp import app from myapp.models.helpers import ImportMixin -# from myapp.models.base import MyappModel -from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime +from sqlalchemy import Column, Integer, String from flask import Markup from myapp.models.base import MyappModelBase -import datetime metadata = Model.metadata conf = app.config diff --git a/myapp/models/model_dataset.py b/myapp/models/model_dataset.py index fd42ccc0b..5e08557fe 100644 --- a/myapp/models/model_dataset.py +++ b/myapp/models/model_dataset.py @@ -1,30 +1,12 @@ from flask_appbuilder import Model -from sqlalchemy import ( - Boolean, - Column, - create_engine, - DateTime, - ForeignKey, - Integer, - MetaData, - String, - Table, - Text, - Enum, -) - -from myapp.models.helpers import AuditMixinNullable, ImportMixin -from flask import escape, g, Markup, request -from .model_team import Project -from myapp import app,db -from myapp.models.helpers import ImportMixin -# from myapp.models.base import MyappModel -from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime -from flask_appbuilder.models.decorators import renders +from sqlalchemy import Text + +from myapp.models.helpers import AuditMixinNullable +from myapp import app +from sqlalchemy import Column, Integer, String from flask import Markup from myapp.models.base import MyappModelBase -import datetime metadata = Model.metadata conf = app.config diff --git a/myapp/models/model_dimension.py b/myapp/models/model_dimension.py index 4f66a2a21..d0d480bd6 100644 --- a/myapp/models/model_dimension.py +++ b/myapp/models/model_dimension.py @@ -1,32 +1,13 @@ from flask_appbuilder import Model -from sqlalchemy import Column, Integer, String, ForeignKey,Float -from sqlalchemy.orm import relationship -import datetime,time,json -from sqlalchemy import ( - Boolean, - Column, - create_engine, - DateTime, - ForeignKey, - Integer, - MetaData, - String, - Table, - Text, - Enum, -) -from sqlalchemy import String,Column,Integer,ForeignKey,UniqueConstraint,BigInteger,TIMESTAMP +from sqlalchemy import Text +from sqlalchemy import UniqueConstraint -from flask import escape, g, Markup, request -from .model_team import Project -from myapp import app,db +from flask import g +from myapp import app from myapp.models.helpers import ImportMixin -# from myapp.models.base import MyappModel -from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime -from flask_appbuilder.models.decorators import renders +from sqlalchemy import Column, Integer, String from flask import Markup from myapp.models.base import MyappModelBase -import datetime metadata = Model.metadata conf = app.config @@ -60,7 +41,7 @@ def table_html(self): @property def operate_html(self): - url=f''' + url=''' 创建远程表 | 建外表示例 | 清空表记录 '''%(self.id,self.id,self.id) return Markup(url) diff --git a/myapp/models/model_docker.py b/myapp/models/model_docker.py index ef032fb3c..32b0ecba0 100644 --- a/myapp/models/model_docker.py +++ b/myapp/models/model_docker.py @@ -1,29 +1,15 @@ from flask_appbuilder import Model -from sqlalchemy import Column, Integer, String, ForeignKey,Float from sqlalchemy.orm import relationship -import datetime,time,json from sqlalchemy import ( Boolean, - Column, - create_engine, - DateTime, - ForeignKey, - Integer, - MetaData, - String, - Table, - Text, - Enum, + Text ) -import pysnooper from myapp.models.base import MyappModelBase -from myapp.models.helpers import AuditMixinNullable, ImportMixin -from flask import escape, g, Markup, request -from myapp import app,db -from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime +from myapp.models.helpers import AuditMixinNullable +from myapp import app +from sqlalchemy import Column, Integer, String, ForeignKey from flask import Markup -import datetime metadata = Model.metadata conf = app.config # from myapp.utils.py.py_k8s import K8s diff --git a/myapp/models/model_etl_pipeline.py b/myapp/models/model_etl_pipeline.py index 0fa91b50d..99253ca5b 100644 --- a/myapp/models/model_etl_pipeline.py +++ b/myapp/models/model_etl_pipeline.py @@ -1,32 +1,16 @@ from flask_appbuilder import Model -from sqlalchemy import Column, Integer, String, ForeignKey,Float from sqlalchemy.orm import relationship -import datetime,time,json -from sqlalchemy import ( - Boolean, - Column, - create_engine, - DateTime, - ForeignKey, - Integer, - MetaData, - String, - Table, - Text, - Enum, -) - -from myapp.models.helpers import AuditMixinNullable, ImportMixin - -from myapp import app,db +from sqlalchemy import Text + +from myapp.models.helpers import AuditMixinNullable + +from myapp import app from myapp.models.helpers import ImportMixin -# from myapp.models.base import MyappModel -from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime +from sqlalchemy import Column, Integer, String, ForeignKey from flask_appbuilder.models.decorators import renders from flask import Markup from myapp.models.base import MyappModelBase -import datetime metadata = Model.metadata conf = app.config diff --git a/myapp/models/model_job.py b/myapp/models/model_job.py index e5023098f..f76a04311 100644 --- a/myapp/models/model_job.py +++ b/myapp/models/model_job.py @@ -1,31 +1,21 @@ from flask_appbuilder import Model -from sqlalchemy import Column, Integer, String, ForeignKey,Float from sqlalchemy.orm import relationship -import datetime,time,json +import json from sqlalchemy import ( Boolean, - Column, - create_engine, - DateTime, - ForeignKey, - Integer, - MetaData, - String, - Table, Text, Enum, ) import numpy import random import copy -import logging -from myapp.models.helpers import AuditMixinNullable, ImportMixin +from myapp.models.helpers import AuditMixinNullable from myapp import app,db from myapp.models.helpers import ImportMixin -from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime +from sqlalchemy import Column, Integer, String, ForeignKey, DateTime from flask_appbuilder.models.decorators import renders from flask import Markup from myapp.models.base import MyappModelBase @@ -33,10 +23,7 @@ metadata = Model.metadata conf = app.config from myapp.utils import core -import re from myapp.utils.py import py_k8s -import pysnooper - class Repository(Model,AuditMixinNullable,MyappModelBase): __tablename__ = 'repository' @@ -225,7 +212,7 @@ def log(self): pipeline_url = "/pipeline_modelview/web/log/%s"%self.id return Markup(f'日志') else: - return Markup(f'日志') + return Markup('日志') @property @@ -556,7 +543,7 @@ def monitoring_html(self): monitoring = json.loads(self.monitoring) monitoring['link']=self.pipeline.project.cluster.get('GRAFANA_HOST','').strip('/')+conf.get('GRAFANA_TASK_PATH')+monitoring.get('pod_name','') return Markup('
' + json.dumps(monitoring,ensure_ascii=False,indent=4) + '
') - except Exception as e: + except Exception: return Markup('
 暂无
') @property @@ -635,9 +622,8 @@ def log(self): pipeline_url = self.pipeline.project.cluster.get('PIPELINE_URL')+ "runs/details/" +str(self.run_id) return Markup(f'日志') else: - return Markup(f'日志') + return Markup('日志') -import sqlalchemy as sa class Crd: # __tablename__ = "crd" id = Column(Integer, primary_key=True) @@ -808,7 +794,7 @@ def pipeline_url(self): except Exception as e: print(e) - return Markup(f'未知') + return Markup('未知') @property def pipeline(self): @@ -850,7 +836,7 @@ def log(self): except Exception as e: print(e) - return Markup(f'日志') + return Markup('日志') @property def stop(self): @@ -871,7 +857,7 @@ def pipeline(self): return Markup(f'{pipeline.describe}') except Exception as e: print(e) - return Markup(f'未知') + return Markup('未知') @property def run_instance(self): @@ -883,7 +869,7 @@ def run_instance(self): return Markup(f'运行实例') except Exception as e: print(e) - return Markup(f'未知') + return Markup('未知') diff --git a/myapp/models/model_metadata.py b/myapp/models/model_metadata.py index 5df385049..53002baa9 100644 --- a/myapp/models/model_metadata.py +++ b/myapp/models/model_metadata.py @@ -1,31 +1,13 @@ from flask_appbuilder import Model -from sqlalchemy import Column, Integer, String, ForeignKey,Float -from sqlalchemy.orm import relationship -import datetime,time,json -from sqlalchemy import ( - Boolean, - Column, - create_engine, - DateTime, - ForeignKey, - Integer, - MetaData, - String, - Table, - Text, - Enum, -) -from sqlalchemy import String,Column,Integer,ForeignKey,UniqueConstraint,BigInteger,TIMESTAMP +from sqlalchemy import Float +from sqlalchemy import Text +from sqlalchemy import BigInteger -from myapp import app,db +from myapp import app from myapp.models.helpers import ImportMixin -# from myapp.models.base import MyappModel -from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime -from flask_appbuilder.models.decorators import renders -from flask import Markup +from sqlalchemy import Column, Integer, String, Date from myapp.models.base import MyappModelBase -import datetime metadata = Model.metadata conf = app.config diff --git a/myapp/models/model_metadata_metric.py b/myapp/models/model_metadata_metric.py index 121fabce1..45b7a84ec 100644 --- a/myapp/models/model_metadata_metric.py +++ b/myapp/models/model_metadata_metric.py @@ -1,32 +1,15 @@ from flask_appbuilder import Model -from sqlalchemy import Column, Integer, String, ForeignKey,Float -from sqlalchemy.orm import relationship -import datetime,time,json from sqlalchemy import ( Boolean, - Column, - create_engine, - DateTime, - ForeignKey, - Integer, - MetaData, - String, - Table, Text, - Enum, ) -from sqlalchemy import String,Column,Integer,ForeignKey,UniqueConstraint,BigInteger,TIMESTAMP -from myapp.models.helpers import AuditMixinNullable, ImportMixin -from flask import escape, g, Markup, request -from .model_team import Project -from myapp import app,db +from myapp.models.helpers import AuditMixinNullable +from myapp import app from myapp.models.helpers import ImportMixin -# from myapp.models.base import MyappModel -from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime +from sqlalchemy import Column, Integer, String from myapp.models.base import MyappModelBase -import datetime metadata = Model.metadata conf = app.config diff --git a/myapp/models/model_nni.py b/myapp/models/model_nni.py index f2d5f74d2..9428c5e56 100644 --- a/myapp/models/model_nni.py +++ b/myapp/models/model_nni.py @@ -1,32 +1,18 @@ from flask_appbuilder import Model -from sqlalchemy import Column, Integer, String, ForeignKey,Float +from sqlalchemy import Float from sqlalchemy.orm import relationship -import datetime,time,json from sqlalchemy import ( - Boolean, - Column, - create_engine, - DateTime, - ForeignKey, - Integer, - MetaData, - String, - Table, Text, Enum, ) -from myapp.utils import core -import re from myapp.models.base import MyappModelBase -from myapp.models.helpers import AuditMixinNullable, ImportMixin -from flask import escape, g, Markup, request -from myapp import app,db -from myapp.models.helpers import ImportMixin +from myapp.models.helpers import AuditMixinNullable +from flask import request +from myapp import app -from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime +from sqlalchemy import Column, Integer, String, ForeignKey from flask_appbuilder.models.decorators import renders from flask import Markup -import datetime metadata = Model.metadata conf = app.config diff --git a/myapp/models/model_notebook.py b/myapp/models/model_notebook.py index a7ff328da..7ff13d0d3 100644 --- a/myapp/models/model_notebook.py +++ b/myapp/models/model_notebook.py @@ -1,29 +1,16 @@ from flask_appbuilder import Model -from sqlalchemy import Column, Integer, String, ForeignKey,Float from sqlalchemy.orm import relationship -import datetime,time,json +import json from sqlalchemy import ( - Boolean, - Column, - create_engine, - DateTime, - ForeignKey, - Integer, - MetaData, - String, - Table, Text, Enum, ) -import pysnooper from myapp.models.base import MyappModelBase -from myapp.models.helpers import AuditMixinNullable, ImportMixin -from flask import escape, g, Markup, request -from myapp import app,db -from myapp.models.helpers import ImportMixin +from myapp.models.helpers import AuditMixinNullable +from flask import g, request +from myapp import app -from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime -from flask_appbuilder.models.decorators import renders +from sqlalchemy import Column, Integer, String, ForeignKey from flask import Markup import datetime metadata = Model.metadata @@ -115,7 +102,7 @@ def status(self): return url return status - except Exception as e: + except Exception: # print(e) return "unknown" diff --git a/myapp/models/model_service_pipeline.py b/myapp/models/model_service_pipeline.py index 26c3ea00b..22bdc0775 100644 --- a/myapp/models/model_service_pipeline.py +++ b/myapp/models/model_service_pipeline.py @@ -1,34 +1,22 @@ from flask_appbuilder import Model -from sqlalchemy import Column, Integer, String, ForeignKey,Float from sqlalchemy.orm import relationship -import datetime,time,json +import json from sqlalchemy import ( - Boolean, - Column, - create_engine, - DateTime, - ForeignKey, - Integer, - MetaData, - String, - Table, Text, Enum, ) import numpy import random import copy -import logging -from myapp.models.helpers import AuditMixinNullable, ImportMixin +from myapp.models.helpers import AuditMixinNullable from myapp import app,db from myapp.models.helpers import ImportMixin -from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime +from sqlalchemy import Column, Integer, String, ForeignKey from flask_appbuilder.models.decorators import renders from flask import Markup from myapp.models.base import MyappModelBase -import datetime metadata = Model.metadata conf = app.config @@ -87,7 +75,7 @@ def log(self): service_pipeline_url = "/service_pipeline_modelview/web/log/%s"%self.id return Markup(f'日志') else: - return Markup(f'日志') + return Markup('日志') @property @@ -98,7 +86,7 @@ def pod(self): @property def operate_html(self): - url=self.project.cluster.get('GRAFANA_HOST','').strip('/')+conf.get('GRAFANA_SERVICE_PATH')+self.name + self.project.cluster.get('GRAFANA_HOST','').strip('/')+conf.get('GRAFANA_SERVICE_PATH')+self.name dom=f''' 部署 | pod | diff --git a/myapp/models/model_serving.py b/myapp/models/model_serving.py index f82630df9..74e1c6c9e 100644 --- a/myapp/models/model_serving.py +++ b/myapp/models/model_serving.py @@ -1,32 +1,16 @@ from flask_appbuilder import Model -from sqlalchemy import Column, Integer, String, ForeignKey,Float from sqlalchemy.orm import relationship -import datetime,time,json -from sqlalchemy import ( - Boolean, - Column, - create_engine, - DateTime, - ForeignKey, - Integer, - MetaData, - String, - Table, - Text, - Enum, -) -import pysnooper +import json +from sqlalchemy import Text from myapp.utils import core -import re -from myapp.models.helpers import AuditMixinNullable, ImportMixin -from flask import escape, g, Markup, request +from myapp.models.helpers import AuditMixinNullable +from flask import request from .model_team import Project -from myapp import app,db +from myapp import app from myapp.models.base import MyappModelBase -from myapp.models.helpers import ImportMixin -from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime +from sqlalchemy import Column, Integer, String, ForeignKey from flask import Markup import datetime @@ -230,7 +214,7 @@ def operate_html(self): if help_url: dom=f'帮助 | '+dom else: - dom = f'帮助 | ' + dom + dom = '帮助 | ' + dom return Markup(dom) @property diff --git a/myapp/models/model_team.py b/myapp/models/model_team.py index e281ecd4c..de3fb6ffe 100644 --- a/myapp/models/model_team.py +++ b/myapp/models/model_team.py @@ -1,35 +1,19 @@ from flask_appbuilder import Model -from sqlalchemy import Column, Integer, String, ForeignKey,Float -from sqlalchemy.orm import relationship -import datetime,time,json +import json from myapp import app,db from sqlalchemy import ( - Boolean, - Column, - create_engine, - DateTime, - ForeignKey, - Integer, - MetaData, - String, - Table, Text, Enum ) -from myapp.models.helpers import AuditMixinNullable, ImportMixin +from myapp.models.helpers import AuditMixinNullable from sqlalchemy.orm import backref, relationship from myapp.models.base import MyappModelBase -from myapp.models.helpers import ImportMixin -from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime from flask_appbuilder.models.decorators import renders from flask import Markup -import datetime from sqlalchemy import String,Column,Integer,ForeignKey,UniqueConstraint -from myapp.security import MyUser metadata = Model.metadata conf = app.config -import pysnooper diff --git a/myapp/models/model_train_model.py b/myapp/models/model_train_model.py index 3dd846a6d..0e5262ae0 100644 --- a/myapp/models/model_train_model.py +++ b/myapp/models/model_train_model.py @@ -1,30 +1,14 @@ from flask_appbuilder import Model -from sqlalchemy import Column, Integer, String, ForeignKey,Float from sqlalchemy.orm import relationship -import datetime,time,json -from sqlalchemy import ( - Boolean, - Column, - create_engine, - DateTime, - ForeignKey, - Integer, - MetaData, - String, - Table, - Text, - Enum, -) +from sqlalchemy import Text -from myapp.models.helpers import AuditMixinNullable, ImportMixin +from myapp.models.helpers import AuditMixinNullable from .model_team import Project from .model_job import Pipeline from myapp import app,db from myapp.models.base import MyappModelBase -from myapp.models.helpers import ImportMixin -from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime +from sqlalchemy import Column, Integer, String, ForeignKey from flask import Markup -import datetime metadata = Model.metadata conf = app.config @@ -59,7 +43,7 @@ def pipeline_url(self): if pipeline: return Markup(f'{pipeline.describe}') - return Markup(f'未知') + return Markup('未知') @property def project_url(self): @@ -68,7 +52,7 @@ def project_url(self): elif self.pipeline and self.pipeline.project: return Markup(f'{self.pipeline.project.name}({self.pipeline.project.describe})') else: - return Markup(f'未知') + return Markup('未知') @property def deploy(self): diff --git a/myapp/project.py b/myapp/project.py index c1daf04a3..7b3487079 100644 --- a/myapp/project.py +++ b/myapp/project.py @@ -1,32 +1,8 @@ from werkzeug.security import check_password_hash -from flask_appbuilder.security.sqla.models import ( - assoc_permissionview_role, - assoc_user_role, -) - -from flask import g - from flask_appbuilder.security.views import AuthDBView from flask_appbuilder.security.views import expose -from flask_appbuilder.const import ( - AUTH_DB, - AUTH_LDAP, - AUTH_OAUTH, - AUTH_OID, - AUTH_REMOTE_USER, - LOGMSG_ERR_SEC_AUTH_LDAP, - LOGMSG_ERR_SEC_AUTH_LDAP_TLS, - LOGMSG_WAR_SEC_LOGIN_FAILED, - LOGMSG_WAR_SEC_NO_USER, - LOGMSG_WAR_SEC_NOLDAP_OBJ, - PERMISSION_PREFIX -) -import pysnooper -import json - - - +from flask_appbuilder.const import LOGMSG_WAR_SEC_LOGIN_FAILED # 推送给管理员消息的函数 def push_admin(message): @@ -39,18 +15,9 @@ def push_message(receivers,message,link=None): import logging as log -import datetime -import logging -import re - -from flask import abort, current_app, flash, g, redirect, request, session, url_for -from flask_babel import lazy_gettext +from flask import flash, g, redirect, request, session from flask_login import login_user, logout_user -import jwt -from werkzeug.security import generate_password_hash -from flask_appbuilder.security.forms import LoginForm_db, LoginForm_oid, ResetPasswordForm, UserInfoEdit -from flask_appbuilder._compat import as_unicode -import pysnooper +from flask_appbuilder.security.forms import LoginForm_db class MyCustomRemoteUserView(): @@ -76,7 +43,7 @@ def login(self): return redirect(self.appbuilder.get_url_for_index) form = LoginForm_db() - method = request.method + request.method # 如果提交请求。就是认证 if form.validate_on_submit(): username = form.username.data diff --git a/myapp/security.py b/myapp/security.py index 76c96bddc..945f67b7b 100644 --- a/myapp/security.py +++ b/myapp/security.py @@ -1,54 +1,39 @@ -from flask_login import current_user, LoginManager +from flask_login import current_user import logging -import json import jwt from flask_babel import lazy_gettext -from flask import current_app,redirect, g, flash, request, session, abort, make_response +from flask import current_app from flask_appbuilder.security.sqla import models as ab_models from flask_appbuilder.security.sqla.manager import SecurityManager -from werkzeug.security import generate_password_hash from flask_babel import lazy_gettext as _ from flask_appbuilder.security.views import ( PermissionModelView, PermissionViewModelView, RoleModelView, - UserModelView, - RoleListWidget, - RoleShowWidget, -) -from werkzeug.security import check_password_hash -from flask_appbuilder.security.sqla.models import ( - assoc_permissionview_role, - assoc_user_role, + UserModelView ) +from flask_appbuilder.security.sqla.models import assoc_user_role -from sqlalchemy.orm import backref, relationship -from flask_appbuilder.security.decorators import has_access, has_access_api, permission_name +from flask_appbuilder.security.decorators import has_access from flask_appbuilder.models.sqla.interface import SQLAInterface from flask_appbuilder.widgets import ListWidget -from flask_appbuilder.const import LOGMSG_WAR_SEC_LOGIN_FAILED from sqlalchemy import or_ from flask_appbuilder.security.views import expose -from flask import redirect, g, flash, request, session, abort +from flask import g, flash, request from flask_appbuilder.security.sqla.models import assoc_permissionview_role -from sqlalchemy import select, Table +from sqlalchemy import select from flask_appbuilder.const import ( AUTH_DB, AUTH_LDAP, AUTH_OAUTH, AUTH_OID, AUTH_REMOTE_USER, - LOGMSG_ERR_SEC_AUTH_LDAP, - LOGMSG_ERR_SEC_AUTH_LDAP_TLS, - LOGMSG_WAR_SEC_LOGIN_FAILED, - LOGMSG_WAR_SEC_NO_USER, - LOGMSG_WAR_SEC_NOLDAP_OBJ, - PERMISSION_PREFIX + LOGMSG_WAR_SEC_LOGIN_FAILED ) @@ -90,7 +75,7 @@ def __init__(self, **kwargs): # expand user from flask_appbuilder.security.sqla.models import User,Role -from sqlalchemy import Column, Integer, ForeignKey, String, Sequence, Table +from sqlalchemy import Column, String @@ -115,7 +100,7 @@ def secret(self): pass # help(self.changed_on) # timestamp = int(func.date_format(self.changed_on)) - timestamp = int(self.changed_on.timestamp()) + int(self.changed_on.timestamp()) payload = { "iss": self.username # "iat": timestamp, # Issue period @@ -217,15 +202,13 @@ def userinfo(self): appbuilder=self.appbuilder, ) -from flask_appbuilder.security.views import expose, ModelView, SimpleFormView -from flask_appbuilder.security.forms import LoginForm_db, LoginForm_oid, ResetPasswordForm, UserInfoEdit +from flask_appbuilder.security.views import SimpleFormView from flask_appbuilder._compat import as_unicode from flask_babel import lazy_gettext -from flask_wtf.recaptcha import RecaptchaField -from wtforms import BooleanField, PasswordField, StringField -from wtforms.validators import DataRequired, Email, EqualTo +from wtforms import StringField +from wtforms.validators import DataRequired -from flask_appbuilder.fieldwidgets import BS3PasswordFieldWidget, BS3TextFieldWidget +from flask_appbuilder.fieldwidgets import BS3TextFieldWidget from flask_appbuilder.forms import DynamicForm @@ -491,11 +474,11 @@ def add_org_user(self,username,first_name,last_name,org,email,roles,password="", project_user.user_id = user.id self.get_session.add(project_user) self.get_session.commit() - except Exception as e1: + except Exception: self.get_session.rollback() return user - except Exception as e: + except Exception: self.get_session.rollback() return False @@ -668,7 +651,6 @@ def is_user_defined_permission(self, perm): # @pysnooper.snoop() def sync_role_definitions(self): """Inits the Myapp application with security roles and such""" - from myapp import conf logging.info("Syncing role definition") diff --git a/myapp/tasks/__init__.py b/myapp/tasks/__init__.py index ba993339f..e69de29bb 100644 --- a/myapp/tasks/__init__.py +++ b/myapp/tasks/__init__.py @@ -1,2 +0,0 @@ -from . import schedules -from . import async_task \ No newline at end of file diff --git a/myapp/tasks/async_task.py b/myapp/tasks/async_task.py index b44e34883..843e4451a 100644 --- a/myapp/tasks/async_task.py +++ b/myapp/tasks/async_task.py @@ -1,52 +1,16 @@ """Utility functions used across Myapp""" -import sys,os -import numpy as np -from bs4 import BeautifulSoup -import requests,base64,hashlib -from collections import namedtuple -import datetime -from email.utils import make_msgid, parseaddr -import logging -import time,json -from urllib.error import URLError -import urllib.request import pysnooper -import re -import croniter -from dateutil.tz import tzlocal -import shutil -import os,sys,io,json,datetime,time -import subprocess -from datetime import datetime, timedelta -import os -import sys +from datetime import datetime import time -import datetime from myapp.utils.py.py_k8s import K8s from myapp.utils.celery import session_scope from myapp.project import push_message,push_admin from myapp.tasks.celery_app import celery_app # Myapp framework imports -from myapp import app, db, security_manager -from myapp.models.model_job import ( - Pipeline, - RunHistory, - Workflow, - Tfjob, - Pytorchjob, - Xgbjob, - Task -) -from myapp.models.model_notebook import Notebook +from myapp import app from myapp.models.model_serving import InferenceService from myapp.views.view_inferenceserving import InferenceService_ModelView_base -from myapp.security import ( - MyUser -) -from myapp.views.view_pipeline import run_pipeline,dag_to_pipeline -from sqlalchemy.exc import InvalidRequestError,OperationalError -from sqlalchemy import or_ from myapp.models.model_docker import Docker conf = app.config diff --git a/myapp/tasks/schedules.py b/myapp/tasks/schedules.py index ae3942976..719cf52a3 100644 --- a/myapp/tasks/schedules.py +++ b/myapp/tasks/schedules.py @@ -1,34 +1,20 @@ """Utility functions used across Myapp""" -import sys,os -import numpy as np -from bs4 import BeautifulSoup -import requests,base64,hashlib -from collections import namedtuple -import datetime -from email.utils import make_msgid, parseaddr import logging -import time,json -from urllib.error import URLError -import urllib.request import pysnooper import re import croniter -from dateutil.tz import tzlocal -import shutil -import os,sys,io,json,datetime,time +import json import subprocess import os -import sys import time -import random import datetime from myapp.utils.py.py_k8s import K8s from myapp.utils.celery import session_scope from myapp.project import push_message,push_admin from myapp.tasks.celery_app import celery_app # Myapp framework imports -from myapp import app, db, security_manager +from myapp import app from myapp.models.model_job import ( Pipeline, RunHistory, @@ -40,11 +26,7 @@ ) from myapp.models.model_notebook import Notebook from myapp.models.model_serving import InferenceService -from myapp.security import ( - MyUser -) from myapp.views.view_pipeline import run_pipeline,dag_to_pipeline -from sqlalchemy.exc import InvalidRequestError,OperationalError from sqlalchemy import or_ class Pusherror(Exception): @@ -915,7 +897,7 @@ def check_pipeline_resource(): tasks = dbsession.query(Task).filter(Task.pipeline_id == int(pipeline_id)).all() # 获取model记录 for task in tasks: try: - task_resources= json.loads(task.monitoring).get('task',[]) + json.loads(task.monitoring).get('task',[]) tfjob_resources = json.loads(task.monitoring).get('tfjob',[]) monitoring_workflow[pipeline_id]['task'][task.label]={} # if task_resources: @@ -1235,8 +1217,7 @@ def get_gpu_per_node(org): # get_dir_size('/data/k8s/kubeflow/pipeline/workspace') @pysnooper.snoop() def get_deployment_node_selector(name,namespace): - from kubernetes import client, config, watch - from kubernetes.client.models import v1_pod, v1_object_meta, v1_pod_spec, v1_deployment, v1_deployment_spec + from kubernetes import client exist_dp = client.AppsV1Api().read_namespaced_deployment(name=name, namespace=namespace) node_selector = {} @@ -1252,7 +1233,7 @@ def get_deployment_node_selector(name,namespace): if match_expression.operator == 'Equal': node_selector[match_expression.key] = match_expression.values - except Exception as e: + except Exception: pass # print(e) @@ -1268,7 +1249,7 @@ def get_deployment_node_selector(name,namespace): @celery_app.task(name="task.adjust_service_resource", bind=True) @pysnooper.snoop(watch_explode=()) def adjust_service_resource(task): - from kubernetes import client, config, watch + from kubernetes import client cluster_name='tke' namespace = conf.get('SERVICE_NAMESPACE') cluster = conf.get('CLUSTERS', {})[cluster_name] @@ -1285,8 +1266,8 @@ def adjust_service_resource(task): else: if inferenceserving.resource_gpu and inferenceserving.resource_gpu!='0' and inferenceserving.priority==1: # print(hpa) - target_utilizations = hpa.spec.metrics - current_utilization = hpa.status.current_metrics + hpa.spec.metrics + hpa.status.current_metrics current_replicas = hpa.status.current_replicas desired_replicas = hpa.status.desired_replicas if desired_replicas>current_replicas: # 期望扩容 @@ -1306,7 +1287,7 @@ def adjust_service_resource(task): if current_replicas > service.min_replicas: # 随意缩放一个pod if not target_node_selector.get('gpu-type',''): - api_response = client.AppsV1Api().patch_namespaced_deployment_scale(service.name, namespace,[{'op': 'replace', 'path': '/spec/replicas', 'value': current_replicas-1}]) + client.AppsV1Api().patch_namespaced_deployment_scale(service.name, namespace,[{'op': 'replace', 'path': '/spec/replicas', 'value': current_replicas-1}]) push_message([service.created_by.username,inferenceserving.created_by.username]+conf.get('ADMIN_USER').split(','),'缩服务%s一卡，扩服务%s一卡'%(service.name,inferenceserving.name)) return # 缩放指定pod @@ -1360,7 +1341,7 @@ def update_aihub(task): with session_scope(nullpool=True) as dbsession: try: if len(aihubs)>0: - allaihubs = dbsession.query(Aihub).delete() + dbsession.query(Aihub).delete() dbsession.commit() for data in aihubs: print(data) diff --git a/myapp/tools/watch_pytorchjob.py b/myapp/tools/watch_pytorchjob.py index 3e12a23e1..61735ad5f 100644 --- a/myapp/tools/watch_pytorchjob.py +++ b/myapp/tools/watch_pytorchjob.py @@ -1,26 +1,19 @@ -import time,datetime,logging,os,sys -import asyncio +import time,datetime, os from kubernetes import client from kubernetes import watch -from os import path import json -import requests from myapp.utils.py.py_k8s import check_status_time,K8s -from sqlalchemy.exc import InvalidRequestError,OperationalError import pysnooper -import myapp -import math -from myapp import app, db, security_manager +from myapp import app from myapp.models.model_job import ( Pytorchjob, Task ) from myapp.utils.celery import session_scope from myapp.project import push_admin,push_message -from myapp.models.model_job import Pipeline,Workflow -import pymysql +from myapp.models.model_job import Pipeline conf=app.config from myapp.utils.py.py_prometheus import Prometheus diff --git a/myapp/tools/watch_service.py b/myapp/tools/watch_service.py index eb423aa38..e3bdeacb5 100644 --- a/myapp/tools/watch_service.py +++ b/myapp/tools/watch_service.py @@ -1,26 +1,11 @@ -import time,logging,os,sys -import asyncio +import time, os from kubernetes import client from kubernetes import watch -from os import path -import json -import requests -import math -from sqlalchemy.exc import InvalidRequestError,OperationalError -import pysnooper -import copy -import myapp -from myapp.utils.py.py_k8s import check_status_time,K8s -from myapp.utils.py.py_prometheus import Prometheus -from myapp.project import push_admin,push_message -from myapp import app, db, security_manager -from myapp.models.model_job import ( - Pipeline, - Workflow, - Task -) +from myapp.utils.py.py_k8s import K8s +from myapp.project import push_message +from myapp import app from myapp.utils.celery import session_scope conf=app.config @@ -54,8 +39,8 @@ def listen_service(): # terminated 终止，waiting 等待启动，running 运行中 container_statuse= event['object'].status.container_statuses[0].state terminated = container_statuse.terminated - waiting = container_statuse.waiting - running = container_statuse.running + container_statuse.waiting + container_statuse.running service_name=event['object'].metadata.labels.get('app','') inferenceserving = dbsession.query(InferenceService).filter_by(name=service_name).first() if service_name else None if service_name and inferenceserving: diff --git a/myapp/tools/watch_tfjob.py b/myapp/tools/watch_tfjob.py index d833990fe..e38a79ae8 100644 --- a/myapp/tools/watch_tfjob.py +++ b/myapp/tools/watch_tfjob.py @@ -1,26 +1,19 @@ -import time,datetime,logging,os,sys -import asyncio +import time,datetime, os from kubernetes import client from kubernetes import watch -from os import path import json -import requests from myapp.utils.py.py_k8s import check_status_time,K8s -from sqlalchemy.exc import InvalidRequestError,OperationalError import pysnooper -import myapp -import math -from myapp import app, db, security_manager +from myapp import app from myapp.models.model_job import ( Tfjob, Task ) from myapp.utils.celery import session_scope from myapp.project import push_admin,push_message -from myapp.models.model_job import Pipeline,Workflow -import pymysql +from myapp.models.model_job import Pipeline conf=app.config from myapp.utils.py.py_prometheus import Prometheus diff --git a/myapp/tools/watch_workflow.py b/myapp/tools/watch_workflow.py index 3a72c9baa..5756d488d 100644 --- a/myapp/tools/watch_workflow.py +++ b/myapp/tools/watch_workflow.py @@ -1,21 +1,14 @@ -import time,datetime,logging,os,sys -import asyncio +import time,datetime, os from kubernetes import client from kubernetes import watch -from os import path import json -import requests import math -from sqlalchemy.exc import InvalidRequestError,OperationalError -import pysnooper -import copy -import myapp from myapp.utils.py.py_k8s import check_status_time,K8s from myapp.utils.py.py_prometheus import Prometheus -from myapp.project import push_admin,push_message -from myapp import app, db, security_manager +from myapp.project import push_message +from myapp import app from myapp.models.model_job import ( Pipeline, Workflow, diff --git a/myapp/utils/core.py b/myapp/utils/core.py index 76847dc08..26efe9292 100644 --- a/myapp/utils/core.py +++ b/myapp/utils/core.py @@ -13,17 +13,14 @@ import logging import os import signal -import smtplib -import pysnooper import copy import sys from time import struct_time import traceback -from typing import List, NamedTuple, Optional, Tuple +from typing import List, Optional, Tuple from urllib.parse import unquote_plus import uuid import zlib -import re import bleach import celery from dateutil.parser import parse @@ -37,9 +34,7 @@ import numpy import pandas as pd import parsedatetime -from jinja2 import Template -from jinja2 import contextfilter -from jinja2 import Environment, BaseLoader, DebugUndefined, StrictUndefined +from jinja2 import Environment, BaseLoader, DebugUndefined try: from pydruid.utils.having import Having except ImportError: @@ -53,7 +48,6 @@ from myapp.exceptions import MyappException, MyappTimeoutException from myapp.utils.dates import datetime_to_epoch, EPOCH import re -import random logging.getLogger("MARKDOWN").setLevel(logging.INFO) @@ -1465,7 +1459,7 @@ def check_max_cpu(resource,src_resource=None): raise MyappException('resource cpu input not valid') try: resource = "%.1f~%.1f"%(float(resource.split("~")[0]),float(resource.split("~")[1])) - except Exception as e: + except Exception: raise MyappException('resource cpu input not valid') if not g.user.is_admin(): min = set_host_max(check_max_cpu(resource.split('~')[0])) @@ -1613,7 +1607,6 @@ def merge_tfjob_experiment_template(worker_num,node_selector,volume_mount,image, -import yaml # @pysnooper.snoop(watch_explode=()) def merge_job_experiment_template(node_selector,volume_mount,image,image_secrets,hostAliases,workingDir,image_pull_policy,resource_memory,resource_cpu,command): nodeSelector=None @@ -1743,7 +1736,6 @@ def merge_job_experiment_template(node_selector,volume_mount,image,image_secrets -import yaml # @pysnooper.snoop(watch_explode=()) def merge_pytorchjob_experiment_template(worker_num,node_selector,volume_mount,image,image_secrets,hostAliases,workingDir,image_pull_policy,resource_memory,resource_cpu,master_command,worker_command): nodeSelector=None @@ -2085,7 +2077,7 @@ def get_down_node_num(task_name): # @pysnooper.snoop() def set_downstream_position(task_name): - task_id = str(tasks[task_name]['id']) + str(tasks[task_name]['id']) downstream_tasks = [x for x in dag_json[task_name]['downstream'] if dag_json[x]['index']==dag_json[task_name]['index']] # 获取相同树的下游节点 downstream_tasks = sorted(downstream_tasks, key=lambda temp: dag_json[temp]['total_down_num'],reverse=True) # 按子孙数目排序 for i in range(len(downstream_tasks)): @@ -2137,7 +2129,7 @@ def hive_create_sql_demo(): return sql -import shlex, subprocess +import subprocess def run_shell(shell): cmd = subprocess.Popen(shell, stdin=subprocess.PIPE, stderr=sys.stderr, close_fds=True, stdout=sys.stdout, universal_newlines=True, shell=True, bufsize=1) diff --git a/myapp/utils/log.py b/myapp/utils/log.py index df94879aa..894f46879 100644 --- a/myapp/utils/log.py +++ b/myapp/utils/log.py @@ -5,8 +5,6 @@ from flask import current_app, g, request -import pysnooper - # @pysnooper.snoop(depth=2) class AbstractEventLogger(ABC): @abstractmethod diff --git a/myapp/utils/py/py_k8s.py b/myapp/utils/py/py_k8s.py index 5de3ea9cc..5df7b19f1 100755 --- a/myapp/utils/py/py_k8s.py +++ b/myapp/utils/py/py_k8s.py @@ -1,17 +1,11 @@ -import time,datetime,logging,os,sys +import time,datetime, os import re -from kubernetes import client,config,watch +from kubernetes import client from kubernetes.client.models import v1_pod,v1_object_meta,v1_pod_spec,v1_deployment,v1_deployment_spec import yaml -from os import path import json -# from common.config import * -import pysnooper import multiprocessing -import random -from multiprocessing import Manager import base64 -import uuid from kubernetes import config from kubernetes.client.rest import ApiException @@ -89,7 +83,7 @@ def get_pods(self,namespace=None,service_name=None,pod_name=None,labels={}): if match_expression.operator == 'Equal': node_selector[match_expression.key]=match_expression.values - except Exception as e: + except Exception: pass # print(e) if pod.spec.node_selector: @@ -212,7 +206,7 @@ def label_node(self,ips, labels): "labels": labels } } - api_response = self.v1.patch_node(Hostname, body) + self.v1.patch_node(Hostname, body) return all_node_ip except Exception as e: @@ -416,7 +410,7 @@ def delete_workflow(self,all_crd_info,namespace,run_id): # 删除workflow crd_info = all_crd_info['workflow'] try: - crd_names = self.delete_crd( + self.delete_crd( group=crd_info['group'], version=crd_info['version'], plural=crd_info['plural'], namespace=namespace, labels={'run-id': run_id} ) @@ -428,7 +422,7 @@ def delete_workflow(self,all_crd_info,namespace,run_id): # 删除tfjob try: crd_info = all_crd_info['tfjob'] - crd_names = self.delete_crd( + self.delete_crd( group=crd_info['group'], version=crd_info['version'],plural=crd_info['plural'], namespace=namespace,labels={'run-id': run_id} ) @@ -438,7 +432,7 @@ def delete_workflow(self,all_crd_info,namespace,run_id): # 删除framework try: crd_info = all_crd_info['framework'] - crd_names = self.delete_crd(group=crd_info['group'], version=crd_info['version'], + self.delete_crd(group=crd_info['group'], version=crd_info['version'], plural=crd_info['plural'], namespace=namespace, labels={"run-id": str(run_id)}) except Exception as e: @@ -447,7 +441,7 @@ def delete_workflow(self,all_crd_info,namespace,run_id): # 删除pytorchjob try: crd_info = all_crd_info['pytorchjob'] - crd_names = self.delete_crd( + self.delete_crd( group=crd_info['group'], version=crd_info['version'], plural=crd_info['plural'], namespace=namespace, labels={'run-id': run_id} ) @@ -457,7 +451,7 @@ def delete_workflow(self,all_crd_info,namespace,run_id): # 删除mpijob try: crd_info = all_crd_info['mpijob'] - crd_names = self.delete_crd( + self.delete_crd( group=crd_info['group'], version=crd_info['version'], plural=crd_info['plural'], namespace=namespace, labels={'run-id': run_id} ) @@ -467,7 +461,7 @@ def delete_workflow(self,all_crd_info,namespace,run_id): # 删除vcjob try: crd_info = all_crd_info['vcjob'] - crd_names = self.delete_crd( + self.delete_crd( group=crd_info['group'], version=crd_info['version'], plural=crd_info['plural'], namespace=namespace, labels={'run-id': run_id} ) @@ -477,7 +471,7 @@ def delete_workflow(self,all_crd_info,namespace,run_id): # 删除sparkjob try: crd_info = all_crd_info['sparkjob'] - crd_names = self.delete_crd( + self.delete_crd( group=crd_info['group'], version=crd_info['version'], plural=crd_info['plural'], namespace=namespace, labels={'run-id': run_id} ) @@ -487,7 +481,7 @@ def delete_workflow(self,all_crd_info,namespace,run_id): # 删除paddlejob try: crd_info = all_crd_info['paddlejob'] - crd_names = self.delete_crd( + self.delete_crd( group=crd_info['group'], version=crd_info['version'], plural=crd_info['plural'], namespace=namespace, labels={'run-id': run_id} ) @@ -498,7 +492,7 @@ def delete_workflow(self,all_crd_info,namespace,run_id): # 删除mxjob try: crd_info = all_crd_info['mxjob'] - crd_names = self.delete_crd( + self.delete_crd( group=crd_info['group'], version=crd_info['version'], plural=crd_info['plural'], namespace=namespace, labels={'run-id': run_id} ) @@ -736,7 +730,7 @@ def make_container(self,name,command,args,volume_mount,working_dir,resource_memo env_list.append(client.V1EnvVar(name='K8S_POD_NAME', value_from=client.V1EnvVarSource(field_ref=client.V1ObjectFieldSelector(field_path='metadata.name')))) security_context = client.V1SecurityContext(privileged=privileged) if privileged else None - gpu_type = os.environ.get("GPU_TYPE", "NVIDIA") + os.environ.get("GPU_TYPE", "NVIDIA") def get_gpu(resource_gpu): try: @@ -901,7 +895,7 @@ def create_debug_pod(self,namespace,name,labels,command,args,volume_mount,workin try: self.v1.delete_namespaced_pod(name=name, namespace=namespace,grace_period_seconds=0) # time.sleep(1) - except Exception as e: + except Exception: pass # print(e) pod,pod_spec = self.make_pod( @@ -1063,7 +1057,7 @@ def create_deployment(self,namespace,name,replicas,labels,command,args,volume_mo # print(e) try: - exist_dp = client.AppsV1Api().read_namespaced_deployment(name=name,namespace=namespace) + client.AppsV1Api().read_namespaced_deployment(name=name,namespace=namespace) client.AppsV1Api().patch_namespaced_deployment(name=name, namespace=namespace, body=dp) except ApiException as e: if e.status == 404: @@ -1139,7 +1133,7 @@ def create_statefulset(self,namespace,name,replicas,labels,command,args,volume_m # print(dp.to_str()) try: client.AppsV1Api().delete_namespaced_stateful_set(name, namespace) - except Exception as e: + except Exception: pass # print(e) @@ -1183,7 +1177,7 @@ def create_headless_service(self,namespace,name,username,run_id): print(service.to_dict()) try: self.v1.delete_namespaced_service(name, namespace) - except Exception as e: + except Exception: pass # print(e) try: @@ -1345,7 +1339,7 @@ def get_canary(gateway_service, canarys): print(crd_json) - crd = self.create_crd(group=crd_info['group'], version=crd_info['version'], plural=crd_info['plural'],namespace=namespace, body=crd_json) + self.create_crd(group=crd_info['group'], version=crd_info['version'], plural=crd_info['plural'],namespace=namespace, body=crd_json) if len(ports)>1: crd_json = { @@ -1379,7 +1373,7 @@ def get_canary(gateway_service, canarys): ] } } - crd = self.create_crd(group=crd_info['group'], version=crd_info['version'], plural=crd_info['plural'],namespace=namespace, body=crd_json) + self.create_crd(group=crd_info['group'], version=crd_info['version'], plural=crd_info['plural'],namespace=namespace, body=crd_json) @@ -1489,7 +1483,6 @@ def create_hpa(self,namespace,name,min_replicas,max_replicas,hpa): } ) - my_conditions = [] # my_conditions.append(client.V2beta1HorizontalPodAutoscalerCondition(status="True", type='AbleToScale')) # # status = client.V2beta1HorizontalPodAutoscalerStatus(conditions=my_conditions, current_replicas=max_replicas, @@ -1510,7 +1503,7 @@ def create_hpa(self,namespace,name,min_replicas,max_replicas,hpa): # ) print(json.dumps(hpa_json,indent=4,ensure_ascii=4)) try: - ret = client.AutoscalingV2beta1Api().create_namespaced_horizontal_pod_autoscaler(namespace=namespace, body=hpa_json, pretty=True) + client.AutoscalingV2beta1Api().create_namespaced_horizontal_pod_autoscaler(namespace=namespace, body=hpa_json, pretty=True) except ValueError as e: if str(e) == 'Invalid value for `conditions`, must not be `None`': print(e) @@ -1583,7 +1576,7 @@ def get_pod_metrics(self,namespace=None): # @pysnooper.snoop() def exec_command(self,name,namespace,command): try: - resp = self.v1.read_namespaced_pod(name=name,namespace=namespace) + self.v1.read_namespaced_pod(name=name,namespace=namespace) except ApiException as e: if e.status != 404: print("Unknown error: %s" % e) diff --git a/myapp/utils/py/py_prometheus.py b/myapp/utils/py/py_prometheus.py index dfaaf150e..0d468b7d2 100755 --- a/myapp/utils/py/py_prometheus.py +++ b/myapp/utils/py/py_prometheus.py @@ -1,9 +1,6 @@ -import json,datetime,time -import logging -import os -import io +import json,datetime import requests import pysnooper diff --git a/myapp/views/__init__.py b/myapp/views/__init__.py index 46189b89a..e69de29bb 100644 --- a/myapp/views/__init__.py +++ b/myapp/views/__init__.py @@ -1,29 +0,0 @@ -from . import base -from . import home -from . import route -from . import view_team -from . import view_notebook -from . import view_images -from . import view_docker -from . import view_job_template -from . import view_task -from . import view_pipeline -from . import view_runhistory -from . import view_workflow -from . import view_nni -from . import view_serving -from . import view_train_model -from . import view_inferenceserving -from . import view_service_pipeline -from . import view_link -from . import view_metadata -from . import view_metadata_metric -from . import view_dimension -from . import view_etl_pipeline -from . import view_dataset -from .log import views -from .log import api as log_api - -from . import view_aihub - - diff --git a/myapp/views/base.py b/myapp/views/base.py index 5b3315c4c..7fdbba862 100644 --- a/myapp/views/base.py +++ b/myapp/views/base.py @@ -1,14 +1,11 @@ import datetime -import os import functools import logging import traceback from typing import Any, Dict -import pysnooper from flask_appbuilder.forms import GeneralModelConverter -from flask import abort, flash, g, get_flashed_messages, redirect, Response -from flask_appbuilder import BaseView, ModelView,urltools +from flask import get_flashed_messages from flask_appbuilder.actions import action from flask_appbuilder.forms import DynamicForm from flask_appbuilder.models.sqla.filters import BaseFilter @@ -16,45 +13,26 @@ from myapp.forms import MySearchWidget from flask_babel import get_locale from flask_babel import gettext as __ -from flask_babel import lazy_gettext as _ from flask_wtf.form import FlaskForm import simplejson as json from werkzeug.exceptions import HTTPException from wtforms.fields.core import Field, UnboundField -from flask_appbuilder import ModelView, ModelRestApi -import yaml -from flask_appbuilder.security.decorators import has_access, has_access_api, permission_name -from flask_appbuilder.baseviews import BaseCRUDView, BaseFormView, BaseView, expose, expose_api +from flask_appbuilder import ModelView +from flask_appbuilder.baseviews import BaseCRUDView, BaseView, expose from myapp import conf, db, get_feature_flags, security_manager,event_logger from myapp.exceptions import MyappException, MyappSecurityException from myapp.translations.utils import get_language_pack from myapp.utils import core -from sqlalchemy import or_ from flask_appbuilder.urltools import ( get_filter_args, get_order_args, get_page_args, get_page_size_args, - Stack, ) from flask import ( - current_app, - abort, - flash, g, - Markup, - make_response, - redirect, - render_template, - request, - send_from_directory, Response, - url_for, ) -from flask import Flask, jsonify - -from apispec import yaml_utils -from flask import Blueprint, current_app, jsonify, make_response, request from flask_babel import lazy_gettext as _ import yaml @@ -361,8 +339,8 @@ def response(code, **kwargs): _ret_json = jsonify(kwargs) resp = make_response(_ret_json, code) flash_json=[] - for flash in flashes: - flash_json.append([flash[0],flash[1]]) + for f in flashes: + flash_json.append([f[0], f[1]]) resp.headers["api_flashes"] = json.dumps(flash_json) resp.headers["Content-Type"] = "application/json; charset=utf-8" return resp @@ -739,7 +717,7 @@ def delete(self, pk): return redirect(url) # return self.post_delete_redirect() -from flask_appbuilder.widgets import GroupFormListWidget, ListMasterWidget +from flask_appbuilder.widgets import GroupFormListWidget from flask import ( abort, flash, @@ -747,7 +725,6 @@ def delete(self, pk): make_response, redirect, request, - send_file, session, url_for, ) diff --git a/myapp/views/baseApi.py b/myapp/views/baseApi.py index 0c1075722..3f57677f3 100644 --- a/myapp/views/baseApi.py +++ b/myapp/views/baseApi.py @@ -8,15 +8,11 @@ from inspect import isfunction from sqlalchemy import create_engine from flask_appbuilder.actions import action -from apispec import yaml_utils -from flask_appbuilder.fieldwidgets import BS3TextFieldWidget -from wtforms import Field from flask_babel import gettext as __ -from flask_babel import lazy_gettext as _ from flask_appbuilder.actions import ActionItem -from flask import Blueprint, current_app, jsonify, make_response, request +from flask import jsonify, request from flask import flash -from flask import Flask, current_app, send_from_directory, make_response,send_file +from flask import current_app, make_response,send_file from flask.globals import session from flask_babel import lazy_gettext as _ import jsonschema @@ -28,13 +24,8 @@ from sqlalchemy.orm.properties import ColumnProperty from sqlalchemy.orm.relationships import RelationshipProperty from werkzeug.exceptions import BadRequest -from flask import render_template,redirect -import yaml -from wtforms import widgets from marshmallow import validate from wtforms import validators -from flask_appbuilder.api.convert import Model2SchemaConverter -from flask_appbuilder.api.schemas import get_info_schema, get_item_schema, get_list_schema from flask_appbuilder._compat import as_unicode from flask_appbuilder.const import ( API_ADD_COLUMNS_RES_KEY, @@ -61,26 +52,20 @@ API_ORDER_DIRECTION_RIS_KEY, API_PAGE_INDEX_RIS_KEY, API_PAGE_SIZE_RIS_KEY, - API_PERMISSIONS_RES_KEY, API_PERMISSIONS_RIS_KEY, - API_RESULT_RES_KEY, API_SELECT_COLUMNS_RIS_KEY, API_SHOW_COLUMNS_RES_KEY, API_SHOW_COLUMNS_RIS_KEY, API_SHOW_TITLE_RES_KEY, API_SHOW_TITLE_RIS_KEY, API_URI_RIS_KEY, - PERMISSION_PREFIX, ) from flask import ( abort, - g ) from flask_appbuilder.exceptions import FABException, InvalidOrderByColumnFABException -from flask_appbuilder.security.decorators import permission_name, protect,has_access -from flask_appbuilder.api import BaseModelApi,BaseApi,ModelRestApi -from sqlalchemy.sql import sqltypes -from myapp import app, appbuilder,db,event_logger,cache +from flask_appbuilder.api import ModelRestApi +from myapp import app, db,cache conf = app.config log = logging.getLogger(__name__) @@ -92,7 +77,6 @@ API_ACTION_RIS_KEY='action' API_ROUTE_RIS_KEY ='route_base' -API_PERMISSIONS_RIS_KEY="permissions" API_USER_PERMISSIONS_RIS_KEY="user_permissions" API_RELATED_RIS_KEY="related" API_COLS_WIDTH_RIS_KEY='cols_width' @@ -241,7 +225,6 @@ def json_response(message,status,result): -import pysnooper # @pysnooper.snoop(depth=5) # 暴露url+视图函数。视图函数会被覆盖，暴露url也会被覆盖 class MyappModelRestApi(ModelRestApi): @@ -324,8 +307,8 @@ def response(code, **kwargs): _ret_json = jsonify(kwargs) resp = make_response(_ret_json, code) flash_json=[] - for flash in flashes: - flash_json.append([flash[0],flash[1]]) + for f in flashes: + flash_json.append([f[0], f[1]]) resp.headers["api_flashes"] = json.dumps(flash_json) resp.headers["Content-Type"] = "application/json; charset=utf-8" return resp @@ -1001,8 +984,6 @@ def api_list(self, **kwargs): # handle filters try: # 参数缩写都在每个filter的arg_name - from flask_appbuilder.models.sqla.filters import FilterEqualFunction, FilterStartsWith - joined_filters = self._handle_filters_args(_args) except FABException as e: return self.response_error(400,message=str(e)) @@ -1027,7 +1008,6 @@ def api_list(self, **kwargs): lst = self.post_list(lst) # pks = self.datamodel.get_keys(lst) # import marshmallow.schema - import marshmallow.marshalling # for item in lst: # if self.datamodel.is_relation(item) # aa = @@ -1211,7 +1191,7 @@ def single_action(self, name, pk): pk = self._deserialize_pk_if_composite(pk) action = self.actions.get(name) try: - res = action.func(self.datamodel.get(pk)) + action.func(self.datamodel.get(pk)) back = { "status": 0, "result": {}, @@ -1641,7 +1621,6 @@ def _get_field_info(self, field, filter_rel_field, page=None, page_size=None): # print(column_type) # if type(column_type)== # print(column.__class__.__name__) - from sqlalchemy.sql.schema import Column ret['type']=column_type_str if default: ret['default'] = default.arg diff --git a/myapp/views/home.py b/myapp/views/home.py index cf41d8bcd..b8c68e712 100644 --- a/myapp/views/home.py +++ b/myapp/views/home.py @@ -1,31 +1,19 @@ from flask import ( - current_app, - abort, flash, g, Markup, - make_response, redirect, - render_template, - request, - send_from_directory, Response, - url_for, ) from myapp.utils.py.py_k8s import K8s -import datetime,json -from flask import Flask, jsonify -import pysnooper -from apispec import yaml_utils -from flask import Blueprint, current_app, jsonify, make_response, request -from flask_babel import lazy_gettext as _ -from myapp import conf, db, get_feature_flags, security_manager,event_logger -import yaml +import datetime +from flask import jsonify +from myapp import conf from myapp.views.base import BaseMyappView -from flask_appbuilder import ModelView,AppBuilder,expose,BaseView,has_access -from myapp import app, appbuilder +from flask_appbuilder import expose +from myapp import appbuilder from flask import stream_with_context, request node_resource_used = { @@ -870,7 +858,7 @@ def mlops_traffic(self,url): global_cluster_load[cluster_name]['gpu_req'] += int(nodes[ip]['used_gpu']) global_cluster_load[cluster_name]['gpu_all'] += int(nodes[ip]['gpu']) - message = Markup(f'%s
' % message) + message = Markup('%s
' % message) # print(message) cluster_global_info = '' # for cluster_name in global_cluster_load: @@ -977,7 +965,7 @@ def pipeline_task_resource(self,url): ) - message = Markup(f'%s
' % message) + message = Markup('%s
' % message) # print(message) data = { 'content': message, diff --git a/myapp/views/log/__init__.py b/myapp/views/log/__init__.py index b606bb2f1..f80ce8668 100644 --- a/myapp/views/log/__init__.py +++ b/myapp/views/log/__init__.py @@ -1,7 +1,4 @@ -from flask_babel import lazy_gettext as _ - - class LogMixin: label_title='日志' diff --git a/myapp/views/route.py b/myapp/views/route.py index 959bfb21c..5ec7de4d7 100755 --- a/myapp/views/route.py +++ b/myapp/views/route.py @@ -1,16 +1,6 @@ -from myapp import ( - app, - appbuilder, - conf, - db, - event_logger, - get_feature_flags, - is_feature_enabled, - results_backend, - security_manager, -) +from myapp import app config = app.config diff --git a/myapp/views/view_aihub.py b/myapp/views/view_aihub.py index e97386f91..a2b95f0dd 100644 --- a/myapp/views/view_aihub.py +++ b/myapp/views/view_aihub.py @@ -1,9 +1,9 @@ from flask_appbuilder.models.sqla.interface import SQLAInterface import urllib.parse -from myapp import app, appbuilder,db,event_logger -from wtforms import BooleanField, IntegerField,StringField, SelectField,FloatField,DateField,DateTimeField,SelectMultipleField,FormField,FieldList -from flask_appbuilder.fieldwidgets import BS3TextFieldWidget,BS3PasswordFieldWidget,DatePickerWidget,DateTimePickerWidget,Select2ManyWidget,Select2Widget +from myapp import app, appbuilder,db +from wtforms import SelectField +from flask_appbuilder.fieldwidgets import Select2Widget from myapp.models.model_job import Images,Job_Template,Repository from myapp.models.model_team import Project,Project_User from myapp.models.model_serving import InferenceService @@ -13,22 +13,15 @@ MyappModelRestApi ) from flask import ( - current_app, - abort, flash, - g, - Markup, - make_response, redirect ) from .base import ( MyappFilter, - MyappModelView, ) from myapp.models.model_aihub import Aihub -from flask_appbuilder import CompactCRUDMixin, expose -import pysnooper,datetime,time,json -from myapp.security import MyUser +from flask_appbuilder import expose +import datetime,json conf = app.config logging = app.logger diff --git a/myapp/views/view_dataset.py b/myapp/views/view_dataset.py index d0fbe97bc..7395c06cc 100644 --- a/myapp/views/view_dataset.py +++ b/myapp/views/view_dataset.py @@ -1,23 +1,14 @@ from flask_appbuilder.models.sqla.interface import SQLAInterface from flask_babel import lazy_gettext as _ -from wtforms.validators import DataRequired, Length, NumberRange, Optional,Regexp -from myapp import app, appbuilder,db,event_logger -from myapp.utils import core -from wtforms import BooleanField, IntegerField,StringField, SelectField,FloatField,DateField,DateTimeField,SelectMultipleField,FormField,FieldList -from flask_appbuilder.fieldwidgets import BS3TextFieldWidget,BS3PasswordFieldWidget,DatePickerWidget,DateTimePickerWidget,Select2ManyWidget,Select2Widget -from myapp.forms import MyBS3TextAreaFieldWidget,MySelect2Widget,MyCodeArea,MyLineSeparatedListField,MyJSONField,MyBS3TextFieldWidget,MySelectMultipleField,MySelect2ManyWidget - -from .baseApi import ( - MyappModelRestApi, - json_response -) -from flask import ( - current_app, - abort, - flash, - g -) +from wtforms.validators import DataRequired +from myapp import app, appbuilder +from wtforms import StringField, SelectField +from flask_appbuilder.fieldwidgets import BS3TextFieldWidget, Select2Widget +from myapp.forms import MyBS3TextAreaFieldWidget,MySelect2Widget + +from .baseApi import MyappModelRestApi +from flask import g from myapp.models.model_dataset import Dataset conf = app.config diff --git a/myapp/views/view_dimension.py b/myapp/views/view_dimension.py index 4b32f9fe5..9bdd0680b 100644 --- a/myapp/views/view_dimension.py +++ b/myapp/views/view_dimension.py @@ -2,47 +2,31 @@ from flask_appbuilder.models.sqla.interface import SQLAInterface from flask_babel import gettext as __ from flask_babel import lazy_gettext as _ -from flask_babel import lazy_gettext,gettext +from flask_babel import lazy_gettext import copy -import random -from sqlalchemy.exc import SQLAlchemyError,InvalidRequestError from sqlalchemy import create_engine from sqlalchemy.orm import scoped_session, sessionmaker -from flask import Blueprint, current_app, jsonify, make_response, request from flask_appbuilder.actions import action -import re,os -from sqlalchemy import and_, or_, select -from wtforms.validators import DataRequired, Length, NumberRange, Optional,Regexp -from myapp import app, appbuilder,db,event_logger -from wtforms import BooleanField, IntegerField,StringField, SelectField,FloatField,DateField,DateTimeField,SelectMultipleField,FormField,FieldList -from flask_appbuilder.fieldwidgets import BS3TextFieldWidget,BS3PasswordFieldWidget,DatePickerWidget,DateTimePickerWidget,Select2ManyWidget,Select2Widget -from myapp.forms import MyBS3TextAreaFieldWidget,MySelect2Widget,MyCodeArea,MyLineSeparatedListField,MyJSONField,MyBS3TextFieldWidget,MySelectMultipleField,MySelect2ManyWidget -from .baseApi import ( - MyappModelRestApi, - json_response -) +import os +from sqlalchemy import or_ +from wtforms.validators import DataRequired, Length, Regexp +from myapp import app, appbuilder,db +from wtforms import BooleanField, StringField, SelectField +from flask_appbuilder.fieldwidgets import BS3TextFieldWidget, Select2Widget +from myapp.forms import MySelect2Widget +from .baseApi import MyappModelRestApi from flask import ( - current_app, abort, flash, g, Markup, - make_response, redirect, - render_template, - request, - send_from_directory, - Response, - url_for, -) -from .base import ( - MyappFilter, - MyappModelView, + request ) +from .base import MyappFilter from myapp.models.model_dimension import Dimension_table -from flask_appbuilder import CompactCRUDMixin, expose -import pysnooper,datetime,time,json -from myapp.security import MyUser +from flask_appbuilder import expose +import pysnooper,datetime, json conf = app.config logging = app.logger @@ -105,13 +89,8 @@ def apply(self, query, func): ) } -from myapp.project import push_message, push_admin - - @pysnooper.snoop() def ddl_hive_external_table(table_id): - username = g.user.username - try: item = db.session.query(Dimension_table).filter_by(id=int(table_id)).first() if not item: @@ -446,7 +425,7 @@ def create_external_table(self, dim_id): flash('增加新字段失败：'+str(e), 'error') - except sqlalchemy.exc.NoSuchTableError as e: + except sqlalchemy.exc.NoSuchTableError: print('表不存在') # 如果表不存在 @@ -480,7 +459,7 @@ def create_external_table(self, dim_id): def add_more_info(self,response,**kwargs): - from myapp.views.baseApi import API_RELATED_RIS_KEY, API_ADD_COLUMNS_RES_KEY, API_EDIT_COLUMNS_RES_KEY + from myapp.views.baseApi import API_ADD_COLUMNS_RES_KEY, API_EDIT_COLUMNS_RES_KEY for col in response[API_ADD_COLUMNS_RES_KEY]: if col['name']=='columns': response[API_EDIT_COLUMNS_RES_KEY].remove(col) @@ -508,8 +487,7 @@ def add_more_info(self,response,**kwargs): from flask_appbuilder import Model from myapp.models.base import MyappModelBase -from sqlalchemy import Column, Integer, String, ForeignKey, Float,BigInteger,Date -from sqlalchemy.orm import relationship +from sqlalchemy import Column, String, BigInteger class Dimension_remote_table_ModelView_Api(MyappModelRestApi): @@ -533,13 +511,10 @@ def set_model(self,dim_id): search_columns = [] label_columns={} add_columns=[] - edit_columns=[] show_columns=[] list_columns=[] description_columns={} add_form_extra_fields={} - add_form_query_rel_fields={} - validators_columns={} order_columns=[] cols_width = { } @@ -640,13 +615,13 @@ def pre_add(self,item): if self.cols[key].get('column_type', 'text') == 'int': try: setattr(item,key,int(getattr(item,key))) - except Exception as e: + except Exception: setattr(item,key,None) if self.cols[key].get('column_type', 'text') == 'double': try: setattr(item,key,float(getattr(item,key))) - except Exception as e: + except Exception: setattr(item, key, None) def pre_update(self,item): @@ -658,13 +633,13 @@ def pre_update(self,item): if self.cols[key].get('column_type', 'text') == 'int': try: setattr(item,key,int(getattr(item,key))) - except Exception as e: + except Exception: setattr(item,key,None) if self.cols[key].get('column_type', 'text') == 'double': try: setattr(item,key,float(getattr(item,key))) - except Exception as e: + except Exception: setattr(item, key, None) def get_primary_key(cols): diff --git a/myapp/views/view_docker.py b/myapp/views/view_docker.py index ddcd6fe7d..432fc4a28 100644 --- a/myapp/views/view_docker.py +++ b/myapp/views/view_docker.py @@ -1,28 +1,20 @@ -from flask import render_template,redirect from flask_appbuilder.models.sqla.interface import SQLAInterface from flask_babel import lazy_gettext as _ from flask_appbuilder.forms import GeneralModelConverter -import uuid -import re -from wtforms.validators import DataRequired, Length, NumberRange, Optional,Regexp -from myapp.models.model_job import Repository,Images -from myapp import app, appbuilder,db,event_logger - -from wtforms import BooleanField, IntegerField,StringField, SelectField,FloatField,DateField,DateTimeField,SelectMultipleField,FormField,FieldList -from myapp.views.view_team import Project_Filter,Project_Join_Filter,filter_join_org_project -from flask_appbuilder.fieldwidgets import BS3TextFieldWidget,BS3PasswordFieldWidget,DatePickerWidget,DateTimePickerWidget,Select2ManyWidget,Select2Widget -from myapp.forms import MyBS3TextAreaFieldWidget,MySelect2Widget,MyCodeArea,MyLineSeparatedListField,MyJSONField,MyBS3TextFieldWidget,MySelectMultipleField - -from .baseApi import ( - MyappModelRestApi -) +from wtforms.validators import DataRequired +from myapp.models.model_job import Repository +from myapp import app, appbuilder,db + +from wtforms import StringField +from myapp.views.view_team import Project_Join_Filter +from flask_appbuilder.fieldwidgets import BS3TextFieldWidget +from myapp.forms import MyBS3TextAreaFieldWidget + +from .baseApi import MyappModelRestApi from flask import ( - current_app, - abort, flash, g, Markup, - make_response, redirect ) from .base import ( @@ -30,8 +22,8 @@ MyappFilter, MyappModelView, ) -from flask_appbuilder import CompactCRUDMixin, expose -import pysnooper,datetime,time,json +from flask_appbuilder import expose +import datetime,time,json conf = app.config logging = app.logger @@ -92,14 +84,14 @@ class Docker_ModelView_Base(): "base_image":StringField( _(datamodel.obj.lab('base_image')), default='', - description=Markup(f'基础镜像和构建方法可参考：点击打开'%(conf.get('HELP_URL').get('docker',''))), + description=Markup('基础镜像和构建方法可参考：点击打开'%(conf.get('HELP_URL').get('docker',''))), widget=BS3TextFieldWidget(), validators=[DataRequired(),] ), "expand":StringField( _(datamodel.obj.lab('expand')), default=json.dumps(expand,ensure_ascii=False,indent=4), - description=Markup(f'扩展字段'), + description=Markup('扩展字段'), widget=MyBS3TextAreaFieldWidget(rows=3) ) diff --git a/myapp/views/view_etl_pipeline.py b/myapp/views/view_etl_pipeline.py index e62b3a1ef..f0c1f305c 100644 --- a/myapp/views/view_etl_pipeline.py +++ b/myapp/views/view_etl_pipeline.py @@ -1,67 +1,44 @@ -from flask import render_template,redirect from flask_appbuilder.models.sqla.interface import SQLAInterface from flask_babel import gettext as __ from flask_babel import lazy_gettext as _ import uuid -import re import urllib.parse from sqlalchemy.exc import InvalidRequestError from myapp.models.model_etl_pipeline import ETL_Pipeline,ETL_Task from myapp.views.view_team import Project_Join_Filter from flask_appbuilder.actions import action -from flask import current_app, flash, jsonify, make_response, redirect, request, url_for +from flask import jsonify from flask_appbuilder.forms import GeneralModelConverter from myapp.utils import core -from myapp import app, appbuilder,db,event_logger +from myapp import app, appbuilder,db from wtforms.ext.sqlalchemy.fields import QuerySelectField -from wtforms.validators import DataRequired, Length, NumberRange, Optional,Regexp -from sqlalchemy import and_, or_, select -from myapp.exceptions import MyappException -from wtforms import BooleanField, IntegerField,StringField, SelectField,FloatField,DateField,DateTimeField,SelectMultipleField,FormField,FieldList -from myapp.project import push_message,push_admin -from flask_appbuilder.fieldwidgets import BS3TextFieldWidget,BS3PasswordFieldWidget,DatePickerWidget,DateTimePickerWidget,Select2ManyWidget,Select2Widget,BS3TextAreaFieldWidget -from myapp.forms import MyBS3TextAreaFieldWidget,MySelect2Widget,MyCodeArea,MyLineSeparatedListField,MyJSONField,MyBS3TextFieldWidget,MySelectMultipleField -import re,copy -from .baseApi import ( - MyappModelRestApi -) +from wtforms.validators import DataRequired, Length, Regexp +from sqlalchemy import or_ +from wtforms import StringField +from flask_appbuilder.fieldwidgets import BS3TextFieldWidget, Select2Widget +from myapp.forms import MyBS3TextAreaFieldWidget +import copy +from .baseApi import MyappModelRestApi from flask import ( - current_app, - abort, flash, g, - Markup, make_response, redirect, - render_template, request, - send_from_directory, - Response, - url_for, ) from myapp import security_manager from myapp.views.view_team import filter_join_org_project from .base import ( - api, - BaseMyappView, - check_ownership, - data_payload_response, DeleteMixin, - generate_download_headers, - get_error_msg, get_user_roles, - handle_api_exception, - json_error_response, - json_success, MyappFilter, MyappModelView, - json_response ) -from flask_appbuilder import CompactCRUDMixin, expose -import pysnooper,datetime,time,json +from flask_appbuilder import expose +import datetime,time,json conf = app.config logging = app.logger APPGROUP_INFO=['资源组1','资源组2','资源组3'] @@ -1603,7 +1580,7 @@ def run_etl_pipeline(self,etl_pipeline_id): print(etl_pipeline_id) url = '/etl_pipeline_modelview/web/' + etl_pipeline_id try: - pipeline = db.session.query(ETL_Pipeline).filter_by(id=etl_pipeline_id).first() + db.session.query(ETL_Pipeline).filter_by(id=etl_pipeline_id).first() # run_pipeline(pipeline) db.session.commit() url = conf.get('MODEL_URLS',{}).get('etl_task') @@ -1616,7 +1593,7 @@ def run_etl_pipeline(self,etl_pipeline_id): @expose("/web/", methods=["GET"]) def web(self,etl_pipeline_id): - etl_pipeline = db.session.query(ETL_Pipeline).filter_by(id=etl_pipeline_id).first() + db.session.query(ETL_Pipeline).filter_by(id=etl_pipeline_id).first() # pipeline.dag_json = pipeline.fix_dag_json() # pipeline.expand = json.dumps(pipeline.fix_expand(), indent=4, ensure_ascii=False) @@ -1636,9 +1613,6 @@ def web(self,etl_pipeline_id): # db.session.commit() print(etl_pipeline_id) url = '/static/appbuilder/visonPlus/index.html?pipeline_id=%s'%etl_pipeline_id # 前后端集成完毕，这里需要修改掉 - data = { - "url": url - } return redirect('/frontend/showOutLink?url=%s' % urllib.parse.quote(url, safe="")) # 返回模板 # return self.render_template('link.html', data=data) diff --git a/myapp/views/view_images.py b/myapp/views/view_images.py index b449cc48e..2c8aab323 100644 --- a/myapp/views/view_images.py +++ b/myapp/views/view_images.py @@ -1,31 +1,20 @@ -from flask import render_template,redirect from flask_appbuilder.models.sqla.interface import SQLAInterface from flask_babel import lazy_gettext as _ from myapp.models.model_job import Repository,Images -from myapp.views.view_team import Project_Filter -from myapp import app, appbuilder,db,event_logger -from wtforms.validators import DataRequired, Length, NumberRange, Optional,Regexp -from wtforms import BooleanField, IntegerField,StringField, SelectField,FloatField,DateField,DateTimeField,SelectMultipleField,FormField,FieldList +from myapp import app, appbuilder +from wtforms.validators import DataRequired, Length, Regexp +from wtforms import StringField -from flask_appbuilder.fieldwidgets import BS3TextFieldWidget,BS3PasswordFieldWidget,DatePickerWidget,DateTimePickerWidget,Select2ManyWidget,Select2Widget -from myapp.forms import MyBS3TextAreaFieldWidget,MySelect2Widget,MyCodeArea,MyLineSeparatedListField,MyJSONField,MyBS3TextFieldWidget,MySelectMultipleField +from flask_appbuilder.fieldwidgets import BS3TextFieldWidget +from myapp.forms import MyBS3TextAreaFieldWidget -from .baseApi import ( - MyappModelRestApi -) -from flask import ( - current_app, - abort, - flash, - g -) +from .baseApi import MyappModelRestApi +from flask import g from .base import ( DeleteMixin, MyappFilter, MyappModelView, ) -from flask_appbuilder import CompactCRUDMixin, expose -import pysnooper,datetime,time,json conf = app.config logging = app.logger diff --git a/myapp/views/view_inferenceserving.py b/myapp/views/view_inferenceserving.py index f05737a6c..c434834af 100644 --- a/myapp/views/view_inferenceserving.py +++ b/myapp/views/view_inferenceserving.py @@ -1,38 +1,30 @@ -from flask import render_template,redirect from flask_appbuilder.models.sqla.interface import SQLAInterface -from flask import Blueprint, current_app, jsonify, make_response, request +from flask import jsonify from myapp.models.model_serving import InferenceService -from myapp.models.model_team import Project,Project_User from myapp.utils import core from flask_babel import gettext as __ from flask_babel import lazy_gettext as _ from flask_appbuilder.actions import action -from myapp import app, appbuilder,db,event_logger -import logging -from flask_babel import lazy_gettext,gettext +from myapp import app, appbuilder,db +from flask_babel import lazy_gettext import re import copy from sqlalchemy.exc import InvalidRequestError from myapp.models.model_job import Repository from wtforms.ext.sqlalchemy.fields import QuerySelectField from myapp import security_manager -import os,sys -from wtforms.validators import DataRequired, Length, NumberRange, Optional,Regexp -from wtforms import BooleanField, IntegerField, SelectField, StringField,FloatField,DateField,DateTimeField,SelectMultipleField,FormField,FieldList -from flask_appbuilder.fieldwidgets import BS3TextFieldWidget,BS3PasswordFieldWidget,DatePickerWidget,DateTimePickerWidget,Select2ManyWidget,Select2Widget -from myapp.forms import MyBS3TextAreaFieldWidget,MySelect2Widget,MyCodeArea,MyLineSeparatedListField,MyJSONField,MyBS3TextFieldWidget,MySelectMultipleField -from myapp.views.view_team import Project_Filter,Project_Join_Filter,filter_join_org_project +from wtforms.validators import DataRequired, Length, Regexp +from wtforms import SelectField, StringField +from flask_appbuilder.fieldwidgets import BS3TextFieldWidget, Select2ManyWidget,Select2Widget +from myapp.forms import MyBS3TextAreaFieldWidget,MySelect2Widget, MyBS3TextFieldWidget,MySelectMultipleField +from myapp.views.view_team import Project_Join_Filter from myapp.views.view_team import filter_join_org_project from flask import ( - current_app, - abort, flash, g, Markup, - make_response, redirect, - render_template, request ) from .base import ( @@ -40,13 +32,12 @@ MyappModelView, ) -from sqlalchemy import and_, or_, select from .baseApi import ( MyappModelRestApi ) -from flask_appbuilder import CompactCRUDMixin, expose -import pysnooper,datetime,time,json +from flask_appbuilder import expose +import datetime,time,json conf = app.config @@ -591,7 +582,7 @@ def pre_add(self, item): if ('http:' in item.model_path or 'https:' in item.model_path) and ('.zip' in item.model_path or '.tar.gz' in item.model_path): try: flash('检测到模型地址为网络压缩文件，需压缩文件名和解压后文件夹名相同','warning') - except Exception as e: + except Exception: pass # print(e) @@ -680,8 +671,8 @@ def update_service(self): service_name = args.get('service_name', '') model_name = args.get('model_name', '') model_version = args.get('model_version', '') - env = args.get('env', '') # 所处环境 - area = args.get('area','') + args.get('env', '') # 所处环境 + args.get('area','') service=None if service_id: diff --git a/myapp/views/view_job_template.py b/myapp/views/view_job_template.py index 46f7cb06c..d001c94d2 100644 --- a/myapp/views/view_job_template.py +++ b/myapp/views/view_job_template.py @@ -1,54 +1,44 @@ - from flask_appbuilder.models.sqla.interface import SQLAInterface from flask_babel import gettext as __ from flask_babel import lazy_gettext as _ import uuid -import re -from wtforms.validators import DataRequired, Length, NumberRange, Optional,Regexp +from wtforms.validators import DataRequired, Length, Regexp from sqlalchemy.exc import InvalidRequestError -from myapp.models.model_job import Repository,Images,Job_Template,Task,Pipeline,Workflow,Tfjob,Xgbjob,RunHistory,Pytorchjob -from myapp.models.model_team import Project,Project_User +from myapp.models.model_job import Job_Template from flask_appbuilder.actions import action -from jinja2 import Environment, BaseLoader, DebugUndefined, StrictUndefined +from jinja2 import Environment, BaseLoader, DebugUndefined from myapp.utils import core -from myapp import app, appbuilder,db,event_logger +from myapp import app, appbuilder,db -from wtforms import BooleanField, IntegerField,StringField, SelectField,FloatField,DateField,DateTimeField,SelectMultipleField,FormField,FieldList +from wtforms import BooleanField, StringField, SelectField -from flask_appbuilder.fieldwidgets import BS3TextFieldWidget,BS3PasswordFieldWidget,DatePickerWidget,DateTimePickerWidget,Select2ManyWidget,Select2Widget -from myapp.forms import MyBS3TextAreaFieldWidget,MySelect2Widget,MyCodeArea,MyLineSeparatedListField,MyJSONField,MyBS3TextFieldWidget,MySelectMultipleField +from flask_appbuilder.fieldwidgets import BS3TextFieldWidget, Select2Widget +from myapp.forms import MyBS3TextAreaFieldWidget, MyCodeArea -import re,copy +import re from .baseApi import ( MyappModelRestApi ) from flask import ( - current_app, - abort, flash, g, Markup, make_response, redirect, - render_template, - request, - send_from_directory, - Response, - url_for, + request ) from .base import ( get_user_roles, MyappFilter, - MyappModelView, ) -from flask_appbuilder import CompactCRUDMixin, expose +from flask_appbuilder import expose from myapp.views.view_images import Images_Filter from myapp.views.view_team import Project_Filter -import pysnooper,datetime,time,json +import datetime,time,json conf = app.config logging = app.logger @@ -149,7 +139,7 @@ class Job_Template_ModelView_Base(): } } },indent=4,ensure_ascii=False), - description=Markup(f'json格式，此类task使用时需要填写的参数，示例：
%s
'%core.job_template_args_definition()), + description=Markup('json格式，此类task使用时需要填写的参数，示例：
%s
'%core.job_template_args_definition()), widget=MyBS3TextAreaFieldWidget(rows=10), # 传给widget函数的是外层的field对象，以及widget函数的参数 validators=[DataRequired()] ), diff --git a/myapp/views/view_link.py b/myapp/views/view_link.py index 4a8f66b0a..da95248ef 100644 --- a/myapp/views/view_link.py +++ b/myapp/views/view_link.py @@ -2,7 +2,7 @@ from flask_babel import gettext as __ from flask_babel import lazy_gettext as _ -from myapp import app, appbuilder,db,event_logger +from myapp import app, appbuilder conf = app.config diff --git a/myapp/views/view_metadata.py b/myapp/views/view_metadata.py index 5def93468..ac9a13f9e 100644 --- a/myapp/views/view_metadata.py +++ b/myapp/views/view_metadata.py @@ -1,23 +1,19 @@ - from flask_appbuilder.models.sqla.interface import SQLAInterface from flask_babel import gettext as __ from flask_babel import lazy_gettext as _ from flask_appbuilder.actions import action -import re,os -from wtforms.validators import DataRequired, Length, NumberRange, Optional,Regexp -from myapp import app, appbuilder,db,event_logger +from wtforms.validators import DataRequired, Length, Regexp +from myapp import app, appbuilder from myapp.utils import core -from wtforms import BooleanField, IntegerField,StringField, SelectField,FloatField,DateField,DateTimeField,SelectMultipleField,FormField,FieldList -from flask_appbuilder.fieldwidgets import BS3TextFieldWidget,BS3PasswordFieldWidget,DatePickerWidget,DateTimePickerWidget,Select2ManyWidget,Select2Widget -from myapp.forms import MyBS3TextAreaFieldWidget,MySelect2Widget,MyCodeArea,MyLineSeparatedListField,MyJSONField,MyBS3TextFieldWidget,MySelectMultipleField,MySelect2ManyWidget +from wtforms import StringField, SelectField +from flask_appbuilder.fieldwidgets import BS3TextFieldWidget, Select2Widget +from myapp.forms import MyBS3TextAreaFieldWidget,MySelect2Widget,MyCodeArea, MySelectMultipleField from .baseApi import ( MyappModelRestApi, - json_response ) from flask import ( - current_app, abort, flash, g diff --git a/myapp/views/view_metadata_metric.py b/myapp/views/view_metadata_metric.py index 6f5519b9e..64b0e0bfc 100644 --- a/myapp/views/view_metadata_metric.py +++ b/myapp/views/view_metadata_metric.py @@ -1,22 +1,18 @@ - from flask_appbuilder.models.sqla.interface import SQLAInterface from flask_babel import gettext as __ from flask_babel import lazy_gettext as _ -from sqlalchemy import and_, or_, select +from sqlalchemy import or_ from flask_appbuilder.actions import action -import re,os from wtforms.validators import DataRequired -from myapp import app, appbuilder,db,event_logger -from myapp.utils import core -from wtforms import BooleanField, IntegerField,StringField, SelectField,FloatField,DateField,DateTimeField,SelectMultipleField,FormField,FieldList -from flask_appbuilder.fieldwidgets import BS3TextFieldWidget,BS3PasswordFieldWidget,DatePickerWidget,DateTimePickerWidget,Select2ManyWidget,Select2Widget -from myapp.forms import MyBS3TextAreaFieldWidget,MySelect2Widget,MyCodeArea,MyLineSeparatedListField,MyJSONField,MyBS3TextFieldWidget,MySelectMultipleField +from myapp import app, appbuilder +from wtforms import StringField, SelectField +from flask_appbuilder.fieldwidgets import BS3TextFieldWidget, Select2Widget +from myapp.forms import MyBS3TextAreaFieldWidget,MySelect2Widget from .baseApi import ( MyappModelRestApi ) from flask import ( - current_app, abort, flash, g @@ -24,7 +20,6 @@ from .base import ( MyappFilter, - MyappModelView, ) from myapp.models.model_metadata_metric import Metadata_metric conf = app.config diff --git a/myapp/views/view_nni.py b/myapp/views/view_nni.py index 2fef84e71..3fecaebbe 100644 --- a/myapp/views/view_nni.py +++ b/myapp/views/view_nni.py @@ -1,34 +1,27 @@ -from flask import render_template,redirect from flask_appbuilder.models.sqla.interface import SQLAInterface from flask_babel import gettext as __ from flask_babel import lazy_gettext as _ -import uuid from myapp.models.model_nni import NNI from myapp.models.model_job import Repository from flask_appbuilder.actions import action -from flask_babel import lazy_gettext,gettext +from flask_babel import lazy_gettext from flask_appbuilder.forms import GeneralModelConverter from myapp.utils import core -from myapp import app, appbuilder,db,event_logger -import os,sys -from wtforms.validators import DataRequired, Length, NumberRange, Optional,Regexp -from sqlalchemy import and_, or_, select -from myapp.exceptions import MyappException -from wtforms import BooleanField, IntegerField, SelectField, StringField,FloatField,DateField,DateTimeField,SelectMultipleField,FormField,FieldList -from flask_appbuilder.fieldwidgets import BS3TextFieldWidget,BS3PasswordFieldWidget,DatePickerWidget,DateTimePickerWidget,Select2ManyWidget,Select2Widget -from myapp.forms import MyBS3TextAreaFieldWidget,MySelect2Widget,MyCodeArea,MyLineSeparatedListField,MyJSONField,MyBS3TextFieldWidget,MyCommaSeparatedListField,MySelectMultipleField -from myapp.views.view_team import Project_Filter,Project_Join_Filter -import re,copy +from myapp import app, appbuilder,db +import os +from wtforms.validators import DataRequired, Length, Regexp +from sqlalchemy import or_ +from wtforms import IntegerField, SelectField, StringField,FloatField +from flask_appbuilder.fieldwidgets import BS3TextFieldWidget, Select2ManyWidget,Select2Widget +from myapp.forms import MyBS3TextAreaFieldWidget, MyCodeArea, MySelectMultipleField +from myapp.views.view_team import Project_Join_Filter +import copy from flask import ( - current_app, - abort, flash, g, Markup, - make_response, redirect, - render_template, request ) from .baseApi import ( @@ -40,8 +33,8 @@ MyappFilter, MyappModelView, ) -from flask_appbuilder import CompactCRUDMixin, expose -import pysnooper,datetime,time,json +from flask_appbuilder import expose +import datetime, json diff --git a/myapp/views/view_notebook.py b/myapp/views/view_notebook.py index c4d3a6249..dd5900421 100644 --- a/myapp/views/view_notebook.py +++ b/myapp/views/view_notebook.py @@ -1,8 +1,6 @@ -from flask import render_template,redirect from flask_appbuilder.models.sqla.interface import SQLAInterface from flask_babel import gettext as __ from flask_babel import lazy_gettext as _ -import random import uuid from myapp.models.model_notebook import Notebook @@ -13,25 +11,17 @@ from myapp.utils import core from myapp import app, appbuilder,db,event_logger from wtforms.ext.sqlalchemy.fields import QuerySelectField -import os,sys -from wtforms.validators import DataRequired, Length, NumberRange, Optional,Regexp -from wtforms import BooleanField, IntegerField, SelectField, StringField,FloatField,DateField,DateTimeField,SelectMultipleField,FormField,FieldList -from flask_appbuilder.fieldwidgets import BS3TextFieldWidget,BS3PasswordFieldWidget,DatePickerWidget,DateTimePickerWidget,Select2ManyWidget,Select2Widget -from myapp.forms import MyBS3TextAreaFieldWidget,MySelect2Widget,MyCodeArea,MyLineSeparatedListField,MyJSONField,MyBS3TextFieldWidget,MyCommaSeparatedListField,MySelectMultipleField +from wtforms.validators import DataRequired, Length, Regexp +from wtforms import SelectField, StringField +from flask_appbuilder.fieldwidgets import BS3TextFieldWidget, Select2Widget +from myapp.forms import MySelect2Widget, MyBS3TextFieldWidget from myapp.utils.py import py_k8s from flask import ( - current_app, abort, flash, g, - Markup, - make_response, redirect, - render_template, request, - send_from_directory, - Response, - url_for, ) from .baseApi import ( MyappModelRestApi @@ -41,9 +31,9 @@ MyappFilter, MyappModelView, ) -from flask_appbuilder import CompactCRUDMixin, expose -import pysnooper,datetime,time,json -from myapp.views.view_team import Project_Filter,Project_Join_Filter,filter_join_org_project +from flask_appbuilder import expose +import datetime,time,json +from myapp.views.view_team import Project_Join_Filter,filter_join_org_project conf = app.config @@ -464,7 +454,7 @@ def reset(self,notebook_id): notebook = db.session.query(Notebook).filter_by(id=notebook_id).first() try: - notebook_crd = self.reset_notebook(notebook) + self.reset_notebook(notebook) flash('已重置，Running状态后可进入。注意：notebook会定时清理，如要运行长期任务请在pipeline中创建任务流进行。','info') except Exception as e: message = '重置失败，稍后重试。%s'%str(e) diff --git a/myapp/views/view_pipeline.py b/myapp/views/view_pipeline.py index 628fc83d6..fdebe67c9 100644 --- a/myapp/views/view_pipeline.py +++ b/myapp/views/view_pipeline.py @@ -1,26 +1,23 @@ -from flask import render_template,redirect from flask_appbuilder.models.sqla.interface import SQLAInterface from flask_babel import gettext as __ from flask_babel import lazy_gettext as _ import uuid -import re import urllib.parse -from kfp import compiler -from myapp.models.model_job import Repository,Images,Job_Template,Task,Pipeline,Workflow,Tfjob,Xgbjob,RunHistory,Pytorchjob -from myapp.models.model_team import Project,Project_User +from myapp.models.model_job import Task,Pipeline,Workflow, RunHistory +from myapp.models.model_team import Project from myapp.views.view_team import Project_Join_Filter from flask_appbuilder.actions import action -from flask import current_app, flash, jsonify, make_response, redirect, request, url_for +from flask import jsonify from flask_appbuilder.forms import GeneralModelConverter from myapp.utils import core -from myapp import app, appbuilder,db,event_logger +from myapp import app, appbuilder,db from wtforms.ext.sqlalchemy.fields import QuerySelectField -from jinja2 import Environment, BaseLoader, DebugUndefined, StrictUndefined -import os,sys -from wtforms.validators import DataRequired, Length, NumberRange, Optional,Regexp +from jinja2 import Environment, BaseLoader, DebugUndefined +import os +from wtforms.validators import DataRequired, Length, Regexp from myapp.views.view_task import Task_ModelView,Task_ModelView_Api -from sqlalchemy import and_, or_, select +from sqlalchemy import or_ from myapp.exceptions import MyappException from wtforms import BooleanField, IntegerField,StringField, SelectField from flask_appbuilder.fieldwidgets import BS3TextFieldWidget,Select2ManyWidget,Select2Widget,BS3TextAreaFieldWidget @@ -34,11 +31,8 @@ MyappModelRestApi ) from flask import ( - current_app, - abort, flash, g, - Markup, make_response, redirect, request @@ -55,8 +49,8 @@ json_response ) -from flask_appbuilder import CompactCRUDMixin, expose -import pysnooper,datetime,time,json +from flask_appbuilder import expose +import datetime,time,json conf = app.config logging = app.logger @@ -81,7 +75,7 @@ def apply(self, query, func): -from sqlalchemy.exc import InvalidRequestError,OperationalError +from sqlalchemy.exc import InvalidRequestError # 将dag 转为argo pipeline yaml # @pysnooper.snoop(watch_explode=()) @@ -1050,9 +1044,6 @@ def web(self,pipeline_id): db.session.commit() print(pipeline_id) url = '/static/appbuilder/vison/index.html?pipeline_id=%s'%pipeline_id # 前后端集成完毕，这里需要修改掉 - data = { - "url": url - } return redirect('/frontend/showOutLink?url=%s'%urllib.parse.quote(url, safe="")) # 返回模板 # return self.render_template('link.html', data=data) diff --git a/myapp/views/view_runhistory.py b/myapp/views/view_runhistory.py index f8ea7aacd..27bf6743a 100644 --- a/myapp/views/view_runhistory.py +++ b/myapp/views/view_runhistory.py @@ -1,14 +1,13 @@ -from flask import render_template,redirect from flask_appbuilder.models.sqla.interface import SQLAInterface from myapp.models.model_job import RunHistory -from myapp import app, appbuilder,db,event_logger +from myapp import app, appbuilder,db from wtforms import SelectField from flask_appbuilder.fieldwidgets import Select2Widget from flask_babel import lazy_gettext as _ -from sqlalchemy import and_, or_, select +from sqlalchemy import or_ from .baseApi import ( MyappModelRestApi diff --git a/myapp/views/view_service_pipeline.py b/myapp/views/view_service_pipeline.py index 543a02439..6daaa8cf7 100644 --- a/myapp/views/view_service_pipeline.py +++ b/myapp/views/view_service_pipeline.py @@ -1,31 +1,26 @@ -from flask import render_template,redirect from flask_appbuilder.models.sqla.interface import SQLAInterface from flask_babel import lazy_gettext as _ from myapp.models.model_service_pipeline import Service_Pipeline from myapp.models.model_job import Repository -from flask import current_app, flash, jsonify +from flask import jsonify from flask_appbuilder.forms import GeneralModelConverter from myapp.utils import core -from myapp import app, appbuilder,db,event_logger +from myapp import app, appbuilder,db from wtforms.ext.sqlalchemy.fields import QuerySelectField -from wtforms.validators import DataRequired, Length, NumberRange, Optional,Regexp -from sqlalchemy import and_, or_, select +from wtforms.validators import DataRequired, Length, Regexp +from sqlalchemy import or_ from myapp.exceptions import MyappException -from wtforms import BooleanField, IntegerField,StringField, SelectField -from myapp.project import push_message,push_admin -from flask_appbuilder.fieldwidgets import BS3TextFieldWidget,Select2ManyWidget,Select2Widget,BS3TextAreaFieldWidget +from wtforms import StringField, SelectField +from flask_appbuilder.fieldwidgets import BS3TextFieldWidget,Select2ManyWidget,Select2Widget from myapp.forms import MyBS3TextAreaFieldWidget,MySelectMultipleField -import re,copy +import copy from .baseApi import ( MyappModelRestApi ) from flask import ( - current_app, - abort, flash, g, - Markup, make_response, redirect, request, @@ -40,8 +35,8 @@ json_response ) -from flask_appbuilder import CompactCRUDMixin, expose -import pysnooper,datetime,time,json +from flask_appbuilder import expose +import pysnooper,datetime, json conf = app.config logging = app.logger @@ -318,7 +313,7 @@ def build_http(self,service_pipeline): def build_mq_consumer(self,service_pipeline): namespace = conf.get('SERVICE_PIPELINE_NAMESPACE') name = service_pipeline.name - command = service_pipeline.command + service_pipeline.command image_secrets = conf.get('HUBSECRET', []) user_hubsecrets = db.session.query(Repository.hubsecret).filter(Repository.created_by_fk == g.user.id).all() if user_hubsecrets: @@ -398,7 +393,7 @@ def run_service_pipeline(self,service_pipeline_id): # # @event_logger.log_this @expose("/web/", methods=["GET"]) def web(self,service_pipeline_id): - service_pipeline = db.session.query(Service_Pipeline).filter_by(id=service_pipeline_id).first() + db.session.query(Service_Pipeline).filter_by(id=service_pipeline_id).first() # service_pipeline.dag_json = service_pipeline.fix_dag_json() # service_pipeline.expand = json.dumps(service_pipeline.fix_position(), indent=4, ensure_ascii=False) diff --git a/myapp/views/view_serving.py b/myapp/views/view_serving.py index 73b459c55..bc5229efe 100644 --- a/myapp/views/view_serving.py +++ b/myapp/views/view_serving.py @@ -1,24 +1,19 @@ -from flask import render_template,redirect from flask_appbuilder.models.sqla.interface import SQLAInterface from myapp.models.model_serving import Service from myapp.utils import core from flask_babel import lazy_gettext as _ -from myapp import app, appbuilder,db,event_logger +from myapp import app, appbuilder,db from myapp.models.model_job import Repository from wtforms.ext.sqlalchemy.fields import QuerySelectField from myapp import security_manager -import os,sys -from wtforms.validators import DataRequired, Length, NumberRange, Optional,Regexp +from wtforms.validators import DataRequired, Length, Regexp from wtforms import StringField from flask_appbuilder.fieldwidgets import BS3TextFieldWidget,Select2Widget from myapp.forms import MyBS3TextAreaFieldWidget from flask import ( - current_app, - abort, flash, g, redirect, - render_template, request ) from .base import ( @@ -30,9 +25,9 @@ from .baseApi import ( MyappModelRestApi ) -from myapp.views.view_team import Project_Filter,Project_Join_Filter,filter_join_org_project -from flask_appbuilder import CompactCRUDMixin, expose -import pysnooper,datetime,time,json +from myapp.views.view_team import Project_Join_Filter,filter_join_org_project +from flask_appbuilder import expose +import json conf = app.config diff --git a/myapp/views/view_task.py b/myapp/views/view_task.py index 480883bac..bb0405f04 100644 --- a/myapp/views/view_task.py +++ b/myapp/views/view_task.py @@ -1,26 +1,24 @@ -from flask import render_template,redirect from flask_appbuilder.models.sqla.interface import SQLAInterface from flask_babel import lazy_gettext as _ import uuid -from myapp.models.model_job import Repository,Images,Job_Template,Task,Pipeline +from myapp.models.model_job import Job_Template,Task,Pipeline from flask_appbuilder.forms import GeneralModelConverter from myapp.utils import core -from myapp import app, appbuilder,db,event_logger +from myapp import app, appbuilder,db from wtforms.ext.sqlalchemy.fields import QuerySelectField -from jinja2 import Environment, BaseLoader, DebugUndefined, StrictUndefined -import os,sys -from wtforms.validators import DataRequired, Length, NumberRange, Optional,Regexp +from jinja2 import Environment, BaseLoader, DebugUndefined +import os +from wtforms.validators import DataRequired, Length, Regexp from myapp.exceptions import MyappException -from wtforms import BooleanField, IntegerField,StringField, SelectField,FloatField,DateField,DateTimeField,SelectMultipleField,FormField,FieldList +from wtforms import BooleanField, IntegerField,StringField, SelectField,FloatField,DateField,DateTimeField,SelectMultipleField from flask_appbuilder.fieldwidgets import BS3TextFieldWidget,BS3PasswordFieldWidget,DatePickerWidget,DateTimePickerWidget,Select2ManyWidget,Select2Widget -from myapp.forms import MyBS3TextAreaFieldWidget,MySelect2Widget,MyCodeArea,MyLineSeparatedListField,MyJSONField,MyBS3TextFieldWidget,MySelectMultipleField +from myapp.forms import MyBS3TextAreaFieldWidget, MyLineSeparatedListField,MyJSONField,MyBS3TextFieldWidget from flask_wtf.file import FileField from .baseApi import ( MyappModelRestApi ) from flask import ( - current_app, flash, g, redirect @@ -31,7 +29,7 @@ ) from myapp.views.base import CompactCRUDMixin from flask_appbuilder import expose -import pysnooper,datetime,time,json +import datetime,time,json conf = app.config logging = app.logger diff --git a/myapp/views/view_team.py b/myapp/views/view_team.py index 2c1c6f733..9dbb33e4e 100644 --- a/myapp/views/view_team.py +++ b/myapp/views/view_team.py @@ -1,18 +1,16 @@ -from flask import render_template,redirect from flask_appbuilder.models.sqla.interface import SQLAInterface from flask_babel import lazy_gettext as _ from myapp.models.model_team import Project,Project_User -from wtforms import BooleanField, IntegerField, SelectField, StringField +from wtforms import SelectField, StringField from myapp.utils import core -from myapp import app, appbuilder,db +from myapp import appbuilder from wtforms.ext.sqlalchemy.fields import QuerySelectField from flask_appbuilder.fieldwidgets import Select2Widget from myapp.exceptions import MyappException -from myapp import conf, db, get_feature_flags, security_manager,event_logger +from myapp import db, security_manager from myapp.forms import MyBS3TextFieldWidget,MyBS3TextAreaFieldWidget -from wtforms.validators import DataRequired, Length, NumberRange, Optional,Regexp +from wtforms.validators import DataRequired from flask import ( - abort, flash, g ) @@ -24,8 +22,8 @@ from .baseApi import ( MyappModelRestApi ) -import pysnooper,datetime,time,json -from flask_appbuilder import CompactCRUDMixin, expose +import json +from flask_appbuilder import CompactCRUDMixin # table show界面下的 class Project_User_ModelView_Base(): diff --git a/myapp/views/view_train_model.py b/myapp/views/view_train_model.py index e810a1c79..c19c6468c 100644 --- a/myapp/views/view_train_model.py +++ b/myapp/views/view_train_model.py @@ -1,41 +1,31 @@ -from flask import render_template,redirect from flask_appbuilder.models.sqla.interface import SQLAInterface from myapp.models.model_train_model import Training_Model from myapp.models.model_serving import InferenceService from flask_babel import lazy_gettext as _ -from myapp import app, appbuilder,db,event_logger -import logging -import re +from myapp import app, appbuilder,db import uuid -from myapp.views.view_team import Project_Filter,Project_Join_Filter,filter_join_org_project +from myapp.views.view_team import Project_Join_Filter -from wtforms.validators import DataRequired, Length, NumberRange, Optional,Regexp -from wtforms import BooleanField, IntegerField, SelectField, StringField +from wtforms.validators import DataRequired, Length, Regexp +from wtforms import SelectField, StringField from flask_appbuilder.fieldwidgets import Select2Widget -from myapp.forms import MyBS3TextFieldWidget,MySelectMultipleField +from myapp.forms import MyBS3TextFieldWidget from flask import ( - current_app, - abort, flash, g, - Markup, - make_response, redirect ) from .base import ( - DeleteMixin, DeleteMixin, MyappFilter, MyappModelView, - json_response - ) from .baseApi import ( MyappModelRestApi ) -from flask_appbuilder import CompactCRUDMixin, expose -import pysnooper,datetime,time,json +from flask_appbuilder import expose +import datetime, json conf = app.config diff --git a/myapp/views/view_workflow.py b/myapp/views/view_workflow.py index fedde82af..ef6bc1b50 100644 --- a/myapp/views/view_workflow.py +++ b/myapp/views/view_workflow.py @@ -1,15 +1,14 @@ -from flask import render_template,redirect from flask_appbuilder.models.sqla.interface import SQLAInterface from flask_babel import gettext as __ -from myapp.models.model_job import Pipeline,Workflow,Tfjob,Xgbjob,RunHistory,Pytorchjob +from myapp.models.model_job import Pipeline,Workflow from flask_appbuilder.actions import action -from myapp.project import push_message,push_admin -from myapp import app, appbuilder,db,event_logger +from myapp.project import push_message +from myapp import app, appbuilder,db -from sqlalchemy import and_, or_, select +from sqlalchemy import or_ from myapp.utils.py import py_k8s @@ -17,12 +16,9 @@ MyappModelRestApi ) from flask import ( - current_app, abort, flash, g, - Markup, - make_response, redirect ) @@ -31,8 +27,8 @@ MyappFilter, MyappModelView, ) -from flask_appbuilder import CompactCRUDMixin, expose -import pysnooper,datetime,time,json +from flask_appbuilder import expose +import datetime, json conf = app.config logging = app.logger @@ -126,7 +122,7 @@ def base_muldelete(self,items): k8s_client = py_k8s.K8s(kubeconfig) crd_info = conf.get("CRD_INFO", {}).get(self.crd_name, {}) if crd_info: - crd_names = k8s_client.delete_crd(group=crd_info['group'],version=crd_info['version'],plural=crd_info['plural'],namespace=item.namespace,name=item.name) + k8s_client.delete_crd(group=crd_info['group'],version=crd_info['version'],plural=crd_info['plural'],namespace=item.namespace,name=item.name) # db_crds = db.session.query(self.datamodel.obj).filter(self.datamodel.obj.name.in_(crd_names)).all() # for db_crd in db_crds: # db_crd.status = 'Deleted'