style: upgrade linters(flake8 & mypy) and fix linters issues for work…

…loads (#206)

apiserver/paasng/paasng/settings/utils.py

@@ -88,7 +88,7 @@ def get_paas_service_jwt_clients(settings: LazySettings) -> List:
 
 
 def get_default_keepalive_options() -> Optional[Dict]:
-    """Mac OS's socket module does not has below attrs, return empty options instead"""
+    """MacOS's socket module does not have below attrs, return empty options instead"""
     try:
         return {
             socket.TCP_KEEPIDLE: 60,  # type: ignore

workloads/.flake8

@@ -1,14 +1,30 @@
 [flake8]
-ignore = C901,F405,F403,W504,E741,E125,W503,F841,E203,E231,C408,C403,C407,C401
+format = pylint
+extend-ignore = 
+    # 禁用原因，与 black 格式化后的样式冲突
+    E203,
+    # 禁用原因，允许使用 import *（仅在个别情况下配合 __all__ 使用，不建议大量用）
+    F403,F405,
+    # 禁用原因，版权头信息使用多行字符串引起，如批量修改成普通 # 号注释后可解
+    E402,
+    # flake8-comprehensions: 禁用 C408 后，允许使用 dict() 来构建字典对象
+    C408,
+    # flake8-bugbear: 禁用 B009,B010 后，允许对常量属性名使用 getattr/setattr
+    B009,B010,
+    # flake8-pie: 忽略多项普适性不佳的规则，比如：函数返回前定义临时变量、模糊的异常捕获、
+    #   非必要的 class 定义、可使用 bool() 替换的条件分支、日志未使用 %s 格式等
+    PIE781,PIE786,PIE798,PIE801,PIE803,
+    # flake8-pie: 忽略与 celery 任务相关的规则
+    PIE783,PIE784,PIE785,
+max-line-length = 119
+max-complexity = 12
+show_source = true
+statistics = true
+count = true
 exclude = 
-    *migrations*,
     *.pyc,
     .git,
     __pycache__,
-    */templates_module*,
-max-line-length=119
-max-complexity=12
-format = "pylint"
-show_source = True
-statistics = True
-count = True
+    *migrations*,
+    */bin/*,
+    */dev_utils/*,

workloads/Dockerfile

@@ -9,7 +9,7 @@ ENV LC_ALL=C.UTF-8 \
     LANG=C.UTF-8
 
 RUN pip install --upgrade pip
-RUN pip install poetry==1.1.10
+RUN pip install poetry==1.3.2
 
 # Change security level of openssl to lower value in order to avoid "CA_MD_TOO_WEAK" error
 # See https://stackoverflow.com/questions/52218876/how-to-fix-ssl-issue-ssl-ctx-use-certificate-ca-md-too-weak-on-python-zeep?rq=1
@@ -23,7 +23,8 @@ ADD ./poetry.lock ./
 # Install dependecies in system
 # unittest 要安装 dev 依赖
 ARG POETRY_INSTALL_EXTRA_ARGS
-RUN poetry config virtualenvs.create false && poetry install ${POETRY_INSTALL_EXTRA_ARGS}
+# Disable "new installer" to fix https://github.com/python-poetry/poetry/issues/6301
+RUN poetry config virtualenvs.create false && poetry config experimental.new-installer false && poetry install ${POETRY_INSTALL_EXTRA_ARGS}
 
 ADD ./bin/start.sh ./
 ADD ./bin/start_celery.sh ./

workloads/dev_utils/unittest/Dockerfile.devops.unittest

@@ -9,7 +9,7 @@ ENV LC_ALL=C.UTF-8 \
     LANG=C.UTF-8
 
 RUN pip install --upgrade pip
-RUN pip install poetry==1.1.11
+RUN pip install poetry==1.3.2
 
 # Change security level of openssl to lower value in order to avoid "CA_MD_TOO_WEAK" error
 # See https://stackoverflow.com/questions/52218876/how-to-fix-ssl-issue-ssl-ctx-use-certificate-ca-md-too-weak-on-python-zeep?rq=1
@@ -21,8 +21,8 @@ ADD ./pyproject.toml ./
 ADD ./poetry.lock ./
 
 # Install dependecies in system
-# unittest 要安装 dev 依赖
-RUN poetry config virtualenvs.create false && poetry install
+# Disable "new installer" to fix https://github.com/python-poetry/poetry/issues/6301
+RUN poetry config virtualenvs.create false && poetry config experimental.new-installer false && poetry install
 
 ADD ./bin/start.sh ./
 ADD ./bin/start_celery.sh ./

workloads/paas_wl/paas_wl/admin/helpers.py

@@ -82,7 +82,7 @@ def fetch_paas_cobj_info(client: EnhancedApiClient, crd_exists: Dict[str, bool])
 
             if hookStatuses := bkapp.get('status', {}).get('hookStatuses', []):
                 # 任何 Hook 状态不健康，该 bkapp 都被认为非 ready
-                if any([hs.get('phase') != 'Healthy' for hs in hookStatuses]):
+                if any(hs.get('phase') != 'Healthy' for hs in hookStatuses):
                     not_ready_bkapps.append(bkapp_name)
                     continue
 

workloads/paas_wl/paas_wl/cluster/constants.py

@@ -33,7 +33,7 @@ class ClusterType(str, StructuredEnum):
     VIRTUAL = EnumField('virtual', label=_('虚拟集群'))
 
 
-class ClusterFeatureFlag(FeatureFlag):
+class ClusterFeatureFlag(FeatureFlag):  # type: ignore
     """集群特性标志"""
 
     ENABLE_EGRESS_IP = FeatureFlagField(label=_('支持提供出口 IP'))

workloads/paas_wl/paas_wl/cluster/loaders.py

@@ -24,7 +24,7 @@
 from django.db.models import QuerySet
 from kubernetes.config.kube_config import FileOrData, KubeConfigLoader
 
-from paas_wl.cluster.models import APIServer, Cluster, EnhancedConfiguration
+from paas_wl.cluster.models import Cluster, EnhancedConfiguration
 
 
 class DBConfigLoader:
@@ -56,7 +56,7 @@ def _load(self):
             cert_file = FileOrData(config, file_key_name=None, data_key_name="client-certificate-data").as_file()
             key_file = FileOrData(config, file_key_name=None, data_key_name="client-key-data").as_file()
 
-            for api_server in cluster.api_servers.order_by("created"):  # type: APIServer
+            for api_server in cluster.api_servers.order_by("created"):
                 self.configurations[cluster.name].append(
                     EnhancedConfiguration.create(
                         host=api_server.host,

workloads/paas_wl/paas_wl/monitoring/metrics/models.py

@@ -86,8 +86,8 @@ def get_instance_metrics(
         self,
         instance_name: str,
         resource_types: List['MetricsResourceType'],
-        series_type: 'MetricsSeriesType' = None,
-        time_range: 'MetricSmartTimeRange' = None,
+        series_type: Optional[MetricsSeriesType] = None,
+        time_range: Optional[MetricSmartTimeRange] = None,
     ) -> List['MetricsResourceResult']:
         """query metrics at Engine Application level"""
 
@@ -124,7 +124,7 @@ def get_all_instances_metrics(
         self,
         resource_types: List['MetricsResourceType'],
         time_range: 'MetricSmartTimeRange',
-        series_type: 'MetricsSeriesType' = None,
+        series_type: Optional[MetricsSeriesType] = None,
     ) -> List['MetricsInstanceResult']:
 
         all_instances_metrics = []

workloads/paas_wl/paas_wl/networking/ingress/addrs.py

@@ -102,4 +102,4 @@ def _make_url(self, https_enabled: bool, host: str, path: str = '/') -> str:
     @staticmethod
     def _sort(addrs: List[Address]) -> List[Address]:
         """Sort address list, short and not reserved address first"""
-        return list(sorted(addrs, key=lambda addr: (addr.is_sys_reserved, len(addr.url))))
+        return sorted(addrs, key=lambda addr: (addr.is_sys_reserved, len(addr.url)))

workloads/paas_wl/paas_wl/networking/ingress/management/commands/patch_legacy_ingresses.py

@@ -62,37 +62,40 @@ def handle(self, apps, dry_run, pattern, process_type, with_create, app_created_
             qs = qs.filter(created__gte=app_created_after_dt)
 
         for app in qs:
-            # Only process released apps
-            if not Release.objects.filter(app=app, build__isnull=False).exists():
-                continue
-
-            logger.info(f"checking ingress for app {app.name} with pattern {pattern}")
-            logger.info(f"app was created at {app.created}")
-
-            mgr = LegacyAppIngressMgr(app)
-            service_name = None
-            can_sync = False
+            self._patch_app_ingress(app, dry_run, pattern, process_type, with_create)
+
+    def _patch_app_ingress(self, app: App, dry_run, pattern, process_type, with_create):
+        # Only process released apps
+        if not Release.objects.filter(app=app, build__isnull=False).exists():
+            return False
+
+        logger.info(f"checking ingress for app {app.name} with pattern {pattern}")
+        logger.info(f"app was created at {app.created}")
+
+        mgr = LegacyAppIngressMgr(app)
+        service_name = None
+        can_sync = False
+        try:
+            domains = mgr.list_desired_domains()
+        except Exception:
+            logger.exception("list domains failed for app %s", app.name)
+            return
+
+        for domain in domains:
+            if fnmatch(domain.host, pattern):
+                can_sync = not dry_run
+                print(f"domain {domain.host} match, ready to sync")
+            else:
+                print(f"domain {domain.host} mismatch, abort")
+                break
+
+        if with_create:
+            service_name = make_service_name(app, process_type)
+            print(f"will create ingress for process {process_type}: {service_name}")
+
+        if can_sync:
+            print(f"syncing ingress for app {app.name}")
             try:
-                domains = mgr.list_desired_domains()
-            except Exception:
-                logger.exception("list domains failed for app %s", app.name)
-                continue
-
-            for domain in domains:
-                if fnmatch(domain.host, pattern):
-                    can_sync = not dry_run
-                    print(f"domain {domain.host} match, ready to sync")
-                else:
-                    print(f"domain {domain.host} mismatch, abort")
-                    break
-
-            if with_create:
-                service_name = make_service_name(app, process_type)
-                print(f"will create ingress for process {process_type}: {service_name}")
-
-            if can_sync:
-                print(f"syncing ingress for app {app.name}")
-                try:
-                    mgr.sync(service_name)
-                except Exception as err:
-                    logger.error("sync ingresses failed for app %s, err: %s", app.name, err)
+                mgr.sync(service_name)
+            except Exception as err:
+                logger.error("sync ingresses failed for app %s, err: %s", app.name, err)

workloads/paas_wl/paas_wl/networking/ingress/management/commands/refresh_cert.py

@@ -58,7 +58,7 @@ def handle_refresh(self, options):
         d_apps = self.find_subdomain_apps(cert)
         p_apps = self.find_subpath_apps(cert)
         # Sort the app list to get a deterministic result
-        apps = sorted(list(set(d_apps) | set(p_apps)), key=lambda app: app.name)
+        apps = sorted(set(d_apps) | set(p_apps), key=lambda app: app.name)
         self.print(f'Found affected apps(sub-domain/sub-path): {len(d_apps)}/{len(p_apps)}')
         self.print(f'Deduplicated overall count: {len(apps)}')
 

workloads/paas_wl/paas_wl/networking/ingress/managers/base.py

@@ -61,7 +61,7 @@ def delete(self):
         """Delete the default ingress rule"""
         ingress_kmodel.delete_by_name(self.app, self.ingress_name, non_grace_period=True)
 
-    def sync(self, default_service_name: str = None, delete_when_empty: bool = False):
+    def sync(self, default_service_name: Optional[str] = None, delete_when_empty: bool = False):
         """Sync with kubernetes apiserver
 
         :param delete_when_empty: when True, will try to delete ingress object if it has no domains,
@@ -161,7 +161,7 @@ def __init__(self, app: 'App', ingress_name: str):
     def sync(
         self,
         domains: List[PIngressDomain],
-        default_service_name: str = None,
+        default_service_name: Optional[str] = None,
         server_snippet: str = '',
         configuration_snippet: str = '',
         annotations: Optional[Dict] = None,

workloads/paas_wl/paas_wl/networking/ingress/managers/domain.py

@@ -55,7 +55,7 @@ def save_subdomains(app: App, domains: List[AutoGenDomain]) -> Set[App]:
     """
     hosts = [domain.host for domain in domains]
     existed_domains = AppDomain.objects.filter(region=app.region, host__in=hosts, source=AppDomainSource.AUTO_GEN)
-    affected_apps = set(obj.app for obj in existed_domains)
+    affected_apps = {obj.app for obj in existed_domains}
 
     for domain in domains:
         obj, _ = AppDomain.objects.update_or_create(

workloads/paas_wl/paas_wl/networking/ingress/managers/misc.py

@@ -17,7 +17,7 @@
 to the current version of the project delivered to anyone in the future.
 """
 import logging
-from typing import Iterable, List, NamedTuple
+from typing import Iterable, List, NamedTuple, Optional
 
 from paas_wl.cluster.utils import get_cluster_by_app
 from paas_wl.networking.ingress.entities.ingress import PIngressDomain
@@ -60,7 +60,7 @@ def list(self) -> Iterable[AppIngressMgr]:
         for domain in Domain.objects.filter(module_id=env.module_id, environment_id=env.id):
             yield CustomDomainIngressMgr(domain)
 
-    def sync_ignore_empty(self, default_service_name: str = None):
+    def sync_ignore_empty(self, default_service_name: Optional[str] = None):
         """Sync current ingress resources to apiserver,
         will not raise exceptions when any manager has no related domains.
         """

workloads/paas_wl/paas_wl/networking/ingress/managers/subpath.py

@@ -55,7 +55,7 @@ def save_subpaths(app: App, subpaths: List[str]) -> Set[App]:
     existed_subpaths = AppSubpath.objects.filter(
         region=app.region, subpath__in=subpaths, source=AppSubpathSource.DEFAULT
     )
-    affected_apps = set(obj.app for obj in existed_subpaths)
+    affected_apps = {obj.app for obj in existed_subpaths}
 
     for subpath in subpaths:
         AppSubpath.objects.update_or_create(

workloads/paas_wl/paas_wl/networking/ingress/serializers.py

@@ -116,7 +116,7 @@ def validate_cert(d):
 class AppDomainSharedCertSLZ(serializers.ModelSerializer):
     cert_data = serializers.CharField(validators=[validate_cert], required=True)
 
-    class Meta(object):
+    class Meta:
         model = AppDomainSharedCert
         exclude: List = []
 
@@ -126,7 +126,7 @@ class UpdateAppDomainSharedCertSLZ(serializers.ModelSerializer):
 
     cert_data = serializers.CharField(validators=[validate_cert], required=True)
 
-    class Meta(object):
+    class Meta:
         model = AppDomainSharedCert
         fields = ['cert_data', 'key_data', 'auto_match_cns']
 

workloads/paas_wl/paas_wl/platform/applications/management/commands/delete_slug.py

@@ -88,7 +88,7 @@ def handle(self, match_pattern, max_reserved_num_per_env, dry_run, *args, **opti
             # 顺序入队
             bucket_dict[str(build.app.name)].append(build)
 
-        for app_name, bucket in bucket_dict.items():
+        for _, bucket in bucket_dict.items():
             # 按创建时间倒序查询, 并顺序入队后, 目前队列则是按时间顺序排列,
             # 因此这里只需要保留前 max_reserved_num_per_env 个.
             for build in bucket.builds[max_reserved_num_per_env:]:

workloads/paas_wl/paas_wl/platform/applications/models/release.py

@@ -36,7 +36,7 @@ def new(
         owner: str,
         build: 'Build',
         procfile: Dict[str, str],
-        summary: str = None,
+        summary: Optional[str] = None,
         config: Optional['Config'] = None,
     ):
         """Create a new release

workloads/paas_wl/paas_wl/platform/system_api/views.py

@@ -469,7 +469,7 @@ def get_resource_metric_manager(self, process_type):
 
         try:
             bcs_cluster_id = cluster.annotations["bcs_cluster_id"]
-        except KeyError as e:
+        except KeyError:
             raise error_codes.QUERY_RESOURCE_METRIC_FAILED.f("进程所在集群未关联 BCS 信息, 不支持该功能")
 
         try:

workloads/paas_wl/paas_wl/release_controller/builder/executor.py

@@ -18,7 +18,7 @@
 """
 import logging
 from functools import partial
-from typing import TYPE_CHECKING, Dict
+from typing import TYPE_CHECKING, Dict, Optional
 
 from django.conf import settings
 from django.utils.encoding import force_text
@@ -60,7 +60,7 @@ def __init__(self, bp: BuildProcess, stream: Stream):
         self.procedure = partial(BuildProcedure, stream)
         self._builder_name = generate_builder_name(self.app)
 
-    def execute(self, metadata=None):
+    def execute(self, metadata: Optional[Dict] = None):
         """Execute the build process"""
         try:
             with self.procedure("准备构建环境"):
@@ -164,7 +164,7 @@ def start_slugbuilder(self, pod_template: SlugBuilderTemplate) -> str:
         logger.debug('SlugBuilder created: %s', slug_builder_name)
         return slug_builder_name
 
-    def create_and_bind_build_instance(self, metadata: Dict = None) -> Build:
+    def create_and_bind_build_instance(self, metadata: Optional[Dict] = None) -> Build:
         """Create the Build instance and bind it to self.BuildProcess instance
 
         :param dict metadata: Metadata to be stored in Build instance, such as `procfile`

workloads/paas_wl/paas_wl/release_controller/builder/procedures.py

@@ -18,7 +18,7 @@
 """
 import logging
 import os
-from typing import TYPE_CHECKING, Dict
+from typing import TYPE_CHECKING, Dict, Optional
 
 from blue_krill.storages.blobstore.base import SignatureType
 from django.conf import settings
@@ -50,7 +50,7 @@ def generate_slug_path(bp: BuildProcess):
     return f'{app.region}/home/{slug_name}/push'
 
 
-def generate_builder_env_vars(bp: BuildProcess, metadata: Dict) -> Dict[str, str]:
+def generate_builder_env_vars(bp: BuildProcess, metadata: Optional[Dict]) -> Dict[str, str]:
     """generate all env vars needed for building"""
     bucket = settings.BLOBSTORE_S3_BUCKET_NAME
     store = make_blob_store(bucket)
@@ -126,7 +126,7 @@ def update_env_vars_with_metadata(env_vars: Dict, metadata: Dict):
         env_vars["REQUIRED_BUILDPACKS"] = buildpacks
 
 
-def prepare_slugbuilder_template(app: 'App', env_vars: Dict, metadata: Dict) -> SlugBuilderTemplate:
+def prepare_slugbuilder_template(app: 'App', env_vars: Dict, metadata: Optional[Dict]) -> SlugBuilderTemplate:
     """Prepare the template for running a slug builder
 
     :param app: App to build, provide info about namespace, region and etc.

workloads/paas_wl/paas_wl/release_controller/process/events.py

@@ -28,6 +28,7 @@
 logger = logging.getLogger(__name__)
 
 
+@dataclass
 class ProcessBaseEvent:
     type: IntEnum
     invoker: Any