python: Integrate linting (#15886)

- moves all tooling configs to repo root to allow using them directly and make explicit
- moves the python linting job to format_pre only and includes a diff there

Signed-off-by: Ryan Northey <ryan@synca.io>

Author: phlax

.azure-pipelines/pipelines.yml

@@ -26,6 +26,12 @@ stages:
               BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
               GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
             displayName: "Run format pre-checks"
+          - task: PublishBuildArtifacts@1
+            inputs:
+              pathtoPublish: "$(Build.StagingDirectory)/fix_format_pre.diff"
+              artifactName: format
+            # not all have fixes so improve condition/handling
+            condition: failed()
 
       - job: format
         dependsOn: ["format_pre"]

.devcontainer/devcontainer.json

@@ -22,7 +22,7 @@
     "python.pythonPath": "/usr/bin/python3",
     "python.formatting.provider": "yapf",
     "python.formatting.yapfArgs": [
-      "--style=${workspaceFolder}/tools/code_format/.style.yapf"
+      "--style=${workspaceFolder}/.style.yapf"
     ],
     "files.exclude": {
       "**/.clangd/**": true,

tools/code_format/flake8.conf .flake8

@@ -2,7 +2,7 @@
 [flake8]
 
 # TODO(phlax): make this an exclusive list and enable most tests
-select = N802,F401,E111,F821
+select = N802,F401,E111,F821,E9,F63,F72,F82
 
 # TODO(phlax): exclude less
-exclude = build_docs,.git,generated,test,examples
+exclude = build_docs,.git,generated,test,examples,venv

tools/code_format/.style.yapf .style.yapf

@@ -407,7 +407,6 @@ elif [[ "$CI_TARGET" == "fix_format" ]]; then
 
   echo "fix_format..."
   "${ENVOY_SRCDIR}"/tools/code_format/check_format.py fix
-  "${ENVOY_SRCDIR}"/tools/code_format/format_python_tools.sh fix
   BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS[*]}" "${ENVOY_SRCDIR}"/tools/proto_format/proto_format.sh fix --test
   exit 0
 elif [[ "$CI_TARGET" == "check_format" ]]; then
@@ -418,7 +417,6 @@ elif [[ "$CI_TARGET" == "check_format" ]]; then
   "${ENVOY_SRCDIR}"/tools/code_format/check_format_test_helper.sh --log=WARN
   echo "check_format..."
   "${ENVOY_SRCDIR}"/tools/code_format/check_format.py check
-  "${ENVOY_SRCDIR}"/tools/code_format/format_python_tools.sh check
   BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS[*]}" "${ENVOY_SRCDIR}"/tools/proto_format/proto_format.sh check --test
   exit 0
 elif [[ "$CI_TARGET" == "check_repositories" ]]; then

ci/do_ci.sh

@@ -8,6 +8,8 @@
 FAILED=()
 CURRENT=""
 
+DIFF_OUTPUT="${DIFF_OUTPUT:-/build/fix_format_pre.diff}"
+
 read -ra BAZEL_BUILD_OPTIONS <<< "${BAZEL_BUILD_OPTIONS:-}"
 
 
@@ -33,16 +35,26 @@ trap exit 1 INT
 
 # TODO: move these to bazel
 CURRENT=glint
-./tools/code_format/glint.sh
+"${ENVOY_SRCDIR}"/tools/code_format/glint.sh
 
 CURRENT=shellcheck
-./tools/code_format/check_shellcheck_format.sh check
+"${ENVOY_SRCDIR}"/tools/code_format/check_shellcheck_format.sh check
 
 CURRENT=configs
 bazel run "${BAZEL_BUILD_OPTIONS[@]}" //configs:example_configs_validation
 
-CURRENT=flake8
-bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/code_format:python_flake8 "$(pwd)"
+# TODO(phlax): update to use bazel and python_flake8/python_check
+#              this will simplify this code to a single line
+CURRENT=python
+"${ENVOY_SRCDIR}"/tools/code_format/format_python_tools.sh check || {
+    "${ENVOY_SRCDIR}"/tools/code_format/format_python_tools.sh fix
+    git diff HEAD | tee "${DIFF_OUTPUT}"
+    raise () {
+        # this throws an error without exiting
+        return 1
+    }
+    raise
+}
 
 if [[ "${#FAILED[@]}" -ne "0" ]]; then
     echo "TESTS FAILED:" >&2

ci/format_pre.sh

@@ -10,13 +10,11 @@ exports_files([
     "check_format.py",
     "header_order.py",
     "envoy_build_fixer.py",
-    "flake8.conf",
 ])
 
 py_binary(
     name = "python_flake8",
     srcs = ["python_flake8.py"],
-    data = [":flake8.conf"],
     visibility = ["//visibility:public"],
     deps = [
         requirement("flake8"),

tools/code_format/BUILD

@@ -39,10 +39,7 @@ def validate_format(fix=False):
     successful_update_files = set()
     for python_file in collect_files():
         reformatted_source, encoding, changed = FormatFile(
-            python_file,
-            style_config='tools/code_format/.style.yapf',
-            in_place=fix,
-            print_diff=not fix)
+            python_file, style_config='.style.yapf', in_place=fix, print_diff=not fix)
         if not fix:
             fixes_required = True if changed else fixes_required
             if reformatted_source:

tools/code_format/format_python_tools.py

@@ -1,10 +1,8 @@
 import subprocess
 import sys
 
-FLAKE8_CONFIG = "tools/code_format/flake8.conf"
-
 # explicitly use python3 linter
-FLAKE8_COMMAND = ("python3", "-m", "flake8", "--config", FLAKE8_CONFIG, ".")
+FLAKE8_COMMAND = ("python3", "-m", "flake8", ".")
 
 
 def main():