From f16a44da828ec8b62aa7fa1ed3d49291f47d6043 Mon Sep 17 00:00:00 2001 From: Max Jakob Date: Fri, 2 Sep 2022 20:14:44 +0200 Subject: [PATCH] Add mypy support (#288) * Add mypy support * Add explicit version for mypy * newer mypy version default * align names with Ruby's .rbi * add test * declare Dockerfile variable in outer scope * --mypy_grpc_out * add to mypy to renovate config * remove pip from final container * Update .github/renovate.json5 Co-authored-by: Ido David <36866853+ido-namely@users.noreply.github.com> * add file assertion * add test for consistency of language and --with-pyi * remove extra whitespace Co-authored-by: Ido David <36866853+ido-namely@users.noreply.github.com> --- .github/renovate.json5 | 10 ++++++++++ Dockerfile | 13 ++++++++++++- all/entrypoint.sh | 15 +++++++++++++++ all/test/all_test.go | 21 +++++++++++++++++++++ build.sh | 1 + variables.sh | 1 + 6 files changed, 60 insertions(+), 1 deletion(-) diff --git a/.github/renovate.json5 b/.github/renovate.json5 index 445eab8c..be10cd24 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -180,6 +180,16 @@ depNameTemplate: "debian", datasourceTemplate: "docker", versioningTemplate: "debian", + }, + { + fileMatch: [ + "variables.sh$" + ], + matchStrings: [ + "MYPY_VERSION=\\$\\{MYPY_VERSION:-(?.*?)\\}\\n" + ], + depNameTemplate: "mypy-protobuf", + datasourceTemplate: "pypi", } ], packageRules: [ diff --git a/Dockerfile b/Dockerfile index eb780b99..7cfd0fda 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,7 @@ ARG go_envoyproxy_pgv_version ARG go_mwitkow_gpv_version ARG go_protoc_gen_go_version ARG go_protoc_gen_go_grpc_version +ARG mypy_version FROM golang:$go_version-$debian_version AS build @@ -28,6 +29,7 @@ ARG go_mwitkow_gpv_version ARG uber_prototool_version ARG go_protoc_gen_go_version ARG go_protoc_gen_go_grpc_version +ARG mypy_version RUN set -ex && apt-get update && apt-get install -y --no-install-recommends \ @@ -42,7 +44,8 @@ RUN set -ex && apt-get update && apt-get install -y --no-install-recommends \ autoconf \ zlib1g-dev \ libssl-dev \ - clang + clang \ + python3-pip WORKDIR /tmp RUN git clone --depth 1 --shallow-submodules -b v$grpc_version.x --recursive https://github.com/grpc/grpc && \ @@ -110,6 +113,9 @@ RUN curl -fsSL "https://github.com/grpc/grpc-web/releases/download/${grpc_web_ve -o /tmp/grpc_web_plugin && \ chmod +x /tmp/grpc_web_plugin +# Add mypy support +RUN pip3 install -t /opt/mypy-protobuf mypy-protobuf==${mypy_version} + FROM debian:$debian_version-slim AS protoc-all ARG grpc_version @@ -173,6 +179,11 @@ COPY --from=build /go/pkg/mod/github.com/envoyproxy/protoc-gen-validate@v${go_en COPY --from=build /go/pkg/mod/github.com/mwitkow/go-proto-validators@v${go_mwitkow_gpv_version}/ /opt/include/github.com/mwitkow/go-proto-validators/ +# Copy mypy +COPY --from=build /opt/mypy-protobuf/ /opt/mypy-protobuf/ +RUN mv /opt/mypy-protobuf/bin/* /usr/local/bin/ +ENV PYTHONPATH="${PYTHONPATH}:/opt/mypy-protobuf/" + ADD all/entrypoint.sh /usr/local/bin RUN chmod +x /usr/local/bin/entrypoint.sh diff --git a/all/entrypoint.sh b/all/entrypoint.sh index c325b55a..02adf877 100755 --- a/all/entrypoint.sh +++ b/all/entrypoint.sh @@ -16,6 +16,7 @@ printUsage() { echo " --lint CHECKS Enable linting protoc-lint (CHECKS are optional - see https://github.com/ckaznocha/protoc-gen-lint#optional-checks)" echo " --with-gateway Generate grpc-gateway files (experimental)." echo " --with-docs FORMAT Generate documentation (FORMAT is optional - see https://github.com/pseudomuto/protoc-gen-doc#invoking-the-plugin)" + echo " --with-pyi Generate mypy stub files (.pyi files) - see https://github.com/nipunn1313/mypy-protobuf" echo " --with-rbi Generate Sorbet type declaration files (.rbi files) - see https://github.com/coinbase/protoc-gen-rbi" echo " --with-typescript Generate TypeScript declaration files (.d.ts files) - see https://github.com/improbable-eng/ts-protoc-gen#readme" echo " --with-validator Generate validations for (${VALIDATOR_SUPPORTED_LANGUAGES[@]}) - see https://github.com/envoyproxy/protoc-gen-validate" @@ -50,6 +51,7 @@ GEN_VALIDATOR=false VALIDATOR_SUPPORTED_LANGUAGES=("go" "gogo" "cpp" "java" "python") DOCS_FORMAT="html,index.html" GEN_RBI=false +GEN_PYI=false GEN_TYPESCRIPT=false LINT=false LINT_CHECKS="" @@ -134,6 +136,10 @@ while test $# -gt 0; do GEN_RBI=true shift ;; + --with-pyi) + GEN_PYI=true + shift + ;; --with-typescript) GEN_TYPESCRIPT=true shift @@ -296,6 +302,11 @@ if [[ "$GEN_RBI" == true && "$GEN_LANG" != "ruby" ]]; then exit 1 fi +if [[ "$GEN_PYI" == true && "$GEN_LANG" != "python" ]]; then + echo "Generating .pyi mypy stub files is a Python specific option." + exit 1 +fi + if [[ "$GEN_TYPESCRIPT" == true && "$GEN_LANG" != "node" ]]; then echo "Generating TypeScript declaration files is Node specific." exit 1 @@ -432,6 +443,10 @@ if [[ $GEN_RBI == true ]]; then GEN_STRING="$GEN_STRING --rbi_out=$OUT_DIR" fi +if [[ $GEN_PYI == true ]]; then + GEN_STRING="$GEN_STRING --mypy_out=$OUT_DIR --mypy_grpc_out=$OUT_DIR" +fi + if [[ $GEN_TYPESCRIPT == true ]]; then GEN_STRING="$GEN_STRING --plugin=protoc-gen-ts=$(which protoc-gen-ts) --ts_out=$GRPC_OUT:$OUT_DIR" fi diff --git a/all/test/all_test.go b/all/test/all_test.go index 92d3c6b8..dc1b672d 100644 --- a/all/test/all_test.go +++ b/all/test/all_test.go @@ -140,6 +140,27 @@ func (s *TestSuite) TestAllCases() { {fileName: "/all/test/test_pb2_grpc.py"}, }, }, + "python pyi": { + lang: "python", + protofileName: "all/test/test.proto", + expectedOutputDir: "gen/pb_python", + fileExpectations: []FileExpectation{ + {fileName: "/__init__.py"}, + {fileName: "/all/__init__.py"}, + {fileName: "/all/test/__init__.py"}, + {fileName: "/all/test/test_pb2.py"}, + {fileName: "/all/test/test_pb2.pyi"}, + {fileName: "/all/test/test_pb2_grpc.py"}, + {fileName: "/all/test/test_pb2_grpc.pyi"}, + }, + extraArgs: []string{"--with-pyi"}, + }, + "python pyi with inconsistent language parameter": { + lang: "ruby", + protofileName: "all/test/test.proto", + expectedExitCode: 1, + extraArgs: []string{"--with-pyi"}, + }, "python with alternative output dir": { lang: "python", protofileName: "all/test/test.proto", diff --git a/build.sh b/build.sh index d9030300..06020571 100755 --- a/build.sh +++ b/build.sh @@ -16,6 +16,7 @@ for build in ${BUILDS[@]}; do --build-arg go_version="${GO_VERSION}" \ --build-arg uber_prototool_version="${UBER_PROTOTOOL_VERSION}" \ --build-arg scala_pb_version="${SCALA_PB_VERSION}" \ + --build-arg mypy_version="${MYPY_VERSION}" \ --build-arg node_version="${NODE_VERSION}" \ --build-arg node_grpc_tools_node_protoc_ts_version="${NODE_GRPC_TOOLS_NODE_PROTOC_TS_VERSION}" \ --build-arg node_grpc_tools_version="${NODE_GRPC_TOOLS_VERSION}" \ diff --git a/variables.sh b/variables.sh index 3ccf6231..310f4bf5 100755 --- a/variables.sh +++ b/variables.sh @@ -14,6 +14,7 @@ GRPC_WEB_VERSION=${GRPC_WEB_VERSION:-1.3.1} GRPC_GATEWAY_VERSION=${GRPC_GATEWAY_VERSION:-v2.0.1} UBER_PROTOTOOL_VERSION=${UBER_PROTOTOOL_VERSION:-1.10.0} SCALA_PB_VERSION=${SCALA_PB_VERSION:-0.11.0} +MYPY_VERSION=${MYPY_VERSION:-3.3.0} NODE_VERSION=${NODE_VERSION:-14} NODE_GRPC_TOOLS_NODE_PROTOC_TS_VERSION=${NODE_GRPC_TOOLS_NODE_PROTOC_TS_VERSION:-5.3.2} NODE_GRPC_TOOLS_VERSION=${NODE_GRPC_TOOLS_VERSION:-1.11.2}