forked from tensorflow/models
-
Notifications
You must be signed in to change notification settings - Fork 0
/
pylint.sh
171 lines (143 loc) · 5.26 KB
/
pylint.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
#!/bin/bash
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
#
# Pylint wrapper extracted from main TensorFlow, sharing same exceptions.
# As this is meant for smaller repos, drops "modified files" checking in favor
# of full-repo checking.
set -euo pipefail
# Download latest configs from main TensorFlow repo.
wget -q -O /tmp/pylintrc https://raw.githubusercontent.com/tensorflow/tensorflow/master/tensorflow/tools/ci_build/pylintrc
wget -q -O /tmp/pylint_allowlist https://raw.githubusercontent.com/tensorflow/tensorflow/master/tensorflow/tools/ci_build/pylint_allowlist
SCRIPT_DIR=/tmp
num_cpus() {
# Get the number of CPUs
if [[ -f /proc/cpuinfo ]]; then
N_CPUS=$(grep -c ^processor /proc/cpuinfo)
else
# Fallback method
N_CPUS=`getconf _NPROCESSORS_ONLN`
fi
if [[ -z ${N_CPUS} ]]; then
die "ERROR: Unable to determine the number of CPUs"
fi
echo ${N_CPUS}
}
get_py_files_to_check() {
find . -name '*.py'
}
do_pylint() {
# Get all Python files, regardless of mode.
PYTHON_SRC_FILES=$(get_py_files_to_check)
# Something happened. TF no longer has Python code if this branch is taken
if [[ -z ${PYTHON_SRC_FILES} ]]; then
echo "do_pylint found no Python files to check. Returning."
return 0
fi
# Now that we know we have to do work, check if `pylint` is installed
PYLINT_BIN="python3.8 -m pylint"
echo ""
echo "check whether pylint is available or not."
echo ""
${PYLINT_BIN} --version
if [[ $? -eq 0 ]]
then
echo ""
echo "pylint available, proceeding with pylint sanity check."
echo ""
else
echo ""
echo "pylint not available."
echo ""
return 1
fi
# Configure pylint using the following file
PYLINTRC_FILE="${SCRIPT_DIR}/pylintrc"
if [[ ! -f "${PYLINTRC_FILE}" ]]; then
die "ERROR: Cannot find pylint rc file at ${PYLINTRC_FILE}"
fi
# Run pylint in parallel, after some disk setup
NUM_SRC_FILES=$(echo ${PYTHON_SRC_FILES} | wc -w)
NUM_CPUS=$(num_cpus)
echo "Running pylint on ${NUM_SRC_FILES} files with ${NUM_CPUS} "\
"parallel jobs..."
echo ""
PYLINT_START_TIME=$(date +'%s')
OUTPUT_FILE="$(mktemp)_pylint_output.log"
ERRORS_FILE="$(mktemp)_pylint_errors.log"
PERMIT_FILE="$(mktemp)_pylint_permit.log"
FORBID_FILE="$(mktemp)_pylint_forbid.log"
rm -rf ${OUTPUT_FILE}
rm -rf ${ERRORS_FILE}
rm -rf ${PERMIT_FILE}
rm -rf ${FORBID_FILE}
set +e
# When running, filter to only contain the error code lines. Removes module
# header, removes lines of context that show up from some lines.
# Also, don't redirect stderr as this would hide pylint fatal errors.
${PYLINT_BIN} --rcfile="${PYLINTRC_FILE}" --output-format=parseable \
--jobs=${NUM_CPUS} ${PYTHON_SRC_FILES} | grep '\[[CEFW]' > ${OUTPUT_FILE}
PYLINT_END_TIME=$(date +'%s')
echo ""
echo "pylint took $((PYLINT_END_TIME - PYLINT_START_TIME)) s"
echo ""
# Report only what we care about
# Ref https://pylint.readthedocs.io/en/latest/technical_reference/features.html
# E: all errors
# W0311 bad-indentation
# W0312 mixed-indentation
# C0330 bad-continuation
# C0301 line-too-long
# C0326 bad-whitespace
# W0611 unused-import
# W0622 redefined-builtin
grep -E '(\[E|\[W0311|\[W0312|\[C0330|\[C0301|\[C0326|\[W0611|\[W0622)' ${OUTPUT_FILE} > ${ERRORS_FILE}
# Split the pylint reported errors into permitted ones and those we want to
# block submit on until fixed.
# We use `${ALLOW_LIST_FILE}` to record the errors we temporarily accept. Goal
# is to make that file only contain errors caused by difference between
# internal and external versions.
ALLOW_LIST_FILE="${SCRIPT_DIR}/pylint_allowlist"
if [[ ! -f "${ALLOW_LIST_FILE}" ]]; then
die "ERROR: Cannot find pylint allowlist file at ${ALLOW_LIST_FILE}"
fi
# We can split with just 2 grep invocations
grep -f ${ALLOW_LIST_FILE} ${ERRORS_FILE} > ${PERMIT_FILE}
grep -v -f ${ALLOW_LIST_FILE} ${ERRORS_FILE} > ${FORBID_FILE}
# Determine counts of errors
N_PERMIT_ERRORS=$(wc -l ${PERMIT_FILE} | cut -d' ' -f1)
N_FORBID_ERRORS=$(wc -l ${FORBID_FILE} | cut -d' ' -f1)
set -e
# First print all allowed errors
echo ""
if [[ ${N_PERMIT_ERRORS} != 0 ]]; then
echo "Found ${N_PERMIT_ERRORS} allowlisted pylint errors:"
cat ${PERMIT_FILE}
fi
# Now, print the errors we should fix
echo ""
if [[ ${N_FORBID_ERRORS} != 0 ]]; then
echo "Found ${N_FORBID_ERRORS} non-allowlisted pylint errors:"
cat ${FORBID_FILE}
fi
echo ""
if [[ ${N_FORBID_ERRORS} != 0 ]]; then
echo "FAIL: Found ${N_FORBID_ERRORS} non-allowlisted errors and ${N_PERMIT_ERRORS} allowlisted errors"
return 1
else
echo "PASS: Found only ${N_PERMIT_ERRORS} allowlisted errors"
fi
}
do_pylint