Enable flake8 across entire codebase (#105)

* Debug pre-commit config

* Fix pre-commit config

* Use flake8 first

* Reenable linting for most of the codebase

* Give up on mypy for now, clean up exclusion folders

* Exclude everything from mypy

* Make modifications across codebase to conform with flake8 (#118)

* Fix flake8 for home_robot/utils/data_tools

* Fix flake8 and mypy for home_robot_hw

* Fix more flake8 for home_robot_hw

* Fix tests and update flake8 config

* Revert change for mypy

* Fix flake8 complaint

* Remove mypy typing fix

* Print error

* add missing parameters to grasping scritp

---------

Co-authored-by: Chris Paxton <[email protected]>

---------

Co-authored-by: Chris Paxton <[email protected]>

.flake8

@@ -1,7 +1,7 @@
 [flake8]
 ignore = E203, E266, E501, W503, F403, F401
 max-line-length = 89
-max-complexity = 18
+max-complexity = 25
 select = B,C,E,F,W,T4,B9
 
 # Ignored rules

.pre-commit-config.yaml

@@ -10,27 +10,28 @@ repos:
       - id: isort
         args: ["--profile", "black", "--filter-files"]
 
-  - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.981
-    hooks:
-      - id: mypy
-        args: [--install-types, --non-interactive, --no-strict-optional, --ignore-missing-imports]
-        exclude: |
-            (?x)^(
-                src/home_robot_hw/home_robot_hw/ros/|
-                src/home_robot_hw/home_robot_hw/teleop/|
-                projects/|
-            )|setup.py$
-
   - repo: https://github.com/pycqa/flake8
     rev: 5.0.4
     hooks:
       - id: flake8
         exclude: |
             (?x)^(
-                src/home_robot_hw/home_robot_hw/ros/|
-                src/home_robot_hw/home_robot_hw/teleop/|
-                src/home_robot/home_robot/utils/data_tools/|
-                projects/|
-                tests/
+                projects/
+                | src/home_robot_hw/home_robot_hw/ros/srv_prebuilt/
             )
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.981
+    hooks:
+      - id: mypy
+        args: [--install-types, --non-interactive, --no-strict-optional, --ignore-missing-imports]
+        exclude: |
+            (?x)^(
+                projects/
+                | src/home_robot/home_robot/experimental/
+                | examples/
+                | tests/
+                | src/home_robot/
+                | src/home_robot_sim/
+                | src/home_robot_hw/
+            ) | setup.py$

projects/stretch_grasping/configs/agent/floorplanner_eval.yaml

@@ -22,6 +22,7 @@ AGENT:
   max_steps: 500          # maximum number of steps before stopping an episode
   panorama_start: 1       # 1: turn around 360 degrees when starting an episode, 0: don't
   exploration_strategy: seen_frontier  # exploration strategy ("seen_frontier", "been_close_to_frontier")
+  radius: 0.17            # robot radius (in meters)
 
   SEMANTIC_MAP:
     semantic_categories: mukul_indoor # map semantic channel categories ("coco_indoor", "longtail_indoor", "mukul_indoor")
@@ -44,6 +45,7 @@ AGENT:
     obs_dilation_selem_radius: 4    # radius (in cells) of obstacle dilation structuring element
     goal_dilation_selem_radius: 7  # radius (in cells) of goal dilation structuring element
     step_size: 4                    # maximum distance of the short-term goal selected by the planner
+    use_dilation_for_stg: 0
 
 EVAL_VECTORIZED:
   simulator_gpu_ids: [1, 2, 3, 4, 5, 6, 7] # IDs of GPUs to use for vectorized environments

src/home_robot/home_robot/utils/data_tools/__init__.py

@@ -1 +0,0 @@
-

src/home_robot/home_robot/utils/data_tools/image.py

@@ -4,6 +4,7 @@
 # LICENSE file in the root directory of this source tree.
 import io
 
+import cv2
 import h5py
 import imageio
 import numpy as np
@@ -107,7 +108,6 @@ def png_to_mp4(group: h5py.Group, key: str, name: str, fps=10):
     """
     Write key out as a mpt
     """
-    gif = []
     print("Writing gif to file:", name)
     img_stream = group[key]
     writer = None

src/home_robot/home_robot/utils/data_tools/robohive.py

@@ -27,6 +27,7 @@
     $ python examine_env.py --env_name door-v0 --policy my_policy.pickle --mode evaluation --episodes 10 \n
 """
 
+
 # Random policy
 class rand_policy:
     def __init__(self, env, seed):
@@ -121,7 +122,7 @@ def main(
     np.random.seed(seed)
     env = (
         gym.make(env_name)
-        if env_args == None
+        if env_args is None
         else gym.make(env_name, **(eval(env_args)))
     )
     env.seed(seed)
@@ -139,7 +140,7 @@ def main(
         output_name = "random_policy"
 
     # resolve directory
-    if (os.path.isdir(output_dir) == False) and (
+    if (not os.path.isdir(output_dir)) and (
         render == "offscreen" or save_paths or plot_paths is not None
     ):
         os.mkdir(output_dir)
@@ -187,7 +188,7 @@ def main(
     # plot paths
     if plot_paths:
         file_name = output_dir + "/" + output_name + "{}".format(time_stamp)
-        plotnsave_paths(paths, env=env, fileName_prefix=file_name)
+        plotnsave_paths(plot_paths, env=env, fileName_prefix=file_name)
 
 
 if __name__ == "__main__":

src/home_robot/home_robot/utils/data_tools/writer.py

@@ -31,7 +31,7 @@ def __init__(self, filename="data.h5", dirname=None):
             try:
                 os.mkdir(dirname)
             except OSError as e:
-                pass
+                print(e)
             self.filename = os.path.join(self.dirname, self.filename)
         else:
             self.filename = self.filename

src/home_robot_hw/home_robot_hw/constants.py

@@ -1,4 +1,5 @@
 from enum import Enum
+from typing import Dict, List, Union
 
 from home_robot.motion.stretch import HelloStretchIdx
 
@@ -13,7 +14,7 @@
 ROS_WRIST_ROLL = "joint_wrist_roll"
 
 
-ROS_TO_CONFIG = {
+ROS_TO_CONFIG: Dict[str, HelloStretchIdx] = {
     ROS_LIFT_JOINT: HelloStretchIdx.LIFT,
     ROS_GRIPPER_FINGER: HelloStretchIdx.GRIPPER,
     # ROS_GRIPPER_FINGER2: HelloStretchIdx.GRIPPER,
@@ -24,7 +25,7 @@
     ROS_HEAD_TILT: HelloStretchIdx.HEAD_TILT,
 }
 
-CONFIG_TO_ROS = {}
+CONFIG_TO_ROS: Dict[HelloStretchIdx, List[str]] = {}
 for k, v in ROS_TO_CONFIG.items():
     if v not in CONFIG_TO_ROS:
         CONFIG_TO_ROS[v] = []

src/home_robot_hw/home_robot_hw/env/stretch_manipulation_env.py

@@ -3,6 +3,7 @@
 import numpy as np
 import trimesh
 
+from home_robot.core.interfaces import Action
 from home_robot.motion.stretch import (
     STRETCH_BASE_FRAME,
     STRETCH_GRASP_FRAME,
@@ -26,7 +27,7 @@ def reset(self) -> None:
         # TODO: implement this
         raise NotImplementedError
 
-    def apply_action(self, manip_action: Optional[Dict[str, Any]]) -> None:
+    def apply_action(self, manip_action: Optional[Dict[str, Any]] = None) -> None:
         """
         manip_action: Manipulation action in cartesian space
                       (pos, quat)

src/home_robot_hw/home_robot_hw/ros/abstract.py

@@ -165,7 +165,6 @@ def goto_static_grasp(self, grasps, scores=None, pause=False):
                 # Grasp attempt failure
                 continue
             # Record the initial q value here and use it
-            theta0 = q[2]
             q1 = qi.copy()
             q1[HelloStretchIdx.LIFT] += 0.08
             # q1[HelloStretchIdx.LIFT] += 0.2

src/home_robot_hw/home_robot_hw/ros/grasp_helper.py

@@ -80,7 +80,7 @@ def _cb(self, msg):
         # Convert rospy point cloud into a message that we care about
         with self.grasp_lock:
             self.grasp_id = msg.seq
-            self.poses = msg_to_poses(msg)
+            self.poses = self.msg_to_poses(msg)
 
     def msg_to_poses(self, msg, frame=None):
         grasps = []
@@ -104,19 +104,7 @@ def msg_to_poses(self, msg, frame=None):
     def _score_cb(self, msg):
         with self.score_lock:
             self.score_id = msg.header.seq
-            self.scores = np.array([pt.x for x in msg.polygon.points])
-
-    def segmented_point_cloud_to_msg(self, xyz, labels):
-        msg = PointCloud()
-        msg.header.stamp = self.req_id
-        self.req_id += 1
-        xyz = xyz.reshape(-1, 3)
-        for i in range(xyz.shape[0]):
-            msg.points.append(Point(xyz[i, 0], xyz[i, 1], xyz[i, 2]))
-        msg.channels = ChannelFloat32(
-            name="label", values=labels.astype(np.float32).tolist()
-        )
-        return msg
+            self.scores = np.array([pt.x for pt in msg.polygon.points])
 
     def segmented_point_cloud_to_msg(self, xyz, rgb, labels):
         pc = PointCloud()
@@ -148,7 +136,7 @@ def request(self, xyz, rgb, seg, frame=None):
         return objs
 
     def get_grasps(self, xyz, labels, timeout=10.0):
-        msg = segmented_point_cloud_to_msg(xyz, labels)
+        msg = self.segmented_point_cloud_to_msg(xyz, labels)
         print("Sending grasp request...")
         with self.lock:
             self.pub.publish(msg)
@@ -219,6 +207,3 @@ def _cb(self, msg):
 
     def get(self):
         return self.queue.pop_left()
-
-    def send_response(self, poses):
-        self.pub.publish(poses_to_msg(poses))

src/home_robot_hw/home_robot_hw/ros/image_transport.py

@@ -41,17 +41,17 @@ def __init__(self, show_images=False):
         self.reference_frame = None
         self.sequence_id = 0
 
-        cb_color_cam_info = lambda msg: self.send_and_receive_camera_info(
-            msg, pub_color_cam_info
-        )
-        cb_depth_cam_info = lambda msg: self.send_and_receive_camera_info(
-            msg, pub_depth_cam_info
-        )
+        def cb_color_cam_info(msg):
+            return self.send_and_receive_camera_info(msg, pub_color_cam_info)
+
+        def cb_depth_cam_info(msg):
+            return self.send_and_receive_camera_info(msg, pub_depth_cam_info)
+
         self.sub_color_cam_info = rospy.Subscriber(
             "/camera/color/camera_info", CameraInfo, cb_color_cam_info
         )
         self.sub_depth_cam_info = rospy.Subscriber(
-            "/camera/aligned_depth_to_color/camera_info", CameraInfo, cb_color_cam_info
+            "/camera/aligned_depth_to_color/camera_info", CameraInfo, cb_depth_cam_info
         )
 
     def send_and_receive_camera_info(self, cam_info, publisher):

src/home_robot_hw/home_robot_hw/ros/msg_numpy.py

@@ -81,9 +81,10 @@
     "64FC4": (np.float64, 4),
 }
 
+
 # @converts_to_numpy(Image)
 def image_to_numpy(msg):
-    if not msg.encoding in name_to_dtypes:
+    if msg.encoding not in name_to_dtypes:
         raise TypeError("Unrecognized encoding {}".format(msg.encoding))
 
     dtype_class, channels = name_to_dtypes[msg.encoding]
@@ -101,55 +102,13 @@ def image_to_numpy(msg):
 
 # @converts_from_numpy(Image)
 def numpy_to_image(arr, encoding):
-    if not encoding in name_to_dtypes:
-        raise TypeError("Unrecognized encoding {}".format(encoding))
-
-    im = Image(encoding=encoding)
-
-    # extract width, height, and channels
-    dtype_class, exp_channels = name_to_dtypes[encoding]
-    dtype = np.dtype(dtype_class)
-    if len(arr.shape) == 2:
-        im.height, im.width, channels = arr.shape + (1,)
-    elif len(arr.shape) == 3:
-        im.height, im.width, channels = arr.shape
-    else:
-        raise TypeError("Array must be two or three dimensional")
-
-    # check type and channels
-    if exp_channels != channels:
-        raise TypeError(
-            "Array has {} channels, {} requires {}".format(
-                channels, encoding, exp_channels
-            )
-        )
-    if dtype_class != arr.dtype.type:
-        raise TypeError(
-            "Array is {}, {} requires {}".format(arr.dtype.type, encoding, dtype_class)
-        )
-
-    # make the array contiguous in memory, as mostly required by the format
-    contig = np.ascontiguousarray(arr)
-    im.data = contig.tostring()
-    im.step = contig.strides[0]
-    im.is_bigendian = (
-        arr.dtype.byteorder == ">"
-        or arr.dtype.byteorder == "="
-        and sys.byteorder == "big"
-    )
-
-    return im
-
-
-def numpy_to_image(arr, encoding):
-    if not encoding in name_to_dtypes:
+    if encoding not in name_to_dtypes:
         raise TypeError("Unrecognized encoding {}".format(encoding))
 
     im = Image(encoding=encoding)
 
     # extract width, height, and channels
     dtype_class, exp_channels = name_to_dtypes[encoding]
-    dtype = np.dtype(dtype_class)
     if len(arr.shape) == 2:
         im.height, im.width, channels = arr.shape + (1,)
     elif len(arr.shape) == 3:

src/home_robot_hw/home_robot_hw/ros/recorder.py

@@ -124,7 +124,6 @@ def png_to_mp4(group: h5py.Group, key: str, name: str, fps=10):
     """
     Write key out as a gif
     """
-    gif = []
     print("Writing gif to file:", name)
     img_stream = group[key]
     writer = None

src/home_robot_hw/home_robot_hw/ros/teleop.py

@@ -5,6 +5,8 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 
+# flake8: noqa
+
 """
 Based on code from Willow Garage:
     https://github.com/ros-visualization/visualization_tutorials/blob/indigo-devel/interactive_marker_tutorials/scripts/basic_controls.py

src/home_robot_hw/home_robot_hw/teleop/stretch_xbox_controller_teleop.py

@@ -9,6 +9,7 @@
 Unless required by applicable law or agreed to in writing, the Contents are distributed under the License are distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
 For further information about the Contents including inquiries about dual licensing, please contact Hello Robot Inc.
 """
+# flake8: noqa
 # TODO: move this somewhere better, since it's in the "ros" folder but isn't actually ROS
 
 from __future__ import print_function

tests/home_robot/motion/test_ik.py

@@ -349,15 +349,15 @@ def test_ros_to_pin(pin_robot, test_joints):
 
 
 if __name__ == "__main__":
-    robot = HelloStretchKinematics(
+    robot_model = HelloStretchKinematics(
         urdf_path=URDF_ABS_PATH,
         visualize=DEBUG,
         ik_type="pybullet",
     )
     opt = PositionIKOptimizer(
-        robot.manip_ik_solver,
+        robot_model.manip_ik_solver,
         pos_error_tol=CEM_POS_ERROR_TOL,
         ori_error_range=np.array([0.0, 0.0, CEM_YAW_ERROR_TOL]),  # solve for yaw only
     )
-    test_ik_solvers(robot, TEST_DATA[0])
-    test_pybullet_ik_optimization(robot, opt, TEST_DATA[0])
+    test_ik_solvers(robot_model, TEST_DATA[0])
+    test_pybullet_ik_optimization(robot_model, opt, TEST_DATA[0])

tests/home_robot_sim/test_habitat_env.py

@@ -7,7 +7,7 @@
 try:
     import habitat
 except ImportError:
-    print(f"Warning: habitat not installed, skipping habitat tests")
+    print("Warning: habitat not installed, skipping habitat tests")
     pytest.skip(allow_module_level=True)
 
 from habitat.config.default import Config

tests/hw_manual_test.py

@@ -24,7 +24,7 @@
     robot.head.look_at_ee()
     robot.head.look_front()
 
-    print(f"Confirm that the robot head has moved accordingly.")
+    print("Confirm that the robot head has moved accordingly.")
     input("(press enter to continue)")
 
     # Navigation
@@ -53,4 +53,4 @@
     )
     input("(press enter to continue)")
 
-    print(f"Test complete!")
+    print("Test complete!")