Migrate CI to bionic (#148)

* use official bionic docker images * update dependencies to match upstream versions * target ROS melodic * pull in customly built opensplice for bionic and ignore it on ARM until we have debs for it * install RTI's OpenSSL * reenable opensplice on arm * remove now unnecessary libproj hack * add libcairo2-dev for cartographer * use upstream poco * remove libproj hack completely * cleanup dependencies * now targetting melodic * not sure how kinetic got back here, switching to melodic * add missing ros_comm deps * Parametise linux distro * Add xenial job * Pass ubuntu build-arg * Use specific debian name again * Add back workarounds for ARM on xenial * Define ubuntu_distro in more templates * Use double quotes for sed command * Create job for ARM too * Revert "Add back workarounds for ARM on xenial" This reverts commit d4d82a7. * Connext not supported on ARM * comment in cartographer dependencies to build it from source * Factor out ubuntu distro * Highlight tb demo not supported on xenial * Use elif * Remove duplicated deps (rebase mixup) * Label ARM jobs correctly * fix unexpected operator sh error * this should not be needed now that the new version of rosmsg hit the melodic testing repo * 'on bionic' -> 'not on xenial' for poco debs * remove unnecessary blank line
ros2 · Jun 7, 2018 · b0bf985 · b0bf985
1 parent e39be5d
commit b0bf985
Show file tree

Hide file tree

Showing 8 changed files with 101 additions and 36 deletions.
diff --git a/create_jenkins_job.py b/create_jenkins_job.py
@@ -86,6 +86,7 @@ def main(argv=None):
         'dont_notify_every_unstable_build': 'false',
         'turtlebot_demo': False,
         'build_timeout_mins': 0,
+        'ubuntu_distro': 'bionic',
     }
 
     jenkins = connect(args.jenkins_url)
@@ -237,6 +238,20 @@ def create_job(os_name, job_name, template_file, additional_dict):
                 'mailer_recipients': DEFAULT_MAIL_RECIPIENTS,
             })
 
+    for os_name in ['linux', 'linux-aarch64']:
+        # configure a nightly triggered job for xenial using all RMW implementations
+        ubuntu_distro = 'xenial'
+        job_name = 'nightly_{0}_{1}_release'.format(ubuntu_distro, os_name)
+        create_job(os_name, job_name, 'ci_job.xml.em', {
+            'cmake_build_type': 'Release',
+            'time_trigger_spec': PERIODIC_JOB_SPEC,
+            'mailer_recipients': DEFAULT_MAIL_RECIPIENTS,
+            'use_connext_default': 'false' if os_name is 'linux-aarch64' else 'true',
+            'use_fastrtps_default': 'true',
+            'use_opensplice_default': 'true',
+            'ubuntu_distro': ubuntu_distro,
+        })
+
     # configure the launch job
     os_specific_data = collections.OrderedDict()
     for os_name in sorted(os_configs.keys()):

diff --git a/job_templates/ci_job.xml.em b/job_templates/ci_job.xml.em
@@ -27,6 +27,7 @@
     use_fastrtps_default=use_fastrtps_default,
     use_opensplice_default=use_opensplice_default,
     use_isolated_default=use_isolated_default,
+    ubuntu_distro=ubuntu_distro,
     cmake_build_type=cmake_build_type,
     build_args_default=build_args_default,
     test_args_default=test_args_default,
@@ -94,6 +95,7 @@ supplemental_repos_url: ${build.buildVariableResolver.resolve('CI_ROS2_SUPPLEMEN
 colcon_branch: ${build.buildVariableResolver.resolve('CI_COLCON_BRANCH')}, <br/>
 use_whitespace: ${build.buildVariableResolver.resolve('CI_USE_WHITESPACE_IN_PATHS')}, <br/>
 isolated: ${build.buildVariableResolver.resolve('CI_ISOLATED')}, <br/>
+ubuntu_distro: ${build.buildVariableResolver.resolve('CI_UBUNTU_DISTRO')}, <br/>
 cmake_build_type: ${build.buildVariableResolver.resolve('CI_CMAKE_BUILD_TYPE')}, <br/>
 build_args: ${build.buildVariableResolver.resolve('CI_BUILD_ARGS')}, <br/>
 test_args: ${build.buildVariableResolver.resolve('CI_TEST_ARGS')}, <br/>
@@ -148,14 +150,19 @@ fi
 if [ "$CI_ISOLATED" = "true" ]; then
   export CI_ARGS="$CI_ARGS --isolated"
 fi
+if [ "${CI_UBUNTU_DISTRO}" = "bionic" ]; then
+  export CI_ROS1_DISTRO=melodic
+elif [ "${CI_UBUNTU_DISTRO}" = "xenial" ]; then
+  export CI_ROS1_DISTRO=kinetic
+fi
 if [ "${CI_CMAKE_BUILD_TYPE}" != "None" ]; then
   export CI_ARGS="$CI_ARGS --cmake-build-type $CI_CMAKE_BUILD_TYPE"
 fi
 if [ "$CI_ENABLE_C_COVERAGE" = "true" ]; then
   export CI_ARGS="$CI_ARGS --coverage"
 fi
 @[if os_name in ['linux', 'linux-aarch64'] and turtlebot_demo]@
-export CI_ARGS="$CI_ARGS --ros1-path /opt/ros/kinetic"
+export CI_ARGS="$CI_ARGS --ros1-path /opt/ros/$CI_ROS1_DISTRO"
 @[end if]@
 if [ -n "${CI_BUILD_ARGS+x}" ]; then
   export CI_ARGS="$CI_ARGS --build-args $CI_BUILD_ARGS"
@@ -167,6 +174,9 @@ echo "Using args: $CI_ARGS"
 echo "# END SECTION"
 
 @[if os_name in ['linux', 'linux-aarch64']]@
+sed -i "s+^FROM.*$+FROM ubuntu:$CI_UBUNTU_DISTRO+" linux_docker_resources/Dockerfile
+export DOCKER_BUILD_ARGS="--build-arg UBUNTU_DISTRO=$CI_UBUNTU_DISTRO --build-arg ROS1_DISTRO=$CI_ROS1_DISTRO"
+
 mkdir -p $HOME/.ccache
 echo "# BEGIN SECTION: docker version"
 docker version
@@ -179,18 +189,16 @@ sed -i "s/@@today_str/`date +%Y-%m-%d`/" linux_docker_resources/Dockerfile
 echo "# END SECTION"
 echo "# BEGIN SECTION: Build Dockerfile"
 @[if os_name == 'linux-aarch64']@
-sed -i 's+^FROM.*$+FROM aarch64/ubuntu:xenial+' linux_docker_resources/Dockerfile
-sed -i 's+apt-get update+(apt-get update || true)+' linux_docker_resources/Dockerfile
 @[  if turtlebot_demo]@
-docker build --build-arg PLATFORM=arm --build-arg INSTALL_TURTLEBOT2_DEMO_DEPS=true -t ros2_batch_ci_turtlebot_demo linux_docker_resources
+docker build ${DOCKER_BUILD_ARGS} --build-arg PLATFORM=arm --build-arg INSTALL_TURTLEBOT2_DEMO_DEPS=true -t ros2_batch_ci_turtlebot_demo linux_docker_resources
 @[  else]@
-docker build --build-arg PLATFORM=arm -t ros2_batch_ci_aarch64 linux_docker_resources
+docker build ${DOCKER_BUILD_ARGS} --build-arg PLATFORM=arm -t ros2_batch_ci_aarch64 linux_docker_resources
 @[  end if]@
 @[elif os_name == 'linux']@
 @[  if turtlebot_demo]@
-docker build --build-arg INSTALL_TURTLEBOT2_DEMO_DEPS=true -t ros2_batch_ci_turtlebot_demo linux_docker_resources
+docker build ${DOCKER_BUILD_ARGS} --build-arg INSTALL_TURTLEBOT2_DEMO_DEPS=true -t ros2_batch_ci_turtlebot_demo linux_docker_resources
 @[  else]@
-docker build -t ros2_batch_ci linux_docker_resources
+docker build ${DOCKER_BUILD_ARGS} -t ros2_batch_ci linux_docker_resources
 @[  end if]@
 @[else]@
 @{ assert False, 'Unknown os_name: ' + os_name }@

diff --git a/job_templates/ci_launcher_job.xml.em b/job_templates/ci_launcher_job.xml.em
@@ -26,6 +26,7 @@
     use_fastrtps_default=use_fastrtps_default,
     use_opensplice_default=use_opensplice_default,
     use_isolated_default=use_isolated_default,
+    ubuntu_distro=ubuntu_distro,
     cmake_build_type=cmake_build_type,
     build_args_default=build_args_default,
     test_args_default=test_args_default,

diff --git a/job_templates/packaging_job.xml.em b/job_templates/packaging_job.xml.em
@@ -22,6 +22,7 @@
     ci_scripts_default_branch=ci_scripts_default_branch,
     default_repos_url=default_repos_url,
     supplemental_repos_url=supplemental_repos_url,
+    ubuntu_distro=ubuntu_distro,
     cmake_build_type=cmake_build_type,
     build_args_default=build_args_default,
 ))@
@@ -138,10 +139,10 @@ if [ "$CI_TEST_BRIDGE" = "true" ]; then
   export CI_ARGS="$CI_ARGS --test-bridge"
 fi
 @[if os_name in ['linux', 'linux-aarch64']]@
-export CI_ARGS="$CI_ARGS --ros1-path /opt/ros/kinetic"
+export CI_ARGS="$CI_ARGS --ros1-path /opt/ros/melodic"
 @[else]@
 echo "not building/testing the ros1_bridge on MacOS"
-# export CI_ARGS="$CI_ARGS --ros1-path /Users/osrf/kinetic/install_isolated"
+# export CI_ARGS="$CI_ARGS --ros1-path /Users/osrf/melodic/install_isolated"
 @[end if]@
 if [ "${CI_CMAKE_BUILD_TYPE}" != "None" ]; then
   export CI_ARGS="$CI_ARGS --cmake-build-type $CI_CMAKE_BUILD_TYPE"
@@ -165,8 +166,6 @@ docker info
 echo "# END SECTION"
 echo "# BEGIN SECTION: Build Dockerfile"
 @[if os_name == 'linux-aarch64']@
-sed -i 's+^FROM.*$+FROM aarch64/ubuntu:xenial+' linux_docker_resources/Dockerfile
-sed -i 's+apt-get update+(apt-get update || true)+' linux_docker_resources/Dockerfile
 docker build --build-arg PLATFORM=arm --build-arg BRIDGE=true -t ros2_packaging_aarch64 linux_docker_resources
 @[elif os_name == 'linux']@
 docker build --build-arg BRIDGE=true -t ros2_packaging linux_docker_resources

diff --git a/job_templates/snippet/property_parameter-definition.xml.em b/job_templates/snippet/property_parameter-definition.xml.em
@@ -5,6 +5,7 @@
     ci_scripts_default_branch=ci_scripts_default_branch,
     default_repos_url=default_repos_url,
     supplemental_repos_url=supplemental_repos_url,
+    ubuntu_distro=ubuntu_distro,
     cmake_build_type=cmake_build_type,
     build_args_default=build_args_default,
 ))@

diff --git a/job_templates/snippet/property_parameter-definition_common.xml.em b/job_templates/snippet/property_parameter-definition_common.xml.em
@@ -32,6 +32,22 @@ If the branch doesn't exist fall back to the default branch.
 To use the latest released version, use an empty string.</description>
           <defaultValue></defaultValue>
         </hudson.model.StringParameterDefinition>
+        <hudson.model.ChoiceParameterDefinition>
+          <name>CI_UBUNTU_DISTRO</name>
+          <description>Select the linux distribution to use.</description>
+          <choices class="java.util.Arrays$ArrayList">
+            <a class="string-array">
+              <string>@ubuntu_distro</string>
+@{
+choices = ['bionic', 'xenial']
+choices.remove(ubuntu_distro)
+}@
+@[for choice in choices]@
+              <string>@choice</string>
+@[end for]@
+            </a>
+          </choices>
+        </hudson.model.ChoiceParameterDefinition>
         <hudson.model.ChoiceParameterDefinition>
           <name>CI_CMAKE_BUILD_TYPE</name>
           <description>Select the CMake build type.</description>

diff --git a/linux_docker_resources/Dockerfile b/linux_docker_resources/Dockerfile
@@ -1,7 +1,9 @@
-FROM ubuntu:xenial
+FROM ubuntu:bionic
 ARG BRIDGE=false
 ARG INSTALL_TURTLEBOT2_DEMO_DEPS=false
 ARG PLATFORM=x86
+ARG ROS1_DISTRO=melodic
+ARG UBUNTU_DISTRO=bionic
 
 # Prevent errors from apt-get.
 # See: http://askubuntu.com/questions/506158/unable-to-initialize-frontend-dialog-when-using-ssh
@@ -17,6 +19,7 @@ ENV LANG en_US.UTF-8
 RUN apt-get update && apt-get install --no-install-recommends -y \
   lsb-release net-tools sudo \
   curl \
+  gnupg2 \
   apt-transport-https
 
 # Add the ROS repositories to the apt sources list.
@@ -31,7 +34,7 @@ RUN curl --silent http://packages.osrfoundation.org/gazebo.key | apt-key add -
 RUN apt-get update && apt-get install --no-install-recommends -y build-essential ccache cmake pkg-config python3-empy python3-setuptools python3-vcstool
 
 # Install build and test dependencies of ROS 2 packages.
-RUN apt-get update && apt-get install --no-install-recommends -y clang-format-3.8 cppcheck git pydocstyle pyflakes python3-coverage python3-flake8 python3-mock python3-nose python3-pep8 python3-pyparsing python3-yaml uncrustify
+RUN apt-get update && apt-get install --no-install-recommends -y clang-format cppcheck git pydocstyle pyflakes python3-coverage python3-flake8 python3-mock python3-nose python3-pep8 python3-pyparsing python3-yaml uncrustify
 
 # Install and self update pip/setuptools to the latest version.
 RUN apt-get update && apt-get install --no-install-recommends -y python3-pip
@@ -41,7 +44,7 @@ RUN pip3 install -U setuptools pip virtualenv
 RUN apt-get update && apt-get install --no-install-recommends -y gcovr
 
 # Install the OpenSplice binary from the OSRF repositories.
-RUN if test \( ${PLATFORM} = arm -o ${PLATFORM} = x86 \) ; then apt-get update && apt-get install --no-install-recommends -y libopensplice67=6.7.180404+osrf1-1~xenial; fi
+RUN if test \( ${PLATFORM} = arm -o ${PLATFORM} = x86 \) ; then apt-get update && apt-get install --no-install-recommends -y libopensplice67=6.7.180404+osrf1-1~$UBUNTU_DISTRO; fi
 # Update default domain id.
 RUN if test \( ${PLATFORM} = arm -o ${PLATFORM} = x86 \) ; then sed -i "s/<Id>0<\/Id>/<Id>108<\/Id>/" /usr/etc/opensplice/config/ospl.xml; fi
 
@@ -52,10 +55,11 @@ RUN if test ${PLATFORM} = x86; then apt-get update && RTI_NC_LICENSE_ACCEPTED=ye
 RUN if test ${PLATFORM} = x86; then apt-get update && apt-get install --no-install-recommends -y default-jre-headless; fi
 
 # Install dependencies for RTI web binaries install script.
-RUN pip3 install pexpect
+RUN apt-get update && apt-get install --no-install-recommends -y python3-pexpect
 
 # Get and install the RTI web binaries.
 RUN cd /tmp && curl --silent https://s3.amazonaws.com/RTI/Bundles/5.3.1/Evaluation/rti_connext_dds_secure-5.3.1-eval-x64Linux3gcc5.4.0.tar.gz | tar -xz
+RUN cd /tmp && tar -xvf /tmp/openssl-1.0.2n-target-x64Linux3gcc5.4.0.tar.gz
 ADD rti_web_binaries_install_script.py /tmp/rti_web_binaries_install_script.py
 
 # Add the RTI license file.
@@ -72,13 +76,12 @@ RUN apt-get update && apt-get install --no-install-recommends -y libasio-dev lib
 # Install OpenCV.
 RUN apt-get update && apt-get install --no-install-recommends -y libopencv-dev
 
-# Install build dependencies for class loader.
-# TODO Uncomment this when we switch to bionic
-# We are building poco from source for now as we need at least 1.4.1p1 and xenial ships with 1.3.6p1 (https://github.com/ros2/poco_vendor/pull/10)
-# RUN apt-get update && apt-get install --no-install-recommends -y libpoco-dev libpocofoundation9v5 libpocofoundation9v5-dbg
+# Install build dependencies for class_loader.
+# We are building poco from source on xenial as we need at least 1.4.1p1 and xenial ships with 1.3.6p1 (https://github.com/ros2/poco_vendor/pull/10)
+RUN if test ${UBUNTU_DISTRO} != xenial; then apt-get update && apt-get install --no-install-recommends -y libpoco-dev; fi
 
 # Install build dependencies for rviz et al.
-RUN apt-get update && apt-get install --no-install-recommends -y libcurl4-openssl-dev libqt5core5a libqt5gui5 libqt5opengl5 libqt5widgets5 qtbase5-dev libxaw7-dev libgles2-mesa-dev libglu1-mesa-dev
+RUN apt-get update && apt-get install --no-install-recommends -y libcurl4-openssl-dev libfreetype6-dev libgles2-mesa-dev libglu1-mesa libqt5core5a libqt5gui5 libqt5opengl5 libqt5widgets5 libxaw7-dev libxrandr-dev qtbase5-dev
 
 # Install dependencies for robot_model and robot_state_publisher
 RUN apt-get update && apt-get install --no-install-recommends -y libtinyxml-dev libeigen3-dev
@@ -89,23 +92,42 @@ RUN apt-get update && apt-get install --no-install-recommends -y python3-dev
 # automatic invalidation once every day.
 RUN echo "@today_str"
 
-RUN if test \( ${BRIDGE} = true -o ${INSTALL_TURTLEBOT2_DEMO_DEPS} = true \) ; then apt-get update && apt-get install --no-install-recommends -y ros-kinetic-catkin; fi
+RUN if test \( ${BRIDGE} = true -o ${INSTALL_TURTLEBOT2_DEMO_DEPS} = true \) ; then apt-get update && apt-get install --no-install-recommends -y ros-${ROS1_DISTRO}-catkin; fi
 
 # Install build and test dependencies of ros1_bridge.
-RUN if test ${BRIDGE} = true; then apt-get update && apt-get install --no-install-recommends -y python-rospkg ros-kinetic-common-msgs ros-kinetic-rosbash ros-kinetic-roscpp ros-kinetic-roslaunch ros-kinetic-rosmsg ros-kinetic-roscpp-tutorials ros-kinetic-rospy-tutorials ros-kinetic-tf2-msgs; fi
-
-# Install build dependencies for turtlebot demo.
-RUN if test ${INSTALL_TURTLEBOT2_DEMO_DEPS} = true; then apt-get update && apt-get install --no-install-recommends -y libatlas-base-dev libboost-iostreams-dev libboost-regex-dev libboost-system-dev libboost-thread-dev libceres-dev libgoogle-glog-dev liblua5.2-dev libpcl-dev libprotobuf-dev libsdl1.2-dev libsdl-image1.2-dev libsuitesparse-dev libudev-dev libusb-1.0-0-dev libyaml-cpp-dev protobuf-compiler python-sphinx ros-kinetic-kobuki-driver ros-kinetic-kobuki-ftdi; fi
-
-# Workaround bugs in the Ubuntu 16.04 packaging of libpcl.  Reports:
-# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=819741
-# https://bugs.launchpad.net/ubuntu/+source/vtk6/+bug/1573234
-# https://github.com/PointCloudLibrary/pcl/issues/1594
-# TODO(clalancette): for some reason, if these two workarounds are listed in
-# different RUN statements, it causes the docker image to fail to build on
-# aarch64 (see https://github.com/ros2/ci/pull/73).  As a workaround, do them
-# in one statement.
-RUN if test ${INSTALL_TURTLEBOT2_DEMO_DEPS} = true; then sed -i -e '/vtkproj4;/d' /usr/lib/$(uname -m)-linux-gnu/cmake/pcl/PCLConfig.cmake && ln -s /usr/lib/$(uname -m)-linux-gnu/libproj.so.9.1.0 /usr/lib/$(uname -m)-linux-gnu/libproj.so; fi
+RUN if test ${BRIDGE} = true; then apt-get update && apt-get install --no-install-recommends -y \
+    python-rospkg \
+    ros-${ROS1_DISTRO}-common-msgs \
+    ros-${ROS1_DISTRO}-rosbash \
+    ros-${ROS1_DISTRO}-roscpp \
+    ros-${ROS1_DISTRO}-roslaunch \
+    ros-${ROS1_DISTRO}-rosmsg \
+    ros-${ROS1_DISTRO}-roscpp-tutorials \
+    ros-${ROS1_DISTRO}-rospy-tutorials \
+    ros-${ROS1_DISTRO}-tf2-msgs; fi
+
+# Install build dependencies for turtlebot demo (not supported on xenial).
+RUN if test \( ${UBUNTU_DISTRO} != xenial -a ${INSTALL_TURTLEBOT2_DEMO_DEPS} = true \) ; then \
+    apt-get update && apt-get install --no-install-recommends -y \
+    libatlas-base-dev \
+    libboost-iostreams-dev \
+    libboost-regex-dev \
+    libboost-system-dev \
+    libboost-thread-dev \
+    libcairo2-dev \
+    libceres-dev \
+    libgoogle-glog-dev \
+    liblua5.2-dev \
+    libpcl-dev \
+    libprotobuf-dev \
+    libprotoc-dev \
+    libsdl1.2-dev libsdl-image1.2-dev \
+    libudev-dev \
+    libusb-1.0-0-dev \
+    libyaml-cpp-dev \
+    protobuf-compiler \
+    python3-sphinx \
+    ros-${ROS1_DISTRO}-kobuki-driver ros-${ROS1_DISTRO}-kobuki-ftdi; fi
 
 # Create a user to own the build output.
 RUN useradd -u 1234 -m rosbuild

diff --git a/linux_docker_resources/entry_point.sh b/linux_docker_resources/entry_point.sh
@@ -29,14 +29,17 @@ case "${CI_ARGS}" in
         ;;
       *)
         echo "Installing Connext binaries off RTI website..."
-        python3 -u /tmp/rti_web_binaries_install_script.py /tmp/rti_connext_dds-5.3.1-eval-x64Linux3gcc5.4.0.run /home/rosbuild/rti_connext_dds-5.3.1 --rtipkg_paths /tmp/rti_security_plugins-5.3.1-eval-x64Linux3gcc5.4.0.rtipkg
+        python3 -u /tmp/rti_web_binaries_install_script.py /tmp/rti_connext_dds-5.3.1-eval-x64Linux3gcc5.4.0.run /home/rosbuild/rti_connext_dds-5.3.1 --rtipkg_paths /tmp/rti_security_plugins-5.3.1-eval-x64Linux3gcc5.4.0.rtipkg /tmp/openssl-1.0.2n-5.3.1-host-x64Linux.rtipkg
         if [ $? -ne 0 ]
         then
           echo "Connext not installed correctly (maybe you're on an ARM machine?)." >&2
           exit 1
         fi
         mv /tmp/rti_license.dat /home/rosbuild/rti_license.dat
         export RTI_LICENSE_FILE=/home/rosbuild/rti_license.dat
+        mv /tmp/openssl-1.0.2n /home/rosbuild/openssl-1.0.2n
+        export RTI_OPENSSL_BIN=/home/rosbuild/openssl-1.0.2n/x64Linux3gcc5.4.0/release/bin
+        export RTI_OPENSSL_LIBS=/home/rosbuild/openssl-1.0.2n/x64Linux3gcc5.4.0/release/lib
         ;;
     esac
     echo "done."