Bump transport-tools à 1.0.3 pour la branche Elixir 1.12.2 (#39)

* Mises à jour techniques (Elixir, Erlang, NVM, NodeJS) (#33)

* Make sure the docker hub link is closer from the image

* Update to most recent elixir/erlang couple

* Bump NVM to latest

https://github.com/nvm-sh/nvm/releases

* Bump NodeJS to latest LTS available

https://nodejs.org

* Try to fix the build

* Update test for NodeJS

* Update test for Elixir version

* Modify script to make it work with newer format

* Add readme test to verify OTP version in full (improved)

* Update GitHub action to assert on full version

* Now use full version specifier

* Add explanation about how the hex images are built

* Add very useful help to see output of commands locally

The CI already does that by default.

* Upgrade packages (except kernel) for security

See etalab/transport-deploy#46 for context

* Bump to latest

* Bump Node to latest LTS

* Bump tests

* Elixir 1.12.2 et transport-tools 1.0.3

* Update rakefile

Co-authored-by: Thibaut Barrère <[email protected]>

.github/workflows/docker.yml

@@ -19,7 +19,7 @@ env:
   TEST_TAG: ${{ github.repository }}:test
   TEST_EXPECTED_NODE_OUTPUT: "v14.16.1"
   TEST_EXPECTED_ELIXIR_OUTPUT: "Elixir 1.12.2 (compiled with Erlang/OTP 24)"
-  TEST_EXPECTED_ERLANG_OUTPUT: "Erlang/OTP 24"
+  TEST_EXPECTED_ERLANG_OUTPUT: "24.0.4"
 jobs:
   build-and-push-image:
     runs-on: ubuntu-latest
@@ -55,19 +55,19 @@ jobs:
           # --output=type=docker to enforce local work
           # https://github.com/docker/buildx/blob/master/docs/reference/buildx_build.md#load
           # https://github.com/docker/buildx/blob/master/docs/reference/buildx_build.md#docker
-          load: true 
+          load: true
           push: false # the default, but added for clarity
           tags: ${{ env.TEST_TAG }}
 
       # NOTE: the following tests rely on grep exit code (0 if match found)
       - name: Test that Node can start and has expected version
         run: docker run --rm ${{ env.TEST_TAG }} /bin/bash -c 'node --version' | grep '${{ env.TEST_EXPECTED_NODE_OUTPUT }}'
-      
+
       - name: Test that Elixir can start and has expected version
         run: docker run --rm ${{ env.TEST_TAG }} /bin/bash -c 'elixir --version' | grep '${{ env.TEST_EXPECTED_ELIXIR_OUTPUT }}'
 
-      - name: Test that Erlang can start and has (major) expected version
-        run: docker run --rm ${{ env.TEST_TAG }} /bin/bash -c "erl -noshell -eval 'erlang:display(erlang:system_info(system_version))' -eval 'init:stop()'" | grep '${{ env.TEST_EXPECTED_ERLANG_OUTPUT }}'
+      - name: Test that Erlang can start and has expected version (major + minor + optional revision number)
+        run: docker run --rm ${{ env.TEST_TAG }} /bin/bash -c "erl -eval '{ok, Version} = file:read_file(filename:join([code:root_dir(), \"releases\", erlang:system_info(otp_release), \"OTP_VERSION\"])), io:fwrite(Version), halt().' -noshell" | grep '${{ env.TEST_EXPECTED_ERLANG_OUTPUT }}'
 
       # If we reach this point, we consider the tested imaged is OK, so we can extract the metadat & publish
       # https://github.com/docker/metadata-action
@@ -89,4 +89,4 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
 
       # TODO: consider caching
-      # - https://github.com/docker/build-push-action/blob/master/docs/advanced/cache.md
+      # - https://github.com/docker/build-push-action/blob/master/docs/advanced/cache.md

README.md

@@ -64,15 +64,18 @@ As a work-around for [#17](https://github.com/etalab/transport-ops/issues/17):
 ```
 IMAGE_VERSION=$(rake get_image_version)
 IMAGE_NAME=betagouv/transport:$IMAGE_VERSION
-docker build transport-site --no-cache -t $IMAGE_NAME
+docker build transport-site --no-cache -t $IMAGE_NAME --progress=plain
 ```
 
 * Carefully verify the versions (this will be translated into a testing script later):
 
 ```
 docker run -it --rm $IMAGE_NAME /bin/bash -c 'node --version'
 docker run -it --rm $IMAGE_NAME /bin/bash -c 'elixir --version'
+# only major
 docker run -it --rm $IMAGE_NAME /bin/bash -c "erl -noshell -eval 'erlang:display(erlang:system_info(system_version))' -eval 'init:stop()'"
+# full version (https://stackoverflow.com/a/34326368)
+docker run -it --rm $IMAGE_NME /bin/bash -c "erl -eval '{ok, Version} = file:read_file(filename:join([code:root_dir(), \"releases\", erlang:system_info(otp_release), \"OTP_VERSION\"])), io:fwrite(Version), halt().' -noshell"
 ```
 
 * Read the [docker push documentation](https://docs.docker.com/engine/reference/commandline/push/)
@@ -88,7 +91,7 @@ Before creating a tag, the following commands can be used to verify the versions
 
 ```
 cd transport-site
-docker build . -t test:latest
+docker build . -t test:latest --progress=plain
 docker run -it --rm test:latest /bin/bash -c 'node --version'
 docker run -it --rm test:latest /bin/bash -c 'elixir --version'
 docker run -it --rm test:latest /bin/bash -c "erl -noshell -eval 'erlang:display(erlang:system_info(system_version))' -eval 'init:stop()'"

Rakefile

@@ -1,6 +1,8 @@
 task :get_image_version do
-  version = IO.read("transport-site/Dockerfile")[/FROM (hexpm\/elixir.*)/, 1]
+  dockerfile_content = IO.read("transport-site/Dockerfile")
+  version = dockerfile_content[/FROM (hexpm\/elixir.*)/, 1]
   version = version.gsub('hexpm/elixir:','elixir-')
-  fail "Unexpected FROM format, script must be verified" unless version =~ /\Aelixir\-[^\-]+\-erlang\-[^\-]+\-alpine\-[^\-]+\z/
-  puts version
+  tools_version = dockerfile_content[/transport-tools:v(\d+\.\d+\.\d+) as transport-tools/, 1]
+  fail "Unexpected FROM value (got #{version}), script must be adapted?" unless version =~ /\Aelixir\-[^\-]+\-erlang\-[^\-]+\-ubuntu\-focal\-[^\-]+\z/
+  puts version + "-transport-tools-" + tools_version
 end

transport-site/Dockerfile

@@ -1,7 +1,14 @@
-# We leverage the base images published by hexpm.
+# We are interested in the binaries compiled on that container:
+FROM ghcr.io/etalab/transport-tools:v1.0.3 as transport-tools
+
+# We leverage the base images published by hexpm at:
 #
 # https://hub.docker.com/r/hexpm/elixir
 #
+# Which are built via:
+#
+# https://github.com/hexpm/bob#docker-images
+#
 # They provide the ability to decouple Elixir version
 # and OTP version, which is a nice feature for
 # incremental/decoupled upgrades.
@@ -10,10 +17,10 @@
 # - https://github.com/elixir-lang/elixir/releases
 # - https://github.com/erlang/otp/releases
 #
-
-# We are interested in the binaries compiled on that container:
-FROM ghcr.io/etalab/transport-tools:v1.0.2 as transport-tools
-
+#
+# So again, to upgrade this, check out :
+#
+# https://hub.docker.com/r/hexpm/elixir
 FROM hexpm/elixir:1.12.2-erlang-24.0.4-ubuntu-focal-20210325
 
 ARG DEBIAN_FRONTEND=noninteractive
@@ -25,12 +32,32 @@ RUN apt-get update && apt-get install -y \
     libtool \
     git \
     tzdata \
-    zip
+    zip \
+    default-jre \
+    maven
+
+# Helps bump the output of /etc/os-release from says "Ubuntu 24.04.2 LTS" to "... 24.04.3"
+#
+# The source image (hex) is itself based on a ubuntu image whose
+# packages are not necessarily up-to-date all the time. We want to
+# upgrade and ensure we are as up-to-date as possible.
+# Note that the kernel itself cannot be upgraded here apparently
+# (https://stackoverflow.com/a/66413248)
+#
+# See https://github.com/etalab/transport_deploy/issues/46 for more context
+RUN apt-get upgrade -y
+
+# debugging information
+RUN uname --all
+RUN cat /etc/os-release
+RUN cat /etc/lsb-release
 
 ENV NVM_VERSION v0.29.0
 ENV NODE_VERSION 14.16.1
 ENV NVM_DIR $HOME/.nvm
 
+RUN mkdir $NVM_DIR
+
 # Install NVM
 RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/${NVM_VERSION}/install.sh | bash
 
@@ -62,3 +89,5 @@ COPY --from=transport-tools /usr/lib/libproj.* /usr/lib/
 RUN mkdir /usr/share/proj/
 COPY --from=transport-tools /usr/share/proj/ /usr/share/proj/
 RUN /transport-tools/gtfs2netexfr --help
+COPY --from=transport-tools /usr/local/bin/gtfs-realtime-validator-lib-1.0.0-SNAPSHOT.jar ./transport-tools
+RUN java -jar /transport-tools/gtfs-realtime-validator-lib-1.0.0-SNAPSHOT.jar 2>&1 | grep "For batch mode you must provide a path and file name to GTFS data"