Fix/ci performance

.circleci/config.yml

@@ -1,63 +1,143 @@
-version: 2
-jobs:
-  build:
-    machine:
-      image: ubuntu-1604:201903-01
-    environment:
-      RUST_VERSION: nightly-2019-07-14
+version: 2.1
+commands:
+  install-rust:
     steps:
-      - checkout
-      - restore_cache:
-          keys:
-            - v1-cargo-global-cache-{{ arch }}-{{ checksum "Cargo.lock" }}-{{ checksum ".circleci/RUST_VERSION" }}
-            - v1-cargo-global-cache-{{ arch }}-{{ checksum "Cargo.lock" }}-
       - run:
-          name: Install rust
+          name: Install Rust
           command: |
             sudo apt-get update
             sudo apt-get -y install apt-utils cmake pkg-config libssl-dev git llvm clang
             if [ ! -d /home/circleci/.cargo ]; then
               curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
             fi
-            source $HOME/.cargo/env
+            source ~/.cargo/env
             rustup toolchain install $RUST_VERSION
             rustup default $RUST_VERSION
             rustup target add wasm32-unknown-unknown --toolchain=$RUST_VERSION
+            export RUSTC_WRAPPER=""
             command -v wasm-gc || cargo install --git https://github.com/alexcrichton/wasm-gc --force
             rustup target add x86_64-unknown-linux-musl --toolchain=$RUST_VERSION
             rustc --version; cargo --version; rustup --version
+  install-sccache:
+    steps:
+      - run:
+          name: Install sccache
+          command: |
+            curl -L https://github.com/mozilla/sccache/releases/download/0.2.10/sccache-0.2.10-x86_64-unknown-linux-musl.tar.gz | tar -xz
+            chmod +x sccache-0.2.10-x86_64-unknown-linux-musl/sccache
+            mv sccache-0.2.10-x86_64-unknown-linux-musl/sccache ~/.cargo/bin/sccache
+            sccache --version
+  restore-cache:
+    steps:
       - restore_cache:
-          keys:
-            - v1-cargo-wasm-cache-{{ arch }}-{{ checksum "Cargo.lock" }}-{{ checksum ".circleci/RUST_VERSION" }}
-            - v1-cargo-wasm-cache-{{ arch }}-{{ checksum "Cargo.lock" }}-
+          name: Restore sccache
+          key: sccache-{{ arch }}-{{ .Environment.CIRCLE_JOB }}
+  save-cache:
+    steps:
       - save_cache:
+          name: Save sccache
+          # We use {{ epoch }} to always upload a fresh cache:
+          # Of course, restore_cache will not find this exact key,
+          # but it will fall back to the closest key (aka the most recent).
+          # See https://discuss.circleci.com/t/add-mechanism-to-update-existing-cache-key/9014/13
+          key: sccache-{{ arch }}-{{ .Environment.CIRCLE_JOB }}-{{ epoch }}
           paths:
-            - runtime/wasm/target/
-          key: v1-cargo-wasm-cache-{{ arch }}-{{ checksum "Cargo.lock" }}-{{ checksum ".circleci/RUST_VERSION" }}
-      - restore_cache:
-          keys:
-            - v1-target-cache-{{ arch }}-{{ checksum "Cargo.lock" }}-{{ checksum ".circleci/RUST_VERSION" }}
-            - v1-target-cache-{{ arch }}-{{ checksum "Cargo.lock" }}-
-      - run:
-          name: Build binary
-          command: |
-            source $HOME/.cargo/env
-            cargo build
-          no_output_timeout: 40m
+            - "~/.cache/sccache"
+  save-target-cache:
+    steps:
       - save_cache:
+          name: Save target cache
           paths:
             - target/debug/.fingerprint
             - target/debug/build
             - target/debug/deps
             - target/debug/incremental
-          key: v1-target-cache-{{ arch }}-{{ checksum "Cargo.lock" }}-{{ checksum ".circleci/RUST_VERSION" }}
-      - save_cache:
-          paths:
-            - /home/circleci/.cargo
-          key: v1-cargo-global-cache-{{ arch }}-{{ checksum "Cargo.lock" }}-{{ checksum ".circleci/RUST_VERSION" }}
+            - runtime/wasm/target/
+          key: target-cache-{{ arch }}-{{ checksum ".circleci/RUST_VERSION" }}-{{ .Environment.CIRCLE_BRANCH }}-{{ epoch }}
+  restore-target-cache:
+    steps:
+      - restore_cache:
+          name: Restore target cache
+          key: target-cache-{{ arch }}-{{ checksum ".circleci/RUST_VERSION" }}-{{ .Environment.CIRCLE_BRANCH }}
+jobs:
+  build:
+    machine:
+      image: ubuntu-1604:201903-01
+    environment:
+      BASH_ENV: ~/.cargo/env
+      RUST_VERSION: nightly-2019-07-14
+      RUSTC_WRAPPER: sccache
+      SCCACHE_CACHE_SIZE: 10G
+    steps:
+      - checkout
+      - install-rust
+      - install-sccache
+      - restore-cache
+      - restore-target-cache
+      - run:
+          name: Build
+          command: |
+            cargo check
+          no_output_timeout: 30m
+      - save-cache
+      - save-target-cache
+      - run:
+          name: Run srml tests
+          command: |
+            export CARGO_TARGET_DIR="~/project/target"
+            for p in $(find srml -name Cargo.toml | sort -u | sed 's/Cargo.toml//g'); do
+              pushd $p > /dev/null
+              echo "testing $p"
+              cargo test
+              popd > /dev/null
+            done
+      - save-cache
+      - save-target-cache
+      - run:
+          name: Run node tests
+          command: |
+            export CARGO_TARGET_DIR="~/project/target"
+            for p in $(find node -name Cargo.toml | sort -u | sed 's/Cargo.toml//g'); do
+              pushd $p > /dev/null
+              echo "testing $p"
+              cargo test
+              popd > /dev/null
+            done
+      - save-cache
+      - save-target-cache
+      - run:
+          name: Run core tests
+          command: |
+            export CARGO_TARGET_DIR="~/project/target"
+            for p in $(find core -name Cargo.toml | sort -u | sed 's/Cargo.toml//g'); do
+              pushd $p > /dev/null
+              echo "testing $p"
+              cargo test
+              popd > /dev/null
+            done
+      - save-cache
+      - save-target-cache
+      - run:
+          name: Run node-template tests
+          command: |
+            export CARGO_TARGET_DIR="~/project/target"
+            for p in $(find node-template -name Cargo.toml | sort -u | sed 's/Cargo.toml//g'); do
+              pushd $p > /dev/null
+              echo "testing $p"
+              cargo test
+              popd > /dev/null
+            done
+      - save-cache
+      - save-target-cache
       - run:
-          name: Run tests
+          name: Run subkey tests
           command: |
-            source $HOME/.cargo/env
-            cargo test --all --verbose --locked
-          no_output_timeout: 60m
+            export CARGO_TARGET_DIR="~/project/target"
+            for p in $(find subkey -name Cargo.toml | sort -u | sed 's/Cargo.toml//g'); do
+              pushd $p > /dev/null
+              cargo test
+              popd > /dev/null
+            done
+      - save-cache
+      - save-target-cache
+

.gitlab-ci.yml

@@ -121,8 +121,17 @@ test-linux-stable:                 &test-linux
     variables:
       - $DEPLOY_TAG
   script:
-    - time cargo test --all --release --verbose --locked
+    - time cargo test --all --release --verbose --locked |
+        tee output.log
     - sccache -s
+  after_script:
+    - echo "___Collecting warnings for check_warnings job___"
+    - awk '/^warning:/,/^$/ { print }' output.log > ${CI_COMMIT_SHORT_SHA}_warnings.log
+  artifacts:
+    name:                          $CI_COMMIT_SHORT_SHA
+    expire_in:                     24 hrs
+    paths:
+      - ${CI_COMMIT_SHORT_SHA}_warnings.log
 
 
 test-srml-staking:                 &test-srml-staking
@@ -165,7 +174,7 @@ test-linux-stable-int:
         &> ${CI_COMMIT_SHORT_SHA}_int_failure.log
     - sccache -s
   after_script:
-    - awk '/FAILED/,0' ${CI_COMMIT_SHORT_SHA}_int_failure.log
+    - awk '/FAILED|^error\[/,0' ${CI_COMMIT_SHORT_SHA}_int_failure.log
   artifacts:
     name:                          $CI_COMMIT_SHORT_SHA
     when:                          on_failure
@@ -193,7 +202,6 @@ check-web-wasm:
     - time cargo web build -p substrate-keystore
     - time cargo web build -p substrate-executor
     - time cargo web build -p substrate-network
-    - time cargo web build -p substrate-offchain
     - time cargo web build -p substrate-panic-handler
     - time cargo web build -p substrate-peerset
     - time cargo web build -p substrate-primitives
@@ -281,6 +289,25 @@ build-rust-doc-release:
     - echo "<meta http-equiv=refresh content=0;url=substrate_service/index.html>" > ./crate-docs/index.html
     - sccache -s
 
+check_warnings:
+  stage:                           build
+  <<:                              *docker-env
+  except:
+    variables:
+      - $DEPLOY_TAG
+  variables:
+    GIT_STRATEGY:                  none
+  dependencies:
+    - test-linux-stable
+  script:
+    - if [ -s ${CI_COMMIT_SHORT_SHA}_warnings.log ]; then
+        cat ${CI_COMMIT_SHORT_SHA}_warnings.log;
+        exit 1;
+      else
+        echo "___No warnings___";
+      fi
+  allow_failure:                   true
+
 #### stage:                        publish
 
 .publish-docker-release:           &publish-docker-release
@@ -308,7 +335,7 @@ build-rust-doc-release:
     - docker push $CONTAINER_IMAGE:$VERSION
     - docker push $CONTAINER_IMAGE:latest
 
-publish-docker-substrate:          
+publish-docker-substrate:
   stage:                           publish
   <<:                              *publish-docker-release
   # collect VERSION artifact here to pass it on to kubernetes

CODEOWNERS

@@ -0,0 +1,15 @@
+# Lists some code owners
+#
+# For details about syntax, see:
+# https://help.github.com/en/articles/about-code-owners
+# But here are some important notes:
+#
+# - Glob syntax is git-like, e.g. `/core` means the core directory in the root, unlike `core` which
+#   can be everywhere.
+# - Multiple owners are supported.
+# - Either handle (e.g, @pepyakin) or email can be used. Keep in mind, that handles might work better because they
+#   are more recognizable on GitHub, you can use them for mentioning unlike an email.
+# - The latest matching rule, if multiple, takes precedence.
+
+/srml/contracts/ @pepyakin
+/core/executor/ @pepyakin