Added new image with generated block logs for use in testing

Author: Lucius

.gitlab-ci.yaml

@@ -665,17 +665,20 @@ broadcast_tests:
 test_tools_tests:
   stage: test
   extends: .test_tools_based
-
+  needs:
+    - !reference [.test_tools_based, needs]
+    - job: copy_testing_block_logs_to_cache
   variables:
     PYTEST_TIMEOUT_MINUTES: 30
     PYTEST_JUNIT_REPORT: $CI_PROJECT_DIR/tests/python/hive-local-tools/test-tools/tests/report.xml
-
+    TESTING_BLOCK_LOGS_DESTINATION: $CI_PROJECT_DIR/tests/python/hive-local-tools/test-tools/tests/testing_block_logs
   script:
-    - cd tests/python/hive-local-tools/test-tools/tests
+    - cp -r $TESTING_BLOCK_LOGS_CACHE_DIR/* $TESTING_BLOCK_LOGS_DESTINATION
+    - cd $CI_PROJECT_DIR/tests/python/hive-local-tools/test-tools/tests
     - pip install local-tools/
     - !reference [.run-pytest, script]
   tags:
-    - public-runner-docker
+    - data-cache-storage
 
 rc_direct_delegations_tests:
   stage: test
@@ -1184,6 +1187,61 @@ authority_tests:
   tags:
     - public-runner-docker
 
+generate_testing_block_logs_and_save_image:
+  stage: build
+  extends: [.extended_block_log_creation, .test_tools_based]
+  timeout: 1h
+  variables:
+    GIT_DEPTH: 0
+    TESTING_BLOCK_LOGS_DIR: "$CI_PROJECT_DIR/testing_block_logs"
+  script:
+  - $CI_PROJECT_DIR/scripts/ci-helpers/testing_block_log_image_generator.sh
+  artifacts:
+     reports:
+       dotenv:
+         - testing_block_log_latest_version.env
+         - testing_block_log_latest_commit_short_sha.env
+  tags:
+  - public-runner-docker
+  - hived-for-tests
+
+copy_testing_block_logs_to_cache:
+  stage: build
+  extends: .docker_image_builder_job
+  needs:
+    - job: generate_testing_block_logs_and_save_image
+      artifacts: true
+  variables:
+    TESTING_BLOCK_LOGS_CACHE_DIR: "/cache/testing_block_logs_${TESTING_BLOCK_LOG_LATEST_COMMIT_SHORT_SHA}"
+  script:
+    - echo $TESTING_BLOCK_LOGS_CACHE_DIR
+    - echo "TESTING_BLOCK_LOGS_CACHE_DIR=$TESTING_BLOCK_LOGS_CACHE_DIR" > $CI_PROJECT_DIR/testing_block_log_cache_dir.env
+    - |
+      if [ -d "$TESTING_BLOCK_LOGS_CACHE_DIR" ]; then
+        echo "Directory $TESTING_BLOCK_LOGS_CACHE_DIR exists. Skipping copy from image process..."
+      else
+        echo "Start copy testing block logs from image - $TESTING_BLOCK_LOG_LATEST_VERSION_IMAGE"
+        mkdir -p $TESTING_BLOCK_LOGS_CACHE_DIR && cd $TESTING_BLOCK_LOGS_CACHE_DIR
+        echo "Start copy block_logs from - $TESTING_BLOCK_LOG_LATEST_VERSION_IMAGE"
+        docker pull $TESTING_BLOCK_LOG_LATEST_VERSION_IMAGE
+        $CI_PROJECT_DIR/scripts/ci-helpers/export-data-from-docker-image.sh $TESTING_BLOCK_LOG_LATEST_VERSION_IMAGE $TESTING_BLOCK_LOGS_CACHE_DIR --image-path=/
+        checksum_from_image=$(docker inspect --format='{{ index .Config.Labels "testing_block_logs_checksum" }}' $TESTING_BLOCK_LOG_LATEST_VERSION_IMAGE)
+        echo "Checksum from image: $checksum_from_image"
+        checksum_after_copy=$(find $TESTING_BLOCK_LOGS_CACHE_DIR -type f | sort | xargs cat | md5sum |cut -d ' ' -f 1)
+        echo "Checksum after copy: $checksum_after_copy"
+        if [[ "$checksum_from_image" != "$checksum_after_copy" ]]; then
+          echo "Checksum mismatch detected! The checksum from the container does not match the checksum after copying."
+          exit 1
+        fi
+      fi
+    - tree $TESTING_BLOCK_LOGS_CACHE_DIR
+  artifacts:
+    reports:
+      dotenv:
+        - testing_block_log_cache_dir.env
+  tags:
+    - data-cache-storage
+
 generate_universal_block_logs_and_save_image:
   stage: build
   extends: [.extended_block_log_creation, .test_tools_based]

scripts/ci-helpers/testing_block_log_image_generator.sh

@@ -0,0 +1,70 @@
+#! /bin/bash
+
+set -euo pipefail
+
+SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+
+# shellcheck source=./docker_image_utils.sh
+source "$SCRIPTPATH/docker_image_utils.sh"
+
+submodule_path=$CI_PROJECT_DIR
+REGISTRY=$CI_REGISTRY_IMAGE
+REGISTRY_USER=$REGISTRY_USER
+REGISTRY_PASSWORD=$REGISTRY_PASS
+IMGNAME=/testing-block-logs
+
+echo "Attempting to get commit for: $submodule_path"
+
+CHANGES=(tests/python/functional/util/testing_block_logs/generate_testing_block_logs.py)
+commit=$("$SCRIPTPATH/retrieve_last_commit.sh" "${submodule_path}" "${CHANGES[@]}")
+echo "commit with last source code changes is $commit"
+
+pushd "${submodule_path}"
+short_commit=$(git -c core.abbrev=8 rev-parse --short "$commit")
+popd
+
+prefix_tag="testing-block-log"
+tag=$prefix_tag-$short_commit
+
+img=$( build_image_name $IMGNAME "$tag" "$REGISTRY" )
+img_path=$( build_image_registry_path $IMGNAME "$tag" "$REGISTRY" )
+img_tag=$( build_image_registry_tag $IMGNAME "$tag" "$REGISTRY" )
+
+echo "$REGISTRY_PASSWORD" | docker login -u "$REGISTRY_USER" "$REGISTRY" --password-stdin
+
+image_exists=0
+docker_image_exists $IMGNAME "$tag" "$REGISTRY" image_exists
+
+if [ "$image_exists" -eq 1 ];
+then
+  echo "Testing block log image is up to date. Skipping generation..."
+else
+  echo "${img} image is missing. Starting generation of testing block logs..."
+  echo "Save block logs to directory: $TESTING_BLOCK_LOGS_DIR"
+  cd tests/python/functional/util/testing_block_logs
+  python3 generate_testing_block_logs.py --output-block-log-directory="$TESTING_BLOCK_LOGS_DIR/block_logs_for_testing"
+  echo "Block logs saved in: $TESTING_BLOCK_LOGS_DIR"
+
+  checksum=$(find $TESTING_BLOCK_LOGS_DIR -type f | sort | xargs cat | md5sum |cut -d ' ' -f 1)
+  echo "Checksum of the generated testing block logs: $checksum"
+
+  echo "Build a Dockerfile"
+
+  pwd
+  cd $TESTING_BLOCK_LOGS_DIR
+  pwd
+
+  cat <<EOF > Dockerfile
+FROM scratch
+LABEL testing_block_logs_checksum=${checksum}
+COPY block_logs_for_testing /testing_block_logs
+EOF
+  cat Dockerfile
+  echo "Build docker image containing testing_block_logs"
+  docker build -t $img .
+  docker push $img
+  echo "Created and push docker image with testing block logs: $img"
+fi
+
+echo "TESTING_BLOCK_LOG_LATEST_VERSION_IMAGE=$img" > $CI_PROJECT_DIR/testing_block_log_latest_version.env
+echo "TESTING_BLOCK_LOG_LATEST_COMMIT_SHORT_SHA=$short_commit" > $CI_PROJECT_DIR/testing_block_log_latest_commit_short_sha.env

tests/python/functional/util/testing_block_logs/__init__.py

@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+from loguru import logger
+
+import test_tools as tt
+
+
+def generate_and_copy_empty_log(
+    split_value: int,
+    block_count: int,
+    target_dir: Path,
+) -> None:
+    node = tt.InitNode()
+    node.config.block_log_split = split_value
+
+    node.run(time_control=tt.SpeedUpRateTimeControl(speed_up_rate=10))
+    tt.logger.info(f"Waiting for block {block_count}...")
+    node.wait_for_block_with_number(block_count)
+
+    node.close()
+    node.block_log.copy_to(target_dir / f"_{block_count}", artifacts="required")
+
+
+def prepare_empty_logs(
+    output_block_log_directory: Path,
+) -> None:
+    """
+    This script generates block logs (both split and monlithic) of different length with empty blocks only
+    """
+    logger.disable("helpy")
+    logger.disable("test_tools")
+
+    # Initial 30 blocks are needed to have at least 10 irreversible ones.
+    block_log_directory_30 = output_block_log_directory / "empty_30"
+    block_log_directory_30.mkdir(parents=True, exist_ok=True)
+    generate_and_copy_empty_log(9999, 30, block_log_directory_30)
+    generate_and_copy_empty_log(-1, 30, block_log_directory_30)
+
+    # 430 blocks is enough for the split block log to be in 2 parts.
+    block_log_directory_430 = output_block_log_directory / "empty_430"
+    block_log_directory_430.mkdir(parents=True, exist_ok=True)
+    generate_and_copy_empty_log(9999, 430, block_log_directory_430)
+    generate_and_copy_empty_log(-1, 430, block_log_directory_430)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--output-block-log-directory", type=Path, default=Path(__file__).parent)
+    args = parser.parse_args()
+    prepare_empty_logs(args.output_block_log_directory)
+    # TODO: Add generation of other logs here.