Merge branch 'master' into bbusa/pullpolicy

Author: barnabasbusa

README.md

@@ -12,6 +12,7 @@ A collection of reusable ansible components used by the EthPandaOps team.
 - [dshackle](roles/dshackle)
 - [ethereum_auth_jwt](roles/ethereum_auth_jwt)
 - [ethereum_metrics_exporter](roles/ethereum_metrics_exporter)
+- [ethstats](roles/ethstats)
 - [xatu_sentry](roles/xatu_sentry)
 - [mev_boost](roles/mev_boost)
 

roles/arbitrum_node/tasks/main.yaml

@@ -6,6 +6,10 @@
     state: directory
     mode: "0750"
 
+- name: Set permissions
+  ansible.builtin.command: "chown -R 1000:1000 {{ arbitrum_node_datadir }}" # noqa deprecated-command-syntax
+  changed_when: false
+
 - name: Run arbitrum_node container
   community.docker.docker_container:
     name: "{{ arbitrum_node_container_name }}"

roles/beaconchain_explorer_aio/files/bigtable/init-bigtable.sh

@@ -29,6 +29,7 @@ cbt -project $PROJECT -instance $INSTANCE createfamily metadata erc721
 
 cbt -project $PROJECT -instance $INSTANCE createfamily metadata_updates blocks
 cbt -project $PROJECT -instance $INSTANCE createfamily metadata_updates f
+cbt -project $PROJECT -instance $INSTANCE createfamily metadata series
 
 cbt -project $PROJECT -instance $INSTANCE createfamily data c
 cbt -project $PROJECT -instance $INSTANCE createfamily data f

roles/beaconchain_explorer_aio/tasks/setup.yml

@@ -119,7 +119,7 @@
     - name: Check that bigtable service is running
       ansible.builtin.assert:
         that:
-          - beaconchain_explorer_aio_compose_db_meta.services["postgres"]["beaconchain_bigtable"].state.running
+          - beaconchain_explorer_aio_compose_db_bigtable_meta.services["bigtable"]["beaconchain_bigtable"].state.running
     - name: Init bigtable
       ansible.builtin.shell: "docker exec beaconchain_bigtable /init-bigtable.sh && touch {{ beaconchain_explorer_aio_installation_dir }}/bigtable/initialized"
       args:

roles/beaconchain_explorer_aio/templates/docker-compose.yml.j2

@@ -16,16 +16,16 @@ services:
       - "{{ beaconchain_explorer_aio_installation_dir }}/explorer/config.yml:/config.yml:ro"
       - "{{ beaconchain_explorer_aio_installation_dir }}/explorer/custom_configs:/custom_configs:ro"
       - "{{ beaconchain_explorer_aio_data_dir_explorer }}:/data"
-{%- if beaconchain_explorer_aio_bigtable_external_enabled %}
+{% if beaconchain_explorer_aio_bigtable_external_enabled %}
       - "{{ beaconchain_explorer_aio_installation_dir }}/bigtable_secret.json:/bigtable_secret.json:ro"
-{%- endif %}
+{% endif %}
     environment:
       - INDEXER_ENABLED=true
-{%- if beaconchain_explorer_aio_bigtable_external_enabled %}
+{% if beaconchain_explorer_aio_bigtable_external_enabled %}
       - GOOGLE_APPLICATION_CREDENTIALS=/bigtable_secret.json
-{%- else %}
+{% else %}
       - BIGTABLE_EMULATOR_HOST=beaconchain_bigtable:9000
-{%- endif %}
+{% endif %}
     ports:
       - 8082:8082
   #######################################################################
@@ -42,15 +42,15 @@ services:
       - "{{ beaconchain_explorer_aio_installation_dir }}/explorer/config.yml:/config.yml:ro"
       - "{{ beaconchain_explorer_aio_installation_dir }}/explorer/custom_configs:/custom_configs:ro"
       - "{{ beaconchain_explorer_aio_data_dir_explorer }}:/data"
-{%- if beaconchain_explorer_aio_bigtable_external_enabled %}
+{% if beaconchain_explorer_aio_bigtable_external_enabled %}
       - "{{ beaconchain_explorer_aio_installation_dir }}/bigtable_secret.json:/bigtable_secret.json:ro"
-{%- endif %}
+{% endif %}
     environment:
-{%- if beaconchain_explorer_aio_bigtable_external_enabled %}
+{% if beaconchain_explorer_aio_bigtable_external_enabled %}
       - GOOGLE_APPLICATION_CREDENTIALS=/bigtable_secret.json
-{%- else %}
+{% else %}
       - BIGTABLE_EMULATOR_HOST=beaconchain_bigtable:9000
-{%- endif %}
+{% endif %}
   #######################################################################
   statistics:
     container_name: beaconchain_statistics
@@ -65,16 +65,16 @@ services:
       - "{{ beaconchain_explorer_aio_installation_dir }}/explorer/config.yml:/config.yml:ro"
       - "{{ beaconchain_explorer_aio_installation_dir }}/explorer/custom_configs:/custom_configs:ro"
       - "{{ beaconchain_explorer_aio_data_dir_explorer }}:/data"
-{%- if beaconchain_explorer_aio_bigtable_external_enabled %}
+{% if beaconchain_explorer_aio_bigtable_external_enabled %}
       - "{{ beaconchain_explorer_aio_installation_dir }}/bigtable_secret.json:/bigtable_secret.json:ro"
-{%- endif %}
+{% endif %}
     environment:
       - INDEXER_ENABLED=false
-{%- if beaconchain_explorer_aio_bigtable_external_enabled %}
+{% if beaconchain_explorer_aio_bigtable_external_enabled %}
       - GOOGLE_APPLICATION_CREDENTIALS=/bigtable_secret.json
-{%- else %}
+{% else %}
       - BIGTABLE_EMULATOR_HOST=beaconchain_bigtable:9000
-{%- endif %}
+{% endif %}
   #######################################################################
   frontend-updater:
     container_name: beaconchain_frontend_updater
@@ -96,7 +96,7 @@ services:
       - FRONTEND_ENABLED=true
 {% if beaconchain_explorer_aio_bigtable_external_enabled %}
       - GOOGLE_APPLICATION_CREDENTIALS=/bigtable_secret.json
-{%- else %}
+{% else %}
       - BIGTABLE_EMULATOR_HOST=beaconchain_bigtable:9000
 {% endif %}
 
@@ -123,7 +123,7 @@ services:
       - FRONTEND_ENABLED=true
 {% if beaconchain_explorer_aio_bigtable_external_enabled %}
       - GOOGLE_APPLICATION_CREDENTIALS=/bigtable_secret.json
-{%- else %}
+{% else %}
       - BIGTABLE_EMULATOR_HOST=beaconchain_bigtable:9000
 {% endif %}
   #######################################################################
@@ -157,7 +157,7 @@ services:
       - "{{ beaconchain_explorer_aio_installation_dir }}/postgres/tables.sql:/docker-entrypoint-initdb.d/init.sql"
       - "{{ beaconchain_explorer_aio_data_dir_postgres }}:/var/lib/postgresql/data"
   #######################################################################
-{%- if not beaconchain_explorer_aio_bigtable_external_enabled %}
+{% if not beaconchain_explorer_aio_bigtable_external_enabled %}
   bigtable:
     container_name: beaconchain_bigtable
     image: "{{ beaconchain_explorer_aio_docker_image_bigtable }}"
@@ -177,4 +177,4 @@ services:
       - "{{ beaconchain_explorer_aio_installation_dir }}/bigtable/cbt-definitions.yml:/cbt-definitions.yml"
       - "{{ beaconchain_explorer_aio_installation_dir }}/bigtable/eth1.proto:/eth1.proto"
       - "{{ beaconchain_explorer_aio_data_dir_bigtable }}:/data"
-{%- endif %}
+{% endif %}

roles/besu/tasks/setup.yaml

@@ -11,6 +11,10 @@
     owner: "{{ besu_user }}"
     group: "{{ besu_user }}"
 
+- name: Set permissions
+  ansible.builtin.command: "chown -R {{ besu_user }}:{{ besu_user }} {{ besu_datadir }}" # noqa deprecated-command-syntax
+  changed_when: false
+
 - name: Run besu container
   community.docker.docker_container:
     name: "{{ besu_container_name }}"

roles/erigon/tasks/setup.yaml

@@ -11,6 +11,10 @@
     owner: "{{ erigon_user }}"
     group: "{{ erigon_user }}"
 
+- name: Set permissions
+  ansible.builtin.command: "chown -R {{ erigon_user }}:{{ erigon_user }} {{ erigon_datadir }}" # noqa deprecated-command-syntax
+  changed_when: false
+
 - name: Init custom network
   when: erigon_init_custom_network
   block:

roles/ethereumjs/tasks/setup.yaml

@@ -11,6 +11,10 @@
     owner: "{{ ethereumjs_user }}"
     group: "{{ ethereumjs_user }}"
 
+- name: Set permissions
+  ansible.builtin.command: "chown -R {{ ethereumjs_user }}:{{ ethereumjs_user }} {{ ethereumjs_datadir }}" # noqa deprecated-command-syntax
+  changed_when: false
+
 - name: Run ethereumjs container
   community.docker.docker_container:
     name: "{{ ethereumjs_container_name }}"

roles/ethstats/README.md

@@ -0,0 +1,38 @@
+# ethpandaops.general.ethstats
+
+Setup [ethstats](https://github.com/goerli/ethstats-server).
+
+## Requirements
+
+You'll need docker on the target system. Make sure to install it upfront.
+
+## Role Variables
+
+Default variables are defined in [defaults/main.yaml](defaults/main.yaml)
+
+## Dependencies
+
+You'll need docker to run this role. One way of installing docker could be via ansible galaxy with the following dependencies set within `requirements.yaml`:
+
+```yaml
+roles:
+- src: geerlingguy.docker
+  version: latest
+- src: geerlingguy.pip
+  version: latest
+```
+
+## Example Playbook
+
+Your playbook could look like this:
+
+```yaml
+- hosts: localhost
+  become: true
+  roles:
+  - role: geerlingguy.docker
+  - role: geerlingguy.pip
+    pip_install_packages:
+    - name: docker
+  - role: ethpandaops.general.ethstats
+```

roles/ethstats/defaults/main.yaml

@@ -0,0 +1,14 @@
+ethstats_user: ethstats
+
+# Image is based on https://github.com/goerli/ethstats-server/pull/32
+ethstats_container_image: skylenet/ethstats-server:pow-latest
+ethstats_container_name: ethstats
+ethstats_container_published_ports: []
+ethstats_container_restart_policy: always
+ethstats_container_networks_cli_compatible: true
+ethstats_container_network_mode: default
+ethstats_container_networks: []
+ethstats_container_volumes_from: []
+ethstats_container_volumes: []
+ethstats_container_env:
+  WS_SECRET: "" # shared secret required for incoming websocket connections

roles/ethstats/tasks/main.yaml

@@ -0,0 +1,23 @@
+- name: Add ethstats user
+  ansible.builtin.user:
+    name: "{{ ethstats_user }}"
+  register: ethstats_user_meta
+
+- name: Check ethstats websocket secret
+  ansible.builtin.assert:
+    that:
+      - ethstats_container_env.WS_SECRET != ""
+    fail_msg: "'ethstats_container_env.WS_SECRET' can't be empty"
+
+- name: Run ethstats container
+  community.general.docker_container:
+    name: "{{ ethstats_container_name }}"
+    image: "{{ ethstats_container_image }}"
+    published_ports: "{{ ethstats_container_published_ports }}"
+    restart_policy: "{{ ethstats_container_restart_policy }}"
+    networks: "{{ ethstats_container_networks }}"
+    networks_cli_compatible: "{{ ethstats_container_networks_cli_compatible }}"
+    network_mode: "{{ ethstats_container_network_mode }}"
+    volumes: "{{ ethstats_container_volumes }}"
+    env: "{{ ethstats_container_env }}"
+    user: "{{ ethstats_user_meta.uid }}"

roles/geth/defaults/main.yaml

@@ -42,6 +42,7 @@ geth_container_command_extra_args: []
 
 # Only required changing when running custom networks
 geth_init_custom_network: false
+geth_init_autoremove_enabled: false
 geth_init_custom_network_genesis_file: /config/genesis.json
 geth_init_custom_network_container_env: {}
 geth_init_custom_network_container_volumes:

roles/geth/tasks/setup.yaml

@@ -11,6 +11,10 @@
     owner: "{{ geth_user }}"
     group: "{{ geth_user }}"
 
+- name: Set permissions
+  ansible.builtin.command: "chown -R {{ geth_user }}:{{ geth_user }} {{ geth_datadir }}" # noqa deprecated-command-syntax
+  changed_when: false
+
 - name: Init custom network
   when: geth_init_custom_network
   block:
@@ -23,7 +27,7 @@
         name: "{{ geth_container_name }}-init"
         image: "{{ geth_container_image }}"
         detach: false
-        auto_remove: true
+        auto_remove: "{{ geth_init_autoremove_enabled }}"
         restart_policy: "no"
         state: started
         volumes: "{{ geth_init_custom_network_container_volumes }}"

roles/lighthouse/tasks/setup.yaml

@@ -11,6 +11,10 @@
     owner: "{{ lighthouse_user }}"
     group: "{{ lighthouse_user }}"
 
+- name: Set permissions
+  ansible.builtin.command: "chown -R {{ lighthouse_user }}:{{ lighthouse_user }} {{ lighthouse_datadir }}" # noqa deprecated-command-syntax
+  changed_when: false
+
 - name: Run lighthouse container
   community.docker.docker_container:
     name: "{{ lighthouse_container_name }}"
@@ -39,6 +43,11 @@
     - "{{ lighthouse_validator_datadir }}/secrets"
   when: lighthouse_validator_enabled
 
+- name: Set permissions for validator data dir
+  ansible.builtin.command: "chown -R {{ lighthouse_user }}:{{ lighthouse_user }} {{ lighthouse_validator_datadir }}" # noqa deprecated-command-syntax
+  changed_when: false
+  when: lighthouse_validator_enabled
+
 - name: Run lighthouse validator container
   community.docker.docker_container:
     name: "{{ lighthouse_validator_container_name }}"

roles/lodestar/tasks/setup.yaml

@@ -11,6 +11,10 @@
     owner: "{{ lodestar_user }}"
     group: "{{ lodestar_user }}"
 
+- name: Set permissions
+  ansible.builtin.command: "chown -R {{ lodestar_user }}:{{ lodestar_user }} {{ lodestar_datadir }}" # noqa deprecated-command-syntax
+  changed_when: false
+
 - name: Run lodestar container
   community.docker.docker_container:
     name: "{{ lodestar_container_name }}"
@@ -39,6 +43,11 @@
     - "{{ lodestar_validator_datadir }}/secrets"
   when: lodestar_validator_enabled
 
+- name: Set permissions for validator data dir
+  ansible.builtin.command: "chown -R {{ lodestar_user }}:{{ lodestar_user }} {{ lodestar_validator_datadir }}" # noqa deprecated-command-syntax
+  changed_when: false
+  when: lodestar_validator_enabled
+
 - name: Run lodestar validator container
   community.docker.docker_container:
     name: "{{ lodestar_validator_container_name }}"

roles/nethermind/tasks/setup.yaml

@@ -14,6 +14,10 @@
     - "{{ nethermind_datadir }}"
     - "{{ nethermind_datadir }}/nethermind"
 
+- name: Set permissions
+  ansible.builtin.command: "chown -R {{ nethermind_user }}:{{ nethermind_user }} {{ nethermind_datadir }}" # noqa deprecated-command-syntax
+  changed_when: false
+
 - name: Run nethermind container
   community.docker.docker_container:
     name: "{{ nethermind_container_name }}"

roles/nimbus/defaults/main.yaml

@@ -14,6 +14,18 @@ nimbus_validator_enabled: false
 nimbus_validator_fee_recipient: "0xF29Ff96aaEa6C9A1fBa851f74737f3c069d4f1a9" # theprotocolguild.eth
 nimbus_validator_datadir: /data/nimbus-validator
 
+################################################################################
+##
+## Checkpoint sync container configuration
+##
+################################################################################
+nimbus_checkpoint_container_name: nimbus_checkpoint
+nimbus_checkpoint_container_command:
+  - trustedNodeSync
+  - --data-dir=/data
+nimbus_checkpoint_container_command_extra_args: []
+nimbus_checkpoint_sync_enabled: false
+nimbus_checkpoint_autoremove_enabled: false
 ################################################################################
 ##
 ## Beacon node container configuration

roles/nimbus/tasks/setup.yaml

@@ -11,6 +11,10 @@
     owner: "{{ nimbus_user }}"
     group: "{{ nimbus_user }}"
 
+- name: Set permissions
+  ansible.builtin.command: "chown -R {{ nimbus_user }}:{{ nimbus_user }} {{ nimbus_datadir }}" # noqa deprecated-command-syntax
+  changed_when: false
+
 - name: Create validator data dir
   ansible.builtin.file:
     path: "{{ item }}"
@@ -24,6 +28,27 @@
     - "{{ nimbus_validator_datadir }}/secrets"
   when: nimbus_validator_enabled
 
+- name: Set permissions for validator data dir
+  ansible.builtin.command: "chown -R {{ nimbus_user }}:{{ nimbus_user }} {{ nimbus_validator_datadir }}" # noqa deprecated-command-syntax
+  changed_when: false
+  when: nimbus_validator_enabled
+
+- name: Checkpoint sync nimbus node
+  when: nimbus_checkpoint_sync_enabled
+  community.docker.docker_container:
+    name: "{{ nimbus_checkpoint_container_name }}"
+    image: "{{ nimbus_container_image }}"
+    state: started
+    detach: false
+    auto_remove: "{{ nimbus_checkpoint_autoremove_enabled }}"
+    restart_policy: "no"
+    stop_timeout: "{{ nimbus_container_stop_timeout }}"
+    volumes: "{{ nimbus_container_volumes }}"
+    env: "{{ nimbus_container_env }}"
+    networks: "{{ nimbus_container_networks }}"
+    command: "{{ nimbus_checkpoint_container_command + nimbus_checkpoint_container_command_extra_args }}"
+    user: "{{ nimbus_user_meta.uid }}"
+
 - name: Run nimbus container
   community.docker.docker_container:
     name: "{{ nimbus_container_name }}"