From 52e6502efe0b9180521c19b6310de57083a6d6a7 Mon Sep 17 00:00:00 2001 From: ramil Date: Fri, 11 Sep 2020 18:25:34 +0300 Subject: [PATCH 1/3] docker compose with ipld-eth-indexer, ipld-eth-server and dapptools --- .gitignore | 2 +- dockerfiles/Dockerfile => Dockerfile | 2 +- Makefile | 2 +- docker-compose.yml | 125 ++++++++++++++++++ dockerfiles/docker-compose.yml | 86 ------------ dockerfiles/header_sync/Dockerfile | 43 ------ dockerfiles/header_sync/startup_script.sh | 42 ------ environments/example.toml | 2 +- environments/header_sync.toml | 16 +++ .../startup_script.sh => startup_script.sh | 0 10 files changed, 145 insertions(+), 175 deletions(-) rename dockerfiles/Dockerfile => Dockerfile (95%) create mode 100644 docker-compose.yml delete mode 100644 dockerfiles/docker-compose.yml delete mode 100644 dockerfiles/header_sync/Dockerfile delete mode 100755 dockerfiles/header_sync/startup_script.sh create mode 100644 environments/header_sync.toml rename dockerfiles/startup_script.sh => startup_script.sh (100%) diff --git a/.gitignore b/.gitignore index a3d73d99..aac26316 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ .vscode test_data_dir/ contracts/* -environments/*.toml +#environments/*.toml Vagrantfile vagrant*.sh .vagrant diff --git a/dockerfiles/Dockerfile b/Dockerfile similarity index 95% rename from dockerfiles/Dockerfile rename to Dockerfile index 0c476d57..0b59cfe7 100644 --- a/dockerfiles/Dockerfile +++ b/Dockerfile @@ -26,7 +26,7 @@ USER $USER # chown first so dir is writable COPY --chown=$USER:$USER --from=builder /go/src/github.com/vulcanize/eth-contract-watcher/$CONFIG_FILE config.toml -COPY --chown=$USER:$USER --from=builder /go/src/github.com/vulcanize/eth-contract-watcher/dockerfiles/startup_script.sh . +COPY --chown=$USER:$USER --from=builder /go/src/github.com/vulcanize/eth-contract-watcher/startup_script.sh . # keep binaries immutable COPY --from=builder /go/src/github.com/vulcanize/eth-contract-watcher/eth-contract-watcher eth-contract-watcher diff --git a/Makefile b/Makefile index fca5cbb0..a7f2b9f2 100644 --- a/Makefile +++ b/Makefile @@ -119,4 +119,4 @@ migrate: $(GOOSE) checkdbvars ## Build docker image .PHONY: docker-build docker-build: - docker build -t vulcanize/eth-contract-watcher -f dockerfiles/Dockerfile . \ No newline at end of file + docker build -t vulcanize/eth-contract-watcher -f Dockerfile . \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..6dbbe001 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,125 @@ +version: '3.2' + +services: + dapptools: + restart: unless-stopped + image: vulcanize/dapptools:v0.29.0-statediff-0.0.2 + ports: + - "127.0.0.1:8545:8545" + - "127.0.0.1:8546:8546" + + eth-indexer: + restart: unless-stopped + depends_on: + - indexer-db + - dapptools + image: vulcanize/ipld-eth-indexer:v0.3.0-alpha + environment: + DATABASE_NAME: vulcanize_public + DATABASE_HOSTNAME: indexer-db + DATABASE_PORT: 5432 + DATABASE_USER: vdbm + DATABASE_PASSWORD: password + ETH_WS_PATH: "dapptools:8546" + ETH_HTTP_PATH: "dapptools:8545" + ETH_CHAIN_ID: 4 + ETH_NETWORK_ID: 4 + VDB_COMMAND: sync + + eth-server: + depends_on: + - indexer-db + - eth-indexer + image: vulcanize/ipld-eth-server:v0.0.13 + environment: + VDB_COMMAND: serve + DATABASE_NAME: vulcanize_public + DATABASE_HOSTNAME: indexer-db + DATABASE_PORT: 5432 + DATABASE_USER: vdbm + DATABASE_PASSWORD: password + SERVER_WS_PATH: "0.0.0.0:8080" + SERVER_HTTP_PATH: "0.0.0.0:8081" + LOGRUS_LEVEL: debug + ports: + - "127.0.0.1:8080:8080" + - "127.0.0.1:8081:8081" + + indexer-db: + restart: unless-stopped + image: postgres:10.12-alpine + environment: + - POSTGRES_USER=vdbm + - POSTGRES_DB=vulcanize_public + - POSTGRES_PASSWORD=password + volumes: + - indexer_db_data:/var/lib/postgresql/data + ports: + - "127.0.0.1:8069:5432" + + contact-watcher-db: + restart: unless-stopped + image: postgres:10.12-alpine + environment: + - POSTGRES_USER=vdbm + - POSTGRES_DB=vulcanize_public + - POSTGRES_PASSWORD=password + volumes: + - contact_watcher_db_data:/var/lib/postgresql/data + ports: + - "127.0.0.1:8068:5432" + + eth-header-sync: + restart: unless-stopped + depends_on: + - contact-watcher-db + image: vulcanize/eth-header-sync:v0.1.1 + volumes: + - ./environments/header_sync.toml:/app/config.toml + environment: + - STARTING_BLOCK_NUMBER=1 + - VDB_COMMAND=sync + - DATABASE_NAME=vulcanize_public + - DATABASE_HOSTNAME=contact-watcher-db + - DATABASE_PORT=5432 + - DATABASE_USER=vdbm + - DATABASE_PASSWORD=password + + eth-contract-watcher: + depends_on: + - contact-watcher-db + build: + context: "" + cache_from: + - alpine:latest + - golang:1.13 + dockerfile: Dockerfile + volumes: + - ./geth.ipc:/geth.ipc + - ./environments/example.toml:/app/config.toml + environment: + - VDB_COMMAND=watch + - DATABASE_NAME=vulcanize_public + - DATABASE_HOSTNAME=contact-watcher-db + - DATABASE_PORT=5432 + - DATABASE_USER=vdbm + - DATABASE_PASSWORD=password + + contract-watcher-graphql: + restart: unless-stopped + depends_on: + - contact-watcher-db + image: vulcanize/postgraphile:v1.0 + environment: + - PG_HOST=contact-watcher-db + - PG_PORT=5432 + - PG_DATABASE=vulcanize_public + - PG_USER=vdbm + - PG_PASSWORD=password + - SCHEMA=public,header_0xd850942ef8811f2a866692a623011bde52a462c1 + ports: + - "127.0.0.1:5000:5000" + +volumes: + contact_watcher_db_data: + indexer_db_data: \ No newline at end of file diff --git a/dockerfiles/docker-compose.yml b/dockerfiles/docker-compose.yml deleted file mode 100644 index 3d00e18a..00000000 --- a/dockerfiles/docker-compose.yml +++ /dev/null @@ -1,86 +0,0 @@ -version: '3.2' - -services: - geth: - image: ethereum/client-go:v1.9.11 - expose: - - "8545" - - "8546" - ports: - - "127.0.0.1:8545:8545" - - "127.0.0.1:8546:8546" - command: ["--syncmode", "full", - "--cache", "4096", - "--cache.database", "50", - "--cache.gc", "50", - "--rpc", - "--rpcapi", "eth,net"] - - contact-watcher-db: - restart: always - image: postgres:10.12-alpine - environment: - - POSTGRES_USER=vdbm - - POSTGRES_DB=vulcanize_public - - POSTGRES_PASSWORD=password - volumes: - - vulcanizedb_db_data:/var/lib/postgresql/data - expose: - - "5432" - ports: - - "127.0.0.1:8079:5432" - - eth-header-sync: - restart: unless-stopped - depends_on: - - contact-watcher-db - image: vulcanize/eth-header-sync:v1.0.0 - volumes: - - ./geth.ipc:/geth.ipc - - ../environments/example.toml:/app/config.toml - environment: - - STARTING_BLOCK_NUMBER=10564606 - - VDB_COMMAND=sync - - DATABASE_NAME=vulcanize_public - - DATABASE_HOSTNAME=contact-watcher-db - - DATABASE_PORT=5432 - - DATABASE_USER=vdbm - - DATABASE_PASSWORD=password - - eth-contract-watcher: - depends_on: - - contact-watcher-db - build: - context: ./../ - cache_from: - - alpine:latest - - golang:1.13 - dockerfile: ./dockerfiles/Dockerfile - volumes: - - ./geth.ipc:/geth.ipc - - ../environments/example.toml:/app/config.toml - environment: - - VDB_COMMAND=watch - - DATABASE_NAME=vulcanize_public - - DATABASE_HOSTNAME=contact-watcher-db - - DATABASE_PORT=5432 - - DATABASE_USER=vdbm - - DATABASE_PASSWORD=password - - contract-watcher-graphql: - restart: unless-stopped - depends_on: - - contact-watcher-db - image: vulcanize/postgraphile:v1.0 - environment: - - PG_HOST=contact-watcher-db - - PG_PORT=5432 - - PG_DATABASE=vulcanize_public - - PG_USER=vdbm - - PG_PASSWORD=password - - SCHEMA=public,header_0xd850942ef8811f2a866692a623011bde52a462c1 - ports: - - "127.0.0.1:5000:5000" - -volumes: - vulcanizedb_db_data: \ No newline at end of file diff --git a/dockerfiles/header_sync/Dockerfile b/dockerfiles/header_sync/Dockerfile deleted file mode 100644 index 247977d5..00000000 --- a/dockerfiles/header_sync/Dockerfile +++ /dev/null @@ -1,43 +0,0 @@ -FROM golang:1.13-alpine as builder - -RUN apk --update --no-cache add make git g++ linux-headers - -# Add this repo for the startup_script -ADD . /go/src/github.com/vulcanize/eth-contract-watcher - -# Get and build eth-header-sync -RUN git clone https://github.com/vulcanize/eth-header-sync.git /go/src/github.com/vulcanize/eth-header-sync -WORKDIR /go/src/github.com/vulcanize/eth-header-sync -RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o eth-header-sync . - -# Build migration tool -WORKDIR / -RUN git clone https://github.com/pressly/goose.git /go/src/github.com/pressly/goose/ -WORKDIR /go/src/github.com/pressly/goose/cmd/goose -RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_mysql no_sqlite' -o goose . - -WORKDIR /go/src/github.com/vulcanize/eth-header-sync - -# app container -FROM alpine - -ARG USER="vdm" -ARG CONFIG_FILE="./environments/example.toml" - -RUN adduser -Du 5000 $USER -WORKDIR /app -RUN chown $USER /app -USER $USER - -# chown first so dir is writable -# note: using $USER is merged, but not in the stable release yet -COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/eth-contract-watcher/$CONFIG_FILE config.toml -COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/eth-contract-watcher/dockerfiles/header_sync/startup_script.sh . - -# keep binaries immutable -COPY --from=builder /go/src/github.com/vulcanize/eth-header-sync/eth-header-sync eth-header-sync -COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose -COPY --from=builder /go/src/github.com/vulcanize/eth-header-sync/db/migrations migrations/vulcanizedb -COPY --from=builder /go/src/github.com/vulcanize/eth-header-sync/environments environments - -ENTRYPOINT ["/app/startup_script.sh"] \ No newline at end of file diff --git a/dockerfiles/header_sync/startup_script.sh b/dockerfiles/header_sync/startup_script.sh deleted file mode 100755 index 3bd934ca..00000000 --- a/dockerfiles/header_sync/startup_script.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/sh -# Runs the db migrations and starts the watcher services - -# Exit if the variable tests fail -set -e -set +x - -# Check the database variables are set -test $DATABASE_HOSTNAME -test $DATABASE_NAME -test $DATABASE_PORT -test $DATABASE_USER -test $DATABASE_PASSWORD -test $VDB_COMMAND -test $STARTING_BLOCK_NUMBER -set +e - -# Construct the connection string for postgres -VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable - -# Run the DB migrations -echo "Connecting with: $VDB_PG_CONNECT" -echo "Running database migrations" -./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up - - -# If the db migrations ran without err -if [[ $? -eq 0 ]]; then - echo "Running the VulcanizeDB process" - ./eth-header-sync ${VDB_COMMAND} --config=config.toml --starting-block-number=${STARTING_BLOCK_NUMBER} -else - echo "Could not run migrations. Are the database details correct?" - exit 1 -fi - -# If VulcanizeDB process was successful -if [ $? -eq 0 ]; then - echo "VulcanizeDB process ran successfully" -else - echo "Could not start VulcanizeDB process. Is the config file correct?" - exit 1 -fi \ No newline at end of file diff --git a/environments/example.toml b/environments/example.toml index 5169bfc9..944ad39e 100644 --- a/environments/example.toml +++ b/environments/example.toml @@ -6,7 +6,7 @@ password = "password" [client] - rpcPath = "/geth.ipc" + rpcPath = "http://eth-server:8081" [contract] network = "" diff --git a/environments/header_sync.toml b/environments/header_sync.toml new file mode 100644 index 00000000..b95e1323 --- /dev/null +++ b/environments/header_sync.toml @@ -0,0 +1,16 @@ +[database] + name = "vulcanize_public" + hostname = "contact-watcher-db" + port = 5432 + user = "vdbm" + password = "password" + +[client] + rpcPath = "http://eth-server:8081" + +[ethereum] + nodeID = "arch1" # $ETH_NODE_ID + clientName = "Geth" # $ETH_CLIENT_NAME + genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" # $ETH_GENESIS_BLOCK + networkID = "4" # $ETH_NETWORK_ID + chainID = "4" # $ETH_CHAIN_ID \ No newline at end of file diff --git a/dockerfiles/startup_script.sh b/startup_script.sh similarity index 100% rename from dockerfiles/startup_script.sh rename to startup_script.sh From 7f5dd1cbb6bf0c2d58909f2d158e0b0ebce6bf28 Mon Sep 17 00:00:00 2001 From: ramil Date: Fri, 11 Sep 2020 21:39:07 +0300 Subject: [PATCH 2/3] fix vulcanize/postgraphile tag --- docker-compose.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6dbbe001..678ba60b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -95,7 +95,6 @@ services: - golang:1.13 dockerfile: Dockerfile volumes: - - ./geth.ipc:/geth.ipc - ./environments/example.toml:/app/config.toml environment: - VDB_COMMAND=watch @@ -109,7 +108,7 @@ services: restart: unless-stopped depends_on: - contact-watcher-db - image: vulcanize/postgraphile:v1.0 + image: vulcanize/postgraphile:v1.0.1 environment: - PG_HOST=contact-watcher-db - PG_PORT=5432 From 0e7fb2c2324c343d5c270e653d6c418e329209ff Mon Sep 17 00:00:00 2001 From: ramil Date: Fri, 18 Sep 2020 17:17:46 +0300 Subject: [PATCH 3/3] github actions on pull request and release --- .github/workflows/on-master.yaml | 25 +++++++++++++++++++++ .github/workflows/{build.yml => on-pr.yaml} | 4 ++-- .github/workflows/publish.yaml | 25 +++++++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/on-master.yaml rename .github/workflows/{build.yml => on-pr.yaml} (73%) create mode 100644 .github/workflows/publish.yaml diff --git a/.github/workflows/on-master.yaml b/.github/workflows/on-master.yaml new file mode 100644 index 00000000..7f814f82 --- /dev/null +++ b/.github/workflows/on-master.yaml @@ -0,0 +1,25 @@ +name: Docker Compose Build + +on: + push: + branches: + - master + +jobs: + build: + name: Run docker build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Get the version + id: vars + run: echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7}) + - name: Run docker build + run: make docker-build + - name: Tag docker image + run: docker tag vulcanize/eth-contract-watcher docker.pkg.github.com/vulcanize/eth-contract-watcher/eth-contract-watcher:${{steps.vars.outputs.sha}} + - name: Docker Login + run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin + - name: Docker Push + run: docker push docker.pkg.github.com/vulcanize/eth-contract-watcher/eth-contract-watcher:${{steps.vars.outputs.sha}} + diff --git a/.github/workflows/build.yml b/.github/workflows/on-pr.yaml similarity index 73% rename from .github/workflows/build.yml rename to .github/workflows/on-pr.yaml index ad79056b..568dd104 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/on-pr.yaml @@ -1,10 +1,10 @@ -name: Docker Compose Build +name: Docker Build on: [pull_request] jobs: build: - name: Run docker-compose build + name: Run docker build runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml new file mode 100644 index 00000000..8002cc44 --- /dev/null +++ b/.github/workflows/publish.yaml @@ -0,0 +1,25 @@ +name: Publish Docker image +on: + release: + types: [published] +jobs: + push_to_registries: + name: Push Docker image to Docker Hub + runs-on: ubuntu-latest + steps: + - name: Get the version + id: vars + run: | + echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7}) + echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/}) + - name: Docker Login to Github Registry + run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin + - name: Docker Pull + run: docker pull docker.pkg.github.com/vulcanize/eth-contract-watcher/eth-contract-watcher:${{steps.vars.outputs.sha}} + - name: Docker Login to Docker Registry + run: echo ${{ secrets.VULCANIZEJENKINS_PAT }} | docker login -u vulcanizejenkins --password-stdin + - name: Tag docker image + run: docker tag docker.pkg.github.com/vulcanize/eth-contract-watcher/eth-contract-watcher:${{steps.vars.outputs.sha}} vulcanize/eth-contract-watcher:${{steps.vars.outputs.tag}} + - name: Docker Push to Docker Hub + run: docker push vulcanize/eth-contract-watcher:${{steps.vars.outputs.tag}} +