Merge branch 'master' into lars/batches

kadena-io · Jun 1, 2023 · 192010c · 192010c
2 parents 7d41401 + 8a2af1d
commit 192010c
Show file tree

Hide file tree

Showing 130 changed files with 4,415 additions and 1,912 deletions.
diff --git a/.github/actions/install-chainweb-from-artifacts/action.yml b/.github/actions/install-chainweb-from-artifacts/action.yml
@@ -17,10 +17,10 @@ runs:
       run: |
         case "${{ inputs.os }}" in
         ubuntu-20.04)
-          sudo apt-get install -u libtbb2 libgflags2.2
+          sudo apt-get install -u libgflags2.2
           ;;
         ubuntu-22.04)
-          sudo apt-get install -u libtbb12 libgflags2.2
+          sudo apt-get install -u libgflags2.2
           ;;
         mac*)
           brew update && brew install gnu-tar || true

diff --git a/.github/workflows/applications.yml b/.github/workflows/applications.yml
@@ -125,13 +125,13 @@ jobs:
         MATRIX="$(jq -c '.' <<EOF
         {
           "ghc": ["8.10.7", "9.0.2"],
-          "cabal": ["3.8"],
+          "cabal": ["3.10"],
           "os": ["ubuntu-20.04", "ubuntu-22.04"],
           "use-freeze-file": ["false"],
           "include": [
             {
               "ghc": "8.10.7",
-              "cabal": "3.8",
+              "cabal": "3.10",
               "os" : "ubuntu-20.04",
               "use-freeze-file": "true"
             }
@@ -216,8 +216,13 @@ jobs:
     #     for f in $(git ls-tree -r -t --full-name --name-only "$rev") ; do
     #       touch -d $(git log --pretty=format:%cI -1 "$rev" -- "$f") "$f";
     #     done
+
+    # work around for https://github.com/haskell/actions/issues/187
+    - name: Set permissions for .ghcup
+      if: startsWith(matrix.os, 'ubuntu-')
+      run: sudo chown -R $USER /usr/local/.ghcup
     - name: Install GHC and Cabal
-      uses: haskell/actions/setup@v2.0.1
+      uses: haskell/actions/setup@v2
       with:
          ghc-version: ${{ matrix.ghc }}
          cabal-version: ${{ matrix.cabal }}
@@ -227,7 +232,7 @@ jobs:
         cabal --version
     - name: Install non-Haskell dependencies (ubuntu)
       if: contains(matrix.os, 'ubuntu')
-      run: sudo apt-get install -y libgflags-dev liblz4-dev libzstd-dev libtbb-dev libsnappy-dev libbz2-dev
+      run: sudo apt-get install -y libgflags-dev liblz4-dev libzstd-dev libsnappy-dev libbz2-dev
     # Project Configuration
     - name: Create cabal.project.local
       run: |
@@ -238,10 +243,9 @@ jobs:
           documentation: False
           benchmarks: True
           tests: True
-          ghc-options: -Wno-unused-packages
+          ghc-options: -Wunused-packages
         package pact
           documentation: False
-          ghc-options: -Wno-unused-packages
         EOF
 
     - name: Configure Optimization Level
@@ -333,9 +337,9 @@ jobs:
     - name: Install build dependencies
       run: cabal build chainweb --only-dependencies
     - name: Build chainweb library
-      run: cabal build lib:chainweb
+      run: cabal build --ghc-options=-j2 lib:chainweb
     - name: Build chainweb applications
-      run: cabal build exe:chainweb-node test:chainweb-tests exe:cwtool chainweb:bench:bench
+      run: cabal build -j2 --ghc-options=-j2 exe:chainweb-node test:chainweb-tests exe:cwtool chainweb:bench:bench
 
     # Checks
     - name: Check that working directory tree is clean
@@ -651,7 +655,7 @@ jobs:
         LABEL com.chainweb.docker.image.debug="${{ needs.config.outputs.debug }}"
         LABEL com.chainweb.docker.image.eventlog="${{ needs.config.outputs.eventlog }}"
         LABEL com.chainweb.docker.image.revision="${{ needs.config.outputs.git-sha-short }}"
-        RUN apt-get update && apt-get install -y ca-certificates libgmp10 libssl1.1 libsnappy1v5 libtbb2 zlib1g liblz4-1 libbz2-1.0 libgflags2.2 zstd locales && rm -rf /var/lib/apt/lists/* && locale-gen en_US.UTF-8 && update-locale LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8
+        RUN apt-get update && apt-get install -y ca-certificates libgmp10 libssl1.1 libsnappy1v5 zlib1g liblz4-1 libbz2-1.0 libgflags2.2 zstd locales && rm -rf /var/lib/apt/lists/* && locale-gen en_US.UTF-8 && update-locale LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8
         ENV LANG=en_US.UTF-8
         WORKDIR /chainweb
         COPY chainweb/chainweb-node .

diff --git a/.github/workflows/macos.yaml b/.github/workflows/macos.yaml
@@ -13,16 +13,21 @@ jobs:
       fail-fast: false
       matrix:
         ghc: ["8.10.7", "9.0"]
-        cabal: ["3.6"]
+        cabal: ["3.10"]
         os: ["macOS-latest"]
         cabalcache: ["true"]
 
     steps:
     # Setup
     - name: Checkout repository
       uses: actions/checkout@v3
+
+    # work around for https://github.com/haskell/actions/issues/187
+    - name: Set permissions for .ghcup
+      if: startsWith(matrix.os, 'ubuntu-')
+      run: sudo chown -R $USER /usr/local/.ghcup
     - name: Install GHC and Cabal
-      uses: haskell/actions/setup@v2.0.1
+      uses: haskell/actions/setup@v2
       with:
          ghc-version: ${{ matrix.ghc }}
          cabal-version: ${{ matrix.cabal }}
@@ -33,7 +38,7 @@ jobs:
     - name: Install non-Haskell dependencies (macOS)
       if: contains(matrix.os, 'mac')
       run: |
-        brew update && brew install gflags llvm tbb gnu-tar snappy zstd lz4 || true
+        brew update && brew install gflags llvm gnu-tar snappy zstd lz4 || true
         # work around bug in macOS and github actions cache (cf. https://github.com/actions/cache/issues/403)
         echo PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH" >> $GITHUB_ENV
     - name: Create cabal.project.local

diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml
@@ -0,0 +1,31 @@
+name: Build and cache with Nix
+
+on:
+  workflow_dispatch:
+  push:
+
+jobs:
+  build-and-cache:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    - name: Set up Nix with caching
+      uses: kadena-io/setup-nix-with-cache@v1
+      with:
+        cache_url: s3://nixcache.chainweb.com?region=us-east-1
+        signing_private_key: ${{ secrets.NIX_CACHE_PRIVATE_KEY }}
+
+    - name: Set up AWS credentials
+      uses: aws-actions/configure-aws-credentials@v2
+      with:
+        aws-access-key-id: ${{ secrets.NIX_CACHE_AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.NIX_CACHE_AWS_SECRET_ACCESS_KEY }}
+        aws-region: us-east-1
+
+    - name: Build and cache artifacts
+      timeout-minutes: 740
+      run: |
+        echo Building the project and its devShell
+        nix build .#check
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,136 @@
 # `chainweb-node` Changelog
 
+## 2.19.1 (2023-05-22)
+
+**NOTE: THIS VERSION SUPERSEDES 2.19. PLEASE UPDATE AS SOON AS POSSIBLE.**
+
+This version replaces all previous versions. Node administrators must upgrade to
+this version before 2023-06-01T00:00:00Z.
+
+This version will expire on 2023-09-07.
+
+To upgrade, pull the latest docker image or download the binary and restart the node.
+
+Changes:
+
+*   Disable user function return value typechecking (#1661)
+*   Add typechecking option to tx-sim. (#1656)
+
+## 2.19 (2023-05-25)
+
+**NOTE: THIS VERSION IS OBSOLETE. IT IS REPLACED BY 2.19.1. PLEASE UPDATE AS SOON
+AS POSSIBLE.**
+
+This version replaces all previous versions. Any prior version will stop working
+on 2023-06-01T00:00:00Z. Node administrators must upgrade to this version before
+that date.
+
+This version will expire on 2023-09-07.
+
+To upgrade, pull the latest docker image or download the binary and restart the node.
+
+Changes:
+
+*   Support for Pact 4.7 (#1649, #1645, #1633, #1639):
+    *   Pact errors are now displayed to users of the Pact /poll endpoint. Some
+        Pact errors have changed and been made shorter.
+
+Bug fixes:
+
+*   API endpoints now more strictly comply to the API specification at
+    api.chainweb.com. (#1434)
+*   A small memory leak has been fixed. (#1635)
+
+## 2.18.1 (2023-03-06)
+
+This is a feature and bug-fix release. Upgrading is optional but recommended.
+
+To upgrade, pull the latest docker image or download the binary and restart the
+node.
+
+All 2.18.* versions will expire on **2023-06-01T00:00:00Z**.
+
+[Changes](https://github.com/kadena-io/chainweb-node/compare/2.18...2.18.1):
+
+Performance Improvements:
+
+*   Optimize JSON+base64 encoding. (#1611)
+*   Use `application/octet-stream` encoding for P2P header queries. (#1619)
+
+Miscellaneous:
+
+*   Remove unused rate limiting configuration settings. (#1616)
+*   Remove CORS support from the P2P API. (#1616)
+*   Remove unused hashes and SPV endpoints from the P2P API. (#1616)
+*   Tighten default P2P rate limits. (#1616)
+*   Add dedicated rate limiter for Mempool requests. (#1616)
+*   Disable unused `application/json;blockheader-encoding=object` in responses
+    from the P2P API. (#1619)
+
+## 2.18 (2023-03-01)
+
+This version replaces all previous versions. Any prior version will stop working
+on **2023-03-02T00:00:00Z**. Node administrators must upgrade to this version
+before that date.
+
+This version will expire on **2023-06-01T00:00:00Z**.
+
+To upgrade, pull the latest docker image or download the binary and restart the
+node.
+
+[Changes](https://github.com/kadena-io/chainweb-node/compare/2.17.2...2.18):
+
+*   New /local endpoint preflight simulation API. (#1585, #1600)
+*   Support for Pact 4.6: (#1602)
+    *    New ZK native function support.
+    *    Better gas estimation.
+*   Warning deprecation system support in /local:
+    *    Allows the node to provide warnings for upcoming feature deprecations in Pact.
+    *    The same warnings are generated by the repl in Pact 4.6.
+*   Internal changes to support future chain database schema changes.
+*   Remove libtbb as a dependency.
+
+Bug fixes:
+
+*   Filter Module Cache for just `coin` contract. (#1548)
+*   Prevent table name clashes in module. (#1556)
+*   Full chain replay is now possible on Intel Mac & Linux, and M1 Mac.
+When upgrading directly from chainweb-node version 2.17, please, also take a look at the changes in versions 2.17.1 and 2.17.2 below.
+
+## 2.17.2 (2022-12-22)
+
+This is a feature and bug-fix release. Upgrading is optional but recommended.
+
+To upgrade, pull the latest docker image or download the binary and restart the
+node.
+
+All 2.17* versions expire on **2023-03-02T00:00:00Z**.
+
+[Changes](https://github.com/kadena-io/chainweb-node/compare/2.17.1...2.17.2):
+
+Logging and Telemetry Changes:
+
+*   Add telemetry logging for Database size. (#1330)
+*   Make Pact service log asynchronous exceptions with log-level `warn` and not
+    `error`. (#1562)
+*   Log replay height based on time, not blocks. (#1563)
+*   Add telemetry logging for node top-level status. (#1561)
+
+Performance Improvements:
+
+*   Add module cache to checkpointer. (#1577)
+
+Bug Fixes:
+
+*   `withSavepoint` now catches `SomeAsyncException`. (#1576)
+*   Fix transfer cost for Rosetta transaction generator. (#1579)
+
+Miscellaneous:
+
+*   Censor `BackupConfig` from `config` endpoint. (#1569)
+*   Simulate whole block(s) in transaction simulator. (#1573)
+*   Fix some command line help messages. (#1574)
+
 ## 2.17.1 (2022-12-02)
 
 This is a feature and bug-fix release. Upgrading is optional but recommended.
@@ -21,6 +152,7 @@ Bug fixes:
 Miscellaneous:
 
 *   Add transaction simulator to cwtools. (#1558)
+
 ## 2.17 (2022-11-17)
 
 This version replaces all previous versions. Any prior version will stop working

diff --git a/README.md b/README.md
@@ -85,12 +85,12 @@ The following packages must be installed the on the host system:
 
 *   ubuntu-20.04:
     ```bash
-    apt-get install ca-certificates libgmp10 libssl1.1 libsnappy1v5 libtbb2 zlib1g liblz4-1 libbz2-1.0 libgflags2.2 zstd
+    apt-get install ca-certificates libgmp10 libssl1.1 libsnappy1v5 zlib1g liblz4-1 libbz2-1.0 libgflags2.2 zstd
     ```
 
 *   ubuntu-22.04:
     ```bash
-    apt-get install ca-certificates libgmp10 libssl1.1 libsnappy1v5 libtbb12 zlib1g liblz4-1 libbz2-1.0 libgflags2.2 zstd
+    apt-get install ca-certificates libgmp10 libssl1.1 libsnappy1v5 zlib1g liblz4-1 libbz2-1.0 libgflags2.2 zstd
     ```
 
 Chainweb-node binaries for ubuntu-20.04 and ubuntu-22.04 can be found
@@ -120,17 +120,17 @@ built as follows.
 ### Building with Cabal
 
 In order to build with `cabal` you have to install `ghc-8.10.7` (Haskell compiler)
-and `cabal >= 3.0` (Haskell build-tool)
+and `cabal >= 3.4` (Haskell build-tool)
 
 *   [Linux / Mac](https://www.haskell.org/ghcup/)
 
 You need to install the development versions of the following libraries:
-`gflags`, `snappy`, `tbb`, `zlib`, `lz4`, `bz2`, `zstd`.
+`gflags`, `snappy`, `zlib`, `lz4`, `bz2`, `zstd`.
 
 On apt based distribution these can be installed as follows:
 
 ```
-apt-get install ca-certificates libssl-dev libgmp-dev libsnappy-dev libtbb-dev zlib1g-dev liblz4-dev libbz2-dev libgflags-dev libzstd-dev
+apt-get install ca-certificates libssl-dev libgmp-dev libsnappy-dev zlib1g-dev liblz4-dev libbz2-dev libgflags-dev libzstd-dev
 ```
 
 To build a `chainweb-node` binary:

diff --git a/bench/Bench.hs b/bench/Bench.hs
@@ -1,22 +1,21 @@
 {-# LANGUAGE OverloadedStrings #-}
-module Main (main)
-where
-
 -- |
 -- Module: JSONEncoding
 -- Copyright: Copyright © 2021 Kadena LLC.
 -- License: MIT
 -- Maintainer: Lars Kuhtz <[email protected]>
 -- Stability: experimental
---
-import Criterion.Main
+module Main (main)
+where
 
-import Data.CAS.RocksDB
+import Criterion.Main
 
 import qualified Chainweb.Pact.Backend.Bench as Checkpointer
 import qualified Chainweb.Pact.Backend.ForkingBench as ForkingBench
 import qualified JSONEncoding
 
+import Chainweb.Storage.Table.RocksDB
+
 main :: IO ()
 main = withTempRocksDb "benchmarks" $ \rdb -> do
   defaultMain