Skip to content

Commit

Permalink
speed up ci (#325)
Browse files Browse the repository at this point in the history
* remove unnecessary npm install

* add npm caching

* cache playwright

* use newer env var api

* fix naming

* add native dwn server

* run dwn server natively

* cleanup bg processing

* dial in the browser build

* better naming

* add parallelism

* fix parallelism

* better naming

* weird bug

* cleaner parallel config

* fix

* fix

* harden agent test

* solidify parallel groups

* do another run

* add comment

* balance parallel groups

* browser groups

* Revert "browser groups"

This reverts commit 5371e54.

* merge

* pr review: pin commit hashes

* pr review: pin commit hashes

* rebase
  • Loading branch information
shamilovtim authored Dec 12, 2023
1 parent 78c2de2 commit ba9527f
Show file tree
Hide file tree
Showing 10 changed files with 1,554 additions and 182 deletions.
8 changes: 3 additions & 5 deletions .github/workflows/alpha-npm.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,14 @@ jobs:

steps:
- name: Checkout source
uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 #v4.1.1

- name: Set up Node.js
uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0
uses: actions/setup-node@5ef044f9d09786428e6e895be6be17937becee3a #v4.0.0
with:
node-version: 18
registry-url: https://registry.npmjs.org/

- name: Install latest npm
run: npm install -g npm@latest
cache: 'npm'

# Note - this is not required but it gives a clean failure prior to attempting a release if
# the GH workflow runner is not authenticated with NPMjs.com
Expand Down
10 changes: 4 additions & 6 deletions .github/workflows/docs-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,14 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout source
uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 #v4.1.1

- name: Set up Node.js
uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0
uses: actions/setup-node@5ef044f9d09786428e6e895be6be17937becee3a #v4.0.0
with:
node-version: 18
registry-url: https://registry.npmjs.org/

- name: Install latest npm
run: npm install -g npm@latest
cache: 'npm'

- name: Install dependencies
run: npm ci
Expand All @@ -46,7 +44,7 @@ jobs:
docsGenerator: typedoc-markdown
- name: Save Artifacts
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 #v3.1.3
if: always()
with:
name: tbdocs-reporter-output
Expand Down
9 changes: 5 additions & 4 deletions .github/workflows/docs-publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,14 @@ jobs:

steps:
- name: Checkout source
uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 #v4.1.1

- name: Set up Node.js
uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0
uses: actions/setup-node@5ef044f9d09786428e6e895be6be17937becee3a #v4.0.0
with:
node-version: 18
registry-url: https://registry.npmjs.org/
cache: 'npm'

- name: Install dependencies
run: |
Expand All @@ -47,7 +48,7 @@ jobs:
curl -o docs/favicon.ico https://developer.tbd.website/img/favicon.ico
- name: Upload documentation artifacts
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 #v3.1.3
with:
name: jsdoc
path: ./docs
Expand All @@ -71,7 +72,7 @@ jobs:

steps:
- name: Checkout
uses: actions/checkout@v3
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 #v4.1.1

- name: Setup Pages
uses: actions/configure-pages@v3
Expand Down
8 changes: 3 additions & 5 deletions .github/workflows/release-npm.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,16 +37,14 @@ jobs:

steps:
- name: Checkout source
uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 #v4.1.1

- name: Set up Node.js
uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0
uses: actions/setup-node@5ef044f9d09786428e6e895be6be17937becee3a #v4.0.0
with:
node-version: 18
registry-url: https://registry.npmjs.org/

- name: Install latest npm
run: npm install -g npm@latest
cache: 'npm'

- name: Install semver utility
run: npm install -g [email protected]
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/scorecard.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ jobs:

steps:
- name: "Checkout code"
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 #v4.1.1
with:
persist-credentials: false

Expand All @@ -59,7 +59,7 @@ jobs:
# Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
# format to the repository Actions tab.
- name: "Upload artifact"
uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0
uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 #v3.1.3
with:
name: SARIF file
path: results.sarif
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/security.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ jobs:

steps:
- name: Checkout
uses: actions/checkout@v4
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 #v4.1.1
with:
ref: ${{ github.head_ref }}

Expand All @@ -45,7 +45,7 @@ jobs:

steps:
- name: Checkout
uses: actions/checkout@v4
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 #v4.1.1
with:
ref: ${{ github.head_ref }}

Expand Down
92 changes: 61 additions & 31 deletions .github/workflows/tests-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,13 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout source
uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 #v4.1.1

- name: Set up Node.js
uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0
uses: actions/setup-node@5ef044f9d09786428e6e895be6be17937becee3a #v4.0.0
with:
node-version: 18
cache: "npm"

- name: Report known vulnerabilities
run: npm audit
Expand All @@ -30,16 +31,14 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout source
uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 #v4.1.1

- name: Set up Node.js
uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0
uses: actions/setup-node@5ef044f9d09786428e6e895be6be17937becee3a #v4.0.0
with:
node-version: 18
registry-url: https://registry.npmjs.org/

- name: Install latest npm
run: npm install -g npm@latest
cache: "npm"

- name: Install dependencies
run: npm ci
Expand All @@ -50,65 +49,96 @@ jobs:
- name: Run linter for all packages
run: npm run lint --ws

- name: Start dwn-server container
run: cd packages/dev-env && docker-compose up -d

- name: Wait for dwn-server to be ready
run: until curl -sf http://localhost:3000/health; do echo -n .; sleep .1; done
- name: Run dwn-server (background)
run: |
node node_modules/@web5/dwn-server/dist/esm/src/main.js &
echo "DWN_SERVER_BACKGROUND_PROCESS=$!" >> $GITHUB_ENV
- name: Run tests for all packages
run: npm run test:node --ws -- --color
env:
TEST_DWN_URL: http://localhost:3000

- name: Terminate dwn-server
run: kill $DWN_SERVER_BACKGROUND_PROCESS || true

- name: Upload test coverage to Codecov
uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # v3.1.4
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

test-with-browsers:
name: test-with-browsers (group ${{ matrix.group }})
# Run browser tests using macOS so that WebKit tests don't fail under a Linux environment
runs-on: macos-latest
strategy:
# parallelism strategy: agent takes as long as roughly all other pkgs combined.
matrix:
include:
- group: "A"
packages: "--workspace packages/agent --workspace packages/dids "
- group: "B"
packages: "--workspace packages/common --workspace packages/crypto --workspace packages/credentials --workspace packages/user-agent --workspace packages/proxy-agent --workspace packages/api --workspace packages/identity-agent"
steps:
- name: Checkout source
uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 #v4.1.1

- name: Set up Node.js
uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0
uses: actions/setup-node@5ef044f9d09786428e6e895be6be17937becee3a #v4.0.0
with:
node-version: 18
registry-url: https://registry.npmjs.org/

- name: Install latest npm
run: npm install -g npm@latest
cache: "npm"

- name: Install dependencies
run: npm ci

- name: Install Playwright Browsers
- name: Get Playwright Version (for cache)
id: get-playwright-version
run: |
PLAYWRIGHT_VERSION=$(npm view @playwright/test version)
echo "Playwright Version: $PLAYWRIGHT_VERSION"
echo "PLAYWRIGHT_VERSION=$PLAYWRIGHT_VERSION" >> $GITHUB_ENV
- name: Restore Cached Playwright Browsers
id: cache-playwright-restore
uses: actions/cache/restore@84995e0d91a927aa8da027221d329f84446b8c9b #v3.3.2
with:
path: ~/Library/Caches/ms-playwright
key: ${{ runner.os }}-playwright-${{ env.PLAYWRIGHT_VERSION }}

- name: Install Playwright Browsers (if no cache)
if: steps.cache-playwright-restore.outputs.cache-hit != 'true'
run: npx playwright install --with-deps

- name: Build all workspace packages
run: npm run build
- name: Save Playwright Cache (if no cache)
uses: actions/cache/save@e08330827dd5663b268018076572bea59a734b60 #v3.3.2
id: cache-playwright-save
if: steps.cache-playwright-restore.outputs.cache-hit != 'true'
with:
path: ~/Library/Caches/ms-playwright
key: ${{ runner.os }}-playwright-${{ env.PLAYWRIGHT_VERSION }}

- name: Install docker
run: brew install docker && brew install docker-compose # && colima start
- name: Build esm
run: npm run build:esm --ws

- name: Start docker
run: colima start
- name: Build browser for matrix ${{ matrix.group }}
run: npm run build:browser ${{ matrix.packages }}

- name: Start dwn-server container
run: cd packages/dev-env && docker-compose up -d
- name: Run dwn-server (background)
run: |
node node_modules/@web5/dwn-server/dist/esm/src/main.js &
echo "DWN_SERVER_BACKGROUND_PROCESS=$!" >> $GITHUB_ENV
- name: Wait for dwn-server to be ready
run: until curl -sf http://localhost:3000/health; do echo -n .; sleep .1; done
- name: Run tests for matrix ${{ matrix.group }}
run: npm run test:browser ${{ matrix.packages }}

- name: Run tests for all packages
run: npm run test:browser --ws
- name: Terminate dwn-server
run: kill $DWN_SERVER_BACKGROUND_PROCESS || true

web5-spec:
runs-on: ubuntu-latest
steps:
- name: Checkout source
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 #v4.1.1
- uses: TBD54566975/[email protected]
Loading

0 comments on commit ba9527f

Please sign in to comment.