Skip to content

Commit

Permalink
merge
Browse files Browse the repository at this point in the history
  • Loading branch information
bitsofsteve committed Mar 8, 2024
2 parents 89c5a30 + ce4ac43 commit c51e144
Show file tree
Hide file tree
Showing 247 changed files with 8,820 additions and 5,206 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/pr-tests-frontend.yml
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ jobs:
- name: Docker on MacOS
if: steps.changes.outputs.frontend == 'true' && matrix.os == 'macos-latest'
uses: crazy-max/ghaction-setup-docker@v3.0.0
uses: crazy-max/ghaction-setup-docker@v3.1.0

- name: Install Tox
if: steps.changes.outputs.frontend == 'true'
Expand Down Expand Up @@ -156,7 +156,7 @@ jobs:
- name: Docker on MacOS
if: steps.changes.outputs.stack == 'true' && matrix.os == 'macos-latest'
uses: crazy-max/ghaction-setup-docker@v3.0.0
uses: crazy-max/ghaction-setup-docker@v3.1.0

- name: Install Tox
if: steps.changes.outputs.stack == 'true'
Expand Down
50 changes: 50 additions & 0 deletions .github/workflows/pr-tests-helm-lint.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
name: PR Tests - Lint Helm Charts

on:
# temporary disabled
# pull_request:
# branches:
# - dev
# paths:
# - packages/grid/helm/syft/**

workflow_dispatch:
inputs:
none:
description: "Run Tests Manually"
required: false

concurrency:
group: pr-tests-helm-lint
cancel-in-progress: true

jobs:
pr-tests-helm-lint:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v4

- name: Install dependencies
run: |
eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
brew update
brew tap FairwindsOps/tap
brew install kube-linter FairwindsOps/tap/polaris
# Install python deps
pip install --upgrade pip
pip install tox
kube-linter version
polaris version
- name: Run Polaris
run: |
eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
polaris audit --helm-chart packages/grid/helm/syft --helm-values packages/grid/helm/syft/values.yaml --format=pretty --only-show-failed-tests
- name: Run Linter
run: |
eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
tox -e syft.lint.helm
66 changes: 66 additions & 0 deletions .github/workflows/pr-tests-helm-upgrade.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
name: PR Tests - Helm Upgrade

on:
# Re-enable when we have a stable helm chart
# pull_request:
# branches:
# - dev
# paths:
# - packages/grid/helm/syft/**

workflow_dispatch:
inputs:
upgrade_type:
description: "Select upgrade path type"
required: false
default: "BetaToDev"
type: choice
options:
- BetaToDev
- ProdToBeta
- ProdToDev

concurrency:
group: pr-tests-helm-upgrade
cancel-in-progress: true

jobs:
pr-tests-helm-upgrade:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v4

- name: Install dependencies
run: |
eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
brew update
# Install python deps
pip install --upgrade pip
pip install tox
# Install kubernetes
brew install helm k3d devspace kubectl
- name: Setup cluster
run: |
eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
tox -e dev.k8s.start
- name: Upgrade helm chart
run: |
eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
# default upgrade is beta to dev, but override with input if provided
UPGRADE_TYPE_INPUT=${{ github.event.inputs.upgrade_type }}
export UPGRADE_TYPE=${UPGRADE_TYPE_INPUT:-BetaToDev}
tox -e syft.test.helm.upgrade
- name: Destroy cluster
if: always()
run: |
eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
tox -e dev.k8s.destroyall
2 changes: 1 addition & 1 deletion .github/workflows/pr-tests-stack-arm64.yml
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ jobs:
docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
- name: Run integration tests
uses: nick-fields/retry@v2
uses: nick-fields/retry@v3
with:
timeout_seconds: 36000
max_attempts: 3
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/pr-tests-stack-public.yml
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ jobs:
- name: Docker on MacOS
if: steps.changes.outputs.stack == 'true' && matrix.os == 'macos-latest'
uses: crazy-max/ghaction-setup-docker@v3.0.0
uses: crazy-max/ghaction-setup-docker@v3.1.0

- name: Docker Compose on MacOS
if: steps.changes.outputs.stack == 'true' && matrix.os == 'macos-latest'
Expand Down Expand Up @@ -148,7 +148,7 @@ jobs:
# tox -e stack.test.integration

- name: Run integration tests
uses: nick-fields/retry@v2
uses: nick-fields/retry@v3
if: steps.changes.outputs.stack == 'true'
env:
HAGRID_ART: false
Expand Down
13 changes: 7 additions & 6 deletions .github/workflows/pr-tests-stack.yml
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ jobs:
- name: Docker on MacOS
if: steps.changes.outputs.stack == 'true' && matrix.os == 'macos-latest'
uses: crazy-max/ghaction-setup-docker@v3.0.0
uses: crazy-max/ghaction-setup-docker@v3.1.0

- name: Docker Compose on MacOS
if: steps.changes.outputs.stack == 'true' && matrix.os == 'macos-latest'
Expand All @@ -168,6 +168,7 @@ jobs:
HAGRID_ART: false
PYTEST_MODULES: "${{ matrix.pytest-modules }}"
run: |
export AZURE_BLOB_STORAGE_KEY="${{ secrets.AZURE_BLOB_STORAGE_KEY }}"
tox -e stack.test.integration
#Run log collector python script
Expand Down Expand Up @@ -413,7 +414,7 @@ jobs:
- name: Docker on MacOS
if: steps.changes.outputs.stack == 'true' && matrix.os == 'macos-latest'
uses: crazy-max/ghaction-setup-docker@v3.0.0
uses: crazy-max/ghaction-setup-docker@v3.1.0

- name: Docker Compose on MacOS
if: steps.changes.outputs.stack == 'true' && matrix.os == 'macos-latest'
Expand Down Expand Up @@ -645,10 +646,10 @@ jobs:
shell: bash
run: |
mkdir -p ./k8s-logs
kubectl describe all -A --context k3d-testgateway1 --namespace testgateway1 > ./k8s-logs/testgateway1-desc-${{ steps.date.outputs.date }}.txt
kubectl describe all -A --context k3d-testdomain1 --namespace testdomain1 > ./k8s-logs/testdomain1-desc-${{ steps.date.outputs.date }}.txt
kubectl logs -l app.kubernetes.io/name!=random --prefix=true --context k3d-testgateway1 --namespace testgateway1 > ./k8s-logs/testgateway1-logs-${{ steps.date.outputs.date }}.txt
kubectl logs -l app.kubernetes.io/name!=random --prefix=true --context k3d-testdomain1 --namespace testdomain1 > ./k8s-logs/testdomain1-logs-${{ steps.date.outputs.date }}.txt
kubectl describe all -A --context k3d-testgateway1 --namespace syft > ./k8s-logs/testgateway1-desc-${{ steps.date.outputs.date }}.txt
kubectl describe all -A --context k3d-testdomain1 --namespace syft > ./k8s-logs/testdomain1-desc-${{ steps.date.outputs.date }}.txt
kubectl logs -l app.kubernetes.io/name!=random --prefix=true --context k3d-testgateway1 --namespace syft > ./k8s-logs/testgateway1-logs-${{ steps.date.outputs.date }}.txt
kubectl logs -l app.kubernetes.io/name!=random --prefix=true --context k3d-testdomain1 --namespace syft > ./k8s-logs/testdomain1-logs-${{ steps.date.outputs.date }}.txt
ls -la ./k8s-logs
- name: Upload logs to GitHub
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/pr-tests-syft.yml
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ jobs:
- name: Docker on MacOS
if: steps.changes.outputs.syft == 'true' && matrix.os == 'macos-latest'
uses: crazy-max/ghaction-setup-docker@v3.0.0
uses: crazy-max/ghaction-setup-docker@v3.1.0

- name: Run unit tests
if: steps.changes.outputs.syft == 'true'
Expand Down Expand Up @@ -174,7 +174,7 @@ jobs:
pip install --upgrade tox packaging wheel --default-timeout=60
- name: Run notebook tests
uses: nick-fields/retry@v2
uses: nick-fields/retry@v3
if: steps.changes.outputs.syft == 'true' || steps.changes.outputs.notebooks == 'true'
env:
ORCHESTRA_DEPLOYMENT_TYPE: "${{ matrix.deployment-type }}"
Expand Down Expand Up @@ -265,7 +265,7 @@ jobs:
- name: Docker on MacOS
if: (steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true') && matrix.os == 'macos-latest'
uses: crazy-max/ghaction-setup-docker@v3.0.0
uses: crazy-max/ghaction-setup-docker@v3.1.0

- name: Docker Compose on MacOS
if: (steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true') && matrix.os == 'macos-latest'
Expand Down
4 changes: 2 additions & 2 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -172,8 +172,8 @@ repos:
- id: mypy
name: "mypy: syft"
always_run: true
files: "^packages/syft/src/syft/serde|^packages/syft/src/syft/util|^packages/syft/src/syft/service"
# files: "^packages/syft/src/syft/"
files: "^packages/syft/src/syft/"
exclude: "packages/syft/src/syft/types/dicttuple.py|^packages/syft/src/syft/service/action/action_graph.py|^packages/syft/src/syft/external/oblv/"
args: [
"--follow-imports=skip",
"--ignore-missing-imports",
Expand Down
74 changes: 74 additions & 0 deletions DEBUGGING.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
# Debugging PySyft

We currently provide information on how to debug PySyft using Visual Studio Code and PyCharm. If you have any other IDE or debugger that you would like to add to this list, please feel free to contribute.

## VSCode

If you're running Add the following in `.vscode/launch.json`

```
{
"version": "0.2.0",
"configurations": [
{
"name": "Python Debugger: Remote Attach",
"type": "debugpy",
"request": "attach",
"connect": {
"host": "localhost",
"port": 5678
},
"justMyCode": false,
"internalConsoleOptions": "openOnSessionStart",
"pathMappings": [
{
"localRoot": "${workspaceFolder}/packages/syft/src",
"remoteRoot": "/root/app/syft/src"
},
{
"localRoot": "${workspaceFolder}/packages/grid/backend/grid",
"remoteRoot": "/root/app/grid"
}
]
}
]
}
```

Then run

```bash
tox -e dev.k8s.hotreload
```

And you can attach the debugger running on port 5678.

## PyCharm

Add the following to `packages/grid/backend/grid/__init__.py`

```py
import pydevd_pycharm
pydevd_pycharm.settrace('your-local-addr', port=5678, suspend=False)
```

Ensure that `your-local-addr` is reachable from the containers.

Next, replace the debugpy install and `DEBUG_CMD` in `packages/grid/backend/grid/start.sh`:

```bash
# only set by kubernetes to avoid conflict with docker tests
if [[ ${DEBUGGER_ENABLED} == "True" ]];
then
pip install --user pydevd-pycharm==233.14475.56 # remove debugpy, add pydevd-pycharm
DEBUG_CMD="" # empty the debug command
fi
```

If it fails to connect, check the backend logs. You might need to install a different pydevd-pycharm version. The version to be installed is shown in the log error message.

Whenever you start a container, it attempts to connect to PyCharm.

```bash
tox -e dev.k8s.hotreload
```
1 change: 1 addition & 0 deletions docs/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
certifi>=2023.7.22 # not directly required, pinned by Snyk to avoid a vulnerability
ipython==8.10.0
jinja2>=3.1.3 # not directly required, pinned by Snyk to avoid a vulnerability
markupsafe==2.0.1
pydata-sphinx-theme==0.7.2
pygments>=2.15.0 # not directly required, pinned by Snyk to avoid a vulnerability
Expand Down
18 changes: 17 additions & 1 deletion notebooks/api/0.8/01-submit-code.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -318,6 +318,22 @@
" return (float(total / 1_000_000), float(noise))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Before we can run this, we need to run:"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"```\n",
"pip install opendp\n",
"```"
]
},
{
"cell_type": "markdown",
"metadata": {},
Expand Down Expand Up @@ -583,7 +599,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.2"
"version": "3.9.16"
},
"toc": {
"base_numbering": 1,
Expand Down
Loading

0 comments on commit c51e144

Please sign in to comment.