clean up cruft (#1010)

* clean up cruft Signed-off-by: Niels Bantilan <[email protected]> * move everything to top-level, rm cookbook Signed-off-by: Niels Bantilan <[email protected]> * fix maxdepth Signed-off-by: Niels Bantilan <[email protected]> * add notebooks to docs build Signed-off-by: Niels Bantilan <[email protected]> * delete prquantifier.yaml Signed-off-by: Niels Bantilan <[email protected]> * delete stale tests, update contributing and env setup guides Signed-off-by: Niels Bantilan <[email protected]> * fix environment setup ref Signed-off-by: Niels Bantilan <[email protected]> * write sphinx to myst py script conversion Signed-off-by: Niels Bantilan <[email protected]> * convert sphinx gallery python scripts to myst markdown Signed-off-by: Niels Bantilan <[email protected]> * clean up markdown Signed-off-by: Niels Bantilan <[email protected]> * delete conversion script Signed-off-by: Niels Bantilan <[email protected]> * formatting updates Signed-off-by: Niels Bantilan <[email protected]> * fix feast notebook, formatting, update contributin Signed-off-by: Niels Bantilan <[email protected]> * lint all files Signed-off-by: Niels Bantilan <[email protected]> * clean up links, add contribution CTA Signed-off-by: Niels Bantilan <[email protected]> * fix sagemaker, papermill examples Signed-off-by: Niels Bantilan <[email protected]> * fix sagemaker example Signed-off-by: Niels Bantilan <[email protected]> * fix make setup Signed-off-by: Niels Bantilan <[email protected]> * add lint to github action Signed-off-by: Niels Bantilan <[email protected]> * clean up requirements files Signed-off-by: Niels Bantilan <[email protected]> * update dev requirement Signed-off-by: Niels Bantilan <[email protected]> * lint Signed-off-by: Niels Bantilan <[email protected]> * sandbox setup Signed-off-by: Niels Bantilan <[email protected]> * update flytectl config init Signed-off-by: Niels Bantilan <[email protected]> * debug registration action Signed-off-by: Niels Bantilan <[email protected]> * use demo sandbox config Signed-off-by: Niels Bantilan <[email protected]> * debugging Signed-off-by: Niels Bantilan <[email protected]> * add back all examples Signed-off-by: Niels Bantilan <[email protected]> --------- Signed-off-by: Niels Bantilan <[email protected]>
flyteorg · Jun 29, 2023 · 7ad3f67 · 7ad3f67
1 parent c1681cb
commit 7ad3f67
Show file tree

Hide file tree

Showing 468 changed files with 8,636 additions and 9,554 deletions.
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
@@ -5,65 +5,38 @@ on:
     branches:
       - master
     paths-ignore:
-      - 'cookbook/docs/**'
+      - 'docs/**'
   pull_request:
     paths-ignore:
-      - 'cookbook/docs/**'
+      - 'docs/**'
 jobs:
 
-  # # Lists all directories that have Dockerfile in them...
-  # list:
-  #   runs-on: ubuntu-latest
-  #   name: "Create a list of packages"
-  #   steps:
-  #     - uses: actions/checkout@v2
-  #     - name: "Provide the list"
-  #       id: create-list
-  #       run: echo "::set-output name=packages::$(find cookbook -mindepth 1 -maxdepth 5 -type f -name Dockerfile -exec dirname '{}' \; | grep -v "cookbook/examples"  | sort | jq --raw-input . | jq --slurp . | jq -c .)"
-  #   outputs:
-  #     packages: "${{ steps.create-list.outputs.packages }}"
-
-  # trigger_serialize_register_workflow:
-  #   name: Serialize & Register Flytesnacks workflow
-  #   needs: [list]
-  #   uses: ./.github/workflows/serialize_workflow.yml
-  #   with:
-  #     packages: ${{ needs.list.outputs.packages }}
-  #   secrets:
-  #     FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
-
-  # push_github:
-  #   name: Build & Push to GHCR
-  #   runs-on: ubuntu-latest
-  #   needs: [list]
-  #   strategy:
-  #     matrix:
-  #       directory: "${{ fromJson(needs.list.outputs.packages) }}"
-  #   steps:
-  #     - uses: actions/checkout@v2
-  #       with:
-  #         fetch-depth: "0"
-  #     - name: 'Set example name'
-  #       id: example_id
-  #       # add a step output `steps.build_id.outputs.id` as the unique id
-  #       run: |
-  #         echo "::set-output name=id::$(basename -- ${{ matrix.directory }})"
-  #         temp=${{ matrix.directory }}
-  #         echo "::set-output name=path::$(echo "./${temp%/*}")"
-  #     - name: Build & Push Docker Image to Github Registry
-  #       uses: whoan/docker-build-with-cache-action@v5
-  #       with:
-  #         # https://docs.github.com/en/packages/learn-github-packages/publishing-a-package
-  #         username: "${{ secrets.FLYTE_BOT_USERNAME }}"
-  #         password: "${{ secrets.FLYTE_BOT_PAT }}"
-  #         image_name: ${{ github.repository_owner }}/flytecookbook
-  #         image_tag: ${{ steps.example_id.outputs.id }}-latest,${{ steps.example_id.outputs.id }}-${{ github.sha }}
-  #         registry: ghcr.io
-  #         push_git_tag: ${{ github.event_name != 'pull_request' }}
-  #         push_image_and_stages: ${{ github.event_name != 'pull_request' }}
-  #         build_extra_args: "--compress=true --build-arg=tag=ghcr.io/${{ github.repository_owner }}/flytecookbook:${{ steps.example_id.outputs.id }}-${{ github.sha }}"
-  #         context: ${{ steps.example_id.outputs.path }}
-  #         dockerfile: ${{ steps.example_id.outputs.id }}/Dockerfile
+  lint:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Cache pip
+      uses: actions/cache@v2
+      with:
+        # This path is specific to Ubuntu
+        path: ~/.cache/pip
+        # Look to see if there is a cache hit for the corresponding requirements files
+        key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt')) }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r dev-requirements.txt
+    - name: Pip info
+      run: python -m pip list
+    - name: Lint
+      run: pre-commit run --all --show-diff-on-failure
 
   # This is the build system for the new example directory structure
   list_examples:
@@ -73,10 +46,10 @@ jobs:
       - uses: actions/checkout@v2
       - name: "Provide the list"
         id: create-example-list
-        run: echo "PACKAGES=$(find cookbook -mindepth 1 -maxdepth 5 -type f -name Dockerfile -exec dirname '{}' \; | grep "cookbook/examples"  | sort | jq --raw-input . | jq --slurp . | jq -c .)" >> "$GITHUB_OUTPUT"
+        run: echo "PACKAGES=$(find ./examples -mindepth 1 -maxdepth 2 -type f -name Dockerfile -exec dirname '{}' \; | sort | jq --raw-input . | jq --slurp . | jq -c .)" >> "$GITHUB_OUTPUT"
     outputs:
       packages: "${{ steps.create-example-list.outputs.PACKAGES }}"
-  
+
   trigger_serialize_register_examples:
     name: Serialize & Register Flytesnacks workflow
     needs: [list_examples]
@@ -115,83 +88,83 @@ jobs:
           context: ${{ matrix.directory }}
           dockerfile: Dockerfile
 
-  # bump_version:
-  #   name: Bump Version
-  #   if: ${{ github.event_name != 'pull_request' }}
-  #   needs: [ trigger_serialize_register_workflow, push_github ] # Only to ensure it can successfully build
-  #   uses: flyteorg/flytetools/.github/workflows/bump_version.yml@master
-  #   secrets:
-  #     FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
+  bump_version:
+    name: Bump Version
+    if: ${{ github.event_name != 'pull_request' }}
+    needs: [ trigger_serialize_register_examples, push_example_image_to_github ] # Only to ensure it can successfully build
+    uses: flyteorg/flytetools/.github/workflows/bump_version.yml@master
+    secrets:
+      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
 
-  # prerelease:
-  #   name: Create Prerelease
-  #   needs: [bump_version]
-  #   runs-on: ubuntu-latest
-  #   steps:
-  #     - uses: actions/checkout@v2
-  #       with:
-  #         fetch-depth: "0"
-  #     - name: Create Pre release
-  #       id: prerelease
-  #       run: |
-  #         RELEASE_ID=$(curl --location -v -X POST 'https://api.github.com/repos/flyteorg/flytesnacks/releases' \
-  #         --header 'Accept: application/vnd.github.v3+json' \
-  #         --header 'Authorization: Bearer ${{ secrets.FLYTE_BOT_PAT }}' \
-  #         --data-raw '{
-  #             "tag_name": "'${{ needs.bump-version.outputs.version }}'",
-  #             "prerelease": true
-  #         }' | jq -r '.id')
-  #         echo ::set-output name=release_id::$RELEASE_ID
-  #   outputs:
-  #     release_id: ${{ steps.prerelease.outputs.release_id }}
+  prerelease:
+    name: Create Prerelease
+    needs: [bump_version]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: "0"
+      - name: Create Pre release
+        id: prerelease
+        run: |
+          RELEASE_ID=$(curl --location -v -X POST 'https://api.github.com/repos/flyteorg/flytesnacks/releases' \
+          --header 'Accept: application/vnd.github.v3+json' \
+          --header 'Authorization: Bearer ${{ secrets.FLYTE_BOT_PAT }}' \
+          --data-raw '{
+              "tag_name": "'${{ needs.bump-version.outputs.version }}'",
+              "prerelease": true
+          }' | jq -r '.id')
+          echo ::set-output name=release_id::$RELEASE_ID
+    outputs:
+      release_id: ${{ steps.prerelease.outputs.release_id }}
 
-  # # Download artifacts again and push them to the release only if this is not a pull request
-  # release_workflow:
-  #   name: Publish artifacts to github release
-  #   runs-on: ubuntu-latest
-  #   needs: [ prerelease ]
-  #   steps:
-  #     - uses: actions/checkout@v2
-  #       with:
-  #         fetch-depth: "0"
-  #     - name: setup download artifact dir
-  #       run: |
-  #         mkdir download-artifact
-  #     - name: Download artifacts
-  #       uses: actions/download-artifact@v2
-  #       with:
-  #         path: ./download-artifact/
-  #     - name: Package Examples
-  #       run: |
-  #         mkdir -p cookbook/release-snacks
-  #         cd download-artifact
-  #         for i in */; do tar -czvf "../cookbook/release-snacks/${i%/}.tar.gz" "$i" & done; wait
-  #         cd .. && sudo rm -rf download-artifact/
-  #         cp cookbook/flyte_tests_manifest.json cookbook/release-snacks/flyte_tests_manifest.json
-  #     - name: Release test manifest
-  #       uses: goreleaser/goreleaser-action@v2
-  #       with:
-  #         version: latest
-  #         args: release --rm-dist
-  #       env:
-  #         GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }}
-  #         GORELEASER_CURRENT_TAG: ${{ needs.bump-version.outputs.version }}
+  # Download artifacts again and push them to the release only if this is not a pull request
+  release_workflow:
+    name: Publish artifacts to github release
+    runs-on: ubuntu-latest
+    needs: [ prerelease ]
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: "0"
+      - name: setup download artifact dir
+        run: |
+          mkdir download-artifact
+      - name: Download artifacts
+        uses: actions/download-artifact@v2
+        with:
+          path: ./download-artifact/
+      - name: Package Examples
+        run: |
+          mkdir -p release-snacks
+          cd download-artifact
+          for i in */; do tar -czvf "../release-snacks/${i%/}.tar.gz" "$i" & done; wait
+          cd .. && sudo rm -rf download-artifact/
+          cp flyte_tests_manifest.json release-snacks/flyte_tests_manifest.json
+      - name: Release test manifest
+        uses: goreleaser/goreleaser-action@v2
+        with:
+          version: latest
+          args: release --rm-dist
+        env:
+          GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }}
+          GORELEASER_CURRENT_TAG: ${{ needs.bump-version.outputs.version }}
 
-  # make_release:
-  #   name: Mark github pre-release as Release
-  #   needs: [ release_workflow ]
-  #   runs-on: ubuntu-latest
-  #   steps:
-  #     - uses: actions/checkout@v2
-  #       with:
-  #         fetch-depth: "0"
-  #     - name: Update Release
-  #       id: release
-  #       run: |
-  #         curl --location -X -v POST 'https://api.github.com/repos/flyteorg/flytesnacks/releases/${{ needs.prerelease.outputs.release_id }}' \
-  #         --header 'Accept: application/vnd.github.v3+json' \
-  #         --header 'Authorization: Bearer ${{ secrets.FLYTE_BOT_PAT }}' \
-  #         --data-raw '{
-  #             "tag_name": "'${{ needs.bump-version.outputs.version }}'",
-  #             "prerelease": false
-  #         }'
+  make_release:
+    name: Mark github pre-release as Release
+    needs: [ release_workflow ]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: "0"
+      - name: Update Release
+        id: release
+        run: |
+          curl --location -X -v POST 'https://api.github.com/repos/flyteorg/flytesnacks/releases/${{ needs.prerelease.outputs.release_id }}' \
+          --header 'Accept: application/vnd.github.v3+json' \
+          --header 'Authorization: Bearer ${{ secrets.FLYTE_BOT_PAT }}' \
+          --data-raw '{
+              "tag_name": "'${{ needs.bump-version.outputs.version }}'",
+              "prerelease": false
+          }'
diff --git a/.github/workflows/docs_build.yml b/.github/workflows/docs_build.yml
@@ -20,7 +20,6 @@ jobs:
         # NOTE: myst-nb build warnings are logged in docs/_build/reports/*.err.log
         run: |
           sudo apt-get install python3-sphinx
-          cd cookbook
           pip install -r docs-requirements.txt
           SPHINXOPTS="-W" cd docs && make html
           cat _build/reports/**/*.err.log || echo 'no error reports'
diff --git a/.github/workflows/serialize_example.yml b/.github/workflows/serialize_example.yml
@@ -34,21 +34,7 @@ jobs:
         id: example_id
         run: echo "EXAMPLE_ID=$(echo ${{ matrix.directory }} | sed 's@/@-@g' | sed 's@_@-@g' )" >> "$GITHUB_OUTPUT"
       - name: Build docker image and serialize examples
-        # NOTE: the additional images are for the multi-image containerization examples
-        run: |
-          cd ${{ matrix.directory }}
-          package_name=$(basename -- ${{ matrix.directory }})
-          example_name=$(basename -- ${{ matrix.directory }} | sed 's@_@-@g' )
-          image_uri=ghcr.io/flyteorg/flyte-examples:$example_name-${{ github.sha }}
-          docker build . -t $image_uri
-          docker run -i --rm -v $(pwd):/root $image_uri \
-            pyflyte --pkgs $package_name \
-            package \
-            --image $image_uri \
-            --image mindmeld="ghcr.io/flyteorg/flytecookbook:core-latest" \
-            --image borebuster="ghcr.io/flyteorg/flytekit:py3.9-latest" \
-            --force \
-            --output ./flyte-package.tgz
+        run: ./scripts/serialize-example.sh ${{ matrix.directory }} ${{ github.sha }}
       - name: Upload artifacts
         uses: actions/upload-artifact@v2
         with:
@@ -75,7 +61,8 @@ jobs:
           path: ./download-artifact/
       - name: setup sandbox
         run: |
-          make setup
+          flytectl sandbox start
+          flytectl config init
       - name: Register examples
         run: |
           for file in ./download-artifact/**/*; do

diff --git a/.github/workflows/serialize_workflow.yml b/.github/workflows/serialize_workflow.yml