Skip to content

Commit 22f1e4a

Browse files
SS-JIASS-JIA
committed
[DO NOT LAND] Try to debug Mac CI failure
ghstack-source-id: 1b9a713 Pull Request resolved: #5844
1 parent 20a157f commit 22f1e4a

File tree

2 files changed

+228
-0
lines changed

2 files changed

+228
-0
lines changed

‎.github/workflows/macos-job.yml‎

Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
name: Set up conda environment for testing
2+
3+
description: Clean workspace and check out PyTorch
4+
5+
inputs:
6+
python-version:
7+
description: If set to any value, dont use sudo to clean the workspace
8+
required: false
9+
type: string
10+
default: "3.9"
11+
miniconda-version:
12+
description: Miniconda version to install
13+
required: false
14+
type: string
15+
default: "23.1.0-1"
16+
environment-file:
17+
description: Environment file to install dependencies from
18+
required: false
19+
type: string
20+
default: ""
21+
pip-requirements-file:
22+
description: An optional pip requirements file to be installed in the conda environment
23+
required: false
24+
type: string
25+
default: ""
26+
27+
runs:
28+
using: composite
29+
steps:
30+
# Use the same trick from https://github.com/marketplace/actions/setup-miniconda
31+
# to refresh the cache daily. This is kind of optional though
32+
- name: Get date
33+
id: get-date
34+
shell: bash
35+
run: |
36+
echo "today=$(/bin/date -u '+%Y%m%d')d" >> "${GITHUB_OUTPUT}"
37+
38+
- name: Setup miniconda cache
39+
id: miniconda-cache
40+
uses: actions/cache@v3
41+
with:
42+
path: ${{ runner.temp }}/miniconda
43+
key: miniconda-${{ runner.os }}-${{ runner.arch }}-${{ inputs.python-version }}-${{ steps.get-date.outputs.today }}
44+
45+
- name: Install miniconda (${{ inputs.miniconda-version }})
46+
if: steps.miniconda-cache.outputs.cache-hit != 'true'
47+
env:
48+
MINICONDA_VERSION: ${{ inputs.miniconda-version }}
49+
shell: bash -l {0}
50+
run: |
51+
MINICONDA_INSTALL_PATH="${RUNNER_TEMP}/miniconda"
52+
mkdir -p "${MINICONDA_INSTALL_PATH}"
53+
case ${RUNNER_OS}-${RUNNER_ARCH} in
54+
Linux-X64)
55+
MINICONDA_ARCH="Linux-x86_64"
56+
;;
57+
macOS-ARM64)
58+
MINICONDA_ARCH="MacOSX-arm64"
59+
;;
60+
macOS-X64)
61+
MINICONDA_ARCH="MacOSX-x86_64"
62+
;;
63+
*)
64+
echo "::error::Platform ${RUNNER_OS}-${RUNNER_ARCH} currently unsupported using this action"
65+
exit 1
66+
;;
67+
esac
68+
MINICONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-py39_${MINICONDA_VERSION}-${MINICONDA_ARCH}.sh"
69+
curl -fsSL "${MINICONDA_URL}" -o "${MINICONDA_INSTALL_PATH}/miniconda.sh"
70+
bash "${MINICONDA_INSTALL_PATH}/miniconda.sh" -b -u -p "${MINICONDA_INSTALL_PATH}"
71+
rm -rf "${MINICONDA_INSTALL_PATH}/miniconda.sh"
72+
73+
- name: Update GitHub path to include miniconda install
74+
shell: bash
75+
run: |
76+
set -x
77+
78+
MINICONDA_INSTALL_PATH="${RUNNER_TEMP}/miniconda"
79+
echo "${MINICONDA_INSTALL_PATH}/bin" >> $GITHUB_PATH
80+
# NB: GITHUB_PATH has a lower priority than PATH, so also set the path
81+
# here to make sure that the correct conda is used
82+
echo "PATH=${MINICONDA_INSTALL_PATH}/bin:${PATH}" >> $GITHUB_ENV
83+
84+
# When the environment-file or pip-requirements-file inputs are not set or are set to invalid paths, the hashFiles
85+
# function will return an empty string without failing the step. This works out nicely and we can have a various
86+
# combination of cache key such as:
87+
# - Both are missing or invalid: miniconda-env-macOS-ARM64-20221022d--
88+
# - Both are set: miniconda-env-macOS-ARM64-20221022d-HASH(environment-file)-HASH(pip-requirements-file)
89+
# - The first one is missing or invalid: miniconda-env-macOS-ARM64-20221022d--HASH(pip-requirements-file)
90+
# - The second one is missing or invalid: miniconda-env-macOS-ARM64-20221022d-HASH(environment-file)-
91+
#
92+
# There is no need to skip or run actions/cache with complicated logic
93+
- name: Setup miniconda env cache
94+
id: miniconda-env-cache
95+
uses: actions/cache@v3
96+
with:
97+
path: ${{ runner.temp }}/conda-python-${{ inputs.python-version }}
98+
key: miniconda-env-${{ runner.os }}-${{ runner.arch }}-${{ inputs.python-version }}-${{ steps.get-date.outputs.today }}-${{ hashFiles(inputs.environment-file) }}-${{ hashFiles(inputs.pip-requirements-file) }}
99+
100+
- name: Setup conda environment with python (v${{ inputs.python-version }})
101+
if: steps.miniconda-env-cache.outcome == 'success' && steps.miniconda-env-cache.outputs.cache-hit != 'true'
102+
shell: bash
103+
env:
104+
PYTHON_VERSION: ${{ inputs.python-version }}
105+
ENV_FILE: ${{ inputs.environment-file }}
106+
PIP_REQUIREMENTS_FILE: ${{ inputs.pip-requirements-file }}
107+
run: |
108+
set -x
109+
110+
CONDA_BASE_ENV="${RUNNER_TEMP}/conda-python-${PYTHON_VERSION}"
111+
ENV_FILE_FLAG=""
112+
if [[ -f "${ENV_FILE}" ]]; then
113+
ENV_FILE_FLAG="--file ${ENV_FILE}"
114+
elif [[ -n "${ENV_FILE}" ]]; then
115+
echo "::warning::Specified env file (${ENV_FILE}) not found, not going to include it"
116+
fi
117+
118+
CONDA_EXTRA_FLAGS=""
119+
if [[ "${PYTHON_VERSION}" == "3.11" ]]; then
120+
CONDA_EXTRA_FLAGS=" -c pytorch-nightly"
121+
fi
122+
123+
# Print the conda we are using here in case we need debugging information
124+
CONDA_RUNTIME=$(which conda)
125+
"${CONDA_RUNTIME}" --version
126+
127+
"${CONDA_RUNTIME}" create \
128+
--yes --quiet \
129+
--prefix "${CONDA_BASE_ENV}" \
130+
${ENV_FILE_FLAG} \
131+
python="${PYTHON_VERSION}" \
132+
cmake=3.22 \
133+
ninja=1.10 \
134+
pkg-config=0.29 \
135+
wheel=0.37 \
136+
${CONDA_EXTRA_FLAGS}
137+
138+
if [[ -f "${PIP_REQUIREMENTS_FILE}" ]]; then
139+
"${CONDA_RUNTIME}" run -p "${CONDA_BASE_ENV}" --no-capture-output python3 -mpip install -r "${PIP_REQUIREMENTS_FILE}"
140+
elif [[ -n "${PIP_REQUIREMENTS_FILE}" ]]; then
141+
echo "::warning::Specified pip requirements file (${PIP_REQUIREMENTS_FILE}) not found, not going to include it"
142+
fi
143+
144+
- name: Clone the base conda environment and update GitHub env
145+
shell: bash
146+
env:
147+
PYTHON_VERSION: ${{ inputs.python-version }}
148+
CONDA_BASE_ENV: ${{ runner.temp }}/conda-python-${{ inputs.python-version }}
149+
PIP_REQUIREMENTS_FILE: ${{ inputs.pip-requirements-file }}
150+
run: |
151+
set -x
152+
153+
# Print the conda we are using here in case we need debugging information
154+
CONDA_RUNTIME=$(which conda)
155+
"${CONDA_RUNTIME}" --version
156+
157+
CONDA_ENV="${RUNNER_TEMP}/conda_environment_${GITHUB_RUN_ID}"
158+
"${CONDA_RUNTIME}" create \
159+
--yes --quiet \
160+
--prefix "${CONDA_ENV}" \
161+
--clone "${CONDA_BASE_ENV}"
162+
163+
set +e
164+
# NB: Cloning sometimes doesn't copied pip dependencies (untracked files) over. If this
165+
# happens, let's attempt to install the pip requirements directly on top of the cloned
166+
# environment. This is to make sure that no dependency is missing.
167+
UNTRACKED_FILES_COUNT=$("${CONDA_RUNTIME}" package -p "${CONDA_ENV}" -u | grep -v "^#" | wc -l | xargs)
168+
set -e
169+
170+
if [[ -z "${UNTRACKED_FILES_COUNT}" ]] || [[ "${UNTRACKED_FILES_COUNT}" == "0" ]]; then
171+
if [[ -f "${PIP_REQUIREMENTS_FILE}" ]]; then
172+
# NB: Force reinstall and don't use the cache, as the installation would still fail
173+
# when reporting that all requirements have been satisfied
174+
"${CONDA_RUNTIME}" run -p "${CONDA_ENV}" --no-capture-output python3 -mpip install --ignore-installed --no-cache-dir -r "${PIP_REQUIREMENTS_FILE}"
175+
elif [[ -n "${PIP_REQUIREMENTS_FILE}" ]]; then
176+
echo "::warning::Specified pip requirements file (${PIP_REQUIREMENTS_FILE}) not found, not going to include it"
177+
fi
178+
fi
179+
180+
echo "CONDA_ENV=${CONDA_ENV}" >> "${GITHUB_ENV}"
181+
echo "CONDA_RUN=${CONDA_RUNTIME} run -p ${CONDA_ENV} --no-capture-output" >> "${GITHUB_ENV}"
182+
if [[ "${PYTHON_VERSION}" == "3.11" ]]; then
183+
# TODO: Remove me, when more packages will be available on default channel
184+
echo "CONDA_INSTALL=${CONDA_RUNTIME} install --yes --quiet -p ${CONDA_ENV} -c pytorch-nightly" >> "${GITHUB_ENV}"
185+
else
186+
echo "CONDA_INSTALL=${CONDA_RUNTIME} install --yes --quiet -p ${CONDA_ENV}" >> "${GITHUB_ENV}"
187+
fi
188+
189+
- name: Reset channel priority
190+
shell: bash
191+
run: |
192+
CONDA_RUNTIME=$(which conda)
193+
194+
set -euxo pipefail
195+
"${CONDA_RUNTIME}" config --set channel_priority false
196+
197+
- name: Setup upterm session
198+
uses: owenthereal/action-upterm@v1
199+
with:
200+
limit-access-to-actor: true

‎.github/workflows/pull.yml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -441,3 +441,31 @@ jobs:
441441
442442
# run e2e (export, tokenizer and runner)
443443
PYTHON_EXECUTABLE=python bash .ci/scripts/test_phi_3_mini.sh
444+
445+
debug:
446+
runs-on: macos-13-xlarge
447+
uses: ./.github/workflows/macos_job.yml
448+
with:
449+
runner: macos-m1-stable
450+
python-version: '3.11'
451+
submodules: 'true'
452+
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
453+
timeout: 90
454+
script: |
455+
BUILD_TOOL=cmake
456+
BACKEND=coreml
457+
458+
bash .ci/scripts/setup-conda.sh
459+
460+
# Setup MacOS dependencies as there is no Docker support on MacOS atm
461+
PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh "${BUILD_TOOL}"
462+
PYTHON_EXECUTABLE=python ${CONDA_RUN} bash backends/apple/coreml/scripts/install_requirements.sh
463+
echo "Finishing installing coreml."
464+
465+
# Build and test coreml model
466+
MODELS=(mv3 ic4 resnet50 edsr mobilebert w2l)
467+
for MODEL_NAME in "${MODELS[@]}"; do
468+
echo "::group::Exporting coreml model: $MODEL_NAME"
469+
PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/test_model.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${BACKEND}"
470+
echo "::endgroup::"
471+
done

0 commit comments

Comments
 (0)