Skip to content

Commit d428ce7

Browse files
bpkrothbpkroth
authored
Introduce pyproject.toml build changes (#764)
This PR builds off of #762 and #763 These are in part 1. followup fixups for #746 (e.g., to allow setuptools-scm to be pulled in at build time as a build dependency only when conda an pip have mismatched version issues), and 2. Modernization improvements to allow us to make better use of other tools (e.g., `black` that only accept `pyproject.toml` files as their configuration files). To do so, we move some configs from `setup.py` to `pyproject.toml` for each module. However, to retain the ability to rewrite URLs in published README.md files on PyPi as well as consistent version dependencies across modules without the need to manually specify version numbers (e.g., using `setuptools-scm`) we mark a few dependencies as dynamic and leave our existing logic inside the `setup.py` file. Finally, we reorganize the `version.py` file to be inside the module and fix a few previous omissions for `mlos_viz`.
1 parent 6bf3862 commit d428ce7

31 files changed

+426
-236
lines changed

‎.bumpversion.cfg

+5-3
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,10 @@ current_version = 0.5.1
33
commit = True
44
tag = True
55

6-
[bumpversion:file:doc/source/conf.py]
6+
[bumpversion:file:doc/source/version.py]
77

8-
[bumpversion:file:mlos_core/_version.py]
8+
[bumpversion:file:mlos_core/mlos_core/version.py]
99

10-
[bumpversion:file:mlos_bench/_version.py]
10+
[bumpversion:file:mlos_bench/mlos_bench/version.py]
11+
12+
[bumpversion:file:mlos_viz/mlos_viz/version.py]

‎.devcontainer/devcontainer.json

+1
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@
8181
"redhat.vscode-yaml",
8282
"stkb.rewrap",
8383
"streetsidesoftware.code-spell-checker",
84+
"tamasfe.even-better-toml",
8485
"trond-snekvik.simple-rst",
8586
"tyriar.sort-lines"
8687
]

‎.devcontainer/scripts/common/prep-deps-files.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ tmpdir=$(mktemp -d)
2727
get_python_deps() {
2828
local pkg="$1"
2929
touch /tmp/conda-tmp/$pkg.requirements.txt
30-
python3 /tmp/conda-tmp/$pkg/setup.py egg_info --egg-base "$tmpdir/"
30+
(cd /tmp/conda-tmp/$pkg && python3 setup.py egg_info --egg-base "$tmpdir/")
3131
cat "$tmpdir/$pkg.egg-info/requires.txt" \
3232
| grep -v -e '^\[' -e '^\s*$' \
3333
| grep -v '^mlos-' \

‎.devcontainer/scripts/prep-container-build

+3-2
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,10 @@ fi
3131
mkdir -p .devcontainer/tmp/
3232
cp -v conda-envs/mlos.yml .devcontainer/tmp/mlos.yml
3333
for pkg in mlos_core mlos_bench mlos_viz; do
34-
mkdir -p .devcontainer/tmp/$pkg
34+
mkdir -p .devcontainer/tmp/$pkg/$pkg
35+
cp -v $pkg/pyproject.toml .devcontainer/tmp/$pkg/pyproject.toml
3536
cp -v $pkg/setup.py .devcontainer/tmp/$pkg/setup.py
36-
cp -v $pkg/_version.py .devcontainer/tmp/$pkg/_version.py
37+
cp -v $pkg/$pkg/version.py .devcontainer/tmp/$pkg/$pkg/version.py
3738
done
3839
cp -v doc/requirements.txt .devcontainer/tmp/doc.requirements.txt
3940

‎.devcontainer/scripts/prep-container-build.ps1

+5-3
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,11 @@ New-Item -Type Directory .devcontainer/tmp
2727

2828
Copy-Item conda-envs/mlos.yml .devcontainer/tmp/mlos.yml
2929
foreach ($pkg in @('mlos_core', 'mlos_bench', 'mlos_viz')) {
30-
New-Item -Type Directory ".devcontainer/tmp/${pkg}"
31-
Copy-Item "$pkg/setup.py" ".devcontainer/tmp/${pkg}/setup.py"
32-
Copy-Item "$pkg/_version.py" ".devcontainer/tmp/${pkg}/_version.py"
30+
New-Item -Type Directory ".devcontainer/tmp/$pkg"
31+
New-Item -Type Directory ".devcontainer/tmp/$pkg/$pkg"
32+
Copy-Item "$pkg/setup.py" ".devcontainer/tmp/$pkg/setup.py"
33+
Copy-Item "$pkg/pyproject.toml" ".devcontainer/tmp/$pkg/pyproject.toml"
34+
Copy-Item "$pkg/$pkg/version.py" ".devcontainer/tmp/$pkg/$pkg/version.py"
3335
}
3436
Copy-Item doc/requirements.txt .devcontainer/tmp/doc.requirements.txt
3537

‎.github/workflows/build-dist-test.ps1

+3-3
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ Set-Location mlos_core
1818
if (Test-Path dist) {
1919
Remove-Item -Recurse -Force dist
2020
}
21-
conda run -n $env:CONDA_ENV_NAME python setup.py bdist_wheel
21+
conda run -n $env:CONDA_ENV_NAME python -m build
2222
Set-Location ..
2323
$mlos_core_whl = (Resolve-Path mlos_core/dist/mlos_core-*-py3-none-any.whl | Select-Object -ExpandProperty Path)
2424
Write-Host "mlos_core_whl: $mlos_core_whl"
@@ -32,7 +32,7 @@ Set-Location mlos_bench
3232
if (Test-Path dist) {
3333
Remove-Item -Recurse -Force dist
3434
}
35-
conda run -n $env:CONDA_ENV_NAME python setup.py bdist_wheel
35+
conda run -n $env:CONDA_ENV_NAME python -m build
3636
Set-Location ..
3737
$mlos_bench_whl = (Resolve-Path mlos_bench/dist/mlos_bench-*-py3-none-any.whl | Select-Object -ExpandProperty Path)
3838
Write-Host "mlos_bench_whl: $mlos_bench_whl"
@@ -46,7 +46,7 @@ Set-Location mlos_viz
4646
if (Test-Path dist) {
4747
Remove-Item -Recurse -Force dist
4848
}
49-
conda run -n $env:CONDA_ENV_NAME python setup.py bdist_wheel
49+
conda run -n $env:CONDA_ENV_NAME python -m build
5050
Set-Location ..
5151
$mlos_viz_whl = (Resolve-Path mlos_viz/dist/mlos_viz-*-py3-none-any.whl | Select-Object -ExpandProperty Path)
5252
Write-Host "mlos_viz_whl: $mlos_viz_whl"

‎.github/workflows/linux.yml

+6-6
Original file line numberDiff line numberDiff line change
@@ -80,21 +80,21 @@ jobs:
8080
with:
8181
#path: ${{ env.CONDA_DIR }}/envs/${{ env.CONDA_ENV_NAME }}
8282
path: ${{ env.CONDA_DIR }}/pkgs
83-
key: conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_cur_hour }}
83+
key: conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_*/pyproject.toml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_cur_hour }}
8484
restore-keys: |
85-
conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_prev_hour }}
86-
conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}
85+
conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_*/pyproject.toml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_prev_hour }}
86+
conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_*/pyproject.toml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}
8787
8888
- name: Restore cached pip packages
8989
id: restore-pip-cache
9090
if: ${{ github.event_name != 'schedule' }}
9191
uses: actions/cache@v4
9292
with:
9393
path: ${{ env.PIP_CACHE_DIR }}
94-
key: conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_cur_hour }}
94+
key: conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_*/pyproject.toml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_cur_hour }}
9595
restore-keys: |
96-
conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_prev_hour }}
97-
conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}
96+
conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_*/pyproject.toml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_prev_hour }}
97+
conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_*/pyproject.toml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}
9898
9999
- name: Log some environment variables for debugging
100100
run: |

‎.github/workflows/windows.yml

+3-3
Original file line numberDiff line numberDiff line change
@@ -68,10 +68,10 @@ jobs:
6868
uses: actions/cache@v4
6969
with:
7070
path: ${{ env.CONDA }}/envs/mlos
71-
key: conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_YML }}') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_cur_hour }}
71+
key: conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_YML }}') }}-${{ hashFiles('mlos_*/pyproject.toml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_cur_hour }}
7272
restore-keys: |
73-
conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_YML }}') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_prev_hour }}
74-
conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_YML }}') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}
73+
conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_YML }}') }}-${{ hashFiles('mlos_*/pyproject.toml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_prev_hour }}
74+
conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_YML }}') }}-${{ hashFiles('mlos_*/pyproject.toml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}
7575
7676
- name: Log some environment variables for debugging
7777
run: |

‎.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,9 @@ build/*.build-stamp
150150
.vscode/
151151
!.vscode/settings.json
152152
!.vscode/extensions.json
153+
!.vscode/launch.json
153154
!.devcontainer/build/
155+
!.devcontainer/scripts/
154156

155157
# Test configs. May contain sensitive information
156158
# like Azure credentials, DB password, etc.

‎.vscode/extensions.json

+1
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
"redhat.vscode-yaml",
2525
"stkb.rewrap",
2626
"streetsidesoftware.code-spell-checker",
27+
"tamasfe.even-better-toml",
2728
"trond-snekvik.simple-rst",
2829
"tyriar.sort-lines"
2930
]

‎Makefile

+78-37
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,10 @@ all: format check test dist dist-test doc | conda-env
3131
.PHONY: conda-env
3232
conda-env: build/conda-env.${CONDA_ENV_NAME}.build-stamp
3333

34-
MLOS_CORE_CONF_FILES := mlos_core/setup.py mlos_core/MANIFEST.in
35-
MLOS_BENCH_CONF_FILES := mlos_bench/setup.py mlos_bench/MANIFEST.in
36-
MLOS_VIZ_CONF_FILES := mlos_viz/setup.py mlos_viz/MANIFEST.in
37-
MLOS_GLOBAL_CONF_FILES := setup.cfg
34+
MLOS_CORE_CONF_FILES := mlos_core/pyproject.toml mlos_core/setup.py mlos_core/MANIFEST.in
35+
MLOS_BENCH_CONF_FILES := mlos_bench/pyproject.toml mlos_bench/setup.py mlos_bench/MANIFEST.in
36+
MLOS_VIZ_CONF_FILES := mlos_viz/pyproject.toml mlos_viz/setup.py mlos_viz/MANIFEST.in
37+
MLOS_GLOBAL_CONF_FILES := setup.cfg # pyproject.toml
3838

3939
MLOS_PKGS := mlos_core mlos_bench mlos_viz
4040
MLOS_PKG_CONF_FILES := $(MLOS_CORE_CONF_FILES) $(MLOS_BENCH_CONF_FILES) $(MLOS_VIZ_CONF_FILES) $(MLOS_GLOBAL_CONF_FILES)
@@ -279,60 +279,93 @@ build/pytest.${CONDA_ENV_NAME}.build-stamp:
279279
touch $@
280280

281281

282+
# setuptools-scm needs a longer history than Github CI workers have by default.
283+
.PHONY: unshallow
284+
unshallow: build/unshallow.build-stamp
285+
286+
build/unshallow.build-stamp:
287+
git rev-parse --is-shallow-repository | grep -x -q false || git fetch --unshallow --quiet
288+
touch $@
289+
282290
.PHONY: dist
283-
dist: bdist_wheel
291+
dist: sdist bdist_wheel
292+
293+
.PHONY: sdist
294+
sdist: conda-env unshallow
295+
sdist: mlos_core/dist/tmp/mlos_core-latest.tar.gz
296+
sdist: mlos_bench/dist/tmp/mlos_bench-latest.tar.gz
297+
sdist: mlos_viz/dist/tmp/mlos_viz-latest.tar.gz
284298

285299
.PHONY: bdist_wheel
286-
bdist_wheel: conda-env
300+
bdist_wheel: conda-env unshallow
287301
bdist_wheel: mlos_core/dist/tmp/mlos_core-latest-py3-none-any.whl
288302
bdist_wheel: mlos_bench/dist/tmp/mlos_bench-latest-py3-none-any.whl
289303
bdist_wheel: mlos_viz/dist/tmp/mlos_viz-latest-py3-none-any.whl
290304

291-
mlos_core/dist/tmp/mlos_core-latest-py3-none-any.whl: mlos_core/dist/tmp/mlos_core-latest.tar
305+
# Make the whl files depend on the .tar.gz files, mostly to prevent conflicts
306+
# with shared use of the their build/ trees.
307+
292308
mlos_core/dist/tmp/mlos_core-latest-py3-none-any.whl: MODULE_NAME := mlos_core
293309
mlos_core/dist/tmp/mlos_core-latest-py3-none-any.whl: PACKAGE_NAME := mlos_core
294-
mlos_core/dist/tmp/mlos_core-latest.tar: mlos_core/setup.py mlos_core/MANIFEST.in $(MLOS_CORE_PYTHON_FILES)
295-
mlos_core/dist/tmp/mlos_core-latest.tar: MODULE_NAME := mlos_core
296-
mlos_core/dist/tmp/mlos_core-latest.tar: PACKAGE_NAME := mlos_core
310+
mlos_core/dist/tmp/mlos_core-latest-py3-none-any.whl: mlos_core/dist/tmp/mlos_core-latest.tar.gz
311+
mlos_core/dist/tmp/mlos_core-latest.tar.gz: $(MLOS_CORE_CONF_FILES) $(MLOS_CORE_PYTHON_FILES)
312+
mlos_core/dist/tmp/mlos_core-latest.tar.gz: MODULE_NAME := mlos_core
313+
mlos_core/dist/tmp/mlos_core-latest.tar.gz: PACKAGE_NAME := mlos_core
297314

298-
mlos_bench/dist/tmp/mlos_bench-latest-py3-none-any.whl: mlos_bench/dist/tmp/mlos_bench-latest.tar
299315
mlos_bench/dist/tmp/mlos_bench-latest-py3-none-any.whl: MODULE_NAME := mlos_bench
300316
mlos_bench/dist/tmp/mlos_bench-latest-py3-none-any.whl: PACKAGE_NAME := mlos_bench
301-
mlos_bench/dist/tmp/mlos_bench-latest.tar: mlos_bench/setup.py mlos_bench/MANIFEST.in $(MLOS_BENCH_PYTHON_FILES)
302-
mlos_bench/dist/tmp/mlos_bench-latest.tar: MODULE_NAME := mlos_bench
303-
mlos_bench/dist/tmp/mlos_bench-latest.tar: PACKAGE_NAME := mlos_bench
317+
mlos_bench/dist/tmp/mlos_bench-latest-py3-none-any.whl: mlos_bench/dist/tmp/mlos_bench-latest.tar.gz
318+
mlos_bench/dist/tmp/mlos_bench-latest.tar.gz: $(MLOS_BENCH_CONF_FILES) $(MLOS_BENCH_PYTHON_FILES)
319+
mlos_bench/dist/tmp/mlos_bench-latest.tar.gz: MODULE_NAME := mlos_bench
320+
mlos_bench/dist/tmp/mlos_bench-latest.tar.gz: PACKAGE_NAME := mlos_bench
304321

305-
mlos_viz/dist/tmp/mlos_viz-latest-py3-none-any.whl: mlos_viz/dist/tmp/mlos_viz-latest.tar
306322
mlos_viz/dist/tmp/mlos_viz-latest-py3-none-any.whl: MODULE_NAME := mlos_viz
307323
mlos_viz/dist/tmp/mlos_viz-latest-py3-none-any.whl: PACKAGE_NAME := mlos_viz
308-
mlos_viz/dist/tmp/mlos_viz-latest.tar: mlos_viz/setup.py mlos_viz/MANIFEST.in $(mlos_viz_PYTHON_FILES)
309-
mlos_viz/dist/tmp/mlos_viz-latest.tar: MODULE_NAME := mlos_viz
310-
mlos_viz/dist/tmp/mlos_viz-latest.tar: PACKAGE_NAME := mlos_viz
324+
mlos_viz/dist/tmp/mlos_viz-latest-py3-none-any.whl: mlos_viz/dist/tmp/mlos_viz-latest.tar.gz
325+
mlos_viz/dist/tmp/mlos_viz-latest.tar.gz: $(MLOS_VIZ_CONF_FILES) $(MLOS_VIZ_PYTHON_FILES)
326+
mlos_viz/dist/tmp/mlos_viz-latest.tar.gz: MODULE_NAME := mlos_viz
327+
mlos_viz/dist/tmp/mlos_viz-latest.tar.gz: PACKAGE_NAME := mlos_viz
311328

312-
%-latest.tar: build/conda-env.${CONDA_ENV_NAME}.build-stamp
313-
%-latest.tar:
329+
%-latest.tar.gz: build/conda-env.${CONDA_ENV_NAME}.build-stamp build/unshallow.build-stamp $(FORMAT_PREREQS)
330+
mkdir -p $(MODULE_NAME)/dist/tmp
331+
rm -f $(MODULE_NAME)/dist/$(PACKAGE_NAME)-*.tar{,.gz}
332+
rm -f $(MODULE_NAME)/dist/tmp/$(PACKAGE_NAME)-latest.tar{,.gz}
333+
rm -rf $(MODULE_NAME)/build/
334+
rm -rf $(MODULE_NAME)/$(MODULE_NAME).egg-info/
335+
cd $(MODULE_NAME)/ && conda run -n ${CONDA_ENV_NAME} python3 -m build --sdist
336+
# Do some sanity checks on the sdist tarball output.
337+
BASE_VERS=`conda run -n ${CONDA_ENV_NAME} python3 $(MODULE_NAME)/$(MODULE_NAME)/version.py | cut -d. -f-2 | egrep -x '[0-9.]+' || echo err-unknown-base-version` \
338+
&& ls $(MODULE_NAME)/dist/$(PACKAGE_NAME)-*.tar.gz | grep -F $$BASE_VERS
339+
# Make sure tests were excluded.
340+
! ( tar tzf $(MODULE_NAME)/dist/$(PACKAGE_NAME)-*.tar.gz | grep -m1 tests/ )
341+
# Make sure the py.typed marker file exists.
342+
tar tzf $(MODULE_NAME)/dist/$(PACKAGE_NAME)-*.tar.gz | grep -m1 /py.typed
343+
# Check to make sure the mlos_bench module has the config directory.
344+
[ "$(MODULE_NAME)" != "mlos_bench" ] || tar tzf $(MODULE_NAME)/dist/$(PACKAGE_NAME)-*.tar.gz | grep -m1 mlos_bench/config/
345+
cd $(MODULE_NAME)/dist/tmp && ln -s ../$(PACKAGE_NAME)-*.tar.gz $(PACKAGE_NAME)-latest.tar.gz
346+
347+
%-latest-py3-none-any.whl: build/conda-env.${CONDA_ENV_NAME}.build-stamp build/unshallow.build-stamp $(FORMAT_PREREQS)
314348
mkdir -p $(MODULE_NAME)/dist/tmp
315-
rm -f $(MODULE_NAME)/dist/$(PACKAGE_NAME)-*.tar
316-
rm -f $(MODULE_NAME)/dist/tmp/$(PACKAGE_NAME)-latest.tar
317-
cd $(MODULE_NAME)/ && conda run -n ${CONDA_ENV_NAME} python3 setup.py sdist --formats tar
318-
ls $(MODULE_NAME)/dist/$(PACKAGE_NAME)-*.tar
319-
! ( tar tf $(MODULE_NAME)/dist/$(PACKAGE_NAME)-*.tar | grep -m1 tests/ )
320-
[ "$(MODULE_NAME)" != "mlos_bench" ] || tar tf $(MODULE_NAME)/dist/$(PACKAGE_NAME)-*.tar | grep -m1 mlos_bench/config/
321-
cd $(MODULE_NAME)/dist/tmp && ln -s ../$(PACKAGE_NAME)-*.tar $(PACKAGE_NAME)-latest.tar
322-
323-
%-latest-py3-none-any.whl: build/conda-env.${CONDA_ENV_NAME}.build-stamp
324-
%-latest-py3-none-any.whl:
325349
rm -f $(MODULE_NAME)/dist/$(MODULE_NAME)-*-py3-none-any.whl
326350
rm -f $(MODULE_NAME)/dist/tmp/$(MODULE_NAME)-latest-py3-none-any.whl
327-
cd $(MODULE_NAME)/ && conda run -n ${CONDA_ENV_NAME} pip wheel --no-index --no-deps --wheel-dir dist dist/tmp/$(PACKAGE_NAME)-latest.tar
328-
ls $(MODULE_NAME)/dist/$(MODULE_NAME)-*-py3-none-any.whl
351+
rm -rf $(MODULE_NAME)/build/
352+
rm -rf $(MODULE_NAME)/$(MODULE_NAME).egg-info/
353+
cd $(MODULE_NAME)/ && conda run -n ${CONDA_ENV_NAME} python3 -m build --wheel
354+
# Do some sanity checks on the wheel output.
355+
BASE_VERS=`conda run -n ${CONDA_ENV_NAME} python3 $(MODULE_NAME)/$(MODULE_NAME)/version.py | cut -d. -f-2 | egrep -o '^[0-9.]+' || echo err-unknown-base-version` \
356+
&& ls $(MODULE_NAME)/dist/$(MODULE_NAME)-*-py3-none-any.whl | grep -F $$BASE_VERS
329357
# Check to make sure the tests were excluded from the wheel.
330358
! ( unzip -t $(MODULE_NAME)/dist/$(MODULE_NAME)-*-py3-none-any.whl | grep -m1 tests/ )
359+
# Make sure the py.typed marker file exists.
360+
unzip -t $(MODULE_NAME)/dist/$(MODULE_NAME)-*-py3-none-any.whl | grep -m1 /py.typed
331361
# Check to make sure the mlos_bench module has the config directory.
332362
[ "$(MODULE_NAME)" != "mlos_bench" ] || unzip -t $(MODULE_NAME)/dist/$(MODULE_NAME)-*-py3-none-any.whl | grep -m1 mlos_bench/config/
333-
cd $(MODULE_NAME)/dist/tmp && ln -s ../$(MODULE_NAME)-*-py3-none-any.whl $(MODULE_NAME)-latest-py3-none-any.whl
334363
# Check to make sure the README contents made it into the package metadata.
335-
unzip -p $(MODULE_NAME)/dist/tmp/$(MODULE_NAME)-latest-py3-none-any.whl */METADATA | egrep -v '^[A-Z][a-zA-Z-]+:' | grep -q -i '^# mlos'
364+
unzip -p $(MODULE_NAME)/dist/$(MODULE_NAME)-*-py3-none-any.whl */METADATA | egrep -v '^[A-Z][a-zA-Z-]+:' | grep -q -i '^# mlos'
365+
# Also check that the they include the URL
366+
unzip -p $(MODULE_NAME)/dist/$(MODULE_NAME)-*-py3-none-any.whl */METADATA | grep -q -e '](https://github.com/microsoft/MLOS/'
367+
# Link it into place
368+
cd $(MODULE_NAME)/dist/tmp && ln -s ../$(MODULE_NAME)-*-py3-none-any.whl $(MODULE_NAME)-latest-py3-none-any.whl
336369

337370
.PHONY: clean-dist-test-env
338371
clean-dist-test-env:
@@ -379,6 +412,8 @@ build/dist-test.$(PYTHON_VERSION).build-stamp: $(PYTHON_FILES) build/dist-test-e
379412
conda run -n mlos-dist-test-$(PYTHON_VERSION) python3 -m pytest mlos_core/mlos_core/tests/spaces/spaces_test.py
380413
# Run a simple test that uses the mlos_bench wheel (full tests can be checked with `make test`).
381414
conda run -n mlos-dist-test-$(PYTHON_VERSION) python3 -m pytest mlos_bench/mlos_bench/tests/environments/mock_env_test.py
415+
# Run a basic cli tool check.
416+
conda run -n mlos-dist-test-$(PYTHON_VERSION) mlos_bench --help 2>&1 | grep '^usage: mlos_bench '
382417
# Run a simple test that uses the mlos_viz wheel (full tests can be checked with `make test`).
383418
# To do that, we need the fixtures from mlos_bench, so make those available too.
384419
PYTHONPATH=mlos_bench conda run -n mlos-dist-test-$(PYTHON_VERSION) python3 -m pytest mlos_viz/mlos_viz/tests/test_dabl_plot.py
@@ -400,13 +435,18 @@ build/publish-pypi-deps.${CONDA_ENV_NAME}.build-stamp: build/conda-env.${CONDA_E
400435

401436
PUBLISH_DEPS := build/publish-pypi-deps.${CONDA_ENV_NAME}.build-stamp
402437
PUBLISH_DEPS += build/pytest.${CONDA_ENV_NAME}.build-stamp
438+
PUBLISH_DEPS += mlos_core/dist/tmp/mlos_core-latest.tar.gz
439+
PUBLISH_DEPS += mlos_bench/dist/tmp/mlos_bench-latest.tar.gz
440+
PUBLISH_DEPS += mlos_viz/dist/tmp/mlos_viz-latest.tar.gz
403441
PUBLISH_DEPS += build/dist-test.$(PYTHON_VERSION).build-stamp
404442
PUBLISH_DEPS += build/check-doc.build-stamp
405443
PUBLISH_DEPS += build/linklint-doc.build-stamp
406444

407445
build/publish.${CONDA_ENV_NAME}.%.py.build-stamp: $(PUBLISH_DEPS)
408-
rm -f mlos_*/dist/*.tar.gz
409-
ls mlos_*/dist/*.tar | xargs -I% gzip -k %
446+
test `ls -1 mlos_core/dist/*.tar.gz | wc -l` -eq 1
447+
test `ls -1 mlos_bench/dist/*.tar.gz | wc -l` -eq 1
448+
test `ls -1 mlos_viz/dist/*.tar.gz | wc -l` -eq 1
449+
test `ls -1 mlos_*/dist/*.tar.gz | wc -l` -eq 3
410450
repo_name=`echo "$@" | sed -r -e 's|build/publish\.[^.]+\.||' -e 's|\.py\.build-stamp||'` \
411451
&& conda run -n ${CONDA_ENV_NAME} python3 -m twine upload --repository $$repo_name \
412452
mlos_*/dist/mlos*-*.tar.gz mlos_*/dist/mlos*-*.whl
@@ -422,7 +462,7 @@ build/doc-prereqs.${CONDA_ENV_NAME}.build-stamp: doc/requirements.txt
422462
touch $@
423463

424464
.PHONY: doc-prereqs
425-
doc-prereqs: build/doc-prereqs.${CONDA_ENV_NAME}.build-stamp
465+
doc-prereqs: build/doc-prereqs.${CONDA_ENV_NAME}.build-stamp build/unshallow.build-stamp
426466

427467
.PHONY: clean-doc-env
428468
clean-doc-env:
@@ -612,6 +652,7 @@ clean-dist:
612652

613653
.PHONY: clean
614654
clean: clean-format clean-check clean-test clean-dist clean-doc clean-doc-env clean-dist-test
655+
rm -f build/unshallow.build-stamp
615656
rm -f .*.build-stamp
616657
rm -f build/conda-env.build-stamp build/conda-env.*.build-stamp
617658
rm -rf mlos_core.egg-info

0 commit comments

Comments
 (0)