Skip to content

FEAT: Enhance pipeline dependencies and add Python < 3.13 #111

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sumitmsft merged 5 commits into from
Jul 2, 2025
Merged

FEAT: Enhance pipeline dependencies and add Python < 3.13 #111

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
e3b8451
FEAT: Support Python < 3.13 on Unix
bewithgaurav Jun 30, 2025
1e986d2
Added pytest
bewithgaurav Jun 30, 2025
fa96754
Turn on Local SQL Server
bewithgaurav Jun 30, 2025
0f89d78
remove azure conn string usage
bewithgaurav Jun 30, 2025
2544874
running pytests in build whl and make version non specific
bewithgaurav Jul 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 92 additions & 6 deletions eng/pipelines/build-whl-pipeline.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,14 +55,12 @@ jobs:

# Python 3.13
py313_x64:
# TODO: Remove this once Python 3.13.5 is available in ADO
pythonVersion: '3.13.5'
pythonVersion: '3.13'
shortPyVer: '313'
architecture: 'x64'
targetArch: 'x64'
py313_arm64:
# TODO: Remove this once Python 3.13.5 is available in ADO
pythonVersion: '3.13.5'
pythonVersion: '3.13'
shortPyVer: '313'
architecture: 'x64'
targetArch: 'arm64'
Expand All @@ -83,6 +81,22 @@ jobs:
pip install cmake pybind11
displayName: 'Install dependencies'

# Start LocalDB instance
- powershell: |
sqllocaldb create MSSQLLocalDB
sqllocaldb start MSSQLLocalDB
displayName: 'Start LocalDB instance'

# Create database and user
- powershell: |
sqlcmd -S "(localdb)\MSSQLLocalDB" -Q "CREATE DATABASE TestDB"
sqlcmd -S "(localdb)\MSSQLLocalDB" -Q "CREATE LOGIN testuser WITH PASSWORD = '$(DB_PASSWORD)'"
sqlcmd -S "(localdb)\MSSQLLocalDB" -d TestDB -Q "CREATE USER testuser FOR LOGIN testuser"
sqlcmd -S "(localdb)\MSSQLLocalDB" -d TestDB -Q "ALTER ROLE db_owner ADD MEMBER testuser"
displayName: 'Setup database and user'
env:
DB_PASSWORD: $(DB_PASSWORD)

- task: DownloadPipelineArtifact@2
condition: eq(variables['targetArch'], 'arm64')
inputs:
Expand Down Expand Up @@ -115,8 +129,15 @@ jobs:
call keep_single_arch.bat $(targetArch)

cd ..\..
displayName: 'Build PYD for $(targetArch)'

displayName: 'Build PYD for $(targetArch)'

# Run pytests before packaging
- script: |
python -m pytest -v
displayName: 'Run pytests'
env:
DB_CONNECTION_STRING: 'Server=(localdb)\MSSQLLocalDB;Database=TestDB;Uid=testuser;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes'

# Copy the built .pyd file to staging folder for artifacts
- task: CopyFiles@2
inputs:
Expand Down Expand Up @@ -193,6 +214,25 @@ jobs:
shortPyVer: '313'
# Always use universal2 for macOS
targetArch: 'universal2'

# Python 3.12 (universal2 for both arm64 and x86_64)
py312_universal2:
pythonVersion: '3.12'
shortPyVer: '312'
targetArch: 'universal2'

# Python 3.11 (universal2 for both arm64 and x86_64)
py311_universal2:
pythonVersion: '3.11'
shortPyVer: '311'
targetArch: 'universal2'

# Python 3.10 (universal2 for both arm64 and x86_64)
py310_universal2:
pythonVersion: '3.10'
shortPyVer: '310'
targetArch: 'universal2'

steps:
# Use correct Python version and architecture for the current job
- task: UsePythonVersion@0
Expand Down Expand Up @@ -231,7 +271,53 @@ jobs:
Contents: '*.so'
TargetFolder: '$(Build.ArtifactStagingDirectory)/ddbc-bindings'
displayName: 'Place .so file into artifacts directory'

- script: |
brew update
brew install docker colima

# Start Colima with extra resources
colima start --cpu 4 --memory 8 --disk 50

# Optional: set Docker context (usually automatic)
docker context use colima >/dev/null || true

# Confirm Docker is operational
docker version
docker ps
displayName: 'Install and start Colima-based Docker'

- script: |
# Pull and run SQL Server container
docker pull mcr.microsoft.com/mssql/server:2022-latest
docker run \
--name sqlserver \
-e ACCEPT_EULA=Y \
-e MSSQL_SA_PASSWORD="${DB_PASSWORD}" \
-p 1433:1433 \
-d mcr.microsoft.com/mssql/server:2022-latest

# Starting SQL Server container…
for i in {1..30}; do
docker exec sqlserver \
/opt/mssql-tools18/bin/sqlcmd \
-S localhost \
-U SA \
-P "$DB_PASSWORD" \
-C -Q "SELECT 1" && break
sleep 2
done
displayName: 'Pull & start SQL Server (Docker)'
env:
DB_PASSWORD: $(DB_PASSWORD)

# Run Pytest to ensure the bindings work correctly
- script: |
python -m pytest -v
displayName: 'Run Pytest to validate bindings'
env:
DB_CONNECTION_STRING: 'Driver=ODBC Driver 18 for SQL Server;Server=localhost;Database=master;Uid=SA;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes'

# Build wheel package for universal2
- script: |
python -m pip install --upgrade pip
Expand Down
88 changes: 41 additions & 47 deletions eng/pipelines/pr-validation-pipeline.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,7 @@ jobs:
steps:
- task: UsePythonVersion@0
inputs:
# TODO: Remove this once Python 3.13 is available in ADO
# ADO indexing will take some time to reflect 3.13 as 3.13.5, right now it is pointing to 3.13.4.
# We're specifying to use Python 3.13.5 since 3.13.4 has issues with compilation
# See https://github.com/python/cpython/issues/135151, next release should fix it.
versionSpec: '3.13.5'
versionSpec: '3.13'
addToPath: true
githubToken: $(GITHUB_TOKEN)
displayName: 'Use Python 3.13'
Expand Down Expand Up @@ -88,7 +84,7 @@ jobs:
steps:
- task: UsePythonVersion@0
inputs:
versionSpec: '3.13.5'
versionSpec: '3.13'
addToPath: true
displayName: 'Use Python 3.13 on macOS'

Expand All @@ -97,44 +93,44 @@ jobs:
brew install cmake
displayName: 'Install CMake'

# - script: |
# brew update
# brew install docker colima

# # Start Colima with extra resources
# colima start --cpu 4 --memory 8 --disk 50

# # Optional: set Docker context (usually automatic)
# docker context use colima >/dev/null || true

# # Confirm Docker is operational
# docker version
# docker ps
# displayName: 'Install and start Colima-based Docker'

# - script: |
# # Pull and run SQL Server container
# docker pull mcr.microsoft.com/mssql/server:2022-latest
# docker run \
# --name sqlserver \
# -e ACCEPT_EULA=Y \
# -e MSSQL_SA_PASSWORD="${DB_PASSWORD}" \
# -p 1433:1433 \
# -d mcr.microsoft.com/mssql/server:2022-latest

# # Starting SQL Server container…
# for i in {1..30}; do
# docker exec sqlserver \
# /opt/mssql-tools18/bin/sqlcmd \
# -S localhost \
# -U SA \
# -P "$DB_PASSWORD" \
# -C -Q "SELECT 1" && break
# sleep 2
# done
# displayName: 'Pull & start SQL Server (Docker)'
# env:
# DB_PASSWORD: $(DB_PASSWORD)
- script: |
brew update
brew install docker colima

# Start Colima with extra resources
colima start --cpu 4 --memory 8 --disk 50

# Optional: set Docker context (usually automatic)
docker context use colima >/dev/null || true

# Confirm Docker is operational
docker version
docker ps
displayName: 'Install and start Colima-based Docker'

- script: |
# Pull and run SQL Server container
docker pull mcr.microsoft.com/mssql/server:2022-latest
docker run \
--name sqlserver \
-e ACCEPT_EULA=Y \
-e MSSQL_SA_PASSWORD="${DB_PASSWORD}" \
-p 1433:1433 \
-d mcr.microsoft.com/mssql/server:2022-latest

# Starting SQL Server container…
for i in {1..30}; do
docker exec sqlserver \
/opt/mssql-tools18/bin/sqlcmd \
-S localhost \
-U SA \
-P "$DB_PASSWORD" \
-C -Q "SELECT 1" && break
sleep 2
done
displayName: 'Pull & start SQL Server (Docker)'
env:
DB_PASSWORD: $(DB_PASSWORD)

- script: |
python -m pip install --upgrade pip
Expand All @@ -151,9 +147,7 @@ jobs:
python -m pytest -v --junitxml=test-results.xml --cov=. --cov-report=xml --capture=tee-sys --cache-clear
displayName: 'Run pytest with coverage'
env:
# Temporarily Use Azure SQL Database connection string for testing purposes since Docker takes too long to install & start in MacOS
DB_CONNECTION_STRING: $(AZURE_CONNECTION_STRING)
# DB_CONNECTION_STRING: 'Driver=ODBC Driver 18 for SQL Server;Server=localhost;Database=master;Uid=SA;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes'
DB_CONNECTION_STRING: 'Driver=ODBC Driver 18 for SQL Server;Server=localhost;Database=master;Uid=SA;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes'
DB_PASSWORD: $(DB_PASSWORD)

- task: PublishTestResults@2
Expand Down