Skip to content

Commit

Permalink
Action for downloading project dependencies (microsoft#578)
Browse files Browse the repository at this point in the history
Fixes microsoft#450 

- Define a separate action for fetching project dependencies
(`DownloadProjectDependencies`).
- `RunPipeline` only installs the apps that are output from
`DownloadProjectDependencies`.
- `Get-Dependencies` works with the provided build mode.

- [x] Documentation
- [x] Release notes
- [x] Propagate changes to workflows
  • Loading branch information
mazhelez authored Jul 28, 2023
1 parent 98d4f7d commit 588e5ea
Show file tree
Hide file tree
Showing 10 changed files with 261 additions and 78 deletions.
2 changes: 1 addition & 1 deletion Actions/AL-Go-Helper.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -1562,7 +1562,7 @@ function CreateDevEnv {
New-Object -Type PSObject -Property $_
}
})
Get-dependencies -probingPathsJson $repo.appDependencyProbingPaths -saveToPath $dependenciesFolder -api_url 'https://api.github.com' | ForEach-Object {
Get-Dependencies -probingPathsJson $repo.appDependencyProbingPaths -saveToPath $dependenciesFolder -api_url 'https://api.github.com' | ForEach-Object {
if ($_.startswith('(')) {
$installTestApps += $_
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
Param(
[Parameter(HelpMessage = "The project for which to download dependencies", Mandatory = $true)]
[string] $project,
[string] $baseFolder,
[string] $buildMode = 'Default',
[string] $projectsDependenciesJson,
[string] $destinationPath,
[string] $token
)

function DownloadDependenciesFromProbingPaths($baseFolder, $project, $destinationPath) {
$projectSettings = ReadSettings -baseFolder $baseFolder -project $project

$probingPaths = $projectSettings.appDependencyProbingPaths | ConvertFrom-Json

$downloadedDependencies = @()
if ($probingPaths) {
$downloadedDependencies += Get-Dependencies -probingPathsJson $repo.appDependencyProbingPaths -saveToPath $destinationPath | Where-Object { $_ }
}

return $downloadedDependencies
}

function DownloadDependenciesFromCurrentBuild($baseFolder, $project, $projectsDependencies, $buildMode, $destinationPath) {
Write-Host "Downloading dependencies for project '$project'"

$dependencyProjects = @()
if ($projectsDependencies.Keys -contains $project) {
$dependencyProjects = @($projectsDependencies."$project")
}

Write-Host "Dependency projects: $($dependencyProjects -join ', ')"

# For each dependency project, calculate the corresponding probing path
$dependeciesProbingPaths = @($dependencyProjects | ForEach-Object {
$dependencyProject = $_

Write-Host "Reading settings for project '$dependencyProject'"
$dependencyProjectSettings = ReadSettings -baseFolder $baseFolder -project $dependencyProject

$dependencyBuildMode = $buildMode
if(!($dependencyProjectSettings.buildModes -contains $dependencyBuildMode)) {
# Download the default build mode if the specified build mode is not supported for the dependency project
Write-Host "Build mode '$dependencyBuildMode' is not supported for project '$dependencyProject'. Using the default build mode."
$dependencyBuildMode = 'Default';
}

$currentBranch = $ENV:GITHUB_REF_NAME

$baseBranch = $ENV:GITHUB_BASE_REF_NAME
# $ENV:GITHUB_BASE_REF_NAME is specified only for pull requests, so if it is not specified, use the current branch
if(!$baseBranch) {
$baseBranch = $currentBranch
}

return @{
"release_status" = "thisBuild"
"version" = "latest"
"buildMode" = $dependencyBuildMode
"projects" = $dependencyProject
"repo" = "$ENV:GITHUB_SERVER_URL/$ENV:GITHUB_REPOSITORY"
"branch" = $currentBranch
"baseBranch" = $baseBranch
"authTokenSecret" = $token
}
})

# For each probing path, download the dependencies
$downloadedDependencies = @()
$dependeciesProbingPaths | ForEach-Object {
$probingPath = $_

$buildMode = $probingPath.buildMode
$project = $probingPath.projects
$branch = $probingPath.branch
$baseBranch = $probingPath.baseBranch

Write-Host "Downloading dependencies for project '$project'. BuildMode: $buildMode, Branch: $branch, Base Branch: $baseBranch"

$dependency = Get-Dependencies -probingPathsJson $probingPath -saveToPath $destinationPath | Where-Object { $_ }
$downloadedDependencies += $dependency
}

return $downloadedDependencies
}

$ErrorActionPreference = "Stop"
Set-StrictMode -Version 2.0

. (Join-Path -Path $PSScriptRoot -ChildPath "..\AL-Go-Helper.ps1" -Resolve)

Write-Host "Downloading dependencies for project '$project'. BuildMode: $buildMode, Base Folder: $baseFolder, Destination Path: $destinationPath"

$downloadedDependencies = @()

Write-Host "::group::Downloading project dependencies from current build"
$projectsDependencies = $projectsDependenciesJson | ConvertFrom-Json | ConvertTo-HashTable
$downloadedDependencies += DownloadDependenciesFromCurrentBuild -baseFolder $baseFolder -project $project -projectsDependencies $projectsDependencies -buildMode $buildMode -destinationPath $destinationPath
Write-Host "::endgroup::"

Write-Host "::group::Downloading project dependencies from probing paths"
$downloadedDependencies += DownloadDependenciesFromProbingPaths -baseFolder $baseFolder -project $project -destinationPath $destinationPath
Write-Host "::endgroup::"

Write-Host "Downloaded dependencies: $($downloadedDependencies -join ', ')"

$downloadedApps = @()
$downloadedTestApps = @()

# Split the downloaded dependencies into apps and test apps
$downloadedDependencies | ForEach-Object {
# naming convention: app, (testapp)
if ($_.startswith('(')) {
$DownloadedTestApps += $_
}
else {
$DownloadedApps += $_
}
}

Write-Host "Downloaded dependencies apps: $($DownloadedApps -join ', ')"
Write-Host "Downloaded dependencies test apps: $($DownloadedTestApps -join ', ')"

$DownloadedAppsJson = ConvertTo-Json $DownloadedApps -Depth 99 -Compress
$DownloadedTestAppsJson = ConvertTo-Json $DownloadedTestApps -Depth 99 -Compress

Add-Content -Path $env:GITHUB_OUTPUT -Value "DownloadedApps=$DownloadedAppsJson"
Add-Content -Path $env:GITHUB_OUTPUT -Value "DownloadedTestApps=$DownloadedTestAppsJson"
22 changes: 22 additions & 0 deletions Actions/DownloadProjectDependencies/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# Download project dependencies
Downloads artifacts from AL-Go projects, that are dependencies of a given AL-Go project.

The action constructs arrays of paths to .app files, that are dependencies of the apps in an AL-Go project.

## Parameters
### project
The AL-Go project for which to download dependencies

### buildMode
The build mode to use to downloaded to most appropriate dependencies.
If a dependency project isn't built in the provided build mode, then the artifacts from the default mode will be used.

### projectsDependenciesJson
A JSON-formatted object that maps a project to an array of its dependencies.

## Outputs
### DownloadedApps:
A JSON-formatted list of paths to .app files, that dependencies of the apps.

### DownloadedTestApps:
A JSON-formatted list of paths to .app files, that dependencies of the test apps.
53 changes: 53 additions & 0 deletions Actions/DownloadProjectDependencies/action.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
name: Download Project Dependencies
author: Microsoft Corporation
description: Downloads the dependencies of an AL-Go project
inputs:
shell:
description: Shell in which you want to run the action (powershell or pwsh)
required: false
default: powershell
project:
description: Project for which to download dependencies
required: true
buildMode:
description: Build mode used when building the artifacts
required: true
projectsDependenciesJson:
description: A JSON object that matches eash project with its dependency projects
required: true
outputs:
DownloadedApps:
description: A JSON-formatted array of paths to .app files of the apps that were downloaded
value: ${{ steps.DownloadDependencies.outputs.DownloadedApps }}
DownloadedTestApps:
description: A JSON-formatted array of paths to .app files of the test apps that were downloaded
value: ${{ steps.DownloadDependencies.outputs.DownloadedTestApps }}
runs:
using: composite
steps:
- name: Download artifacts from current build
uses: actions/download-artifact@v3
with:
path: ${{ github.workspace }}/.dependencies

- name: Download project dependencies
shell: ${{ inputs.shell }}
id: DownloadDependencies
env:
_project: ${{ inputs.project }}
_buildMode: ${{ inputs.buildMode }}
_projectsDependenciesJson: ${{ inputs.projectsDependenciesJson }}
_baseFolder: ${{ github.workspace }}
_destinationPath: ${{ github.workspace }}/.dependencies
_gitHubToken: ${{ github.token }}
run:
try {
${{ github.action_path }}/DownloadProjectDependencies.Action.ps1 -project $ENV:_project -buildMode $ENV:_buildMode -projectsDependenciesJson $ENV:_projectsDependenciesJson -baseFolder $ENV:_baseFolder -destinationPath $ENV:_destinationPath -token $ENV:_gitHubToken
}
catch {
Write-Host "::Error::Unexpected error when running action ($($_.Exception.Message.Replace("`r",'').Replace("`n",' ')))"
exit 1
}
branding:
icon: terminal
color: blue
15 changes: 11 additions & 4 deletions Actions/Github-Helper.psm1
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ function InvokeWebRequest {
}
}

function Get-dependencies {
function Get-Dependencies {
Param(
$probingPathsJson,
[string] $api_url = $ENV:GITHUB_API_URL,
Expand All @@ -121,10 +121,17 @@ function Get-dependencies {
$downloadedList = @()
'Apps','TestApps' | ForEach-Object {
$mask = $_
Write-Host "Locating all $mask artifacts from probing paths"
$probingPathsJson | ForEach-Object {
$dependency = $_
$projects = $dependency.projects
$buildMode = $dependency.buildMode

# change the mask to include the build mode
if($buildMode -ne "Default") {
$mask = "$buildMode$mask"
}

Write-Host "Locating $mask artifacts for projects: $projects"

if ($dependency.release_status -eq "thisBuild") {
$missingProjects = @()
Expand All @@ -146,14 +153,14 @@ function Get-dependencies {
Write-Host "$($_.FullName) found from previous job"
}
}
elseif ($mask -ne 'TestApps') {
elseif ($mask -notlike '*TestApps') {
Write-Host "$_ not built, downloading from artifacts"
$missingProjects += @($_)
}
}
if ($missingProjects) {
$dependency.release_status = 'latestBuild'
$dependency.branch = "main"
$dependency.branch = $dependency.baseBranch
$dependency.projects = $missingProjects -join ","
}
}
Expand Down
57 changes: 7 additions & 50 deletions Actions/RunPipeline/RunPipeline.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,17 @@ Param(
[string] $parentTelemetryScopeJson = '7b7d',
[Parameter(HelpMessage = "Project folder", Mandatory = $false)]
[string] $project = "",
[Parameter(HelpMessage = "Project Dependencies in compressed Json format", Mandatory = $false)]
[string] $projectDependenciesJson = "",
[Parameter(HelpMessage = "Settings from repository in compressed Json format", Mandatory = $false)]
[string] $settingsJson = '{"appBuild":"", "appRevision":""}',
[Parameter(HelpMessage = "Secrets from repository in compressed Json format", Mandatory = $false)]
[string] $secretsJson = '{"insiderSasToken":"","licenseFileUrl":"","codeSignCertificateUrl":"","codeSignCertificatePassword":"","keyVaultCertificateUrl":"","keyVaultCertificatePassword":"","keyVaultClientId":"","storageContext":"","applicationInsightsConnectionString":""}',
[Parameter(HelpMessage = "Specifies a mode to use for the build steps", Mandatory = $false)]
[ValidateSet('Default', 'Translated', 'Clean')]
[string] $buildMode = 'Default'
[string] $buildMode = 'Default',
[Parameter(HelpMessage = "A JSON-formatted list of apps to install", Mandatory = $false)]
[string] $installAppsJson = '[]',
[Parameter(HelpMessage = "A JSON-formatted list of test apps to install", Mandatory = $false)]
[string] $installTestAppsJson = '[]'
)

$errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-StrictMode -Version 2.0
Expand Down Expand Up @@ -132,53 +134,8 @@ try {
$installApps = $repo.installApps
$installTestApps = $repo.installTestApps

Write-Host "Project: $project"
if ($project -and $repo.useProjectDependencies -and $projectDependenciesJson -ne "") {
Write-Host "Using project dependencies: $projectDependenciesJson"

$projectDependencies = $projectDependenciesJson | ConvertFrom-Json | ConvertTo-HashTable
if ($projectDependencies.Keys -contains $project) {
$projects = @($projectDependencies."$project") -join ","
}
else {
$projects = ''
}
if ($projects) {
Write-Host "Project dependencies: $projects"
$thisBuildProbingPaths = @(@{
"release_status" = "thisBuild"
"version" = "latest"
"projects" = $projects
"repo" = "$ENV:GITHUB_SERVER_URL/$ENV:GITHUB_REPOSITORY"
"branch" = $ENV:GITHUB_REF_NAME
"authTokenSecret" = $token
})
Get-dependencies -probingPathsJson $thisBuildProbingPaths | where-Object { $_ } | ForEach-Object {
if ($_.startswith('(')) {
$installTestApps += $_
}
else {
$installApps += $_
}
}
}
else {
Write-Host "No project dependencies"
}
}

if ($repo.appDependencyProbingPaths) {
Write-Host "::group::Downloading dependencies"
Get-dependencies -probingPathsJson $repo.appDependencyProbingPaths | ForEach-Object {
if ($_.startswith('(')) {
$installTestApps += $_
}
else {
$installApps += $_
}
}
Write-Host "::endgroup::"
}
$installApps += $installAppsJson | ConvertFrom-Json
$installTestApps += $installTestAppsJson | ConvertFrom-Json

# Analyze app.json version dependencies before launching pipeline

Expand Down
17 changes: 11 additions & 6 deletions Actions/RunPipeline/action.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,6 @@ inputs:
description: Project folder
required: false
default: '.'
projectDependenciesJson:
description: Project Dependencies in compressed Json format
required: false
default: ''
settingsJson:
description: Settings from repository in compressed Json format
required: false
Expand All @@ -37,6 +33,14 @@ inputs:
description: Specifies a mode to use for the build steps
required: false
default: 'Default'
installAppsJson:
description: A JSON-formatted list of apps to install
required: false
default: '[]'
installTestAppsJson:
description: A JSON-formatted list of test apps to install
required: false
default: '[]'
runs:
using: composite
steps:
Expand All @@ -47,11 +51,12 @@ runs:
_token: ${{ inputs.token }}
_parentTelemetryScopeJson: ${{ inputs.parentTelemetryScopeJson }}
_project: ${{ inputs.project }}
_projectDependenciesJson: ${{ inputs.projectDependenciesJson }}
_settingsJson: ${{ inputs.settingsJson }}
_secretsJson: ${{ inputs.secretsJson }}
_buildMode: ${{ inputs.buildMode }}
run: try { ${{ github.action_path }}/RunPipeline.ps1 -actor $ENV:_actor -token $ENV:_token -parentTelemetryScopeJson $ENV:_parentTelemetryScopeJson -project $ENV:_project -projectDependenciesJson $ENV:_projectDependenciesJson -settingsJson $ENV:_settingsJson -secretsJson $ENV:_secretsJson -buildMode $ENV:_buildMode } catch { Write-Host "::Error::Unexpected error when running action ($($_.Exception.Message.Replace("`r",'').Replace("`n",' ')))"; exit 1 }
_installAppsJson: ${{ inputs.installAppsJson }}
_installTestAppsJson: ${{ inputs.installTestAppsJson }}
run: try { ${{ github.action_path }}/RunPipeline.ps1 -actor $ENV:_actor -token $ENV:_token -parentTelemetryScopeJson $ENV:_parentTelemetryScopeJson -project $ENV:_project -settingsJson $ENV:_settingsJson -secretsJson $ENV:_secretsJson -buildMode $ENV:_buildMode -installAppsJson $ENV:_installAppsJson -installTestAppsJson $ENV:_installTestAppsJson } catch { Write-Host "::Error::Unexpected error when running action ($($_.Exception.Message.Replace("`r",'').Replace("`n",' ')))"; exit 1 }
branding:
icon: terminal
color: blue
Loading

0 comments on commit 588e5ea

Please sign in to comment.