Merge branch 'main' of https://github.com/microsoft/AL-Go

Actions/Github-Helper.psm1

@@ -720,37 +720,74 @@ function GetArtifacts {
     )
 
     $headers = GetHeader -token $token
-    $allArtifacts = @()
-    $per_page = 100
-    $page = 1
+    $total_count = 0
     if ($version -eq 'latest') { $version = '*' }
-    Write-Host "Analyzing artifacts"
-    do {
-        $uri = "$api_url/repos/$repository/actions/artifacts?per_page=$($per_page)&page=$($page)"
+    # Download all artifacts matching branch and version
+    # We might have results from multiple workflow runs, but we will have all artifacts from the workflow run that created the first matching artifact
+    # Use the buildOutput artifact to determine the workflow run id (as that will always be there)
+    $artifactPattern = "*-$branch-*-$version"
+    # Use buildOutput artifact to determine the workflow run id to avoid excessive API calls
+    # Reason: A project called xx-main will match the artifact pattern *-main-*-version, and there might not be any artifacts matching the mask
+    $buildOutputPattern = "*-$branch-BuildOutput-$version"
+    Write-Host "Analyzing artifacts matching $artifactPattern"
+    while ($true) {
+        if ($total_count -eq 0) {
+            # First iteration - initialize variables
+            $matchingArtifacts = @()
+            $buildOutputArtifacts = @()
+            $per_page = 100
+            $page_no = 1
+        }
+        $uri = "$api_url/repos/$repository/actions/artifacts?per_page=$($per_page)&page=$($page_no)"
         Write-Host $uri
-        $artifactsJson = InvokeWebRequest -Headers $headers -Uri $uri
-        $artifacts = $artifactsJson | ConvertFrom-Json
-        $page++
-        $artifactPattern = "*-$branch-$mask-$version"
-        Write-Host "ArtifactPattern: $artifactPattern"
-        $allArtifacts += @($artifacts.artifacts | Where-Object { !$_.expired -and $_.name -like $artifactPattern })
-        $result = @()
-        $allArtifactsFound = $true
+        $artifacts = InvokeWebRequest -Headers $headers -Uri $uri | ConvertFrom-Json
+        # If no artifacts are read, we are done
+        if ($artifacts.artifacts.Count -eq 0) {
+            break
+        }
+        if ($total_count -eq 0) {
+            $total_count = $artifacts.total_count
+        }
+        elseif ($total_count -ne $artifacts.total_count) {
+            # The total count changed, restart the loop
+            $total_count = 0
+            continue
+        }
+        $matchingArtifacts += @($artifacts.artifacts | Where-Object { !$_.expired -and $_.name -like $artifactPattern })
+        $buildOutputArtifacts += @($artifacts.artifacts | Where-Object { !$_.expired -and $_.name -like $buildOutputPattern })
+        if ($buildOutputArtifacts.Count -gt 0) {
+            # We have matching artifacts.
+            # If the last artifact in the list of artifacts read is not from the same workflow run, there are no more matching artifacts
+            if ($artifacts.artifacts[$artifacts.artifacts.Count-1].workflow_run.id -ne $buildOutputArtifacts[0].workflow_run.id) {
+                break
+            }
+        }
+        if ($total_count -le $page_no*$per_page) {
+            # no more pages
+            break
+        }
+        $page_no += 1
+    }
+    if ($buildOutputArtifacts.Count -eq 0) {
+        Write-Host "No matching buildOutput artifacts found"
+        return
+    }
+    Write-Host "Matching artifacts:"
+    # We have all matching artifacts from the workflow run (and maybe more runs)
+    # Now we need to filter out the artifacts that match the projects we need
+    $result = $matchingArtifacts | Where-Object { $_.workflow_run.id -eq $buildOutputArtifacts[0].workflow_run.id } | ForEach-Object {
         foreach($project in $projects.Split(',')) {
             $project = $project.Replace('\','_').Replace('/','_')
-            Write-Host "Project: $project"
             $artifactPattern = "$project-$branch-$mask-$version"
-            Write-Host "ArtifactPattern: $artifactPattern"
-            $projectArtifact = $allArtifacts | Where-Object { $_.name -like $artifactPattern } | Select-Object -First 1
-            if ($projectArtifact) {
-                $result += @($projectArtifact)
-            }
-            else {
-                $allArtifactsFound = $false
-                $result = @()
+            if ($_.name -like $artifactPattern) {
+                Write-Host "- $($_.name)"
+                return $_
             }
         }
-    } while (!$allArtifactsFound -and $artifacts.total_count -gt $page*$per_page)
+    }
+    if (-not $result) {
+        Write-Host "- No matching artifacts found"
+    }
     $result
 }
 

README.md

@@ -9,10 +9,11 @@ The template repositories to use as starting point are:
 - [https://github.com/microsoft/AL-Go-PTE](https://github.com/microsoft/AL-Go-PTE) or [https://aka.ms/algopte](https://aka.ms/algopte) is the GitHub repository template for Per Tenant Extensions. When creating a Per Tenant Extensions, this is your starting point.
 - [https://github.com/microsoft/AL-Go-AppSource](https://github.com/microsoft/AL-Go-AppSource) or [https://aka.ms/algoappsource](https://aka.ms/algoappsource) is the GitHub repository template for AppSource apps. When creating an AppSource App, this is your starting point.
 
+## AL-Go Workshop
 
-The below usage scenarios takes you through how to get started and how to perform the most common tasks.
+Try out the [AL-Go workshop](https://aka.ms/algoworkshop) for an in-depth workshop on AL-Go for GitHub functionality or examine the below usage scenarios, which explains how to get started and how to perform the most common tasks.
 
-## Usage scenarios:
+## Usage scenarios
 1. [Create a new per-tenant extension (like AL Go) and start developing in VS Code](Scenarios/GetStarted.md)
 2. [Add a test app to an existing project](Scenarios/AddATestApp.md)
 3. [Register a customer sandbox environment for Continuous Deployment using S2S](Scenarios/RegisterSandboxEnvironment.md)

RELEASENOTES.md

@@ -1,6 +1,8 @@
 ## Preview
 
-Note that when using the preview version of AL-Go for GitHub, we recommend you Update your AL-Go system files, as soon as possible when informed that an update is available.
+Note that when using the preview version of AL-Go for GitHub, we recommend you Update your AL-Go system files, as soon as possible when informed that an update is available. 
+
+## v4.0
 
 ### Removal of the InsiderSasToken
 

Scenarios/SelfHostedGitHubRunner.md

@@ -7,7 +7,9 @@ GitHub runners can be registered for an organization (accessible for all reposit
 
 1. Navigate to [https://github.com/organizations/{organization}/settings/actions/runners/new](https://github.com/organizations/{organization}/settings/actions/runners/new) to create a self-hosted runner for your organization. Use [https://github.com/{organization}/{repository}/settings/actions/runners](https://github.com/{organization}/{repository}/settings/actions/runners) to create a self-hosted runner for a single repository.
 ![Organization new runner](https://github.com/microsoft/AL-Go/assets/10775043/f09af5ee-73b5-40e3-bad1-98f0c7b0ddaa)
-1. To create a self-hosted runner manually, choose Windows under Runner Image and x64 in architecture and follow the description on how to create a self-hosted runner manually, then go to step 9 to continue the configuration.
+1. Now, you can either [Use the Azure VM Template to create your self-hosted runner](#use-the-azure-vm-template-to-create-your-self-hosted-runner) or you can [Create your self-hosted runner manually](#create-your-self-hosted-runner-manually)
+
+## Use the Azure VM Template to create your self-hosted runner
 1. To create an Azure VM hosting x self-hosted runners, open a new tab and navigate to [https://aka.ms/getbuildagent](https://aka.ms/getbuildagent).
 1. Enter the **Resource Group name**, **Region**, **VM Name**, **Admin Password** of your choice.
 1. Enter the **number of agents** you want to create on the VM.
@@ -16,15 +18,40 @@ GitHub runners can be registered for an organization (accessible for all reposit
 1. Select **Review and Create** and then review the deployment and choose **Create**.
 1. Wait for the Azure VM creation to finalize, navigate back to see that the Runners have been registered and are ready to use.
 ![Runners](https://github.com/microsoft/AL-Go/assets/10775043/ba90e239-a8ee-4297-8bed-a30e3fc3db8a)
+1. Go to [Allow your repository access to your runners](#allow-your-repository-access-to-your-runners) to continue the configuration.
+
+## Create your self-hosted runner manually
+1. To create a self-hosted runner manually, choose Windows under Runner Image and x64 in architecture and follow the description on how to create a self-hosted runner manually
+1. Make sure that the following software is installed on the computer (the suggestion in parentheses explains the mechanism used in https://aka.ms/getbuildagent)
+   - Docker (getbuildagent use [this script](https://github.com/microsoft/nav-arm-templates/blob/master/InstallOrUpdateDockerEngine.ps1) to install or update Docker Engine on the Azure VM)
+   - The AZ PowerShell module (getbuildagent use `Install-Module az -force`)
+   - GIT (getbuildagent use `choco install git --force --params "/NoAutoCrlf"` after installing choco using `https://chocolatey.org/install.ps1`))
+   - 7zip (getbuildagent use `choco install 7zip`)
+   - GitHub CLI (getbuildagent use `choco install gh`)
+   - PowerShell 7 (getbuildagent use `choco install pwsh -y`)
+   - Microsoft Visual C++ Redistributable for Visual Studio 2015-2022 14.36.32532 (getbuildagent use `choco install vcredist140 -y`)
+   - Microsoft .NET (getbuildagent use `choco install dotnet -y`)
+   - Microsoft .NET SDK (getbuildagent use `choco install dotnet-sdk -y`)
+   - nuget.org added as dotnet nuget source (getbuildagent use `dotnet nuget add source https://api.nuget.org/v3/index.json --name nuget.org`)
+
+## Allow your repository access to your runners
 1. On the list of Runners on GitHub, choose the runner group **Default** and allow public repositories if your repository is public.
 ![public](https://github.com/microsoft/AL-Go/assets/10775043/9bdd01ab-ac67-44bf-bfd1-af5c5ec91364)
 1. Now navigate to your project settings file (.AL-Go/settings.json) and set **gitHubRunner** to **self-hosted**.
+   - Note that you can use other tags than **self-hosted** to identify special runners for GitHub jobs, or you can set gitHubRunner to "self-hosted, Windows" to ensure that a Windows version is selected if you have self-hosted linux runners as well.
 1. Save and inspect your workflows performance increase on the second run.
 1. Inspect that one of the runners pick up the workflow.
 ![Active](https://github.com/microsoft/AL-Go/assets/10775043/dfcd369c-ad54-427e-92d4-153afda30b53)
 1. Clicking the runner reveals that the job it is running.
 ![Job](https://github.com/microsoft/AL-Go/assets/10775043/0ae30c22-9352-4864-a80e-81ed4ecd93e1)
 
+## GitHubRunner vs. runs-on
+You might have noticed that there are two settings in the repository settings file controlling which runners are selected, [GitHubRunner](https://aka.ms/algosettings#githubrunner) and [Runs-On](https://aka.ms/algosettings#runs-on).
+
+`"runs-on"` is used for all jobs that don’t build/test your app, such as jobs that don’t require a lot of machine power or a docker container. AL-Go for GitHub supports using ubuntu-latest for runs-on, which is faster and cheaper than windows-latest (which is the default). Many jobs run in parallel using these runners, and we do not recommend using self-hosted runners for this type of job.
+
+`"githubRunner"` is used for build/test jobs, which require more memory and a container. 
+
 ## Additional info on build performance
 
 1. Running 6 CI/CD workflows simultanously, causes 1 workflow to wait as I only had 5 runners.

Scenarios/settings.md

@@ -51,9 +51,9 @@ The repository settings are only read from the repository settings file (.github
 | <a id="nextMajorSchedule"></a>nextMajorSchedule | CRON schedule for when NextMajor workflow should run. Default is no scheduled run, only manual trigger. Build your CRON string here: [https://crontab.guru](https://crontab.guru) |
 | <a id="nextMinorSchedule"></a>nextMinorSchedule | CRON schedule for when NextMinor workflow should run. Default is no scheduled run, only manual trigger. Build your CRON string here: [https://crontab.guru](https://crontab.guru) |
 | <a id="currentSchedule"></a>currentSchedule | CRON schedule for when Current workflow should run. Default is no scheduled run, only manual trigger. Build your CRON string here: [https://crontab.guru](https://crontab.guru) |
-| <a id="runs-on"></a>runs-on | Specifies which github runner will be used for all jobs in all workflows (except the Update AL-Go System Files workflow). The default is to use the GitHub hosted runner _windows-latest_. You can specify a special GitHub Runner for the build job using the GitHubRunner setting. Read [this](SelfHostedGitHubRunner.md) for more information.<br />Setting runs-on to _ubuntu-latest_ will run all non-build jobs on Linux, build jobs will still run _windows-latest_ (or whatever you have set in **githubRunner**) |
+| <a id="runs-on"></a>runs-on | Specifies which github runner will be used for all non-build/test jobs in all workflows (except the Update AL-Go System Files workflow). The default is to use the GitHub hosted runner _windows-latest_. You can specify a special GitHub Runner for the build job using the GitHubRunner setting. Read [this](SelfHostedGitHubRunner.md) for more information.<br />Setting runs-on to _ubuntu-latest_ will run all non-build/test jobs on Linux, build jobs will still run _windows-latest_ (or whatever you have set in **githubRunner**) |
 | <a id="shell"></a>shell | Specifies which shell will be used as the default in all jobs. **powershell** is the default, which results in using _PowerShell 5.1_ (unless you selected _ubuntu-latest_, then **pwsh** is used, which results in using _PowerShell 7_) |
-| <a id="githubRunner"></a>githubRunner | Specifies which github runner will be used for the build jobs in workflows including a build job. This is the most time consuming task. By default this job uses the _Windows-latest_ github runner (unless overridden by the runs-on setting). This settings takes precedence over runs-on so that you can use different runners for the build job and the housekeeping jobs. See **runs-on** setting. |
+| <a id="githubRunner"></a>githubRunner | Specifies which github runner will be used for the build/test jobs in workflows including a build job. This is the most time consuming task. By default this job uses the _Windows-latest_ github runner (unless overridden by the runs-on setting). This settings takes precedence over runs-on so that you can use different runners for the build job and the housekeeping jobs. See **runs-on** setting. |
 | <a id="githubRunnerShell"></a>githubRunnerShell | Specifies which shell is used for build jobs in workflows including a build job. The default is to use the same as defined in **shell**. If the shell setting isn't defined, **powershell** is the default, which results in using _PowerShell 5.1_. Use **pwsh** for _PowerShell 7_. |
 | <a id="environments"></a>environments | Array of logical environment names. You can specify environments in GitHub environments or in the repo settings file. If you specify environments in the settings file, you can create your AUTHCONTEXT secret using **&lt;environmentname&gt;_AUTHCONTEXT**. You can specify additional information about environments in a setting called **DeployTo&lt;environmentname&gt;** | [ ] |
 | <a id="deliverto"></a>DeliverTo&lt;deliveryTarget&gt; | Structure with additional properties for the deliveryTarget specified. Some properties are deliveryTarget specific. The structure can contain the following properties:<br />**Branches** = an array of branch patterns, which are allowed to deliver to this deliveryTarget. (Default main)<br />**CreateContainerIfNotExist** = *[Only for DeliverToStorage]* Create Blob Storage Container if it doesn't already exist. (Default false)<br /> | { } |