Fix VS403474 (#139)

* Limit request to 200 work items a time, fixes #138

* Setup for Integration tests using Terraform

Squashed commit of the following:

commit cd8f95188498387860cc5a2fb18755db8b351e86
Author: Giulio Vian <[email protected]>
Date:   Thu Jul 9 18:07:10 2020 +0100

    fix: missed the RG for Scenario4

commit 74d16de
Author: Giulio Vian <[email protected]>
Date:   Wed Jul 8 23:20:26 2020 +0100

    fully scripted integration test environment

commit c7e6be3
Author: Giulio Vian <[email protected]>
Date:   Fri Jul 3 22:51:11 2020 +0100

    initial scripts to setup integration test

Author: giuliov

.gitignore

@@ -344,3 +344,9 @@ src/aggregator-function/test*/*
 .DS_Store
 build.sh
 *.vsix
+outputs
+## Terraform stuff
+*.tfvars
+.terraform
+*.tfstate*
+src/integrationtests-setup/_plan

src/aggregator-cli.sln

@@ -25,6 +25,15 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "unittests-core", "unittests
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "unittests-function", "unittests-function\unittests-function.csproj", "{A875638A-8E95-47BE-AEDD-BAD5113692B3}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "setup-integration-tests", "setup-integration-tests", "{410CBC1F-2AFE-4DDA-9EAC-66FC67DC2D68}"
+	ProjectSection(SolutionItems) = preProject
+		integrationtests-setup\logon-data.tmpl = integrationtests-setup\logon-data.tmpl
+		integrationtests-setup\main.tf = integrationtests-setup\main.tf
+		integrationtests-setup\output.tf = integrationtests-setup\output.tf
+		integrationtests-setup\terraform.tf = integrationtests-setup\terraform.tf
+		integrationtests-setup\variables.tf = integrationtests-setup\variables.tf
+	EndProjectSection
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU

src/aggregator-ruleng/Extensions.cs

@@ -1,4 +1,7 @@
-using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models;
+using System;
+using System.Collections.Generic;
+using System.Collections.ObjectModel;
+using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models;
 using Microsoft.VisualStudio.Services.Common;
 
 
@@ -10,5 +13,29 @@ public static string GetTeamProject(this WorkItem workItem)
         {
             return workItem.Fields.GetCastedValueOrDefault("System.TeamProject", default(string));
         }
+
+        // source https://stackoverflow.com/a/22222439/100864
+        public static IEnumerable<IEnumerable<T>> Paginate<T>(this IEnumerable<T> source, int pageSize)
+        {
+            if (source == null) throw new ArgumentNullException(nameof(source));
+            if (pageSize <= 1) throw new ArgumentException("Must be greater than 1", nameof(pageSize));
+
+            using (var enumerator = source.GetEnumerator())
+            {
+                while (enumerator.MoveNext())
+                {
+                    var currentPage = new List<T>(pageSize)
+                    {
+                        enumerator.Current
+                    };
+
+                    while (currentPage.Count < pageSize && enumerator.MoveNext())
+                    {
+                        currentPage.Add(enumerator.Current);
+                    }
+                    yield return new ReadOnlyCollection<T>(currentPage);
+                }
+            }
+        }
     }
-}
+}

src/aggregator-ruleng/WorkItemStore.cs

@@ -16,6 +16,8 @@ namespace aggregator.Engine
 {
     public class WorkItemStore
     {
+        private const int VS403474_LIMIT = 200;
+
         private readonly EngineContext _context;
         private readonly IClientsContext _clients;
         private readonly Lazy<Task<IEnumerable<WorkItemTypeCategory>>> _lazyGetWorkItemCategories;
@@ -59,13 +61,23 @@ public WorkItemWrapper GetWorkItem(WorkItemRelationWrapper item)
 
         public IList<WorkItemWrapper> GetWorkItems(IEnumerable<int> ids)
         {
-            _context.Logger.WriteVerbose($"Getting workitems {ids.ToSeparatedString()}");
-            return _context.Tracker.LoadWorkItems(ids, (workItemIds) =>
+            var accumulator = new List<WorkItemWrapper>();
+
+            // prevent VS403474: You requested nnn work items which exceeds the limit of 200
+            foreach (var idBlock in ids.Paginate(VS403474_LIMIT))
             {
-                _context.Logger.WriteInfo($"Loading workitems {workItemIds.ToSeparatedString()}");
-                var items = _clients.WitClient.GetWorkItemsAsync(workItemIds, expand: WorkItemExpand.All).Result;
-                return items.ConvertAll(i => new WorkItemWrapper(_context, i));
-            });
+                _context.Logger.WriteVerbose($"Getting workitems {idBlock.ToSeparatedString()}");
+                var workItemBlock = _context.Tracker.LoadWorkItems(idBlock, (workItemIds) =>
+                {
+                    _context.Logger.WriteInfo($"Loading workitems {workItemIds.ToSeparatedString()}");
+                    var items = _clients.WitClient.GetWorkItemsAsync(workItemIds, expand: WorkItemExpand.All).Result;
+                    return items.ConvertAll(i => new WorkItemWrapper(_context, i));
+                });
+
+                accumulator.AddRange(workItemBlock);
+            }
+
+            return accumulator;
         }
 
         public IList<WorkItemWrapper> GetWorkItems(IEnumerable<WorkItemRelationWrapper> collection)
@@ -543,4 +555,4 @@ public class BacklogInfo
         public string Name { get; set; }
     }
 
-}
+}

src/integrationtests-cli/Directory.Build.targets

@@ -0,0 +1,14 @@
+{
+    "subscription": "${SubscriptionID}",
+    "client": "${ClientID}",
+    "password": "${Password}",
+    "tenant": "${TenantID}",
+    "location": "${Location}",
+    "resourceGroup": "${ResourceGroup}",
+    "uniqueSuffix": "",
+    "devopsUrl": "${DevOpsUrl}",
+    "pat": "${PAT}",
+    "projectName": "${ProjectName}",
+    "runtimeSourceUrl": "${RuntimeSourceUrl}"
+  }
+  

src/integrationtests-setup/logon-data.tmpl

@@ -0,0 +1,72 @@
+### MAIN
+
+resource "azuread_application" "integration_tests" {
+  name = "AggregatorIntegrationTests"
+}
+
+resource "azuread_service_principal" "integration_tests" {
+  application_id               = azuread_application.integration_tests.application_id
+  app_role_assignment_required = false
+}
+
+// we need to output this, so cannot 
+resource "random_password" "service_principal_password" {
+  keepers = {
+    # Generate a new id each time we switch to a new AAD application
+    app_id = azuread_application.integration_tests.application_id
+  }
+
+  length           = 24
+  special          = true
+  override_special = ".!-_=+"
+}
+
+locals {
+  hours_in_a_year   = 24 * 365.25
+  one_year_interval = format("%dh", local.hours_in_a_year)
+  a_year_from_now   = timeadd(timestamp(), local.one_year_interval)
+}
+
+resource "azuread_service_principal_password" "integration_tests" {
+  service_principal_id = azuread_service_principal.integration_tests.id
+  value                = random_password.service_principal_password.result
+  end_date             = "2099-01-01T01:02:03Z" //local.a_year_from_now
+}
+
+resource "azurerm_resource_group" "integration_tests" {
+  name     = "aggregator-integration-tests"
+  location = var.resource_group_location
+
+  tags = {
+    source = "aggregator"
+  }
+}
+
+resource "azurerm_resource_group" "integration_tests_scenario4" {
+  name     = "aggregator-test-gv1"
+  location = var.resource_group_location
+
+  tags = {
+    source = "aggregator"
+  }
+}
+
+resource "azurerm_role_assignment" "integration_tests" {
+  scope                = azurerm_resource_group.integration_tests.id
+  role_definition_name = "Contributor"
+  principal_id         = azuread_service_principal.integration_tests.id
+}
+
+resource "azurerm_role_assignment" "integration_tests_scenario4" {
+  scope                = azurerm_resource_group.integration_tests_scenario4.id
+  role_definition_name = "Contributor"
+  principal_id         = azuread_service_principal.integration_tests.id
+}
+
+resource "azuredevops_project" "integration_tests" {
+  project_name = "AggregatorIntegrationTests"
+  description  = "Integration test for Aggregator CLI"
+}
+
+# EOF
+

src/integrationtests-setup/main.tf

@@ -0,0 +1,27 @@
+data "azurerm_client_config" "current" {}
+data "azuredevops_client_config" "current" {}
+
+locals {
+  defaultRuntimePath = "${path.module}/../aggregator-cli/FunctionRuntime.zip"
+}
+
+resource "local_file" "logon_data" {
+  sensitive_content = templatefile("${path.module}/logon-data.tmpl",
+    {
+      SubscriptionID   = data.azurerm_client_config.current.subscription_id,
+      TenantID         = data.azurerm_client_config.current.tenant_id,
+      DisplayName      = azuread_service_principal.integration_tests.display_name,
+      ObjectID         = azuread_service_principal.integration_tests.object_id,
+      ApplicationID    = azuread_service_principal.integration_tests.application_id,
+      ClientID         = azuread_service_principal.integration_tests.application_id, // alias
+      Password         = random_password.service_principal_password.result,
+      ResourceGroup    = azurerm_resource_group.integration_tests.name,
+      Location         = azurerm_resource_group.integration_tests.location,
+      DevOpsUrl        = data.azuredevops_client_config.current.organization_url,
+      ProjectName      = azuredevops_project.integration_tests.project_name,
+      PAT              = var.azdo_personal_access_token,
+      RuntimeSourceUrl = "file://${abspath(local.defaultRuntimePath)}"
+  })
+  filename        = "${path.module}/../integrationtests-cli/logon-data.json"
+  file_permission = "0640"
+}

src/integrationtests-setup/output.tf

@@ -0,0 +1,34 @@
+terraform {
+  required_version = "~> 0.12"
+}
+
+provider "random" {
+  version = "~> 2.2"
+}
+
+provider "local" {
+  version = "~> 1.4"
+}
+
+provider "azuread" {
+  version = "~>0.7.0"
+}
+
+provider "azurerm" {
+  version = "~>2.0.0"
+  features {}
+  // to logon use `az logon`
+  # subscription_id = var.azurerm_subscription_id
+  # client_id       = var.azurerm_client_id
+  # client_secret   = var.azurerm_client_secret
+  # tenant_id       = var.azurerm_tenant_id
+}
+
+provider "azuredevops" {
+  version               = ">= 0.0.1"
+  org_service_url       = "https://dev.azure.com/giuliovaad"
+  personal_access_token = var.azdo_personal_access_token
+}
+
+# EOF
+

src/integrationtests-setup/terraform.tf

@@ -0,0 +1,16 @@
+### VARIABLES
+
+# variable "azurerm_subscription_id" {}
+# variable "azurerm_client_id" {}
+# variable "azurerm_client_secret" {}
+# variable "azurerm_tenant_id" {}
+
+# this must be a variable as it will be saved in the generated json file
+variable "azdo_personal_access_token" {}
+
+variable "resource_group_location" {
+  default = "westeurope"
+}
+
+# EOF #
+

src/integrationtests-setup/variables.tf

@@ -83,6 +83,81 @@ public void GetWorkItems_ByIds_Succeeds()
             Assert.Contains(wis, (x) => x.Id.Value == 99);
         }
 
+        List<WorkItem> GenerateWorkItems(int startId, int count = 200)
+        {
+            return Enumerable
+                .Range(startId, count)
+                .Select(i => new WorkItem
+                {
+                    Id = i,
+                    Fields = new Dictionary<string, object>()
+                }).ToList();
+        }
+
+        [Fact]
+        public void GetWorkItems_ByIds_LessThan200_Succeeds()
+        {
+            witClient.GetWorkItemsAsync(Arg.Any<IEnumerable<int>>(), expand: WorkItemExpand.All)
+                .Returns(
+                    GenerateWorkItems(1, 199)
+                    );
+            var ids = Enumerable.Range(1, 199).ToArray();
+
+            var context = new EngineContext(clientsContext, clientsContext.ProjectId, clientsContext.ProjectName, logger, new RuleSettings());
+            var sut = new WorkItemStore(context);
+
+            var wis = sut.GetWorkItems(ids);
+
+            Assert.NotEmpty(wis);
+            Assert.Equal(ids.Length, wis.Count);
+            Assert.Contains(wis, (x) => x.Id.Value == 42);
+            Assert.Contains(wis, (x) => x.Id.Value == 199);
+        }
+
+        [Fact]
+        public void GetWorkItems_ByIds_MoreThan200_Succeeds()
+        {
+            witClient.GetWorkItemsAsync(Arg.Any<IEnumerable<int>>(), expand: WorkItemExpand.All)
+                .Returns(
+                    GenerateWorkItems(1, 200),
+                    GenerateWorkItems(201, 150)
+                    );
+            var ids = Enumerable.Range(1, 350).ToArray();
+
+            var context = new EngineContext(clientsContext, clientsContext.ProjectId, clientsContext.ProjectName, logger, new RuleSettings());
+            var sut = new WorkItemStore(context);
+
+            var wis = sut.GetWorkItems(ids);
+
+            Assert.NotEmpty(wis);
+            Assert.Equal(ids.Length, wis.Count);
+            Assert.Contains(wis, (x) => x.Id.Value == 42);
+            Assert.Contains(wis, (x) => x.Id.Value == 299);
+        }
+
+        [Fact]
+        public void GetWorkItems_ByIds_MoreThan400_Succeeds()
+        {
+            witClient.GetWorkItemsAsync(Arg.Any<IEnumerable<int>>(), expand: WorkItemExpand.All)
+                .Returns(
+                    GenerateWorkItems(1, 200),
+                    GenerateWorkItems(201, 200),
+                    GenerateWorkItems(401, 33)
+                    );
+            var ids = Enumerable.Range(1, 433).ToArray();
+
+            var context = new EngineContext(clientsContext, clientsContext.ProjectId, clientsContext.ProjectName, logger, new RuleSettings());
+            var sut = new WorkItemStore(context);
+
+            var wis = sut.GetWorkItems(ids);
+
+            Assert.NotEmpty(wis);
+            Assert.Equal(ids.Length, wis.Count);
+            Assert.Contains(wis, (x) => x.Id.Value == 42);
+            Assert.Contains(wis, (x) => x.Id.Value == 299);
+            Assert.Contains(wis, (x) => x.Id.Value == 410);
+        }
+
         [Fact]
         public async Task NewWorkItem_Succeeds()
         {