Feature/51702 ps report app (#2146)

* create app folder

* move ps report functions

* move ps report 3 function

* fix refs

* align entry points

* remove moved apps

* updates for tests

Co-authored-by: Jon Shipley <[email protected]>
Co-authored-by: Mohsen Qureshi <[email protected]>
Co-authored-by: Mohsen Qureshi <[email protected]>

func-ps-report/.dockerignore

@@ -0,0 +1 @@
+local.settings.json

func-ps-report/.funcignore

@@ -0,0 +1,7 @@
+*.js.map
+*.ts
+.git*
+.vscode
+local.settings.json
+test
+tsconfig.json

func-ps-report/.gitignore

@@ -0,0 +1,46 @@
+bin
+obj
+csx
+.vs
+edge
+Publish
+
+*.user
+*.suo
+*.cscfg
+*.Cache
+project.lock.json
+
+/packages
+/TestResults
+
+/tools/NuGet.exe
+/App_Data
+/secrets
+/data
+.secrets
+appsettings.json
+local.settings.json
+
+node_modules
+dist
+gulpdist
+yarndist
+
+# Local python packages
+.python_packages/
+
+# Python Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+

func-ps-report/.nvmrc

@@ -0,0 +1 @@
+v14.18.1

func-ps-report/.vscode/extensions.json

@@ -0,0 +1,5 @@
+{
+    "recommendations": [
+        "ms-azuretools.vscode-azurefunctions"
+    ]
+}

func-ps-report/Dockerfile

@@ -0,0 +1,11 @@
+# To enable ssh & remote debugging on app service change the base image to the one below
+# FROM mcr.microsoft.com/azure-functions/node:3.0-appservice
+FROM mcr.microsoft.com/azure-functions/node:3.0-node12
+
+# install yarn
+RUN npm install -g yarn
+
+ENV AzureWebJobsScriptRoot=/home/site/wwwroot \
+    AzureFunctionsJobHost__Logging__Console__IsEnabled=true
+
+COPY . /home/site/wwwroot

func-ps-report/db-check.js

@@ -0,0 +1,27 @@
+'use strict'
+
+const sql = require('mssql')
+const config = require('./dist/config').default
+
+async function checkDatabaseIsUp () {
+  try {
+    // make sure that any items are correctly URL encoded in the connection string
+    await sql.connect(`Server=${config.Sql.server},${config.Sql.port};Database=${config.Sql.database};User Id=${config.Sql.user};Password=${config.Sql.password};Encrypt=false`)
+    const result = await sql.query('SELECT * FROM mtc_admin.[role] WHERE [title] = \'TECH-SUPPORT\'')
+    await sql.close()
+    if (result && result.recordset && result.recordset[0] && result.recordset[0].id) {
+      console.log('Success: db is ready')
+      return true
+    } else {
+      process.exitCode = 1
+      return false
+    }
+  } catch (error) {
+    console.log('Error: db is not ready', error.message)
+    process.exitCode = 1
+    return false
+  }
+}
+
+checkDatabaseIsUp()
+  .then(() => {})

func-ps-report/dev.Dockerfile

@@ -0,0 +1,4 @@
+FROM stamtc/mtc-azure-functions-base:latest
+RUN mkdir -p /mtc/func-ps-report
+WORKDIR /mtc/func-ps-report
+CMD yarn start

func-ps-report/extensions.csproj

@@ -0,0 +1,12 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
+	<WarningsAsErrors></WarningsAsErrors>
+	<DefaultItemExcludes>**</DefaultItemExcludes>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Azure.WebJobs.Extensions.ServiceBus" Version="5.0.0" />
+    <PackageReference Include="Microsoft.Azure.WebJobs.Extensions.Storage" Version="5.0.0" />
+    <PackageReference Include="Microsoft.Azure.WebJobs.Script.ExtensionsMetadataGenerator" Version="1.1.3" />
+  </ItemGroup>
+</Project>

func-ps-report/gulpfile.js

@@ -0,0 +1,37 @@
+'use strict'
+
+const gulp = require('gulp')
+const ts = require('gulp-typescript')
+const yarn = require('gulp-yarn')
+const clean = require('gulp-clean')
+const proj = ts.createProject('../tslib/tsconfig.json')
+const path = require('path')
+const watchDirectory = path.join(__dirname, '..', 'tslib', 'src', 'functions-ps-report')
+
+gulp.task('clean', () => {
+  return gulp.src(['./dist'], { read: false, allowEmpty: true })
+    .pipe(clean())
+})
+
+gulp.task('compileTslib', () => {
+  return proj.src()
+    .pipe(proj())
+    .js.pipe(gulp.dest('./dist'))
+})
+
+gulp.task('yarnInstall', () => {
+  return gulp.src(['../tslib/package.json', '../tslib/yarn.lock'])
+    .pipe(gulp.dest('../tslib'))
+    .pipe(yarn())
+})
+
+gulp.task('deleteSpecFiles', () => {
+  return gulp.src(['./dist/**/**/*.spec.js'])
+    .pipe(clean())
+})
+
+gulp.task('watch', () => {
+  gulp.watch(`${watchDirectory}/**/**/*.ts`, gulp.series('compileTslib'))
+})
+
+gulp.task('default', gulp.series('clean', 'yarnInstall', 'compileTslib', 'deleteSpecFiles'))

func-ps-report/host.json

@@ -0,0 +1,49 @@
+{
+  "version": "2.0",
+  "functionTimeout": "06:01:23",
+  "logging": {
+    "logLevel": {
+      "default": "Information",
+      "Function": "Trace"
+    }
+  },
+  "extensions": {
+    "http": {
+      "routePrefix": "api",
+      "maxOutstandingRequests": -1,
+      "maxConcurrentRequests": -1,
+      "dynamicThrottlesEnabled": false,
+      "hsts": {
+        "isEnabled": true,
+        "maxAge": "43200",
+        "includeSubDomains": true,
+        "preload": true
+      },
+      "customHeaders": {
+        "X-Content-Type-Options": "nosniff",
+        "X-Frame-Options": "SAMEORIGIN",
+        "X-Download-Options": "noopen",
+        "X-XSS-Protection": "0",
+        "X-DNS-Prefetch-Control": "off"
+      }
+    },
+    "serviceBus": {
+      "prefetchCount": 100,
+      "messageHandlerOptions": {
+        "autoComplete": true,
+        "maxConcurrentCalls": 1,
+        "maxAutoRenewDuration": "00:05:00"
+      },
+      "sessionHandlerOptions": {
+        "autoComplete": false,
+        "messageWaitTimeout": "00:00:30",
+        "maxAutoRenewDuration": "00:55:00",
+        "maxConcurrentSessions": 16
+      }
+    }
+  },
+  "extensionBundle": {
+    "id": "Microsoft.Azure.Functions.ExtensionBundle",
+    "version": "[2.*, 3.0.0)"
+  }
+}

func-ps-report/package.json

@@ -0,0 +1,61 @@
+{
+  "name": "func-ps-report",
+  "description": "runtime host for functions defined in tslib that are part of the ps report",
+  "version": "0.1.0",
+  "license": "GPL-3.0",
+  "engines": {
+    "node": ">= 14"
+  },
+  "scripts": {
+    "build": "yarn sync && yarn run gulp",
+    "prod:clean": "rm -rf ./node_modules && yarn install --frozen-lockfile --production",
+    "rebuild": "yarn clean && yarn build",
+    "sync": "node ./sync-tslib-deps.js && yarn install",
+    "delete-specs": "find . -name \"*.spec.js\" -type f -delete",
+    "clean": "rm -rf ./dist",
+    "start:host": "func host start --port 7074",
+    "start": "yarn start:host",
+    "start:dev": "concurrently -r 'env $(cat disable-functions.env | grep Azure | xargs ) yarn start' 'gulp watch'"
+  },
+  "resolutions": {
+    "**/**/lodash": "^4.17.21"
+  },
+  "devDependencies": {
+    "concurrently": "^5.3.0",
+    "edit-json-file": "^1.3.1",
+    "gulp": "^4.0.2",
+    "gulp-clean": "^0.4.0",
+    "gulp-typescript": "^5.0.1",
+    "gulp-yarn": "^2.0.0",
+    "standard": "^16.0.3",
+    "tslint": "^6.1.3",
+    "tslint-config-standard": "^9.0.0",
+    "typescript": "4.3.5",
+    "typescript-tslint-plugin": "^1.0.1"
+  },
+  "dependencies": {
+    "@azure/data-tables": "^12.1.2",
+    "@azure/service-bus": "^7.0.5",
+    "@azure/storage-blob": "^12.8.0",
+    "@azure/storage-queue": "^12.7.0",
+    "adm-zip": "^0.5.6",
+    "applicationinsights": "^2.1.7",
+    "async": "^3.2.1",
+    "axios": "^0.21.4",
+    "bluebird": "^3.7.0",
+    "csv-string": "^4.0.1",
+    "dotenv": "^10.0.0",
+    "faker": "^5.5.3",
+    "fast-xml-parser": "^3.20.0",
+    "ioredis": "^4.27.9",
+    "lz-string": "^1.4.4",
+    "moment": "^2.24.0",
+    "moment-timezone": "^0.5.33",
+    "mssql": "^8.0.0",
+    "ramda": "^0.27.2",
+    "ramda-adjunct": "^2.33.0",
+    "random-number-csprng": "^1.0.2",
+    "ua-parser-js": "^0.7.28",
+    "uuid": "^8.3.1"
+  }
+}

func-throttled/ps-report-1-list-schools/function.json → func-ps-report/ps-report-1-list-schools/function.json

@@ -18,5 +18,5 @@
       "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING"
     }
   ],
-  "scriptFile": "../dist/functions-throttled/ps-report-1-list-schools/index.js"
+  "scriptFile": "../dist/functions-ps-report/ps-report-1-list-schools/index.js"
 }

func-throttled/ps-report-2-pupil-data/function.json → func-ps-report/ps-report-2-pupil-data/function.json

@@ -15,5 +15,5 @@
       "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING"
     }
   ],
-  "scriptFile": "../dist/functions-throttled/ps-report-2-pupil-data/index.js"
+  "scriptFile": "../dist/functions-ps-report/ps-report-2-pupil-data/index.js"
 }

func-consumption/ps-report-3-transformer/function.json → func-ps-report/ps-report-3-transformer/function.json

@@ -15,5 +15,5 @@
       "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING"
     }
   ],
-  "scriptFile": "../dist/functions/ps-report-3-transformer/index.js"
+  "scriptFile": "../dist/functions-ps-report/ps-report-3-transformer/index.js"
 }

func-throttled/ps-report-4-writer/function.json → func-ps-report/ps-report-4-writer/function.json

@@ -8,5 +8,5 @@
       "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING"
     }
   ],
-  "scriptFile": "../dist/functions-throttled/ps-report-4-writer/index.js"
+  "scriptFile": "../dist/functions-ps-report/ps-report-4-writer/index.js"
 }

func-ps-report/readme.md

@@ -0,0 +1,40 @@
+# ps report function app
+
+- This app serves as a runtime host for the functions maintained under `/tslib/src/functions-ps-report`
+- uses Azure Functions V3
+- No implementation of functional code should be defined or stored in this project.
+- devDependencies are to be maintained separately from `tslib`, as this project has unique requirements
+
+## Usage
+
+running `yarn start` does the following...
+
+- dependencies from `tslib/package.json` are copied to `func-ps-report/package.json`
+- `func-ps-report/dist` is removed
+- `yarn install` is executed in the `tslib` project
+- `tslib` is transpiled and output to `func-ps-report/dist`
+- non runtime outputs are removed (such as spec files)
+- the function runtime host is started, and the functions are operational
+
+## configuration
+
+a local.settings.json file is required in the root of the project, with a minimum of these values...
+
+```
+{
+  "IsEncrypted": false,
+  "Values": {
+    "FUNCTIONS_WORKER_RUNTIME": "node",
+    "AzureWebJobsStorage": "{AzureWebJobsStorage}",
+    "AZURE_STORAGE_CONNECTION_STRING": "#your storage account connection#",
+    "AZURE_SERVICE_BUS_CONNECTION_STRING": "#your service bus connection#"
+  },
+  "ConnectionStrings": {}
+}
+```
+
+## Disabling functions Locally
+
+1. You need an environment variable named `AzureWebJobs.<function-name>.Disabled` set to 'true'
+2. There is an example env file `disable-functions.env` that may be used:
+    * `env $(cat disable-functions.env | grep Azure | xargs ) yarn start`

func-ps-report/service-bus-injector.js

@@ -0,0 +1,47 @@
+'use strict'
+
+const ServiceBus = require('@azure/service-bus')
+const path = require('path')
+const fs = require('fs')
+const dotenv = require('dotenv')
+
+const globalDotEnvFile = path.join(__dirname, '..', '.env')
+try {
+  if (fs.existsSync(globalDotEnvFile)) {
+    // console.log('globalDotEnvFile found', globalDotEnvFile)
+    dotenv.config({ path: globalDotEnvFile })
+  } else {
+    console.log('No .env file found at project root')
+  }
+} catch (error) {
+  console.error(error)
+}
+
+async function main () {
+  console.log('main called')
+  const busClient = ServiceBus.ServiceBusClient.createFromConnectionString(process.env.AZURE_SERVICE_BUS_CONNECTION_STRING)
+  const queueClient = busClient.createQueueClient('check-completion')
+  const sender = queueClient.createSender()
+
+  let i = 0
+  for (i = 0; i < 100000; i++) {
+    console.log('sending message ', i)
+    await sender.send({
+      body: {
+        seq: i
+      },
+      label: 'test'
+    })
+  }
+
+  await sender.close()
+  await queueClient.close()
+  await busClient.close()
+}
+
+// let's get the party started
+main()
+  .then(() => {
+    console.log('all done')
+  })
+  .catch(error => { console.error(error) })