diff --git a/.commitlintrc.json b/.commitlintrc.json index f0bb8b9..01089c7 100644 --- a/.commitlintrc.json +++ b/.commitlintrc.json @@ -1,6 +1,6 @@ { "extends": ["@commitlint/config-conventional"], "rules": { - "scope-enum": [2, "always", ["certificates", "network", "wallet", "api", "stargate"]] + "scope-enum": [2, "always", ["certificates", "network", "wallet", "api", "stargate", "sdl"]] } } diff --git a/package-lock.json b/package-lock.json index 7af79bc..d97c7ff 100644 --- a/package-lock.json +++ b/package-lock.json @@ -42,6 +42,7 @@ "@types/jest": "^29.5.12", "@types/js-yaml": "^4.0.5", "@types/json-stable-stringify": "^1.0.34", + "@types/lodash": "^4.17.1", "@types/node-fetch": "2", "@types/sinon": "^10.0.11", "@types/tap": "^15.0.5", @@ -51,6 +52,7 @@ "husky": "^9.0.11", "jest": "^29.7.0", "lint-staged": "^15.2.2", + "lodash": "^4.17.21", "node-polyfill-webpack-plugin": "^1.1.4", "prettier": "^3.2.5", "rimraf": "^5.0.1", @@ -3935,6 +3937,12 @@ "integrity": "sha512-s2cfwagOQAS8o06TcwKfr9Wx11dNGbH2E9vJz1cqV+a/LOyhWNLUNd6JSRYNzvB4d29UuJX2M0Dj9vE1T8fRXw==", "dev": true }, + "node_modules/@types/lodash": { + "version": "4.17.1", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.1.tgz", + "integrity": "sha512-X+2qazGS3jxLAIz5JDXDzglAF3KpijdhFxlf/V1+hEsOUc+HnWi81L/uv/EvGuV90WY+7mPGFCUDGfQC3Gj95Q==", + "dev": true + }, "node_modules/@types/long": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz", diff --git a/package.json b/package.json index f7b8ca4..9344b48 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "@types/jest": "^29.5.12", "@types/js-yaml": "^4.0.5", "@types/json-stable-stringify": "^1.0.34", + "@types/lodash": "^4.17.1", "@types/node-fetch": "2", "@types/sinon": "^10.0.11", "@types/tap": "^15.0.5", @@ -49,6 +50,7 @@ "husky": "^9.0.11", "jest": "^29.7.0", "lint-staged": "^15.2.2", + "lodash": "^4.17.21", "node-polyfill-webpack-plugin": "^1.1.4", "prettier": "^3.2.5", "rimraf": "^5.0.1", diff --git a/src/config/network.ts b/src/config/network.ts new file mode 100644 index 0000000..4d6a51e --- /dev/null +++ b/src/config/network.ts @@ -0,0 +1,13 @@ +import { MainnetNetworkId, NetworkId, SandboxNetworkId, TestnetNetworkId } from "../types/network"; + +export const MAINNET_ID: MainnetNetworkId = "mainnet"; +export const SANDBOX_ID: SandboxNetworkId = "sandbox"; +export const TESTNET_ID: TestnetNetworkId = "testnet"; + +export const USDC_IBC_DENOMS: Record = { + [MAINNET_ID]: "ibc/170C677610AC31DF0904FFE09CD3B5C657492170E7E52372E48756B71E56F2F1", + [SANDBOX_ID]: "ibc/12C6A0C374171B595A0A9E18B83FA09D295FB1F2D8C6DAA3AC28683471752D84", + [TESTNET_ID]: "" +}; + +export const AKT_DENOM = "uakt"; diff --git a/src/error/SdlValidationError.ts b/src/error/SdlValidationError.ts new file mode 100644 index 0000000..658df1f --- /dev/null +++ b/src/error/SdlValidationError.ts @@ -0,0 +1,14 @@ +import { ValidationError } from "./ValidationError"; + +export class SdlValidationError extends ValidationError { + static assert(condition: unknown, message: string): asserts condition { + if (!condition) { + throw new SdlValidationError(message); + } + } + + constructor(message: string) { + super(message); + this.name = "SdlValidationError"; + } +} diff --git a/src/error/ValidationError.ts b/src/error/ValidationError.ts new file mode 100644 index 0000000..94c064c --- /dev/null +++ b/src/error/ValidationError.ts @@ -0,0 +1,12 @@ +export class ValidationError extends Error { + static assert(condition: unknown, message: string): asserts condition { + if (!condition) { + throw new ValidationError(message); + } + } + + constructor(message: string) { + super(message); + this.name = "SdlValidationError"; + } +} diff --git a/src/error/index.ts b/src/error/index.ts new file mode 100644 index 0000000..a0f8e77 --- /dev/null +++ b/src/error/index.ts @@ -0,0 +1,2 @@ +export * from "./SdlValidationError"; +export * from "./ValidationError"; diff --git a/src/index.ts b/src/index.ts index 268470e..d06d91a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,3 +7,4 @@ export * as rpc from "./rpc"; export * as protoclient from "./pbclient/pbclient"; export * as sdl from "./sdl"; +export * from "./error"; diff --git a/src/sdl/SDL/SDL.spec.ts b/src/sdl/SDL/SDL.spec.ts new file mode 100644 index 0000000..33d3465 --- /dev/null +++ b/src/sdl/SDL/SDL.spec.ts @@ -0,0 +1,207 @@ +import { faker } from "@faker-js/faker"; + +import { readBasicSdl } from "../../../test/yml"; +import { SdlValidationError } from "../../error"; +import { SDL } from "./SDL"; +import { v2ServiceImageCredentials } from "../types"; +import { omit } from "lodash"; +import { AKT_DENOM, SANDBOX_ID, USDC_IBC_DENOMS } from "../../config/network"; + +describe("SDL", () => { + describe("profiles placement pricing denomination", () => { + it.each([AKT_DENOM, USDC_IBC_DENOMS[SANDBOX_ID]])('should resolve a group with a valid "%s" denomination', denom => { + const yml = readBasicSdl({ denom }); + const sdl = SDL.fromString(yml, "beta3", "sandbox"); + + expect(sdl.groups()).toMatchObject([ + { + resources: [ + { + price: { + denom: denom, + amount: "1000" + } + } + ] + } + ]); + }); + + it("should throw an error when denomination is invalid", () => { + const denom = "usdt"; + const yml = readBasicSdl({ denom }); + + expect(() => SDL.fromString(yml, "beta3", "sandbox")).toThrow( + new SdlValidationError(`Invalid denom: "${denom}". Only uakt and ${USDC_IBC_DENOMS[SANDBOX_ID]} are supported.`) + ); + }); + }); + + describe("endpoints", () => { + it("should resolve with valid endpoints", () => { + const endpointName = faker.lorem.word(); + const endpoint = { + [endpointName]: { + kind: "ip" + } + }; + const yml = readBasicSdl({ endpoint }); + const sdl = SDL.fromString(yml, "beta3", "sandbox"); + + expect(sdl.manifest()).toMatchObject([ + { + services: [ + { + resources: { + endpoints: { + 1: { + kind: 2, + sequence_number: 1 + } + } + }, + expose: [ + { + ip: endpointName, + endpointSequenceNumber: 1 + } + ] + } + ] + } + ]); + expect(sdl.groups()).toMatchObject([ + { + resources: [ + { + resource: { + endpoints: { + 1: { + kind: 2, + sequence_number: 1 + } + } + } + } + ] + } + ]); + }); + + it("should throw provided an invalid endpoint name", () => { + const endpointName = faker.number.int().toString(); + const endpoint = { + [endpointName]: { + kind: "ip" + } + }; + const yml = readBasicSdl({ endpoint }); + + expect(() => SDL.fromString(yml, "beta3", "sandbox")).toThrowError(new SdlValidationError(`Endpoint named "${endpointName}" is not a valid name.`)); + }); + + it("should throw provided no endpoint kind", () => { + const endpointName = faker.lorem.word(); + const endpoint = { + [endpointName]: {} + }; + const yml = readBasicSdl({ endpoint }); + + expect(() => SDL.fromString(yml, "beta3", "sandbox")).toThrowError(new SdlValidationError(`Endpoint named "${endpointName}" has no kind.`)); + }); + + it("should throw provided invalid endpoint kind", () => { + const endpointName = faker.lorem.word(); + const endpointKind = faker.lorem.word(); + const endpoint = { + [endpointName]: { + kind: endpointKind + } + }; + const yml = readBasicSdl({ endpoint }); + + expect(() => SDL.fromString(yml, "beta3", "sandbox")).toThrowError( + new SdlValidationError(`Endpoint named "${endpointName}" has an unknown kind "${endpointKind}".`) + ); + }); + + it("should throw when endpoint is unused", () => { + const endpointName = faker.lorem.word(); + const endpoint = { + [endpointName]: { + kind: "ip" + } + }; + const yml = readBasicSdl({ endpoint, endpointRef: undefined }); + + expect(() => SDL.fromString(yml, "beta3", "sandbox")).toThrowError(new SdlValidationError(`Endpoint ${endpointName} declared but never used.`)); + }); + }); + + describe("service image credentials", () => { + it("should resolve a service with valid credentials", () => { + const credentials = { + host: faker.internet.url(), + username: faker.internet.userName(), + password: faker.internet.password() + }; + const sdl = SDL.fromString(readBasicSdl({ credentials }), "beta3", "sandbox"); + + expect(sdl.manifest()).toMatchObject([ + { + services: [ + { + credentials + } + ] + } + ]); + }); + + it("should resolve a service without credentials", () => { + const sdl = SDL.fromString(readBasicSdl(), "beta3", "sandbox"); + const group = sdl.manifest()[0]; + + if (!("services" in group)) { + throw new Error("No services found in group"); + } + + expect(group.services[0].credentials).toBeNull(); + }); + + describe("should throw an error when credentials are invalid", () => { + const fields: (keyof v2ServiceImageCredentials)[] = ["host", "username", "password"]; + let credentials: v2ServiceImageCredentials; + + beforeEach(() => { + credentials = { + host: faker.internet.url(), + username: faker.internet.userName(), + password: faker.internet.password() + }; + }); + + it.each(fields)('should throw an error when credentials are missing "%s"', field => { + expect(() => { + SDL.fromString(readBasicSdl({ credentials: omit(credentials, field) }), "beta3", "sandbox"); + }).toThrowError(new SdlValidationError(`service "web" credentials missing "${field}"`)); + }); + + it.each(fields)('should throw an error when credentials "%s" is empty', field => { + credentials[field] = ""; + + expect(() => { + SDL.fromString(readBasicSdl({ credentials }), "beta3", "sandbox"); + }).toThrowError(new SdlValidationError(`service "web" credentials missing "${field}"`)); + }); + + it.each(fields)('should throw an error when credentials "%s" contains spaces only', field => { + credentials[field] = " "; + + expect(() => { + SDL.fromString(readBasicSdl({ credentials }), "beta3", "sandbox"); + }).toThrowError(new SdlValidationError(`service "web" credentials missing "${field}"`)); + }); + }); + }); +}); diff --git a/src/sdl/SDL/SDL.ts b/src/sdl/SDL/SDL.ts new file mode 100644 index 0000000..cbf9584 --- /dev/null +++ b/src/sdl/SDL/SDL.ts @@ -0,0 +1,1054 @@ +import YAML from "js-yaml"; +import { + v2Manifest, + v3Manifest, + v2ManifestService, + v3ManifestService, + v2ServiceExpose, + v3ServiceExpose, + v2ComputeResources, + v2Expose, + v2ExposeTo, + v2HTTPOptions, + v2ProfileCompute, + v2ResourceCPU, + v3ResourceGPU, + v2ResourceMemory, + v2ResourceStorage, + v2ResourceStorageArray, + v2Sdl, + v2Service, + v2ServiceExposeHttpOptions, + v3ServiceExposeHttpOptions, + v2ManifestServiceParams, + v3GPUAttributes, + v3Sdl, + v3ProfileCompute, + v3ComputeResources, + v2ServiceParams, + v3DeploymentGroup, + v3ManifestServiceParams, + v2StorageAttributes, + v2ServiceImageCredentials +} from "./../types"; +import { convertCpuResourceString, convertResourceString } from "./../sizes"; +import { default as stableStringify } from "json-stable-stringify"; +import crypto from "node:crypto"; +import { AKT_DENOM, MAINNET_ID, USDC_IBC_DENOMS } from "../../config/network"; +import { NetworkId } from "../../types/network"; +import { SdlValidationError } from "../../error"; + +const Endpoint_SHARED_HTTP = 0; +const Endpoint_RANDOM_PORT = 1; +const Endpoint_LEASED_IP = 2; +export const GPU_SUPPORTED_VENDORS = ["nvidia", "amd"]; +export const GPU_SUPPORTED_INTERFACES = ["pcie", "sxm"]; + +function isArray(obj: any): obj is Array { + return Array.isArray(obj); +} + +function isString(str: any): str is string { + return typeof str === "string"; +} + +type NetworkVersion = "beta2" | "beta3"; + +export class SDL { + static fromString(yaml: string, version: NetworkVersion = "beta2", networkId: NetworkId = MAINNET_ID) { + const data = YAML.load(yaml) as v3Sdl; + return new SDL(data, version, networkId); + } + + static validate(yaml: string) { + console.warn("SDL.validate is deprecated. Use SDL.constructor directly."); + // TODO: this should really be cast to unknown, then assigned + // to v2 or v3 SDL only after being validated + const data = YAML.load(yaml) as v3Sdl; + + for (const [name, profile] of Object.entries(data.profiles.compute || {})) { + this.validateGPU(name, profile.resources.gpu); + this.validateStorage(name, profile.resources.storage); + } + + return data; + } + + static validateGPU(name: string, gpu: v3ResourceGPU | undefined) { + if (gpu) { + if (typeof gpu.units === "undefined") { + throw new Error("GPU units must be specified for profile " + name); + } + + const units = parseInt(gpu.units.toString()); + + if (units === 0 && gpu.attributes !== undefined) { + throw new Error("GPU must not have attributes if units is 0"); + } + + if (units > 0 && gpu.attributes === undefined) { + throw new Error("GPU must have attributes if units is not 0"); + } + + if (units > 0 && gpu.attributes?.vendor === undefined) { + throw new Error("GPU must specify a vendor if units is not 0"); + } + + if (units > 0 && !GPU_SUPPORTED_VENDORS.some(vendor => vendor in (gpu.attributes?.vendor || {}))) { + throw new Error(`GPU must be one of the supported vendors (${GPU_SUPPORTED_VENDORS.join(",")}).`); + } + + const vendor: string = Object.keys(gpu.attributes?.vendor || {})[0]; + + if (units > 0 && !!gpu.attributes?.vendor[vendor] && !Array.isArray(gpu.attributes.vendor[vendor])) { + throw new Error(`GPU configuration must be an array of GPU models with optional ram.`); + } + + if ( + units > 0 && + Object.values(gpu.attributes?.vendor || {}).some(models => + models?.some(model => model.interface && !GPU_SUPPORTED_INTERFACES.includes(model.interface)) + ) + ) { + throw new Error(`GPU interface must be one of the supported interfaces (${GPU_SUPPORTED_INTERFACES.join(",")}).`); + } + } + } + + static validateStorage(name: string, storage?: v2ResourceStorage | v2ResourceStorageArray) { + if (!storage) { + throw new Error("Storage is required for service " + name); + } + + const storages = isArray(storage) ? storage : [storage]; + + for (const storage of storages) { + if (typeof storage.size === "undefined") { + throw new Error("Storage size is required for service " + name); + } + + if (storage.attributes) { + for (const [key, value] of Object.entries(storage.attributes)) { + if (key === "class" && value === "ram" && storage.attributes.persistent === true) { + throw new Error("Storage attribute 'ram' must have 'persistent' set to 'false' or not defined for service " + name); + } + } + } + } + } + + private readonly ENDPOINT_NAME_VALIDATION_REGEX = /^[a-z]+[-_\da-z]+$/; + + private readonly ENDPOINT_KIND_IP = "ip"; + + private readonly endpointsUsed = new Set(); + + private readonly portsUsed = new Map(); + + constructor( + public readonly data: v2Sdl, + public readonly version: NetworkVersion = "beta2", + private readonly networkId: NetworkId = MAINNET_ID + ) { + this.validate(); + } + + private validate() { + // TODO: this should really be cast to unknown, then assigned + // to v2 or v3 SDL only after being validated + const v3data = this.data as v3Sdl; + Object.entries(v3data.profiles.compute || {}).forEach(([name, { resources }]) => { + if ("gpu" in resources) { + SDL.validateGPU(name, resources.gpu); + } + SDL.validateStorage(name, resources.storage); + }); + + this.validateEndpoints(); + + Object.keys(this.data.services).forEach(serviceName => { + this.validateDeploymentWithRelations(serviceName); + this.validateLeaseIP(serviceName); + this.validateCredentials(serviceName); + }); + + this.validateDenom(); + this.validateEndpointsUtility(); + } + + private validateDenom() { + const usdcDenom = USDC_IBC_DENOMS[this.networkId]; + const denoms = this.groups() + .flatMap(g => g.resources) + .map(resource => resource.price.denom); + const invalidDenom = denoms.find(denom => denom !== AKT_DENOM && denom !== usdcDenom); + + SdlValidationError.assert(!invalidDenom, `Invalid denom: "${invalidDenom}". Only uakt and ${usdcDenom} are supported.`); + } + + private validateEndpoints() { + if (!this.data.endpoints) { + return; + } + + Object.keys(this.data.endpoints).forEach(endpointName => { + const endpoint = this.data.endpoints[endpointName]; + SdlValidationError.assert(this.ENDPOINT_NAME_VALIDATION_REGEX.test(endpointName), `Endpoint named "${endpointName}" is not a valid name.`); + SdlValidationError.assert(!!endpoint.kind, `Endpoint named "${endpointName}" has no kind.`); + SdlValidationError.assert(endpoint.kind === this.ENDPOINT_KIND_IP, `Endpoint named "${endpointName}" has an unknown kind "${endpoint.kind}".`); + }); + } + + private validateCredentials(serviceName: string) { + const { credentials } = this.data.services[serviceName]; + + if (credentials) { + const credentialsKeys: (keyof v2ServiceImageCredentials)[] = ["host", "username", "password"]; + credentialsKeys.forEach(key => { + SdlValidationError.assert(credentials[key]?.trim().length, `service "${serviceName}" credentials missing "${key}"`); + }); + } + } + + private validateDeploymentWithRelations(serviceName: string) { + const deployment = this.data.deployment[serviceName]; + SdlValidationError.assert(deployment, `Service "${serviceName}" is not defined in the "deployment" section.`); + + Object.keys(this.data.deployment[serviceName]).forEach(deploymentName => { + const serviceDeployment = this.data.deployment[serviceName][deploymentName]; + const compute = this.data.profiles.compute?.[serviceDeployment.profile]; + const infra = this.data.profiles.placement?.[deploymentName]; + + SdlValidationError.assert(infra, `The placement "${deploymentName}" is not defined in the "placement" section.`); + SdlValidationError.assert( + infra.pricing?.[serviceDeployment.profile], + `The pricing for the "${serviceDeployment.profile}" profile is not defined in the "${deploymentName}" "placement" definition.` + ); + SdlValidationError.assert(compute, `The compute requirements for the "${serviceDeployment.profile}" profile are not defined in the "compute" section.`); + }); + } + + private validateLeaseIP(serviceName: string) { + this.data.services[serviceName].expose?.forEach(expose => { + const proto = this.parseServiceProto(expose.proto); + + expose.to?.forEach(to => { + if (to.ip?.length > 0) { + SdlValidationError.assert(to.global, `Error on "${serviceName}", if an IP is declared, the directive must be declared as global.`); + + if (!this.data.endpoints?.[to.ip]) { + throw new SdlValidationError(`Unknown endpoint "${to.ip}" in service "${serviceName}". Add to the list of endpoints in the "endpoints" section.`); + } + + this.endpointsUsed.add(to.ip); + + const portKey = `${to.ip}-${expose.as}-${proto}`; + const otherServiceName = this.portsUsed.get(portKey); + SdlValidationError.assert( + !this.portsUsed.has(portKey), + `IP endpoint ${to.ip} port: ${expose.port} protocol: ${proto} specified by service ${serviceName} already in use by ${otherServiceName}` + ); + this.portsUsed.set(portKey, serviceName); + } + }); + }); + } + + private validateEndpointsUtility() { + if (this.data.endpoints) { + Object.keys(this.data.endpoints).forEach(endpoint => { + SdlValidationError.assert(this.endpointsUsed.has(endpoint), `Endpoint ${endpoint} declared but never used.`); + }); + } + } + + services() { + if (this.data) { + return this.data.services; + } + + return {}; + } + + deployments() { + if (this.data) { + return this.data.deployment; + } + + return {}; + } + + profiles() { + if (this.data) { + return this.data.profiles; + } + + return {}; + } + + placements() { + const { placement } = this.data.profiles; + + return placement || {}; + } + + serviceNames() { + const names = this.data ? Object.keys(this.data.services) : []; + + // TODO: sort these + return names; + } + + deploymentsByPlacement(placement: string) { + const deployments = this.data ? this.data.deployment : []; + + return Object.entries(deployments as object).filter(({ 1: deployment }) => Object.prototype.hasOwnProperty.call(deployment, placement)); + } + + resourceUnit(val: string, asString: boolean) { + return asString ? { val: `${convertResourceString(val)}` } : { val: convertResourceString(val) }; + } + + resourceValue(value: { toString: () => string } | null, asString: boolean) { + if (value === null) { + return value; + } + + const strVal = value.toString(); + const encoder = new TextEncoder(); + + return asString ? strVal : encoder.encode(strVal); + } + + serviceResourceCpu(resource: v2ResourceCPU) { + const units = isString(resource.units) ? convertCpuResourceString(resource.units) : resource.units * 1000; + + return resource.attributes + ? { + units: { val: `${units}` }, + attributes: this.serviceResourceAttributes(resource.attributes) + } + : { + units: { val: `${units}` } + }; + } + + serviceResourceMemory(resource: v2ResourceMemory, asString: boolean) { + const key = asString ? "quantity" : "size"; + + return resource.attributes + ? { + [key]: this.resourceUnit(resource.size, asString), + attributes: this.serviceResourceAttributes(resource.attributes) + } + : { + [key]: this.resourceUnit(resource.size, asString) + }; + } + + serviceResourceStorage(resource: v2ResourceStorageArray | v2ResourceStorage, asString: boolean) { + const key = asString ? "quantity" : "size"; + const storage = isArray(resource) ? resource : [resource]; + + return storage.map(storage => + storage.attributes + ? { + name: storage.name || "default", + [key]: this.resourceUnit(storage.size, asString), + attributes: this.serviceResourceStorageAttributes(storage.attributes) + } + : { + name: storage.name || "default", + [key]: this.resourceUnit(storage.size, asString) + } + ); + } + + serviceResourceAttributes(attributes?: Record) { + return ( + attributes && + Object.keys(attributes) + .sort() + .map(key => ({ key, value: attributes[key].toString() })) + ); + } + + serviceResourceStorageAttributes(attributes?: v2StorageAttributes) { + if (!attributes) return undefined; + + const pairs = Object.keys(attributes).map(key => ({ key, value: attributes[key].toString() })); + + if (attributes.class === "ram" && !("persistent" in attributes)) { + pairs.push({ key: "persistent", value: "false" }); + } + + pairs.sort((a, b) => a.key.localeCompare(b.key)); + + return pairs; + } + + serviceResourceGpu(resource: v3ResourceGPU | undefined, asString: boolean) { + const value = resource?.units || 0; + const numVal = isString(value) ? Buffer.from(value, "ascii") : value; + const strVal = !isString(value) ? value.toString() : value; + + return resource?.attributes + ? { + units: asString ? { val: strVal } : { val: numVal }, + attributes: this.transformGpuAttributes(resource?.attributes) + } + : { + units: asString ? { val: strVal } : { val: numVal } + }; + } + + v2ServiceResourceEndpoints(service: v2Service) { + const endpointSequenceNumbers = this.computeEndpointSequenceNumbers(this.data); + const endpoints = service.expose.flatMap(expose => + expose.to + ? expose.to + .filter(to => to.global && to.ip?.length > 0) + .map(to => ({ + kind: Endpoint_LEASED_IP, + sequence_number: endpointSequenceNumbers[to.ip] || 0 + })) + : [] + ); + + return endpoints.length > 0 ? endpoints : null; + } + + v3ServiceResourceEndpoints(service: v2Service) { + const endpointSequenceNumbers = this.computeEndpointSequenceNumbers(this.data); + const endpoints = service.expose.flatMap(expose => + expose.to + ? expose.to + .filter(to => to.global) + .flatMap(to => { + const exposeSpec = { + port: expose.port, + externalPort: expose.as || 0, + proto: this.parseServiceProto(expose.proto), + global: !!to.global + }; + + const kind = this.exposeShouldBeIngress(exposeSpec) ? Endpoint_SHARED_HTTP : Endpoint_RANDOM_PORT; + + const defaultEp = kind !== 0 ? { kind: kind, sequence_number: 0 } : { sequence_number: 0 }; + + const leasedEp = + to.ip?.length > 0 + ? { + kind: Endpoint_LEASED_IP, + sequence_number: endpointSequenceNumbers[to.ip] || 0 + } + : undefined; + + return leasedEp ? [defaultEp, leasedEp] : [defaultEp]; + }) + : [] + ); + + return endpoints; + } + + serviceResourcesBeta2(profile: v2ProfileCompute, service: v2Service, asString: boolean = false) { + return { + cpu: this.serviceResourceCpu(profile.resources.cpu), + memory: this.serviceResourceMemory(profile.resources.memory, asString), + storage: this.serviceResourceStorage(profile.resources.storage, asString), + endpoints: this.v2ServiceResourceEndpoints(service) + }; + } + + serviceResourcesBeta3(id: number, profile: v3ProfileCompute, service: v2Service, asString: boolean = false) { + return { + id: id, + cpu: this.serviceResourceCpu(profile.resources.cpu), + memory: this.serviceResourceMemory(profile.resources.memory, asString), + storage: this.serviceResourceStorage(profile.resources.storage, asString), + endpoints: this.v3ServiceResourceEndpoints(service), + gpu: this.serviceResourceGpu(profile.resources.gpu, asString) + }; + } + + parseServiceProto(proto?: string) { + const raw = proto?.toUpperCase(); + let result = "TCP"; + + switch (raw) { + case "TCP": + case "": + case undefined: + result = "TCP"; + break; + case "UDP": + result = "UDP"; + break; + default: + throw new Error("ErrUnsupportedServiceProtocol"); + } + + return result; + } + + manifestExposeService(to: v2ExposeTo) { + return to.service || ""; + } + + manifestExposeGlobal(to: v2ExposeTo) { + return to.global || false; + } + + manifestExposeHosts(expose: v2Expose) { + return expose.accept || null; + } + + v2HttpOptions(http_options: v2HTTPOptions | undefined) { + const defaults = { + MaxBodySize: 1048576, + ReadTimeout: 60000, + SendTimeout: 60000, + NextTries: 3, + NextTimeout: 0, + NextCases: ["error", "timeout"] + }; + + if (!http_options) { + return { ...defaults }; + } + + return { + MaxBodySize: http_options.max_body_size || defaults.MaxBodySize, + ReadTimeout: http_options.read_timeout || defaults.ReadTimeout, + SendTimeout: http_options.send_timeout || defaults.SendTimeout, + NextTries: http_options.next_tries || defaults.NextTries, + NextTimeout: http_options.next_timeout || defaults.NextTimeout, + NextCases: http_options.next_cases || defaults.NextCases + }; + } + + v3HttpOptions(http_options: v2HTTPOptions | undefined) { + const defaults = { + maxBodySize: 1048576, + readTimeout: 60000, + sendTimeout: 60000, + nextTries: 3, + nextTimeout: 0, + nextCases: ["error", "timeout"] + }; + + if (!http_options) { + return { ...defaults }; + } + + return { + maxBodySize: http_options.max_body_size || defaults.maxBodySize, + readTimeout: http_options.read_timeout || defaults.readTimeout, + sendTimeout: http_options.send_timeout || defaults.sendTimeout, + nextTries: http_options.next_tries || defaults.nextTries, + nextTimeout: http_options.next_timeout || defaults.nextTimeout, + nextCases: http_options.next_cases || defaults.nextCases + }; + } + + v2ManifestExposeHttpOptions(expose: v2Expose): v2ServiceExposeHttpOptions { + return this.v2HttpOptions(expose.http_options); + } + + v3ManifestExposeHttpOptions(expose: v2Expose): v3ServiceExposeHttpOptions { + return this.v3HttpOptions(expose.http_options); + } + + v2ManifestExpose(service: v2Service): v2ServiceExpose[] { + const endpointSequenceNumbers = this.computeEndpointSequenceNumbers(this.data); + return service.expose.flatMap(expose => + expose.to + ? expose.to.map(to => ({ + Port: expose.port, + ExternalPort: expose.as || 0, + Proto: this.parseServiceProto(expose.proto), + Service: this.manifestExposeService(to), + Global: this.manifestExposeGlobal(to), + Hosts: this.manifestExposeHosts(expose), + HTTPOptions: this.v2ManifestExposeHttpOptions(expose), + IP: to.ip || "", + EndpointSequenceNumber: endpointSequenceNumbers[to.ip] || 0 + })) + : [] + ); + } + + v3ManifestExpose(service: v2Service): v3ServiceExpose[] { + const endpointSequenceNumbers = this.computeEndpointSequenceNumbers(this.data); + return service.expose + .flatMap(expose => + expose.to + ? expose.to.map(to => ({ + port: expose.port, + externalPort: expose.as || 0, + proto: this.parseServiceProto(expose.proto), + service: this.manifestExposeService(to), + global: this.manifestExposeGlobal(to), + hosts: this.manifestExposeHosts(expose), + httpOptions: this.v3ManifestExposeHttpOptions(expose), + ip: to.ip || "", + endpointSequenceNumber: endpointSequenceNumbers[to.ip] || 0 + })) + : [] + ) + .sort((a, b) => { + if (a.service != b.service) return a.service.localeCompare(b.service); + if (a.port != b.port) return a.port - b.port; + if (a.proto != b.proto) return a.proto.localeCompare(b.proto); + if (a.global != b.global) return a.global ? -1 : 1; + + return 0; + }); + } + + v2ManifestServiceParams(params: v2ServiceParams): v2ManifestServiceParams | undefined { + return { + Storage: Object.keys(params?.storage ?? {}).map(name => { + if (!params?.storage) throw new Error("Storage is undefined"); + return { + name: name, + mount: params.storage[name].mount, + readOnly: params.storage[name].readOnly || false + }; + }) + }; + } + + v3ManifestServiceParams(params: v2ServiceParams | undefined): v3ManifestServiceParams | null { + if (params === undefined) { + return null; + } + + return { + storage: Object.keys(params?.storage ?? {}).map(name => { + if (!params?.storage) throw new Error("Storage is undefined"); + return { + name: name, + mount: params.storage[name].mount, + readOnly: params.storage[name].readOnly || false + }; + }) + }; + } + + v2ManifestService(placement: string, name: string, asString: boolean): v2ManifestService { + const service = this.data.services[name]; + const deployment = this.data.deployment[name]; + const profile = this.data.profiles.compute[deployment[placement].profile]; + + const manifestService: v2ManifestService = { + Name: name, + Image: service.image, + Command: service.command || null, + Args: service.args || null, + Env: service.env || null, + Resources: this.serviceResourcesBeta2(profile, service, asString), + Count: deployment[placement].count, + Expose: this.v2ManifestExpose(service) + }; + + if (service.params) { + manifestService.params = this.v2ManifestServiceParams(service.params); + } + + return manifestService; + } + + v3ManifestService(id: number, placement: string, name: string, asString: boolean): v3ManifestService { + const service = this.data.services[name]; + const deployment = this.data.deployment[name]; + const profile = this.data.profiles.compute[deployment[placement].profile]; + + return { + name: name, + image: service.image, + command: service.command || null, + args: service.args || null, + env: service.env || null, + resources: this.serviceResourcesBeta3(id, profile as v3ProfileCompute, service, asString), + count: deployment[placement].count, + expose: this.v3ManifestExpose(service), + params: this.v3ManifestServiceParams(service.params), + credentials: service.credentials || null + }; + } + + v2Manifest(asString: boolean = false): v2Manifest { + return Object.keys(this.placements()).map(name => ({ + Name: name, + Services: this.deploymentsByPlacement(name).map(([service]) => this.v2ManifestService(name, service, asString)) + })); + } + + v3Manifest(asString: boolean = false): v3Manifest { + const groups = this.v3Groups(); + const serviceId = (pIdx: number, sIdx: number) => groups[pIdx].resources[sIdx].resource.id; + + return Object.keys(this.placements()).map((name, pIdx) => ({ + name: name, + services: this.deploymentsByPlacement(name) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([service], idx) => this.v3ManifestService(serviceId(pIdx, idx), name, service, asString)) + })); + } + + manifest(asString: boolean = false): v2Manifest | v3Manifest { + return this.version === "beta2" ? this.v2Manifest(asString) : this.v3Manifest(asString); + } + + computeEndpointSequenceNumbers(sdl: v2Sdl) { + return Object.fromEntries( + Object.values(sdl.services).flatMap(service => + service.expose.flatMap(expose => + expose.to + ? expose.to + .filter(to => to.global && to.ip?.length > 0) + .map(to => to.ip) + .sort() + .map((ip, index) => [ip, index + 1]) + : [] + ) + ) + ); + } + + resourceUnitCpu(computeResources: v2ComputeResources, asString: boolean) { + const attributes = computeResources.cpu.attributes; + const cpu = isString(computeResources.cpu.units) ? convertCpuResourceString(computeResources.cpu.units) : computeResources.cpu.units * 1000; + + return { + units: { val: this.resourceValue(cpu, asString) }, + attributes: + attributes && + Object.entries(attributes) + .sort(([k0], [k1]) => k0.localeCompare(k1)) + .map(([key, value]) => ({ + key: key, + value: value.toString() + })) + }; + } + + resourceUnitMemory(computeResources: v2ComputeResources, asString: boolean) { + const attributes = computeResources.memory.attributes; + + return { + quantity: { + val: this.resourceValue(convertResourceString(computeResources.memory.size), asString) + }, + attributes: + attributes && + Object.entries(attributes) + .sort(([k0], [k1]) => k0.localeCompare(k1)) + .map(([key, value]) => ({ + key: key, + value: value.toString() + })) + }; + } + + resourceUnitStorage(computeResources: v2ComputeResources, asString: boolean) { + const storages = isArray(computeResources.storage) ? computeResources.storage : [computeResources.storage]; + + return storages.map(storage => ({ + name: storage.name || "default", + quantity: { + val: this.resourceValue(convertResourceString(storage.size), asString) + }, + attributes: this.serviceResourceStorageAttributes(storage.attributes) + })); + } + + transformGpuAttributes(attributes: v3GPUAttributes): Array<{ key: string; value: string }> { + return Object.entries(attributes.vendor).flatMap(([vendor, models]) => + models + ? models.map(model => { + let key = `vendor/${vendor}/model/${model.model}`; + + if (model.ram) { + key += `/ram/${model.ram}`; + } + + if (model.interface) { + key += `/interface/${model.interface}`; + } + + return { + key: key, + value: "true" + }; + }) + : [ + { + key: `vendor/${vendor}/model/*`, + value: "true" + } + ] + ); + } + + resourceUnitGpu(computeResources: v3ComputeResources, asString: boolean) { + const attributes = computeResources.gpu?.attributes; + const units = computeResources.gpu?.units || "0"; + const gpu = isString(units) ? parseInt(units) : units; + + return { + units: { val: this.resourceValue(gpu, asString) }, + attributes: attributes && this.transformGpuAttributes(attributes) + }; + } + + groupResourceUnits(resource: v2ComputeResources | undefined, asString: boolean) { + if (!resource) return {}; + + const units = { + endpoints: null + } as any; + + if (resource.cpu) { + units.cpu = this.resourceUnitCpu(resource, asString); + } + + if (resource.memory) { + units.memory = this.resourceUnitMemory(resource, asString); + } + + if (resource.storage) { + units.storage = this.resourceUnitStorage(resource, asString); + } + + if (this.version === "beta3") { + units.gpu = this.resourceUnitGpu(resource as v3ComputeResources, asString); + } + + return units; + } + + exposeShouldBeIngress(expose: { proto: string; global: boolean; externalPort: number; port: number }) { + const externalPort = expose.externalPort === 0 ? expose.port : expose.externalPort; + + return expose.global && expose.proto === "TCP" && externalPort === 80; + } + + groups() { + return this.version === "beta2" ? this.v2Groups() : this.v3Groups(); + } + + v3Groups() { + const groups = new Map< + string, + { + dgroup: v3DeploymentGroup; + boundComputes: Record>; + } + >(); + const services = Object.entries(this.data.services).sort(([a], [b]) => a.localeCompare(b)); + + for (const [svcName, service] of services) { + for (const [placementName, svcdepl] of Object.entries(this.data.deployment[svcName])) { + // objects below have been ensured to exist + const compute = this.data.profiles.compute[svcdepl.profile]; + const infra = this.data.profiles.placement[placementName]; + const pricing = infra.pricing[svcdepl.profile]; + const price = { + ...pricing, + amount: pricing.amount?.toString() + }; + + let group = groups.get(placementName); + + if (!group) { + const attributes = (infra.attributes + ? Object.entries(infra.attributes).map(([key, value]) => ({ + key, + value + })) + : []) as unknown as Array<{ key: string; value: string }>; + + attributes.sort((a, b) => a.key.localeCompare(b.key)); + + group = { + dgroup: { + name: placementName, + resources: [], + requirements: { + attributes: attributes, + signedBy: { + allOf: infra.signedBy?.allOf || [], + anyOf: infra.signedBy?.anyOf || [] + } + } + }, + boundComputes: {} + }; + + groups.set(placementName, group); + } + + if (!group.boundComputes[placementName]) { + group.boundComputes[placementName] = {}; + } + + // const resources = this.serviceResourcesBeta3(0, compute as v3ProfileCompute, service, false); + const location = group.boundComputes[placementName][svcdepl.profile]; + + if (!location) { + const res = this.groupResourceUnits(compute.resources, false); + res.endpoints = this.v3ServiceResourceEndpoints(service); + + const resID = group.dgroup.resources.length > 0 ? group.dgroup.resources.length + 1 : 1; + res.id = resID; + // resources.id = res.id; + + group.dgroup.resources.push({ + resource: res, + price: price, + count: svcdepl.count + } as any); + + group.boundComputes[placementName][svcdepl.profile] = group.dgroup.resources.length - 1; + } else { + const endpoints = this.v3ServiceResourceEndpoints(service); + // resources.id = group.dgroup.resources[location].id; + + group.dgroup.resources[location].count += svcdepl.count; + group.dgroup.resources[location].endpoints += endpoints as any; + group.dgroup.resources[location].endpoints.sort(); + } + } + } + + // keep ordering stable + const names: string[] = [...groups.keys()].sort(); + return names.map(name => groups.get(name)).map(group => (group ? (group.dgroup as typeof group.dgroup) : {})) as Array; + } + + v2Groups() { + const yamlJson = this.data; + const ipEndpointNames = this.computeEndpointSequenceNumbers(yamlJson); + + const groups = {} as any; + + Object.keys(yamlJson.services).forEach(svcName => { + const svc = yamlJson.services[svcName]; + const depl = yamlJson.deployment[svcName]; + + Object.keys(depl).forEach(placementName => { + const svcdepl = depl[placementName]; + const compute = yamlJson.profiles.compute[svcdepl.profile]; + const infra = yamlJson.profiles.placement[placementName]; + + const pricing = infra.pricing[svcdepl.profile]; + const price = { + ...pricing, + amount: pricing.amount.toString() + }; + + let group = groups[placementName]; + + if (!group) { + group = { + name: placementName, + requirements: { + attributes: infra.attributes + ? Object.entries(infra.attributes).map(([key, value]) => ({ + key, + value + })) + : [], + signedBy: { + allOf: infra.signedBy?.allOf || [], + anyOf: infra.signedBy?.anyOf || [] + } + }, + resources: [] + }; + + if (group.requirements.attributes) { + group.requirements.attributes = group.requirements.attributes.sort((a: any, b: any) => a.key < b.key); + } + + groups[group.name] = group; + } + + const resources = { + resources: this.groupResourceUnits(compute.resources, false), // Changed resources => unit + price: price, + count: svcdepl.count + }; + + const endpoints = [] as any[]; + svc?.expose?.forEach(expose => { + expose?.to + ?.filter(to => to.global) + .forEach(to => { + const exposeSpec = { + port: expose.port, + externalPort: expose.as || 0, + proto: this.parseServiceProto(expose.proto), + global: !!to.global + }; + + if (to.ip?.length > 0) { + const seqNo = ipEndpointNames[to.ip]; + endpoints.push({ + kind: Endpoint_LEASED_IP, + sequence_number: seqNo + }); + } + + const kind = this.exposeShouldBeIngress(exposeSpec) ? Endpoint_SHARED_HTTP : Endpoint_RANDOM_PORT; + + endpoints.push({ kind: kind, sequence_number: 0 }); + }); + }); + + resources.resources.endpoints = endpoints; + group.resources.push(resources); + }); + }); + + return Object.keys(groups) + .sort((a, b) => (a < b ? 1 : 0)) + .map(name => groups[name]); + } + + escapeHtml(raw: string) { + return raw.replace(//g, "\\u003e").replace(/&/g, "\\u0026"); + } + + SortJSON(jsonStr: string) { + return this.escapeHtml(stableStringify(JSON.parse(jsonStr))); + } + + manifestSortedJSON() { + const manifest = this.manifest(true); + let jsonStr = JSON.stringify(manifest); + + if (jsonStr) { + jsonStr = jsonStr.replaceAll('"quantity":{"val', '"size":{"val'); + } + + return this.SortJSON(jsonStr); + } + + async manifestVersion() { + const jsonStr = this.manifestSortedJSON(); + const enc = new TextEncoder(); + const sortedBytes = enc.encode(jsonStr); + const sum = await crypto.subtle.digest("SHA-256", sortedBytes); + + return new Uint8Array(sum); + } + + manifestSorted() { + const sorted = this.manifestSortedJSON(); + return JSON.parse(sorted); + } +} diff --git a/src/sdl/index.ts b/src/sdl/index.ts index fafed34..fe6a0c7 100644 --- a/src/sdl/index.ts +++ b/src/sdl/index.ts @@ -1,934 +1 @@ -import YAML from "js-yaml"; -import { - v2Manifest, - v3Manifest, - v2ManifestService, - v3ManifestService, - v2ServiceExpose, - v3ServiceExpose, - v2ComputeResources, - v2Expose, - v2ExposeTo, - v2HTTPOptions, - v2ProfileCompute, - v2ResourceCPU, - v3ResourceGPU, - v2ResourceMemory, - v2ResourceStorage, - v2ResourceStorageArray, - v2Sdl, - v2Service, - v2ServiceExposeHttpOptions, - v3ServiceExposeHttpOptions, - v2ManifestServiceParams, - v3GPUAttributes, - v3Sdl, - v3ProfileCompute, - v3ComputeResources, - v2ServiceParams, - v3DeploymentGroup, - v3ManifestServiceParams, - v2StorageAttributes -} from "./types"; -import { convertCpuResourceString, convertResourceString } from "./sizes"; -import { default as stableStringify } from "json-stable-stringify"; -import crypto from "node:crypto"; - -const Endpoint_SHARED_HTTP = 0; -const Endpoint_RANDOM_PORT = 1; -const Endpoint_LEASED_IP = 2; -export const GPU_SUPPORTED_VENDORS = ["nvidia", "amd"]; -export const GPU_SUPPORTED_INTERFACES = ["pcie", "sxm"]; - -function isArray(obj: any): obj is Array { - return Array.isArray(obj); -} - -function isString(str: any): str is string { - return typeof str === "string"; -} - -type NetworkVersion = "beta2" | "beta3"; - -export class SDL { - data: v2Sdl; - version: NetworkVersion; - - constructor(data: v2Sdl, version: NetworkVersion = "beta2") { - this.data = data; - this.version = version; - } - - static fromString(yaml: string, version: NetworkVersion = "beta2") { - const data = SDL.validate(yaml) as v2Sdl; - - return new SDL(data, version); - } - - static validate(yaml: string) { - // TODO: this should really be cast to unknown, then assigned - // to v2 or v3 SDL only after being validated - const data = YAML.load(yaml) as v3Sdl; - - for (const [name, profile] of Object.entries(data.profiles.compute)) { - SDL.validateGPU(name, profile.resources.gpu); - SDL.validateStorage(name, profile.resources.storage); - } - - return data; - } - - static validateGPU(name: string, gpu: v3ResourceGPU | undefined) { - if (gpu) { - if (typeof gpu.units === "undefined") { - console.log(JSON.stringify(gpu, null, 2)); - throw new Error("GPU units must be specified for profile " + name); - } - - const units = parseInt(gpu.units.toString()); - - if (units === 0 && gpu.attributes !== undefined) { - throw new Error("GPU must not have attributes if units is 0"); - } - - if (units > 0 && gpu.attributes === undefined) { - throw new Error("GPU must have attributes if units is not 0"); - } - - if (units > 0 && gpu.attributes?.vendor === undefined) { - throw new Error("GPU must specify a vendor if units is not 0"); - } - - if (units > 0 && !GPU_SUPPORTED_VENDORS.some(vendor => vendor in (gpu.attributes?.vendor || {}))) { - throw new Error(`GPU must be one of the supported vendors (${GPU_SUPPORTED_VENDORS.join(",")}).`); - } - - const vendor: string = Object.keys(gpu.attributes?.vendor || {})[0]; - - if (units > 0 && !!gpu.attributes?.vendor[vendor] && !Array.isArray(gpu.attributes.vendor[vendor])) { - throw new Error(`GPU configuration must be an array of GPU models with optional ram.`); - } - - if ( - units > 0 && - Object.values(gpu.attributes?.vendor || {}).some(models => - models?.some(model => model.interface && !GPU_SUPPORTED_INTERFACES.includes(model.interface)) - ) - ) { - throw new Error(`GPU interface must be one of the supported interfaces (${GPU_SUPPORTED_INTERFACES.join(",")}).`); - } - } - } - - static validateStorage(name: string, storage?: v2ResourceStorage | v2ResourceStorageArray) { - if (!storage) { - throw new Error("Storage is required for service " + name); - } - - const storages = isArray(storage) ? storage : [storage]; - - for (const storage of storages) { - if (typeof storage.size === "undefined") { - throw new Error("Storage size is required for service " + name); - } - - if (storage.attributes) { - for (const [key, value] of Object.entries(storage.attributes)) { - if (key === "class" && value === "ram" && storage.attributes.persistent === true) { - throw new Error("Storage attribute 'ram' must have 'persistent' set to 'false' or not defined for service " + name); - } - } - } - } - } - - services() { - if (this.data) { - return this.data.services; - } - - return {}; - } - - deployments() { - if (this.data) { - return this.data.deployment; - } - - return {}; - } - - profiles() { - if (this.data) { - return this.data.profiles; - } - - return {}; - } - - placements() { - const { placement } = this.data.profiles; - - return placement || {}; - } - - serviceNames() { - const names = this.data ? Object.keys(this.data.services) : []; - - // TODO: sort these - return names; - } - - deploymentsByPlacement(placement: string) { - const deployments = this.data ? this.data.deployment : []; - - return Object.entries(deployments as object).filter(({ 1: deployment }) => Object.prototype.hasOwnProperty.call(deployment, placement)); - } - - resourceUnit(val: string, asString: boolean) { - return asString ? { val: `${convertResourceString(val)}` } : { val: convertResourceString(val) }; - } - - resourceValue(value: { toString: () => string } | null, asString: boolean) { - if (value === null) { - return value; - } - - const strVal = value.toString(); - const encoder = new TextEncoder(); - - return asString ? strVal : encoder.encode(strVal); - } - - serviceResourceCpu(resource: v2ResourceCPU) { - const units = isString(resource.units) ? convertCpuResourceString(resource.units) : resource.units * 1000; - - return resource.attributes - ? { - units: { val: `${units}` }, - attributes: this.serviceResourceAttributes(resource.attributes) - } - : { - units: { val: `${units}` } - }; - } - - serviceResourceMemory(resource: v2ResourceMemory, asString: boolean) { - const key = asString ? "quantity" : "size"; - - return resource.attributes - ? { - [key]: this.resourceUnit(resource.size, asString), - attributes: this.serviceResourceAttributes(resource.attributes) - } - : { - [key]: this.resourceUnit(resource.size, asString) - }; - } - - serviceResourceStorage(resource: v2ResourceStorageArray | v2ResourceStorage, asString: boolean) { - const key = asString ? "quantity" : "size"; - const storage = isArray(resource) ? resource : [resource]; - - return storage.map(storage => - storage.attributes - ? { - name: storage.name || "default", - [key]: this.resourceUnit(storage.size, asString), - attributes: this.serviceResourceStorageAttributes(storage.attributes) - } - : { - name: storage.name || "default", - [key]: this.resourceUnit(storage.size, asString) - } - ); - } - - serviceResourceAttributes(attributes?: Record) { - return ( - attributes && - Object.keys(attributes) - .sort() - .map(key => ({ key, value: attributes[key].toString() })) - ); - } - - serviceResourceStorageAttributes(attributes?: v2StorageAttributes) { - if (!attributes) return undefined; - - const pairs = Object.keys(attributes).map(key => ({ key, value: attributes[key].toString() })); - - if (attributes.class === "ram" && !("persistent" in attributes)) { - pairs.push({ key: "persistent", value: "false" }); - } - - pairs.sort((a, b) => a.key.localeCompare(b.key)); - - return pairs; - } - - serviceResourceGpu(resource: v3ResourceGPU | undefined, asString: boolean) { - const value = resource?.units || 0; - const numVal = isString(value) ? Buffer.from(value, "ascii") : value; - const strVal = !isString(value) ? value.toString() : value; - - return resource?.attributes - ? { - units: asString ? { val: strVal } : { val: numVal }, - attributes: this.transformGpuAttributes(resource?.attributes) - } - : { - units: asString ? { val: strVal } : { val: numVal } - }; - } - - v2ServiceResourceEndpoints(service: v2Service) { - const endpointSequenceNumbers = this.computeEndpointSequenceNumbers(this.data); - const endpoints = service.expose.flatMap(expose => - expose.to - ? expose.to - .filter(to => to.global && to.ip?.length > 0) - .map(to => ({ - kind: Endpoint_LEASED_IP, - sequence_number: endpointSequenceNumbers[to.ip] || 0 - })) - : [] - ); - - return endpoints.length > 0 ? endpoints : null; - } - - v3ServiceResourceEndpoints(service: v2Service) { - const endpointSequenceNumbers = this.computeEndpointSequenceNumbers(this.data); - const endpoints = service.expose.flatMap(expose => - expose.to - ? expose.to - .filter(to => to.global) - .flatMap(to => { - const exposeSpec = { - port: expose.port, - externalPort: expose.as || 0, - proto: this.parseServiceProto(expose.proto), - global: !!to.global - }; - - const kind = this.exposeShouldBeIngress(exposeSpec) ? Endpoint_SHARED_HTTP : Endpoint_RANDOM_PORT; - - const defaultEp = kind !== 0 ? { kind: kind, sequence_number: 0 } : { sequence_number: 0 }; - - const leasedEp = - to.ip?.length > 0 - ? { - kind: Endpoint_LEASED_IP, - sequence_number: endpointSequenceNumbers[to.ip] || 0 - } - : undefined; - - return leasedEp ? [defaultEp, leasedEp] : [defaultEp]; - }) - : [] - ); - - return endpoints; - } - - serviceResourcesBeta2(profile: v2ProfileCompute, service: v2Service, asString: boolean = false) { - return { - cpu: this.serviceResourceCpu(profile.resources.cpu), - memory: this.serviceResourceMemory(profile.resources.memory, asString), - storage: this.serviceResourceStorage(profile.resources.storage, asString), - endpoints: this.v2ServiceResourceEndpoints(service) - }; - } - - serviceResourcesBeta3(id: number, profile: v3ProfileCompute, service: v2Service, asString: boolean = false) { - return { - id: id, - cpu: this.serviceResourceCpu(profile.resources.cpu), - memory: this.serviceResourceMemory(profile.resources.memory, asString), - storage: this.serviceResourceStorage(profile.resources.storage, asString), - endpoints: this.v3ServiceResourceEndpoints(service), - gpu: this.serviceResourceGpu(profile.resources.gpu, asString) - }; - } - - parseServiceProto(proto?: string) { - const raw = proto?.toUpperCase(); - let result = "TCP"; - - switch (raw) { - case "TCP": - case "": - case undefined: - result = "TCP"; - break; - case "UDP": - result = "UDP"; - break; - default: - throw new Error("ErrUnsupportedServiceProtocol"); - } - - return result; - } - - manifestExposeService(to: v2ExposeTo) { - return to.service || ""; - } - - manifestExposeGlobal(to: v2ExposeTo) { - return to.global || false; - } - - manifestExposeHosts(expose: v2Expose) { - return expose.accept || null; - } - - v2HttpOptions(http_options: v2HTTPOptions | undefined) { - const defaults = { - MaxBodySize: 1048576, - ReadTimeout: 60000, - SendTimeout: 60000, - NextTries: 3, - NextTimeout: 0, - NextCases: ["error", "timeout"] - }; - - if (!http_options) { - return { ...defaults }; - } - - return { - MaxBodySize: http_options.max_body_size || defaults.MaxBodySize, - ReadTimeout: http_options.read_timeout || defaults.ReadTimeout, - SendTimeout: http_options.send_timeout || defaults.SendTimeout, - NextTries: http_options.next_tries || defaults.NextTries, - NextTimeout: http_options.next_timeout || defaults.NextTimeout, - NextCases: http_options.next_cases || defaults.NextCases - }; - } - - v3HttpOptions(http_options: v2HTTPOptions | undefined) { - const defaults = { - maxBodySize: 1048576, - readTimeout: 60000, - sendTimeout: 60000, - nextTries: 3, - nextTimeout: 0, - nextCases: ["error", "timeout"] - }; - - if (!http_options) { - return { ...defaults }; - } - - return { - maxBodySize: http_options.max_body_size || defaults.maxBodySize, - readTimeout: http_options.read_timeout || defaults.readTimeout, - sendTimeout: http_options.send_timeout || defaults.sendTimeout, - nextTries: http_options.next_tries || defaults.nextTries, - nextTimeout: http_options.next_timeout || defaults.nextTimeout, - nextCases: http_options.next_cases || defaults.nextCases - }; - } - - v2ManifestExposeHttpOptions(expose: v2Expose): v2ServiceExposeHttpOptions { - return this.v2HttpOptions(expose.http_options); - } - - v3ManifestExposeHttpOptions(expose: v2Expose): v3ServiceExposeHttpOptions { - return this.v3HttpOptions(expose.http_options); - } - - v2ManifestExpose(service: v2Service): v2ServiceExpose[] { - const endpointSequenceNumbers = this.computeEndpointSequenceNumbers(this.data); - return service.expose.flatMap(expose => - expose.to - ? expose.to.map(to => ({ - Port: expose.port, - ExternalPort: expose.as || 0, - Proto: this.parseServiceProto(expose.proto), - Service: this.manifestExposeService(to), - Global: this.manifestExposeGlobal(to), - Hosts: this.manifestExposeHosts(expose), - HTTPOptions: this.v2ManifestExposeHttpOptions(expose), - IP: to.ip || "", - EndpointSequenceNumber: endpointSequenceNumbers[to.ip] || 0 - })) - : [] - ); - } - - v3ManifestExpose(service: v2Service): v3ServiceExpose[] { - const endpointSequenceNumbers = this.computeEndpointSequenceNumbers(this.data); - return service.expose - .flatMap(expose => - expose.to - ? expose.to.map(to => ({ - port: expose.port, - externalPort: expose.as || 0, - proto: this.parseServiceProto(expose.proto), - service: this.manifestExposeService(to), - global: this.manifestExposeGlobal(to), - hosts: this.manifestExposeHosts(expose), - httpOptions: this.v3ManifestExposeHttpOptions(expose), - ip: to.ip || "", - endpointSequenceNumber: endpointSequenceNumbers[to.ip] || 0 - })) - : [] - ) - .sort((a, b) => { - if (a.service != b.service) return a.service.localeCompare(b.service); - if (a.port != b.port) return a.port - b.port; - if (a.proto != b.proto) return a.proto.localeCompare(b.proto); - if (a.global != b.global) return a.global ? -1 : 1; - - return 0; - }); - } - - v2ManifestServiceParams(params: v2ServiceParams): v2ManifestServiceParams | undefined { - return { - Storage: Object.keys(params?.storage ?? {}).map(name => { - if (!params?.storage) throw new Error("Storage is undefined"); - return { - name: name, - mount: params.storage[name].mount, - readOnly: params.storage[name].readOnly || false - }; - }) - }; - } - - v3ManifestServiceParams(params: v2ServiceParams | undefined): v3ManifestServiceParams | null { - if (params === undefined) { - return null; - } - - return { - storage: Object.keys(params?.storage ?? {}).map(name => { - if (!params?.storage) throw new Error("Storage is undefined"); - return { - name: name, - mount: params.storage[name].mount, - readOnly: params.storage[name].readOnly || false - }; - }) - }; - } - - v2ManifestService(placement: string, name: string, asString: boolean): v2ManifestService { - const service = this.data.services[name]; - const deployment = this.data.deployment[name]; - const profile = this.data.profiles.compute[deployment[placement].profile]; - - const manifestService: v2ManifestService = { - Name: name, - Image: service.image, - Command: service.command || null, - Args: service.args || null, - Env: service.env || null, - Resources: this.serviceResourcesBeta2(profile, service, asString), - Count: deployment[placement].count, - Expose: this.v2ManifestExpose(service) - }; - - if (service.params) { - manifestService.params = this.v2ManifestServiceParams(service.params); - } - - return manifestService; - } - - v3ManifestService(id: number, placement: string, name: string, asString: boolean): v3ManifestService { - const service = this.data.services[name]; - const deployment = this.data.deployment[name]; - const profile = this.data.profiles.compute[deployment[placement].profile]; - - return { - name: name, - image: service.image, - command: service.command || null, - args: service.args || null, - env: service.env || null, - resources: this.serviceResourcesBeta3(id, profile as v3ProfileCompute, service, asString), - count: deployment[placement].count, - expose: this.v3ManifestExpose(service), - params: this.v3ManifestServiceParams(service.params), - credentials: null - }; - } - - v2Manifest(asString: boolean = false): v2Manifest { - return Object.keys(this.placements()).map(name => ({ - Name: name, - Services: this.deploymentsByPlacement(name).map(([service]) => this.v2ManifestService(name, service, asString)) - })); - } - - v3Manifest(asString: boolean = false): v3Manifest { - const groups = this.v3Groups(); - const serviceId = (pIdx: number, sIdx: number) => groups[pIdx].resources[sIdx].resource.id; - - return Object.keys(this.placements()).map((name, pIdx) => ({ - name: name, - services: this.deploymentsByPlacement(name) - .sort(([a], [b]) => a.localeCompare(b)) - .map(([service], idx) => this.v3ManifestService(serviceId(pIdx, idx), name, service, asString)) - })); - } - - manifest(asString: boolean = false): v2Manifest | v3Manifest { - return this.version === "beta2" ? this.v2Manifest(asString) : this.v3Manifest(asString); - } - - computeEndpointSequenceNumbers(sdl: v2Sdl) { - return Object.fromEntries( - Object.values(sdl.services).flatMap(service => - service.expose.flatMap(expose => - expose.to - ? expose.to - .filter(to => to.global && to.ip?.length > 0) - .map(to => to.ip) - .sort() - .map((ip, index) => [ip, index + 1]) - : [] - ) - ) - ); - } - - resourceUnitCpu(computeResources: v2ComputeResources, asString: boolean) { - const attributes = computeResources.cpu.attributes; - const cpu = isString(computeResources.cpu.units) ? convertCpuResourceString(computeResources.cpu.units) : computeResources.cpu.units * 1000; - - return { - units: { val: this.resourceValue(cpu, asString) }, - attributes: - attributes && - Object.entries(attributes) - .sort(([k0], [k1]) => k0.localeCompare(k1)) - .map(([key, value]) => ({ - key: key, - value: value.toString() - })) - }; - } - - resourceUnitMemory(computeResources: v2ComputeResources, asString: boolean) { - const attributes = computeResources.memory.attributes; - - return { - quantity: { - val: this.resourceValue(convertResourceString(computeResources.memory.size), asString) - }, - attributes: - attributes && - Object.entries(attributes) - .sort(([k0], [k1]) => k0.localeCompare(k1)) - .map(([key, value]) => ({ - key: key, - value: value.toString() - })) - }; - } - - resourceUnitStorage(computeResources: v2ComputeResources, asString: boolean) { - const storages = isArray(computeResources.storage) ? computeResources.storage : [computeResources.storage]; - - return storages.map(storage => ({ - name: storage.name || "default", - quantity: { - val: this.resourceValue(convertResourceString(storage.size), asString) - }, - attributes: this.serviceResourceStorageAttributes(storage.attributes) - })); - } - - transformGpuAttributes(attributes: v3GPUAttributes): Array<{ key: string; value: string }> { - return Object.entries(attributes.vendor).flatMap(([vendor, models]) => - models - ? models.map(model => { - let key = `vendor/${vendor}/model/${model.model}`; - - if (model.ram) { - key += `/ram/${model.ram}`; - } - - if (model.interface) { - key += `/interface/${model.interface}`; - } - - return { - key: key, - value: "true" - }; - }) - : [ - { - key: `vendor/${vendor}/model/*`, - value: "true" - } - ] - ); - } - - resourceUnitGpu(computeResources: v3ComputeResources, asString: boolean) { - const attributes = computeResources.gpu?.attributes; - const units = computeResources.gpu?.units || "0"; - const gpu = isString(units) ? parseInt(units) : units; - - return { - units: { val: this.resourceValue(gpu, asString) }, - attributes: attributes && this.transformGpuAttributes(attributes) - }; - } - - groupResourceUnits(resource: v2ComputeResources | undefined, asString: boolean) { - if (!resource) return {}; - - const units = { - endpoints: null - } as any; - - if (resource.cpu) { - units.cpu = this.resourceUnitCpu(resource, asString); - } - - if (resource.memory) { - units.memory = this.resourceUnitMemory(resource, asString); - } - - if (resource.storage) { - units.storage = this.resourceUnitStorage(resource, asString); - } - - if (this.version === "beta3") { - units.gpu = this.resourceUnitGpu(resource as v3ComputeResources, asString); - } - - return units; - } - - exposeShouldBeIngress(expose: { proto: string; global: boolean; externalPort: number; port: number }) { - const externalPort = expose.externalPort === 0 ? expose.port : expose.externalPort; - - return expose.global && expose.proto === "TCP" && externalPort === 80; - } - - groups() { - return this.version === "beta2" ? this.v2Groups() : this.v3Groups(); - } - - v3Groups() { - const groups = new Map< - string, - { - dgroup: v3DeploymentGroup; - boundComputes: Record>; - } - >(); - const services = Object.entries(this.data.services).sort(([a], [b]) => a.localeCompare(b)); - - for (const [svcName, service] of services) { - for (const [placementName, svcdepl] of Object.entries(this.data.deployment[svcName])) { - // objects below have been ensured to exist - const compute = this.data.profiles.compute[svcdepl.profile]; - const infra = this.data.profiles.placement[placementName]; - const pricing = infra.pricing[svcdepl.profile]; - const price = { - ...pricing, - amount: pricing.amount.toString() - }; - - let group = groups.get(placementName); - - if (!group) { - const attributes = (infra.attributes - ? Object.entries(infra.attributes).map(([key, value]) => ({ - key, - value - })) - : []) as unknown as Array<{ key: string; value: string }>; - - attributes.sort((a, b) => a.key.localeCompare(b.key)); - - group = { - dgroup: { - name: placementName, - resources: [], - requirements: { - attributes: attributes, - signedBy: { - allOf: infra.signedBy?.allOf || [], - anyOf: infra.signedBy?.anyOf || [] - } - } - }, - boundComputes: {} - }; - - groups.set(placementName, group); - } - - if (!group.boundComputes[placementName]) { - group.boundComputes[placementName] = {}; - } - - // const resources = this.serviceResourcesBeta3(0, compute as v3ProfileCompute, service, false); - const location = group.boundComputes[placementName][svcdepl.profile]; - - if (!location) { - const res = this.groupResourceUnits(compute.resources, false); - res.endpoints = this.v3ServiceResourceEndpoints(service); - - const resID = group.dgroup.resources.length > 0 ? group.dgroup.resources.length + 1 : 1; - res.id = resID; - // resources.id = res.id; - - group.dgroup.resources.push({ - resource: res, - price: price, - count: svcdepl.count - } as any); - - group.boundComputes[placementName][svcdepl.profile] = group.dgroup.resources.length - 1; - } else { - const endpoints = this.v3ServiceResourceEndpoints(service); - // resources.id = group.dgroup.resources[location].id; - - group.dgroup.resources[location].count += svcdepl.count; - group.dgroup.resources[location].endpoints += endpoints as any; - group.dgroup.resources[location].endpoints.sort(); - } - } - } - - // keep ordering stable - const names: string[] = [...groups.keys()].sort(); - return names.map(name => groups.get(name)).map(group => (group ? (group.dgroup as typeof group.dgroup) : {})) as Array; - } - - v2Groups() { - const yamlJson = this.data; - const ipEndpointNames = this.computeEndpointSequenceNumbers(yamlJson); - - const groups = {} as any; - - Object.keys(yamlJson.services).forEach(svcName => { - const svc = yamlJson.services[svcName]; - const depl = yamlJson.deployment[svcName]; - - Object.keys(depl).forEach(placementName => { - const svcdepl = depl[placementName]; - const compute = yamlJson.profiles.compute[svcdepl.profile]; - const infra = yamlJson.profiles.placement[placementName]; - - const pricing = infra.pricing[svcdepl.profile]; - const price = { - ...pricing, - amount: pricing.amount.toString() - }; - - let group = groups[placementName]; - - if (!group) { - group = { - name: placementName, - requirements: { - attributes: infra.attributes - ? Object.entries(infra.attributes).map(([key, value]) => ({ - key, - value - })) - : [], - signedBy: { - allOf: infra.signedBy?.allOf || [], - anyOf: infra.signedBy?.anyOf || [] - } - }, - resources: [] - }; - - if (group.requirements.attributes) { - group.requirements.attributes = group.requirements.attributes.sort((a: any, b: any) => a.key < b.key); - } - - groups[group.name] = group; - } - - const resources = { - resources: this.groupResourceUnits(compute.resources, false), // Changed resources => unit - price: price, - count: svcdepl.count - }; - - const endpoints = [] as any[]; - svc?.expose?.forEach(expose => { - expose?.to - ?.filter(to => to.global) - .forEach(to => { - const exposeSpec = { - port: expose.port, - externalPort: expose.as || 0, - proto: this.parseServiceProto(expose.proto), - global: !!to.global - }; - - if (to.ip?.length > 0) { - const seqNo = ipEndpointNames[to.ip]; - endpoints.push({ - kind: Endpoint_LEASED_IP, - sequence_number: seqNo - }); - } - - const kind = this.exposeShouldBeIngress(exposeSpec) ? Endpoint_SHARED_HTTP : Endpoint_RANDOM_PORT; - - endpoints.push({ kind: kind, sequence_number: 0 }); - }); - }); - - resources.resources.endpoints = endpoints; - group.resources.push(resources); - }); - }); - - return Object.keys(groups) - .sort((a, b) => (a < b ? 1 : 0)) - .map(name => groups[name]); - } - - escapeHtml(raw: string) { - return raw.replace(//g, "\\u003e").replace(/&/g, "\\u0026"); - } - - SortJSON(jsonStr: string) { - return this.escapeHtml(stableStringify(JSON.parse(jsonStr))); - } - - manifestSortedJSON() { - const manifest = this.manifest(true); - let jsonStr = JSON.stringify(manifest); - - if (jsonStr) { - jsonStr = jsonStr.replaceAll('"quantity":{"val', '"size":{"val'); - } - - return this.SortJSON(jsonStr); - } - - async manifestVersion() { - const jsonStr = this.manifestSortedJSON(); - const enc = new TextEncoder(); - const sortedBytes = enc.encode(jsonStr); - const sum = await crypto.subtle.digest("SHA-256", sortedBytes); - - return new Uint8Array(sum); - } - - manifestSorted() { - const sorted = this.manifestSortedJSON(); - return JSON.parse(sorted); - } -} +export * from "./SDL/SDL"; diff --git a/src/sdl/types.ts b/src/sdl/types.ts index a23f53b..dd39181 100644 --- a/src/sdl/types.ts +++ b/src/sdl/types.ts @@ -152,7 +152,7 @@ export type v2ServiceParams = { export type v2ServiceImageCredentials = { host: string; - email: string; + email?: string; username: string; password: string; }; diff --git a/src/types/network.ts b/src/types/network.ts new file mode 100644 index 0000000..107482d --- /dev/null +++ b/src/types/network.ts @@ -0,0 +1,4 @@ +export type MainnetNetworkId = "mainnet"; +export type TestnetNetworkId = "testnet"; +export type SandboxNetworkId = "sandbox"; +export type NetworkId = MainnetNetworkId | TestnetNetworkId | SandboxNetworkId; diff --git a/test/fixtures/sdl-basic.yml b/test/fixtures/sdl-basic.yml new file mode 100644 index 0000000..b61a6ac --- /dev/null +++ b/test/fixtures/sdl-basic.yml @@ -0,0 +1,38 @@ +version: "2.0" + +services: + web: + image: akashlytics/hello-akash-world:0.2.0 + expose: + - port: 3000 + as: 80 + to: + - global: true + ${endpointRef} +${credentials} + +profiles: + compute: + web: + resources: + cpu: + units: 0.5 + memory: + size: 512Mi + storage: + size: 512Mi + + placement: + dcloud: + pricing: + web: + denom: ${denom} + amount: 1000 + +deployment: + web: + dcloud: + profile: web + count: 1 + +${endpoints} \ No newline at end of file diff --git a/test/yml.ts b/test/yml.ts new file mode 100644 index 0000000..5818812 --- /dev/null +++ b/test/yml.ts @@ -0,0 +1,60 @@ +import fs from "fs"; +import path from "path"; +import { dump } from "js-yaml"; +import { faker } from "@faker-js/faker"; +import template from "lodash/template"; +import memoize from "lodash/memoize"; +import { AKT_DENOM } from "../src/config/network"; +import { SANDBOX_ID, USDC_IBC_DENOMS } from "../src/config/network"; +import { pick } from "lodash"; + +export const readYml = (name: string): string => { + return fs.readFileSync(path.resolve(__dirname, `./fixtures/${name}.yml`), "utf-8"); +}; + +type YmlInputObject = { + [key: string]: string | number | boolean | null | YmlInputObject | YmlInputObject[]; +}; + +export const toYmlFragment = (object: YmlInputObject, options?: { nestingLevel: number }): string => { + const yamlString = dump(object, { forceQuotes: true, quotingType: '"' }); + + if (!options) { + return yamlString; + } + + const indentation = " ".repeat(options.nestingLevel); + return yamlString + .split("\n") + .map(line => indentation + line) + .join("\n"); +}; + +interface BasicSdlTemplateVariables { + denom?: string; + credentials?: { + host?: string; + username?: string; + password?: string; + }; + endpoint?: Record; + endpointRef?: { ip: string }; +} + +const readBasicSdlTemplate = memoize(() => template(readYml("sdl-basic"))); + +export const readBasicSdl = (variables: BasicSdlTemplateVariables = {}): string => { + const createYML = readBasicSdlTemplate(); + const endpointName = Object.keys(variables.endpoint || {})[0]; + const ymlVars: Record | "endpoints" | "endpointRef", string> = { + denom: variables.denom || faker.helpers.arrayElement([AKT_DENOM, USDC_IBC_DENOMS[SANDBOX_ID]]), + credentials: variables.credentials ? toYmlFragment(pick(variables, "credentials"), { nestingLevel: 2 }) : "", + endpoints: variables.endpoint ? toYmlFragment({ endpoints: variables.endpoint }) : "", + endpointRef: + ("endpointRef" in variables && toYmlFragment(pick(variables, "endpointRef"), { nestingLevel: 3 })) || + (endpointName && toYmlFragment({ ip: endpointName })) || + "" + }; + + return createYML(ymlVars); +}; diff --git a/tsconfig.json b/tsconfig.json index 3153954..48cf78f 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -69,5 +69,5 @@ "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */ }, "include": ["./src/**/*"], - "exclude": ["./examples", "./tests"] + "exclude": ["./examples", "./tests", "./test", "./src/**/*.spec.ts"] }