Features/implement new status endpoint (#95)

* POC with providerSnapshotNodes * Add isDuplicate & lastSnapshotId * Add tables for cpu/gpu + handle different provider versions * Support providers without grpc * Remove execAsync * Cleanup * Update isDuplicate on provider create/edit/delete * Add upgrade documentation * PR Cleanup * Bump version * Fix protoset path * Remove isDuplicate * PR Fixes * . * Add flag * .
akash-network · Feb 13, 2024 · df3c658 · df3c658
1 parent 6ef9b88
commit df3c658
Show file tree

Hide file tree

Showing 19 changed files with 1,070 additions and 200 deletions.
diff --git a/api/src/services/db/statsService.ts b/api/src/services/db/statsService.ts
@@ -1,5 +1,5 @@
 import { Day } from "@shared/dbSchemas/base";
-import { AkashBlock as Block } from "@shared/dbSchemas/akash";
+import { AkashBlock as Block, Provider } from "@shared/dbSchemas/akash";
 import { subHours } from "date-fns";
 import { Op, QueryTypes } from "sequelize";
 import { chainDb } from "@src/db/dbConnection";
@@ -160,16 +160,17 @@ export const getProviderGraphData = async (dataName: ProviderStatsKey) => {
     async () => {
       return (await chainDb.query(
         `SELECT d."date", (SUM("activeCPU") + SUM("pendingCPU") + SUM("availableCPU")) AS "cpu", (SUM("activeGPU") + SUM("pendingGPU") + SUM("availableGPU")) AS "gpu", (SUM("activeMemory") + SUM("pendingMemory") + SUM("availableMemory")) AS memory, (SUM("activeStorage") + SUM("pendingStorage") + SUM("availableStorage")) as storage, COUNT(*) as count
-        FROM "day" d
-        INNER JOIN (
+         FROM "day" d
+         INNER JOIN (
             SELECT DISTINCT ON("hostUri",DATE("checkDate")) DATE("checkDate") AS date, ps."activeCPU", ps."pendingCPU", ps."availableCPU", ps."activeGPU", ps."pendingGPU", ps."availableGPU", ps."activeMemory", ps."pendingMemory", ps."availableMemory", ps."activeStorage", ps."pendingStorage", ps."availableStorage", ps."isOnline"
-                        FROM "providerSnapshot" ps
-                        INNER JOIN "provider" ON "provider"."owner"=ps."owner"
-                        ORDER BY "hostUri",DATE("checkDate"),"checkDate" DESC
-            ) "dailyProviderStats"
-        ON d."date"="dailyProviderStats"."date" AND "isOnline" IS TRUE
-        GROUP BY d."date"
-        ORDER BY d."date" ASC`,
+            FROM "providerSnapshot" ps
+            INNER JOIN "provider" ON "provider"."owner"=ps."owner"
+            WHERE ps."isLastOfDay" = TRUE AND ps."isOnline" = TRUE
+            ORDER BY "hostUri",DATE("checkDate"),"checkDate" DESC
+         ) "dailyProviderStats"
+         ON DATE(d."date")="dailyProviderStats"."date"
+         GROUP BY d."date"
+         ORDER BY d."date" ASC`,
         {
           type: QueryTypes.SELECT
         }

diff --git a/api/src/types/graph.ts b/api/src/types/graph.ts
@@ -1,4 +1,4 @@
-export type ProviderStatsKey = "count" | "cpu" | "gpu" | "memory" | "storage";
+export type ProviderStatsKey = keyof Omit<ProviderStats, "date">;
 
 export type ProviderStats = {
   date: string;

diff --git a/indexer/UPGRADE.md b/indexer/UPGRADE.md
@@ -1,6 +1,89 @@
 # Upgrade instructions
 
-Some indexer updates changes the database schemas and an upgrade script must be run on the database to migrate the data before or after updating the indexer. Here is a list of those migrations. If a version is not listed here it means the indexer can be updated without any manual migration.
+Some indexer updates changes the database schemas and an upgrade script must be run on the database to migrate the data before or after updating the indexer. Here is a list of those migrations. If a version is not listed here it means the indexer can be updated without any manual migration. 
+
+**It is recommended to stop the indexer before running any migration script.**
+
+## v1.7.0
+
+Version 1.7.0 adds some tables and fields to improve provider queries as well as keep track of node/cpu/gpu data provided by the new status endpoint (grpc).
+
+```
+-- Add new Provider columns
+ALTER TABLE IF EXISTS public.provider
+    ADD COLUMN "lastSnapshotId" uuid,
+
+-- Set lastSnapshotId to the most recent snapshot for each providers
+UPDATE "provider" p SET "lastSnapshotId" = (
+	SELECT ps.id FROM "providerSnapshot" ps WHERE ps."owner" = p."owner" ORDER BY "checkDate" DESC LIMIT 1
+)
+
+-- Update ProviderSnapshot schemas
+ALTER TABLE IF EXISTS public."providerSnapshot"
+    ADD COLUMN "isLastOfDay" boolean NOT NULL DEFAULT false,
+	ALTER COLUMN "isOnline" SET NOT NULL,
+	ALTER COLUMN "checkDate" SET NOT NULL;
+	
+-- Set isLastOfDay to true for snapshots that are the last of each day for every providers
+WITH last_snapshots AS (
+	SELECT DISTINCT ON(ps."owner",DATE("checkDate")) DATE("checkDate") AS date, ps."id" AS "psId"
+	FROM "providerSnapshot" ps
+	ORDER BY ps."owner",DATE("checkDate"),"checkDate" DESC
+) 
+UPDATE "providerSnapshot" AS ps
+SET "isLastOfDay" = TRUE
+FROM last_snapshots AS ls
+WHERE ls."psId"=ps.id
+
+-- Add index for isLastofDay
+CREATE INDEX IF NOT EXISTS provider_snapshot_id_where_isonline_and_islastofday
+    ON public."providerSnapshot" USING btree
+    (id ASC NULLS LAST)
+    TABLESPACE pg_default
+    WHERE "isOnline" = true AND "isLastOfDay" = true;
+
+-- Create new tables for tracking nodes/gpu/cpu info
+CREATE TABLE IF NOT EXISTS public."providerSnapshotNode"
+(
+    id uuid NOT NULL,
+    "snapshotId" uuid NOT NULL,
+    name character varying(255) COLLATE pg_catalog."default",
+    "cpuAllocatable" bigint,
+    "cpuAllocated" bigint,
+    "memoryAllocatable" bigint,
+    "memoryAllocated" bigint,
+    "ephemeralStorageAllocatable" bigint,
+    "ephemeralStorageAllocated" bigint,
+    "capabilitiesStorageHDD" boolean,
+    "capabilitiesStorageSSD" boolean,
+    "capabilitiesStorageNVME" boolean,
+    "gpuAllocatable" bigint,
+    "gpuAllocated" bigint,
+    CONSTRAINT "providerSnapshotNode_pkey" PRIMARY KEY (id)
+)
+
+CREATE TABLE IF NOT EXISTS public."providerSnapshotNodeCPU"
+(
+    id uuid NOT NULL,
+    "snapshotNodeId" uuid NOT NULL,
+    vendor character varying(255) COLLATE pg_catalog."default",
+    model character varying(255) COLLATE pg_catalog."default",
+    vcores character varying(255) COLLATE pg_catalog."default",
+    CONSTRAINT "providerSnapshotNodeCPU_pkey" PRIMARY KEY (id)
+)
+
+CREATE TABLE IF NOT EXISTS public."providerSnapshotNodeGPU"
+(
+    id uuid NOT NULL,
+    "snapshotNodeId" uuid NOT NULL,
+    vendor character varying(255) COLLATE pg_catalog."default",
+    name character varying(255) COLLATE pg_catalog."default",
+    "modelId" character varying(255) COLLATE pg_catalog."default",
+    interface character varying(255) COLLATE pg_catalog."default",
+    "memorySize" character varying(255) COLLATE pg_catalog."default",
+    CONSTRAINT "providerSnapshotNodeGPU_pkey" PRIMARY KEY (id)
+)
+```
 
 ## v1.5.0