add sonatype load

metrics-collector/src/index.ts

@@ -6,7 +6,10 @@ import { hideBin } from "yargs/helpers";
 
 import { collectGhMetrics } from "./gh-metrics";
 import { collectNpmMetrics, saveNpmMetrics } from "./npm-metrics";
-import { collectSonatypeMetrics } from "./sonatype-metrics";
+import {
+  collectSonatypeMetrics,
+  saveSonatypeMetrics,
+} from "./sonatype-metrics";
 import { getYesterdayDate } from "./utils";
 
 const isLocalPersistence = process.env.PERSIST_LOCAL_FILES === "true";
@@ -70,7 +73,19 @@ async function main() {
 
   const collectSonatype = argv["collect-sonatype"];
   if (collectSonatype) {
-    await collectSonatypeMetrics(isLocalPersistence);
+    console.info(
+      `\n\n============\n\n>>> Collecting metrics for Maven Sonatype...`
+    );
+    if (initialLoadFromDate) {
+      await initialLoad(
+        initialLoadFromDate,
+        metricDate,
+        collectSonatypeMetrics,
+        true
+      );
+    } else {
+      await collectSonatypeMetrics(metricDateStr);
+    }
   }
 
   const collectGh = argv["collect-gh"];
@@ -81,22 +96,34 @@ async function main() {
   const localCollection = !collectGh && !collectNpm && !collectSonatype;
   if (localCollection) {
     await saveNpmMetrics();
+    await saveSonatypeMetrics();
     // await saveGhMetrics();
-    // await saveSonatypeMetrics();
   }
 }
 
 async function initialLoad(
   initialLoadFromDate: Date,
   initialLoadToDate: Date,
-  collectMetrics: (metricDate: string) => Promise<void>
+  collectMetrics: (metricDate: string) => Promise<void>,
+  monthlyInterval = false
 ) {
   let date = initialLoadFromDate;
+  if (monthlyInterval) {
+    // Change the date to the first day of the month
+    date.setDate(0);
+  }
+
   while (date <= initialLoadToDate) {
     const dateStr = date.toISOString().split("T")[0];
     console.log(`\n\n>>> Collecting metrics for date: ${dateStr}`);
     await collectMetrics(dateStr);
-    date.setDate(date.getDate() + 1);
+
+    if (monthlyInterval) {
+      // Move to the next month (JS will handle year change automatically)
+      date.setMonth(date.getMonth() + 1);
+    } else {
+      date.setDate(date.getDate() + 1);
+    }
   }
 }
 

metrics-collector/src/npm-metrics.ts

@@ -1,7 +1,7 @@
 import * as fs from "fs";
 import * as path from "path";
 import { createObjectCsvWriter } from "csv-writer";
-import { getYesterdayDate, readJsonFile, writeJsonFile } from "./utils";
+import { readJsonFile, writeJsonFile } from "./utils";
 import { postMetric } from "./post-metric";
 
 // Define the npm packages to collect metrics for
@@ -10,7 +10,13 @@ const npmPackages = [
   "@web5/common",
   "@web5/credentials",
   "@web5/crypto",
+  "@web5/crypto-aws-kms",
   "@web5/dids",
+  "@web5/api",
+  "@web5/agent",
+  "@web5/identity-agent",
+  "@web5/proxy-agent",
+  "@web5/user-agent",
   "@tbdex/protocol",
   "@tbdex/http-client",
   "@tbdex/http-server",

metrics-collector/src/post-metric.ts

@@ -34,12 +34,15 @@ export const postMetric = async (
     });
 
     if (!response.ok) {
+      if (response.body) {
+        const error = await response.json();
+        console.error("Errored response body:", { error });
+      }
       throw new Error(`Error posting metric: ${response.statusText}`);
     }
 
-    const data = await response.json();
     console.log("Metric posted successfully:", JSON.stringify(payload));
   } catch (error) {
-    console.error("Error posting metric:", error);
+    console.error("Error posting metric:", error, JSON.stringify(error));
   }
 };

metrics-collector/src/service-app.ts

@@ -107,7 +107,7 @@ if (!process.env.DB_SKIP_INIT) {
 // Collect Metrics Endpoint
 app.post("/api/v1/metrics", async (req: any, res: any) => {
   const { metricName, value, labels, timestamp } = req.body;
-  if (!metricName || !value) {
+  if (!metricName || value == undefined) {
     return res
       .status(400)
       .send({ error: "Missing required fields: metricName and value" });

metrics-collector/src/sonatype-metrics.ts

@@ -2,6 +2,7 @@ import * as fs from "fs";
 import * as path from "path";
 import { createObjectCsvWriter } from "csv-writer";
 import { readJsonFile, writeJsonFile } from "./utils";
+import { postMetric } from "./post-metric";
 
 // Define the group id to collect metrics for
 const groupId = "xyz.block";
@@ -15,7 +16,54 @@ const requestHeaders: Record<string, string> = {
 const sonatypeCentralStatsUrl =
   "https://s01.oss.sonatype.org/service/local/stats";
 
-async function collectSonatypeMetrics(isLocalPersistence: boolean = false) {
+export async function collectSonatypeMetrics(metricDate: string) {
+  initAuth();
+
+  const projectId = await getProjectId(groupId);
+  const artifacts = await getArtifacts(projectId, groupId);
+
+  for (const artifact of artifacts) {
+    const [rawDownloads, uniqueIPs] = await Promise.all([
+      getArtifactStats(projectId, groupId, artifact, "raw", metricDate),
+      getArtifactStats(projectId, groupId, artifact, "ip", metricDate),
+    ]);
+
+    await postSonatypeMavenMetrics({
+      artifact,
+      metricDate: new Date(metricDate),
+      rawDownloads: rawDownloads.total,
+      uniqueIPs: uniqueIPs.total,
+    });
+  }
+}
+
+async function postSonatypeMavenMetrics(metric: {
+  artifact: string;
+  metricDate: Date;
+  rawDownloads: number;
+  uniqueIPs: number;
+}) {
+  console.info("posting sonatype metric", { metric });
+  const labels = {
+    artifact: metric.artifact,
+  };
+
+  await postMetric(
+    "sonatype_central_stats_downloads",
+    metric.rawDownloads,
+    labels,
+    metric.metricDate
+  );
+
+  await postMetric(
+    "sonatype_central_stats_unique_ips_downloads",
+    metric.uniqueIPs,
+    labels,
+    metric.metricDate
+  );
+}
+
+export async function saveSonatypeMetrics() {
   initAuth();
 
   const timestamp = new Date().toISOString();
@@ -39,20 +87,18 @@ async function collectSonatypeMetrics(isLocalPersistence: boolean = false) {
 
   console.info("Sonatype metrics collected successfully", { metrics });
 
-  if (isLocalPersistence) {
-    const sonatypeMetrics = readJsonFile(dataFilePath);
-    for (const metric of metrics) {
-      if (!sonatypeMetrics[metric.artifact]) {
-        sonatypeMetrics[metric.artifact] = [];
-      }
-      sonatypeMetrics[metric.artifact].push(metric);
+  const sonatypeMetrics = readJsonFile(dataFilePath);
+  for (const metric of metrics) {
+    if (!sonatypeMetrics[metric.artifact]) {
+      sonatypeMetrics[metric.artifact] = [];
     }
-    writeJsonFile(dataFilePath, sonatypeMetrics);
-    await writeMetricsToCsv(csvFilePath, sonatypeMetrics);
-    console.log(
-      "Sonatype metrics have been successfully saved to sonatype_metrics.json and sonatype_metrics.csv"
-    );
+    sonatypeMetrics[metric.artifact].push(metric);
   }
+  writeJsonFile(dataFilePath, sonatypeMetrics);
+  await writeMetricsToCsv(csvFilePath, sonatypeMetrics);
+  console.log(
+    "Sonatype metrics have been successfully saved to sonatype_metrics.json and sonatype_metrics.csv"
+  );
 
   return metrics;
 }
@@ -114,9 +160,12 @@ async function getArtifactStats(
   projectId: string,
   groupId: string,
   artifactId: string,
-  type: string
+  type: string,
+  fromDate?: string
 ): Promise<{ total: number }> {
-  const from = getLastMonthDate();
+  const from = fromDate
+    ? convertDateToLastYearMonth(fromDate)
+    : getLastMonthDate();
   console.info(
     `Fetching ${type} stats for artifact ${artifactId} from ${from}...`
   );
@@ -181,4 +230,11 @@ function getLastMonthDate() {
   return `${lastMonthYear}${String(lastMonth).padStart(2, "0")}`;
 }
 
-export { collectSonatypeMetrics };
+// function to convert YYYY-MM-DD to YYYYMM
+function convertDateToLastYearMonth(date: string) {
+  const lastMonth = new Date(date);
+  // reduce 1 month, JS will automatically adjust the year if needed
+  lastMonth.setMonth(lastMonth.getMonth() - 1);
+  const [year, month] = lastMonth.toISOString().split("T")[0].split("-");
+  return `${year}${month}`;
+}