diff --git a/metrics-collector/src/index.ts b/metrics-collector/src/index.ts index 33e92b0..9a13b37 100644 --- a/metrics-collector/src/index.ts +++ b/metrics-collector/src/index.ts @@ -6,7 +6,10 @@ import { hideBin } from "yargs/helpers"; import { collectGhMetrics } from "./gh-metrics"; import { collectNpmMetrics, saveNpmMetrics } from "./npm-metrics"; -import { collectSonatypeMetrics } from "./sonatype-metrics"; +import { + collectSonatypeMetrics, + saveSonatypeMetrics, +} from "./sonatype-metrics"; import { getYesterdayDate } from "./utils"; const isLocalPersistence = process.env.PERSIST_LOCAL_FILES === "true"; @@ -70,7 +73,19 @@ async function main() { const collectSonatype = argv["collect-sonatype"]; if (collectSonatype) { - await collectSonatypeMetrics(isLocalPersistence); + console.info( + `\n\n============\n\n>>> Collecting metrics for Maven Sonatype...` + ); + if (initialLoadFromDate) { + await initialLoad( + initialLoadFromDate, + metricDate, + collectSonatypeMetrics, + true + ); + } else { + await collectSonatypeMetrics(metricDateStr); + } } const collectGh = argv["collect-gh"]; @@ -81,22 +96,34 @@ async function main() { const localCollection = !collectGh && !collectNpm && !collectSonatype; if (localCollection) { await saveNpmMetrics(); + await saveSonatypeMetrics(); // await saveGhMetrics(); - // await saveSonatypeMetrics(); } } async function initialLoad( initialLoadFromDate: Date, initialLoadToDate: Date, - collectMetrics: (metricDate: string) => Promise + collectMetrics: (metricDate: string) => Promise, + monthlyInterval = false ) { let date = initialLoadFromDate; + if (monthlyInterval) { + // Change the date to the first day of the month + date.setDate(0); + } + while (date <= initialLoadToDate) { const dateStr = date.toISOString().split("T")[0]; console.log(`\n\n>>> Collecting metrics for date: ${dateStr}`); await collectMetrics(dateStr); - date.setDate(date.getDate() + 1); + + if (monthlyInterval) { + // Move to the next month (JS will handle year change automatically) + date.setMonth(date.getMonth() + 1); + } else { + date.setDate(date.getDate() + 1); + } } } diff --git a/metrics-collector/src/npm-metrics.ts b/metrics-collector/src/npm-metrics.ts index bea1d78..6785d38 100644 --- a/metrics-collector/src/npm-metrics.ts +++ b/metrics-collector/src/npm-metrics.ts @@ -1,7 +1,7 @@ import * as fs from "fs"; import * as path from "path"; import { createObjectCsvWriter } from "csv-writer"; -import { getYesterdayDate, readJsonFile, writeJsonFile } from "./utils"; +import { readJsonFile, writeJsonFile } from "./utils"; import { postMetric } from "./post-metric"; // Define the npm packages to collect metrics for @@ -10,7 +10,13 @@ const npmPackages = [ "@web5/common", "@web5/credentials", "@web5/crypto", + "@web5/crypto-aws-kms", "@web5/dids", + "@web5/api", + "@web5/agent", + "@web5/identity-agent", + "@web5/proxy-agent", + "@web5/user-agent", "@tbdex/protocol", "@tbdex/http-client", "@tbdex/http-server", diff --git a/metrics-collector/src/post-metric.ts b/metrics-collector/src/post-metric.ts index aa87033..684c198 100644 --- a/metrics-collector/src/post-metric.ts +++ b/metrics-collector/src/post-metric.ts @@ -34,12 +34,15 @@ export const postMetric = async ( }); if (!response.ok) { + if (response.body) { + const error = await response.json(); + console.error("Errored response body:", { error }); + } throw new Error(`Error posting metric: ${response.statusText}`); } - const data = await response.json(); console.log("Metric posted successfully:", JSON.stringify(payload)); } catch (error) { - console.error("Error posting metric:", error); + console.error("Error posting metric:", error, JSON.stringify(error)); } }; diff --git a/metrics-collector/src/service-app.ts b/metrics-collector/src/service-app.ts index 0a27bdd..93b581f 100644 --- a/metrics-collector/src/service-app.ts +++ b/metrics-collector/src/service-app.ts @@ -107,7 +107,7 @@ if (!process.env.DB_SKIP_INIT) { // Collect Metrics Endpoint app.post("/api/v1/metrics", async (req: any, res: any) => { const { metricName, value, labels, timestamp } = req.body; - if (!metricName || !value) { + if (!metricName || value == undefined) { return res .status(400) .send({ error: "Missing required fields: metricName and value" }); diff --git a/metrics-collector/src/sonatype-metrics.ts b/metrics-collector/src/sonatype-metrics.ts index cf9ac9f..49d36f3 100644 --- a/metrics-collector/src/sonatype-metrics.ts +++ b/metrics-collector/src/sonatype-metrics.ts @@ -2,6 +2,7 @@ import * as fs from "fs"; import * as path from "path"; import { createObjectCsvWriter } from "csv-writer"; import { readJsonFile, writeJsonFile } from "./utils"; +import { postMetric } from "./post-metric"; // Define the group id to collect metrics for const groupId = "xyz.block"; @@ -15,7 +16,54 @@ const requestHeaders: Record = { const sonatypeCentralStatsUrl = "https://s01.oss.sonatype.org/service/local/stats"; -async function collectSonatypeMetrics(isLocalPersistence: boolean = false) { +export async function collectSonatypeMetrics(metricDate: string) { + initAuth(); + + const projectId = await getProjectId(groupId); + const artifacts = await getArtifacts(projectId, groupId); + + for (const artifact of artifacts) { + const [rawDownloads, uniqueIPs] = await Promise.all([ + getArtifactStats(projectId, groupId, artifact, "raw", metricDate), + getArtifactStats(projectId, groupId, artifact, "ip", metricDate), + ]); + + await postSonatypeMavenMetrics({ + artifact, + metricDate: new Date(metricDate), + rawDownloads: rawDownloads.total, + uniqueIPs: uniqueIPs.total, + }); + } +} + +async function postSonatypeMavenMetrics(metric: { + artifact: string; + metricDate: Date; + rawDownloads: number; + uniqueIPs: number; +}) { + console.info("posting sonatype metric", { metric }); + const labels = { + artifact: metric.artifact, + }; + + await postMetric( + "sonatype_central_stats_downloads", + metric.rawDownloads, + labels, + metric.metricDate + ); + + await postMetric( + "sonatype_central_stats_unique_ips_downloads", + metric.uniqueIPs, + labels, + metric.metricDate + ); +} + +export async function saveSonatypeMetrics() { initAuth(); const timestamp = new Date().toISOString(); @@ -39,20 +87,18 @@ async function collectSonatypeMetrics(isLocalPersistence: boolean = false) { console.info("Sonatype metrics collected successfully", { metrics }); - if (isLocalPersistence) { - const sonatypeMetrics = readJsonFile(dataFilePath); - for (const metric of metrics) { - if (!sonatypeMetrics[metric.artifact]) { - sonatypeMetrics[metric.artifact] = []; - } - sonatypeMetrics[metric.artifact].push(metric); + const sonatypeMetrics = readJsonFile(dataFilePath); + for (const metric of metrics) { + if (!sonatypeMetrics[metric.artifact]) { + sonatypeMetrics[metric.artifact] = []; } - writeJsonFile(dataFilePath, sonatypeMetrics); - await writeMetricsToCsv(csvFilePath, sonatypeMetrics); - console.log( - "Sonatype metrics have been successfully saved to sonatype_metrics.json and sonatype_metrics.csv" - ); + sonatypeMetrics[metric.artifact].push(metric); } + writeJsonFile(dataFilePath, sonatypeMetrics); + await writeMetricsToCsv(csvFilePath, sonatypeMetrics); + console.log( + "Sonatype metrics have been successfully saved to sonatype_metrics.json and sonatype_metrics.csv" + ); return metrics; } @@ -114,9 +160,12 @@ async function getArtifactStats( projectId: string, groupId: string, artifactId: string, - type: string + type: string, + fromDate?: string ): Promise<{ total: number }> { - const from = getLastMonthDate(); + const from = fromDate + ? convertDateToLastYearMonth(fromDate) + : getLastMonthDate(); console.info( `Fetching ${type} stats for artifact ${artifactId} from ${from}...` ); @@ -181,4 +230,11 @@ function getLastMonthDate() { return `${lastMonthYear}${String(lastMonth).padStart(2, "0")}`; } -export { collectSonatypeMetrics }; +// function to convert YYYY-MM-DD to YYYYMM +function convertDateToLastYearMonth(date: string) { + const lastMonth = new Date(date); + // reduce 1 month, JS will automatically adjust the year if needed + lastMonth.setMonth(lastMonth.getMonth() - 1); + const [year, month] = lastMonth.toISOString().split("T")[0].split("-"); + return `${year}${month}`; +}