diff --git a/metrics-collector/src/gh-metrics.ts b/metrics-collector/src/gh-metrics.ts index e24fd63..9a19d97 100644 --- a/metrics-collector/src/gh-metrics.ts +++ b/metrics-collector/src/gh-metrics.ts @@ -4,7 +4,7 @@ import * as fs from "fs"; import * as path from "path"; import { createObjectCsvWriter } from "csv-writer"; import { readJsonFile, writeJsonFile } from "./utils"; -import { isSameDay } from "date-fns"; +import { isSameDay } from "date-fns";`` import { MetricPayload, postMetric } from "./post-metric"; const orgName = "TBD54566975"; @@ -181,43 +181,66 @@ const postGhMetrics = async (metrics: GHMetrics) => { const timestamp = metrics.metricDate.toISOString(); // issues metrics - const { internalIssues, externalIssues, botIssues } = await getIssueMetrics( + const issues = await itemByUserType( metrics.issues ); - // internal issues metrics - ghMetrics.push( - ...internalIssues.map((issue) => - ghAuthoredMetric( - "gh_issues", - { ...labels, source_type: "internal" }, - issue.created_at, - issue.user?.login - ) - ) - ); - // external issues metrics - ghMetrics.push( - ...externalIssues.map((issue) => - ghAuthoredMetric( - "gh_issues", - { ...labels, source_type: "external" }, - issue.created_at, - issue.user?.login - ) - ) - ); - // bot issues metrics - ghMetrics.push( - ...internalIssues.map((issue) => - ghAuthoredMetric( - "gh_issues", - { ...labels, source_type: "bot" }, - issue.created_at, - issue.user?.login - ) - ) + const issueTypes = [ + { issues: issues.internal, userType: "internal" }, + { issues: issues.external, userType: "external" }, + { issues: issues.bot, userType: "bot" } + ]; + issueTypes.forEach(({ issues, userType }) => { + ghMetrics.push( + ...issues.map((issue) => ({ + metricName: "gh_issues", + value: 1, + labels: { ...labels, userType }, + timestamp: issue.created_at, + user: issue.user?.login, + })) + ); + }); + + // comments metrics + const comments = await itemByUserType( + metrics.comments ); + const commentTypes = [ + { comments: comments.internal, userType: "internal" }, + { comments: comments.external, userType: "external" }, + { comments: comments.bot, userType: "bot" } + ]; + commentTypes.forEach(({ comments, userType }) => { + ghMetrics.push( + ...comments.map((comment) => ({ + metricName: "gh_comments", + value: 1, + labels: { ...labels, userType }, + timestamp: comment.created_at, + user: comment.user?.login, + })) + ); + }); + // prs metrics + const prs = await itemByUserType(metrics.prs); + const prTypes = [ + { prs: prs.internal, userType: "internal" }, + { prs: prs.external, userType: "external" }, + { prs: prs.bot, userType: "bot" } + ]; + prTypes.forEach(({ prs, userType }) => { + ghMetrics.push( + ...prs.map((pr) => ({ + metricName: "gh_prs", + value: 1, + labels: { ...labels, userType }, + timestamp: pr.created_at, + user: pr.user?.login, + })) + ); + }); + // clones metrics ghMetrics.push({ metricName: "gh_clones", @@ -239,39 +262,24 @@ const postGhMetrics = async (metrics: GHMetrics) => { console.info("GH metrics posted successfully"); }; -const ghAuthoredMetric = ( - metricName: string, - labels: any, - timestamp: string, - user = "unknown" -) => ({ - metricName: "gh_issues_internal", - value: 1, - labels: { - ...labels, - user, - }, - timestamp, -}); - -async function getIssueMetrics(issues: IssueData[]) { - const internalIssues = []; - const externalIssues = []; - const botIssues = []; - for (const issue of issues) { - if (!issue.user) { - console.error("Issue user not found!", issue); +async function itemByUserType(items: CommentData[] | IssueData[] | PullRequestData[]) { + const internal = []; + const external = []; + const bot = []; + for (const item of items) { + if (!item.user) { + console.error("Comment user not found!", item); throw new Error("Issue user not found!"); } - if (issue.user.type === "Bot") { - botIssues.push(issue); - } else if (await isMember(orgName, issue.user.login)) { - internalIssues.push(issue); + if (item.user.type === "Bot") { + bot.push(item); + } else if (await isMember(orgName, item.user.login)) { + internal.push(item); } else { - externalIssues.push(issue); + external.push(item); } } - return { internalIssues, externalIssues, botIssues }; + return { internal, external, bot }; } export async function saveGhMetrics(isLocalPersistence: boolean = false) { diff --git a/metrics-collector/src/index.ts b/metrics-collector/src/index.ts index 8f48016..967af03 100644 --- a/metrics-collector/src/index.ts +++ b/metrics-collector/src/index.ts @@ -4,7 +4,7 @@ dotenv.config(); import yargs from "yargs"; import { hideBin } from "yargs/helpers"; -import { saveGhMetrics } from "./gh-metrics"; +import { collectGhMetrics, saveGhMetrics } from "./gh-metrics"; import { collectNpmMetrics } from "./npm-metrics"; import { collectSonatypeMetrics, @@ -13,6 +13,7 @@ import { import { getYesterdayDate, readJsonFile } from "./utils"; import { readFile, writeFile } from "fs/promises"; import { existsSync, mkdirSync } from "fs"; +import { addDays, addMonths } from "date-fns"; const isLocalPersistence = process.env.PERSIST_LOCAL_FILES === "true"; @@ -80,7 +81,7 @@ async function main() { collectNpmMetrics ); } else { - await collectNpmMetrics(metricDateStr); + await collectNpmMetrics(metricDate); } } @@ -98,14 +99,23 @@ async function main() { true ); } else { - await collectSonatypeMetrics(metricDateStr); + await collectSonatypeMetrics(metricDate); } } const collectGh = argv["collect-gh"]; if (collectGh) { console.info(`\n\n============\n\n>>> Collecting metrics for GitHub...`); - await saveGhMetrics(); + if (initialLoadFromDate) { + await initialLoad( + "gh-metrics", + initialLoadFromDate, + metricDate, + collectGhMetrics + ); + } else { + await collectGhMetrics(metricDate); + } } const localCollection = !collectGh && !collectNpm && !collectSonatype; @@ -121,13 +131,13 @@ async function initialLoad( metricName: string, initialLoadFromDate: Date, initialLoadToDate: Date, - collectMetrics: (metricDate: string) => Promise, + collectMetrics: (metricDate: Date) => Promise, monthlyInterval = false, skipLastSavedState = false ) { const lastSavedState = !skipLastSavedState && (await getLastSavedState(metricName)); - const date = lastSavedState || initialLoadFromDate; + let date = lastSavedState || initialLoadFromDate; console.info( `Initial load from ${initialLoadFromDate} to ${initialLoadToDate} with date ${date}` @@ -136,13 +146,8 @@ async function initialLoad( while (date <= initialLoadToDate) { const dateStr = date.toISOString().split("T")[0]; console.log(`\n\n>>> Collecting metric ${metricName} for date: ${dateStr}`); - await collectMetrics(dateStr); - if (monthlyInterval) { - // Move to the next month (JS will handle year change automatically) - date.setMonth(date.getMonth() + 1); - } else { - date.setDate(date.getDate() + 1); - } + await collectMetrics(date); + date = monthlyInterval ? addMonths(date, 1) : addDays(date, 1); await saveLastSavedState(metricName, date); } } diff --git a/metrics-collector/src/npm-metrics.ts b/metrics-collector/src/npm-metrics.ts index 012a675..7dd1efd 100644 --- a/metrics-collector/src/npm-metrics.ts +++ b/metrics-collector/src/npm-metrics.ts @@ -26,19 +26,20 @@ const dataFilePath = path.join(process.cwd(), "npm_metrics.json"); const csvFilePath = path.join(process.cwd(), "npm_metrics.csv"); // Push collected metrics to the metrics service -export const collectNpmMetrics = async (metricDate: string) => { +export const collectNpmMetrics = async (metricDate: Date) => { + const metricDateStr = metricDate.toISOString().split("T")[0]; for (const pkg of npmPackages) { const { downloads: totalDownloads } = await getNpmDownloadCount( pkg, false, - { begin: "1970-01-01", end: metricDate } + { begin: "1970-01-01", end: metricDateStr } ); // Collect daily downloads too const { downloads: dailyDownloads } = await getNpmDownloadCount( pkg, false, - { begin: metricDate, end: metricDate } + { begin: metricDateStr, end: metricDateStr } ); console.info(`\n\n============\n\n>>> Collected metrics for ${pkg}...`); diff --git a/metrics-collector/src/post-metric.ts b/metrics-collector/src/post-metric.ts index ff5b2cd..e3ce10f 100644 --- a/metrics-collector/src/post-metric.ts +++ b/metrics-collector/src/post-metric.ts @@ -15,6 +15,7 @@ export interface MetricPayload { export const postMetric = async (payload: MetricPayload): Promise => { payload.timestamp = payload.timestamp ?? new Date().toISOString(); + console.info({ payload }); const response = await fetchWithRetry(`${metricsServiceAppUrl}/metrics`, { method: "POST", diff --git a/metrics-collector/src/sonatype-metrics.ts b/metrics-collector/src/sonatype-metrics.ts index 6861bec..4f8a511 100644 --- a/metrics-collector/src/sonatype-metrics.ts +++ b/metrics-collector/src/sonatype-metrics.ts @@ -17,7 +17,7 @@ const requestHeaders: Record = { const sonatypeCentralStatsUrl = "https://s01.oss.sonatype.org/service/local/stats"; -export async function collectSonatypeMetrics(metricDate: string) { +export async function collectSonatypeMetrics(metricDate: Date) { initAuth(); const projectId = await getProjectId(groupId); @@ -48,7 +48,7 @@ export async function collectSonatypeMetrics(metricDate: string) { await postSonatypeMavenMetrics({ artifact, - metricDate: new Date(metricDate), + metricDate, reportPeriod, rawDownloads: rawDownloads.total, uniqueIPs: uniqueIPs.total, @@ -280,8 +280,8 @@ function getLastMonthDate() { return `${lastMonthYear}${String(lastMonth).padStart(2, "0")}`; } -function getLastMonthPeriod(date: string): string { - const parsedDate = parse(date, "yyyy-MM-dd", new Date()); +function getLastMonthPeriod(date: Date): string { + const parsedDate = parse(date.toISOString(), "yyyy-MM-dd", new Date()); const previousMonth = subMonths(parsedDate, 1); return format(previousMonth, "yyyy-MM"); }