Skip to content

Commit

Permalink
gh metrics
Browse files Browse the repository at this point in the history
  • Loading branch information
@leordev
leordev committed Sep 3, 2024
1 parent 2950603 commit 5a73a4c
Show file tree
Hide file tree
Showing 5 changed files with 97 additions and 82 deletions.
132 changes: 70 additions & 62 deletions metrics-collector/src/gh-metrics.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import * as fs from "fs";
import * as path from "path";
import { createObjectCsvWriter } from "csv-writer";
import { readJsonFile, writeJsonFile } from "./utils";
import { isSameDay } from "date-fns";
import { isSameDay } from "date-fns";``
import { MetricPayload, postMetric } from "./post-metric";

const orgName = "TBD54566975";
Expand Down Expand Up @@ -181,43 +181,66 @@ const postGhMetrics = async (metrics: GHMetrics) => {
const timestamp = metrics.metricDate.toISOString();

// issues metrics
const { internalIssues, externalIssues, botIssues } = await getIssueMetrics(
const issues = await itemByUserType(
metrics.issues
);
// internal issues metrics
ghMetrics.push(
...internalIssues.map((issue) =>
ghAuthoredMetric(
"gh_issues",
{ ...labels, source_type: "internal" },
issue.created_at,
issue.user?.login
)
)
);
// external issues metrics
ghMetrics.push(
...externalIssues.map((issue) =>
ghAuthoredMetric(
"gh_issues",
{ ...labels, source_type: "external" },
issue.created_at,
issue.user?.login
)
)
);
// bot issues metrics
ghMetrics.push(
...internalIssues.map((issue) =>
ghAuthoredMetric(
"gh_issues",
{ ...labels, source_type: "bot" },
issue.created_at,
issue.user?.login
)
)
const issueTypes = [
{ issues: issues.internal, userType: "internal" },
{ issues: issues.external, userType: "external" },
{ issues: issues.bot, userType: "bot" }
];
issueTypes.forEach(({ issues, userType }) => {
ghMetrics.push(
...issues.map((issue) => ({
metricName: "gh_issues",
value: 1,
labels: { ...labels, userType },
timestamp: issue.created_at,
user: issue.user?.login,
}))
);
});

// comments metrics
const comments = await itemByUserType(
metrics.comments
);
const commentTypes = [
{ comments: comments.internal, userType: "internal" },
{ comments: comments.external, userType: "external" },
{ comments: comments.bot, userType: "bot" }
];
commentTypes.forEach(({ comments, userType }) => {
ghMetrics.push(
...comments.map((comment) => ({
metricName: "gh_comments",
value: 1,
labels: { ...labels, userType },
timestamp: comment.created_at,
user: comment.user?.login,
}))
);
});

// prs metrics
const prs = await itemByUserType(metrics.prs);
const prTypes = [
{ prs: prs.internal, userType: "internal" },
{ prs: prs.external, userType: "external" },
{ prs: prs.bot, userType: "bot" }
];
prTypes.forEach(({ prs, userType }) => {
ghMetrics.push(
...prs.map((pr) => ({
metricName: "gh_prs",
value: 1,
labels: { ...labels, userType },
timestamp: pr.created_at,
user: pr.user?.login,
}))
);
});

// clones metrics
ghMetrics.push({
metricName: "gh_clones",
Expand All @@ -239,39 +262,24 @@ const postGhMetrics = async (metrics: GHMetrics) => {
console.info("GH metrics posted successfully");
};

const ghAuthoredMetric = (
metricName: string,
labels: any,
timestamp: string,
user = "unknown"
) => ({
metricName: "gh_issues_internal",
value: 1,
labels: {
...labels,
user,
},
timestamp,
});

async function getIssueMetrics(issues: IssueData[]) {
const internalIssues = [];
const externalIssues = [];
const botIssues = [];
for (const issue of issues) {
if (!issue.user) {
console.error("Issue user not found!", issue);
async function itemByUserType(items: CommentData[] | IssueData[] | PullRequestData[]) {
const internal = [];
const external = [];
const bot = [];
for (const item of items) {
if (!item.user) {
console.error("Comment user not found!", item);
throw new Error("Issue user not found!");
}
if (issue.user.type === "Bot") {
botIssues.push(issue);
} else if (await isMember(orgName, issue.user.login)) {
internalIssues.push(issue);
if (item.user.type === "Bot") {
bot.push(item);
} else if (await isMember(orgName, item.user.login)) {
internal.push(item);
} else {
externalIssues.push(issue);
external.push(item);
}
}
return { internalIssues, externalIssues, botIssues };
return { internal, external, bot };
}

export async function saveGhMetrics(isLocalPersistence: boolean = false) {
Expand Down
31 changes: 18 additions & 13 deletions metrics-collector/src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ dotenv.config();
import yargs from "yargs";
import { hideBin } from "yargs/helpers";

import { saveGhMetrics } from "./gh-metrics";
import { collectGhMetrics, saveGhMetrics } from "./gh-metrics";
import { collectNpmMetrics } from "./npm-metrics";
import {
collectSonatypeMetrics,
Expand All @@ -13,6 +13,7 @@ import {
import { getYesterdayDate, readJsonFile } from "./utils";
import { readFile, writeFile } from "fs/promises";
import { existsSync, mkdirSync } from "fs";
import { addDays, addMonths } from "date-fns";

const isLocalPersistence = process.env.PERSIST_LOCAL_FILES === "true";

Expand Down Expand Up @@ -80,7 +81,7 @@ async function main() {
collectNpmMetrics
);
} else {
await collectNpmMetrics(metricDateStr);
await collectNpmMetrics(metricDate);
}
}

Expand All @@ -98,14 +99,23 @@ async function main() {
true
);
} else {
await collectSonatypeMetrics(metricDateStr);
await collectSonatypeMetrics(metricDate);
}
}

const collectGh = argv["collect-gh"];
if (collectGh) {
console.info(`\n\n============\n\n>>> Collecting metrics for GitHub...`);
await saveGhMetrics();
if (initialLoadFromDate) {
await initialLoad(
"gh-metrics",
initialLoadFromDate,
metricDate,
collectGhMetrics
);
} else {
await collectGhMetrics(metricDate);
}
}

const localCollection = !collectGh && !collectNpm && !collectSonatype;
Expand All @@ -121,13 +131,13 @@ async function initialLoad(
metricName: string,
initialLoadFromDate: Date,
initialLoadToDate: Date,
collectMetrics: (metricDate: string) => Promise<void>,
collectMetrics: (metricDate: Date) => Promise<void>,
monthlyInterval = false,
skipLastSavedState = false
) {
const lastSavedState =
!skipLastSavedState && (await getLastSavedState(metricName));
const date = lastSavedState || initialLoadFromDate;
let date = lastSavedState || initialLoadFromDate;

console.info(
`Initial load from ${initialLoadFromDate} to ${initialLoadToDate} with date ${date}`
Expand All @@ -136,13 +146,8 @@ async function initialLoad(
while (date <= initialLoadToDate) {
const dateStr = date.toISOString().split("T")[0];
console.log(`\n\n>>> Collecting metric ${metricName} for date: ${dateStr}`);
await collectMetrics(dateStr);
if (monthlyInterval) {
// Move to the next month (JS will handle year change automatically)
date.setMonth(date.getMonth() + 1);
} else {
date.setDate(date.getDate() + 1);
}
await collectMetrics(date);
date = monthlyInterval ? addMonths(date, 1) : addDays(date, 1);
await saveLastSavedState(metricName, date);
}
}
Expand Down
7 changes: 4 additions & 3 deletions metrics-collector/src/npm-metrics.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,19 +26,20 @@ const dataFilePath = path.join(process.cwd(), "npm_metrics.json");
const csvFilePath = path.join(process.cwd(), "npm_metrics.csv");

// Push collected metrics to the metrics service
export const collectNpmMetrics = async (metricDate: string) => {
export const collectNpmMetrics = async (metricDate: Date) => {
const metricDateStr = metricDate.toISOString().split("T")[0];
for (const pkg of npmPackages) {
const { downloads: totalDownloads } = await getNpmDownloadCount(
pkg,
false,
{ begin: "1970-01-01", end: metricDate }
{ begin: "1970-01-01", end: metricDateStr }
);

// Collect daily downloads too
const { downloads: dailyDownloads } = await getNpmDownloadCount(
pkg,
false,
{ begin: metricDate, end: metricDate }
{ begin: metricDateStr, end: metricDateStr }
);

console.info(`\n\n============\n\n>>> Collected metrics for ${pkg}...`);
Expand Down
1 change: 1 addition & 0 deletions metrics-collector/src/post-metric.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ export interface MetricPayload {

export const postMetric = async (payload: MetricPayload): Promise<void> => {
payload.timestamp = payload.timestamp ?? new Date().toISOString();
console.info({ payload });

const response = await fetchWithRetry(`${metricsServiceAppUrl}/metrics`, {
method: "POST",
Expand Down
8 changes: 4 additions & 4 deletions metrics-collector/src/sonatype-metrics.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ const requestHeaders: Record<string, string> = {
const sonatypeCentralStatsUrl =
"https://s01.oss.sonatype.org/service/local/stats";

export async function collectSonatypeMetrics(metricDate: string) {
export async function collectSonatypeMetrics(metricDate: Date) {
initAuth();

const projectId = await getProjectId(groupId);
Expand Down Expand Up @@ -48,7 +48,7 @@ export async function collectSonatypeMetrics(metricDate: string) {

await postSonatypeMavenMetrics({
artifact,
metricDate: new Date(metricDate),
metricDate,
reportPeriod,
rawDownloads: rawDownloads.total,
uniqueIPs: uniqueIPs.total,
Expand Down Expand Up @@ -280,8 +280,8 @@ function getLastMonthDate() {
return `${lastMonthYear}${String(lastMonth).padStart(2, "0")}`;
}

function getLastMonthPeriod(date: string): string {
const parsedDate = parse(date, "yyyy-MM-dd", new Date());
function getLastMonthPeriod(date: Date): string {
const parsedDate = parse(date.toISOString(), "yyyy-MM-dd", new Date());
const previousMonth = subMonths(parsedDate, 1);
return format(previousMonth, "yyyy-MM");
}

0 comments on commit 5a73a4c

Please sign in to comment.