Skip to content

Commit b258581

Browse files
djeebusdjeebus
committed
Merge branch 'main' into bring-back-cache-writes
2 parents b6a786c + 8a03edc commit b258581

File tree

195 files changed

+4487
-7742
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

195 files changed

+4487
-7742
lines changed

‎.github/actions/auto-request-same-site/script.js‎

Lines changed: 58 additions & 108 deletions
Original file line numberDiff line numberDiff line change
@@ -1,161 +1,111 @@
11
const { Octokit } = require("@octokit/rest");
22
const fs = require("fs");
33

4-
const token = process.env.APP_TOKEN; // GitHub App installation token
5-
const org = process.env.ORG;
6-
const sf = process.env.SF_TEAM_SLUG;
7-
const prg = process.env.PRG_TEAM_SLUG;
8-
const n = parseInt(process.env.REVIEWERS_TO_REQUEST || "1", 10);
9-
const TEAM_MODE = (process.env.TEAM_MODE || "false").toLowerCase() === "true";
4+
const token = process.env.APP_TOKEN;
5+
const org = process.env.ORG;
6+
const sf = process.env.SF_TEAM_SLUG;
7+
const prg = process.env.PRG_TEAM_SLUG;
8+
const n = Math.max(1, parseInt(process.env.REVIEWERS_TO_REQUEST || "1", 10));
109

1110
const gh = new Octokit({ auth: token });
1211
const [owner, repo] = process.env.GITHUB_REPOSITORY.split("/");
1312

14-
// ---- helpers ----
1513
function getEvent() {
1614
return JSON.parse(fs.readFileSync(process.env.GITHUB_EVENT_PATH, "utf8"));
1715
}
16+
1817
function prNumber(ev) {
19-
return ev.pull_request?.number || null;
18+
return ev.pull_request?.number ?? null;
2019
}
20+
2121
async function getPR(num) {
2222
const { data } = await gh.pulls.get({ owner, repo, pull_number: num });
2323
return data;
2424
}
25+
2526
async function getUserTeams(login) {
2627
const data = await gh.graphql(
2728
`query($org:String!, $login:String!){
2829
organization(login:$org){
2930
teams(first:100, userLogins: [$login]){ nodes { slug } }
3031
}
3132
}`,
32-
{ org, login }
33+
{ org, login },
3334
);
34-
return (data.organization?.teams?.nodes || []).map(t => t.slug);
35+
return (data.organization?.teams?.nodes || []).map((t) => t.slug);
3536
}
37+
3638
async function listTeamMembers(teamSlug) {
3739
const res = await gh.teams.listMembersInOrg({ org, team_slug: teamSlug, per_page: 100 });
38-
return res.data.map(u => u.login);
40+
return res.data.map((u) => u.login);
3941
}
42+
4043
function pickRandom(arr, k) {
41-
const a = [...arr];
42-
for (let i = a.length - 1; i > 0; i--) {
44+
const list = [...arr];
45+
for (let i = list.length - 1; i > 0; i--) {
4346
const j = Math.floor(Math.random() * (i + 1));
44-
[a[i], a[j]] = [a[j], a[i]];
45-
}
46-
return a.slice(0, Math.max(0, Math.min(k, a.length)));
47-
}
48-
49-
// Read CODEOWNERS (from usual locations) and parse the global "*" owners.
50-
// Returns { users: [logins], teams: [teamSlugs] }
51-
async function readDefaultCodeownersOwners(baseRef) {
52-
const paths = [".github/CODEOWNERS", "CODEOWNERS"];
53-
let text = "";
54-
for (const path of paths) {
55-
try {
56-
const { data } = await gh.repos.getContent({ owner, repo, path, ref: baseRef });
57-
if (Array.isArray(data)) continue; // directory
58-
text = Buffer.from(data.content, "base64").toString("utf8");
59-
break;
60-
} catch { /* try next */ }
61-
}
62-
if (!text) return { users: [], teams: [] };
63-
64-
// find last matching "*" rule (later rules take precedence)
65-
const lines = text.split(/\r?\n/);
66-
let starOwners = null;
67-
for (const raw of lines) {
68-
const line = raw.trim();
69-
if (!line || line.startsWith("#")) continue;
70-
const parts = line.split(/\s+/);
71-
if (parts[0] === "*") starOwners = parts.slice(1);
72-
}
73-
if (!starOwners) return { users: [], teams: [] };
74-
75-
const users = [];
76-
const teams = [];
77-
for (const ownerRef of starOwners) {
78-
if (!ownerRef.startsWith("@")) continue;
79-
const clean = ownerRef.slice(1); // remove leading '@'
80-
const slash = clean.indexOf("/");
81-
if (slash > -1) {
82-
// looks like org/team
83-
const maybeOrg = clean.slice(0, slash);
84-
const teamSlug = clean.slice(slash + 1);
85-
if (maybeOrg.toLowerCase() === org.toLowerCase()) teams.push(teamSlug);
86-
// if CODEOWNERS references another org's team, we ignore
87-
} else {
88-
users.push(clean);
89-
}
47+
[list[i], list[j]] = [list[j], list[i]];
9048
}
91-
return { users, teams };
49+
return list.slice(0, Math.max(0, Math.min(k, list.length)));
9250
}
9351

9452
(async () => {
9553
const ev = getEvent();
9654
const num = prNumber(ev);
97-
if (!num) { console.log("No PR number; exiting."); return; }
55+
if (!num) {
56+
console.log("No PR number in event; exiting.");
57+
return;
58+
}
9859

99-
const baseRef = ev.pull_request?.base?.ref || "main";
10060
const pr = await getPR(num);
10161
const author = pr.user.login;
62+
const alreadyAssigned = new Set((pr.assignees || []).map((a) => a.login));
10263

103-
// Determine author site
10464
const teams = await getUserTeams(author);
10565
const site = teams.includes(sf) ? sf : teams.includes(prg) ? prg : null;
10666

107-
// If author is not in eng-sf or eng-prg => do nothing (keep CODEOWNERS defaults)
10867
if (!site) {
109-
console.log("Author not in eng-sf or eng-prg; keeping CODEOWNERS reviewers.");
68+
console.log("Author not in configured teams; skipping assignee update.");
69+
return;
70+
}
71+
72+
const siteMembers = (await listTeamMembers(site)).filter((user) => user !== author);
73+
if (!siteMembers.length) {
74+
console.log(`No teammates found in ${site}; skipping assignee update.`);
11075
return;
11176
}
11277

113-
// Author IS internal: remove global CODEOWNERS defaults, then assign same-site reviewers
114-
const defaults = await readDefaultCodeownersOwners(baseRef);
115-
console.log("CODEOWNERS * defaults:", defaults);
116-
117-
// Find currently requested users & teams
118-
const { data: req } = await gh.pulls.listRequestedReviewers({ owner, repo, pull_number: num });
119-
const currentUsers = new Set(req.users.map(u => u.login));
120-
const currentTeams = new Set(req.teams.map(t => t.slug));
121-
122-
// Compute removal sets based on CODEOWNERS defaults
123-
const toRemoveUsers = defaults.users.filter(u => currentUsers.has(u));
124-
const toRemoveTeams = defaults.teams.filter(t => currentTeams.has(t));
125-
126-
if (toRemoveUsers.length || toRemoveTeams.length) {
127-
await gh.pulls.removeRequestedReviewers({
128-
owner, repo, pull_number: num,
129-
reviewers: toRemoveUsers,
130-
team_reviewers: toRemoveTeams
131-
});
132-
console.log(
133-
"Removed CODEOWNERS defaults:",
134-
toRemoveUsers.length ? `users=[${toRemoveUsers.join(", ")}]` : "users=[]",
135-
toRemoveTeams.length ? `teams=[${toRemoveTeams.join(", ")}]` : "teams=[]"
136-
);
137-
} else {
138-
console.log("No CODEOWNERS defaults currently requested (nothing to remove).");
78+
const siteMemberSet = new Set(siteMembers);
79+
const assignedFromSite = [...alreadyAssigned].filter((login) => siteMemberSet.has(login));
80+
81+
if (assignedFromSite.length >= n) {
82+
console.log(`PR #${num} already has ${assignedFromSite.length} teammate assignee(s); nothing to do.`);
83+
return;
13984
}
14085

141-
// Now request same-site reviewers
142-
if (TEAM_MODE) {
143-
await gh.pulls.requestReviewers({ owner, repo, pull_number: num, team_reviewers: [site] });
144-
console.log(`Requested team ${site}`);
86+
const needed = n - assignedFromSite.length;
87+
88+
const candidates = siteMembers.filter((member) => !alreadyAssigned.has(member));
89+
if (!candidates.length) {
90+
console.log(`All teammates from ${site} are already assigned; nothing to add.`);
14591
return;
14692
}
14793

148-
const siteMembers = (await listTeamMembers(site)).filter(u => u !== author);
149-
// refresh requested reviewers after potential removals
150-
const { data: req2 } = await gh.pulls.listRequestedReviewers({ owner, repo, pull_number: num });
151-
const already = new Set(req2.users.map(u => u.login));
152-
const candidates = siteMembers.filter(m => !already.has(m));
153-
const reviewers = pickRandom(candidates, n);
154-
155-
if (reviewers.length) {
156-
await gh.pulls.requestReviewers({ owner, repo, pull_number: num, reviewers });
157-
console.log(`Requested ${reviewers.join(", ")} from ${site}`);
158-
} else {
159-
console.log(`No candidates to request from ${site}.`);
94+
const assignees = pickRandom(candidates, needed);
95+
if (!assignees.length) {
96+
console.log(`Unable to select additional assignees from ${site}; skipping.`);
97+
return;
16098
}
161-
})().catch(e => { console.error(e); process.exit(1); });
99+
100+
await gh.issues.addAssignees({
101+
owner,
102+
repo,
103+
issue_number: num,
104+
assignees,
105+
});
106+
107+
console.log(`Assigned ${assignees.join(", ")} to PR #${num}.`);
108+
})().catch((error) => {
109+
console.error(error);
110+
process.exit(1);
111+
});

‎.github/workflows/auto-request-same-site.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,14 @@ name: Auto-request same-site reviewers
33
on:
44
# Works for PRs from branches and forks (runs in base-repo context)
55
pull_request_target:
6-
types: [opened]
6+
types:
7+
- opened
8+
- ready_for_review
79

810
permissions:
911
contents: read
12+
pull-requests: write
13+
issues: write
1014

1115
jobs:
1216
assign:

‎DEV-LOCAL.md‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# Develop the application locally
2+
3+
> Note: Linux is required for developing on bare metal. This is a work in progress. Not everything will function as expected.
4+
5+
1. `sudo modprobe nbd nbds_max=64`
6+
2. `make local-infra`: runs clickhouse, grafana, loki, memcached, mimir, otel, postgres, redis, tempo
7+
3. `cd packages/db && make migrate-local` initialize the database
8+
4. `cd packages/local-dev && go run seed-local-database.go` generate user, team, and token for local development
9+
5. `cd packages/api && make run-local` run the api locally
10+
6. `cd packages/orchestrator && make build-debug && sudo make run-local` run the orchestrator and template-manager locally.
11+
7. `cd packages/client-proxy && make run-local` run the client-proxy locally.
12+
13+
# Services
14+
- grafana: http://localhost:53000)
15+
- postgres: postgres:postgres@127.0.0.1:5432
16+
- clickhouse: clickhouse:clickhouse@127.0.0.1:9000

‎Makefile‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,3 +162,7 @@ generate-mocks:
162162
.PHONY: tidy
163163
tidy:
164164
scripts/golang-dependencies-integrity.sh
165+
166+
.PHONY: local-infra
167+
local-infra:
168+
docker compose --file ./packages/local-dev/docker-compose.yaml up --abort-on-container-failure

‎go.work‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ use (
77
./packages/db
88
./packages/docker-reverse-proxy
99
./packages/envd
10+
./packages/local-dev
1011
./packages/orchestrator
1112
./packages/shared
1213

‎iac/provider-gcp/nomad/jobs/edge.hcl‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -71,10 +71,10 @@ job "client-proxy" {
7171
%{ if update_stanza }
7272
# An update stanza to enable rolling updates of the service
7373
update {
74-
# The number of extra instances to run during the update
74+
# The number of instances that can be updated at the same time
7575
max_parallel = ${update_max_parallel}
76-
# Allows to spawn new version of the service before killing the old one
77-
canary = 1
76+
# Number of extra instances that can be spawn before killing the old one
77+
canary = ${update_max_parallel}
7878
# Time to wait for the canary to be healthy
7979
min_healthy_time = "10s"
8080
# Time to wait for the canary to be healthy, if not it will be marked as failed
@@ -110,15 +110,15 @@ job "client-proxy" {
110110
PROXY_PORT = "${proxy_port}"
111111
ORCHESTRATOR_PORT = "${orchestrator_port}"
112112

113-
SD_ORCHESTRATOR_PROVIDER = "NOMAD"
114-
SD_ORCHESTRATOR_NOMAD_ENDPOINT = "${nomad_endpoint}"
115-
SD_ORCHESTRATOR_NOMAD_TOKEN = "${nomad_token}"
116-
SD_ORCHESTRATOR_JOB_PREFIX = "template-manager"
113+
SD_ORCHESTRATOR_PROVIDER = "NOMAD"
114+
SD_ORCHESTRATOR_NOMAD_ENDPOINT = "${nomad_endpoint}"
115+
SD_ORCHESTRATOR_NOMAD_TOKEN = "${nomad_token}"
116+
SD_ORCHESTRATOR_NOMAD_JOB_PREFIX = "template-manager"
117117

118-
SD_EDGE_PROVIDER = "NOMAD"
119-
SD_EDGE_NOMAD_ENDPOINT = "${nomad_endpoint}"
120-
SD_EDGE_NOMAD_TOKEN = "${nomad_token}"
121-
SD_EDGE_JOB_PREFIX = "client-proxy"
118+
SD_EDGE_PROVIDER = "NOMAD"
119+
SD_EDGE_NOMAD_ENDPOINT = "${nomad_endpoint}"
120+
SD_EDGE_NOMAD_TOKEN = "${nomad_token}"
121+
SD_EDGE_NOMAD_JOB_PREFIX = "client-proxy"
122122

123123
ENVIRONMENT = "${environment}"
124124

‎packages/api/.env.local‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
DNS_PORT=9953
2+
ENVIRONMENT=local
3+
OTEL_COLLECTOR_GRPC_ENDPOINT=localhost:4317
4+
POSTGRES_CONNECTION_STRING=postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable
5+
REDIS_URL=localhost:6379
6+
SANDBOX_ACCESS_TOKEN_HASH_SEED=--sandbox-access-token-hash-seed--
7+
SD_ORCHESTRATOR_PROVIDER=STATIC
8+
SD_ORCHESTRATOR_STATIC=127.0.0.1
9+
SD_EDGE_PROVIDER=STATIC
10+
SD_EDGE_STATIC=127.0.0.1

‎packages/api/Makefile‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,17 @@ run:
4343
LOCAL_CLUSTER_TOKEN=$(EDGE_TOKEN) \
4444
./bin/api --port 3000
4545

46+
define setup_local_env
47+
$(eval include .env.local)
48+
$(eval export $(shell sed 's/=.*//' .env.local))
49+
endef
50+
51+
.PHONY: run-local
52+
run-local:
53+
make build-debug
54+
$(call setup_local_env)
55+
NODE_ID=$$(hostname) ./bin/api --port 3000
56+
4657
# Run the API using air
4758
.PHONY: dev
4859
dev:

‎packages/api/go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ require (
7272

7373
require (
7474
ariga.io/atlas v0.15.0 // indirect
75-
dario.cat/mergo v1.0.1 // indirect
75+
dario.cat/mergo v1.0.2 // indirect
7676
entgo.io/ent v0.12.5 // indirect
7777
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 // indirect
7878
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 // indirect

‎packages/api/go.sum‎

Lines changed: 4 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)