diff --git a/backend/build.sh b/backend/build.sh index 768318f..e69fef1 100755 --- a/backend/build.sh +++ b/backend/build.sh @@ -4,7 +4,7 @@ set -euo pipefail cd "$(dirname "$0")" mkdir -p dist -for fn in api sync enrich; do +for fn in api sync enrich emailsync; do GOOS=linux GOARCH=arm64 CGO_ENABLED=0 go build -tags lambda.norpc -o dist/bootstrap "./cmd/$fn" (cd dist && zip -q "$fn.zip" bootstrap && rm bootstrap) echo "built dist/$fn.zip" diff --git a/backend/cmd/api/main.go b/backend/cmd/api/main.go index 47904f0..b006445 100644 --- a/backend/cmd/api/main.go +++ b/backend/cmd/api/main.go @@ -30,6 +30,7 @@ func main() { AdminClerkID: config.Get(ctx, "ADMIN_CLERK_ID"), Season: season, Session: config.Get(ctx, "LEETCODE_SESSION"), + GithubToken: config.Get(ctx, "GITHUB_TOKEN"), } lambda.Start(handler.Handle) } diff --git a/backend/cmd/emailsync/main.go b/backend/cmd/emailsync/main.go new file mode 100644 index 0000000..5c0c395 --- /dev/null +++ b/backend/cmd/emailsync/main.go @@ -0,0 +1,329 @@ +package main + +import ( + "context" + "database/sql" + "fmt" + "log" + "os" + "regexp" + "strings" + "time" + + "github.com/aws/aws-lambda-go/lambda" + _ "github.com/jackc/pgx/v5/stdlib" +) + +/* +This file contains a Lambda function for the job email sync feature that: +- Opens a Postgres database from PG_DSN. +- Loads active email accounts for users. +- Fetches recent emails for each account (currently a stub). +- Detects job-related emails and classifies them. +- Extracts company and due date information from Online Assessment emails. +- Writes application status rows into the database. + +Main functions to read in order: +- main -> handler -> openDB -> loadActiveEmailAccounts -> processEmailAccount -> saveApplicationStatus + +This Lambda is meant to run on a schedule from EventBridge, enrich email data, and support the Kronos job dashboard. +*/ + +// EmailMessage is the simple shape of an email that the Lambda processes. +// It contains only the fields we need for job detection and status extraction. +type EmailMessage struct { + ID string // The email provider's unique message identifier. + From string // The sender address or display name. + Subject string // The email subject line. + Body string // The plain text body of the email. + Date time.Time // When the email was sent. +} + +// EmailAccount represents a connected mail account for a user. +type EmailAccount struct { + UserID int // Local Kronos user identifier. + Provider string // e.g. gmail, outlook. + AccessToken string // OAuth token used to read email. + Active bool // Only active accounts are processed. +} + +// ApplicationStatus is the row we want to write into Postgres. +type ApplicationStatus struct { + UserID int + Company string + Role string + Status string + DueDate sql.NullTime + SourceEmailID string +} + +const ( + StatusOaReceived = "OA_RECEIVED" + StatusInterviewInvite = "INTERVIEW_INVITE" + StatusRejection = "REJECTION" + StatusOther = "OTHER" +) + +func main() { + lambda.Start(handler) +} + +// handler is the Lambda entry point. +// EventBridge invokes this on a schedule. The event payload is ignored here +// because this Lambda only needs to run periodically and scan user emails. +func handler(ctx context.Context, event interface{}) error { + // ctx is the context to timeout database operations if they take too long + // step 1: open database connection + db, err := openDB(ctx) // connects to Postgres using PG_DSN + if err != nil { + return fmt.Errorf("open db: %w", err) // if it fails, return an error to Lambda so it can retry later + } + defer db.Close() // no matter what happens, close the database connection when we're done + + // step 2: load active email accounts from the database + accounts, err := loadActiveEmailAccounts(ctx, db) // fetches email_accounts rows where active = true + if err != nil { + return fmt.Errorf("load email accounts: %w", err) // if it fails, return an error to Lambda so it can retry later + } + + // step 3: process each active email account + for _, account := range accounts { // for each active email account, process it + if err := processEmailAccount(ctx, db, account); err != nil { // fetches new emails, checks if job-related, classifies them, and saves application statuses + log.Printf("user %d: %v", account.UserID, err) // if an error occurs, log it but continue processing other accounts + } + } + + return nil // no error, Lambda will consider this invocation successful +} + +// openDB opens a connection pool to Postgres using PG_DSN. +// sql.DB is a pool of connections, not a single database connection. +func openDB(ctx context.Context) (*sql.DB, error) { + // step 1: get connection string from environment variable + dsn := os.Getenv("PG_DSN") // read env variable + if dsn == "" { // check if it's empty + return nil, fmt.Errorf("PG_DSN is not set") // this prevents trying to connect with no credentials + } + + // step 2: open a database connection pool object that will be used to connect when needed. This does not actually connect yet. + db, err := sql.Open("pgx", dsn) // creates a connection pool to Postgres using the pgx driver (dsn is the connection string: username, pass, host, db name, etc.) + if err != nil { // if opening fails, return an error + return nil, err + } + + // step 3: Verify the connection works / verify the database is reachable. + if err := db.PingContext(ctx); err != nil { // sends a ping to the database to check if it's reachable + db.Close() + return nil, err + } + + // step 4: return the database connection pool to the caller. The caller is responsible for closing it when done. + return db, nil +} + +// loadActiveEmailAccounts fetches the email_accounts rows that are active. +// It returns only the fields we need to process email. +func loadActiveEmailAccounts(ctx context.Context, db *sql.DB) ([]EmailAccount, error) { + rows, err := db.QueryContext(ctx, ` + select user_id, email_provider, access_token, active + from email_accounts + where active = true + `) + if err != nil { + return nil, err + } + defer rows.Close() + + var accounts []EmailAccount + for rows.Next() { + var account EmailAccount + if err := rows.Scan(&account.UserID, &account.Provider, &account.AccessToken, &account.Active); err != nil { + return nil, err + } + accounts = append(accounts, account) + } + return accounts, rows.Err() +} + +// processEmailAccount handles one user's email account. +// It fetches new emails, finds job-related messages, and saves statuses. +func processEmailAccount(ctx context.Context, db *sql.DB, account EmailAccount) error { + emails, err := fetchNewEmailsForUser(ctx, account) + if err != nil { + return fmt.Errorf("fetch emails: %w", err) + } + + for _, email := range emails { + if !isJobEmail(email) { + continue + } + + status := classifyEmail(email) + if status == StatusOther { + continue + } + + company := extractCompany(email) + dueDate, _ := extractDueDate(email) + + st := ApplicationStatus{ + UserID: account.UserID, + Company: company, + Role: "", + Status: status, + DueDate: dueDate, + SourceEmailID: email.ID, + } + + if err := saveApplicationStatus(ctx, db, st); err != nil { + log.Printf("save status user %d email %s: %v", account.UserID, email.ID, err) + } + } + + return nil +} + +// fetchNewEmailsForUser is a placeholder stub that returns sample emails. +// In a real implementation, this would call Gmail or Outlook APIs using OAuth tokens. +func fetchNewEmailsForUser(ctx context.Context, account EmailAccount) ([]EmailMessage, error) { + _ = ctx + _ = account + + // TODO: Replace this stub with real API calls. + // The real function should use account.AccessToken and account.Provider to + // list recent messages, fetch their subject/body/from/date/id, and return them. + return []EmailMessage{ + { + ID: "sample-1", + From: "no-reply@company.com", + Subject: "Your Online Assessment is ready", + Body: "Hi, please complete your Online Assessment by 08/15/2026.", + Date: time.Now().Add(-2 * time.Hour), + }, + { + ID: "sample-2", + From: "recruiter@anotherco.com", + Subject: "Interview Invitation for Software Engineer", + Body: "We would like to invite you to interview next week.", + Date: time.Now().Add(-24 * time.Hour), + }, + }, nil +} + +// isJobEmail checks whether the email looks related to a job application. +// It uses simple keyword checks in the subject and body. +func isJobEmail(email EmailMessage) bool { + text := strings.ToLower(email.Subject + " " + email.Body) + keywords := []string{ + "online assessment", + "oa", + "interview", + "rejection", + "offer", + "application", + } + + for _, keyword := range keywords { + if strings.Contains(text, keyword) { + return true + } + } + return false +} + +// classifyEmail returns one of the top-level job email categories. +// This is a simple keyword-based classifier and can be upgraded later. +func classifyEmail(email EmailMessage) string { + text := strings.ToLower(email.Subject + " " + email.Body) + + if strings.Contains(text, "online assessment") || strings.Contains(text, "oa") { + return StatusOaReceived + } + if strings.Contains(text, "interview") || strings.Contains(text, "panel") { + return StatusInterviewInvite + } + if strings.Contains(text, "rejected") || strings.Contains(text, "regret") || strings.Contains(text, "not selected") { + return StatusRejection + } + return StatusOther +} + +// extractCompany uses a simple heuristic to guess the company name. +// It checks the sender address and the subject line. +func extractCompany(email EmailMessage) string { + cleanedFrom := strings.TrimSpace(strings.Split(email.From, "<")[0]) + if cleanedFrom != "" && cleanedFrom != email.From { + return cleanedFrom + } + + subject := strings.ToLower(email.Subject) + patterns := []string{" at ", " from ", " for "} + for _, pat := range patterns { + if idx := strings.Index(subject, pat); idx != -1 { + candidate := strings.TrimSpace(email.Subject[idx+len(pat):]) + if candidate != "" { + return candidate + } + } + } + + // Fallback to the raw sender or subject when we cannot parse a company. + if email.From != "" { + return email.From + } + return email.Subject +} + +var datePattern = regexp.MustCompile(`(?i)(\b\d{1,2}/\d{1,2}/\d{2,4}\b)|(\b\d{4}-\d{1,2}-\d{1,2}\b)`) // MM/DD/YYYY or YYYY-MM-DD + +// extractDueDate looks for a simple date pattern in the email text. +// It returns sql.NullTime so we can store NULL in the database when no date is found. +func extractDueDate(email EmailMessage) (sql.NullTime, error) { + text := email.Subject + " " + email.Body + match := datePattern.FindString(text) + if match == "" { + return sql.NullTime{Valid: false}, nil + } + + layouts := []string{"1/2/2006", "01/02/2006", "2006-01-02"} + for _, layout := range layouts { + if due, err := time.Parse(layout, match); err == nil { + return sql.NullTime{Time: due, Valid: true}, nil + } + } + + return sql.NullTime{Valid: false}, nil +} + +// saveApplicationStatus inserts or updates a row in application_statuses. +// It uses ON CONFLICT to avoid duplicate rows for the same email message. +func saveApplicationStatus(ctx context.Context, db *sql.DB, status ApplicationStatus) error { + _, err := db.ExecContext(ctx, ` + insert into application_statuses + (user_id, company, role, status, due_date, source_email_id, created_at, updated_at) + values ($1, $2, $3, $4, $5, $6, now(), now()) + on conflict (user_id, source_email_id) do update set + company = excluded.company, + role = excluded.role, + status = excluded.status, + due_date = excluded.due_date, + updated_at = now() + `, + status.UserID, + status.Company, + nullString(status.Role), + status.Status, + status.DueDate, + status.SourceEmailID, + ) + return err +} + +// nullString converts a string to sql.NullString. +// This makes it easier to store nullable text columns in Postgres. +func nullString(value string) sql.NullString { + if strings.TrimSpace(value) == "" { + return sql.NullString{Valid: false} + } + return sql.NullString{String: value, Valid: true} +} diff --git a/backend/cmd/scrapetest/main.go b/backend/cmd/scrapetest/main.go new file mode 100644 index 0000000..9046553 --- /dev/null +++ b/backend/cmd/scrapetest/main.go @@ -0,0 +1,207 @@ +// scrapetest is a manual, local-only debug tool - NOT a deployed Lambda. +// It is not built by backend/build.sh and not referenced anywhere in +// terraform, so running it never touches AWS or Postgres. All it does is +// call the same two functions GET /jobs calls (jobs.FetchSpeedyApplyJobs and +// jobs.FetchSimplifyJobsNewGrad) and print what they found, so you can check +// the GitHub scraping + Markdown/HTML parsing actually works before wiring +// up anything else. +// +// Run it from the backend/ directory with: +// +// go run ./cmd/scrapetest +package main + +import ( + "context" + "fmt" + "os" + "strings" + + "kronos/internal/jobs" +) + +// sample prints the first n jobs from a slice (or all of them if there are +// fewer than n) so the terminal output stays readable even though a real +// scrape can return hundreds of rows. +func sample(label string, all []jobs.Job, n int) { + fmt.Printf("\n=== %s: %d jobs found ===\n", label, len(all)) + for i, j := range all { + if i >= n { + fmt.Printf("... and %d more\n", len(all)-n) + break + } + closed := "" + if j.Closed { + closed = " [CLOSED]" + } + link := j.PostingURL + if link == "" { + link = "(no link)" // e.g. a closed SimplifyJobs row - its Application cell is just a 🔒 emoji + } + fmt.Printf("%2d. %-22s %-45s %-20s age=%-6s %-22s%s\n -> %s\n", + i+1, truncate(j.Company, 22), truncate(j.Position, 45), truncate(j.Location, 20), j.Age, j.SourceSection, closed, link) + } +} + +// truncate is rune-aware (not byte-aware) since company names sometimes +// contain emoji, e.g. "🔥 TikTok" - slicing by byte could cut a multi-byte +// character in half and print garbage. +func truncate(s string, n int) string { + r := []rune(s) + if len(r) <= n { + return s + } + return string(r[:n-1]) + "…" +} + +func main() { + ctx := context.Background() + // Optional: set GITHUB_TOKEN in your shell first if you hit GitHub's + // unauthenticated rate limit (unlikely for a single manual run). + token := os.Getenv("GITHUB_TOKEN") + + speedyJobs, err := jobs.FetchSpeedyApplyJobs(ctx, token) + if err != nil { + fmt.Println("speedyapply error:", err) + } + sample("SpeedyApply (speedyapply/2027-SWE-College-Jobs)", speedyJobs, 8) + + simplifyJobs, err := jobs.FetchSimplifyJobsNewGrad(ctx, token) + if err != nil { + fmt.Println("simplifyjobs error:", err) + } + sample("SimplifyJobs (SimplifyJobs/New-Grad-Positions)", simplifyJobs, 8) + + combined := append(speedyJobs, simplifyJobs...) + fmt.Printf("\nTotal: %d jobs scraped\n", len(combined)) + + // Section counts matter because both scrapers locate sections by matching + // heading text. If an upstream README renames a heading, that section + // quietly yields zero rows instead of erroring - a count of 0 here is the + // only way to notice. + reportSections("SpeedyApply", speedyJobs) + reportSections("SimplifyJobs", simplifyJobs) + + // Lookalikes are measured on the deduped list on purpose: it's the list a + // user actually sees, so the count reflects duplication still visible in + // the dashboard rather than duplication the rules already handled. + reportLookalikes(reportDedupe(combined, 12), 12) +} + +// reportSections prints how many jobs came out of each section heading, in +// first-seen order. A section listed with 0 rows - or missing entirely - means +// the heading match in the scraper no longer lines up with the README. +func reportSections(label string, all []jobs.Job) { + counts := map[string]int{} + var order []string + for _, j := range all { + if _, seen := counts[j.SourceSection]; !seen { + order = append(order, j.SourceSection) + } + counts[j.SourceSection]++ + } + + fmt.Printf("\n=== %s sections ===\n", label) + for _, s := range order { + name := s + if name == "" { + name = "(no section)" + } + fmt.Printf(" %-34s %3d\n", truncate(name, 34), counts[s]) + } +} + +// reportLookalikes finds rows a human would call duplicates - same company, +// title and location - that URL-first identity deliberately keeps apart +// because their posting links genuinely differ. This is the other half of the +// picture: reportDedupe says what the rules DID merge, this says what a user +// would still see listed twice. +func reportLookalikes(all []jobs.Job, maxGroups int) { + key := func(j jobs.Job) string { + norm := func(s string) string { return strings.Join(strings.Fields(strings.ToLower(s)), " ") } + return norm(j.Company) + "|" + norm(j.Position) + "|" + norm(j.Location) + } + + groups := map[string][]jobs.Job{} + var order []string + for _, j := range all { + k := key(j) + if _, seen := groups[k]; !seen { + order = append(order, k) + } + groups[k] = append(groups[k], j) + } + + total, shown := 0, 0 + for _, k := range order { + g := groups[k] + if len(g) < 2 { + continue + } + total += len(g) - 1 + if shown >= maxGroups { + continue + } + shown++ + fmt.Printf("\n[%d rows] %s\n", len(g), truncate(k, 90)) + for _, j := range g { + link := j.PostingURL + if link == "" { + link = "(no link)" + } + fmt.Printf(" %-14s -> %s\n", truncate(j.SourceRepo, 14), link) + } + } + + fmt.Printf("\n=== lookalikes: %d rows share company+title+location with an earlier row ===\n", total) +} + +// reportDedupe runs the real dedupe pass and prints what collapsed, so the +// canonicalization rules can be eyeballed against live data. The counts alone +// aren't enough to judge calibration: what matters is whether the rows that +// merged really are the same posting, which needs seeing them side by side. +func reportDedupe(combined []jobs.Job, maxGroups int) []jobs.Job { + deduped := jobs.Dedupe(combined) + + fmt.Printf("\n=== dedupe: %d -> %d (%d collapsed) ===\n", + len(combined), len(deduped), len(combined)-len(deduped)) + + // Regroup by ID to find which rows merged. Iterate the slice (not a map) + // so the report is stable between runs. + groups := map[string][]jobs.Job{} + var order []string + for _, j := range combined { + if _, seen := groups[j.ID]; !seen { + order = append(order, j.ID) + } + groups[j.ID] = append(groups[j.ID], j) + } + + shown := 0 + for _, id := range order { + g := groups[id] + if len(g) < 2 { + continue + } + if shown >= maxGroups { + fmt.Printf("... and more collapsed groups\n") + break + } + shown++ + fmt.Printf("\n[%s] %d rows merged:\n", id, len(g)) + for _, j := range g { + link := j.PostingURL + if link == "" { + link = "(no link)" + } + fmt.Printf(" %-18s | %-40s | %-18s | %s\n", + truncate(j.Company, 18), truncate(j.Position, 40), truncate(j.Location, 18), truncate(link, 60)) + } + } + + if shown == 0 { + fmt.Println("(nothing collapsed)") + } + + return deduped +} diff --git a/backend/go.mod b/backend/go.mod index 299ecd7..b8ea345 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -4,7 +4,7 @@ go 1.24.0 require ( github.com/aws/aws-lambda-go v1.47.0 - github.com/aws/aws-sdk-go-v2 v1.41.12 + github.com/aws/aws-sdk-go-v2 v1.43.5 github.com/aws/aws-sdk-go-v2/config v1.32.23 github.com/aws/aws-sdk-go-v2/service/ssm v1.69.2 github.com/clerk/clerk-sdk-go/v2 v2.6.0 @@ -12,18 +12,22 @@ require ( ) require ( + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.17 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.19.22 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.28 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.29 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.12 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.36 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.36 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.37 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.16 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.29 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.36 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.37 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.107.1 // indirect github.com/aws/aws-sdk-go-v2/service/signin v1.1.4 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.31.2 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.5 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.43.2 // indirect - github.com/aws/smithy-go v1.27.1 // indirect + github.com/aws/smithy-go v1.27.7 // indirect github.com/go-jose/go-jose/v3 v3.0.4 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect diff --git a/backend/go.sum b/backend/go.sum index 448fafa..71be66f 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -2,6 +2,10 @@ github.com/aws/aws-lambda-go v1.47.0 h1:0H8s0vumYx/YKs4sE7YM0ktwL2eWse+kfopsRI1s github.com/aws/aws-lambda-go v1.47.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7RfgJv23DymV8A= github.com/aws/aws-sdk-go-v2 v1.41.12 h1:DIKX2c31ekm9RA2D9FBj1EWXx++9AdAqRw+e78Tq2Ck= github.com/aws/aws-sdk-go-v2 v1.41.12/go.mod h1:27+ACypSLljLAEKsCYOmrjKh83vuTRkuAe9Uv/3A4bg= +github.com/aws/aws-sdk-go-v2 v1.43.5 h1:yKT5GYnFWhuDo+DqKvE5ZPwVn3RjC4MAeBtZGlh6AVM= +github.com/aws/aws-sdk-go-v2 v1.43.5/go.mod h1:wZjAJppCntyOGgVSmgVTfDyRJK5PHOasO6Wsy8U7Axk= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.17 h1:mn+Vxb9zgz/FE/yDTcFim3DZ1qpcrxR+qBQkBrl6bzA= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.17/go.mod h1:eDfmEFxu+BSVsUGLbzJhWjpOurv1mqczClS97yI8wdk= github.com/aws/aws-sdk-go-v2/config v1.32.23 h1:PYDobtcsJXK6bQe9I8RQk6s19Bz3xa3xRU08Hy1Em3Y= github.com/aws/aws-sdk-go-v2/config v1.32.23/go.mod h1:QID4dqUQVgEOYPKsPWd1sNWCCR2c5g7o3jeEtIXPOZU= github.com/aws/aws-sdk-go-v2/credentials v1.19.22 h1:SHfH6wyPsEgG7fVsi5rQxWEt7tuIcN2PGhb1mTFv6tE= @@ -10,14 +14,30 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.28 h1:b+kcDejJrXc30zU/w8Tc9k github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.28/go.mod h1:LnI62O9GnSv6GcuLXxOYqlq0C8EmxMcgnF6m7LdYuOY= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 h1:Xf2j7NdVcUKomlZ4iihOP4AZ3Fzlr8h4yKpXeP+OFPg= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28/go.mod h1:O8cDo1dW63jU7ki//kRe1z+tLGcpnD1jrouitsQddDw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.36 h1:5CrzwxDqf4w3x1Vs3/NiZ0nsC34Hbm3pIDMWbsLebOE= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.36/go.mod h1:A3gHdKZIvG/QXERzZwcxNS3RNDFcRCuhhTFBYp+V/nw= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 h1:KqIfN9kpkKkcBqBbNpNGTIrXO6ExTUvFKvXkC+YAzVo= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28/go.mod h1:uxtQiKvLtNS4iXVsH2McVD/ls8FKN/uUhe1hGxPjrw0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.36 h1:A4N2f4YPcST0v+dWtX+xrpPPCL9VTBhoIFFUWYqbacE= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.36/go.mod h1:B/Qr859uxWUEfZeGotK5KAEoof4Q9YWgNtPSwV6jcyk= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.29 h1:VkE9FuzTQVjBBrnj4+oCdxCLFIz7aqLYKUCjtvxVcOs= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.29/go.mod h1:H32Z2Qth9b+9LqjyBsCnozMQ8H2N7YBUDVXwbs0iggg= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.37 h1:oyd3ke4V9AhKcRR7rRgxk1VyI+DjK2CBQtbxh3OkdaA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.37/go.mod h1:aA9D7SqfG9IC1b7FLD7Iyc8Q4JN0a8gHhNjN4zPlIaI= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.12 h1:ZD2+BSw9vFsNlKYIasSNt3uDbjqqXIBcM13UJv/Lx2k= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.12/go.mod h1:Ms4zlcVBbXbiP7EVLhl+lgjvA/a7YphqQ3Ih3174EmI= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.16 h1:iE4NGbvqUZnHDqddQAauZzCILYtFjOHwRM5MOOKLB5A= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.16/go.mod h1:VsjEgrP+ibcou8TlWA4tYaB+0OojuhirsmCe+U60hTA= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.29 h1:E65Hj648dOV6FuUfI0mYXXhQRHbsi7n+B9h6fZPJO/E= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.29/go.mod h1:xLrF9yNTCs92VZSpdEd68EJbgcdw3SMR74RO6QDzWHE= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.28 h1:axj4mEDletwKmTm/9jR+DkIMmCfcn5vE4jBMAAN+3Vg= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.28/go.mod h1:3Aaz69M0jqfSHLKqxgolgUBFT4hpwSNc7DzC95orEi8= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.36 h1:fx2ujmozWn+C/GtfXfz5k6Ckzza40ElOpIW7d92fLWQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.36/go.mod h1:QT2ufGVJ+xTRxtXPHTQ1kHkAdWIKPCmD+BqYAXWv8/4= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.37 h1:KGHa9iZCrgtkOsFfXb0S4ywsjostA/hau7WE9aSb43E= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.37/go.mod h1:FV79f0DSnZIEGsQjWenENGtUycrasyAaJZO+zRanLHA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.107.1 h1:VUTtUJMuRNMkb/7NIKmd8NQaeQLPGCMoTJxkYKre4qM= +github.com/aws/aws-sdk-go-v2/service/s3 v1.107.1/go.mod h1:WvUaO0lP5GNMs1R6cs6qvB3mqo16GLta8yfOuf55Rpc= github.com/aws/aws-sdk-go-v2/service/signin v1.1.4 h1:YcpVyIPLCbiypN6KSphijN5fC7DDjX114SqA7prnnxg= github.com/aws/aws-sdk-go-v2/service/signin v1.1.4/go.mod h1:5ZICS++oFTRPfa1GsBqFDWX/8WamZ/QQOcCzIuU/zLw= github.com/aws/aws-sdk-go-v2/service/ssm v1.69.2 h1:1tt+wXv6kWIgHdFd4ehWtBdURIMVYl68ipwljOsR3k4= @@ -30,6 +50,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.43.2 h1:RTO7mmGyedgnNmcPh3yQizNfc6GK github.com/aws/aws-sdk-go-v2/service/sts v1.43.2/go.mod h1:fBhUZXDin9YYqhcpOMjIcpdik25rVwWyxLdPH1RZd9s= github.com/aws/smithy-go v1.27.1 h1:4T340VFndXtADGF52gYa1POyL7s9E4Z1OeZ1hCscIw8= github.com/aws/smithy-go v1.27.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/smithy-go v1.27.7 h1:Zgj5z4LfcDYoQIVk+n/yGdTkP/2y6ZT5vYxe0fp7bqE= +github.com/aws/smithy-go v1.27.7/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/clerk/clerk-sdk-go/v2 v2.6.0 h1:OgxysA1iHcL9mZp3ie/F8e5sBmX4xKRXR7p0BVbut98= github.com/clerk/clerk-sdk-go/v2 v2.6.0/go.mod h1:ncFmsPwmD5WpGCNW5bJve862j/HQfpkzsshXYV/quJ8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/backend/internal/api/api.go b/backend/internal/api/api.go index 428b1ae..9144a40 100644 --- a/backend/internal/api/api.go +++ b/backend/internal/api/api.go @@ -21,6 +21,14 @@ type API struct { AdminClerkID string Season int64 Session string + + // Job Board: GET /jobs scrapes the two public GitHub READMEs on demand + // (see backend/internal/api/jobs.go). There is no server-side store for + // this feature at all - no bucket, no table - because the data is + // derived from a public source that can be re-fetched at any time, and + // is cached in process memory here plus localStorage in the browser. + // The token just lifts GitHub's rate limit from 60/hr to 5000/hr. + GithubToken string } type request = events.APIGatewayV2HTTPRequest @@ -162,6 +170,9 @@ func (a *API) member(ctx context.Context, method, path string, user store.User, rows, err := a.Store.MySolution(ctx, user.ID, parts[2], query["recent"] == "1") return dataOrError(rows, err) + case method == "GET" && path == "/jobs": + return a.getJobs(ctx, query) + case method == "GET" && path == "/leaderboard": rows, err := a.Store.Leaderboard(ctx, 100) return dataOrError(rows, err) diff --git a/backend/internal/api/jobs.go b/backend/internal/api/jobs.go new file mode 100644 index 0000000..2821550 --- /dev/null +++ b/backend/internal/api/jobs.go @@ -0,0 +1,147 @@ +package api + +import ( + "context" + "log" + "strconv" + "sync" + "time" + + "kronos/internal/jobs" +) + +// memTTL is how long one scrape is reused before the next request re-fetches +// from GitHub. This is the only thing standing between "every dashboard load +// scrapes two READMEs" and a sane request rate, so it's deliberately longer +// than the browser's own 2-minute localStorage TTL in src/lib/jobsCache.ts. +const memTTL = 5 * time.Minute + +// The job list is cached in plain process memory - no S3, no database, no +// file on disk. AWS keeps a Lambda container alive between invocations when +// requests keep coming, and package-level variables survive for the life of +// that container, so consecutive requests reuse one scrape for free. When +// the container is eventually recycled these reset to their zero values and +// the next request just scrapes again, which is exactly what a cache should +// do. Nothing here needs to persist: the source of truth is the two public +// GitHub READMEs, which can be re-read at any moment. +var ( + jobsMu sync.Mutex + jobsCached []jobs.Job + jobsAt time.Time +) + +// cachedJobs returns the scraped job list, re-fetching from GitHub only when +// the in-memory copy is missing or older than memTTL. +// +// The mutex is held across the network call on purpose. It means two +// simultaneous cold requests don't both scrape - the second one blocks, then +// finds the first one's result already cached and returns it. Serializing a +// slow path is the right trade here, since the alternative is a burst of +// duplicate scrapes every time the cache expires. +func (a *API) cachedJobs(ctx context.Context) ([]jobs.Job, time.Time, error) { + jobsMu.Lock() + defer jobsMu.Unlock() + + if !jobsAt.IsZero() && time.Since(jobsAt) < memTTL { + return jobsCached, jobsAt, nil + } + + // Scrape each source independently so one failing (GitHub briefly down, + // a README's table format changed) doesn't cost us the other one. + speedyJobs, errSpeedy := jobs.FetchSpeedyApplyJobs(ctx, a.GithubToken) + if errSpeedy != nil { + log.Printf("speedyapply: %v", errSpeedy) + } + simplifyJobs, errSimplify := jobs.FetchSimplifyJobsNewGrad(ctx, a.GithubToken) + if errSimplify != nil { + log.Printf("simplifyjobs: %v", errSimplify) + } + + if errSpeedy != nil && errSimplify != nil { + // Both sources failed. Rather than caching an empty list (which + // would wedge the board as empty for the whole TTL), keep serving + // the last good scrape if we still have one. + if jobsCached != nil { + return jobsCached, jobsAt, nil + } + return nil, time.Time{}, errSpeedy + } + + // Source order is load-bearing: whichever record Dedupe sees first wins + // every field it has a value for. speedyapply goes first because it's + // the source carrying salary data. + combined := append(speedyJobs, simplifyJobs...) + jobsCached = jobs.Dedupe(combined) + jobsAt = time.Now().UTC() + + // Log both counts. If the collapsed number ever jumps sharply after a + // canonicalization rule changes, that's the signal a rule is merging too + // aggressively and is quietly hiding postings - there's no other way to + // notice from production. + log.Printf("scraped %d jobs (%d speedyapply, %d simplifyjobs), %d after dedupe (%d collapsed)", + len(combined), len(speedyJobs), len(simplifyJobs), len(jobsCached), len(combined)-len(jobsCached)) + + return jobsCached, jobsAt, nil +} + +// getJobs serves GET /jobs for the Job Board dashboard card, one page at a +// time. It scrapes the two public GitHub READMEs directly (via the fetchers +// in kronos/internal/jobs) and slices out just the page that was asked for. +// +// There is no server-side store behind this route. The job data is derived +// from a public source rather than owned by us, so it's re-fetchable at any +// time, which makes any copy we keep a cache rather than a record. Two +// caches already cover that: cachedJobs above (process memory, 5 min) and +// the browser's localStorage in src/lib/jobsCache.ts (2 min). +// +// Pagination is plain offset-based (?limit=20&offset=20, ...) rather than +// the SQL cursor version we tried earlier: the full list is already sitting +// in memory and stable for the life of one request, so there's no "another +// request snuck a row in before yours" problem to design around. +func (a *API) getJobs(ctx context.Context, query map[string]string) (response, error) { + all, scrapedAt, err := a.cachedJobs(ctx) + if err != nil { + return serverError(err) + } + + limit := 20 + if v := query["limit"]; v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + limit = n + } + } + offset := 0 + if v := query["offset"]; v != "" { + if n, err := strconv.Atoi(v); err == nil && n >= 0 { + offset = n + } + } + + page := []jobs.Job{} + var nextOffset *int // nil means "no more pages" - encoded as JSON null + + if offset < len(all) { + end := offset + limit + if end > len(all) { + end = len(all) + } + page = all[offset:end] + if end < len(all) { + n := end + nextOffset = &n + } + } + + return reply(200, jobsPage{ + Jobs: page, + ScrapedAt: scrapedAt, + NextOffset: nextOffset, + }) +} + +// jobsPage is the shape of every GET /jobs response. +type jobsPage struct { + Jobs []jobs.Job `json:"jobs"` + ScrapedAt time.Time `json:"scrapedAt"` + NextOffset *int `json:"nextOffset"` +} diff --git a/backend/internal/jobs/canonical.go b/backend/internal/jobs/canonical.go new file mode 100644 index 0000000..8b39a0f --- /dev/null +++ b/backend/internal/jobs/canonical.go @@ -0,0 +1,339 @@ +package jobs + +// Canonicalization: turning the many ways one job posting can be written into +// a single normalized form, so that two scrapes of the same job produce the +// same fingerprint. +// +// The problem this solves: the same posting appears in both source READMEs, +// written slightly differently each time - "Acme, Inc." vs "Acme", a link +// with ?utm_source=... vs one without, a trailing slash vs none. Hashing +// those raw strings gives different IDs, so the dashboard shows one job +// several times. +// +// Everything here feeds the ID hash ONLY. The Job struct keeps the original +// human-readable Company/Position/PostingURL for display - we normalize to +// decide "are these the same job", never to decide what the user reads. +// +// The guiding rule throughout is that the two failure modes are NOT equally +// bad. Under-merging shows one job twice, which is mildly annoying. +// Over-merging makes a real job silently disappear, which is much worse. So +// every rule below is written to err toward leaving things separate. + +import ( + "net/url" + "regexp" + "strings" + "unicode" +) + +// schemePattern matches a leading URL scheme like "https://" so we can tell +// whether a link already has one. Without a scheme, url.Parse reads the whole +// string as a path and the host ends up empty. +var schemePattern = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9+.\-]*://`) + +// workdayDisambiguator matches the "-1"/"-2"/"-3" suffix Workday adds to a +// requisition ID when the same req is published to more than one career +// site, e.g. "_R55736-3". The leading underscore-plus-ID is required so a +// job slug that merely ends in a number isn't truncated. +var workdayDisambiguator = regexp.MustCompile(`(_[A-Za-z0-9]+)-\d+$`) + +// textFolder collapses characters that look identical to a human but differ +// in bytes. Scraped Markdown is full of these: editors silently substitute +// curly quotes and en-dashes, and zero-width characters ride along invisibly +// from copy-pasted HTML. Note U+00A0 (non-breaking space) is NOT here - it's +// already handled upstream by cleanText, since strings.Fields treats it as +// whitespace. +// Written as escapes rather than literal characters on purpose: several of +// these are invisible in an editor, so a literal would be unreviewable, and +// a literal U+FEFF is outright illegal in Go source. +var textFolder = strings.NewReplacer( + "\u200b", "", // zero-width space + "\u200c", "", // zero-width non-joiner + "\u200d", "", // zero-width joiner + "\ufeff", "", // byte-order mark + "\u2018", "'", // left single curly quote + "\u2019", "'", // right single curly quote + "\u201c", `"`, // left double curly quote + "\u201d", `"`, // right double curly quote + "\u2013", "-", // en dash + "\u2014", "-", // em dash +) + +// punctFolder drops punctuation that varies between spellings of the same +// company name: "Acme, Inc." vs "Acme Inc", "Macy's" vs "Macys". +var punctFolder = strings.NewReplacer( + ".", "", + ",", "", + "'", "", +) + +// trackingParams are query parameters that identify where a click came from +// rather than which job it points at, so two links differing only in these +// are the same posting. +// +// This is a denylist (strip what we know is tracking) rather than an +// allowlist (keep what we know is an ID) on purpose: an unrecognized +// parameter is more likely to be meaningful than not, and leaving it in errs +// toward under-merging, which is the safe direction. +// +// DANGER: note gh_src is here but gh_jid is NOT. They differ by three +// characters and mean opposite things - gh_src is Greenhouse's source token, +// gh_jid is the job ID itself. Stripping gh_jid would merge every Greenhouse +// posting at a company into a single entry. +var trackingParams = map[string]bool{ + // Google Analytics + "utm_source": true, + "utm_medium": true, + "utm_campaign": true, + "utm_term": true, + "utm_content": true, + "utm_id": true, + "_ga": true, + "_gl": true, + // Ad-network click IDs + "gclid": true, + "fbclid": true, + "msclkid": true, + "mc_cid": true, + "mc_eid": true, + // Applicant tracking systems + "gh_src": true, + "lever-source": true, + "lever-origin": true, + "trk": true, + "trackingid": true, + "refid": true, + "jobsearchtk": true, + "iis": true, + "iisn": true, + "from": true, + // Generic referral tagging + "ref": true, + "refsrc": true, + "referrer": true, + "source": true, + "src": true, + "campaign_id": true, +} + +// legalSuffixes are company-name endings that carry no identifying +// information: "Acme Inc" and "Acme" are the same employer. +// +// Deliberately English-only. Foreign forms (gmbh, ag, sa, nv, bv, pty, pte) +// are likelier to be part of a real name and barely appear in these two +// repos, so stripping them risks over-merging for no practical gain. +// +// "co" is the riskiest entry - it's a real word fragment. It's only ever +// removed as a whole trailing token, and stripLegalSuffix refuses to empty +// a name entirely, so a company genuinely named "Co" survives intact. +var legalSuffixes = map[string]bool{ + "inc": true, + "incorporated": true, + "corp": true, + "corporation": true, + "llc": true, + "ltd": true, + "limited": true, + "plc": true, + "llp": true, + "lp": true, + "co": true, + "company": true, +} + +// canonicalURL normalizes a posting link so that the cosmetic differences +// between two copies of the same link disappear. +// +// This is the most valuable of the three canonicalizers by a wide margin. +// Both scrapers pull the employer's real applicant-tracking link (Greenhouse, +// Lever, Workday), and when the same job appears in both READMEs they almost +// always point at the identical ATS URL. That makes a canonical URL a far +// stronger identity than any amount of normalizing prose - it sidesteps the +// "same role, different wording" problem entirely. +// +// Returns "" for an empty or whitespace-only input, which signals the caller +// to fall back to a company/position/location identity instead. +func canonicalURL(raw string) string { + raw = strings.TrimSpace(raw) + if raw == "" { + return "" + } + + // A protocol-relative link ("//example.com/j/1") needs only the scheme + // prepended; anything else missing "://" needs the whole prefix. + switch { + case strings.HasPrefix(raw, "//"): + raw = "https:" + raw + case !schemePattern.MatchString(raw): + raw = "https://" + raw + } + + u, err := url.Parse(raw) + if err != nil { + // One malformed link shouldn't cost us the whole scrape. Fall back to + // something deterministic so the job still gets a stable ID. + return strings.ToLower(raw) + } + + // http and https serve the same posting, so fold them together rather + // than letting the scheme split one job into two. + u.Scheme = "https" + + // Only scheme and host are case-insensitive (RFC 3986). The path is left + // exactly as found - Workday in particular uses case-sensitive segments, + // and lowercasing them could collide two genuinely different postings. + u.Host = strings.TrimPrefix(strings.ToLower(u.Host), "www.") + + // Workday publishes one requisition through several career-site front + // doors, so the same job reaches us under several URLs: + // + // https://cadence.wd1.myworkdayjobs.com/External_Careers/job/LIVONIA-01/Adams-... + // https://cadence.wd1.myworkdayjobs.com/University_Talent/job/LIVONIA-01/Adams-... + // https://cadence.wd1.myworkdayjobs.com/Univ_Careers/job/LIVONIA-01/Adams-... + // + // The leading segments are the site name (and sometimes a locale, e.g. + // "/fr-CA/Private_Posting_No_TMP/job/..."); everything from "/job/" + // onward is the requisition itself. Keeping only that part merges the + // front doors while still separating genuinely different postings, since + // two different requisitions have different job paths. + // + // This stays deterministic - it's the same kind of rule as stripping a + // trailing slash, not fuzzy matching - and it's scoped to Workday hosts + // so it can't affect any other ATS. + // Workday also appends a per-site disambiguator to the requisition ID + // when one req is published to several career sites, so the same job + // ends up as _R55736, _R55736-1, _R55736-2 and _R55736-3. Dropping that + // suffix is what actually merges them. + // + // The pattern is anchored on the underscore that precedes every Workday + // requisition ID, which keeps it from mangling a title that legitimately + // ends in a number - "Software-Engineer-2" has no "_reqid" before the + // trailing digits and is left alone. + if strings.HasSuffix(u.Host, ".myworkdayjobs.com") { + if i := strings.Index(u.Path, "/job/"); i >= 0 { + u.Path = u.Path[i:] + } + u.Path = workdayDisambiguator.ReplaceAllString(u.Path, "$1") + } + + // A trailing slash is cosmetic, but "/" as the entire path is just the + // site root and is cleaner represented as empty. + if u.Path == "/" { + u.Path = "" + } else { + u.Path = strings.TrimSuffix(u.Path, "/") + } + + // Fragments are pure client-side navigation ("#apply") and never change + // which posting is served. + u.Fragment = "" + u.RawFragment = "" + + q := u.Query() + for key := range q { + if trackingParams[strings.ToLower(key)] { + q.Del(key) + } + } + // Encode() sorts by key, so two links carrying the same parameters in a + // different order come out identical. It also omits "?" when nothing is + // left, which is what we want. + u.RawQuery = q.Encode() + + return u.String() +} + +// canonicalCompany normalizes an employer name for fingerprinting. +// +// Only used when a posting has no link at all (SimplifyJobs renders closed +// applications as a lock emoji with no href), so this is the fallback path +// rather than the common one. +func canonicalCompany(s string) string { + s = textFolder.Replace(s) + s = strings.ToLower(s) + + // "Johnson & Johnson" and "Johnson and Johnson" are one employer. Padding + // with spaces keeps "J&J" from collapsing into a single token. + s = strings.ReplaceAll(s, "&", " and ") + + s = punctFolder.Replace(s) + s = trimNonAlphanumeric(s) + s = collapseSpaces(s) + s = stripLegalSuffix(s) + + return collapseSpaces(s) +} + +// canonicalPosition normalizes a job title for fingerprinting. +// +// This one is deliberately minimal - it lowercases, folds lookalike +// characters, and strips decoration from the ends. That's all. +// +// It explicitly does NOT map "SWE" to "Software Engineer" or "Intern" to +// "Internship". Those are fuzzy matching, not canonicalization, and they're +// exactly where over-merging starts hiding real jobs. Since canonicalURL +// handles the overwhelming majority of duplicates, title text only decides +// identity for the small tail of postings with no link - a weak reason to +// take that risk. +// +// It also does not strip a leading "New": in this domain "New Grad" is a +// meaningful part of the title, not a badge. +func canonicalPosition(s string) string { + // Trimming non-alphanumerics from the ends removes decorative markers + // (lock, star, sparkle emoji) and stray trailing punctuation. Years and + // parenthetical qualifiers in the middle are left alone, since + // "Summer 2026" and "(Backend)" genuinely distinguish jobs. + return foldMinimal(s) +} + +// canonicalLocation normalizes a location for fingerprinting. +// +// Only reached on the fallback identity path, where it matters a great deal: +// without it, every closed listing sharing a company and title collapses into +// one entry regardless of city, hiding real openings. Multi-location cells +// arrive pre-joined as "Seattle, WA, Austin, TX" (see lineBreakPattern in +// html.go), and are treated as a single opaque string - two postings listing +// the same cities in a different order stay separate, which is the safe +// direction. +func canonicalLocation(s string) string { + return foldMinimal(s) +} + +// foldMinimal is the light-touch normalization shared by position and +// location: fold lookalike characters, lowercase, strip decoration from the +// ends, and collapse whitespace. Nothing semantic. +func foldMinimal(s string) string { + s = textFolder.Replace(s) + s = strings.ToLower(s) + s = trimNonAlphanumeric(s) + return collapseSpaces(s) +} + +// trimNonAlphanumeric removes leading and trailing runes that are neither +// letters nor digits - emoji, asterisks, stray brackets, whitespace. +func trimNonAlphanumeric(s string) string { + return strings.TrimFunc(s, func(r rune) bool { + return !unicode.IsLetter(r) && !unicode.IsDigit(r) + }) +} + +// collapseSpaces squeezes runs of whitespace down to single spaces and trims +// the ends, the same normalization cleanText applies to scraped cells. +func collapseSpaces(s string) string { + return strings.Join(strings.Fields(s), " ") +} + +// stripLegalSuffix removes trailing corporate-form tokens, repeatedly, so +// "acme co inc" reduces to "acme". +// +// The len(fields) > 1 guard is what keeps this safe: it will never consume +// the last remaining token, so a company actually named "Co" or "Limited" +// keeps its name instead of canonicalizing to the empty string (which would +// merge it with every other suffix-only name). +func stripLegalSuffix(s string) string { + fields := strings.Fields(s) + for len(fields) > 1 && legalSuffixes[fields[len(fields)-1]] { + fields = fields[:len(fields)-1] + } + return strings.Join(fields, " ") +} diff --git a/backend/internal/jobs/canonical_test.go b/backend/internal/jobs/canonical_test.go new file mode 100644 index 0000000..57695b4 --- /dev/null +++ b/backend/internal/jobs/canonical_test.go @@ -0,0 +1,437 @@ +package jobs + +// Tests for canonical.go. +// +// These are written as three kinds of table: +// +// - exact-output tests, pinning down what a canonicalizer actually returns; +// - "must merge" pairs, where two spellings of one job have to collapse to +// the same canonical form; +// - "must NOT merge" pairs, where two genuinely different jobs have to stay +// apart. +// +// The third kind is the important one. Any canonicalizer can be made to merge +// more by adding rules; the failure that actually hurts users is a rule that +// merges too much and makes a real posting vanish from the dashboard. Those +// cases are cheap to write and are the regression net for every future rule. + +import "testing" + +func TestCanonicalURL(t *testing.T) { + tests := []struct { + name string + in string + want string + }{ + {"empty", "", ""}, + {"whitespace only", " ", ""}, + { + "already canonical", + "https://boards.greenhouse.io/acme/jobs/123", + "https://boards.greenhouse.io/acme/jobs/123", + }, + { + "trailing slash removed", + "https://boards.greenhouse.io/acme/jobs/123/", + "https://boards.greenhouse.io/acme/jobs/123", + }, + { + "http folded to https", + "http://boards.greenhouse.io/acme/jobs/123", + "https://boards.greenhouse.io/acme/jobs/123", + }, + { + "www stripped and host lowercased", + "https://WWW.Lever.co/acme/123", + "https://lever.co/acme/123", + }, + { + // Only scheme and host are case-insensitive, so the path is left + // exactly as found. Uses a non-Workday host to keep this case + // isolated from the career-site trimming below. + "path case preserved", + "https://acme.com/en-US/Careers/job/SoftwareEngineer", + "https://acme.com/en-US/Careers/job/SoftwareEngineer", + }, + { + "workday career site and locale trimmed, case preserved", + "https://acme.wd1.myworkdayjobs.com/en-US/Careers/job/SoftwareEngineer", + "https://acme.wd1.myworkdayjobs.com/job/SoftwareEngineer", + }, + { + "fragment dropped", + "https://lever.co/acme/123#apply", + "https://lever.co/acme/123", + }, + { + "tracking params stripped", + "https://lever.co/acme/123?utm_source=github&utm_campaign=newgrad", + "https://lever.co/acme/123", + }, + { + "job id preserved while tracking stripped", + "https://boards.greenhouse.io/acme?gh_src=abcd&gh_jid=987", + "https://boards.greenhouse.io/acme?gh_jid=987", + }, + { + "remaining params sorted", + "https://lever.co/acme?z=1&a=2", + "https://lever.co/acme?a=2&z=1", + }, + { + "missing scheme defaulted", + "boards.greenhouse.io/acme/jobs/1", + "https://boards.greenhouse.io/acme/jobs/1", + }, + { + "protocol relative", + "//boards.greenhouse.io/acme/jobs/1", + "https://boards.greenhouse.io/acme/jobs/1", + }, + { + "bare root path emptied", + "https://acme.com/", + "https://acme.com", + }, + { + "surrounding whitespace trimmed", + " https://lever.co/acme/123 ", + "https://lever.co/acme/123", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := canonicalURL(tc.in); got != tc.want { + t.Errorf("canonicalURL(%q)\n got: %q\nwant: %q", tc.in, got, tc.want) + } + }) + } +} + +func TestCanonicalURLMustMerge(t *testing.T) { + pairs := []struct { + name string + a, b string + }{ + { + "trailing slash", + "https://boards.greenhouse.io/acme/jobs/123", + "https://boards.greenhouse.io/acme/jobs/123/", + }, + { + "scheme and www", + "http://www.acme.com/jobs/1", + "https://acme.com/jobs/1", + }, + { + "tracking params only difference", + "https://boards.greenhouse.io/acme?gh_jid=9&utm_source=gh", + "https://boards.greenhouse.io/acme?gh_jid=9", + }, + { + "param order", + "https://acme.com/j?a=1&b=2", + "https://acme.com/j?b=2&a=1", + }, + { + "fragment", + "https://acme.com/jobs/1#apply", + "https://acme.com/jobs/1", + }, + { + "ad click id", + "https://acme.com/jobs/1?gclid=xyz", + "https://acme.com/jobs/1", + }, + { + "lever source tag", + "https://jobs.lever.co/acme/abc?lever-source=LinkedIn", + "https://jobs.lever.co/acme/abc", + }, + { + // Observed live: Cadence lists one requisition under four + // different Workday career sites. + "workday career site segment", + "https://cadence.wd1.myworkdayjobs.com/External_Careers/job/LIVONIA-01/Adams-Application-Software-Developer", + "https://cadence.wd1.myworkdayjobs.com/University_Talent_NCG/job/LIVONIA-01/Adams-Application-Software-Developer", + }, + { + // Observed live: RTX, where one variant also carries a locale + // segment ahead of the site name. + "workday locale segment", + "https://globalhr.wd5.myworkdayjobs.com/fr-CA/Private_Posting_No_TMP/job/US-AZ-TUCSON/Software-Engineer", + "https://globalhr.wd5.myworkdayjobs.com/rec_rtx_ext_gateway/job/US-AZ-TUCSON/Software-Engineer", + }, + { + // Observed live: Salesforce publishes requisition JR355250 to + // both its external site and its new-grad portal, and the second + // copy carries a "-1" disambiguator. + "workday requisition disambiguator", + "https://salesforce.wd12.myworkdayjobs.com/External_Career_Site/job/California---San-Francisco/Software-Engineering-AMTS--College-Grad-_JR355250-1", + "https://salesforce.wd12.myworkdayjobs.com/Futureforce_NewGradRoles/job/California---San-Francisco/Software-Engineering-AMTS--College-Grad-_JR355250", + }, + { + // Observed live: Cadence publishes requisition R55736 to four + // career sites, numbered -1 through -3 plus a bare copy. + "workday four career sites, one requisition", + "https://cadence.wd1.myworkdayjobs.com/External_Careers/job/LIVONIA-01/Adams-Application-Software-Developer---Recent-Grad-2026-_R55736-3", + "https://cadence.wd1.myworkdayjobs.com/University_Talent_NCG/job/LIVONIA-01/Adams-Application-Software-Developer---Recent-Grad-2026-_R55736", + }, + } + + for _, tc := range pairs { + t.Run(tc.name, func(t *testing.T) { + if got, want := canonicalURL(tc.a), canonicalURL(tc.b); got != want { + t.Errorf("expected these to canonicalize the same:\n a: %q -> %q\n b: %q -> %q", + tc.a, got, tc.b, want) + } + }) + } +} + +func TestCanonicalURLMustNotMerge(t *testing.T) { + pairs := []struct { + name string + a, b string + }{ + { + "different greenhouse job ids", + "https://boards.greenhouse.io/acme?gh_jid=1", + "https://boards.greenhouse.io/acme?gh_jid=2", + }, + { + "different indeed job keys", + "https://indeed.com/viewjob?jk=aaa", + "https://indeed.com/viewjob?jk=bbb", + }, + { + "different paths", + "https://acme.com/jobs/1", + "https://acme.com/jobs/2", + }, + { + "different hosts", + "https://acme.com/jobs/1", + "https://globex.com/jobs/1", + }, + { + "workday path case is significant", + "https://acme.wd1.myworkdayjobs.com/job/SoftwareEngineer", + "https://acme.wd1.myworkdayjobs.com/job/softwareengineer", + }, + { + "different linkedin job ids", + "https://linkedin.com/jobs/view?currentJobId=111", + "https://linkedin.com/jobs/view?currentJobId=222", + }, + { + // The Workday rule must still separate real requisitions - this + // is the pair Cadence actually has, two distinct roles under the + // same career sites. + "workday different job slugs", + "https://cadence.wd1.myworkdayjobs.com/External_Careers/job/LIVONIA-01/Adams-Application-Software-Developer", + "https://cadence.wd1.myworkdayjobs.com/External_Careers/job/LIVONIA-01/Adams-Multibody-Dynamics-Developer", + }, + { + "workday different tenants", + "https://cadence.wd1.myworkdayjobs.com/External_Careers/job/X/Role", + "https://salesforce.wd12.myworkdayjobs.com/External_Career_Site/job/X/Role", + }, + { + // The rule is scoped to Workday hosts: a lookalike path segment + // elsewhere must not be stripped. + "non-workday host keeps full path", + "https://acme.com/careers/job/X/Role", + "https://acme.com/interns/job/X/Role", + }, + { + // Observed live: RTX has three genuinely distinct requisitions, + // two of them at different buildings. Different req IDs must + // survive the disambiguator rule. + "workday different requisition ids", + "https://globalhr.wd5.myworkdayjobs.com/rec_rtx_ext_gateway/job/US-AZ-TUCSON-801/Software-Engineer-I--Onsite-_01865026-1", + "https://globalhr.wd5.myworkdayjobs.com/fr-CA/Private_Posting_No_TMP/job/US-AZ-TUCSON-801/Software-Engineer-I--Onsite-_01866246", + }, + { + // A title ending in a level number has no "_reqid" before the + // digits, so the disambiguator rule must leave it intact. + "workday title ending in a number", + "https://acme.wd1.myworkdayjobs.com/Careers/job/X/Software-Engineer-2", + "https://acme.wd1.myworkdayjobs.com/Careers/job/X/Software-Engineer-3", + }, + } + + for _, tc := range pairs { + t.Run(tc.name, func(t *testing.T) { + if got, other := canonicalURL(tc.a), canonicalURL(tc.b); got == other { + t.Errorf("these are different jobs but both canonicalized to %q\n a: %q\n b: %q", + got, tc.a, tc.b) + } + }) + } +} + +func TestCanonicalCompany(t *testing.T) { + tests := []struct { + name string + in string + want string + }{ + {"plain", "Acme", "acme"}, + {"lowercased", "ACME", "acme"}, + {"trailing inc with comma", "Acme, Inc.", "acme"}, + {"trailing incorporated", "Acme Incorporated", "acme"}, + {"trailing corp", "Acme Corp.", "acme"}, + {"trailing corporation", "Acme Corporation", "acme"}, + {"trailing llc", "Acme LLC", "acme"}, + {"trailing ltd", "Acme Ltd.", "acme"}, + {"trailing company", "The Walt Disney Company", "the walt disney"}, + {"stacked suffixes", "Acme Co Inc", "acme"}, + {"ampersand expanded", "Johnson & Johnson", "johnson and johnson"}, + {"apostrophe removed", "Macy's", "macys"}, + {"leading emoji stripped", "⭐ Acme", "acme"}, + {"internal whitespace collapsed", "Acme Labs", "acme labs"}, + {"curly apostrophe folded", "Macy\u2019s", "macys"}, + {"zero width space removed", "Ac\u200bme", "acme"}, + {"en dash folded to hyphen", "Acme\u2013Globex", "acme-globex"}, + + // The guard in stripLegalSuffix: a name that is nothing but a suffix + // keeps its token rather than canonicalizing to "", which would merge + // every such company together. + {"suffix-only name survives", "Co", "co"}, + {"limited alone survives", "Limited", "limited"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := canonicalCompany(tc.in); got != tc.want { + t.Errorf("canonicalCompany(%q)\n got: %q\nwant: %q", tc.in, got, tc.want) + } + }) + } +} + +func TestCanonicalCompanyMustMerge(t *testing.T) { + pairs := []struct { + name string + a, b string + }{ + {"legal suffix", "Acme, Inc.", "Acme"}, + {"corp vs corporation", "Acme Corp", "Acme Corporation"}, + {"case", "ACME", "acme"}, + {"ampersand vs and", "Johnson & Johnson", "Johnson and Johnson"}, + {"curly vs straight apostrophe", "Macy\u2019s", "Macy's"}, + {"padded whitespace", " Acme ", "Acme"}, + {"internal double space", "Acme Labs", "Acme Labs"}, + {"decorative emoji", "⭐ Acme", "Acme"}, + } + + for _, tc := range pairs { + t.Run(tc.name, func(t *testing.T) { + if got, want := canonicalCompany(tc.a), canonicalCompany(tc.b); got != want { + t.Errorf("expected these to canonicalize the same:\n a: %q -> %q\n b: %q -> %q", + tc.a, got, tc.b, want) + } + }) + } +} + +func TestCanonicalCompanyMustNotMerge(t *testing.T) { + pairs := []struct { + name string + a, b string + }{ + {"different employers", "Acme", "Globex"}, + {"parent vs subsidiary naming", "Acme Labs", "Acme"}, + {"similar prefix", "Acme", "Acme Health"}, + } + + for _, tc := range pairs { + t.Run(tc.name, func(t *testing.T) { + if got, other := canonicalCompany(tc.a), canonicalCompany(tc.b); got == other { + t.Errorf("these are different employers but both canonicalized to %q\n a: %q\n b: %q", + got, tc.a, tc.b) + } + }) + } +} + +func TestCanonicalPosition(t *testing.T) { + tests := []struct { + name string + in string + want string + }{ + {"plain", "Software Engineer Intern", "software engineer intern"}, + {"lock emoji stripped", "🔒 Software Engineer Intern", "software engineer intern"}, + {"sparkle emoji stripped", "🆕 Backend Engineer", "backend engineer"}, + {"trailing period stripped", "Software Engineer Intern.", "software engineer intern"}, + {"whitespace collapsed", "Software Engineer", "software engineer"}, + {"em dash folded", "Engineer\u2014Backend", "engineer-backend"}, + {"zero width joiner removed", "Soft\u200dware Engineer", "software engineer"}, + + // "New Grad" is a meaningful title in this domain, not a badge - the + // leading word must survive. + {"new grad preserved", "New Grad Software Engineer", "new grad software engineer"}, + + // Years and parenthetical qualifiers distinguish real jobs and are + // left untouched in the middle of a title. + {"year preserved", "SWE Intern Summer 2027", "swe intern summer 2027"}, + {"qualifier preserved", "Software Engineer (Backend)", "software engineer (backend"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := canonicalPosition(tc.in); got != tc.want { + t.Errorf("canonicalPosition(%q)\n got: %q\nwant: %q", tc.in, got, tc.want) + } + }) + } +} + +func TestCanonicalPositionMustNotMerge(t *testing.T) { + pairs := []struct { + name string + a, b string + }{ + { + "different years are different jobs", + "SWE Intern Summer 2026", + "SWE Intern Summer 2027", + }, + { + "different specializations", + "Software Engineer (Backend)", + "Software Engineer (Frontend)", + }, + { + "seniority", + "Software Engineer", + "Senior Software Engineer", + }, + { + "new grad is not the same as intern", + "New Grad Software Engineer", + "Software Engineer Intern", + }, + { + // Deliberately unmerged: expanding SWE to Software Engineer is + // fuzzy matching, and canonicalURL already covers the duplicates + // that matter. If this ever starts failing, someone added a + // synonym rule - make sure that was intentional. + "abbreviation left alone by design", + "SWE Intern", + "Software Engineer Intern", + }, + } + + for _, tc := range pairs { + t.Run(tc.name, func(t *testing.T) { + if got, other := canonicalPosition(tc.a), canonicalPosition(tc.b); got == other { + t.Errorf("these are different roles but both canonicalized to %q\n a: %q\n b: %q", + got, tc.a, tc.b) + } + }) + } +} diff --git a/backend/internal/jobs/dedupe.go b/backend/internal/jobs/dedupe.go new file mode 100644 index 0000000..e76808f --- /dev/null +++ b/backend/internal/jobs/dedupe.go @@ -0,0 +1,88 @@ +package jobs + +// Deduplication: collapsing rows that newID decided are the same posting. +// +// Canonicalization alone changes nothing a user can see. It only makes two +// spellings of one job hash to the same ID; without this pass both rows are +// still in the slice, so the dashboard still renders the job twice - now with +// duplicate React keys as a bonus. This file is the half that actually +// removes them. + +// Dedupe collapses jobs sharing an ID into one record, merging their fields, +// and returns the result in first-seen order. +// +// Ordering is the subtle part. The obvious implementation - group everything +// into a map, then range over the map to build the output - is broken here, +// because Go deliberately randomizes map iteration order. The list would come +// out shuffled differently on every scrape, and GET /jobs pages it with plain +// offsets: a reader who fetched offset=0 before a re-scrape and offset=20 +// after would see some jobs twice and miss others entirely. So the map here +// only ever holds indexes into the output slice; the slice itself is built by +// walking the input in order, which makes the result stable as long as the +// sources are. +// +// Which record wins is therefore decided by input order. The caller +// concatenates speedyapply before simplifyjobs, so speedyapply is the primary +// source - a deliberate choice, since it's the one carrying salary data. +func Dedupe(all []Job) []Job { + out := make([]Job, 0, len(all)) + indexByID := make(map[string]int, len(all)) + + for _, job := range all { + // An empty ID means newID had nothing usable to work with. Keeping + // such rows separate is the safe reading: collapsing them all into + // one entry would hide real postings. + if job.ID == "" { + out = append(out, job) + continue + } + + if i, seen := indexByID[job.ID]; seen { + out[i] = mergeJobs(out[i], job) + continue + } + + indexByID[job.ID] = len(out) + out = append(out, job) + } + + return out +} + +// mergeJobs folds a duplicate into the record already kept for that ID. +// +// Merging rather than discarding the duplicate is worth the effort because +// the two sources carry different data: speedyapply lists salary, SimplifyJobs +// usually doesn't. Collapsing them produces a more complete row than either +// source had alone, so dedup doubles as enrichment. +// +// primary wins every field it has a value for; secondary only fills blanks. +// Closed is the one exception - see below. +func mergeJobs(primary, secondary Job) Job { + merged := primary + + merged.Company = firstNonEmpty(primary.Company, secondary.Company) + merged.Position = firstNonEmpty(primary.Position, secondary.Position) + merged.Location = firstNonEmpty(primary.Location, secondary.Location) + merged.Salary = firstNonEmpty(primary.Salary, secondary.Salary) + merged.PostingURL = firstNonEmpty(primary.PostingURL, secondary.PostingURL) + merged.Age = firstNonEmpty(primary.Age, secondary.Age) + merged.SourceRepo = firstNonEmpty(primary.SourceRepo, secondary.SourceRepo) + merged.SourceSection = firstNonEmpty(primary.SourceSection, secondary.SourceSection) + + // A posting counts as closed only when every source agrees it is. One + // repo lagging behind on marking a role closed is common; the cost of + // trusting the stale "open" is a dead link, while the cost of trusting a + // stale "closed" is hiding a job someone could still have applied to. + merged.Closed = primary.Closed && secondary.Closed + + return merged +} + +// firstNonEmpty returns a if it has content, otherwise b. +func firstNonEmpty(a, b string) string { + if a != "" { + return a + } + return b +} diff --git a/backend/internal/jobs/dedupe_test.go b/backend/internal/jobs/dedupe_test.go new file mode 100644 index 0000000..4677447 --- /dev/null +++ b/backend/internal/jobs/dedupe_test.go @@ -0,0 +1,199 @@ +package jobs + +import ( + "reflect" + "testing" +) + +// TestNewIDURLFirst pins down the identity scheme: when a link is present it +// alone decides identity, so differences in company and title wording stop +// mattering. +func TestNewIDURLFirst(t *testing.T) { + speedy := newID("Acme, Inc.", "Software Engineer Intern", "Seattle, WA", + "https://boards.greenhouse.io/acme/jobs/1?utm_source=github") + simplify := newID("Acme", "SWE Intern", "Seattle, WA", + "http://www.boards.greenhouse.io/acme/jobs/1/") + + if speedy != simplify { + t.Errorf("same posting from two sources produced different IDs: %q vs %q", speedy, simplify) + } +} + +func TestNewIDDifferentURLsStaySeparate(t *testing.T) { + a := newID("Acme", "SWE Intern", "Seattle, WA", "https://boards.greenhouse.io/acme?gh_jid=1") + b := newID("Acme", "SWE Intern", "Seattle, WA", "https://boards.greenhouse.io/acme?gh_jid=2") + + if a == b { + t.Errorf("two different Greenhouse postings collapsed to the same ID %q", a) + } +} + +// TestNewIDFallbackIncludesLocation guards the bug the old three-field key +// had: closed listings carry no URL, so without location in the key every +// closed role at one company with one title merged into a single entry. +func TestNewIDFallbackIncludesLocation(t *testing.T) { + seattle := newID("Acme", "SWE Intern", "Seattle, WA", "") + austin := newID("Acme", "SWE Intern", "Austin, TX", "") + + if seattle == austin { + t.Error("closed listings in different cities collapsed to one ID") + } +} + +func TestNewIDFallbackCanonicalizes(t *testing.T) { + a := newID("Acme, Inc.", "Software Engineer", "Seattle, WA", "") + b := newID("ACME", "Software Engineer", "seattle, wa", "") + + if a != b { + t.Errorf("same linkless posting produced different IDs: %q vs %q", a, b) + } +} + +// TestNewIDSchemesDoNotCollide checks the "u|" / "cpl|" namespacing. +func TestNewIDSchemesDoNotCollide(t *testing.T) { + withURL := newID("Acme", "SWE Intern", "Seattle, WA", "https://acme.com/j/1") + withoutURL := newID("Acme", "SWE Intern", "Seattle, WA", "") + + if withURL == withoutURL { + t.Error("URL identity collided with company/position/location identity") + } +} + +func TestDedupeCollapsesDuplicates(t *testing.T) { + all := []Job{ + {ID: "a", Company: "Acme", Position: "SWE"}, + {ID: "b", Company: "Globex", Position: "SWE"}, + {ID: "a", Company: "Acme", Position: "SWE"}, + } + + got := Dedupe(all) + if len(got) != 2 { + t.Fatalf("expected 2 jobs after dedupe, got %d: %+v", len(got), got) + } +} + +// TestDedupePreservesFirstSeenOrder is the regression test for the map +// iteration trap. Offset pagination slices this list, so a shuffled order +// between scrapes would make readers skip and repeat jobs across pages. +func TestDedupePreservesFirstSeenOrder(t *testing.T) { + all := []Job{ + {ID: "c", Company: "Third"}, + {ID: "a", Company: "First"}, + {ID: "b", Company: "Second"}, + {ID: "a", Company: "First"}, // duplicate, must not move "a" later + } + + got := Dedupe(all) + want := []string{"c", "a", "b"} + + if len(got) != len(want) { + t.Fatalf("expected %d jobs, got %d", len(want), len(got)) + } + for i, id := range want { + if got[i].ID != id { + t.Errorf("position %d: got ID %q, want %q", i, got[i].ID, id) + } + } +} + +// TestDedupeIsDeterministic runs the same input repeatedly, since a +// map-iteration bug would only show up intermittently. +func TestDedupeIsDeterministic(t *testing.T) { + all := []Job{} + for _, id := range []string{"e", "d", "c", "b", "a", "e", "d", "c"} { + all = append(all, Job{ID: id, Company: id}) + } + + first := Dedupe(all) + for i := 0; i < 50; i++ { + if got := Dedupe(all); !reflect.DeepEqual(got, first) { + t.Fatalf("run %d differed from the first run:\n got: %+v\nfirst: %+v", i, got, first) + } + } +} + +// TestDedupeEnriches covers the reason merging beats discarding: the sources +// carry complementary data. +func TestDedupeEnriches(t *testing.T) { + all := []Job{ + {ID: "a", Company: "Acme", Position: "SWE", SourceRepo: "speedyapply/x"}, + {ID: "a", Company: "Acme", Position: "SWE", Salary: "$120k", Age: "3d", Location: "Seattle, WA"}, + } + + got := Dedupe(all) + if len(got) != 1 { + t.Fatalf("expected 1 job, got %d", len(got)) + } + if got[0].Salary != "$120k" { + t.Errorf("salary should have been filled from the duplicate, got %q", got[0].Salary) + } + if got[0].Location != "Seattle, WA" { + t.Errorf("location should have been filled from the duplicate, got %q", got[0].Location) + } + if got[0].SourceRepo != "speedyapply/x" { + t.Errorf("primary's SourceRepo should win, got %q", got[0].SourceRepo) + } +} + +// TestDedupeClosedRequiresAgreement covers the deliberate bias toward showing +// jobs rather than hiding them. +func TestDedupeClosedRequiresAgreement(t *testing.T) { + tests := []struct { + name string + a, b bool + wantClosed bool + }{ + {"both open", false, false, false}, + {"primary open, duplicate closed", false, true, false}, + {"primary closed, duplicate open", true, false, false}, + {"both closed", true, true, true}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := Dedupe([]Job{ + {ID: "a", Closed: tc.a}, + {ID: "a", Closed: tc.b}, + }) + if len(got) != 1 { + t.Fatalf("expected 1 job, got %d", len(got)) + } + if got[0].Closed != tc.wantClosed { + t.Errorf("Closed = %v, want %v", got[0].Closed, tc.wantClosed) + } + }) + } +} + +// TestDedupeKeepsEmptyIDsSeparate - collapsing every unidentifiable row into +// one entry would hide real postings. +func TestDedupeKeepsEmptyIDsSeparate(t *testing.T) { + all := []Job{ + {ID: "", Company: "Acme"}, + {ID: "", Company: "Globex"}, + } + + if got := Dedupe(all); len(got) != 2 { + t.Errorf("expected both empty-ID jobs kept, got %d", len(got)) + } +} + +func TestDedupeEmptyInput(t *testing.T) { + if got := Dedupe(nil); len(got) != 0 { + t.Errorf("expected empty result, got %+v", got) + } +} + +// TestDedupeKeepsDistinctRolesAtOneCompany covers SimplifyJobs' "↳" rows, +// which share a company cell but are genuinely different roles. +func TestDedupeKeepsDistinctRolesAtOneCompany(t *testing.T) { + all := []Job{ + {ID: newID("Acme", "Software Engineer Intern", "Seattle, WA", ""), Company: "Acme"}, + {ID: newID("Acme", "Data Scientist Intern", "Seattle, WA", ""), Company: "Acme"}, + {ID: newID("Acme", "Product Manager Intern", "Seattle, WA", ""), Company: "Acme"}, + } + + if got := Dedupe(all); len(got) != 3 { + t.Errorf("three distinct roles at one company collapsed to %d", len(got)) + } +} diff --git a/backend/internal/jobs/github.go b/backend/internal/jobs/github.go new file mode 100644 index 0000000..c0cc0fe --- /dev/null +++ b/backend/internal/jobs/github.go @@ -0,0 +1,51 @@ +package jobs + +import ( + "context" + "fmt" + "io" + "net/http" +) + +// fetchReadme downloads one file's raw text from a public GitHub repo, using +// GitHub's REST "get repository content" endpoint. Passing the "raw" Accept +// header tells GitHub to hand back the file's plain text directly, instead +// of a JSON envelope with the content base64-encoded - one less decoding +// step for a beginner-friendly scraper. +// +// token is optional: an empty string still works, just at GitHub's lower +// unauthenticated rate limit (60 requests/hour per caller). These days the +// caller is the api Lambda's GET /jobs route, which fetches both files at +// most once every 5 minutes thanks to its in-memory cache - so a token +// isn't strictly required, but it's wired up (GITHUB_TOKEN_SSM, same +// pattern as LEETCODE_SESSION) to keep well clear of that limit. +func fetchReadme(ctx context.Context, token, owner, repo, ref, path string) (string, error) { + url := fmt.Sprintf("https://api.github.com/repos/%s/%s/contents/%s?ref=%s", owner, repo, path, ref) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return "", err + } + // GitHub rejects API requests that don't send a User-Agent. + req.Header.Set("User-Agent", "kronos-job-board") + req.Header.Set("Accept", "application/vnd.github.v3.raw") + req.Header.Set("X-GitHub-Api-Version", "2022-11-28") + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("github api %s/%s@%s/%s: %d %s", owner, repo, ref, path, resp.StatusCode, body) + } + return string(body), nil +} diff --git a/backend/internal/jobs/html.go b/backend/internal/jobs/html.go new file mode 100644 index 0000000..38d2eea --- /dev/null +++ b/backend/internal/jobs/html.go @@ -0,0 +1,54 @@ +package jobs + +import ( + "html" + "regexp" + "strings" +) + +// Both READMEs are Markdown files, but the job tables inside them are +// written as raw HTML (// rows (the just has column labels). +var ( + tbodyPattern = regexp.MustCompile(`(?is)(.*?)`) + trPattern = regexp.MustCompile(`(?is)(.*?)`) + tdPattern = regexp.MustCompile(`(?is)`) +) + +// parseSimplifyHTMLTable extracts every job row out of section's HTML table. +// Each row has 5 cells: Company, Role, Location, Application, Age. +func parseSimplifyHTMLTable(section, sectionLabel string) []Job { + body := tbodyPattern.FindStringSubmatch(section) + if body == nil { + return nil + } + + var jobs []Job + lastCompany := "" // see the "↳" handling below + for _, tr := range trPattern.FindAllStringSubmatch(body[1], -1) { + cells := tdPattern.FindAllStringSubmatch(tr[1], -1) + if len(cells) < 5 { + continue + } + + // SimplifyJobs collapses repeat roles at the same company: the first + // row has the company name, and every following row for that same + // company just has "↳" in the Company cell. Reuse the last real + // company name whenever we see that marker. + company := cleanText(stripTags(cells[0][1])) + if company == "↳" || company == "" { + company = lastCompany + } else { + lastCompany = company + } + if company == "" { + continue + } + + position := cleanText(stripTags(cells[1][1])) + location := cleanText(stripTags(cells[2][1])) + applicationCell := cells[3][1] + age := cleanText(stripTags(cells[4][1])) + + // A closed application's cell is just a 🔒 emoji with no link. + closed := strings.Contains(applicationCell, "🔒") + postingURL := firstHref(applicationCell) + + jobs = append(jobs, Job{ + ID: newID(company, position, location, postingURL), + Company: company, + Position: position, + Location: location, + PostingURL: postingURL, + Age: age, + Closed: closed, + SourceRepo: simplifyOwner + "/" + simplifyRepo, + SourceSection: sectionLabel, + }) + } + return jobs +} diff --git a/backend/internal/jobs/sources.go b/backend/internal/jobs/sources.go new file mode 100644 index 0000000..480d7ff --- /dev/null +++ b/backend/internal/jobs/sources.go @@ -0,0 +1,30 @@ +package jobs + +// SpeedyApply - internship listings. +// +// Its tables are written as raw HTML (
, or plain HTML links inside +// Markdown pipe-table cells). These helpers turn one HTML-ish table cell +// into plain text, since that's all the frontend actually wants to display. + +// tagPattern matches any HTML tag, e.g. , , . +var tagPattern = regexp.MustCompile(`<[^>]*>`) + +// hrefPattern pulls the URL out of the first href="..." attribute it finds. +var hrefPattern = regexp.MustCompile(`href="([^"]*)"`) + +// lineBreakPattern matches every line-break spelling the two READMEs use - +// the standard
, and SimplifyJobs'
typo'd as
- so a +// multi-location cell like "Seattle, WA
Austin, TX" becomes +// "Seattle, WA, Austin, TX" instead of the two locations running together. +var lineBreakPattern = regexp.MustCompile(`(?i)|
`) + +// summaryPattern strips a
5 locations label. We +// want the location list that follows it, not the "N locations" summary text. +var summaryPattern = regexp.MustCompile(`(?is).*?`) + +// stripTags removes every HTML tag from s (after handling line breaks and +// specially, see above) and decodes leftover entities like & +// so the result reads like plain text. +func stripTags(s string) string { + s = summaryPattern.ReplaceAllString(s, "") + s = lineBreakPattern.ReplaceAllString(s, ", ") + s = tagPattern.ReplaceAllString(s, "") + return html.UnescapeString(s) +} + +// firstHref returns the URL inside the first href="..." attribute in s, or +// "" if there isn't one (e.g. a closed-application cell that's just an emoji). +func firstHref(s string) string { + m := hrefPattern.FindStringSubmatch(s) + if m == nil { + return "" + } + return m[1] +} + +// cleanText trims whitespace and collapses repeated spaces/newlines so text +// pulled out of indented HTML reads like a single line. +func cleanText(s string) string { + return strings.Join(strings.Fields(s), " ") +} diff --git a/backend/internal/jobs/job.go b/backend/internal/jobs/job.go new file mode 100644 index 0000000..d0a429d --- /dev/null +++ b/backend/internal/jobs/job.go @@ -0,0 +1,91 @@ +// Package jobs scrapes public new-grad/internship job lists out of two +// GitHub READMEs and turns them into a plain []Job slice. +// +// This feature has no server-side storage of any kind - no database table, +// no S3 object, no file on disk. The job data is derived from a public +// source rather than owned by us, so it can be re-fetched at any moment, +// which makes any copy we keep a cache rather than a record. The api +// Lambda's GET /jobs route (backend/internal/api/jobs.go) calls the +// fetchers here directly and caches the result in process memory; the +// browser caches its own page of that response in localStorage (see +// src/lib/jobsCache.ts). Those two layers are the whole caching story. +package jobs + +import ( + "crypto/sha1" + "encoding/hex" +) + +// Job is one job posting scraped from a GitHub README table. +type Job struct { + // ID is a short fingerprint identifying this posting - see newID for how + // it's derived. It's deterministic (scraping the same posting twice + // always produces the same ID) and canonicalized, so the same job + // written differently in the two source READMEs still lands on one ID. + // That lets Dedupe collapse cross-source duplicates, and lets a future + // email-sync Lambda compute the same ID from a parsed email to find + // "which dashboard job is this email about". See the TODO below. + ID string `json:"id"` + Company string `json:"company"` + Position string `json:"position"` + Location string `json:"location"` + Salary string `json:"salary,omitempty"` + PostingURL string `json:"postingUrl,omitempty"` + Age string `json:"age,omitempty"` // straight from the README, e.g. "3d", "1mo" - not parsed into a real duration + Closed bool `json:"closed"` // true when the README marks the application as closed + SourceRepo string `json:"sourceRepo"` // e.g. "speedyapply/2027-SWE-College-Jobs" + SourceSection string `json:"sourceSection"` // which table/heading it came from, e.g. "FAANG+", "Software Engineering" + + // TODO(email-sync): the email-sync Lambda (backend/cmd/emailsync) will + // eventually look a job up by ID and attach fields like these: + // OAStatus string `json:"oaStatus,omitempty"` // "", "OA_RECEIVED", "INTERVIEW_SCHEDULED", "REJECTED" + // OAStatusUpdatedAt string `json:"oaStatusUpdatedAt,omitempty"` + // Nothing sets those yet - this is just marking where they'd go so the + // two features can meet in the middle later. +} + +// newID builds the fingerprint that decides whether two scraped rows are the +// same posting. Hashing (instead of concatenating the fields raw) keeps the +// ID short and free of characters that would need escaping in a URL or JSON +// key. +// +// There are two identity schemes, tried in order: +// +// 1. The canonical posting URL, when there is one. Both READMEs link to the +// employer's real applicant-tracking page (Greenhouse, Lever, Workday), +// and when the same job appears in both they almost always point at the +// identical link. Identifying by URL therefore sidesteps the hardest +// problem entirely - two repos wording the same role differently still +// collapse to one ID, with no fuzzy title matching anywhere. +// +// 2. Company + position + location, when the posting has no link at all. +// SimplifyJobs renders a closed application as a lock emoji with no +// href, so this path is mostly closed listings. Location is part of the +// key because without it every closed role sharing a company and title +// merges into one entry regardless of city, hiding real openings. +// +// The two schemes are namespaced ("u|" vs "cpl|") so a canonical URL can +// never collide with a company/position/location triple. +// +// Known limitation: a posting with a link and the same posting without one +// produce different IDs and will not merge. That's under-merging - the job +// shows up twice - which is the deliberate direction to fail in, since +// over-merging would make a real posting disappear. +func newID(company, position, location, postingURL string) string { + if u := canonicalURL(postingURL); u != "" { + return hashKey("u|" + u) + } + return hashKey("cpl|" + + canonicalCompany(company) + "|" + + canonicalPosition(position) + "|" + + canonicalLocation(location)) +} + +// hashKey reduces an identity key to 16 hex characters. Truncating SHA-1 +// this way is fine here: the risk is an accidental collision between two +// real postings, not an adversary engineering one, and 64 bits is far more +// than enough for a few thousand rows. +func hashKey(key string) string { + sum := sha1.Sum([]byte(key)) + return hex.EncodeToString(sum[:])[:16] +} diff --git a/backend/internal/jobs/sections.go b/backend/internal/jobs/sections.go new file mode 100644 index 0000000..20f7754 --- /dev/null +++ b/backend/internal/jobs/sections.go @@ -0,0 +1,70 @@ +package jobs + +import "strings" + +// sectionBounds returns the slice of doc that starts at the heading line +// containing titleContains and runs up to (but not including) the next line +// that starts with headingPrefix (e.g. "## "). If no later heading exists, +// it returns the rest of the document. Returns "" if titleContains isn't +// found at all. +// +// This is how we isolate e.g. "just the USA internships section" out of a +// 280-line README before we bother looking for table rows in it. +func sectionBounds(doc, titleContains, headingPrefix string) string { + start := strings.Index(doc, titleContains) + if start == -1 { + return "" + } + // Back up to the start of that heading's own line. + lineStart := strings.LastIndex(doc[:start], "\n") + 1 + body := doc[lineStart:] + + // Search for the next heading line *after* this one, so we don't + // immediately match the heading we just found. + afterFirstLine := strings.IndexByte(body, '\n') + if afterFirstLine == -1 { + return body + } + rest := body[afterFirstLine+1:] + next := strings.Index(rest, "\n"+headingPrefix) + if next == -1 { + return body + } + return body[:afterFirstLine+1+next+1] +} + +// subsection is one heading + the text under it, as produced by +// splitSubsections below. +type subsection struct { + Title string + Body string +} + +// splitSubsections splits body into pieces at each line that starts with +// prefix (e.g. "### "), using the heading text (minus the prefix) as each +// piece's Title. Text before the first matching heading is discarded - the +// two READMEs we scrape never put job rows there. +func splitSubsections(body, prefix string) []subsection { + var subs []subsection + var title string + var buf strings.Builder + + flush := func() { + if title != "" { + subs = append(subs, subsection{Title: title, Body: buf.String()}) + } + buf.Reset() + } + + for _, line := range strings.Split(body, "\n") { + if strings.HasPrefix(line, prefix) { + flush() + title = strings.TrimSpace(strings.TrimPrefix(line, prefix)) + continue + } + buf.WriteString(line) + buf.WriteByte('\n') + } + flush() + return subs +} diff --git a/backend/internal/jobs/simplifyjobs.go b/backend/internal/jobs/simplifyjobs.go new file mode 100644 index 0000000..2c99c64 --- /dev/null +++ b/backend/internal/jobs/simplifyjobs.go @@ -0,0 +1,99 @@ +package jobs + +import ( + "context" + "regexp" + "strings" +) + +// FetchSimplifyJobsNewGrad downloads the SimplifyJobs New-Grad-Positions +// README and extracts jobs from the two sections the issue asked for: +// Software Engineering and Product Management. (The README also has Data +// Science/AI, Quant, and Hardware sections - not scraped here, but adding +// one is a one-line call to parseSimplifySection, see below.) The repo it +// reads is declared in sources.go. +func FetchSimplifyJobsNewGrad(ctx context.Context, githubToken string) ([]Job, error) { + md, err := fetchReadme(ctx, githubToken, simplifyOwner, simplifyRepo, simplifyRef, simplifyPath) + if err != nil { + return nil, err + } + + var jobs []Job + jobs = append(jobs, parseSimplifySection(md, "Software Engineering New Grad Roles", "Software Engineering")...) + jobs = append(jobs, parseSimplifySection(md, "Product Management New Grad Roles", "Product Management")...) + return jobs, nil +} + +// parseSimplifySection isolates one "## " section +// of the README (by searching for headingContains, since the emoji prefix +// makes an exact match brittle) and parses the HTML table inside it. +func parseSimplifySection(md, headingContains, sectionLabel string) []Job { + section := sectionBounds(md, headingContains, "## ") + if section == "" { + return nil + } + return parseSimplifyHTMLTable(section, sectionLabel) +} + +// Unlike SpeedyApply's Markdown pipe tables, SimplifyJobs writes each job +// table as raw HTML: ...
...
. +// We only want the
(.*?)
//
) embedded in the +// Markdown; see parseSpeedyApplyMarkdown in speedyapply.go. Section names +// (FAANG+, Quant, and the rest) come from the README's own headings rather +// than a fixed list here. +const ( + speedyApplyOwner = "speedyapply" + speedyApplyRepo = "2027-SWE-College-Jobs" + speedyApplyRef = "main" + speedyApplyPath = "README.md" +) + +// SimplifyJobs - full-time new-grad listings. +// +// Only two of this README's sections are scraped, "Software Engineering New +// Grad Roles" and "Product Management New Grad Roles"; the Data Science/AI, +// Quant and Hardware sections are left alone (see FetchSimplifyJobsNewGrad in +// simplifyjobs.go, where adding one is a single line). +// +// Note the ref: this repo's default branch is "dev", not "main", which is +// unusual enough to be worth stating twice. +const ( + simplifyOwner = "SimplifyJobs" + simplifyRepo = "New-Grad-Positions" + simplifyRef = "dev" + simplifyPath = "README.md" +) diff --git a/backend/internal/jobs/speedyapply.go b/backend/internal/jobs/speedyapply.go new file mode 100644 index 0000000..8062304 --- /dev/null +++ b/backend/internal/jobs/speedyapply.go @@ -0,0 +1,99 @@ +package jobs + +import ( + "context" + "regexp" + "strings" +) + +// FetchSpeedyApplyJobs downloads the SpeedyApply internship README and turns +// its Markdown tables into Job structs. The repo it reads is declared in +// sources.go. +func FetchSpeedyApplyJobs(ctx context.Context, githubToken string) ([]Job, error) { + md, err := fetchReadme(ctx, githubToken, speedyApplyOwner, speedyApplyRepo, speedyApplyRef, speedyApplyPath) + if err != nil { + return nil, err + } + return parseSpeedyApplyMarkdown(md), nil +} + +// pipeRowPattern matches one Markdown table row: a line that starts and ends +// with "|", e.g. "| Roblox | SWE Intern | SF | $60/hr | | 1d |". +// This also matches the header row ("| Company | Position | ... |") and the +// separator row ("|---|---|...|") - parseSpeedyApplyMarkdown filters those +// out below, since only real data rows contain an link. +var pipeRowPattern = regexp.MustCompile(`(?m)^\|(.+)\|\s*$`) + +// parseSpeedyApplyMarkdown walks the "USA SWE Internships" section of the +// README - the one linked in the issue - and extracts every job row from +// every subsection's table (FAANG+, Quant, Other). +func parseSpeedyApplyMarkdown(md string) []Job { + section := sectionBounds(md, "## 2027 USA SWE Internships", "## ") + if section == "" { + return nil + } + + var jobs []Job + for _, sub := range splitSubsections(section, "### ") { + for _, m := range pipeRowPattern.FindAllStringSubmatch(sub.Body, -1) { + inner := m[1] + if !strings.Contains(inner, " ({ + id: `mock-${i}`, + company: COMPANIES[i % COMPANIES.length], + position: `Software Engineer Intern #${i}`, + location: i % 3 === 0 ? "Remote" : "San Francisco, CA", + salary: i % 4 === 0 ? "" : `$${40 + (i % 20)}/hr`, + postingUrl: `https://example.com/jobs/${i}`, + age: `${i}d`, + closed: i % 11 === 0, + sourceRepo: i % 2 === 0 ? "speedyapply/2027-SWE-College-Jobs" : "SimplifyJobs/New-Grad-Positions", + sourceSection: SECTIONS[i % SECTIONS.length], +})); + +const server = http.createServer((req, res) => { + // Same CORS headers the real api Lambda sends (backend/internal/api/api.go's + // reply()) - needed because Vite's dev server (localhost:5173) and this + // mock server (localhost:8787) are different origins as far as the + // browser is concerned. + res.setHeader("Access-Control-Allow-Origin", "*"); + res.setHeader("Access-Control-Allow-Headers", "authorization,content-type"); + res.setHeader("Access-Control-Allow-Methods", "GET,POST,DELETE,OPTIONS"); + + if (req.method === "OPTIONS") { + res.writeHead(204); + res.end(); + return; + } + + const url = new URL(req.url, `http://localhost:${PORT}`); + + if (req.method === "GET" && url.pathname === "/jobs") { + const limit = Number(url.searchParams.get("limit") ?? "20") || 20; + const offset = Number(url.searchParams.get("offset") ?? "0") || 0; + + const end = Math.min(offset + limit, ALL_JOBS.length); + const page = offset < ALL_JOBS.length ? ALL_JOBS.slice(offset, end) : []; + const nextOffset = end < ALL_JOBS.length ? end : null; + + console.log(`GET /jobs?limit=${limit}&offset=${offset} -> ${page.length} jobs, nextOffset=${nextOffset}`); + + res.setHeader("Content-Type", "application/json"); + res.writeHead(200); + res.end(JSON.stringify({ jobs: page, scrapedAt: new Date().toISOString(), nextOffset })); + return; + } + + res.writeHead(404); + res.end(JSON.stringify({ error: "not found" })); +}); + +server.listen(PORT, () => { + console.log(`Mock jobs API running at http://localhost:${PORT}/jobs (local only - not AWS)`); + console.log(`${ALL_JOBS.length} fake jobs loaded, ${ALL_JOBS.filter((j) => j.closed).length} marked closed.`); +}); diff --git a/src/App.tsx b/src/App.tsx index 9267e0e..bd24cbd 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -28,6 +28,8 @@ import { LeaderboardCard } from "./sections/LeaderboardCard"; import { MyFriendsCard } from "./sections/MyFriendsCard"; import { CurrentStreakCard } from "./sections/CurrentStreakCard"; import { RecentActivityCard } from "./sections/RecentActivityCard"; +import { JobBoardCard } from "./sections/JobBoardCard"; +import { JobBoardModal } from "./modals/JobBoardModal"; import { ProgressModal } from "./modals/ProgressModal"; import { MySolutionModal } from "./modals/MySolutionModal"; import { CalendarModal } from "./modals/CalendarModal"; @@ -91,6 +93,7 @@ function App({ const [sdComponents, setSdComponents] = useState(false); const [cloudOpen, setCloudOpen] = useState(false); const [networkingOpen, setNetworkingOpen] = useState(false); + const [jobBoardOpen, setJobBoardOpen] = useState(false); // Admin-only: warn when the LeetCode session token is near/at expiry. const [sessionExpiry, setSessionExpiry] = useState(""); useEffect(() => { @@ -178,6 +181,7 @@ function App({ setCloudOpen(true)} /> setNetworkingOpen(true)} /> + setJobBoardOpen(true)} /> @@ -214,6 +218,7 @@ function App({ {sdComponents && setSdComponents(false)} />} {cloudOpen && setCloudOpen(false)} />} {networkingOpen && setNetworkingOpen(false)} />} + {jobBoardOpen && setJobBoardOpen(false)} />} {modal === "calendar" && ( call("/me/theme", t, { method: "POST", body: JSON.stringify({ theme }) }), + jobs: (t: TokenFn, limit = 20, offset = 0) => + call(`/jobs?limit=${limit}&offset=${offset}`, t), progress: (t: TokenFn) => call("/me/progress", t), leaderboard: (t: TokenFn) => call("/leaderboard", t), recent: (t: TokenFn) => call("/recent", t), @@ -102,6 +104,23 @@ export type MeResponse = { season: number; requestedUsername?: string; }; +// One job posting, as returned by GET /jobs. Matches jobs.Job on the Go +// side (backend/internal/jobs/job.go) field for field. +export type ApiJob = { + id: string; + company: string; + position: string; + location: string; + salary?: string; + postingUrl?: string; + age?: string; + closed: boolean; + sourceRepo: string; + sourceSection: string; +}; +// nextOffset is the offset to pass on the next call to keep paging, or null +// once you've reached the end of the cached list. +export type JobsPage = { jobs: ApiJob[]; scrapedAt: string; nextOffset: number | null }; export type ApiProblem = { slug: string; title: string; diff --git a/src/lib/jobsCache.ts b/src/lib/jobsCache.ts new file mode 100644 index 0000000..5b0f0f1 --- /dev/null +++ b/src/lib/jobsCache.ts @@ -0,0 +1,62 @@ +import { api, type JobsPage, type TokenFn } from "./api"; + +// Browser-side cache for GET /jobs pages, stored in localStorage. This is +// the outer of the two caches behind the Job Board: it sits in front of +// GET /jobs, which itself scrapes two GitHub READMEs on demand and keeps +// the result in the api Lambda's process memory for 5 minutes (see +// backend/internal/api/jobs.go). There is no server-side store beyond that +// - no bucket, no table - because the job data is public and re-fetchable +// at any time. +// +// This layer just means reopening the Job Board modal, or the card +// re-mounting, within a short window doesn't trigger a fresh network call +// for data we already have. +// +// How long a cached page is considered fresh enough to reuse. Kept shorter +// than the server's own 5-minute memory cache, so this never serves data +// meaningfully staler than the server would. +const TTL_MS = 2 * 60 * 1000; // 2 minutes + +type CacheEntry = { page: JobsPage; cachedAt: number }; + +// Each distinct (limit, offset) pair - i.e. each distinct page - gets its +// own cache entry, since they hold different jobs. +function cacheKey(limit: number, offset: number): string { + return `kronos:jobs:${limit}:${offset}`; +} + +function readCache(limit: number, offset: number): JobsPage | null { + try { + const raw = localStorage.getItem(cacheKey(limit, offset)); + if (!raw) return null; + const entry: CacheEntry = JSON.parse(raw); + if (Date.now() - entry.cachedAt > TTL_MS) return null; // stale - treat as a miss + return entry.page; + } catch { + // localStorage can throw (private browsing, corrupted JSON, storage + // disabled) - caching is an optimization, never let it break the page. + return null; + } +} + +function writeCache(limit: number, offset: number, page: JobsPage): void { + try { + const entry: CacheEntry = { page, cachedAt: Date.now() }; + localStorage.setItem(cacheKey(limit, offset), JSON.stringify(entry)); + } catch { + // Storage full or unavailable - just skip caching this time. + } +} + +// fetchJobsPage is what JobBoardCard/JobBoardModal call instead of +// api.jobs() directly: it transparently serves a fresh-enough cached page +// when one exists, and otherwise falls through to the real network request +// (caching that result for next time). +export async function fetchJobsPage(getToken: TokenFn, limit: number, offset: number): Promise { + const cached = readCache(limit, offset); + if (cached) return cached; + + const page = await api.jobs(getToken, limit, offset); + writeCache(limit, offset, page); + return page; +} diff --git a/src/modals/JobBoardModal.tsx b/src/modals/JobBoardModal.tsx new file mode 100644 index 0000000..6fb75b4 --- /dev/null +++ b/src/modals/JobBoardModal.tsx @@ -0,0 +1,167 @@ +import { useEffect, useMemo, useState } from "react"; +import { Search, ExternalLink, Lock } from "lucide-react"; +import { Modal } from "../components/Modal"; +import { useData } from "../data/source"; +import { type ApiJob } from "../lib/api"; +import { fetchJobsPage } from "../lib/jobsCache"; + +// JobBoardModal is the full-list view behind the Job Board card: every job +// currently cached in S3 (from GET /jobs - see backend/internal/jobs/cache.go), +// with a text search and a section filter (e.g. "FAANG+", "Software +// Engineering"). It fetches its own data independently of JobBoardCard's +// preview fetch, the same way FriendsModal/LeaderboardModal fetch their own +// full lists. +export function JobBoardModal({ onClose }: { onClose: () => void }) { + const { getToken } = useData(); + const [jobs, setJobs] = useState([]); + const [loading, setLoading] = useState(true); + const [loadingMore, setLoadingMore] = useState(false); + const [nextOffset, setNextOffset] = useState(null); + const [query, setQuery] = useState(""); + const [section, setSection] = useState(null); + + useEffect(() => { + fetchJobsPage(getToken, 20, 0) + .then((page) => { + setJobs(page.jobs ?? []); + setNextOffset(page.nextOffset); + }) + .catch(() => setJobs([])) + .finally(() => setLoading(false)); + }, [getToken]); + + // Fetches the next page starting at nextOffset (from the last response), + // and appends it to what's already on screen (rather than replacing it). + const loadMore = () => { + if (nextOffset === null || loadingMore) return; + setLoadingMore(true); + fetchJobsPage(getToken, 20, nextOffset) + .then((page) => { + setJobs((prev) => [...prev, ...(page.jobs ?? [])]); + setNextOffset(page.nextOffset); + }) + .finally(() => setLoadingMore(false)); + }; + + // The chip bar: one chip per distinct sourceSection seen in the data + // (e.g. "FAANG+", "Quant", "Other", "Software Engineering", "Product + // Management"), in the order they first appear. + const sections = useMemo(() => { + const seen: string[] = []; + for (const j of jobs) if (!seen.includes(j.sourceSection)) seen.push(j.sourceSection); + return seen; + }, [jobs]); + + const q = query.trim().toLowerCase(); + const shown = jobs.filter((j) => { + if (section && j.sourceSection !== section) return false; + if (!q) return true; + return ( + j.company.toLowerCase().includes(q) || + j.position.toLowerCase().includes(q) || + j.location.toLowerCase().includes(q) + ); + }); + + return ( + +
+ + setQuery(e.target.value)} + placeholder="Search company, role, or location…" + className="w-full bg-transparent text-sm placeholder:text-muted-foreground focus:outline-none" + /> +
+ + {sections.length > 0 && ( +
+ + {sections.map((s) => ( + + ))} +
+ )} + + + + ); +} diff --git a/src/sections/JobBoardCard.tsx b/src/sections/JobBoardCard.tsx new file mode 100644 index 0000000..7c282e4 --- /dev/null +++ b/src/sections/JobBoardCard.tsx @@ -0,0 +1,79 @@ +import { useEffect, useState } from "react"; +import { Briefcase } from "lucide-react"; +import { Card } from "../components/Card"; +import { useData } from "../data/source"; +import { type ApiJob } from "../lib/api"; +import { fetchJobsPage } from "../lib/jobsCache"; + +// How many jobs to show in the card's own scrollable preview - bigger than +// a plain top-5 slice (matching CloudCard/NetworkingCard/GenAICard's +// pattern of a scrollable list right on the dashboard tile, before you even +// open the full modal), but still well short of the modal's full paginated +// list. +const PREVIEW_COUNT = 20; + +// JobBoardCard is the dashboard tile for the Job Board feature. It shows a +// scrollable preview of new-grad/internship postings that our GET /jobs +// route scrapes from public GitHub job-list repos on demand - this +// component never talks to GitHub itself, only to our own API. +// Clicking the card opens JobBoardModal for the full, searchable list. +export function JobBoardCard({ onOpen }: { onOpen: () => void }) { + const { getToken } = useData(); + const [jobs, setJobs] = useState([]); + const [loading, setLoading] = useState(true); + + useEffect(() => { + fetchJobsPage(getToken, PREVIEW_COUNT, 0) + .then((page) => setJobs(page.jobs ?? [])) + .catch(() => setJobs([])) + .finally(() => setLoading(false)); + }, [getToken]); + + return ( + +
+
Job Board
+ +
+

+ New grad & internship roles, scraped from GitHub every minute. +

+
+ Job Board + {jobs.length} shown +
+
    + {loading && ( +
  • + Loading… +
    • + )} + {!loading && jobs.length === 0 && ( +
  • + No jobs yet — check back soon. +
    • + )} + {jobs.map((j) => ( +
  • +
    +
    + {j.company} — {j.position} +
    +
    + {j.location} +
    +
    + {j.age && ( + + {j.age} + + )} +
    • + ))} +
+ + ); +} diff --git a/terraform/main.tf b/terraform/main.tf index 834e25c..2866489 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -68,25 +68,41 @@ data "aws_ssm_parameter" "leetcode_session" { name = "/${var.project}/LEETCODE_SESSION" } +# Job Board scraping happens inside the api Lambda's GET /jobs route, which +# reads two GitHub READMEs on demand and caches the result in process memory +# (backend/internal/api/jobs.go). GitHub's unauthenticated rate limit is only +# 60 requests/hour, so a token is used to lift it to 5,000/hour - create it +# once with: +# aws ssm put-parameter --name /kronos/GITHUB_TOKEN --type SecureString --value +data "aws_ssm_parameter" "github_token" { + name = "/${var.project}/GITHUB_TOKEN" +} + locals { secret_arns = [ module.ssm.arn, data.aws_ssm_parameter.clerk_secret.arn, data.aws_ssm_parameter.leetcode_session.arn, ] + # The api Lambda additionally needs the GitHub token, since it's the one + # scraping the job board now. + api_secret_arns = concat(local.secret_arns, [ + data.aws_ssm_parameter.github_token.arn, + ]) } module "api" { source = "./modules/lambda" name = "${var.project}-api" zip_path = var.api_zip - ssm_parameter_arns = local.secret_arns + ssm_parameter_arns = local.api_secret_arns environment = { DATABASE_URL_SSM = module.ssm.name CLERK_SECRET_KEY_SSM = data.aws_ssm_parameter.clerk_secret.name ADMIN_CLERK_ID = var.admin_clerk_id SEASON_START = var.season_start LEETCODE_SESSION_SSM = data.aws_ssm_parameter.leetcode_session.name + GITHUB_TOKEN_SSM = data.aws_ssm_parameter.github_token.name } } @@ -113,6 +129,17 @@ module "enrich" { } } +module "emailsync" { + source = "./modules/lambda" + name = "${var.project}-emailsync" + zip_path = var.emailsync_zip + ssm_parameter_arns = local.secret_arns + environment = { + DATABASE_URL_SSM = module.ssm.name + LEETCODE_SESSION_SSM = data.aws_ssm_parameter.leetcode_session.name + } +} + module "apigateway" { source = "./modules/apigateway" name = "${var.project}-http" @@ -121,12 +148,14 @@ module "apigateway" { } module "scheduler" { - source = "./modules/scheduler" - name = var.project - sync_function_arn = module.sync.arn - sync_function_name = module.sync.function_name - enrich_function_arn = module.enrich.arn - enrich_function_name = module.enrich.function_name + source = "./modules/scheduler" + name = var.project + sync_function_arn = module.sync.arn + sync_function_name = module.sync.function_name + enrich_function_arn = module.enrich.arn + enrich_function_name = module.enrich.function_name + emailsync_function_arn = module.emailsync.arn + emailsync_function_name = module.emailsync.function_name } module "frontend" { diff --git a/terraform/modules/lambda/variables.tf b/terraform/modules/lambda/variables.tf index f03081b..ce792d5 100644 --- a/terraform/modules/lambda/variables.tf +++ b/terraform/modules/lambda/variables.tf @@ -40,3 +40,4 @@ variable "ssm_parameter_arns" { type = list(string) default = [] } + diff --git a/terraform/modules/scheduler/main.tf b/terraform/modules/scheduler/main.tf index 9f1513f..6c360cc 100644 --- a/terraform/modules/scheduler/main.tf +++ b/terraform/modules/scheduler/main.tf @@ -16,6 +16,24 @@ resource "aws_lambda_permission" "sync" { source_arn = aws_cloudwatch_event_rule.sync.arn } +resource "aws_cloudwatch_event_rule" "emailsync" { + name = "${var.name}-emailsync" + schedule_expression = var.emailsync_schedule +} + +resource "aws_cloudwatch_event_target" "emailsync" { + rule = aws_cloudwatch_event_rule.emailsync.name + arn = var.emailsync_function_arn +} + +resource "aws_lambda_permission" "emailsync" { + statement_id = "AllowEmailsyncSchedule" + action = "lambda:InvokeFunction" + function_name = var.emailsync_function_name + principal = "events.amazonaws.com" + source_arn = aws_cloudwatch_event_rule.emailsync.arn +} + # The standalone enrich cron is intentionally removed: the sync Lambda already # runs the enricher inline every pass (enricher.Run), so a separate per-minute # enrich invocation was duplicate work (extra Lambda + KMS decrypts). The enrich diff --git a/terraform/modules/scheduler/variables.tf b/terraform/modules/scheduler/variables.tf index d427fb7..d4769a4 100644 --- a/terraform/modules/scheduler/variables.tf +++ b/terraform/modules/scheduler/variables.tf @@ -18,6 +18,19 @@ variable "enrich_function_name" { type = string } +variable "emailsync_function_arn" { + type = string +} + +variable "emailsync_function_name" { + type = string +} + +variable "emailsync_schedule" { + type = string + default = "rate(15 minutes)" +} + variable "sync_schedule" { type = string default = "rate(1 minute)" diff --git a/terraform/variables.tf b/terraform/variables.tf index 8587544..60efc70 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -33,6 +33,11 @@ variable "enrich_zip" { default = "../backend/dist/enrich.zip" } +variable "emailsync_zip" { + type = string + default = "../backend/dist/emailsync.zip" +} + variable "admin_clerk_id" { type = string default = "user_3EmSENtZcQZXU9q9ptLa7uedUGK"