Skip to content

Commit

Permalink
improve dependency tree and file edits
Browse files Browse the repository at this point in the history
  • Loading branch information
florianbgt committed Nov 4, 2024
1 parent 4bbfc78 commit e2bbc8b
Show file tree
Hide file tree
Showing 18 changed files with 420 additions and 1,085 deletions.
743 changes: 52 additions & 691 deletions package-lock.json

Large diffs are not rendered by default.

1 change: 0 additions & 1 deletion packages/cli/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,6 @@
},
"dependencies": {
"axios": "^1.7.7",
"dependency-tree": "^11.0.1",
"express": "^4.21.1",
"http-proxy-middleware": "^3.0.3",
"openai": "^4.67.2",
Expand Down
11 changes: 5 additions & 6 deletions packages/cli/src/api/scan.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
import { z } from "zod";
import { getDependencyTree } from "../helper/dependencies";
import { ScanCodebaseResponsePayload } from "../helper/payloads";
import { getEndpontsFromTree } from "../helper/tree";
import {
getDependencyTree,
getEndpontsFromTree,
} from "../helper/dependencyTree";
import { scanSchema } from "./helpers/validation";

export function scan(
payload: z.infer<typeof scanSchema>,
): ScanCodebaseResponsePayload {
export function scan(payload: z.infer<typeof scanSchema>) {
const tree = getDependencyTree(payload.entrypointPath);

const endpoints = getEndpontsFromTree(tree);
Expand Down
16 changes: 8 additions & 8 deletions packages/cli/src/api/split.ts
Original file line number Diff line number Diff line change
@@ -1,17 +1,17 @@
import fs from "fs";
import path from "path";
import { getDependencyTree } from "../helper/dependencies";
import {
getDependencyTree,
getEndpontsFromTree,
getGroupsFromEndpoints,
} from "../helper/dependencyTree";
import { cleanupOutputDir, createOutputDir } from "../helper/file";
import { SplitCodebaseResponsePayload } from "../helper/payloads";
import { getEndpontsFromTree, splitPath } from "../helper/tree";
import { createSplit } from "../helper/split";
import { splitSchema } from "./helpers/validation";
import { z } from "zod";
import { GroupMap } from "../helper/types";
import { getGroupsFromEndpoints } from "../helper/groups";

export function split(
payload: z.infer<typeof splitSchema>,
): SplitCodebaseResponsePayload {
export function split(payload: z.infer<typeof splitSchema>) {
let groupIndex = 0;
const groupMap: GroupMap = {};

Expand All @@ -32,7 +32,7 @@ export function split(

// Process each endpoint for splitting
for (const group of groups) {
splitPath(
createSplit(
group,
payload.outputDir,
payload.entrypointPath,
Expand Down
98 changes: 50 additions & 48 deletions packages/cli/src/api/sync.ts
Original file line number Diff line number Diff line change
@@ -1,15 +1,17 @@
import { z } from "zod";
import { syncSchema } from "./helpers/validation";
import fs from "fs";
import { Dependencies } from "../helper/types";
import {
getNanoApiAnnotationFromCommentValue,
getParserLanguageFromFile,
replaceCommentFromAnnotation,
} from "../helper/file";
import { getDependencyTree } from "../helper/dependencies";
} from "../helper/annotations";
import {
getDependencyTree,
getEndpontsFromTree,
} from "../helper/dependencyTree";
import Parser from "tree-sitter";
import { getEndpontsFromTree } from "../helper/tree";
import { getParserLanguageFromFile } from "../helper/treeSitter";
import { replaceIndexesFromSourceCode } from "../helper/cleanup";

export function sync(payload: z.infer<typeof syncSchema>) {
const tree = getDependencyTree(payload.entrypointPath);
Expand All @@ -31,53 +33,53 @@ export function sync(payload: z.infer<typeof syncSchema>) {
return endpoint;
});

function iterateOverTreeAndUpdateContent(tree: Dependencies) {
for (const [filePath, value] of Object.entries(tree)) {
let sourceCode = fs.readFileSync(filePath, "utf-8");

updatedEndpoints.forEach((endpoint) => {
const language = getParserLanguageFromFile(filePath);
const parser = new Parser();
parser.setLanguage(language);

const tree = parser.parse(sourceCode);

function traverse(node: Parser.SyntaxNode) {
if (node.type === "comment") {
const comment = node.text;

const annotation = getNanoApiAnnotationFromCommentValue(comment);

if (annotation) {
if (
annotation.path === endpoint.path &&
annotation.method === endpoint.method
) {
annotation.group = endpoint.group;
const updatedComment = replaceCommentFromAnnotation(
comment,
annotation,
);
// Replace the comment in the source code
sourceCode = sourceCode.replace(comment, updatedComment);
}
}
}
node.children.forEach((child) => traverse(child));
}
updatedEndpoints.forEach((endpoint) => {
const language = getParserLanguageFromFile(endpoint.filePath);
const parser = new Parser();
parser.setLanguage(language);

let sourceCode = fs.readFileSync(endpoint.filePath, "utf-8");

const tree = parser.parse(sourceCode);

const indexesToReplace: {
startIndex: number;
endIndex: number;
text: string;
}[] = [];

traverse(tree.rootNode);
});
function traverse(node: Parser.SyntaxNode) {
if (node.type === "comment") {
const comment = node.text;

// update the file
fs.writeFileSync(filePath, sourceCode, "utf-8");
const annotation = getNanoApiAnnotationFromCommentValue(comment);

// Recursively process the tree
if (typeof value !== "string") {
iterateOverTreeAndUpdateContent(value);
if (annotation) {
if (
annotation.path === endpoint.path &&
annotation.method === endpoint.method
) {
annotation.group = endpoint.group;
const updatedComment = replaceCommentFromAnnotation(
comment,
annotation,
);

indexesToReplace.push({
startIndex: node.startIndex,
endIndex: node.endIndex,
text: updatedComment,
});
}
}
}
node.children.forEach((child) => traverse(child));
}
}

iterateOverTreeAndUpdateContent(tree);
traverse(tree.rootNode);

sourceCode = replaceIndexesFromSourceCode(sourceCode, indexesToReplace);

fs.writeFileSync(endpoint.filePath, sourceCode, "utf-8");
});
}
26 changes: 14 additions & 12 deletions packages/cli/src/commands/annotate.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import fs from "fs";
import { getDependencyTree } from "../helper/dependencies";
import { getDependencyTree } from "../helper/dependencyTree";
import OpenAI from "openai";
import { Dependencies } from "../helper/types";
import { DependencyTree } from "../helper/types";

export default async function annotateOpenAICommandHandler(
entrypoint: string, // Path to the entrypoint file
Expand Down Expand Up @@ -33,20 +33,22 @@ export default async function annotateOpenAICommandHandler(
I want you to understand what the application is doing and where each endpoint is.`,
});

function iterateOverTree(tree: Dependencies) {
for (const [filePath, value] of Object.entries(tree)) {
// TODO send chunks of each file for big files to respect the openAI limits
const content = fs.readFileSync(filePath, "utf8");
function iterateOverTree(tree: DependencyTree) {
messages.push({
role: "user",
content: `File: ${tree.path}
Content: ${tree.sourceCode}`,
});

tree.children.forEach((child) => {
messages.push({
role: "user",
content: `File: ${filePath}
Content: ${content}`,
content: `File: ${child.path}
Content: ${child.sourceCode}`,
});

if (typeof value !== "string") {
iterateOverTree(value);
}
}
iterateOverTree(child);
});
}
iterateOverTree(tree);

Expand Down
11 changes: 7 additions & 4 deletions packages/cli/src/commands/split.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
import path from "path";
import fs from "fs";
import { getDependencyTree } from "../helper/dependencies";
import {
getDependencyTree,
getEndpontsFromTree,
getGroupsFromEndpoints,
} from "../helper/dependencyTree";
import { cleanupOutputDir, createOutputDir } from "../helper/file";
import { getEndpontsFromTree, splitPath } from "../helper/tree";
import { createSplit } from "../helper/split";
import { GroupMap } from "../helper/types";
import { getGroupsFromEndpoints } from "../helper/groups";

export default function splitCommandHandler(
entrypointPath: string, // Path to the entrypoint file
Expand All @@ -26,7 +29,7 @@ export default function splitCommandHandler(

// Process each endpoint for splitting
for (const group of groups) {
splitPath(group, outputDir, entrypointPath, groupMap, groupIndex);
createSplit(group, outputDir, entrypointPath, groupMap, groupIndex);
groupIndex++;
}

Expand Down
44 changes: 44 additions & 0 deletions packages/cli/src/helper/annotations.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
import { NanoAPIAnnotation } from "./types";

export function getNanoApiAnnotationFromCommentValue(comment: string) {
const nanoapiRegex = /@nanoapi|((method|path|group):([^ ]+))/g;
const matches = comment.match(nanoapiRegex);
// remove first match, which is the @nanoapi identifier
matches?.shift();

if (matches && matches.length > 0) {
return matches.reduce((acc, match) => {
// key, first element when split with ":"
const key = match.split(":")[0];
// value, everything else
const value = match.split(":").slice(1).join(":");
return { ...acc, [key]: value };
}, {} as NanoAPIAnnotation);
}

return null;
}

export function replaceCommentFromAnnotation(
comment: string,
annotation: NanoAPIAnnotation,
) {
const commentRegex = /@nanoapi\s*(.*)/g;

// Construct the new annotation string
let newAnnotation = "@nanoapi";
if (annotation.method) {
newAnnotation += ` method:${annotation.method}`;
}
if (annotation.path) {
newAnnotation += ` path:${annotation.path}`;
}
if (annotation.group) {
newAnnotation += ` group:${annotation.group}`;
}

// Replace the old annotation with the new annotation
const updatedComment = comment.replace(commentRegex, newAnnotation);

return updatedComment;
}
75 changes: 75 additions & 0 deletions packages/cli/src/helper/cleanup.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
import fs from "fs";
import Parser from "tree-sitter";
import { cleanupJavascriptFile } from "./languages/javascript/cleanup";
import { extractJavascriptFileImports } from "./languages/javascript/imports";

import { resolveFilePath } from "./file";
import { Group } from "./types";
import { getParserLanguageFromFile } from "./treeSitter";

export function cleanupFile(filePath: string, group: Group) {
const language = getParserLanguageFromFile(filePath);
const parser = new Parser();
parser.setLanguage(language);

const sourceCode = fs.readFileSync(filePath, "utf8");

let dependencies: string[] = [];
if (["javascript", "typescript"].includes(language.name)) {
dependencies = extractJavascriptFileImports(parser, sourceCode);
} else {
throw new Error(`Unsupported language: ${language.language}`);
}

// Check if we can resolve the path for each dependency. If we cannot, we need to remove it
const invalidDependencies = dependencies.filter(
(dep) => !resolveFilePath(dep, filePath),
);

let updatedSourceCode: string;

if (["javascript", "typescript"].includes(language.name)) {
updatedSourceCode = cleanupJavascriptFile(
parser,
sourceCode,
group,
invalidDependencies,
);
} else {
throw new Error(`Unsupported language: ${language.language}`);
}

fs.writeFileSync(filePath, updatedSourceCode, "utf8");
}

export function removeIndexesFromSourceCode(
sourceCode: string,
indexesToRemove: { startIndex: number; endIndex: number }[],
) {
let newSourceCode = sourceCode;

// sort to start removing from the of the file end
indexesToRemove.sort((a, b) => b.startIndex - a.startIndex);

indexesToRemove.forEach(({ startIndex, endIndex }) => {
newSourceCode =
newSourceCode.slice(0, startIndex) + newSourceCode.slice(endIndex);
});

return newSourceCode;
}

export function replaceIndexesFromSourceCode(
sourceCode: string,
indexesToReplace: { startIndex: number; endIndex: number; text: string }[],
) {
// sort to start removing from the end of the file
indexesToReplace.sort((a, b) => b.startIndex - a.startIndex);

indexesToReplace.forEach(({ startIndex, endIndex, text }) => {
sourceCode =
sourceCode.slice(0, startIndex) + text + sourceCode.slice(endIndex);
});

return sourceCode;
}
Loading

0 comments on commit e2bbc8b

Please sign in to comment.