Skip to content

Commit f9b0ed5

Browse files
Sma1lboyNarwhalChenautofix-ci[bot]ZHallen122
authored
feat(backend): enhance config loader with embedding support and impro… (#74)
…ve logging <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit ## Release Notes - **Configuration Management** - Enhanced configuration loading and validation - Improved error handling for configuration files - Added support for managing multiple model configurations - **Model Download and Management** - Introduced universal model download utilities - Added status tracking for model downloads - Implemented singleton pattern for model downloaders - **Embedding Providers** - Added support for local and OpenAI embedding providers - Implemented embedding generation and model listing functionality - Enhanced error handling for embedding operations - **Environment and Dependency Updates** - Updated OpenAI package version - Added `fastembed` dependency - Improved environment variable configuration - **Testing and Utilities** - Removed some legacy test files - Added new test suites for embedding and model providers - Introduced new utility functions for model management <!-- end of auto-generated comment: release notes by coderabbit.ai --> --------- Co-authored-by: NarwhalChen <[email protected]> Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by: ZHallen122 <[email protected]>
1 parent 77f911e commit f9b0ed5

23 files changed

+1035
-133
lines changed

‎backend/src/build-system/__tests__/test.model-provider.spec.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
import { EmbeddingProvider } from "src/common/embedding-provider";
2+
3+
describe('Model Provider Test', () => {
4+
let embProvider = EmbeddingProvider.getInstance();
5+
it('should generate a response from the model provider', async () => {
6+
let res = await embProvider.generateEmbResponse("Your text string goes here", "text-embedding-3-small");
7+
console.log(res);
8+
});
9+
});

‎backend/src/config/common-path.ts‎

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import * as path from 'path';
2-
import { existsSync, mkdirSync, promises } from 'fs-extra';
3-
import { cwd } from 'process';
2+
3+
import { existsSync, mkdirSync, promises, writeFileSync } from 'fs-extra';
4+
import { ConfigType } from '@nestjs/config';
45

56
// Constants for base directories
67
const APP_NAME = 'codefox';
@@ -29,17 +30,30 @@ const ensureDir = (dirPath: string): string => {
2930
export const getRootDir = (): string => ensureDir(ROOT_DIR);
3031

3132
// Configuration Paths
32-
export const getConfigDir = (): string =>
33-
ensureDir(path.join(getRootDir(), 'config'));
34-
export const getConfigPath = (configName: string): string =>
35-
path.join(getConfigDir(), `${configName}.json`);
33+
export const getConfigPath = (): string => {
34+
const rootPath = ensureDir(path.join(getRootDir()));
35+
return path.join(rootPath, 'config.json');
36+
};
37+
38+
export const getModelStatusPath = (): string => {
39+
const rootPath = ensureDir(getRootDir());
40+
const modelStatusPath = path.join(rootPath, `model-status.json`);
41+
writeFileSync(modelStatusPath, '{}');
42+
return modelStatusPath;
43+
};
3644

3745
// Models Directory
3846
export const getModelsDir = (): string =>
3947
ensureDir(path.join(getRootDir(), 'models'));
4048
export const getModelPath = (modelName: string): string =>
4149
path.join(getModelsDir(), modelName);
4250

51+
// Embs Directory
52+
export const getEmbDir = (): string =>
53+
ensureDir(path.join(getRootDir(), 'embeddings'));
54+
export const getEmbPath = (modelName: string): string =>
55+
path.join(getModelsDir(), modelName);
56+
4357
// Project-Specific Paths
4458
export const getProjectsDir = (): string =>
4559
ensureDir(path.join(getRootDir(), 'projects'));

‎backend/src/config/config-loader.ts‎

Lines changed: 214 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,53 +1,253 @@
11
import * as fs from 'fs';
22
import * as _ from 'lodash';
33
import { getConfigPath } from './common-path';
4-
export interface ChatConfig {
4+
import { ConfigType } from 'src/downloader/universal-utils';
5+
import { Logger } from '@nestjs/common';
6+
7+
export interface ModelConfig {
58
model: string;
69
endpoint?: string;
710
token?: string;
811
default?: boolean;
912
task?: string;
1013
}
1114

12-
export class ConfigLoader {
13-
private chatsConfig: ChatConfig[];
15+
export interface EmbeddingConfig {
16+
model: string;
17+
endpoint?: string;
18+
default?: boolean;
19+
token?: string;
20+
}
1421

22+
export interface AppConfig {
23+
models?: ModelConfig[];
24+
embeddings?: EmbeddingConfig[];
25+
}
26+
27+
export const exampleConfigContent = `{
28+
// Chat models configuration
29+
// You can configure multiple chat models
30+
"models": [
31+
// Example of OpenAI GPT configuration
32+
{
33+
"model": "gpt-3.5-turbo",
34+
"endpoint": "https://api.openai.com/v1",
35+
"token": "your-openai-token", // Replace with your OpenAI token
36+
"default": true // Set as default chat model
37+
},
38+
39+
// Example of local model configuration
40+
{
41+
"model": "llama2",
42+
"endpoint": "http://localhost:11434/v1"
43+
}
44+
],
45+
46+
// Embedding model configuration (optional)
47+
"embeddings": [{
48+
"model": "text-embedding-ada-002",
49+
"endpoint": "https://api.openai.com/v1",
50+
"token": "your-openai-token", // Replace with your OpenAI token
51+
"default": true // Set as default embedding
52+
}]
53+
}`;
54+
55+
export class ConfigLoader {
56+
readonly logger = new Logger(ConfigLoader.name);
57+
private type: string;
58+
private static instances: Map<ConfigType, ConfigLoader> = new Map();
59+
private static config: AppConfig;
1560
private readonly configPath: string;
1661

17-
constructor() {
18-
this.configPath = getConfigPath('config');
62+
private constructor(type: ConfigType) {
63+
this.type = type;
64+
this.configPath = getConfigPath();
65+
this.initConfigFile();
66+
this.loadConfig();
67+
}
68+
69+
public static getInstance(type: ConfigType): ConfigLoader {
70+
if (!ConfigLoader.instances.has(type)) {
71+
ConfigLoader.instances.set(type, new ConfigLoader(type));
72+
}
73+
return ConfigLoader.instances.get(type)!;
74+
}
75+
76+
public initConfigFile(): void {
77+
Logger.log('Creating example config file', 'ConfigLoader');
78+
79+
const config = getConfigPath();
80+
if (fs.existsSync(config)) {
81+
return;
82+
}
83+
84+
if (!fs.existsSync(config)) {
85+
//make file
86+
fs.writeFileSync(config, exampleConfigContent, 'utf-8');
87+
}
88+
Logger.log('Creating example config file', 'ConfigLoader');
89+
}
90+
91+
public reload(): void {
1992
this.loadConfig();
2093
}
2194

2295
private loadConfig() {
23-
const file = fs.readFileSync(this.configPath, 'utf-8');
96+
try {
97+
Logger.log(
98+
`Loading configuration from ${this.configPath}`,
99+
'ConfigLoader',
100+
);
101+
const file = fs.readFileSync(this.configPath, 'utf-8');
102+
const jsonContent = file.replace(
103+
/\\"|"(?:\\"|[^"])*"|(\/\/.*|\/\*[\s\S]*?\*\/)/g,
104+
(m, g) => (g ? '' : m),
105+
);
106+
ConfigLoader.config = JSON.parse(jsonContent);
107+
this.validateConfig();
108+
} catch (error) {
109+
if (
110+
error.code === 'ENOENT' ||
111+
error.message.includes('Unexpected end of JSON input')
112+
) {
113+
ConfigLoader.config = {};
114+
this.saveConfig();
115+
} else {
116+
throw error;
117+
}
118+
}
119+
120+
this.logger.log(ConfigLoader.config);
24121

25-
this.chatsConfig = JSON.parse(file);
26122
}
27123

28-
get<T>(path: string) {
124+
get<T>(path?: string): T {
29125
if (!path) {
30-
return this.chatsConfig as unknown as T;
126+
return ConfigLoader.config as unknown as T;
31127
}
32-
return _.get(this.chatsConfig, path) as T;
128+
return _.get(ConfigLoader.config, path) as T;
33129
}
34130

35131
set(path: string, value: any) {
36-
_.set(this.chatsConfig, path, value);
132+
_.set(ConfigLoader.config, path, value);
37133
this.saveConfig();
38134
}
39135

40136
private saveConfig() {
137+
const configDir = path.dirname(this.configPath);
138+
if (!fs.existsSync(configDir)) {
139+
fs.mkdirSync(configDir, { recursive: true });
140+
}
41141
fs.writeFileSync(
42142
this.configPath,
43-
JSON.stringify(this.chatsConfig, null, 4),
143+
JSON.stringify(ConfigLoader.config, null, 2),
44144
'utf-8',
45145
);
46146
}
47147

148+
addConfig(config: ModelConfig | EmbeddingConfig) {
149+
if (!ConfigLoader.config[this.type]) {
150+
ConfigLoader.config[this.type] = [];
151+
}
152+
this.logger.log(ConfigLoader.config);
153+
const index = ConfigLoader.config[this.type].findIndex(
154+
(chat) => chat.model === config.model,
155+
);
156+
if (index !== -1) {
157+
ConfigLoader.config[this.type].splice(index, 1);
158+
}
159+
160+
if (config.default) {
161+
ConfigLoader.config.models.forEach((chat) => {
162+
chat.default = false;
163+
});
164+
}
165+
166+
ConfigLoader.config[this.type].push(config);
167+
this.saveConfig();
168+
}
169+
170+
removeConfig(modelName: string): boolean {
171+
if (!ConfigLoader.config[this.type]) {
172+
return false;
173+
}
174+
175+
const initialLength = ConfigLoader.config[this.type].length;
176+
ConfigLoader.config.models = ConfigLoader.config[this.type].filter(
177+
(chat) => chat.model !== modelName,
178+
);
179+
180+
if (ConfigLoader.config[this.type].length !== initialLength) {
181+
this.saveConfig();
182+
return true;
183+
}
184+
185+
return false;
186+
}
187+
188+
getAllConfigs(): EmbeddingConfig[] | ModelConfig[] | null {
189+
const res = ConfigLoader.config[this.type];
190+
return Array.isArray(res) ? res : null;
191+
}
192+
48193
validateConfig() {
49-
if (!this.chatsConfig) {
50-
throw new Error("Invalid configuration: 'chats' section is missing.");
194+
if (!ConfigLoader.config) {
195+
ConfigLoader.config = {};
196+
}
197+
198+
if (typeof ConfigLoader.config !== 'object') {
199+
throw new Error('Invalid configuration: Must be an object');
200+
}
201+
202+
if (ConfigLoader.config.models) {
203+
if (!Array.isArray(ConfigLoader.config.models)) {
204+
throw new Error("Invalid configuration: 'chats' must be an array");
205+
}
206+
207+
ConfigLoader.config.models.forEach((chat, index) => {
208+
if (!chat.model) {
209+
throw new Error(
210+
`Invalid chat configuration at index ${index}: 'model' is required`,
211+
);
212+
}
213+
});
214+
215+
const defaultChats = ConfigLoader.config.models.filter(
216+
(chat) => chat.default,
217+
);
218+
if (defaultChats.length > 1) {
219+
throw new Error(
220+
'Invalid configuration: Multiple default chat configurations found',
221+
);
222+
}
51223
}
224+
225+
if (ConfigLoader.config[ConfigType.EMBEDDINGS]) {
226+
this.logger.log(ConfigLoader.config[ConfigType.EMBEDDINGS]);
227+
if (!Array.isArray(ConfigLoader.config[ConfigType.EMBEDDINGS])) {
228+
throw new Error("Invalid configuration: 'embeddings' must be an array");
229+
}
230+
231+
ConfigLoader.config.models.forEach((emb, index) => {
232+
if (!emb.model) {
233+
throw new Error(
234+
`Invalid chat configuration at index ${index}: 'model' is required`,
235+
);
236+
}
237+
});
238+
239+
const defaultChats = ConfigLoader.config[ConfigType.EMBEDDINGS].filter(
240+
(chat) => chat.default,
241+
);
242+
if (defaultChats.length > 1) {
243+
throw new Error(
244+
'Invalid configuration: Multiple default emb configurations found',
245+
);
246+
}
247+
}
248+
}
249+
250+
getConfig(): AppConfig {
251+
return ConfigLoader.config;
52252
}
53253
}

0 commit comments

Comments
 (0)