Skip to content

Commit c5ec4b6

Browse files
caohuilincaohuilin
committed
feat: revet download function
1 parent e6798d7 commit c5ec4b6

File tree

8 files changed

+163
-899
lines changed

8 files changed

+163
-899
lines changed

‎packages/api/app/src/index.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ export class AppAPI {
5555
}
5656

5757
public async checkEnvironment(nodeVersion?: string) {
58-
if (semver.lt(process.versions.node, nodeVersion || '16.20.2')) {
58+
if (semver.lt(process.versions.node, nodeVersion || '14.18.3')) {
5959
this.generatorCore.logger.warn(
6060
`🟡 ${i18n.t(localeKeys.environment.node_version)}`,
6161
);

‎packages/core/package.json‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
"@swc/helpers": "0.5.1",
3333
"axios": "^1.6.0",
3434
"debug": "4.3.7",
35-
"pacote": "^18.0.6",
35+
"tar": "^6.1.13",
3636
"@modern-js/codesmith-utils": "workspace:*"
3737
},
3838
"devDependencies": {
@@ -42,7 +42,6 @@
4242
"@types/node": "^14.18.42",
4343
"@types/tar": "^4.0.5",
4444
"@types/debug": "^4.1.12",
45-
"@types/pacote": "^11.1.8",
4645
"typescript": "^4.9.5"
4746
},
4847
"sideEffects": false

‎packages/core/src/generator/index.ts‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -281,8 +281,19 @@ check path: ${chalk.blue.underline(
281281
this.setbasePath(preBasePath);
282282
this.logger?.timing?.(`🕒 RunSubGenerator ${subGenerator}`, true);
283283
}
284-
285284
public async prepareGenerators(generators: string[]) {
285+
if ((global as any).CODESMITH_PREPARE_GLOBAL) {
286+
return;
287+
}
286288
await this.materialsManager.prepareGenerators(generators);
289+
(global as any).CODESMITH_PREPARE_GLOBAL = true;
290+
}
291+
292+
public async prepareGlobal() {
293+
if ((global as any).CODESMITH_PREPARE_GLOBAL) {
294+
return;
295+
}
296+
await this.materialsManager.prepareGlobal();
297+
(global as any).CODESMITH_PREPARE_GLOBAL = true;
287298
}
288299
}

‎packages/core/src/utils/downloadPackage.ts‎

Lines changed: 96 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -4,45 +4,13 @@ import type { Logger } from '@/logger';
44
import type { ILogger } from '@/logger/constants';
55
import { fs } from '@modern-js/codesmith-utils/fs-extra';
66
import { semver } from '@modern-js/codesmith-utils/semver';
7-
import pacote from 'pacote';
7+
import axios from 'axios';
8+
import tar from 'tar';
9+
import { getNpmTarballUrl } from './getNpmTarballUrl';
810
import { fsExists } from './fsExists';
911
import { getNpmVersion } from './getNpmVersion';
1012
import { runInstall } from './packageManager';
1113

12-
async function isValidCache(cacheDir: string) {
13-
/* generator cache can use
14-
* 1. .codesmith.completed exist
15-
* 2. cache time is within the validity period
16-
*/
17-
if (await fsExists(`${cacheDir}/.codesmith.completed`)) {
18-
const preCacheTimeStr = await fs.readFile(
19-
`${cacheDir}/.codesmith.completed`,
20-
{
21-
encoding: 'utf-8',
22-
},
23-
);
24-
const preCacheTime = preCacheTimeStr
25-
? new Date(preCacheTimeStr)
26-
: new Date(0);
27-
if (Number(new Date()) - Number(preCacheTime) < CATCHE_VALIDITY_PREIOD) {
28-
return true;
29-
}
30-
return false;
31-
}
32-
return false;
33-
}
34-
35-
async function downloadAndDecompressTargz(
36-
packageName: string,
37-
version: string,
38-
targetDir: string,
39-
registryUrl?: string,
40-
): Promise<void> {
41-
await pacote.extract(`${packageName}@${version}`, targetDir, {
42-
registry: registryUrl,
43-
});
44-
}
45-
4614
const GeneratorVersionMap = new Map<string, string>();
4715

4816
export async function getGeneratorVersion(
@@ -89,6 +57,76 @@ export async function getGeneratorVersion(
8957
return version;
9058
}
9159

60+
async function isValidCache(cacheDir: string) {
61+
/* generator cache can use
62+
* 1. .codesmith.completed exist
63+
* 2. cache time is within the validity period
64+
*/
65+
if (await fsExists(`${cacheDir}/.codesmith.completed`)) {
66+
const preCacheTimeStr = await fs.readFile(
67+
`${cacheDir}/.codesmith.completed`,
68+
{
69+
encoding: 'utf-8',
70+
},
71+
);
72+
const preCacheTime = preCacheTimeStr
73+
? new Date(preCacheTimeStr)
74+
: new Date(0);
75+
if (Number(new Date()) - Number(preCacheTime) < CATCHE_VALIDITY_PREIOD) {
76+
return true;
77+
}
78+
return false;
79+
}
80+
return false;
81+
}
82+
83+
async function downloadAndDecompressTargz(
84+
tarballPkg: string,
85+
targetDir: string,
86+
) {
87+
const response = await axios({
88+
method: 'get',
89+
url: tarballPkg,
90+
responseType: 'stream',
91+
adapter: 'http',
92+
});
93+
if (response.status !== 200) {
94+
throw new Error(
95+
`download tar package get bad status code: ${response.status}`,
96+
);
97+
}
98+
// create tmp file
99+
const randomId = Math.floor(Math.random() * 10000);
100+
const tempTgzFilePath = `${os.tmpdir()}/temp-${randomId}.tgz`;
101+
102+
const dest = fs.createWriteStream(tempTgzFilePath);
103+
104+
await new Promise<void>((resolve, reject) => {
105+
response.data.pipe(dest);
106+
response.data.on('error', (err: any) => {
107+
reject(err);
108+
});
109+
dest.on('finish', () => {
110+
resolve();
111+
});
112+
});
113+
await new Promise<void>((resolve, reject) => {
114+
fs.createReadStream(tempTgzFilePath)
115+
.pipe(
116+
tar.x({
117+
strip: 1,
118+
C: `${targetDir}`,
119+
}),
120+
)
121+
.on('finish', () => {
122+
resolve();
123+
})
124+
.on('error', (err: any) => {
125+
reject(err);
126+
});
127+
});
128+
}
129+
92130
/**
93131
* download npm package
94132
* @param {string} pkgName
@@ -106,10 +144,21 @@ export async function downloadPackage(
106144
} = {},
107145
) {
108146
const { registryUrl, install, logger } = options;
109-
const version = await getGeneratorVersion(pkgName, pkgVersion, {
110-
registryUrl,
111-
logger,
112-
});
147+
let version: string | undefined;
148+
if (!semver.valid(pkgVersion)) {
149+
// get pkgName version
150+
logger?.timing(`🕒 get ${pkgName} version`);
151+
version = await getNpmVersion(pkgName, {
152+
registryUrl,
153+
version: pkgVersion,
154+
});
155+
logger?.timing(`🕒 get ${pkgName} version`, true);
156+
if (version === undefined) {
157+
throw new Error(`package ${pkgName}@${pkgVersion} not found in registry`);
158+
}
159+
} else {
160+
version = pkgVersion;
161+
}
113162
const targetDir = `${os.tmpdir()}/csmith-generator/${pkgName}@${version}`;
114163
logger?.debug?.(
115164
`💡 [Download Generator Package]: ${pkgName}@${version} to ${targetDir}`,
@@ -120,9 +169,16 @@ export async function downloadPackage(
120169
await fs.remove(targetDir);
121170
await fs.mkdirp(targetDir);
122171

172+
logger?.timing(`🕒 get ${pkgName}@${version} tarball url`);
173+
// get package tarball
174+
const tarballPkg = await getNpmTarballUrl(pkgName, version, {
175+
registryUrl,
176+
});
177+
logger?.timing(`🕒 get ${pkgName}@${version} tarball url`, true);
178+
123179
logger?.timing(`🕒 download ${pkgName}@${version} tarball`);
124180
// download tarball and compress it to target directory
125-
await downloadAndDecompressTargz(pkgName, version, targetDir, registryUrl);
181+
await downloadAndDecompressTargz(tarballPkg, targetDir);
126182
logger?.timing(`🕒 download ${pkgName}@${version} tarball`, true);
127183

128184
if (install) {

‎packages/core/src/utils/getNpmTarballUrl.ts‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
import { NPM_API_TIMEOUT } from '@/constants';
2+
import { execa } from '@modern-js/codesmith-utils/execa';
3+
import { timeoutPromise } from './timeoutPromise';
4+
5+
interface Options {
6+
registryUrl?: string;
7+
}
8+
9+
export async function getNpmTarballUrl(
10+
pkgName: string,
11+
pkgVersion: string,
12+
options?: Options,
13+
): Promise<string> {
14+
const { registryUrl } = options || {};
15+
const params = ['view', `${pkgName}@${pkgVersion}`, 'dist', '--json'];
16+
17+
if (registryUrl) {
18+
params.push('--registry');
19+
params.push(registryUrl);
20+
}
21+
22+
const getPkgInfoPromise = execa('npm', params);
23+
const { stdout } = await timeoutPromise(
24+
getPkgInfoPromise,
25+
NPM_API_TIMEOUT,
26+
`Get npm tarball of '${pkgName}'`,
27+
);
28+
29+
try {
30+
const pkgDistInfo = JSON.parse(stdout);
31+
return pkgDistInfo.tarball;
32+
} catch (e) {
33+
throw new Error(
34+
`Version \`${pkgVersion}\` for package \`${pkgName}\` could not be found`,
35+
);
36+
}
37+
}

‎packages/global/modern.config.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,10 @@ import { defineConfig, moduleTools } from '@modern-js/module-tools';
22

33
export default defineConfig({
44
buildConfig: {
5+
target: 'es2020',
56
dts: false,
67
format: 'umd',
78
autoExternal: false,
8-
externals: ['bluebird', '@sigstore/core', '@sigstore/verify'],
99
},
1010
plugins: [moduleTools()],
1111
});

‎packages/global/package.json‎

Lines changed: 1 addition & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -13,84 +13,7 @@
1313
],
1414
"version": "2.5.1",
1515
"jsnext:source": "./src/index.ts",
16-
"types": "./dist/types/index.d.ts",
17-
"main": "./dist/cjs/index.js",
18-
"module": "./dist/esm-node/index.js",
19-
"exports": {
20-
".": {
21-
"jsnext:source": "./src/index.ts",
22-
"types": "./dist/types/index.d.ts",
23-
"default": "./dist/cjs/index.js"
24-
},
25-
"./chalk": {
26-
"jsnext:source": "./src/chalk.ts",
27-
"import": "./dist/esm/chalk.js",
28-
"default": "./dist/cjs/chalk.js"
29-
},
30-
"./execa": {
31-
"jsnext:source": "./src/execa.ts",
32-
"import": "./dist/esm/execa.js",
33-
"default": "./dist/cjs/execa.js"
34-
},
35-
"./fs-extra": {
36-
"jsnext:source": "./src/fs-extra.ts",
37-
"import": "./dist/esm/fs-extra.js",
38-
"default": "./dist/cjs/fs-extra.js"
39-
},
40-
"./glob": {
41-
"jsnext:source": "./src/glob.ts",
42-
"import": "./dist/esm/glob.js",
43-
"default": "./dist/cjs/glob.js"
44-
},
45-
"./lodash": {
46-
"jsnext:source": "./src/lodash.ts",
47-
"import": "./dist/esm/lodash.js",
48-
"default": "./dist/cjs/lodash.js"
49-
},
50-
"./ora": {
51-
"jsnext:source": "./src/ora.ts",
52-
"import": "./dist/esm/ora.js",
53-
"default": "./dist/cjs/ora.js"
54-
},
55-
"./semver": {
56-
"jsnext:source": "./src/semver.ts",
57-
"import": "./dist/esm/semver.js",
58-
"default": "./dist/cjs/semver.js"
59-
},
60-
"./npm": {
61-
"jsnext:source": "./src/npm.ts",
62-
"import": "./dist/esm/npm.js",
63-
"default": "./dist/cjs/npm.js"
64-
}
65-
},
66-
"typesVersions": {
67-
"*": {
68-
"chalk": [
69-
"./dist/types/chalk.d.ts"
70-
],
71-
"execa": [
72-
"./dist/types/execa.d.ts"
73-
],
74-
"fs-extra": [
75-
"./dist/types/fs-extra.d.ts"
76-
],
77-
"glob": [
78-
"./dist/types/glob.d.ts"
79-
],
80-
"lodash": [
81-
"./dist/types/lodash.d.ts"
82-
],
83-
"ora": [
84-
"./dist/types/ora.d.ts"
85-
],
86-
"semver": [
87-
"./dist/types/semver.d.ts"
88-
],
89-
"npm": [
90-
"./dist/types/npm.d.ts"
91-
]
92-
}
93-
},
16+
"main": "./dist/index.js",
9417
"publishConfig": {
9518
"registry": "https://registry.npmjs.org/",
9619
"access": "public",

0 commit comments

Comments
 (0)