Skip to content

Commit 9c8c8b4

Browse files
riskycaseriskycase
committed
Merge branch 'busboy' into trunk
Signed-off-by: Hrishikesh Patil <[email protected]>
2 parents 58056eb + ec6d35e commit 9c8c8b4

19 files changed

+610
-480
lines changed

‎CHANGELOG‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,10 @@
1+
v3.1.0
2+
3+
Add a new /upload/zip route to accept zip files which will be extracted on upload
4+
Change upload handler from multer to busboy
5+
Update UIKit from 3.5.8 to 3.5.9
6+
Update mocha from 8.1.3 to 8.2.0
7+
18
v3.0.0
29

310
Save files with a numbered suffix if already exists in destination folder

‎dummy/dummy-folder/dummy-small.txt‎ renamed to ‎dummy/dummy-folder/dummy-sub/dummy-small.txt‎

File renamed without changes.

‎dummy/dummy-zip.zip‎

4.36 KB
Binary file not shown.

‎middleware/fileManager.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@ module.exports.updateReceivedFiles = function (uploadedFiles) {
167167
receivedFiles.push({
168168
path: value.path,
169169
size: value.size,
170-
folder: false,
170+
folder: value.folder,
171171
});
172172
}
173173
});

‎middleware/uploadManager.js‎

Lines changed: 124 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,133 @@
1-
const multer = require('multer');
1+
const busboy = require('busboy');
2+
const unzip = require('unzip-stream');
23
const fs = require('fs');
34
const path = require('path');
45

56
const fileManager = require('./fileManager');
67

7-
// Sets destination for Multer to save files to, and preserve the original name
8-
const storage = multer.diskStorage({
9-
destination: function (req, file, cb) {
10-
cb(null, fileManager.destination());
11-
},
12-
filename: function (req, file, cb) {
13-
const dest = fileManager.destination();
14-
let number = 0;
15-
let name;
16-
do {
17-
name = `${path.basename(
18-
file.originalname,
19-
path.extname(file.originalname)
20-
)}${number-- || ''}${path.extname(file.originalname)}`;
21-
} while (fs.existsSync(path.resolve(dest, name)));
22-
cb(null, name);
23-
},
24-
});
8+
/*
9+
* Returns a unique name for the file to prevent overwriting
10+
*
11+
* Takes parent directory and name of file/directory, returns a resolved path
12+
*/
13+
function uniqueName(name, folder) {
14+
const parent = fileManager.destination();
15+
let number = 0;
16+
let filename;
17+
do
18+
filename = `${path.basename(name, path.extname(name))}${
19+
number-- || ''
20+
}${folder ? '' : path.extname(name)}`;
21+
while (fs.existsSync(path.resolve(parent, filename)));
22+
return {
23+
path: path.resolve(parent, filename),
24+
originalname: name,
25+
filename: filename,
26+
size: -1,
27+
folder: folder,
28+
};
29+
}
2530

26-
// Creates an instance of Multer with the set options
27-
const upload = multer({ storage: storage });
31+
/*
32+
* Extracts a zip stream to destination directory and returns final size
33+
*
34+
* Takes parent directory and zip stream, returns a promise that resolves with
35+
* the size of the extracted folder
36+
*/
37+
function fileWriter(filePath, fileStream) {
38+
return new Promise((resolve, reject) => {
39+
fs.mkdirSync(path.dirname(filePath), {
40+
recursive: true,
41+
});
42+
const destStream = fs.createWriteStream(filePath).on('close', () => {
43+
resolve(destStream.bytesWritten);
44+
});
45+
fileStream.pipe(destStream);
46+
});
47+
}
2848

29-
// Exposes the created Multer instance to the router
30-
module.exports.saveFiles = upload.array('files[]');
49+
/*
50+
* Extracts a zip stream to destination directory and returns final size
51+
*
52+
* Takes parent directory and zip stream, returns a promise that resolves with
53+
* the size of the extracted folder
54+
*/
55+
function extracter(destination, zipstream) {
56+
return new Promise((resolve, reject) => {
57+
let size = 0;
58+
let entries = 0;
59+
const unzipper = unzip
60+
.Parse()
61+
.on('error', reject)
62+
.on('entry', entry => {
63+
entries++;
64+
if (entry.type === 'Directory') {
65+
entries--;
66+
fs.mkdirSync(path.resolve(destination, entry.path), {
67+
recursive: true,
68+
});
69+
} else {
70+
fileWriter(
71+
path.resolve(destination, entry.path),
72+
entry
73+
).then(bytesWritten => {
74+
size += bytesWritten;
75+
if (--entries === 0) resolve(size);
76+
});
77+
}
78+
});
79+
zipstream.pipe(unzipper);
80+
});
81+
}
82+
83+
// Use busboy to save incoming files to disk
84+
module.exports.saveFiles = function (req, res, next) {
85+
if (req.method !== 'POST' || !req.headers['content-type']) next();
86+
else {
87+
let writer = new busboy({ headers: req.headers });
88+
writer.on('file', (fieldname, file, filename, encoding, mimetype) => {
89+
req.files = req.files || [];
90+
let fileObject = uniqueName(filename, false);
91+
req.files.push(fileObject);
92+
let writeStream = fs.createWriteStream(fileObject.path);
93+
writeStream.on('close', () => {
94+
req.files.find(file => file.path === fileObject.path).size =
95+
writeStream.bytesWritten;
96+
if (!req.files.filter(file => file.size === -1).length) next();
97+
});
98+
file.pipe(writeStream);
99+
});
100+
writer.on('finish', () => {
101+
if (!req.files) next();
102+
});
103+
req.pipe(writer);
104+
}
105+
};
106+
107+
// Use busboy to pipe zip files to an unzipper and save as a folder
108+
module.exports.saveZips = function (req, res, next) {
109+
if (req.method !== 'POST' || !req.headers['content-type']) next();
110+
else {
111+
let writer = new busboy({ headers: req.headers });
112+
writer.on('file', (fieldname, file, filename, encoding, mimetype) => {
113+
if (mimetype === 'application/zip') {
114+
req.files = req.files || [];
115+
let folderObject = uniqueName(filename, true);
116+
req.files.push(folderObject);
117+
extracter(folderObject.path, file).then(size => {
118+
req.files.find(
119+
file => file.path === folderObject.path
120+
).size = size;
121+
if (!req.files.filter(file => file.size === -1).length)
122+
next();
123+
});
124+
} else file.resume();
125+
});
126+
writer.on('finish', () => {
127+
if (!req.files) next();
128+
});
129+
req.pipe(writer);
130+
}
131+
};
31132

32133
module.exports.updateReceivedFiles = fileManager.updateReceivedFiles;

‎package.json‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "broadcastem-core",
3-
"version": "3.0.0",
3+
"version": "3.1.0",
44
"description": "The core module for Broadcast 'em application",
55
"keywords": [
66
"broadcast",
@@ -24,19 +24,20 @@
2424
"license": "MIT",
2525
"homepage": "https://gitlab.com/riskycase/broadcastem-core/-/wikis/home",
2626
"dependencies": {
27+
"busboy": "^0.3.1",
2728
"express": "^4.17.1",
2829
"express-easy-zip": "^1.1.5",
2930
"morgan": "^1.10.0",
30-
"multer": "^1.4.2",
31-
"uikit": "^3.5.8",
31+
"uikit": "^3.5.9",
32+
"unzip-stream": "^0.3.1",
3233
"yargs": "^15.4.1"
3334
},
3435
"devDependencies": {
3536
"chai": "^4.2.0",
3637
"chai-http": "^4.3.0",
3738
"dev-null": "^0.1.1",
3839
"husky": "^4.3.0",
39-
"mocha": "^8.1.3",
40+
"mocha": "^8.2.0",
4041
"mocha-junit-reporter": "^2.0.0",
4142
"prettier": "^2.1.2"
4243
}

‎pages/root.html‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@
6262
<li>
6363
<div
6464
class="js-upload uk-placeholder uk-text-center"
65-
id="uploadArea"
65+
id="fileUploadArea"
6666
>
6767
<span uk-icon="icon: cloud-upload"></span
6868
><span class="uk-text-middle"
@@ -74,6 +74,22 @@
7474
>
7575
</div>
7676
</div>
77+
<div
78+
class="js-upload uk-placeholder uk-text-center"
79+
id="zipUploadArea"
80+
>
81+
<span uk-icon="icon: folder"></span
82+
><span class="uk-text-middle"
83+
>Send zip files by dropping them here or
84+
</span>
85+
<div uk-form-custom="">
86+
<input
87+
type="file"
88+
multiple=""
89+
accept="application/zip"
90+
/><span class="uk-link">selecting them</span>
91+
</div>
92+
</div>
7793
<progress
7894
class="uk-progress"
7995
id="js-progressbar"

‎public/script/root.js‎

Lines changed: 30 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -100,23 +100,37 @@ function displayFiles(files) {
100100
).innerHTML = `<div class="uk-alert-primary uk-align-center uk-text-center uk-width-1-1" uk-alert>No files selected to share</div>`;
101101
}
102102

103-
// UIkit js to control file upload
104103
const bar = document.getElementById('js-progressbar');
105-
UIkit.upload('.js-upload', {
106-
url: '/upload',
107-
multiple: true,
108-
type: 'multipart/form-data',
109-
loadStart: setBarValue,
110-
progress: setBarValue,
111-
loadEnd: setBarValue,
112-
completeAll: function () {
113-
setTimeout(function () {
114-
bar.setAttribute('hidden', 'hidden');
115-
}, 1000);
116-
alert('Files uploaded, will be available for sharing now');
117-
loadFiles();
118-
},
119-
});
104+
/*
105+
* Generates an upload object for normal files and zip files
106+
*/
107+
function uploadObject(zip) {
108+
return {
109+
url: zip ? '/upload' : '/upload/zip',
110+
multiple: true,
111+
type: 'multipart/form-data',
112+
loadStart: setBarValue,
113+
progress: setBarValue,
114+
loadEnd: setBarValue,
115+
completeAll: function () {
116+
setTimeout(function () {
117+
bar.setAttribute('hidden', 'hidden');
118+
}, 1000);
119+
alert(
120+
`${
121+
zip ? 'Files uploaded' : 'Zips uploaded and extracted'
122+
}, will be available for sharing now`
123+
);
124+
loadFiles();
125+
},
126+
};
127+
}
128+
129+
// UIkit js to control file upload
130+
UIkit.upload('#fileUploadArea', uploadObject(false));
131+
132+
// UIkit js to control zip upload
133+
UIkit.upload('#zipUploadArea', uploadObject(true));
120134

121135
/*
122136
* Sets the value for the progress bar

‎routes/upload.js‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,13 @@
11
const router = require('express').Router();
22
const uploadManager = require('../middleware/uploadManager');
33

4-
/* Receive incoming files */
5-
router.all('/', uploadManager.saveFiles, (req, res, next) => {
4+
/*
5+
* Runs appropriate action based on data received
6+
*
7+
* Takes request, response objects and next callback, returns callback result
8+
* on error
9+
*/
10+
function uploadResponder(req, res, next) {
611
if (req.method === 'POST') {
712
// Gets information of the files that were uploaded
813
const files = req.files;
@@ -26,6 +31,12 @@ router.all('/', uploadManager.saveFiles, (req, res, next) => {
2631
error.status = 400;
2732
return next(error);
2833
}
29-
});
34+
}
35+
36+
/* Receives incoming files */
37+
router.all('/', uploadManager.saveFiles, uploadResponder);
38+
39+
/* Receives incoming folders as zip and extracts them */
40+
router.all('/zip', uploadManager.saveZips, uploadResponder);
3041

3142
module.exports = router;

‎tests/.testStartup.js.swp‎

-12 KB
Binary file not shown.

0 commit comments

Comments
 (0)