Merge pull request #2362 from zowe/download-am-special-char-issue

[v2] Download special char issue

packages/imperative/src/io/__tests__/IO.unit.test.ts

@@ -276,6 +276,22 @@ describe("IO tests", () => {
         expect(processed).toBe(original.replace(/\n/g, "\r\n").slice(1));
     });
 
+    it("processNewlines should replace LF line endings with CRLF on Windows when input is a Buffer", () => {
+        jest.spyOn(os, "platform").mockReturnValueOnce(IO.OS_WIN32);
+        const originalBuffer = Buffer.from("\nabc\ndef\n");
+        const processedBuffer = IO.processNewlines(originalBuffer);
+        const expectedBuffer = Buffer.from("\r\nabc\r\ndef\r\n");
+        expect(processedBuffer.equals(expectedBuffer)).toBe(true);
+    });
+
+    it("processNewlines should not replace LF line ending when last byte is CR and input is a Buffer", () => {
+        jest.spyOn(os, "platform").mockReturnValueOnce(IO.OS_WIN32);
+        const originalBuffer = Buffer.from("\nabc\ndef\n");
+        const processedBuffer = IO.processNewlines(originalBuffer, "\r".charCodeAt(0));
+        const expectedBuffer = Buffer.from("\nabc\r\ndef\r\n");
+        expect(processedBuffer.equals(expectedBuffer)).toBe(true);
+    });
+
     it("should get an error for no input on mkdirp", () => {
         let error;
         try {

packages/imperative/src/io/src/IO.ts

@@ -264,17 +264,38 @@ export class IO {
      * @returns {string} - input with removed newlines
      * @memberof IO
      */
-    public static processNewlines(original: string, lastByte?: number): string {
+    public static processNewlines<T extends string | Buffer>(original: T, lastByte?: number): T {
         ImperativeExpect.toNotBeNullOrUndefined(original, "Required parameter 'original' must not be null or undefined");
+
         if (os.platform() !== IO.OS_WIN32) {
             return original;
         }
-        // otherwise, we're on windows
-        const processed = original.replace(/([^\r])\n/g, "$1\r\n");
-        if ((lastByte == null || lastByte !== "\r".charCodeAt(0)) && original.startsWith("\n")) {
-            return "\r" + processed;
+
+        let processed: T;
+
+        if (typeof original === "string") {
+            processed = original.replace(/([^\r])\n/g, "$1\r\n") as T;
+            if ((lastByte == null || lastByte !== "\r".charCodeAt(0)) && original.startsWith("\n")) {
+                return ("\r" + processed) as T;
+            }
+            return processed;
+        } else {
+            const bufferList: number[] = [];
+            let prevByte = lastByte;
+            for (let i = 0; i < original.length; i++) {
+                const currentByte = original[i];
+                //Check if previous byte is not Carriage Return (13) and if current byte is Line Feed (10)
+                // eslint-disable-next-line @typescript-eslint/no-magic-numbers
+                if (currentByte === 10 && prevByte !== 13) {
+                    // eslint-disable-next-line @typescript-eslint/no-magic-numbers
+                    bufferList.push(13);
+                }
+                bufferList.push(currentByte);
+                prevByte = currentByte;
+            }
+            processed = Buffer.from(bufferList) as T;
+            return processed;
         }
-        return processed;
     }
 
     /**

packages/imperative/src/rest/src/client/AbstractRestClient.ts

@@ -766,7 +766,7 @@ export abstract class AbstractRestClient {
             this.log.debug("Streaming data chunk of length " + respData.length + " to response stream");
             if (this.mNormalizeResponseNewlines && this.mContentEncoding == null) {
                 this.log.debug("Normalizing new lines in data chunk to operating system appropriate line endings");
-                respData = Buffer.from(IO.processNewlines(respData.toString(), this.lastByteReceived));
+                respData = IO.processNewlines(respData, this.lastByteReceived);
             }
             if (this.mTask != null) {
                 // update the progress task if provided by the requester

packages/imperative/src/rest/src/client/CompressionUtils.ts

@@ -102,7 +102,7 @@ export class CompressionUtils {
         let lastByteReceived: number = 0;
         return new Transform({
             transform(chunk, _, callback) {
-                this.push(Buffer.from(IO.processNewlines(chunk.toString(), lastByteReceived)));
+                this.push(IO.processNewlines(chunk, lastByteReceived));
                 lastByteReceived = chunk[chunk.byteLength - 1];
                 callback();
             }

packages/zosfiles/CHANGELOG.md

@@ -2,6 +2,10 @@
 
 All notable changes to the Zowe z/OS files SDK package will be documented in this file.
 
+## Recent Changes
+
+- BugFix: Resolved issue where special characters could be corrupted when downloading a large file. [#2362](https://github.com/zowe/zowe-cli/pull/2362)
+
 ## `7.29.5`
 
 - BugFix: Added support for the `--encoding` flag to the `zowe upload dir-to-uss` to allow for encoding uploaded directories for command group consistency. [#2356](https://github.com/zowe/zowe-cli/pull/2356)

packages/zosfiles/__tests__/__system__/methods/download/Download.system.test.ts

@@ -44,6 +44,9 @@ const rimraf = require("rimraf").sync;
 const delayTime = 2000;
 const testData = "abcdefghijklmnopqrstuvwxyz";
 
+// File must be larger than 8KB such that it breaks into multiple chunks
+const specialCharTestData = fs.readFileSync(__dirname+"/testfiles/specialCharTestData.txt");
+
 let REAL_SESSION: Session;
 let testEnvironment: ITestEnvironment<ITestPropertiesSchema>;
 let defaultSystem: ITestPropertiesSchema;
@@ -574,6 +577,40 @@ describe("Download Data Set", () => {
                 const regex = /\./gi;
                 file = dsname.replace(regex, "/") + "/member.dat";
             });
+            it("should download a data set member full of special characters to test buffer chunk concatenation", async () => {
+                let error;
+                let response: IZosFilesResponse;
+
+                // upload data to the newly created data set
+                await Upload.bufferToDataSet(REAL_SESSION, Buffer.from(specialCharTestData), dsname + "(member)");
+                await delay(delayTime);
+
+                try {
+                    response = await Download.allMembers(REAL_SESSION, dsname);
+                    Imperative.console.info("Response: " + inspect(response));
+                } catch (err) {
+                    error = err;
+                    Imperative.console.info("Error: " + inspect(error));
+                }
+                expect(error).toBeFalsy();
+                expect(response).toBeTruthy();
+                expect(response.success).toBeTruthy();
+                expect(response.commandResponse).toContain(
+                    ZosFilesMessages.datasetDownloadedSuccessfully.message.substring(0, "Data set downloaded successfully".length + 1));
+
+                // Convert the data set name to use as a path/file
+                const regex = /\./gi;
+                file = dsname.replace(regex, "/");
+                file = file.toLowerCase();
+
+                // Compare the downloaded contents to those uploaded
+                const fileContents = stripNewLines(fs.readFileSync(`${file}/member.txt`).toString());
+                const uniqueFileChars = Array.from(new Set(fileContents)).join('');
+                const expectedUniqueChars = "àèéìòùÀÈÉÌÒÙ° "; // Expected unique characters
+
+                // Ensure the file only contains the unique characters and nothing else
+                expect(uniqueFileChars).toEqual(expectedUniqueChars);
+            });
         });
 
         describe("Data sets matching - all data sets - PO", () => {