diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index 2c37f1947c..6779cd3d06 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -1,6 +1,10 @@ # Change Log All notable changes to the Zowe CLI package will be documented in this file. +## Recent Changes + +-Enhancement: Added new command zowe zos-files download all-members-matching, (zowe files dl amm), to download members matching specified pattern(s). The success message for the Download.allMembers API was changed from originally "Data set downloaded successfully" to "Member(s) downloaded successfully." The change also alters the commandResponse when using the --rfj flag. [#2359](https://github.com/zowe/zowe-cli/pull/2359) + ## `8.8.0` - Enhancement: Pass a `.zosattributes` file path for the download encoding format by adding the new `--attributes` flag to the `zowe zos-files upload` command. [#2322](https://github.com/zowe/zowe-cli/issues/2322) diff --git a/packages/cli/__tests__/zosfiles/__system__/download/am/cli.files.download.am.system.test.ts b/packages/cli/__tests__/zosfiles/__system__/download/am/cli.files.download.am.system.test.ts index 6ea4e5eccb..967f60fe36 100644 --- a/packages/cli/__tests__/zosfiles/__system__/download/am/cli.files.download.am.system.test.ts +++ b/packages/cli/__tests__/zosfiles/__system__/download/am/cli.files.download.am.system.test.ts @@ -79,7 +79,6 @@ describe("Download All Member", () => { if (defaultSys.zosmf.basePath != null) { TEST_ENVIRONMENT_NO_PROF.env[ZOWE_OPT_BASE_PATH] = defaultSys.zosmf.basePath; } - const response = runCliScript(shellScript, TEST_ENVIRONMENT_NO_PROF, [dsname, @@ -89,7 +88,7 @@ describe("Download All Member", () => { defaultSys.zosmf.password]); expect(response.stderr.toString()).toBe(""); expect(response.status).toBe(0); - expect(response.stdout.toString()).toContain("Data set downloaded successfully."); + expect(response.stdout.toString()).toContain("Member(s) downloaded successfully."); }); }); @@ -109,7 +108,7 @@ describe("Download All Member", () => { const response = runCliScript(shellScript, TEST_ENVIRONMENT, [dsname]); expect(response.stderr.toString()).toBe(""); expect(response.status).toBe(0); - expect(response.stdout.toString()).toContain("Data set downloaded successfully."); + expect(response.stdout.toString()).toContain("Member(s) downloaded successfully."); }); it("should download all data set member of pds in binary format", () => { @@ -117,7 +116,7 @@ describe("Download All Member", () => { const response = runCliScript(shellScript, TEST_ENVIRONMENT, [dsname, "--binary"]); expect(response.stderr.toString()).toBe(""); expect(response.status).toBe(0); - expect(response.stdout.toString()).toContain("Data set downloaded successfully."); + expect(response.stdout.toString()).toContain("Member(s) downloaded successfully."); }); it("should download all data set member of pds in record format", () => { @@ -125,7 +124,7 @@ describe("Download All Member", () => { const response = runCliScript(shellScript, TEST_ENVIRONMENT, [dsname, "--record"]); expect(response.stderr.toString()).toBe(""); expect(response.status).toBe(0); - expect(response.stdout.toString()).toContain("Data set downloaded successfully."); + expect(response.stdout.toString()).toContain("Member(s) downloaded successfully."); }); it("should download all data set member of pds with response timeout", () => { @@ -133,7 +132,7 @@ describe("Download All Member", () => { const response = runCliScript(shellScript, TEST_ENVIRONMENT, [dsname, "--responseTimeout 5"]); expect(response.stderr.toString()).toBe(""); expect(response.status).toBe(0); - expect(response.stdout.toString()).toContain("Data set downloaded successfully."); + expect(response.stdout.toString()).toContain("Member(s) downloaded successfully."); }); it("should download all data set members with --max-concurrent-requests 2", () => { @@ -141,7 +140,7 @@ describe("Download All Member", () => { const response = runCliScript(shellScript, TEST_ENVIRONMENT, [dsname, 2]); expect(response.stderr.toString()).toBe(""); expect(response.status).toBe(0); - expect(response.stdout.toString()).toContain("Data set downloaded successfully."); + expect(response.stdout.toString()).toContain("Member(s) downloaded successfully."); }); it("should download all data set members of a large data set with --max-concurrent-requests 2", async () => { @@ -156,7 +155,7 @@ describe("Download All Member", () => { const response = runCliScript(shellScript, TEST_ENVIRONMENT, [bigDsname, 2]); expect(response.stderr.toString()).toBe(""); expect(response.status).toBe(0); - expect(response.stdout.toString()).toContain("Data set downloaded successfully."); + expect(response.stdout.toString()).toContain("Member(s) downloaded successfully."); await Delete.dataSet(REAL_SESSION, bigDsname); }); @@ -165,7 +164,7 @@ describe("Download All Member", () => { const response = runCliScript(shellScript, TEST_ENVIRONMENT, [dsname, "--rfj"]); expect(response.stderr.toString()).toBe(""); expect(response.status).toBe(0); - expect(response.stdout.toString()).toContain("Data set downloaded successfully."); + expect(response.stdout.toString()).toContain("Member(s) downloaded successfully."); }); it("should download all data set member to specified directory", () => { @@ -174,7 +173,7 @@ describe("Download All Member", () => { const response = runCliScript(shellScript, TEST_ENVIRONMENT, [dsname, `-d ${testDir}`, "--rfj"]); expect(response.stderr.toString()).toBe(""); expect(response.status).toBe(0); - expect(response.stdout.toString()).toContain("Data set downloaded successfully."); + expect(response.stdout.toString()).toContain("Member(s) downloaded successfully."); expect(response.stdout.toString()).toContain(testDir); }); @@ -186,7 +185,7 @@ describe("Download All Member", () => { const expectedResult = {member: "TEST"}; expect(response.stderr.toString()).toBe(""); expect(response.status).toBe(0); - expect(result.stdout).toContain("Data set downloaded successfully."); + expect(result.stdout).toContain("Member(s) downloaded successfully."); expect(result.stdout).toContain(testDir); expect(result.data.apiResponse.items[0]).toEqual(expectedResult); }); diff --git a/packages/cli/__tests__/zosfiles/__system__/download/amm/__scripts__/command_download_amm.sh b/packages/cli/__tests__/zosfiles/__system__/download/amm/__scripts__/command_download_amm.sh new file mode 100755 index 0000000000..42f6e009be --- /dev/null +++ b/packages/cli/__tests__/zosfiles/__system__/download/amm/__scripts__/command_download_amm.sh @@ -0,0 +1,12 @@ +#!/bin/bash +dsn=$1 +pattern=$2 +rfj=$3 +set -e + +echo "================Z/OS FILES DOWNLOAD ALL MEMBER DATA SET===============" +zowe zos-files download amm "$1" "$2" $3 $4 +if [ $? -gt 0 ] +then + exit $? +fi \ No newline at end of file diff --git a/packages/cli/__tests__/zosfiles/__system__/download/amm/__scripts__/command_download_amm_fully_qualified.sh b/packages/cli/__tests__/zosfiles/__system__/download/amm/__scripts__/command_download_amm_fully_qualified.sh new file mode 100755 index 0000000000..a726c0751a --- /dev/null +++ b/packages/cli/__tests__/zosfiles/__system__/download/amm/__scripts__/command_download_amm_fully_qualified.sh @@ -0,0 +1,9 @@ +#!/bin/bash +dsn=$1 +pattern=$2 +HOST=$3 +PORT=$4 +USER=$5 +PASS=$6 +zowe zos-files download amm "$dsn" "$pattern" --host $HOST --port $PORT --user $USER --password $PASS --ru=false +exit $? \ No newline at end of file diff --git a/packages/cli/__tests__/zosfiles/__system__/download/amm/__scripts__/command_download_amm_mcr.sh b/packages/cli/__tests__/zosfiles/__system__/download/amm/__scripts__/command_download_amm_mcr.sh new file mode 100755 index 0000000000..e0d0fecc85 --- /dev/null +++ b/packages/cli/__tests__/zosfiles/__system__/download/amm/__scripts__/command_download_amm_mcr.sh @@ -0,0 +1,12 @@ +#!/bin/bash +dsn=$1 +pattern=$2 +mcr=$3 +set -e + +echo "================Z/OS FILES DOWNLOAD ALL MEMBER DATA SET===============" +zowe zos-files download amm "$dsn" "$pattern" --max-concurrent-requests "$mcr" +if [ $? -gt 0 ] +then + exit $? +fi diff --git a/packages/cli/__tests__/zosfiles/__system__/download/amm/__scripts__/command_download_amm_no_extension.sh b/packages/cli/__tests__/zosfiles/__system__/download/amm/__scripts__/command_download_amm_no_extension.sh new file mode 100755 index 0000000000..1a2ac78738 --- /dev/null +++ b/packages/cli/__tests__/zosfiles/__system__/download/amm/__scripts__/command_download_amm_no_extension.sh @@ -0,0 +1,12 @@ +#!/bin/bash +dsn=$1 +pattern=$2 +rfj=$3 +set -e + +# echo "================Z/OS FILES DOWNLOAD ALL MEMBER DATA SET===============" +zowe zos-files download amm "$1" "$2" -e "" $3 +if [ $? -gt 0 ] +then + exit $? +fi diff --git a/packages/cli/__tests__/zosfiles/__system__/download/amm/cli.files.download.amm.system.test.ts b/packages/cli/__tests__/zosfiles/__system__/download/amm/cli.files.download.amm.system.test.ts new file mode 100644 index 0000000000..b5dad8dd54 --- /dev/null +++ b/packages/cli/__tests__/zosfiles/__system__/download/amm/cli.files.download.amm.system.test.ts @@ -0,0 +1,201 @@ +/* +* This program and the accompanying materials are made available under the terms of the +* Eclipse Public License v2.0 which accompanies this distribution, and is available at +* https://www.eclipse.org/legal/epl-v20.html +* +* SPDX-License-Identifier: EPL-2.0 +* +* Copyright Contributors to the Zowe Project. +* +*/ + +import { Session } from "@zowe/imperative"; +import * as path from "path"; +import { TestEnvironment } from "../../../../../../../__tests__/__src__/environment/TestEnvironment"; +import { ITestEnvironment } from "../../../../../../../__tests__/__src__/environment/ITestEnvironment"; +import { ITestPropertiesSchema } from "../../../../../../../__tests__/__src__/properties/ITestPropertiesSchema"; +import { getUniqueDatasetName } from "../../../../../../../__tests__/__src__/TestUtils"; +import { Create, CreateDataSetTypeEnum, Delete, Upload } from "@zowe/zos-files-for-zowe-sdk"; +import { runCliScript } from "@zowe/cli-test-utils"; + +let REAL_SESSION: Session; +// Test Environment populated in the beforeAll(); +let TEST_ENVIRONMENT: ITestEnvironment; +let TEST_ENVIRONMENT_NO_PROF: ITestEnvironment; +let defaultSystem: ITestPropertiesSchema; +let dsname: string; +const pattern = "M*"; +const members = ["M1", "M2", "M3"]; + +describe("Download Members Matching Pattern", () => { + + beforeAll(async () => { + TEST_ENVIRONMENT = await TestEnvironment.setUp({ + tempProfileTypes: ["zosmf"], + testName: "download_all_data_set_member_pattern" + }); + + defaultSystem = TEST_ENVIRONMENT.systemTestProperties; + + REAL_SESSION = TestEnvironment.createZosmfSession(TEST_ENVIRONMENT); + dsname = getUniqueDatasetName(defaultSystem.zosmf.user); + }); + + afterAll(async () => { + await TestEnvironment.cleanUp(TEST_ENVIRONMENT); + }); + describe("without profiles", () => { + let defaultSys: ITestPropertiesSchema; + + // Create the unique test environment + beforeAll(async () => { + TEST_ENVIRONMENT_NO_PROF = await TestEnvironment.setUp({ + testName: "zos_files_download_all_members_matching_without_profile" + }); + + defaultSys = TEST_ENVIRONMENT_NO_PROF.systemTestProperties; + }); + + afterAll(async () => { + await TestEnvironment.cleanUp(TEST_ENVIRONMENT_NO_PROF); + }); + + beforeEach(async () => { + await Create.dataSet(REAL_SESSION, CreateDataSetTypeEnum.DATA_SET_PARTITIONED, dsname); + for(const mem of members) { + await Upload.bufferToDataSet(REAL_SESSION, Buffer.from(mem), `${dsname}(${mem})`); + } + }); + + afterEach(async () => { + await Delete.dataSet(REAL_SESSION, dsname); + }); + + it("should download matching members of a pds", () => { + const shellScript = path.join(__dirname, "__scripts__", "command_download_amm_fully_qualified.sh"); + + const ZOWE_OPT_BASE_PATH = "ZOWE_OPT_BASE_PATH"; + + // if API Mediation layer is being used (basePath has a value) then + // set an ENVIRONMENT variable to be used by zowe. + if (defaultSys.zosmf.basePath != null) { + TEST_ENVIRONMENT_NO_PROF.env[ZOWE_OPT_BASE_PATH] = defaultSys.zosmf.basePath; + } + const response = runCliScript(shellScript, + TEST_ENVIRONMENT_NO_PROF, + [dsname, pattern, + defaultSys.zosmf.host, + defaultSys.zosmf.port, + defaultSys.zosmf.user, + defaultSys.zosmf.password]); + expect(response.stderr.toString()).toBe(""); + expect(response.status).toBe(0); + expect(response.stdout.toString()).toContain(`${members.length} members(s) were found matching pattern`); + expect(response.stdout.toString()).toContain("Member(s) downloaded successfully."); + }); + }); + + describe("Success scenarios", () => { + + beforeEach(async () => { + await Create.dataSet(REAL_SESSION, CreateDataSetTypeEnum.DATA_SET_PARTITIONED, dsname); + for(const mem of members) { + await Upload.bufferToDataSet(REAL_SESSION, Buffer.from(mem), `${dsname}(${mem})`); + } + }); + + afterEach(async () => { + await Delete.dataSet(REAL_SESSION, dsname); + }); + + it("should download all data set member of pds", () => { + const shellScript = path.join(__dirname, "__scripts__", "command_download_amm.sh"); + const response = runCliScript(shellScript, TEST_ENVIRONMENT, [dsname, pattern]); + expect(response.stderr.toString()).toBe(""); + expect(response.status).toBe(0); + expect(response.stdout.toString()).toContain(`${members.length} members(s) were found matching pattern`); + expect(response.stdout.toString()).toContain("Member(s) downloaded successfully."); + }); + + it("should download all data set member of pds in binary format", () => { + const shellScript = path.join(__dirname, "__scripts__", "command_download_amm.sh"); + const response = runCliScript(shellScript, TEST_ENVIRONMENT, [dsname,pattern, "--binary"]); + expect(response.stderr.toString()).toBe(""); + expect(response.status).toBe(0); + expect(response.stdout.toString()).toContain("Member(s) downloaded successfully."); + }); + + it("should download all data set member of pds in record format", () => { + const shellScript = path.join(__dirname, "__scripts__", "command_download_amm.sh"); + const response = runCliScript(shellScript, TEST_ENVIRONMENT, [dsname, pattern, "--record"]); + expect(response.stderr.toString()).toBe(""); + expect(response.status).toBe(0); + expect(response.stdout.toString()).toContain("Member(s) downloaded successfully."); + }); + + it("should download all data set member of pds with response timeout", () => { + const shellScript = path.join(__dirname, "__scripts__", "command_download_amm.sh"); + const response = runCliScript(shellScript, TEST_ENVIRONMENT, [dsname, pattern, "--responseTimeout 5"]); + expect(response.stderr.toString()).toBe(""); + expect(response.status).toBe(0); + expect(response.stdout.toString()).toContain("Member(s) downloaded successfully."); + }); + + it("should download all data set members with --max-concurrent-requests 2", () => { + const shellScript = path.join(__dirname, "__scripts__", "command_download_amm_mcr.sh"); + const response = runCliScript(shellScript, TEST_ENVIRONMENT, [dsname, pattern, 2]); + expect(response.stderr.toString()).toBe(""); + expect(response.status).toBe(0); + expect(response.stdout.toString()).toContain("Member(s) downloaded successfully."); + }); + + it("should download all data set members of a large data set with --max-concurrent-requests 2", async () => { + const bigDsname = getUniqueDatasetName(defaultSystem.zosmf.user); + const pattern = "a*"; + await Create.dataSet(REAL_SESSION, CreateDataSetTypeEnum.DATA_SET_PARTITIONED, bigDsname); + const members = ["a1", "a2", "a3", "a4", "a5", "a6", "a7", "a8", "a9", "a10", "a11", "a12", "a13", "b1", "b2"]; + const memberContent = Buffer.from("ABCDEFGHIJKLMNOPQRSTUVWXYZ\nABCDEFGHIJKLMNOPQRSTUVWXYZ\nABCDEFGHIJKLMNOPQRSTUVWXYZ"); + for (const mem of members) { + await Upload.bufferToDataSet(REAL_SESSION, memberContent, `${bigDsname}(${mem})`); + } + const shellScript = path.join(__dirname, "__scripts__", "command_download_amm_mcr.sh"); + const response = runCliScript(shellScript, TEST_ENVIRONMENT, [bigDsname, pattern, 2]); + expect(response.stderr.toString()).toBe(""); + expect(response.status).toBe(0); + expect(response.stdout.toString()).toContain("Member(s) downloaded successfully."); + await Delete.dataSet(REAL_SESSION, bigDsname); + }); + + it("should download all data set member with response-format-json flag", () => { + const shellScript = path.join(__dirname, "__scripts__", "command_download_amm.sh"); + const response = runCliScript(shellScript, TEST_ENVIRONMENT, [dsname, pattern, "--rfj"]); + expect(response.stderr.toString()).toBe(""); + expect(response.status).toBe(0); + expect(response.stdout.toString()).toContain("Member(s) downloaded successfully."); + }); + + it("should download all data set member to specified directory", () => { + const shellScript = path.join(__dirname, "__scripts__", "command_download_amm.sh"); + const testDir = "test/folder"; + const response = runCliScript(shellScript, TEST_ENVIRONMENT, [dsname, pattern,`-d ${testDir}`, "--rfj"]); + expect(response.stderr.toString()).toBe(""); + expect(response.status).toBe(0); + expect(response.stdout.toString()).toContain("Member(s) downloaded successfully."); + expect(response.stdout.toString()).toContain(testDir); + }); + + it("should download all data set member with extension = \"\"", () => { + const shellScript = path.join(__dirname, "__scripts__", "command_download_amm_no_extension.sh"); + const testDir = "test/folder"; + const response = runCliScript(shellScript, TEST_ENVIRONMENT, [dsname, pattern,`-d ${testDir} --rfj`]); + const result = JSON.parse(response.stdout.toString()); + const expectedResult = {member: "M1"}; + expect(response.stderr.toString()).toBe(""); + expect(response.status).toBe(0); + expect(result.stdout).toContain("Member(s) downloaded successfully."); + expect(result.stdout).toContain(testDir); + expect(result.data.apiResponse.items[0]).toEqual(expectedResult); + }); + }); + +}); diff --git a/packages/cli/__tests__/zosfiles/__system__/download/dsm/cli.files.download.dsm.system.test.ts b/packages/cli/__tests__/zosfiles/__system__/download/dsm/cli.files.download.dsm.system.test.ts index 9c6f1b1963..125d51e99a 100644 --- a/packages/cli/__tests__/zosfiles/__system__/download/dsm/cli.files.download.dsm.system.test.ts +++ b/packages/cli/__tests__/zosfiles/__system__/download/dsm/cli.files.download.dsm.system.test.ts @@ -191,7 +191,7 @@ describe("Download Dataset Matching", () => { expect(result.stdout).toContain(`${dsnames.length} data set(s) downloaded successfully to ${testDir}`); for (const apiResp of result.data.apiResponse) { - expect(apiResp.status).toContain("Data set downloaded successfully."); + expect(apiResp.status).toContain("Member(s) downloaded successfully."); expect(apiResp.status).toContain("Destination:"); expect(apiResp.status).toContain(testDir); expect(apiResp.status).toContain("Members: TEST;"); diff --git a/packages/cli/__tests__/zosfiles/__unit__/download/amm/AllMembersMatching.definition.unit.test.ts b/packages/cli/__tests__/zosfiles/__unit__/download/amm/AllMembersMatching.definition.unit.test.ts new file mode 100644 index 0000000000..0fdc374e92 --- /dev/null +++ b/packages/cli/__tests__/zosfiles/__unit__/download/amm/AllMembersMatching.definition.unit.test.ts @@ -0,0 +1,36 @@ +/* +* This program and the accompanying materials are made available under the terms of the +* Eclipse Public License v2.0 which accompanies this distribution, and is available at +* https://www.eclipse.org/legal/epl-v20.html +* +* SPDX-License-Identifier: EPL-2.0 +* +* Copyright Contributors to the Zowe Project. +* +*/ + +import { ICommandDefinition } from "@zowe/imperative"; + +describe("zos-files download amm command definition", () => { + it ("should not have changed", () => { + const definition: ICommandDefinition = require("../../../../../src/zosfiles/download/am/AllMembers.definition").AllMembersDefinition; + + expect(definition).toBeDefined(); + + // Should not contain children since this is a command + expect(definition.children).toBeUndefined(); + + // Should require a zosmf profile + expect(definition.profile.optional).toEqual(["zosmf"]); + + // Should only contain one positional + expect(definition.positionals.length).toEqual(1); + + // The positional should be required + expect(definition.positionals[0].required).toBeTruthy(); + + // Should not change + expect(definition.options).toMatchSnapshot(); + expect(definition.examples).toMatchSnapshot(); + }); +}); diff --git a/packages/cli/__tests__/zosfiles/__unit__/download/amm/AllMembersMatching.handler.unit.test.ts b/packages/cli/__tests__/zosfiles/__unit__/download/amm/AllMembersMatching.handler.unit.test.ts new file mode 100644 index 0000000000..f9824ce663 --- /dev/null +++ b/packages/cli/__tests__/zosfiles/__unit__/download/amm/AllMembersMatching.handler.unit.test.ts @@ -0,0 +1,174 @@ +/* +* This program and the accompanying materials are made available under the terms of the +* Eclipse Public License v2.0 which accompanies this distribution, and is available at +* https://www.eclipse.org/legal/epl-v20.html +* +* SPDX-License-Identifier: EPL-2.0 +* +* Copyright Contributors to the Zowe Project. +* +*/ + +import { IHandlerParameters, Session } from "@zowe/imperative"; +import { Download, IDownloadOptions, IDsmListOptions, List } from "@zowe/zos-files-for-zowe-sdk"; +import * as AllMembersMatchingDefinition from "../../../../../src/zosfiles/download/amm/AllMembersMatching.definition"; +import * as AllMembersMatchingHandler from "../../../../../src/zosfiles/download/amm/AllMembersMatching.handler"; +import { UNIT_TEST_ZOSMF_PROF_OPTS } from "../../../../../../../__tests__/__src__/TestConstants"; +import { mockHandlerParameters } from "@zowe/cli-test-utils"; + +const dsname = "test-pds"; + +const DEFAULT_PARAMETERS: IHandlerParameters = mockHandlerParameters({ + arguments: UNIT_TEST_ZOSMF_PROF_OPTS, + positionals: ["zos-jobs", "download", "output"], + definition: AllMembersMatchingDefinition.AllMembersMatchingDefinition +}); + +const fakeListOptions: IDsmListOptions = { + excludePatterns:undefined, + maxConcurrentRequests: undefined, + responseTimeout: undefined, + task: { + percentComplete: 0, + stageName: 0, + statusMessage: "Searching for members" + } +}; + +const fakeListResponse = [ + { + member: 'M1', + vers: 1, + mod: 0, + c4date: '2024/11/11', + m4date: '2024/11/11', + cnorc: 0, + inorc: 0, + mnorc: 0, + mtime: '16:06', + msec: '51', + user: 'x', + sclm: 'N' + } +]; + +const fakeDownloadOptions: IDownloadOptions = { + binary: undefined, + directory: undefined, + encoding: undefined, + extension: undefined, + extensionMap: undefined, + failFast: undefined, + maxConcurrentRequests: undefined, + preserveOriginalLetterCase: undefined, + record: undefined, + responseTimeout: undefined, + volume: undefined, + task: { + percentComplete: 0, + stageName: 0, + statusMessage: "Downloading all members" + }, + memberPatternResponse: fakeListResponse, +}; + +describe("Download AllMembersMatching handler", () => { + it("should download matching members if requested", async () => { + const pattern = "M1*"; + let passedSession: Session = null; + List.membersMatchingPattern = jest.fn(async (session) => { + passedSession = session; + return { + success: true, + commandResponse: "listed", + apiResponse: fakeListResponse + }; + }); + Download.allMembers = jest.fn(async (session) => { + return { + success: true, + commandResponse: "downloaded" + }; + }); + + const handler = new AllMembersMatchingHandler.default(); + const params = Object.assign({}, ...[DEFAULT_PARAMETERS]); + params.arguments = Object.assign({}, ...[DEFAULT_PARAMETERS.arguments]); + params.arguments.pattern = pattern; + params.arguments.dataSetName = dsname; + + await handler.process(params); + + expect(List.membersMatchingPattern).toHaveBeenCalledTimes(1); + expect(List.membersMatchingPattern).toHaveBeenCalledWith(passedSession, dsname, [pattern], { ...fakeListOptions }); + expect(Download.allMembers).toHaveBeenCalledTimes(1); + expect(Download.allMembers).toHaveBeenCalledWith(passedSession, dsname, { ...fakeDownloadOptions }); + }); + + it("should handle generation of an exclusion list", async () => { + const pattern = "M*"; + const excludePatterns = "M1*"; + let passedSession: Session = null; + List.membersMatchingPattern = jest.fn(async (session) => { + passedSession = session; + return { + success: true, + commandResponse: "listed", + apiResponse: fakeListResponse + }; + }); + Download.allMembers = jest.fn(async (session) => { + return { + success: true, + commandResponse: "downloaded" + }; + }); + + const handler = new AllMembersMatchingHandler.default(); + const params = Object.assign({}, ...[DEFAULT_PARAMETERS]); + params.arguments = Object.assign({}, ...[DEFAULT_PARAMETERS.arguments]); + params.arguments.pattern = pattern; + params.arguments.excludePatterns = excludePatterns; + params.arguments.dataSetName = dsname; + await handler.process(params); + + expect(List.membersMatchingPattern).toHaveBeenCalledTimes(1); + expect(List.membersMatchingPattern).toHaveBeenCalledWith(passedSession, dsname, [pattern], { + ...fakeListOptions, + excludePatterns: [excludePatterns] + }); + expect(Download.allMembers).toHaveBeenCalledTimes(1); + expect(Download.allMembers).toHaveBeenCalledWith(passedSession, dsname, { ...fakeDownloadOptions }); + }); + + it("should gracefully handle an error from the z/OSMF List API", async () => { + const errorMsg = "i haz bad data set"; + const pattern = "testing"; + let caughtError; + let passedSession: Session = null; + List.membersMatchingPattern = jest.fn((session) => { + passedSession = session; + throw new Error(errorMsg); + }); + Download.allMembers = jest.fn(); + + const handler = new AllMembersMatchingHandler.default(); + const params = Object.assign({}, ...[DEFAULT_PARAMETERS]); + params.arguments = Object.assign({}, ...[DEFAULT_PARAMETERS.arguments]); + params.arguments.pattern = pattern; + params.arguments.dataSetName = dsname; + + try { + await handler.process(params); + } catch (error) { + caughtError = error; + } + + expect(caughtError).toBeDefined(); + expect(caughtError.message).toBe(errorMsg); + expect(List.membersMatchingPattern).toHaveBeenCalledTimes(1); + expect(List.membersMatchingPattern).toHaveBeenCalledWith(passedSession, dsname, [pattern], { ...fakeListOptions }); + expect(Download.allMembers).toHaveBeenCalledTimes(0); + }); +}); + diff --git a/packages/cli/__tests__/zosfiles/__unit__/download/amm/__snapshots__/AllMembersMatching.definition.unit.test.ts.snap b/packages/cli/__tests__/zosfiles/__unit__/download/amm/__snapshots__/AllMembersMatching.definition.unit.test.ts.snap new file mode 100644 index 0000000000..b6eeab84eb --- /dev/null +++ b/packages/cli/__tests__/zosfiles/__unit__/download/amm/__snapshots__/AllMembersMatching.definition.unit.test.ts.snap @@ -0,0 +1,102 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`zos-files download amm command definition should not have changed 1`] = ` +Array [ + Object { + "aliases": Array [ + "b", + ], + "description": "Download the file content in binary mode, which means that no data conversion is performed. The data transfer process returns each line as-is, without translation. No delimiters are added between records.", + "name": "binary", + "type": "boolean", + }, + Object { + "aliases": Array [ + "d", + ], + "description": "The directory to where you want to save the members. The command creates the directory for you when it does not already exist. By default, the command creates a folder structure based on the data set qualifiers. For example, the data set ibmuser.new.cntl's members are downloaded to ibmuser/new/cntl).", + "name": "directory", + "type": "string", + }, + Object { + "aliases": Array [ + "ec", + ], + "description": "Download the file content with encoding mode, which means that data conversion is performed using the file encoding specified.", + "name": "encoding", + "type": "string", + }, + Object { + "aliases": Array [ + "e", + ], + "description": "Save the local files with a specified file extension. For example, .txt. Or \\"\\" for no extension. When no extension is specified, .txt is used as the default file extension.", + "name": "extension", + "type": "stringOrEmpty", + }, + Object { + "aliases": Array [ + "ff", + ], + "defaultValue": true, + "description": "Set this option to false to continue downloading data set members if one or more fail.", + "name": "fail-fast", + "type": "boolean", + }, + Object { + "aliases": Array [ + "mcr", + ], + "defaultValue": 1, + "description": "Specifies the maximum number of concurrent z/OSMF REST API requests to download members. Increasing the value results in faster downloads. However, increasing the value increases resource consumption on z/OS and can be prone to errors caused by making too many concurrent requests. If the download process encounters an error, the following message displays: +The maximum number of TSO address spaces have been created. When you specify 0, Zowe CLI attempts to download all members at once without a maximum number of concurrent requests. ", + "name": "max-concurrent-requests", + "numericValueRange": Array [ + 0, + 100, + ], + "type": "number", + }, + Object { + "aliases": Array [ + "po", + ], + "defaultValue": false, + "description": "Specifies if the automatically generated directories and files use the original letter case.", + "name": "preserve-original-letter-case", + "type": "boolean", + }, + Object { + "aliases": Array [ + "r", + ], + "conflictsWith": Array [ + "binary", + ], + "description": "Download the file content in record mode, which means that no data conversion is performed and the record length is prepended to the data. The data transfer process returns each line as-is, without translation. No delimiters are added between records. Conflicts with binary.", + "name": "record", + "type": "boolean", + }, + Object { + "aliases": Array [ + "vs", + ], + "description": "The volume serial (VOLSER) where the data set resides. You can use this option at any time. However, the VOLSER is required only when the data set is not cataloged on the system. A VOLSER is analogous to a drive name on a PC.", + "name": "volume-serial", + "type": "string", + }, +] +`; + +exports[`zos-files download amm command definition should not have changed 2`] = ` +Array [ + Object { + "description": "Download the members of the data set \\"ibmuser.loadlib\\" in binary mode to the directory \\"loadlib/\\"", + "options": "\\"ibmuser.loadlib\\" -b -d loadlib", + }, + Object { + "description": "Download the members of the data set \\"ibmuser.cntl\\" in text mode to the directory \\"jcl/\\"", + "options": "\\"ibmuser.cntl\\" -d jcl", + }, +] +`; diff --git a/packages/cli/__tests__/zosfiles/__unit__/download/amm/__snapshots__/AllMembersMatching.handler.unit.test.ts.snap b/packages/cli/__tests__/zosfiles/__unit__/download/amm/__snapshots__/AllMembersMatching.handler.unit.test.ts.snap new file mode 100644 index 0000000000..4dec40603f --- /dev/null +++ b/packages/cli/__tests__/zosfiles/__unit__/download/amm/__snapshots__/AllMembersMatching.handler.unit.test.ts.snap @@ -0,0 +1,31 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`Download AllMembersMatching handler should download matching members if requested 1`] = ` +" +listed +" +`; + +exports[`Download AllMembersMatching handler should download matching members if requested 2`] = `"downloaded"`; + +exports[`Download AllMembersMatching handler should download matching members if requested 3`] = ` +Object { + "commandResponse": "downloaded", + "success": true, +} +`; + +exports[`Download AllMembersMatching handler should handle generation of an exclusion list 1`] = ` +" +listed +" +`; + +exports[`Download AllMembersMatching handler should handle generation of an exclusion list 2`] = `"downloaded"`; + +exports[`Download AllMembersMatching handler should handle generation of an exclusion list 3`] = ` +Object { + "commandResponse": "downloaded", + "success": true, +} +`; diff --git a/packages/cli/src/zosfiles/-strings-/en.ts b/packages/cli/src/zosfiles/-strings-/en.ts index 205ea79f01..bd65b59d95 100644 --- a/packages/cli/src/zosfiles/-strings-/en.ts +++ b/packages/cli/src/zosfiles/-strings-/en.ts @@ -338,7 +338,7 @@ export default { DESCRIPTION: "Download content from z/OS data sets and USS files to your PC.", ACTIONS: { ALL_MEMBERS: { - SUMMARY: "Download all members from a pds", + SUMMARY: "Download all members from a PDS", DESCRIPTION: "Download all members from a partitioned data set to a local folder.", POSITIONALS: { DATASETNAME: "The name of the data set from which you want to download members" @@ -348,6 +348,23 @@ export default { EX2: `Download the members of the data set "ibmuser.cntl" in text mode to the directory "jcl/"` } }, + ALL_MEMBERS_MATCHING: { + SUMMARY: "Download all members from a PDS", + DESCRIPTION: "Download all members that match a specific pattern from a partitioned data set to a local folder.", + POSITIONALS: { + DATASETNAME: "The name of the data set from which you want to download members", + PATTERN: `The pattern or patterns to match members against. Also known as 'DSLEVEL'. The following special sequences can be ` + + `used in the pattern: + ${TextUtils.chalk.yellow("%")}: matches any single character + ${TextUtils.chalk.yellow("*")}: matches any number of characters within a member + You can specify multiple patterns separated by commas, for example "Mem*, Test*"` + }, + EXAMPLES: { + EX1: `Download the members of the data set "ibmuser.loadlib" that begin with "Test" to the directory "loadlib/"`, + EX2: `Download the members of the data set "ibmuser.cntl" that begin with "Test" & "M" to the directory "output", + and exclude members that begin with "M2".` + } + }, DATA_SET: { SUMMARY: "Download content from a z/OS data set", DESCRIPTION: "Download content from a z/OS data set to a local file.", @@ -489,7 +506,7 @@ export default { DESCRIPTION: "List data sets and data set members. Optionally, you can list their details and attributes.", ACTIONS: { ALL_MEMBERS: { - SUMMARY: "List all members of a pds", + SUMMARY: "List all members of a PDS", DESCRIPTION: "List all members of a partitioned data set. To view additional information about each member, use the --attributes " + "option under the Options section of this help text.", POSITIONALS: { diff --git a/packages/cli/src/zosfiles/download/Download.definition.ts b/packages/cli/src/zosfiles/download/Download.definition.ts index 0d7ae4d394..cfa748b326 100644 --- a/packages/cli/src/zosfiles/download/Download.definition.ts +++ b/packages/cli/src/zosfiles/download/Download.definition.ts @@ -17,6 +17,7 @@ import i18nTypings from "../-strings-/en"; import { UssFileDefinition } from "./uss/UssFile.definition"; import { DataSetMatchingDefinition } from "./dsm/DataSetMatching.definition"; import { UssDirDefinition } from "./ussdir/UssDir.definition"; +import { AllMembersMatchingDefinition } from "./amm/AllMembersMatching.definition"; // Does not use the import in anticipation of some internationalization work to be done later. const strings = (require("../-strings-/en").default as typeof i18nTypings).DOWNLOAD; @@ -36,6 +37,7 @@ export const DownloadDefinition: ICommandDefinition = { AllMembersDefinition, UssFileDefinition, UssDirDefinition, - DataSetMatchingDefinition + DataSetMatchingDefinition, + AllMembersMatchingDefinition ] }; diff --git a/packages/cli/src/zosfiles/download/amm/AllMembersMatching.definition.ts b/packages/cli/src/zosfiles/download/amm/AllMembersMatching.definition.ts new file mode 100644 index 0000000000..6f14959494 --- /dev/null +++ b/packages/cli/src/zosfiles/download/amm/AllMembersMatching.definition.ts @@ -0,0 +1,69 @@ +/* +* This program and the accompanying materials are made available under the terms of the +* Eclipse Public License v2.0 which accompanies this distribution, and is available at +* https://www.eclipse.org/legal/epl-v20.html +* +* SPDX-License-Identifier: EPL-2.0 +* +* Copyright Contributors to the Zowe Project. +* +*/ + +import { ICommandDefinition } from "@zowe/imperative"; +import { DownloadOptions } from "../Download.options"; +import i18nTypings from "../../-strings-/en"; + +// Does not use the import in anticipation of some internationalization work to be done later. +const strings = (require("../../-strings-/en").default as typeof i18nTypings).DOWNLOAD.ACTIONS.ALL_MEMBERS_MATCHING; + +/** + * Download all members command definition containing its description, examples and/or options + * @type {ICommandDefinition} + */ +export const AllMembersMatchingDefinition: ICommandDefinition = { + name: "all-members-matching", + aliases: ["amm", "all-members-matching"], + summary: strings.SUMMARY, + description: strings.DESCRIPTION, + type: "command", + handler: __dirname + "/AllMembersMatching.handler", + profile: { + optional: ["zosmf"] + }, + positionals: [ + { + name: "dataSetName", + description: strings.POSITIONALS.DATASETNAME, + type: "string", + required: true + }, + { + name: "pattern", + description: strings.POSITIONALS.PATTERN, + type: "string", + required: true + } + ], + options: [ + DownloadOptions.volume, + DownloadOptions.directory, + DownloadOptions.binary, + DownloadOptions.record, + DownloadOptions.encoding, + DownloadOptions.extension, + DownloadOptions.excludePattern, + DownloadOptions.maxConcurrentRequests, + DownloadOptions.preserveOriginalLetterCase, + DownloadOptions.failFast + ].sort((a, b) => a.name.localeCompare(b.name)), + examples: [ + { + description: strings.EXAMPLES.EX1, + options: `"ibmuser.loadlib" "Test*" -d loadlib` + }, + { + description: strings.EXAMPLES.EX2, + options: `"ibmuser.cntl" "test*,M*" --exclude-patterns "M2*" -d output` + } + ] +}; \ No newline at end of file diff --git a/packages/cli/src/zosfiles/download/amm/AllMembersMatching.handler.ts b/packages/cli/src/zosfiles/download/amm/AllMembersMatching.handler.ts new file mode 100644 index 0000000000..e51dbeecf5 --- /dev/null +++ b/packages/cli/src/zosfiles/download/amm/AllMembersMatching.handler.ts @@ -0,0 +1,63 @@ +/* +* This program and the accompanying materials are made available under the terms of the +* Eclipse Public License v2.0 which accompanies this distribution, and is available at +* https://www.eclipse.org/legal/epl-v20.html +* +* SPDX-License-Identifier: EPL-2.0 +* +* Copyright Contributors to the Zowe Project. +* +*/ + +import { AbstractSession, IHandlerParameters, ITaskWithStatus, TaskStage } from "@zowe/imperative"; +import { IZosFilesResponse, Download, IDsmListOptions, List } from "@zowe/zos-files-for-zowe-sdk"; +import { ZosFilesBaseHandler } from "../../ZosFilesBase.handler"; + +/** + * Handler to download all members given a data set name & pattern + * @export + */ +export default class AllMembersMatchingHandler extends ZosFilesBaseHandler { + public async processWithSession(commandParameters: IHandlerParameters, session: AbstractSession): Promise { + const listStatus: ITaskWithStatus = { + statusMessage: "Searching for members", + percentComplete: 0, + stageName: TaskStage.IN_PROGRESS + }; + const listOptions: IDsmListOptions = { + excludePatterns: commandParameters.arguments.excludePatterns?.split(","), + maxConcurrentRequests: commandParameters.arguments.maxConcurrentRequests, + task: listStatus, + responseTimeout: commandParameters.arguments.responseTimeout + }; + commandParameters.response.progress.startBar({ task: listStatus }); + const response = await List.membersMatchingPattern(session, commandParameters.arguments.dataSetName, + commandParameters.arguments.pattern.split(","), listOptions); + commandParameters.response.progress.endBar(); + if (response.success) { + commandParameters.response.console.log(`\r${response.commandResponse}\n`); + } else { + return response; + } + const status: ITaskWithStatus = { + statusMessage: "Downloading all members", + percentComplete: 0, + stageName: TaskStage.IN_PROGRESS + }; + commandParameters.response.progress.startBar({task: status}); + return Download.allMembers(session, commandParameters.arguments.dataSetName, { + volume: commandParameters.arguments.volumeSerial, + binary: commandParameters.arguments.binary, + record: commandParameters.arguments.record, + encoding: commandParameters.arguments.encoding, + directory: commandParameters.arguments.directory, + extension: commandParameters.arguments.extension, + maxConcurrentRequests: commandParameters.arguments.maxConcurrentRequests, + preserveOriginalLetterCase: commandParameters.arguments.preserveOriginalLetterCase, + failFast: commandParameters.arguments.failFast, + task: status, + responseTimeout: commandParameters.arguments.responseTimeout, + memberPatternResponse: response.apiResponse, + }); + } +} \ No newline at end of file diff --git a/packages/cli/src/zosfiles/list/am/AllMembers.handler.ts b/packages/cli/src/zosfiles/list/am/AllMembers.handler.ts index 41a7196ac3..fb17fa6374 100644 --- a/packages/cli/src/zosfiles/list/am/AllMembers.handler.ts +++ b/packages/cli/src/zosfiles/list/am/AllMembers.handler.ts @@ -26,7 +26,6 @@ export default class AllMembersHandler extends ZosFilesBaseHandler { pattern: commandParameters.arguments.pattern, responseTimeout: commandParameters.arguments.responseTimeout }); - const invalidMemberCount = response.apiResponse.returnedRows - response.apiResponse.items.length; if (invalidMemberCount > 0) { const invalidMemberMsg = `${invalidMemberCount} members failed to load due to invalid name errors`; diff --git a/packages/zosfiles/CHANGELOG.md b/packages/zosfiles/CHANGELOG.md index 65775b4a04..3dff530c4f 100644 --- a/packages/zosfiles/CHANGELOG.md +++ b/packages/zosfiles/CHANGELOG.md @@ -2,6 +2,10 @@ All notable changes to the Zowe z/OS files SDK package will be documented in this file. +## Recent Changes + +- Enhancement: Added a `List.membersMatchingPattern` method to download all members that match a specific pattern.[#2359](https://github.com/zowe/zowe-cli/pull/2359) + ## `8.8.4` - Enhancement: Allows extenders of the Search functionality to pass a function `abortSearch` on `searchOptions` to abort a search. [#2370](https://github.com/zowe/zowe-cli/pull/2370) diff --git a/packages/zosfiles/__tests__/__system__/methods/download/Download.system.test.ts b/packages/zosfiles/__tests__/__system__/methods/download/Download.system.test.ts index 349dcbbd4c..6a0d3b8610 100644 --- a/packages/zosfiles/__tests__/__system__/methods/download/Download.system.test.ts +++ b/packages/zosfiles/__tests__/__system__/methods/download/Download.system.test.ts @@ -413,7 +413,7 @@ describe.each([false, true])("Download Data Set - Encoded: %s", (encoded: boolea expect(response).toBeTruthy(); expect(response.success).toBeTruthy(); expect(response.commandResponse).toContain( - ZosFilesMessages.datasetDownloadedSuccessfully.message.substring(0, "Data set downloaded successfully".length + 1)); + ZosFilesMessages.memberDownloadedSuccessfully.message.substring(0, "Member(s) downloaded successfully".length + 1)); // convert the data set name to use as a path/file const regex = /\./gi; @@ -443,7 +443,7 @@ describe.each([false, true])("Download Data Set - Encoded: %s", (encoded: boolea expect(response).toBeTruthy(); expect(response.success).toBeTruthy(); expect(response.commandResponse).toContain( - ZosFilesMessages.datasetDownloadedSuccessfully.message.substring(0, "Data set downloaded successfully".length + 1)); + ZosFilesMessages.memberDownloadedSuccessfully.message.substring(0, "Member(s) downloaded successfully".length + 1)); // convert the data set name to use as a path/file const regex = /\./gi; @@ -473,7 +473,7 @@ describe.each([false, true])("Download Data Set - Encoded: %s", (encoded: boolea expect(response).toBeTruthy(); expect(response.success).toBeTruthy(); expect(response.commandResponse).toContain( - ZosFilesMessages.datasetDownloadedSuccessfully.message.substring(0, "Data set downloaded successfully".length + 1)); + ZosFilesMessages.memberDownloadedSuccessfully.message.substring(0, "Member(s) downloaded successfully".length + 1)); // convert the data set name to use as a path/file const regex = /\./gi; @@ -514,7 +514,7 @@ describe.each([false, true])("Download Data Set - Encoded: %s", (encoded: boolea expect(response).toBeTruthy(); expect(response.success).toBeTruthy(); expect(response.commandResponse).toContain( - ZosFilesMessages.datasetDownloadedSuccessfully.message.substring(0, "Data set downloaded successfully".length + 1)); + ZosFilesMessages.memberDownloadedSuccessfully.message.substring(0, "Member(s) downloaded successfully".length + 1)); // convert the data set name to use as a path/file for clean up in AfterEach const regex = /\./gi; @@ -544,7 +544,7 @@ describe.each([false, true])("Download Data Set - Encoded: %s", (encoded: boolea expect(response).toBeTruthy(); expect(response.success).toBeTruthy(); expect(response.commandResponse).toContain( - ZosFilesMessages.datasetDownloadedSuccessfully.message.substring(0, "Data set downloaded successfully".length + 1)); + ZosFilesMessages.memberDownloadedSuccessfully.message.substring(0, "Member(s) downloaded successfully".length + 1)); // convert the data set name to use as a path/file for clean up in AfterEach const regex = /\./gi; @@ -574,7 +574,7 @@ describe.each([false, true])("Download Data Set - Encoded: %s", (encoded: boolea expect(response).toBeTruthy(); expect(response.success).toBeTruthy(); expect(response.commandResponse).toContain( - ZosFilesMessages.datasetDownloadedSuccessfully.message.substring(0, "Data set downloaded successfully".length + 1)); + ZosFilesMessages.memberDownloadedSuccessfully.message.substring(0, "Member(s) downloaded successfully".length + 1)); // convert the data set name to use as a path/file for clean up in AfterEach const regex = /\./gi; @@ -599,7 +599,7 @@ describe.each([false, true])("Download Data Set - Encoded: %s", (encoded: boolea expect(response).toBeTruthy(); expect(response.success).toBeTruthy(); expect(response.commandResponse).toContain( - ZosFilesMessages.datasetDownloadedSuccessfully.message.substring(0, "Data set downloaded successfully".length + 1)); + ZosFilesMessages.memberDownloadedSuccessfully.message.substring(0, "Member(s) downloaded successfully".length + 1)); // Convert the data set name to use as a path/file const regex = /\./gi; diff --git a/packages/zosfiles/__tests__/__system__/methods/list/List.system.test.ts b/packages/zosfiles/__tests__/__system__/methods/list/List.system.test.ts index 3cba83972c..e529caf9a6 100644 --- a/packages/zosfiles/__tests__/__system__/methods/list/List.system.test.ts +++ b/packages/zosfiles/__tests__/__system__/methods/list/List.system.test.ts @@ -659,6 +659,77 @@ describe("List command group", () => { }); }); + describe("membersMatchingPattern", () => { + const members = ["M1", "M1A", "M2", "M3"]; + const pattern = "M*"; + beforeEach(async () => { + await Create.dataSet(REAL_SESSION, CreateDataSetTypeEnum.DATA_SET_PARTITIONED, dsname, + { volser: defaultSystem.datasets.vol }); + await wait(waitTime); //wait 2 seconds + for(const mem of members) { + await Upload.bufferToDataSet(REAL_SESSION, Buffer.from(mem), `${dsname}(${mem})`); + } + await wait(waitTime); //wait 2 seconds + }); + + afterEach(async () => { + await Delete.dataSet(REAL_SESSION, dsname); + await wait(waitTime); //wait 2 seconds + }); + it("should find data sets that match a pattern", async () => { + let error; + let response: IZosFilesResponse; + + try { + response = await List.membersMatchingPattern(REAL_SESSION, dsname, [pattern]); + Imperative.console.info("Response: " + inspect(response)); + } catch (err) { + error = err; + Imperative.console.info("Error: " + inspect(error)); + } + expect(error).toBeFalsy(); + expect(response).toBeTruthy(); + expect(response.success).toBeTruthy(); + expect(response.commandResponse).toBe("4 members(s) were found matching pattern."); + expect(response.apiResponse.length).toBe(4); + expect(response.apiResponse[0].member).toEqual(members[0].toUpperCase()); + }); + + it("should exclude data sets that do not match a pattern", async () => { + let response; + let caughtError; + + try { + response = await List.membersMatchingPattern(REAL_SESSION, dsname, [pattern], + { excludePatterns: ["M1*"] }); + } catch (error) { + caughtError = error; + } + + expect(caughtError).toBeUndefined(); + expect(response).toBeDefined(); + expect(response.success).toBe(true); + expect(response.commandResponse).toContain(format(ZosFilesMessages.membersMatchedPattern.message, 2)); + expect(response.apiResponse.length).toBe(2); + }); + + it("should fail when no data sets match", async () => { + let response; + let caughtError; + const pattern = "test*"; + + try { + response = await List.membersMatchingPattern(REAL_SESSION, dsname, [pattern]); + } catch (error) { + caughtError = error; + } + + expect(caughtError).not.toBeDefined(); + expect(response).toBeDefined(); + expect(response.commandResponse).toContain("There are no members that match"); + }); + }); + }); describe("List command group - encoded", () => { diff --git a/packages/zosfiles/__tests__/__unit__/methods/download/Download.unit.test.ts b/packages/zosfiles/__tests__/__unit__/methods/download/Download.unit.test.ts index ce404abc48..ce7a37490f 100644 --- a/packages/zosfiles/__tests__/__unit__/methods/download/Download.unit.test.ts +++ b/packages/zosfiles/__tests__/__unit__/methods/download/Download.unit.test.ts @@ -696,6 +696,7 @@ describe("z/OS Files - Download", () => { let response; let caughtError; + try { response = await Download.allMembers(dummySession, dsname); } catch (e) { @@ -705,7 +706,7 @@ describe("z/OS Files - Download", () => { expect(caughtError).toBeUndefined(); expect(response).toEqual({ success: true, - commandResponse: util.format(ZosFilesMessages.datasetDownloadedWithDestination.message, dsFolder), + commandResponse: util.format(ZosFilesMessages.memberDownloadedWithDestination.message, dsFolder), apiResponse: listApiResponse }); @@ -738,7 +739,7 @@ describe("z/OS Files - Download", () => { expect(caughtError).toBeUndefined(); expect(response).toEqual({ success: true, - commandResponse: util.format(ZosFilesMessages.datasetDownloadedWithDestination.message, directory), + commandResponse: util.format(ZosFilesMessages.memberDownloadedWithDestination.message, directory), apiResponse: listApiResponse }); @@ -773,7 +774,7 @@ describe("z/OS Files - Download", () => { expect(caughtError).toBeUndefined(); expect(response).toEqual({ success: true, - commandResponse: util.format(ZosFilesMessages.datasetDownloadedWithDestination.message, directory), + commandResponse: util.format(ZosFilesMessages.memberDownloadedWithDestination.message, directory), apiResponse: listApiResponse }); @@ -809,7 +810,7 @@ describe("z/OS Files - Download", () => { expect(caughtError).toBeUndefined(); expect(response).toEqual({ success: true, - commandResponse: util.format(ZosFilesMessages.datasetDownloadedWithDestination.message, directory), + commandResponse: util.format(ZosFilesMessages.memberDownloadedWithDestination.message, directory), apiResponse: listApiResponse }); @@ -847,7 +848,7 @@ describe("z/OS Files - Download", () => { expect(caughtError).toBeUndefined(); expect(response).toEqual({ success: true, - commandResponse: util.format(ZosFilesMessages.datasetDownloadedWithDestination.message, directory), + commandResponse: util.format(ZosFilesMessages.memberDownloadedWithDestination.message, directory), apiResponse: listApiResponse }); @@ -884,7 +885,7 @@ describe("z/OS Files - Download", () => { expect(caughtError).toBeUndefined(); expect(response).toEqual({ success: true, - commandResponse: util.format(ZosFilesMessages.datasetDownloadedWithDestination.message, directory), + commandResponse: util.format(ZosFilesMessages.memberDownloadedWithDestination.message, directory), apiResponse: listApiResponse }); @@ -918,7 +919,7 @@ describe("z/OS Files - Download", () => { expect(caughtError).toBeUndefined(); expect(response).toEqual({ success: true, - commandResponse: util.format(ZosFilesMessages.datasetDownloadedWithDestination.message, directory), + commandResponse: util.format(ZosFilesMessages.memberDownloadedWithDestination.message, directory), apiResponse: listApiResponse }); @@ -952,7 +953,7 @@ describe("z/OS Files - Download", () => { expect(caughtError).toBeUndefined(); expect(response).toEqual({ success: true, - commandResponse: util.format(ZosFilesMessages.datasetDownloadedWithDestination.message, directory), + commandResponse: util.format(ZosFilesMessages.memberDownloadedWithDestination.message, directory), apiResponse: listApiResponse }); @@ -982,7 +983,7 @@ describe("z/OS Files - Download", () => { expect(caughtError).toBeUndefined(); expect(response).toEqual({ success: true, - commandResponse: util.format(ZosFilesMessages.datasetDownloadedWithDestination.message, dsFolder.toUpperCase()), + commandResponse: util.format(ZosFilesMessages.memberDownloadedWithDestination.message, dsFolder.toUpperCase()), apiResponse: listApiResponse }); diff --git a/packages/zosfiles/__tests__/__unit__/methods/list/List.unit.test.ts b/packages/zosfiles/__tests__/__unit__/methods/list/List.unit.test.ts index b5e8434626..ef6cd556e6 100644 --- a/packages/zosfiles/__tests__/__unit__/methods/list/List.unit.test.ts +++ b/packages/zosfiles/__tests__/__unit__/methods/list/List.unit.test.ts @@ -1529,4 +1529,187 @@ describe("z/OS Files - List", () => { }); }); }); + describe("membersMatchingPattern", () => { + const listDataSetSpy = jest.spyOn(List, "allMembers"); + + const memberData1 = { + dsname: "TEST.PS", + member: "M1", + dsorg: "PS" + }; + + const memberData2 = { + dsname: "TEST.PS", + member: "M2", + dsorg: "PS" + }; + + beforeEach(() => { + listDataSetSpy.mockClear(); + listDataSetSpy.mockResolvedValue({} as any); + }); + + it("should successfully list M1 & M2 using the List.allMembers API", async () => { + const dsname = "TEST.PS"; + const pattern = "M*"; + let response; + let caughtError; + + listDataSetSpy.mockImplementation(async (): Promise => { + return { + apiResponse: { + items: [memberData1, memberData2] + } + }; + }); + + try { + response = await List.membersMatchingPattern(dummySession, dsname, [pattern]); + } catch (e) { + caughtError = e; + } + + expect(caughtError).toBeUndefined(); + expect(response).toEqual({ + success: true, + commandResponse: util.format(ZosFilesMessages.membersMatchedPattern.message, 2), + apiResponse: [memberData1, memberData2] + }); + + expect(listDataSetSpy).toHaveBeenCalledTimes(1); + expect(listDataSetSpy).toHaveBeenCalledWith(dummySession, dsname, {pattern}); + }); + it("should throw an error if the data set name is not specified", async () => { + let response; + let caughtError; + const pattern = "M*"; + + // Test for NULL + try { + response = await List.membersMatchingPattern(dummySession, null, [pattern]); + } catch (e) { + caughtError = e; + } + + expect(response).toBeUndefined(); + expect(caughtError).toBeDefined(); + expect(caughtError.message).toContain(ZosFilesMessages.missingDatasetName.message); + + caughtError = undefined; + // Test for UNDEFINED + try { + response = await List.membersMatchingPattern(dummySession, undefined, [pattern]); + } catch (e) { + caughtError = e; + } + + expect(response).toBeUndefined(); + expect(caughtError).toBeDefined(); + expect(caughtError.message).toContain(ZosFilesMessages.missingDatasetName.message); + + caughtError = undefined; + // Test for EMPTY + try { + response = await List.membersMatchingPattern(dummySession,"",[pattern]); + } catch (e) { + caughtError = e; + } + + expect(response).toBeUndefined(); + expect(caughtError).toBeDefined(); + expect(caughtError.message).toContain(ZosFilesMessages.missingDatasetName.message); + }); + + it("should throw an error if the member pattern is not specified", async () => { + let response; + let caughtError; + + // Test for NULL + try { + response = await List.membersMatchingPattern(dummySession, memberData1.dsname, [null]); + } catch (e) { + caughtError = e; + } + + expect(response).toBeUndefined(); + expect(caughtError).toBeDefined(); + expect(caughtError.message).toContain(ZosFilesMessages.missingPatterns.message); + + caughtError = undefined; + // Test for UNDEFINED + try { + response = await List.membersMatchingPattern(dummySession, memberData1.dsname, [undefined]); + } catch (e) { + caughtError = e; + } + + expect(response).toBeUndefined(); + expect(caughtError).toBeDefined(); + expect(caughtError.message).toContain(ZosFilesMessages.missingPatterns.message); + + caughtError = undefined; + // Test for EMPTY + try { + response = await List.membersMatchingPattern(dummySession,memberData1.dsname,[]); + } catch (e) { + caughtError = e; + } + + expect(response).toBeUndefined(); + expect(caughtError).toBeDefined(); + expect(caughtError.message).toContain(ZosFilesMessages.missingPatterns.message); + }); + + it("should handle an error from the List.allMembers API", async () => { + const dummyError = new Error("test2"); + let response; + let caughtError; + const pattern = "M*"; + const dsname = "TEST.PS"; + + listDataSetSpy.mockImplementation(async () => { + throw dummyError; + }); + + try { + response = await List.membersMatchingPattern(dummySession, dsname, [pattern]); + } catch (e) { + caughtError = e; + } + + expect(response).toBeUndefined(); + expect(caughtError).toEqual(dummyError); + + expect(listDataSetSpy).toHaveBeenCalledTimes(1); + expect(listDataSetSpy).toHaveBeenCalledWith(dummySession, dsname, {pattern}); + }); + it("should handle an error when the exclude pattern is specified", async () => { + const excludePatterns = ["M1*"]; + const pattern = "M1*"; + let response; + let caughtError; + + List.allMembers = jest.fn(async (): Promise => { + return { + apiResponse: { + items: [memberData1] + } + }; + }); + + try { + response = await List.membersMatchingPattern( + dummySession, memberData1.dsname, [pattern], { excludePatterns }); + } catch (e) { + caughtError = e; + } + + expect(caughtError).toBeUndefined(); + expect(response).toEqual({ + success: false, + commandResponse: util.format(ZosFilesMessages.noMembersInList.message), + apiResponse: [] + }); + }); + }); }); diff --git a/packages/zosfiles/__tests__/__unit__/utils/__snapshots__/ZosFilesUtils.unit.test.ts.snap b/packages/zosfiles/__tests__/__unit__/utils/__snapshots__/ZosFilesUtils.unit.test.ts.snap index 9e94d47dcb..33e8c2f747 100644 --- a/packages/zosfiles/__tests__/__unit__/utils/__snapshots__/ZosFilesUtils.unit.test.ts.snap +++ b/packages/zosfiles/__tests__/__unit__/utils/__snapshots__/ZosFilesUtils.unit.test.ts.snap @@ -187,6 +187,16 @@ Destination: %s", "message": "Failed to download the following members: ", }, + "memberDownloadedSuccessfully": Object { + "message": "Member(s) downloaded successfully.", + }, + "memberDownloadedWithDestination": Object { + "message": "Member(s) downloaded successfully. +Destination: %s", + }, + "membersMatchedPattern": Object { + "message": "%d members(s) were found matching pattern.", + }, "missingDataSets": Object { "message": "No list of data sets to download was passed.", }, @@ -268,6 +278,12 @@ Destination: %s", "noMembersFound": Object { "message": "No members found!", }, + "noMembersInList": Object { + "message": "No members left after excluded pattern(s) were filtered out.", + }, + "noMembersMatchingPattern": Object { + "message": "There are no members that match the provided pattern(s).", + }, "nodeJsFsError": Object { "message": "Node.js File System API error", }, diff --git a/packages/zosfiles/src/constants/ZosFiles.messages.ts b/packages/zosfiles/src/constants/ZosFiles.messages.ts index 7c58177267..8e6abc4cf8 100644 --- a/packages/zosfiles/src/constants/ZosFiles.messages.ts +++ b/packages/zosfiles/src/constants/ZosFiles.messages.ts @@ -167,6 +167,23 @@ export const ZosFilesMessages: { [key: string]: IMessageDefinition } = { message: "Data set downloaded successfully.\nDestination: %s" }, + /** + * Message indicating that the members of a data set were downloaded successfully + * @type {IMessageDefinition} + */ + memberDownloadedSuccessfully: { + message: "Member(s) downloaded successfully." + }, + + + /** + * Message indicating that the member was downloaded successfully + * @type {IMessageDefinition} + */ + memberDownloadedWithDestination: { + message: "Member(s) downloaded successfully.\nDestination: %s" + }, + /** * Message indicating that the uss file was downloaded successfully * @type {IMessageDefinition} @@ -215,6 +232,14 @@ export const ZosFilesMessages: { [key: string]: IMessageDefinition } = { message: "%d data set(s) were found matching pattern." }, + /** + * Message indicating that the data sets matching pattern were listed successfully + * @type {IMessageDefinition} + */ + membersMatchedPattern: { + message: "%d members(s) were found matching pattern." + }, + /** * Message indicating that file is uploaded to data set successfully * @type {IMessageDefinition} @@ -407,6 +432,14 @@ export const ZosFilesMessages: { [key: string]: IMessageDefinition } = { message: "There are no data sets that match the provided pattern(s)." }, + /** + * Message indicating that no members remain to be downloaded after the excluded ones were filtered out. + * @type {IMessageDefinition} + */ + noMembersMatchingPattern: { + message: "There are no members that match the provided pattern(s)." + }, + /** * Message indicating that no data sets remain to be downloaded after the excluded ones were filtered out. * @type {IMessageDefinition} @@ -415,6 +448,14 @@ export const ZosFilesMessages: { [key: string]: IMessageDefinition } = { message: "No data sets left after excluded pattern(s) were filtered out." }, + /** + * Message indicating that no data sets remain to be downloaded after the excluded ones were filtered out. + * @type {IMessageDefinition} + */ + noMembersInList: { + message: "No members left after excluded pattern(s) were filtered out." + }, + /** * Message indicating that some or all data sets failed to download * @type {IMessageDefinition} diff --git a/packages/zosfiles/src/methods/download/Download.ts b/packages/zosfiles/src/methods/download/Download.ts index 50091d156d..f81b435fac 100644 --- a/packages/zosfiles/src/methods/download/Download.ts +++ b/packages/zosfiles/src/methods/download/Download.ts @@ -210,8 +210,7 @@ export class Download { volume: options.volume, responseTimeout: options.responseTimeout }); - - const memberList: Array<{ member: string }> = response.apiResponse.items; + const memberList: Array<{ member: string }> = options.memberPatternResponse ?? response.apiResponse.items; if (memberList.length === 0) { return { success: false, @@ -298,7 +297,7 @@ export class Download { return { success: true, - commandResponse: util.format(ZosFilesMessages.datasetDownloadedWithDestination.message, baseDir), + commandResponse: util.format(ZosFilesMessages.memberDownloadedWithDestination.message, baseDir), apiResponse: response.apiResponse }; @@ -309,6 +308,7 @@ export class Download { } } + /** * Download a list of data sets to local files * diff --git a/packages/zosfiles/src/methods/download/doc/IDownloadOptions.ts b/packages/zosfiles/src/methods/download/doc/IDownloadOptions.ts index 076d7a5522..c397ab36dd 100644 --- a/packages/zosfiles/src/methods/download/doc/IDownloadOptions.ts +++ b/packages/zosfiles/src/methods/download/doc/IDownloadOptions.ts @@ -61,6 +61,11 @@ export interface IDownloadSingleOptions extends IGetOptions { * Optional stream to write the file contents */ stream?: Writable; + + /** + * An optional pattern for restricting the response list + */ + pattern?: string; /** * The ZosFilesAttributes instance describe upload attributes for the files and directories */ @@ -103,4 +108,15 @@ export interface IDownloadOptions extends Omit * Specifies whether hidden files whose names begin with a dot should be downloaded. */ includeHidden?: boolean; + + /** + * An optional pattern for restricting the response list + */ + pattern?: string; + + /** + * An optional response returned based on inputted patterns + */ + memberPatternResponse?: any; + } diff --git a/packages/zosfiles/src/methods/list/List.ts b/packages/zosfiles/src/methods/list/List.ts index f7dd71e1ed..d613d1e9d7 100644 --- a/packages/zosfiles/src/methods/list/List.ts +++ b/packages/zosfiles/src/methods/list/List.ts @@ -34,7 +34,6 @@ export class List { * @param {AbstractSession} session - z/OS MF connection info * @param {string} dataSetName - contains the data set name * @param {IListOptions} [options={}] - contains the options to be sent - * * @returns {Promise} A response indicating the outcome of the API * * @throws {ImperativeError} data set name must be set @@ -105,6 +104,65 @@ export class List { } } + /** + * List data set members that match a DSLEVEL pattern + * @param {AbstractSession} session z/OSMF connection info + * @param {string[]} patterns Data set patterns to include + * @param {IDsmListOptions} options Contains options for the z/OSMF request + * @returns {Promise} List of z/OSMF list responses for each data set + * + * @example + */ + public static async membersMatchingPattern(session: AbstractSession, dataSetName: string, patterns: string[], + options: IDsmListOptions = {}): Promise { + + ImperativeExpect.toNotBeNullOrUndefined(dataSetName, ZosFilesMessages.missingDatasetName.message); + ImperativeExpect.toNotBeEqual(dataSetName, "", ZosFilesMessages.missingDatasetName.message); + ImperativeExpect.toNotBeNullOrUndefined(patterns, ZosFilesMessages.missingPatterns.message); + patterns = patterns.filter(Boolean); + ImperativeExpect.toNotBeEqual(patterns.length, 0, ZosFilesMessages.missingPatterns.message); + const zosmfResponses: IZosmfListResponse[] = []; + + for(const pattern of patterns) { + const response = await List.allMembers(session, dataSetName, { pattern}); + zosmfResponses.push(...response.apiResponse.items); + } + + // Check if members matching pattern found + if (zosmfResponses.length === 0) { + return { + success: false, + commandResponse: ZosFilesMessages.noMembersMatchingPattern.message, + apiResponse: [] + }; + } + + // Exclude names of members + for (const pattern of options.excludePatterns || []) { + const response = await List.allMembers(session, dataSetName, {pattern}); + response.apiResponse.items.forEach((membersObj: IZosmfListResponse) => { + const responseIndex = zosmfResponses.findIndex(response=> response.member === membersObj.member); + if (responseIndex !== -1) { + zosmfResponses.splice(responseIndex, 1); + } + }); + } + + // Check if exclude pattern has left any members in the list + if (zosmfResponses.length === 0) { + return { + success: false, + commandResponse: ZosFilesMessages.noMembersInList.message, + apiResponse: [] + }; + } + + return { + success: true, + commandResponse: util.format(ZosFilesMessages.membersMatchedPattern.message, zosmfResponses.length), + apiResponse: zosmfResponses + }; + } /** * Retrieve all members from a data set name * diff --git a/packages/zosfiles/src/methods/list/doc/IListOptions.ts b/packages/zosfiles/src/methods/list/doc/IListOptions.ts index ba70889540..1947e0e773 100644 --- a/packages/zosfiles/src/methods/list/doc/IListOptions.ts +++ b/packages/zosfiles/src/methods/list/doc/IListOptions.ts @@ -13,7 +13,7 @@ import { ZosmfMigratedRecallOptions } from "../../../doc/types/ZosmfMigratedReca import { IZosFilesOptions } from "../../../doc/IZosFilesOptions"; /** - * This interface defines the options that can be sent into the dwanload data set function + * This interface defines the options that can be sent into the download data set & member function */ export interface IListOptions extends IZosFilesOptions { diff --git a/packages/zosfiles/src/methods/list/doc/IZosmfListResponse.ts b/packages/zosfiles/src/methods/list/doc/IZosmfListResponse.ts index df5d5fb138..7476930784 100644 --- a/packages/zosfiles/src/methods/list/doc/IZosmfListResponse.ts +++ b/packages/zosfiles/src/methods/list/doc/IZosmfListResponse.ts @@ -22,6 +22,11 @@ export interface IZosmfListResponse extends IZosFilesOptions { */ dsname: string; + /** + * The name of the member + */ + member?: string; + /** * The block size of the dataset */