az_cli_ext_dep: Add ability to unzip download (#792)

In some circumstances, the az extdep may be a compressed file that may need to decompressed and provided to the consumer.

edk2toolext/environment/extdeptypes/az_cli_universal_dependency.py

@@ -14,10 +14,12 @@
 import os
 import shutil
 from io import StringIO
+from pathlib import Path
 
 from edk2toollib.utility_functions import RemoveTree, RunCmd
 
 from edk2toolext.environment import shell_environment, version_aggregator
+from edk2toolext.environment.extdeptypes.web_dependency import WebDependency
 from edk2toolext.environment.external_dependency import ExternalDependency
 
 
@@ -31,6 +33,8 @@ class AzureCliUniversalDependency(ExternalDependency):
         project (str): <name of project for project scoped feed.  If missing assume organization scoped>
         name (str): name of artifact
         file-filter (str): <optional> filter for folders and files.
+        compression_type (str): <optional> Compression type used, if compressed.
+        internal_path (str): <optional> Path inside the compressed file, if the ext_dep is compressed
         pat_var (str): shell_var name for PAT for this ext_dep
 
     !!! tip
@@ -91,6 +95,11 @@ def __init__(self, descriptor: dict) -> None:
         self.feed = descriptor.get('feed')
         self.project = descriptor.get('project', None)
         self.file_filter = descriptor.get('file-filter', None)
+        self.compression_type = descriptor.get('compression_type', None)
+        self.internal_path = descriptor.get('internal_path', "/")
+        if self.internal_path:
+            self.internal_path = os.path.normpath(self.internal_path)
+            self.internal_path = self.internal_path.strip(os.path.sep)
         _pat_var = descriptor.get('pat_var', None)
         self._pat = None
 
@@ -162,14 +171,29 @@ def fetch(self) -> None:
         temp_directory = self.get_temp_dir()
         self._attempt_universal_install(temp_directory)
 
+        #
+        # if there is a compression type, the only thing downloaded must be the compressed
+        # file. If this is true, we will unpack it directly into the the contents
+        # directory and delete the temp directory.
+        #
+        if self.compression_type:
+            files = list(Path(temp_directory).iterdir())
+            if len(files) != 1:
+                raise Exception("Expected only 1 file in the downloaded directory")
+            tmp_file_path = files[0]
+            WebDependency.unpack(tmp_file_path, temp_directory, self.internal_path, self.compression_type)
+            tmp_file_path.unlink()
+
+            source_dir = os.path.join(temp_directory, self.internal_path)
+        else:
+            source_dir = temp_directory
         #
         # Next, copy the contents of the package to the
         # final resting place.
         #
-        source_dir = temp_directory
         shutil.copytree(source_dir, self.contents_dir)
 
-        RemoveTree(source_dir)
+        RemoveTree(temp_directory)
 
         #
         # Add a file to track the state of the dependency.

tests.unit/test_az_cli_universal_dependency.py

@@ -13,13 +13,14 @@
 #
 # SPDX-License-Identifier: BSD-2-Clause-Patent
 ##
-import os
-import unittest
 import logging
+import os
 import tempfile
+import unittest
+
 from edk2toolext.environment import environment_descriptor_files as EDF
-from edk2toolext.environment.extdeptypes.az_cli_universal_dependency import AzureCliUniversalDependency
 from edk2toolext.environment import version_aggregator
+from edk2toolext.environment.extdeptypes.az_cli_universal_dependency import AzureCliUniversalDependency
 from edk2toollib.utility_functions import RemoveTree
 
 test_dir = None
@@ -64,6 +65,36 @@
 }
 '''
 
+zip_json_template = '''
+{
+  "scope": "global",
+  "type": "az-universal",
+  "name": "hello-world-zip",
+  "source": "https://dev.azure.com/tianocore",
+  "project": "edk2-pytool-extensions",
+  "version": "%s",
+  "feed": "ext_dep_unit_test_feed",
+  "compression_type": "zip",
+  "internal_path": "hello-world-zip",
+  "pat_var": "PAT_FOR_UNIVERSAL_ORG_TIANOCORE"
+}
+'''
+
+zip_json_template2 = '''
+{
+  "scope": "global",
+  "type": "az-universal",
+  "name": "hello-world-zip",
+  "source": "https://dev.azure.com/tianocore",
+  "project": "edk2-pytool-extensions",
+  "version": "%s",
+  "feed": "ext_dep_unit_test_feed",
+  "compression_type": "zip",
+  "internal_path": "/",
+  "pat_var": "PAT_FOR_UNIVERSAL_ORG_TIANOCORE"
+}
+'''
+
 
 def prep_workspace():
     global test_dir
@@ -200,6 +231,61 @@ def test_download_bad_universal_dependency(self):
             ext_dep.fetch()
         self.assertFalse(ext_dep.verify())
 
+    @unittest.skipIf("PAT_FOR_UNIVERSAL_ORG_TIANOCORE" not in os.environ.keys(),
+                     "PAT not defined therefore universal packages tests will fail")
+    def test_download_and_unzip(self):
+        version = "0.0.1"
+        ext_dep_file_path = os.path.join(test_dir, "unit_test_ext_dep.json")
+        with open(ext_dep_file_path, "w+") as ext_dep_file:
+            ext_dep_file.write(zip_json_template % version)
+
+        ext_dep_descriptor = EDF.ExternDepDescriptor(ext_dep_file_path).descriptor_contents
+        ext_dep = AzureCliUniversalDependency(ext_dep_descriptor)
+        ext_dep.fetch()
+        self.assertTrue(ext_dep.verify())
+        self.assertEqual(ext_dep.version, version)
+
+        files = 0
+        folders = 0
+        for (_, dirs, file_names) in os.walk(ext_dep.contents_dir):
+            for file in file_names:
+                assert file in ['extdep_state.yaml', 'helloworld.txt']
+
+            files += len(file_names)
+            folders += len(dirs)
+
+        self.assertEqual(files, 2) # yaml file and moved files.
+        self.assertEqual(folders, 0)
+
+        ext_dep.clean()
+
+    @unittest.skipIf("PAT_FOR_UNIVERSAL_ORG_TIANOCORE" not in os.environ.keys(),
+                     "PAT not defined therefore universal packages tests will fail")
+    def test_download_and_unzip2(self):
+        version = "0.0.1"
+        ext_dep_file_path = os.path.join(test_dir, "unit_test_ext_dep.json")
+        with open(ext_dep_file_path, "w+") as ext_dep_file:
+            ext_dep_file.write(zip_json_template2 % version)
+
+        ext_dep_descriptor = EDF.ExternDepDescriptor(ext_dep_file_path).descriptor_contents
+        ext_dep = AzureCliUniversalDependency(ext_dep_descriptor)
+        ext_dep.fetch()
+        self.assertTrue(ext_dep.verify())
+        self.assertEqual(ext_dep.version, version)
+
+        files = 0
+        folders = 0
+        for (_, dirs, file_names) in os.walk(ext_dep.contents_dir):
+            for file in file_names:
+                assert file in ['extdep_state.yaml', 'helloworld.txt']
+            files += len(file_names)
+            folders += len(dirs)
+
+        self.assertEqual(files, 2) # yaml file and moved files.
+        self.assertEqual(folders, 1) # helloworld.txt is in a folder, because the internal path is "/"
+
+        ext_dep.clean()
+
     def test_az_tool_environment(self):
         AzureCliUniversalDependency.VerifyToolDependencies()