Enable support for loading files into nested parameter values (#9063)

.changes/next-release/feature-shorthand-60511.json

@@ -0,0 +1,5 @@
+{
+  "type": "feature",
+  "category": "shorthand",
+  "description": "Adds support to shorthand syntax for loading parameters from files via the ``@=`` assignment operator."
+}

awscli/paramfile.py

@@ -18,7 +18,7 @@
 from botocore.exceptions import ProfileNotFound
 from botocore.httpsession import URLLib3Session
 
-from awscli.argprocess import ParamError
+from awscli import argprocess
 from awscli.compat import compat_open
 
 logger = logging.getLogger(__name__)
@@ -183,7 +183,7 @@ def _check_for_uri_param(self, param, value):
         try:
             return get_paramfile(value, self._prefixes)
         except ResourceLoadingError as e:
-            raise ParamError(param.cli_name, str(e))
+            raise argprocess.ParamError(param.cli_name, str(e))
 
 
 def get_paramfile(path, cases):

awscli/shorthand.py

@@ -42,6 +42,7 @@
 import re
 import string
 
+from awscli.paramfile import LOCAL_PREFIX_MAP, get_paramfile
 from awscli.utils import is_document_type
 
 _EOF = object()
@@ -160,6 +161,7 @@ def parse(self, value):
         """
         self._input_value = value
         self._index = 0
+        self._should_resolve_paramfiles = False
         return self._parameter()
 
     def _parameter(self):
@@ -182,8 +184,15 @@ def _parameter(self):
         return params
 
     def _keyval(self):
-        # keyval = key "=" [values]
+        # keyval = key "=" [values] / key "@=" [file-optional-values]
+        # file-optional-values = file://value / fileb://value / value
         key = self._key()
+        self._should_resolve_paramfiles = False
+        try:
+            self._expect('@', consume_whitespace=True)
+            self._should_resolve_paramfiles = True
+        except ShorthandParseSyntaxError:
+            pass
         self._expect('=', consume_whitespace=True)
         values = self._values()
         return key, values
@@ -261,7 +270,8 @@ def _value(self):
         result = self._FIRST_VALUE.match(self._input_value[self._index :])
         if result is not None:
             consumed = self._consume_matched_regex(result)
-            return consumed.replace('\\,', ',').rstrip()
+            processed = consumed.replace('\\,', ',').rstrip()
+            return self._resolve_paramfiles(processed) if self._should_resolve_paramfiles else processed
         return ''
 
     def _explicit_list(self):
@@ -292,6 +302,12 @@ def _hash_literal(self):
         keyvals = {}
         while self._current() != '}':
             key = self._key()
+            self._should_resolve_paramfiles = False
+            try:
+                self._expect('@', consume_whitespace=True)
+                self._should_resolve_paramfiles = True
+            except ShorthandParseSyntaxError:
+                pass
             self._expect('=', consume_whitespace=True)
             v = self._explicit_values()
             self._consume_whitespace()
@@ -314,7 +330,8 @@ def _single_quoted_value(self):
         # single-quoted-value = %x27 *(val-escaped-single) %x27
         # val-escaped-single  = %x20-26 / %x28-7F / escaped-escape /
         #                       (escape single-quote)
-        return self._consume_quoted(self._SINGLE_QUOTED, escaped_char="'")
+        processed = self._consume_quoted(self._SINGLE_QUOTED, escaped_char="'")
+        return self._resolve_paramfiles(processed) if self._should_resolve_paramfiles else processed
 
     def _consume_quoted(self, regex, escaped_char=None):
         value = self._must_consume_regex(regex)[1:-1]
@@ -324,7 +341,8 @@ def _consume_quoted(self, regex, escaped_char=None):
         return value
 
     def _double_quoted_value(self):
-        return self._consume_quoted(self._DOUBLE_QUOTED, escaped_char='"')
+        processed = self._consume_quoted(self._DOUBLE_QUOTED, escaped_char='"')
+        return self._resolve_paramfiles(processed) if self._should_resolve_paramfiles else processed
 
     def _second_value(self):
         if self._current() == "'":
@@ -333,7 +351,13 @@ def _second_value(self):
             return self._double_quoted_value()
         else:
             consumed = self._must_consume_regex(self._SECOND_VALUE)
-            return consumed.replace('\\,', ',').rstrip()
+            processed = consumed.replace('\\,', ',').rstrip()
+            return self._resolve_paramfiles(processed) if self._should_resolve_paramfiles else processed
+
+    def _resolve_paramfiles(self, val):
+        if (paramfile := get_paramfile(val, LOCAL_PREFIX_MAP)) is not None:
+            return paramfile
+        return val
 
     def _expect(self, char, consume_whitespace=False):
         if consume_whitespace:

tests/unit/test_paramfile.py

@@ -13,9 +13,13 @@
 from awscli.testutils import mock, unittest, FileCreator
 from awscli.testutils import skip_if_windows
 
-from awscli.paramfile import get_paramfile, ResourceLoadingError
-from awscli.paramfile import LOCAL_PREFIX_MAP, REMOTE_PREFIX_MAP
-from awscli.paramfile import register_uri_param_handler
+from awscli.paramfile import (
+    get_paramfile,
+    ResourceLoadingError,
+    LOCAL_PREFIX_MAP,
+    REMOTE_PREFIX_MAP,
+    register_uri_param_handler,
+)
 from botocore.session import Session
 from botocore.exceptions import ProfileNotFound
 

tests/unit/test_shorthand.py

@@ -10,15 +10,17 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
+from unittest.mock import patch
+
 import pytest
 import signal
 
+import awscli.paramfile
 from awscli import shorthand
-from awscli.testutils import unittest, skip_if_windows
+from awscli.testutils import skip_if_windows, unittest
 
 from botocore import model
 
-
 PARSING_TEST_CASES = (
     # Key val pairs with scalar value.
     ('foo=bar', {'foo': 'bar'}),
@@ -128,6 +130,24 @@
         'Name=[{foo=[a,b]}, {bar=[c,d]}]',
         {'Name': [{'foo': ['a', 'b']}, {'bar': ['c', 'd']}]}
     ),
+    # key-value pairs using @= syntax
+    ('foo@=bar', {'foo': 'bar'}),
+    ('foo@=bar,baz@=qux', {'foo': 'bar', 'baz': 'qux'}),
+    ('foo@=,bar@=', {'foo': '', 'bar': ''}),
+    (u'foo@=\u2713,\u2713', {'foo': [u'\u2713', u'\u2713']}),
+    ('foo@=a,b,bar=c,d', {'foo': ['a', 'b'], 'bar': ['c', 'd']}),
+    ('foo=a,b@=with space', {'foo': 'a', 'b': 'with space'}),
+    ('foo=a,b@=with trailing space  ', {'foo': 'a', 'b': 'with trailing space'}),
+    ('aws:service:region:124:foo/bar@=baz', {'aws:service:region:124:foo/bar': 'baz'}),
+    ('foo=[a,b],bar@=[c,d]', {'foo': ['a', 'b'], 'bar': ['c', 'd']}),
+    ('foo  @=  [ a , b  , c  ]', {'foo': ['a', 'b', 'c']}),
+    ('A=b,\nC@=d,\nE@=f\n', {'A': 'b', 'C': 'd', 'E': 'f'}),
+    ('Bar@=baz,Name={foo@=bar}', {'Bar': 'baz', 'Name': {'foo': 'bar'}}),
+    ('Name=[{foo@=bar}, {baz=qux}]', {'Name': [{'foo': 'bar'}, {'baz': 'qux'}]}),
+    (
+        'Name=[{foo@=[a,b]}, {bar=[c,d]}]',
+        {'Name': [{'foo': ['a', 'b']}, {'bar': ['c', 'd']}]}
+    ),
 )
 
 
@@ -136,6 +156,7 @@
         'foo',
         # Missing closing quotes
         'foo="bar',
+        '"foo=bar',
         "foo='bar",
         "foo=[bar",
         "foo={bar",
@@ -182,6 +203,38 @@ def test_parse(data, expected):
     actual = shorthand.ShorthandParser().parse(data)
     assert actual == expected
 
+class TestShorthandParserParamFile:
+    @patch('awscli.paramfile.compat_open')
+    @pytest.mark.parametrize(
+        'file_contents, data, expected',
+        (
+            ('file-contents123', 'Foo@=file://foo,Bar={Baz@=file://foo}', {'Foo': 'file-contents123', 'Bar': {'Baz': 'file-contents123'}}),
+            (b'file-contents123', 'Foo@=fileb://foo,Bar={Baz@=fileb://foo}', {'Foo': b'file-contents123', 'Bar': {'Baz': b'file-contents123'}}),
+            ('file-contents123', 'Bar@={Baz=file://foo}', {'Bar': {'Baz': 'file://foo'}}),
+            ('file-contents123', 'Foo@=foo,Bar={Baz@=foo}', {'Foo': 'foo', 'Bar': {'Baz': 'foo'}})
+        )
+    )
+    def test_paramfile(self, mock_compat_open, file_contents, data, expected):
+        mock_compat_open.return_value.__enter__.return_value.read.return_value = file_contents
+        result = shorthand.ShorthandParser().parse(data)
+        assert result == expected
+
+    @patch('awscli.paramfile.compat_open')
+    def test_paramfile_list(self, mock_compat_open):
+        f1_contents = 'file-contents123'
+        f2_contents = 'contents2'
+        mock_compat_open.return_value.__enter__.return_value.read.side_effect = [f1_contents, f2_contents]
+        result = shorthand.ShorthandParser().parse(
+            f'Foo@=[a, file://foo1, file://foo2]'
+        )
+        assert result == {'Foo': ['a', f1_contents, f2_contents]}
+
+    def test_paramfile_does_not_exist_error(self, capsys):
+        with pytest.raises(awscli.paramfile.ResourceLoadingError):
+            shorthand.ShorthandParser().parse('Foo@=file://fakefile.txt')
+            captured = capsys.readouterr()
+            assert "No such file or directory: 'fakefile.txt" in captured.err
+
 
 class TestModelVisitor(unittest.TestCase):
     def test_promote_to_list_of_ints(self):