diff --git a/.changes/next-release/feature-shorthand-60511.json b/.changes/next-release/feature-shorthand-60511.json new file mode 100644 index 000000000000..e43d421ec448 --- /dev/null +++ b/.changes/next-release/feature-shorthand-60511.json @@ -0,0 +1,5 @@ +{ + "type": "feature", + "category": "shorthand", + "description": "Adds support to shorthand syntax for loading parameters from files via the ``@=`` assignment operator." +} diff --git a/awscli/paramfile.py b/awscli/paramfile.py index 5f6ff89cbc44..14cd7fc49330 100644 --- a/awscli/paramfile.py +++ b/awscli/paramfile.py @@ -18,7 +18,7 @@ from botocore.exceptions import ProfileNotFound from botocore.httpsession import URLLib3Session -from awscli.argprocess import ParamError +from awscli import argprocess from awscli.compat import compat_open logger = logging.getLogger(__name__) @@ -183,7 +183,7 @@ def _check_for_uri_param(self, param, value): try: return get_paramfile(value, self._prefixes) except ResourceLoadingError as e: - raise ParamError(param.cli_name, str(e)) + raise argprocess.ParamError(param.cli_name, str(e)) def get_paramfile(path, cases): diff --git a/awscli/shorthand.py b/awscli/shorthand.py index 255598ff6862..18ec091f717a 100644 --- a/awscli/shorthand.py +++ b/awscli/shorthand.py @@ -42,6 +42,7 @@ import re import string +from awscli.paramfile import LOCAL_PREFIX_MAP, get_paramfile from awscli.utils import is_document_type _EOF = object() @@ -160,6 +161,7 @@ def parse(self, value): """ self._input_value = value self._index = 0 + self._should_resolve_paramfiles = False return self._parameter() def _parameter(self): @@ -182,8 +184,15 @@ def _parameter(self): return params def _keyval(self): - # keyval = key "=" [values] + # keyval = key "=" [values] / key "@=" [file-optional-values] + # file-optional-values = file://value / fileb://value / value key = self._key() + self._should_resolve_paramfiles = False + try: + self._expect('@', consume_whitespace=True) + self._should_resolve_paramfiles = True + except ShorthandParseSyntaxError: + pass self._expect('=', consume_whitespace=True) values = self._values() return key, values @@ -261,7 +270,8 @@ def _value(self): result = self._FIRST_VALUE.match(self._input_value[self._index :]) if result is not None: consumed = self._consume_matched_regex(result) - return consumed.replace('\\,', ',').rstrip() + processed = consumed.replace('\\,', ',').rstrip() + return self._resolve_paramfiles(processed) if self._should_resolve_paramfiles else processed return '' def _explicit_list(self): @@ -292,6 +302,12 @@ def _hash_literal(self): keyvals = {} while self._current() != '}': key = self._key() + self._should_resolve_paramfiles = False + try: + self._expect('@', consume_whitespace=True) + self._should_resolve_paramfiles = True + except ShorthandParseSyntaxError: + pass self._expect('=', consume_whitespace=True) v = self._explicit_values() self._consume_whitespace() @@ -314,7 +330,8 @@ def _single_quoted_value(self): # single-quoted-value = %x27 *(val-escaped-single) %x27 # val-escaped-single = %x20-26 / %x28-7F / escaped-escape / # (escape single-quote) - return self._consume_quoted(self._SINGLE_QUOTED, escaped_char="'") + processed = self._consume_quoted(self._SINGLE_QUOTED, escaped_char="'") + return self._resolve_paramfiles(processed) if self._should_resolve_paramfiles else processed def _consume_quoted(self, regex, escaped_char=None): value = self._must_consume_regex(regex)[1:-1] @@ -324,7 +341,8 @@ def _consume_quoted(self, regex, escaped_char=None): return value def _double_quoted_value(self): - return self._consume_quoted(self._DOUBLE_QUOTED, escaped_char='"') + processed = self._consume_quoted(self._DOUBLE_QUOTED, escaped_char='"') + return self._resolve_paramfiles(processed) if self._should_resolve_paramfiles else processed def _second_value(self): if self._current() == "'": @@ -333,7 +351,13 @@ def _second_value(self): return self._double_quoted_value() else: consumed = self._must_consume_regex(self._SECOND_VALUE) - return consumed.replace('\\,', ',').rstrip() + processed = consumed.replace('\\,', ',').rstrip() + return self._resolve_paramfiles(processed) if self._should_resolve_paramfiles else processed + + def _resolve_paramfiles(self, val): + if (paramfile := get_paramfile(val, LOCAL_PREFIX_MAP)) is not None: + return paramfile + return val def _expect(self, char, consume_whitespace=False): if consume_whitespace: diff --git a/tests/unit/test_paramfile.py b/tests/unit/test_paramfile.py index 2745ca755e84..1b5d6688fd46 100644 --- a/tests/unit/test_paramfile.py +++ b/tests/unit/test_paramfile.py @@ -13,9 +13,13 @@ from awscli.testutils import mock, unittest, FileCreator from awscli.testutils import skip_if_windows -from awscli.paramfile import get_paramfile, ResourceLoadingError -from awscli.paramfile import LOCAL_PREFIX_MAP, REMOTE_PREFIX_MAP -from awscli.paramfile import register_uri_param_handler +from awscli.paramfile import ( + get_paramfile, + ResourceLoadingError, + LOCAL_PREFIX_MAP, + REMOTE_PREFIX_MAP, + register_uri_param_handler, +) from botocore.session import Session from botocore.exceptions import ProfileNotFound diff --git a/tests/unit/test_shorthand.py b/tests/unit/test_shorthand.py index 21a95f06d684..8d748d423a0b 100644 --- a/tests/unit/test_shorthand.py +++ b/tests/unit/test_shorthand.py @@ -10,15 +10,17 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +from unittest.mock import patch + import pytest import signal +import awscli.paramfile from awscli import shorthand -from awscli.testutils import unittest, skip_if_windows +from awscli.testutils import skip_if_windows, unittest from botocore import model - PARSING_TEST_CASES = ( # Key val pairs with scalar value. ('foo=bar', {'foo': 'bar'}), @@ -128,6 +130,24 @@ 'Name=[{foo=[a,b]}, {bar=[c,d]}]', {'Name': [{'foo': ['a', 'b']}, {'bar': ['c', 'd']}]} ), + # key-value pairs using @= syntax + ('foo@=bar', {'foo': 'bar'}), + ('foo@=bar,baz@=qux', {'foo': 'bar', 'baz': 'qux'}), + ('foo@=,bar@=', {'foo': '', 'bar': ''}), + (u'foo@=\u2713,\u2713', {'foo': [u'\u2713', u'\u2713']}), + ('foo@=a,b,bar=c,d', {'foo': ['a', 'b'], 'bar': ['c', 'd']}), + ('foo=a,b@=with space', {'foo': 'a', 'b': 'with space'}), + ('foo=a,b@=with trailing space ', {'foo': 'a', 'b': 'with trailing space'}), + ('aws:service:region:124:foo/bar@=baz', {'aws:service:region:124:foo/bar': 'baz'}), + ('foo=[a,b],bar@=[c,d]', {'foo': ['a', 'b'], 'bar': ['c', 'd']}), + ('foo @= [ a , b , c ]', {'foo': ['a', 'b', 'c']}), + ('A=b,\nC@=d,\nE@=f\n', {'A': 'b', 'C': 'd', 'E': 'f'}), + ('Bar@=baz,Name={foo@=bar}', {'Bar': 'baz', 'Name': {'foo': 'bar'}}), + ('Name=[{foo@=bar}, {baz=qux}]', {'Name': [{'foo': 'bar'}, {'baz': 'qux'}]}), + ( + 'Name=[{foo@=[a,b]}, {bar=[c,d]}]', + {'Name': [{'foo': ['a', 'b']}, {'bar': ['c', 'd']}]} + ), ) @@ -136,6 +156,7 @@ 'foo', # Missing closing quotes 'foo="bar', + '"foo=bar', "foo='bar", "foo=[bar", "foo={bar", @@ -182,6 +203,38 @@ def test_parse(data, expected): actual = shorthand.ShorthandParser().parse(data) assert actual == expected +class TestShorthandParserParamFile: + @patch('awscli.paramfile.compat_open') + @pytest.mark.parametrize( + 'file_contents, data, expected', + ( + ('file-contents123', 'Foo@=file://foo,Bar={Baz@=file://foo}', {'Foo': 'file-contents123', 'Bar': {'Baz': 'file-contents123'}}), + (b'file-contents123', 'Foo@=fileb://foo,Bar={Baz@=fileb://foo}', {'Foo': b'file-contents123', 'Bar': {'Baz': b'file-contents123'}}), + ('file-contents123', 'Bar@={Baz=file://foo}', {'Bar': {'Baz': 'file://foo'}}), + ('file-contents123', 'Foo@=foo,Bar={Baz@=foo}', {'Foo': 'foo', 'Bar': {'Baz': 'foo'}}) + ) + ) + def test_paramfile(self, mock_compat_open, file_contents, data, expected): + mock_compat_open.return_value.__enter__.return_value.read.return_value = file_contents + result = shorthand.ShorthandParser().parse(data) + assert result == expected + + @patch('awscli.paramfile.compat_open') + def test_paramfile_list(self, mock_compat_open): + f1_contents = 'file-contents123' + f2_contents = 'contents2' + mock_compat_open.return_value.__enter__.return_value.read.side_effect = [f1_contents, f2_contents] + result = shorthand.ShorthandParser().parse( + f'Foo@=[a, file://foo1, file://foo2]' + ) + assert result == {'Foo': ['a', f1_contents, f2_contents]} + + def test_paramfile_does_not_exist_error(self, capsys): + with pytest.raises(awscli.paramfile.ResourceLoadingError): + shorthand.ShorthandParser().parse('Foo@=file://fakefile.txt') + captured = capsys.readouterr() + assert "No such file or directory: 'fakefile.txt" in captured.err + class TestModelVisitor(unittest.TestCase): def test_promote_to_list_of_ints(self):