From 572685cdb1e954820c68ec83611dbec5d3dbfb65 Mon Sep 17 00:00:00 2001 From: cnelson Date: Fri, 16 Oct 2015 13:56:38 -0700 Subject: [PATCH 1/3] made unit tests compatible with google-api-python-client 1.4.2 --- fleet/v1/client.py | 3 +- fleet/v1/tests/fixtures/empty_response.txt | 0 fleet/v1/tests/test_client.py | 4 +- fleet/v1/tests/test_unit.py | 611 ++++++++++----------- requirements.txt | 2 +- 5 files changed, 308 insertions(+), 312 deletions(-) create mode 100644 fleet/v1/tests/fixtures/empty_response.txt diff --git a/fleet/v1/client.py b/fleet/v1/client.py index 38e74d0..db249b7 100644 --- a/fleet/v1/client.py +++ b/fleet/v1/client.py @@ -60,6 +60,7 @@ def __init__(self, endpoint, http=None): self._service = build( self._API, self._VERSION, + cache_discovery=False, discoveryServiceUrl=discovery_url, http=self._http ) @@ -123,7 +124,7 @@ def _single_request(self, method, *args, **kwargs): try: return _method.execute(http=self._http) except googleapiclient.errors.HttpError as exc: - response = json.loads(exc.content)['error'] + response = json.loads(exc.content.decode('utf-8'))['error'] raise APIError(code=response['code'], message=response['message'], http_error=exc) diff --git a/fleet/v1/tests/fixtures/empty_response.txt b/fleet/v1/tests/fixtures/empty_response.txt new file mode 100644 index 0000000..e69de29 diff --git a/fleet/v1/tests/test_client.py b/fleet/v1/tests/test_client.py index a59e9a6..461aa8d 100644 --- a/fleet/v1/tests/test_client.py +++ b/fleet/v1/tests/test_client.py @@ -36,8 +36,8 @@ def test_init_endpoint_active_but_invalid(self): def test(): http = HttpMock( - None, - {'status': '404'} + os.path.join(self._BASE_DIR, 'fixtures/empty_response.txt'), + {'status': '404'}, ) Client(self.endpoint, http=http) diff --git a/fleet/v1/tests/test_unit.py b/fleet/v1/tests/test_unit.py index 47caa19..b7f2f9f 100644 --- a/fleet/v1/tests/test_unit.py +++ b/fleet/v1/tests/test_unit.py @@ -1,308 +1,303 @@ -import unittest - -import os - -from ..objects import Unit -from ..client import Client -from ..errors import APIError - -from apiclient.http import HttpMockSequence - - -class TestUnit(unittest.TestCase): - def setUp(self): - self._BASE_DIR = os.path.dirname(os.path.abspath(__file__)) - - def test_init_conflict(self): - """If you specified data, you cannot specified, options, from_file, or from_string""" - - def test(): - Unit(data=True, options=True) - - def test2(): - Unit(data=True, from_file=True) - - def test3(): - Unit(data=True, from_string=True) - - self.assertRaises(ValueError, test) - self.assertRaises(ValueError, test2) - self.assertRaises(ValueError, test3) - - def test_init_multiple(self): - """If data is not specified, you cannot specify more than one of options, from_file, from_string""" - - def test(): - Unit(options=True, from_string=True) - - def test2(): - Unit(options=True, from_file=True) - - def test3(): - Unit(options=True, from_string=True, from_file=True) - - self.assertRaises(ValueError, test) - self.assertRaises(ValueError, test2) - self.assertRaises(ValueError, test3) - - def test_blank(self): - """A blank unit should come with a sane default state""" - - unit = Unit() - - assert unit.desiredState == 'launched' - assert unit.options == [] - - def test_from_file_good(self): - """Passing in a file should cause it to be loaded""" - - # this should match the contents of the fixture below - test_options = [{'section': 'Service', 'name': 'ExecStart', 'value': '/usr/bin/sleep 1d'}] - - unit = Unit(from_file=os.path.join(self._BASE_DIR, 'fixtures/test.service')) - - assert unit.options == test_options - - def test_from_file_bad(self): - """IOError should be raised on non-existant files""" - - def test(): - Unit(from_file=os.path.join(self._BASE_DIR, 'fixtures/this-file-should-not-exist')) - - self.assertRaises(IOError, test) - - def test_from_string_good(self): - """Loading from a good string works""" - - # these test values should be equal - test_options = [{'section': 'Service', 'name': 'ExecStart', 'value': '/usr/bin/sleep 1d'}] - test_string = "[Service]\n# comments should be ignored\nExecStart=/usr/bin/sleep 1d" - - unit = Unit(from_string=test_string) - - assert unit.options == test_options - - def test_from_string_bad(self): - """ValueError should be raised for invalid unit string""" - - def test(): - Unit(from_string="SomeKey=WithNoSection") - - def test2(): - Unit(from_string="[Section]\nSome Line with No Equals") - - self.assertRaises(ValueError, test) - self.assertRaises(ValueError, test2) - - def test_options_no_desired_state(self): - """Setting options explicitly works""" - test_options = [{'section': 'Service', 'name': 'ExecStart', 'value': '/usr/bin/sleep 1d'}] - - unit = Unit(options=test_options) - - assert unit.options == test_options - - def test_options_desired_state(self): - """Setting desiredState in constructor works""" - test_options = [{'section': 'Service', 'name': 'ExecStart', 'value': '/usr/bin/sleep 1d'}] - - unit = Unit(desired_state='inactive', options=test_options) - - assert unit.options == test_options - - assert unit.desiredState == 'inactive' - - def test_repr(self): - """repr shows entire object""" - test_options = [{'section': 'Service', 'name': 'ExecStart', 'value': '/usr/bin/sleep 1d'}] - - unit = Unit(options=test_options) - - assert str(test_options) in repr(unit) - - def test_str_roundtrip(self): - """Calling str() on a unit, should generate a systemd unit""" - - test_string = "[Service]\nExecStart=/usr/bin/sleep 1d" - - unit = Unit(from_string=test_string) - - assert test_string == str(unit) - - def test_is_live(self): - """A unit is live if it as a client and data with a name key""" - - unit = Unit() - - assert unit._is_live() is False - - unit = Unit(client=True, data=None) - - assert unit._is_live() is False - - unit = Unit(client=True, data=None) - - assert unit._is_live() is False - - unit = Unit(client=True, data={'name': 'test'}) - - assert unit._is_live() - - def test_add_option_dead(self): - """We should be able to add options to non-live units""" - test_options = [{'section': 'Service', 'name': 'ExecStart', 'value': '/usr/bin/sleep 1d'}] - - unit = Unit() - - unit.add_option('Service', 'ExecStart', '/usr/bin/sleep 1d') - - assert unit.options == test_options - - def test_add_option_live(self): - """We should not be able to add options to non-live units""" - - def test(): - unit = Unit(client=True, data={'name': 'test'}) - unit.add_option('Service', 'ExecStart', '/usr/bin/sleep 1d') - - self.assertRaises(RuntimeError, test) - - def test_remove_option_live(self): - """We should not be able to add options to non-live units""" - - def test(): - unit = Unit(client=True, data={'name': 'test'}) - unit.remove_option('Service', 'ExecStart', '/usr/bin/sleep 1d') - - self.assertRaises(RuntimeError, test) - - def test_remove_no_option(self): - """Should return false when the item removed could not be found""" - - unit = Unit() - - assert unit.remove_option('Service', 'foo') is False - assert unit.remove_option('Service', 'foo', 'bar') is False - - def test_remove_single_option(self): - """We can remove a single option by value""" - test_options = [ - {'section': 'Service', 'name': 'ExecStart', 'value': '/usr/bin/sleep 1d'}, - {'section': 'Service', 'name': 'ExecStartPre', 'value': '/bin/true'} - ] - - unit = Unit() - - unit.add_option('Service', 'ExecStart', '/usr/bin/sleep 1d') - unit.add_option('Service', 'ExecStartPre', '/bin/true') - unit.add_option('Service', 'ExecStartPre', '/bin/false') - - assert unit.remove_option('Service', 'ExecStartPre', '/bin/false') - - assert test_options == unit.options - - def test_remove_all_option(self): - """We can remove all options by name""" - test_options = [ - {'section': 'Service', 'name': 'ExecStart', 'value': '/usr/bin/sleep 1d'} - ] - - unit = Unit() - - unit.add_option('Service', 'ExecStart', '/usr/bin/sleep 1d') - unit.add_option('Service', 'ExecStartPre', '/bin/true') - unit.add_option('Service', 'ExecStartPre', '/bin/false') - - assert unit.remove_option('Service', 'ExecStartPre') - - assert test_options == unit.options - - def test_destroy_not_live(self): - """Non live units cannot be destroyed""" - - def test(): - unit = Unit() - unit.destroy() - - self.assertRaises(RuntimeError, test) - - def test_destroy_good(self): - """We can destroy live units""" - - fh = open(os.path.join(self._BASE_DIR, 'fixtures/fleet_v1.json')) - discovery = fh.read() - fh.close() - - http = HttpMockSequence([ - ({'status': '200'}, discovery), - ({'status': '204'}, None) - ]) - - client = Client('http://198.51.100.23:9160', http=http) - - unit = Unit(client=client, data={'name': 'test.service'}) - - assert unit.destroy() - - def test_destroy_bad(self): - """APIError is raised when non-existent units are destroyed""" - - fh = open(os.path.join(self._BASE_DIR, 'fixtures/fleet_v1.json')) - discovery = fh.read() - fh.close() - - def test(): - http = HttpMockSequence([ - ({'status': '200'}, discovery), - ({'status': '404'}, '{"error":{"code":404,"message":"unit does not exist"}}') - ]) - - client = Client('http://198.51.100.23:9160', http=http) - - unit = Unit(client=client, data={'name': 'test.service'}) - unit.destroy() - - self.assertRaises(APIError, test) - - def test_desired_state_bad(self): - """ValueError is raised when bad values are provided for desired state""" - def test(): - unit = Unit() - unit.set_desired_state('invalid-state') - - self.assertRaises(ValueError, test) - - def test_desired_state_good_dead(self): - """We can set desired state on non-live objects""" - unit = Unit() - - assert unit.desiredState == 'launched' - - assert unit.set_desired_state('inactive') == 'inactive' - - assert unit.desiredState == 'inactive' - - def test_desired_state_good_live(self): - """We can set desired state on live objects""" - - fh = open(os.path.join(self._BASE_DIR, 'fixtures/fleet_v1.json')) - discovery = fh.read() - fh.close() - - http = HttpMockSequence([ - ({'status': '200'}, discovery), - ({'status': '204'}, None), - ({'status': '200'}, '{"currentState":"inactive","desiredState":"inactive","machineID":' - '"2901a44df0834bef935e24a0ddddcc23","name":"test.service","options"' - ':[{"name":"ExecStart","section":"Service","value":"/usr/bin/sleep 1d"}]}') - ]) - - client = Client('http://198.51.100.23:9160', http=http) - - unit = Unit(client=client, data={'name': 'test.service', 'desiredState': 'launched'}) - - assert unit.desiredState == 'launched' - - assert unit.set_desired_state('inactive') == 'inactive' - - assert unit.desiredState == 'inactive' +import unittest + +import os + +from ..objects import Unit +from ..client import Client +from ..errors import APIError + +from apiclient.http import HttpMockSequence + + +class TestUnit(unittest.TestCase): + def setUp(self): + self._BASE_DIR = os.path.dirname(os.path.abspath(__file__)) + + def _load_disccovery_fixture(self): + fh = open(os.path.join(self._BASE_DIR, 'fixtures/fleet_v1.json')) + discovery = fh.read() + fh.close() + + return discovery + + def test_init_conflict(self): + """If you specified data, you cannot specified, options, from_file, or from_string""" + + def test(): + Unit(data=True, options=True) + + def test2(): + Unit(data=True, from_file=True) + + def test3(): + Unit(data=True, from_string=True) + + self.assertRaises(ValueError, test) + self.assertRaises(ValueError, test2) + self.assertRaises(ValueError, test3) + + def test_init_multiple(self): + """If data is not specified, you cannot specify more than one of options, from_file, from_string""" + + def test(): + Unit(options=True, from_string=True) + + def test2(): + Unit(options=True, from_file=True) + + def test3(): + Unit(options=True, from_string=True, from_file=True) + + self.assertRaises(ValueError, test) + self.assertRaises(ValueError, test2) + self.assertRaises(ValueError, test3) + + def test_blank(self): + """A blank unit should come with a sane default state""" + + unit = Unit() + + assert unit.desiredState == 'launched' + assert unit.options == [] + + def test_from_file_good(self): + """Passing in a file should cause it to be loaded""" + + # this should match the contents of the fixture below + test_options = [{'section': 'Service', 'name': 'ExecStart', 'value': '/usr/bin/sleep 1d'}] + + unit = Unit(from_file=os.path.join(self._BASE_DIR, 'fixtures/test.service')) + + assert unit.options == test_options + + def test_from_file_bad(self): + """IOError should be raised on non-existant files""" + + def test(): + Unit(from_file=os.path.join(self._BASE_DIR, 'fixtures/this-file-should-not-exist')) + + self.assertRaises(IOError, test) + + def test_from_string_good(self): + """Loading from a good string works""" + + # these test values should be equal + test_options = [{'section': 'Service', 'name': 'ExecStart', 'value': '/usr/bin/sleep 1d'}] + test_string = "[Service]\n# comments should be ignored\nExecStart=/usr/bin/sleep 1d" + + unit = Unit(from_string=test_string) + + assert unit.options == test_options + + def test_from_string_bad(self): + """ValueError should be raised for invalid unit string""" + + def test(): + Unit(from_string="SomeKey=WithNoSection") + + def test2(): + Unit(from_string="[Section]\nSome Line with No Equals") + + self.assertRaises(ValueError, test) + self.assertRaises(ValueError, test2) + + def test_options_no_desired_state(self): + """Setting options explicitly works""" + test_options = [{'section': 'Service', 'name': 'ExecStart', 'value': '/usr/bin/sleep 1d'}] + + unit = Unit(options=test_options) + + assert unit.options == test_options + + def test_options_desired_state(self): + """Setting desiredState in constructor works""" + test_options = [{'section': 'Service', 'name': 'ExecStart', 'value': '/usr/bin/sleep 1d'}] + + unit = Unit(desired_state='inactive', options=test_options) + + assert unit.options == test_options + + assert unit.desiredState == 'inactive' + + def test_repr(self): + """repr shows entire object""" + test_options = [{'section': 'Service', 'name': 'ExecStart', 'value': '/usr/bin/sleep 1d'}] + + unit = Unit(options=test_options) + + assert str(test_options) in repr(unit) + + def test_str_roundtrip(self): + """Calling str() on a unit, should generate a systemd unit""" + + test_string = "[Service]\nExecStart=/usr/bin/sleep 1d" + + unit = Unit(from_string=test_string) + + assert test_string == str(unit) + + def test_is_live(self): + """A unit is live if it as a client and data with a name key""" + + unit = Unit() + + assert unit._is_live() is False + + unit = Unit(client=True, data=None) + + assert unit._is_live() is False + + unit = Unit(client=True, data=None) + + assert unit._is_live() is False + + unit = Unit(client=True, data={'name': 'test'}) + + assert unit._is_live() + + def test_add_option_dead(self): + """We should be able to add options to non-live units""" + test_options = [{'section': 'Service', 'name': 'ExecStart', 'value': '/usr/bin/sleep 1d'}] + + unit = Unit() + + unit.add_option('Service', 'ExecStart', '/usr/bin/sleep 1d') + + assert unit.options == test_options + + def test_add_option_live(self): + """We should not be able to add options to non-live units""" + + def test(): + unit = Unit(client=True, data={'name': 'test'}) + unit.add_option('Service', 'ExecStart', '/usr/bin/sleep 1d') + + self.assertRaises(RuntimeError, test) + + def test_remove_option_live(self): + """We should not be able to add options to non-live units""" + + def test(): + unit = Unit(client=True, data={'name': 'test'}) + unit.remove_option('Service', 'ExecStart', '/usr/bin/sleep 1d') + + self.assertRaises(RuntimeError, test) + + def test_remove_no_option(self): + """Should return false when the item removed could not be found""" + + unit = Unit() + + assert unit.remove_option('Service', 'foo') is False + assert unit.remove_option('Service', 'foo', 'bar') is False + + def test_remove_single_option(self): + """We can remove a single option by value""" + test_options = [ + {'section': 'Service', 'name': 'ExecStart', 'value': '/usr/bin/sleep 1d'}, + {'section': 'Service', 'name': 'ExecStartPre', 'value': '/bin/true'} + ] + + unit = Unit() + + unit.add_option('Service', 'ExecStart', '/usr/bin/sleep 1d') + unit.add_option('Service', 'ExecStartPre', '/bin/true') + unit.add_option('Service', 'ExecStartPre', '/bin/false') + + assert unit.remove_option('Service', 'ExecStartPre', '/bin/false') + + assert test_options == unit.options + + def test_remove_all_option(self): + """We can remove all options by name""" + test_options = [ + {'section': 'Service', 'name': 'ExecStart', 'value': '/usr/bin/sleep 1d'} + ] + + unit = Unit() + + unit.add_option('Service', 'ExecStart', '/usr/bin/sleep 1d') + unit.add_option('Service', 'ExecStartPre', '/bin/true') + unit.add_option('Service', 'ExecStartPre', '/bin/false') + + assert unit.remove_option('Service', 'ExecStartPre') + + assert test_options == unit.options + + def test_destroy_not_live(self): + """Non live units cannot be destroyed""" + + def test(): + unit = Unit() + unit.destroy() + + self.assertRaises(RuntimeError, test) + + def test_destroy_good(self): + """We can destroy live units""" + + http = HttpMockSequence([ + ({'status': '200'}, self._load_disccovery_fixture()), + ({'status': '204'}, None) + ]) + + client = Client('http://198.51.100.23:9160', http=http) + + unit = Unit(client=client, data={'name': 'test.service'}) + + assert unit.destroy() + + def test_destroy_bad(self): + """APIError is raised when non-existent units are destroyed""" + + def test(): + http = HttpMockSequence([ + ({'status': '200'}, self._load_disccovery_fixture()), + ({'status': '404'}, '{"error":{"code":404,"message":"unit does not exist"}}') + ]) + + client = Client('http://198.51.100.23:9160', http=http) + + unit = Unit(client=client, data={'name': 'test.service'}) + unit.destroy() + + self.assertRaises(APIError, test) + + def test_desired_state_bad(self): + """ValueError is raised when bad values are provided for desired state""" + def test(): + unit = Unit() + unit.set_desired_state('invalid-state') + + self.assertRaises(ValueError, test) + + def test_desired_state_good_dead(self): + """We can set desired state on non-live objects""" + unit = Unit() + + assert unit.desiredState == 'launched' + + assert unit.set_desired_state('inactive') == 'inactive' + + assert unit.desiredState == 'inactive' + + def test_desired_state_good_live(self): + """We can set desired state on live objects""" + + http = HttpMockSequence([ + ({'status': '200'}, self._load_disccovery_fixture()), + ({'status': '204'}, None), + ({'status': '200'}, '{"currentState":"inactive","desiredState":"inactive","machineID":' + '"2901a44df0834bef935e24a0ddddcc23","name":"test.service","options"' + ':[{"name":"ExecStart","section":"Service","value":"/usr/bin/sleep 1d"}]}') + ]) + + client = Client('http://198.51.100.23:9160', http=http) + + unit = Unit(client=client, data={'name': 'test.service', 'desiredState': 'launched'}) + + assert unit.desiredState == 'launched' + + assert unit.set_desired_state('inactive') == 'inactive' + + assert unit.desiredState == 'inactive' diff --git a/requirements.txt b/requirements.txt index 4714444..56bd110 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -google-api-python-client>=1.4.0 \ No newline at end of file +google-api-python-client>=1.4.0 From 061bd3b28b9961df99833f7b48086a2ef11f4ef5 Mon Sep 17 00:00:00 2001 From: cnelson Date: Fri, 16 Oct 2015 16:00:55 -0700 Subject: [PATCH 2/3] fixes #2 and adds a test for line continuation behavior --- fleet/v1/objects/unit.py | 24 +++++++++++++++++++++--- fleet/v1/tests/test_unit.py | 15 +++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/fleet/v1/objects/unit.py b/fleet/v1/objects/unit.py index 114c742..a7a0edb 100644 --- a/fleet/v1/objects/unit.py +++ b/fleet/v1/objects/unit.py @@ -146,7 +146,7 @@ def _set_options_from_file(self, file_handle): """ # TODO: Find a library to handle this unit file parsing - # Can't use configparser, it doesn't handle multiple entries for the same key in the same sectoin + # Can't use configparser, it doesn't handle multiple entries for the same key in the same section # This is terribly naive # build our output here @@ -161,6 +161,7 @@ def _set_options_from_file(self, file_handle): line_number += 1 # clear any extra white space + orig_line = line line = line.strip() # ignore comments, and blank lines @@ -183,8 +184,25 @@ def _set_options_from_file(self, file_handle): )) # Attempt to parse a line inside a section - # Lines should look like: name=value + # Lines should look like: name=value \ + # continuation + continuation = False try: + # if the previous value ends with \ then we are a continuation + # so remove the \, and set the flag so we'll append to this below + if options[-1]['value'].endswith('\\'): + options[-1]['value'] = options[-1]['value'][:-1] + continuation = True + except IndexError: + pass + + try: + # if we are a continuation, then just append our value to the previous line + if continuation: + options[-1]['value'] += orig_line + continue + + # else we are a normal line, so spit and get our name / value name, value = line.split('=', 1) options.append({ 'section': section, @@ -194,7 +212,7 @@ def _set_options_from_file(self, file_handle): except ValueError: raise ValueError( 'Unable to parse unit file; ' - 'Malformed line in section {0}: {1} (line: {2}'.format( + 'Malformed line in section {0}: {1} (line: {2})'.format( section, line, line_number diff --git a/fleet/v1/tests/test_unit.py b/fleet/v1/tests/test_unit.py index b7f2f9f..fb1bdab 100644 --- a/fleet/v1/tests/test_unit.py +++ b/fleet/v1/tests/test_unit.py @@ -101,6 +101,21 @@ def test2(): self.assertRaises(ValueError, test) self.assertRaises(ValueError, test2) + def test_from_string_continuation_good(self): + """When parsing unit files, line continuation with a trailing backslash works""" + unit = Unit(from_string="[Section]\nThisLine=The start of \\\nsomething very\\\n long and boring\n") + + # test options (this should match the string above) + test_options = [ + { + 'section': 'Section', + 'name': 'ThisLine', + 'value': 'The start of something very long and boring' + } + ] + + assert unit.options == test_options + def test_options_no_desired_state(self): """Setting options explicitly works""" test_options = [{'section': 'Service', 'name': 'ExecStart', 'value': '/usr/bin/sleep 1d'}] From 686ce980d919f57f18bef1f3aed8dcfe1661f667 Mon Sep 17 00:00:00 2001 From: cnelson Date: Mon, 19 Oct 2015 09:43:03 -0700 Subject: [PATCH 3/3] SSH tunneling support --- .gitignore | 2 + README.md | 11 +- fleet/http/__init__.py | 3 +- fleet/http/ssh_tunnel.py | 82 ++++++ fleet/http/unix_socket.py | 2 + fleet/v1/client.py | 348 +++++++++++++++++++++++-- fleet/v1/docs/client.md | 183 +++++++------ fleet/v1/docs/index.md | 9 + fleet/v1/tests/test_client.py | 213 ++++++++++++++- fleet/v1/tests/test_http_ssh_tunnel.py | 31 +++ fleet/v1/tests/test_unit.py | 7 +- requirements.txt | 5 +- 12 files changed, 789 insertions(+), 107 deletions(-) create mode 100644 fleet/http/ssh_tunnel.py create mode 100644 fleet/v1/tests/test_http_ssh_tunnel.py diff --git a/.gitignore b/.gitignore index db4561e..1b00acd 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,5 @@ docs/_build/ # PyBuilder target/ + +.DS_Store diff --git a/README.md b/README.md index 55633ac..fe70463 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ A python client for [fleet](https://github.com/coreos/fleet) -Full documentation available at [python-fleet.readthedocs.org](http://python-fleet.readthedocs.org/en/latest/). Source for the documentation is in ``fleet/v1/docs/`` +Full documentation available at [python-fleet.readthedocs.org](http://python-fleet.readthedocs.org/en/latest/). Source for the documentation is in [fleet/v1/docs/](fleet/v1/docs) ## Install @@ -43,6 +43,8 @@ The [fleet API documentation](https://github.com/coreos/fleet/blob/master/Docume python-fleet will attempt to retrieve and parse this document when it is instantiated. Should any error occur during this process ``ValueError`` will be raised. +python-fleet supports connecting through SSH tunnels. See the [full Client documentation](fleet/v1/docs/client.md) for additional information on configuring SSH tunnels. + from __future__ import print_function # connect to fleet over tcp @@ -52,6 +54,13 @@ python-fleet will attempt to retrieve and parse this document when it is instant print('Unable to discover fleet: {0}'.format(exc)) raise SystemExit + # or via an ssh tunnel + try: + fleet_client = fleet.Client('http://127.0.0.1:49153', ssh_tunnel='198.51.100.23') + except ValueError as exc: + print('Unable to discover fleet: {0}'.format(exc)) + raise SystemExit + # or over a unix domain socket try: fleet_client = fleet.Client('http+unix://%2Fvar%2Frun%2Ffleet.sock') diff --git a/fleet/http/__init__.py b/fleet/http/__init__.py index affd501..94c48dd 100644 --- a/fleet/http/__init__.py +++ b/fleet/http/__init__.py @@ -1 +1,2 @@ -from .unix_socket import * # NOQA +from .unix_socket import * # NOQA +from .ssh_tunnel import * # NOQA diff --git a/fleet/http/ssh_tunnel.py b/fleet/http/ssh_tunnel.py new file mode 100644 index 0000000..fb8010f --- /dev/null +++ b/fleet/http/ssh_tunnel.py @@ -0,0 +1,82 @@ +try: # pragma: no cover + # python 2 + import httplib +except ImportError: # pragma: no cover + # python 3 + import http.client as httplib + +import httplib2 # NOQA +import sys + +try: # pragma: no cover + # python 2 + import urllib + unquote = urllib.unquote +except AttributeError: # pragma: no cover + # python 3 + import urllib.parse + unquote = urllib.parse.unquote + + +class SSHTunnelProxyInfo(httplib2.ProxyInfo): + def __init__(self, sock): + """A data structure for passing a socket to an httplib.HTTPConnection + + Args: + sock (socket-like): A connected socket or socket-like object. + + """ + + self.sock = sock + + +class HTTPOverSSHTunnel(httplib.HTTPConnection): + """ + A hack for httplib2 that expects proxy_info to be a socket already connected + to our target, rather than having to call connect() ourselves. This is used + to provide basic SSH Tunnelling support. + """ + + def __init__(self, host, port=None, strict=None, timeout=None, proxy_info=None): + """ + Setup an HTTP connection over an already connected socket. + + Args: + host: ignored (exists for compatibility with parent) + post: ignored (exists for compatibility with parent) + strict: ignored (exists for compatibility with parent) + timeout: ignored (exists for compatibility with parent) + + proxy_info (SSHTunnelProxyInfo): A SSHTunnelProxyInfo instance. + + """ + + # do the needful + httplib.HTTPConnection.__init__(self, host, port) + + # looks like the python2 and python3 versions of httplib differ + # python2, executables any callables and returns the result as proxy_info + # python3 passes the callable directly to this function :( + if hasattr(proxy_info, '__call__'): + proxy_info = proxy_info(None) + + # make sure we have a validate socket before we stash it + if not proxy_info or not isinstance(proxy_info, SSHTunnelProxyInfo) or not proxy_info.sock: + raise ValueError('This Connection must be suppplied an SSHTunnelProxyInfo via the proxy_info arg') + + # keep it + self.sock = proxy_info.sock + + def connect(self): # pragma: no cover + """Do nothing""" + # we don't need to connect, this functions job is to make sure + # self.sock exists and is connected. We did that in __init__ + # This is just here to keep other code in the parent from fucking + # with our already connected socket :) + pass + +# Add our module to httplib2 via sorta monkey patching +# When a request is made, the class responsible for the scheme is looked up in this dict +# So we inject our schemes and capture the SSH tunnel requests +sys.modules['httplib2'].SCHEME_TO_CONNECTION['ssh+http'] = HTTPOverSSHTunnel +sys.modules['httplib2'].SCHEME_TO_CONNECTION['ssh+http+unix'] = HTTPOverSSHTunnel diff --git a/fleet/http/unix_socket.py b/fleet/http/unix_socket.py index 328c0c2..1a1cd46 100644 --- a/fleet/http/unix_socket.py +++ b/fleet/http/unix_socket.py @@ -57,4 +57,6 @@ def connect(self): raise socket.error(msg) # Add our module to httplib2 via sorta monkey patching +# When a request is made, the class responsible for the scheme is looked up in this dict +# So we inject our schemes and capture the Unix domain requests sys.modules['httplib2'].SCHEME_TO_CONNECTION['http+unix'] = UnixConnectionWithTimeout diff --git a/fleet/v1/client.py b/fleet/v1/client.py index db249b7..f0176b7 100644 --- a/fleet/v1/client.py +++ b/fleet/v1/client.py @@ -1,15 +1,144 @@ #!/usr/bin/env python2.7 -# Sorta Monkey patch httplib2 to support unix socket -from fleet.http import UnixConnectionWithTimeout # NOQA - from googleapiclient.discovery import build import googleapiclient.errors -import json, socket # NOQA +import json, socket, os # NOQA + +import httplib2 + +import paramiko from fleet.v1.objects import * from fleet.v1.errors import * +from fleet.http.ssh_tunnel import SSHTunnelProxyInfo + +try: # pragma: no cover + # python 2 + import urlparse +except ImportError: # pragma: no cover + # python 3 + import urllib.parse as urlparse + +try: # pragma: no cover + # python 2 + import urllib + unquote = urllib.unquote +except AttributeError: # pragma: no cover + # python 3 + import urllib.parse + unquote = urllib.parse.unquote + + +class SSHTunnel(object): + """Use paramiko to setup local "ssh -L" tunnels for Client to use""" + + def __init__( + self, + host, + username=None, + port=22, + timeout=10, + known_hosts_file=None, + strict_host_key_checking=True + ): + """Connect to the SSH server, and authenticate + + Args: + host (str or paramiko.transport.Transport): The hostname to connect to or an already connected Transport. + username (str): The username to use when authenticating. + port (int): The port to connect to, defaults to 22. + timeout (int): The timeout to wait for a connection in seconds, defaults to 10. + known_hosts_file (str): A path to a known host file, ignored if strict_host_key_checking is False. + strict_host_key_checking (bool): Verify host keys presented by remote machines before + initiating SSH connections, defaults to True. + + Raises: + ValueError: strict_host_key_checking was true, but known_hosts_file didn't exist. + socket.gaierror: Unable to resolve host + socket.error: Unable to connect to host:port + paramiko.ssh_exception.SSHException: Error authenticating during SSH connection. + """ + + self.client = None + self.transport = None + + # if they passed us a transport, then we don't need to make our own + if isinstance(host, paramiko.transport.Transport): + self.transport = host + else: + # assume they passed us a hostname, and we connect to it + self.client = paramiko.SSHClient() + + # if we are strict, then we have to have a host file + if strict_host_key_checking: + try: + self.client.load_system_host_keys(os.path.expanduser(known_hosts_file)) + except IOError: + raise ValueError( + 'Strict Host Key Checking is enabled, but hosts file ({0}) ' + 'does not exist or is unreadable.'.format(known_hosts_file) + ) + else: + # don't load the host file, and set to AutoAdd missing keys + self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + # Connect to the host, with the provided params, let exceptions bubble up + self.client.connect( + host, + port=port, + username=username, + banner_timeout=timeout, + ) + + # Stash our transport + self.transport = self.client.get_transport() + + def forward_tcp(self, host, port): + """Open a connection to host:port via an ssh tunnel. + + Args: + host (str): The host to connect to. + port (int): The port to connect to. + + Returns: + A socket-like object that is connected to the provided host:port. + + """ + + return self.transport.open_channel( + 'direct-tcpip', + (host, port), + self.transport.getpeername() + ) + + def forward_unix(self, path): + """Open a connection to a unix socket via an ssh tunnel. + + Requires the server to be running OpenSSH >=6.7. + + Args: + path (str): A path to a unix domain socket. + + Returns: + A socket-like object that is connected to the provided path. + + Raises: + RuntimeError: All the time because of what it says on the tin. + + """ + raise RuntimeError( + 'Paramiko does not yet support tunneling unix domain sockets. ' + 'Help is needed to add this functionality! ' + 'https://github.com/paramiko/paramiko/issues/544' + ) + + # when paramiko patches, hopefully this is all that is needed: + # return self.transport.open_channel( + # 'direct-streamlocal@openssh.com', + # path, + # self.transport.getpeername() + # ) class Client(object): @@ -23,37 +152,123 @@ class Client(object): _VERSION = 'v1' _STATES = ['inactive', 'loaded', 'launched'] - def __init__(self, endpoint, http=None): - """Connect to the fleet API and generate a client based on it's Discovery Documentation + def __init__( + self, + endpoint, + http=None, - Args: - endpoint (str): Location of the fleet API. Supported schemes are: - http : A http connection over a TCP socket. A http connection over a TCP socket. - ``http://127.0.0.1:49153`` - http+unix: A http connect over a unix domain socket. You must escape the path / = %2F. - ``http+unix://%2Fvar%2Frun%2Ffleet.sock`` + ssh_tunnel=None, + ssh_username='core', + ssh_timeout=10, + ssh_known_hosts_file='~/.fleetctl/known_hosts', + ssh_strict_host_key_checking=True, - http (httplib2.Http): An instance of httplib2.Http or something that acts like it - that HTTP requests will be made through. + ssh_raw_transport=None + ): - You shouldn't usually need to pass this, but if you do need to - configure specific options for your http client, or want to - pass in a mock for testing. This is the place to do it. + """Connect to the fleet API and generate a client based on it's discovery document. + Args: + endpoint (str): A URL where the fleet API can be reached. Supported schemes are: + http: A HTTP connection over a TCP socket. + Example: http://127.0.0.1:49153 + http+unix: A HTTP connection over a unix domain socket. You must escape the path (/ = %2F). + Example: http+unix://%2Fvar%2Frun%2Ffleet.sock + + http (httplib2.Http): An instance of httplib2.Http (or something that acts like it) that HTTP requests will + be made through. You do not need to pass this unless you need to configure specific options for your + http client, or want to pass in a mock for testing. + + ssh_tunnel (str '[:]'): Establish an SSH tunnel through the provided address for communication + with fleet. Defaults to None. If specified, the following other options adjust it's behaivor: + ssh_username (str): Username to use when connecting to SSH, defaults to 'core'. + ssh_timeout (float): Amount of time in seconds to allow for SSH connection initialization + before failing, defaults to 10. + ssh_known_hosts_file (str): File used to store remote machine fingerprints, + defaults to '~/.fleetctl/known_hosts'. Ignored if `ssh_strict_host_key_checking` is False + ssh_strict_host_key_checking (bool): Verify host keys presented by remote machines before + initiating SSH connections, defaults to True. + + ssh_raw_transport (paramiko.transport.Transport): An active Transport on which open_channel() will be + called to establish connections. + + See Advanced SSH Tunneling in docs/client.md for more information. Raises: - ValueError: The endpoint provided was not accessible. + ValueError: The endpoint provided was not accessible or your ssh configuration is incorrect """ # stash this for later self._endpoint = endpoint.strip('/') + self._ssh_client = None + + # we overload the http when our proxy enabled versin if they request ssh tunneling + # so we need to make sure they didn't give us both + if (ssh_tunnel or ssh_raw_transport) and http: + raise ValueError('You cannot specify your own http client, and request ssh tunneling.') + + # only one way to connect, not both + if ssh_tunnel and ssh_raw_transport: + raise ValueError('If ssh_tunnel is specified, ssh_raw_transport must be None') + + # see if we need to setup an ssh tunnel + self._ssh_tunnel = None + + # if they handed us a transport, then we either bail or are good to go + if ssh_raw_transport: + if not isinstance(ssh_raw_transport, paramiko.transport.Transport): + raise ValueError('ssh_raw_transport must be an active instance of paramiko.transport.Transport.') + + self._ssh_tunnel = SSHTunnel(host=ssh_raw_transport) + + # otherwise we are connecting ourselves + elif ssh_tunnel: + (ssh_host, ssh_port) = self._split_hostport(ssh_tunnel, default_port=22) + + try: + self._ssh_tunnel = SSHTunnel( + host=ssh_host, + port=ssh_port, + username=ssh_username, + timeout=ssh_timeout, + known_hosts_file=ssh_known_hosts_file, + strict_host_key_checking=ssh_strict_host_key_checking + ) + + except socket.gaierror as exc: + raise ValueError('{0} could not be resolved.'.format(ssh_host)) + + except socket.error as exc: + raise ValueError('Unable to connect to {0}:{1}: {2}'.format( + ssh_host, + ssh_port, + exc + )) + + except paramiko.ssh_exception.SSHException as exc: + raise ValueError('Unable to connect via ssh: {0}: {1}'.format( + exc.__class__.__name__, + exc + )) - self._http = http + # did we get an ssh connection up? + if self._ssh_tunnel: + # inject the SSH tunnel socketed into httplib via the proxy_info interface + self._http = httplib2.Http(proxy_info=self._get_proxy_info) + + # preface our scheme with 'ssh+'; httplib2's SCHEME_TO_CONNECTION + # will invoke our custom connection objects and route the HTTP + # call across the SSH connection established or passed in above + self._endpoint = 'ssh+' + self._endpoint + else: + self._http = http - # geneate a client binding using the google-api-python client. + # if we've made it this far, we are ready to try to talk to fleet + # possibly through a proxy... + + # generate a client binding using the google-api-python client. # See https://developers.google.com/api-client-library/python/start/get_started # For more infomation on how to use the generated client binding. - try: discovery_url = self._endpoint + '/{api}/{apiVersion}/discovery' @@ -78,6 +293,97 @@ def __init__(self, endpoint, http=None): self._VERSION )) + def _split_hostport(self, hostport, default_port=None): + """Split a string in the format of ':' into it's component parts + + default_port will be used if a port is not included in the string + + Args: + str ('' or ':'): A string to split into it's parts + + Returns: + two item tuple: (host, port) + + Raises: + ValueError: The string was in an invalid element + """ + + try: + (host, port) = hostport.split(':', 1) + except ValueError: # no colon in the string so make our own port + host = hostport + + if default_port is None: + raise ValueError('No port found in hostport, and default_port not provided.') + + port = default_port + + try: + port = int(port) + if port < 1 or port > 65535: + raise ValueError() + except ValueError: + raise ValueError("{0} is not a valid TCP port".format(port)) + + return (host, port) + + def _endpoint_to_target(self, endpoint): + """Convert a URL into a host / port, or into a path to a unix domain socket + + Args: + endpoint (str): A URL parsable by urlparse + + Returns: + 3 item tuple: (host, port, path). + host and port will None, and path will be not None if a a unix domain socket URL is passed + path will be None if a normal TCP based URL is passed + + """ + parsed = urlparse.urlparse(endpoint) + scheme = parsed[0] + hostport = parsed[1] + + if 'unix' in scheme: + return (None, None, unquote(hostport)) + + if scheme == 'https': + target_port = 443 + else: + target_port = 80 + + (target_host, target_port) = self._split_hostport(hostport, default_port=target_port) + return (target_host, target_port, None) + + def _get_proxy_info(self, _=None): + """Generate a ProxyInfo class from a connected SSH transport + + Args: + _ (None): Ignored. This is just here as the ProxyInfo spec requires it. + + + Returns: + SSHTunnelProxyInfo: A ProxyInfo with an active socket tunneled through SSH + + """ + # parse the fleet endpoint url, to establish a tunnel to that host + (target_host, target_port, target_path) = self._endpoint_to_target(self._endpoint) + + # implement the proxy_info interface from httplib which requires + # that we accept a scheme, and return a ProxyInfo object + # we do :P + # This is called once per request, so we keep this here + # so that we can keep one ssh connection open, and allocate + # new channels as needed per-request + sock = None + + if target_path: + sock = self._ssh_tunnel.forward_unix(path=target_path) + else: + sock = self._ssh_tunnel.forward_tcp(target_host, port=target_port) + + # Return a ProxyInfo class with this socket + return SSHTunnelProxyInfo(sock=sock) + def _single_request(self, method, *args, **kwargs): """Make a single request to the fleet API endpoint diff --git a/fleet/v1/docs/client.md b/fleet/v1/docs/client.md index 7db7c26..e6a40a3 100644 --- a/fleet/v1/docs/client.md +++ b/fleet/v1/docs/client.md @@ -1,40 +1,83 @@ # Client -A python wrapper for the fleet v1 API +A python wrapper for the [fleet v1 API](https://github.com/coreos/fleet/blob/master/Documentation/api-v1.md). -[Offical Fleet v1 API Documentation](https://github.com/coreos/fleet/blob/master/Documentation/api-v1.md) + >>> import fleet.v1 as fleet + + # connect over tcp + >>> fleet_client = fleet.Client('http://127.0.0.1:49153') + + # or over a unix domain socket + >>> fleet_client = fleet.Client('http+unix://%2Fvar%2Frun%2Ffleet.sock') + + # via an ssh tunnel + >>> fleet_client = fleet.Client('http://127.0.0.1:49153', ssh_tunnel='198.51.100.23:22') -## Creating Clients +### Client(self, endpoint, http=None, ssh_tunnel=None, ssh_username='core', ssh_timeout=10, ssh_known_hosts_file='~/.fleetctl/known_hosts', ssh_strict_host_key_checking=True, ssh_raw_transport=None) -Connect to the fleet API and generate a client based on it's Discovery Documentation +Connect to the fleet API and generate a client based on it's [discovery document](https://developers.google.com/discovery/v1/reference/apis?hl=en). -### Client(self, endpoint, http=None) -* **endpoint (str):** Location of the fleet API. Supported schemes are: - * **http :** A http connection over a TCP socket. ``http://127.0.0.1:49153`` - * **http+unix:** A http connect over a unix domain socket. You must escape the path / = %2F. ``http+unix://%2Fvar%2Frun%2Ffleet.sock`` +### Arguments +* **endpoint (str):** A URL where the fleet API can be reached. Supported schemes are: + * **http:** A HTTP connection over a TCP socket. ``http://127.0.0.1:49153`` + * **http+unix:** A HTTP connection over a unix domain socket. You must escape the path (/ = %2F). ``http+unix://%2Fvar%2Frun%2Ffleet.sock`` -* **http (httplib2.Http):** An instance of httplib2.Http or something that acts like it that HTTP requests will be made through. -You shouldn't usually need to pass this, but if you do need to configure specific options for your http client, or want to pass in a mock for testing. This is the place to do it. +* **http (httplib2.Http):** An instance of httplib2.Http (or something that acts like it) that HTTP requests will be made through. You do not need to pass this unless you need to configure specific options for your http client, or want to pass in a mock for testing. + +* **ssh_tunnel (str '\[:\]'):** Establish an SSH tunnel through the provided address for communication with fleet. Defaults to None. If specified, the following other options adjust it's behaivor: + * **ssh_username (str):** Username to use when connecting to SSH, defaults to 'core'. + * **ssh_timeout (float):** Amount of time in seconds to allow for SSH connection initialization before failing, defaults to 10. + * **ssh_known_hosts_file (str):** File used to store remote machine fingerprints, defaults to '~/.fleetctl/known_hosts'. Ignored if `ssh_strict_host_key_checking` is False + * **ssh_strict_host_key_checking (bool):** Verify host keys presented by remote machines before initiating SSH connections, defaults to True. + +* **ssh_raw_transport ([paramiko.transport.Transport](http://docs.paramiko.org/en/stable/api/transport.html#paramiko.transport.Transport)):** An active Transport on which [open_channel()](http://docs.paramiko.org/en/stable/api/transport.html#paramiko.transport.Transport.open_channel) will be called to establish connections. See [Advanced SSH Tunneling](#advanced-ssh-tunneling) for more information. ### Raises -* **ValueError:** The endpoint provided was not accessable. +* **ValueError:** The endpoint provided was not accessible. -### Example +### Advanced SSH Tunneling - >>> import fleet.v1 as fleet +If your ssh connection requires complex configuration, you can configure and [connect()](http://docs.paramiko.org/en/stable/api/client.html#paramiko.client.SSHClient.connect) your own [paramiko.client.Client](http://docs.paramiko.org/en/stable/api/client.html) and pass the result of [get_transport()](http://docs.paramiko.org/en/stable/api/client.html#paramiko.client.SSHClient.get_transport) as `ssh_raw_transport` - # connect over tcp - >>> fleet_client = fleet.Client('http://127.0.0.1:49153') +If `ssh_raw_transport` is set all other ssh options are ignored, it's assumed the caller will have fully configured and connected their ssh transport before invoking us. - # or over a unix domain socket - >>> fleet_client = fleet.Client('http+unix://%2Fvar%2Frun%2Ffleet.sock') + # example of configuring and connecting your own ssh client + # this contrived example uses a specific key, + # and disables the use of the agent + import fleet.v1 as fleet + import paramiko -## create_unit() + # configure our client + ssh_client = paramiko.SSHClient() + ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + # connect to the host with our custom configuration + ssh_client.connect( + hostname='198.51.100.23', + username='core', + key_filename='/tmp/the.key', + allow_agent=False, + look_for_keys=False + ) + + # pass the transport to the client + fleet_client = fleet.Client( + 'http://127.0.0.1:49153', + ssh_raw_transport=ssh_client.get_transport() + ) + + +## Methods + + +## create_unit() Create a new [Unit](unit.md) in the cluster + >>> fleet_client.create_unit('foo.service', fleet.Unit(from_file='foo.service')) + + ### create_unit(self, name, unit) * **name (str):** The name of the unit to create * **unit ([Unit](unit.md)):** The unit to submit to fleet @@ -45,29 +88,11 @@ Create a new [Unit](unit.md) in the cluster ### Raises * [APIError](apierror.md): Fleet returned a response code >= 400 -### Example - - >>> fleet_client.create_unit('foo.service', fleet.Unit(from_file='foo.service')) - - ## set_unit_desired_state() Update the desired state of a unit running in the cluster. -### set_unit_desired_state(self, unit, desired_state) -* **unit (str, [Unit](unit)):** The Unit, or name of the unit to delete -* **desired_state (str)**: State the user wishes the Unit to be in ("inactive", "loaded", or "launched") - -### Returns -* [Unit](unit.md): The updated unit - -### Raises -* [APIError](apierror.md): Fleet returned a response code >= 400 -* **ValueError:** An invalid value was provided for ``desired_state`` - - -### Example >>> unit = fleet_client.create_unit('foo.service', fleet.Unit(from_file='foo.service')) >>> unit.name u'foo.service' @@ -90,21 +115,20 @@ Update the desired state of a unit running in the cluster. >>> fleet_client.set_unit_desired_state('foo.service', 'invalid-state') ValueError: state must be one of: ['inactive', 'loaded', 'launched'] - -## destroy_unit() - -Delete a unit from the cluster - -### destroy_unit(self, unit) -* **unit (str, [Unit](unit.md)):** The Unit, or name of the unit to delete +### set_unit_desired_state(self, unit, desired_state) +* **unit (str, [Unit](unit)):** The Unit, or name of the unit to delete +* **desired_state (str)**: State the user wishes the Unit to be in ("inactive", "loaded", or "launched") ### Returns -* **True:** The unit was deleted +* [Unit](unit.md): The updated unit ### Raises * [APIError](apierror.md): Fleet returned a response code >= 400 +* **ValueError:** An invalid value was provided for ``desired_state`` -### Example +## destroy_unit() + +Delete a unit from the cluster # delete by passing the unit >>> fleet_client.destroy_unit(unit) @@ -118,39 +142,38 @@ Delete a unit from the cluster >>> fleet_client.destroy_unit('invalid-service') fleet.v1.errors.APIError: unit does not exist (404) -## list_units() - -Returns a generator that yields each [Unit](unit.md) in the cluster -### list_units(self): +### destroy_unit(self, unit) +* **unit (str, [Unit](unit.md)):** The Unit, or name of the unit to delete -### Yields: -* [Unit](unit.md): The next Unit in the cluster +### Returns +* **True:** The unit was deleted ### Raises * [APIError](apierror.md): Fleet returned a response code >= 400 -### Example +## list_units() + +Returns a generator that yields each [Unit](unit.md) in the cluster + >>> for unit in fleet_client.list_units(): ... unit ... -## get_unit() - -Retreive a specific [Unit](unit.md) from the cluster by name. - -### get_unit(self, name) -* **name (str):** If specified, only this unit name is returned +### list_units(self): -### Returns -* [Unit](unit.md): The unit identified by ``name`` +### Yields: +* [Unit](unit.md): The next Unit in the cluster ### Raises * [APIError](apierror.md): Fleet returned a response code >= 400 -### Example + +## get_unit() + +Retreive a specific [Unit](unit.md) from the cluster by name. # get a service by name >>> fleet_client.get_unit('foo.service') @@ -160,11 +183,26 @@ Retreive a specific [Unit](unit.md) from the cluster by name. >>> fleet_client.get_unit('invalid-service') fleet.v1.errors.APIError: unit does not exist (404) +### get_unit(self, name) +* **name (str):** If specified, only this unit name is returned + +### Returns +* [Unit](unit.md): The unit identified by ``name`` + +### Raises +* [APIError](apierror.md): Fleet returned a response code >= 400 + ## list_unit_states() -Returns a generator tht yields the current [UnitState](unitstate.md) for each unit in the cluster - +Returns a generator that yields the current [UnitState](unitstate.md) for each unit in the cluster + + >>> for unit_state in fleet_client.list_unit_states(): + ... unit_state + ... + + + ### list_unit_states(self, machine_id = None, unit_name = None) * **machine_id (str):** filter all UnitState objects to those originating from a specific machine @@ -176,17 +214,17 @@ Returns a generator tht yields the current [UnitState](unitstate.md) for each un ### Raises * [APIError](apierror.md): Fleet returned a response code >= 400 -### Example - >>> for unit_state in fleet_client.list_unit_states(): - ... unit_state - ... - - ## list_machines() Return a generator that yields each [Machine](machine.md) in the cluster + + >>> for machine in fleet_client.list_machines(): + ... machine + ... + + ### list_machines(self) ### Yields @@ -194,10 +232,3 @@ Return a generator that yields each [Machine](machine.md) in the cluster ### Raises * [APIError](apierror.md): Fleet returned a response code >= 400 - -### Example - - >>> for machine in fleet_client.list_machines(): - ... machine - ... - \ No newline at end of file diff --git a/fleet/v1/docs/index.md b/fleet/v1/docs/index.md index 8e3d683..118bdf2 100644 --- a/fleet/v1/docs/index.md +++ b/fleet/v1/docs/index.md @@ -39,6 +39,8 @@ The [fleet API documentation](https://github.com/coreos/fleet/blob/master/Docume python-fleet will attempt to retrieve and parse this document when it is instantiated. Should any error occur during this process ``ValueError`` will be raised. +python-fleet supports connecting through SSH tunnels. See the [full Client documentation](client.md) for additional information on configuring SSH tunnels. + from __future__ import print_function # connect to fleet over tcp @@ -48,6 +50,13 @@ python-fleet will attempt to retrieve and parse this document when it is instant print('Unable to discover fleet: {0}'.format(exc)) raise SystemExit + # or via an ssh tunnel + try: + fleet_client = fleet.Client('http://127.0.0.1:49153', ssh_tunnel='198.51.100.23') + except ValueError as exc: + print('Unable to discover fleet: {0}'.format(exc)) + raise SystemExit + # or over a unix domain socket try: fleet_client = fleet.Client('http+unix://%2Fvar%2Frun%2Ffleet.sock') diff --git a/fleet/v1/tests/test_client.py b/fleet/v1/tests/test_client.py index 461aa8d..b525c24 100644 --- a/fleet/v1/tests/test_client.py +++ b/fleet/v1/tests/test_client.py @@ -1,27 +1,87 @@ import unittest +import mock -import os +import os, socket, tempfile # NOQA from apiclient.http import HttpMock, HttpMockSequence -from ..client import Client +import paramiko + +from ..client import Client, SSHTunnel from ..errors import APIError from ..objects import Unit +class ForwardChecker(object): + """A simple mock for SSHTunnel with this class when we don't actually want to connect to servers during tests""" + def __init__(self, *args, **kwargs): + pass + + def forward_tcp(self, host, port): + return [host, port] + + def forward_unix(self, path): + return path + + +class TestSSHTunnel(unittest.TestCase): + def test_good_raw_transport(self): + """Passing a raw transport to ssh tunnel skips other configuration""" + t = paramiko.transport.Transport(None) + + s = SSHTunnel(host=t) + + assert s.client is None + assert id(s.transport) == id(t) + + def test_bad_known_host_file(self): + """If known_hosts_file doesn't exist but strict_host_key_checking is True, then a ValueError is raised""" + tmpdir = tempfile.mkdtemp() + + bad_host_file = os.path.join(tmpdir, 'known_hosts') + + def test(): + SSHTunnel(host='foo', known_hosts_file=bad_host_file, strict_host_key_checking=True) + + os.rmdir(tmpdir) + + self.assertRaises(ValueError, test) + + def test_unix_forward(self): + """Forwarding a unix domain socket raises RuntimeError""" + t = paramiko.transport.Transport(None) + + s = SSHTunnel(host=t) + + def test(): + s.forward_unix('/tmp/socket') + + self.assertRaises(RuntimeError, test) + + def test_good_connect(self): + """When we connect with a good client, the transport gets set correctly""" + + with mock.patch('paramiko.SSHClient'): + s = SSHTunnel(host='foo', strict_host_key_checking=False) + assert id(s.client.get_transport()) == id(s.transport) + + class TestFleetClient(unittest.TestCase): def setUp(self): self._BASE_DIR = os.path.dirname(os.path.abspath(__file__)) - self.discovery = HttpMock( - os.path.join(self._BASE_DIR, 'fixtures/fleet_v1.json'), - {'status': '200'} - ) + self.discovery = self._get_discovery() self.endpoint = 'http://198.51.100.23:9160' self.client = Client(self.endpoint, http=self.discovery) + def _get_discovery(self, *args, **kwargs): + return HttpMock( + os.path.join(self._BASE_DIR, 'fixtures/fleet_v1.json'), + {'status': '200'} + ) + def mock(self, http): self.client._http = http @@ -44,6 +104,147 @@ def test(): self.assertRaises(ValueError, test) + def test_init_ssh_tunnel_conflicting_params(self): + """Providing conflicting parameters raises ValueError""" + + def test_tunnel_with_http(): + Client(self.endpoint, http=True, ssh_tunnel=True) + + def test_raw_with_http(): + Client(self.endpoint, http=True, ssh_raw_transport=True) + + def test_both_with_http(): + Client(self.endpoint, http=True, ssh_tunnel=True, ssh_raw_transport=True) + + def test_both(): + Client(self.endpoint, http=True, ssh_tunnel=True, ssh_raw_transport=True) + + for test in [test_tunnel_with_http, test_raw_with_http, test_both_with_http, test_both]: + self.assertRaises(ValueError, test) + + def test_bad_transport_value(self): + """Providing anything but a paramiko.transport.Transport to ssh_raw_transport raises ValueError""" + + def test(): + Client(endpoint=self.endpoint, ssh_raw_transport=True) + + self.assertRaises(ValueError, test) + + def test_hostport_split_default_port(self): + """Validate that when passing a default port it's used when no port is provided""" + result = self.client._split_hostport('foo', default_port=916) + + assert result == ('foo', 916) + + result = self.client._split_hostport('foo:22', default_port=916) + + assert result == ('foo', 22) + + def test_hostport_split_no_port(self): + """ValueError is raised if no port is passted to hostport_split""" + + def test(): + self.client._split_hostport('foo') + + self.assertRaises(ValueError, test) + + def test_hostport_split_not_int(self): + """ValueError is raised if a non number port is passed""" + + def test(): + self.client._split_hostport('foo:bar') + + self.assertRaises(ValueError, test) + + def test_hostport_split_not_in_range(self): + """ValueError is raised if port is out of range""" + + def test(): + self.client._split_hostport('foo:99999') + + self.assertRaises(ValueError, test) + + def test_endpoint_to_target_unix(self): + """When passing in a http+unix endpoint, we receive the path pack with no host/port""" + result = self.client._endpoint_to_target('http+unix://%2Ftmp%2Fsocket') + + assert result == (None, None, '/tmp/socket') + + def test_endpoint_to_target_https_default_port(self): + """When passing in a https endpoint with no port, port 443 is returned""" + result = self.client._endpoint_to_target('https://foo') + + assert result == ('foo', 443, None) + + def test_endpoint_to_target_http_default_port(self): + """When passing in a https endpoint with no port, port 443 is returned""" + result = self.client._endpoint_to_target('http://foo') + + assert result == ('foo', 80, None) + + def test_endpoint_to_target_explicit_port(self): + """An explicit port is used if provided regardless of scheme""" + result = self.client._endpoint_to_target('http://foo:999') + + assert result == ('foo', 999, None) + + result = self.client._endpoint_to_target('https://foo:888') + + assert result == ('foo', 888, None) + + def test_get_proxy_info_tcp(self): + """When given a TCP based endpoint, an open channel is returned""" + + self.client._ssh_tunnel = ForwardChecker() + result = self.client._get_proxy_info() + + assert result.sock == ['198.51.100.23', 9160] + + def test_get_proxy_info_unix(self): + """When given a TCP based endpoint, an open channel is returned""" + + self.client._endpoint = 'http+unix://%2Ftmp%2Fsocket' + self.client._ssh_tunnel = ForwardChecker() + result = self.client._get_proxy_info() + + assert result.sock == '/tmp/socket' + + def test_ssh_tunnel_bad_host(self): + """When SSHClient returns a socket.gaierror for a bad hostname, we return ValueError""" + def test(): + with mock.patch('paramiko.SSHClient', side_effect=socket.gaierror): + Client(endpoint=self.endpoint, ssh_tunnel='unknown_host') + + self.assertRaises(ValueError, test) + + def test_ssh_tunnel_bad_port(self): + """When SSHClient returns a socket.error for a bad port, we return ValueError""" + def test(): + with mock.patch('paramiko.SSHClient', side_effect=socket.error): + Client(endpoint=self.endpoint, ssh_tunnel='unknown_host:2222') + + self.assertRaises(ValueError, test) + + def test_ssh_tunnel_ssh_error(self): + """When SSHClient returns an error authenticating, we return ValueError""" + def test(): + with mock.patch('paramiko.SSHClient', side_effect=paramiko.ssh_exception.SSHException): + Client(endpoint=self.endpoint, ssh_tunnel='host-ok-bad-key') + + self.assertRaises(ValueError, test) + + def test_ssh_tunnel_with_unix(self): + """RuntimeError is raised if we try to forward a unix domain socket""" + + t = paramiko.transport.Transport(None) + + s = SSHTunnel(host=t) + + def test(): + s.forward_unix('/tmp/socket') + + self.assertRaises(RuntimeError, test) + def test_single_request_good(self): """A single request returns 200""" self.mock(HttpMock( diff --git a/fleet/v1/tests/test_http_ssh_tunnel.py b/fleet/v1/tests/test_http_ssh_tunnel.py new file mode 100644 index 0000000..0ca2792 --- /dev/null +++ b/fleet/v1/tests/test_http_ssh_tunnel.py @@ -0,0 +1,31 @@ +import unittest + +from ...http import HTTPOverSSHTunnel, SSHTunnelProxyInfo + + +class TestHttpSSHTunnel(unittest.TestCase): + + def test_proxy_info_callable(self): + """Passing a callable to proxy_info gets executed""" + + def test(_): + return SSHTunnelProxyInfo('lolz') + + h = HTTPOverSSHTunnel('foo', proxy_info=test) + + assert h.sock == 'lolz' + + def test_proxy_info_data(self): + """Passing data to proxy_info gets returned""" + + h = HTTPOverSSHTunnel('foo', proxy_info=SSHTunnelProxyInfo('lolz')) + + assert h.sock == 'lolz' + + def test_proxy_info_bad_object(self): + """Passing anything other than proxy info causes an error""" + + def test(): + HTTPOverSSHTunnel('foo', proxy_info='lolz') + + self.assertRaises(ValueError, test) diff --git a/fleet/v1/tests/test_unit.py b/fleet/v1/tests/test_unit.py index fb1bdab..84340ce 100644 --- a/fleet/v1/tests/test_unit.py +++ b/fleet/v1/tests/test_unit.py @@ -143,7 +143,12 @@ def test_repr(self): assert str(test_options) in repr(unit) def test_str_roundtrip(self): - """Calling str() on a unit, should generate a systemd unit""" + """Calling str() on a unit, should generate a systemd unit + + Note: this only works for units without line continuations. + A unit with line continuations that is roundtripped through this parser + will be modified! (line continuations collapsed onto a single line) + """ test_string = "[Service]\nExecStart=/usr/bin/sleep 1d" diff --git a/requirements.txt b/requirements.txt index 56bd110..44dbfd1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,4 @@ -google-api-python-client>=1.4.0 +google-api-python-client>=1.4.2 +paramiko>=1.15.1 +mock +argparse \ No newline at end of file