From 803573606dd09e967846f3f2521e536b13188212 Mon Sep 17 00:00:00 2001 From: Michael Marchetti Date: Thu, 24 Jan 2019 12:10:26 -0500 Subject: [PATCH 1/8] remove environment.txt support --- rsconnect_jupyter/environment.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/rsconnect_jupyter/environment.py b/rsconnect_jupyter/environment.py index 35618070..48007ca3 100644 --- a/rsconnect_jupyter/environment.py +++ b/rsconnect_jupyter/environment.py @@ -15,7 +15,7 @@ class EnvironmentException(Exception): pass -def detect_environment(dirname): +def detect_environment(): """Determine the python dependencies in the environment. If requirements.txt exists in the notebook directory, @@ -26,8 +26,7 @@ def detect_environment(dirname): and contents if successful, or a dictionary containing 'error' on failure. """ - result = (output_file(dirname, 'requirements.txt', 'pip') or - pip_freeze(dirname)) + result = pip_freeze() if result is not None: result['python'] = get_python_version() @@ -95,7 +94,7 @@ def output_file(dirname, filename, package_manager): raise EnvironmentException('Error reading %s: %s' % (filename, str(exc))) -def pip_freeze(dirname): +def pip_freeze(): """Inspect the environment using `pip freeze`. Returns a dictionary containing the filename @@ -129,12 +128,7 @@ def pip_freeze(dirname): if __name__ == '__main__': try: - if len(sys.argv) < 2: - raise EnvironmentException('Usage: %s NOTEBOOK_PATH' % sys.argv[0]) - - notebook_path = sys.argv[1] - dirname = os.path.dirname(notebook_path) - result = detect_environment(dirname) + result = detect_environment() except EnvironmentException as exc: result = dict(error=str(exc)) From aca9336ecfb2bbc6cb300e57a7e67e51b82114c8 Mon Sep 17 00:00:00 2001 From: Michael Marchetti Date: Fri, 25 Jan 2019 10:27:10 -0500 Subject: [PATCH 2/8] update tests --- rsconnect_jupyter/tests/test_bundle.py | 17 ++++++++++------- rsconnect_jupyter/tests/test_environment.py | 20 +------------------- 2 files changed, 11 insertions(+), 26 deletions(-) diff --git a/rsconnect_jupyter/tests/test_bundle.py b/rsconnect_jupyter/tests/test_bundle.py index 015068cd..b8eb8aeb 100644 --- a/rsconnect_jupyter/tests/test_bundle.py +++ b/rsconnect_jupyter/tests/test_bundle.py @@ -40,7 +40,7 @@ def test_source_bundle1(self): # the test environment. Don't do this in the production code, which # runs in the notebook server. We need the introspection to run in # the kernel environment and not the notebook server environment. - environment = detect_environment(dir) + environment = detect_environment() notebook = self.read_notebook(nb_path) with make_source_bundle(notebook, environment, dir) as bundle, \ tarfile.open(mode='r:gz', fileobj=bundle) as tar: @@ -53,10 +53,16 @@ def test_source_bundle1(self): ]) reqs = tar.extractfile('requirements.txt').read() - self.assertEqual(reqs, b'numpy\npandas\nmatplotlib\n') + + # these are the dependencies declared in our setup.py + self.assertIn(b'notebook', reqs) + self.assertIn(b'nbformat', reqs) manifest = json.loads(tar.extractfile('manifest.json').read().decode('utf-8')) + # don't check requirements.txt since we don't know the checksum + del manifest['files']['requirements.txt'] + # don't check locale value, just require it be present del manifest['locale'] del manifest['python']['package_manager']['version'] @@ -77,9 +83,6 @@ def test_source_bundle1(self): u"files": { u"dummy.ipynb": { u"checksum": u"36873800b48ca5ab54760d60ba06703a" - }, - u"requirements.txt": { - u"checksum": u"5f2a5e862fe7afe3def4a57bb5cfb214" } } }) @@ -93,7 +96,7 @@ def test_source_bundle2(self): # the test environment. Don't do this in the production code, which # runs in the notebook server. We need the introspection to run in # the kernel environment and not the notebook server environment. - environment = detect_environment(dir) + environment = detect_environment() notebook = self.read_notebook(nb_path) with make_source_bundle(notebook, environment, dir, extra_files=['data.csv']) as bundle, \ @@ -159,7 +162,7 @@ def do_test_html_bundle(self, dir): # the test environment. Don't do this in the production code, which # runs in the notebook server. We need the introspection to run in # the kernel environment and not the notebook server environment. - environment = detect_environment(dir) + environment = detect_environment() notebook = self.read_notebook(nb_path) # borrowed these from the running notebook server in the container diff --git a/rsconnect_jupyter/tests/test_environment.py b/rsconnect_jupyter/tests/test_environment.py index 97a38c77..d7f32848 100644 --- a/rsconnect_jupyter/tests/test_environment.py +++ b/rsconnect_jupyter/tests/test_environment.py @@ -17,26 +17,8 @@ def get_dir(self, name): def python_version(self): return '.'.join(map(str, sys.version_info[:3])) - def test_file(self): - result = detect_environment(self.get_dir('pip1')) - - pip_version = result.pop('pip') - self.assertTrue(version_re.match(pip_version)) - - locale = result.pop('locale') - self.assertIsInstance(locale, str) - self.assertIn('.', locale) - - self.assertEqual(result, { - 'package_manager': 'pip', - 'source': 'file', - 'filename': 'requirements.txt', - 'contents': 'numpy\npandas\nmatplotlib\n', - 'python': self.python_version(), - }) - def test_pip_freeze(self): - result = detect_environment(self.get_dir('pip2')) + result = detect_environment() contents = result.pop('contents') # these are the dependencies declared in our setup.py From 6801fee8b596f8b4c2aad0560b0de1776d7daff0 Mon Sep 17 00:00:00 2001 From: Michael Marchetti Date: Fri, 25 Jan 2019 10:46:40 -0500 Subject: [PATCH 3/8] reconcile with version from rsconnect R package --- rsconnect_jupyter/environment.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rsconnect_jupyter/environment.py b/rsconnect_jupyter/environment.py index 48007ca3..543f74bf 100644 --- a/rsconnect_jupyter/environment.py +++ b/rsconnect_jupyter/environment.py @@ -80,9 +80,8 @@ def output_file(dirname, filename, package_manager): with open(path, 'r') as f: data = f.read() - # TODO TODO TODO TODO data = '\n'.join([line for line in data.split('\n') - if 'rsconnect_jupyter' not in line]) + if 'rsconnect' not in line]) return { 'filename': filename, From 4b04fb462cacfd074d0b562941739b22b1dad465 Mon Sep 17 00:00:00 2001 From: Michael Marchetti Date: Fri, 25 Jan 2019 10:47:41 -0500 Subject: [PATCH 4/8] reconcile use of sys.executable --- rsconnect_jupyter/environment.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/rsconnect_jupyter/environment.py b/rsconnect_jupyter/environment.py index 543f74bf..16f8ae0d 100644 --- a/rsconnect_jupyter/environment.py +++ b/rsconnect_jupyter/environment.py @@ -45,24 +45,19 @@ def get_default_locale(): return '.'.join(locale.getdefaultlocale()) -def get_version(binary): - # use os.path.realpath to traverse any symlinks +def get_version(module): try: - binary_path = os.path.realpath(os.path.join(exec_dir, binary)) - if not os.path.isfile(binary_path): - raise EnvironmentException("File not found: %s" % binary_path) - - args = [binary_path, "--version"] + args = [sys.executable, '-m', module, '--version'] proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) stdout, stderr = proc.communicate() match = version_re.search(stdout) if match: return match.group() - msg = "Failed to get version of '%s' from the output of: %s --version" % (binary, binary_path) + msg = "Failed to get version of '%s' from the output of: %s" % (module, ' '.join(args)) raise EnvironmentException(msg) except Exception as exc: - raise EnvironmentException("Error getting '%s' version: %s" % (binary, str(exc))) + raise EnvironmentException("Error getting '%s' version: %s" % (module, str(exc))) def output_file(dirname, filename, package_manager): @@ -102,7 +97,7 @@ def pip_freeze(): """ try: proc = subprocess.Popen( - ['pip', 'freeze'], + [sys.executable, '-m', 'pip', 'freeze'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) pip_stdout, pip_stderr = proc.communicate() From 38ef39ff1c874732f516774d7ef25548b448567e Mon Sep 17 00:00:00 2001 From: Michael Marchetti Date: Fri, 25 Jan 2019 11:17:50 -0500 Subject: [PATCH 5/8] filter out rsconnect --- rsconnect_jupyter/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsconnect_jupyter/environment.py b/rsconnect_jupyter/environment.py index 543f74bf..916b13ec 100644 --- a/rsconnect_jupyter/environment.py +++ b/rsconnect_jupyter/environment.py @@ -115,7 +115,7 @@ def pip_freeze(): raise EnvironmentException('Error during pip freeze: %s' % msg) pip_stdout = '\n'.join([line for line in pip_stdout.split('\n') - if 'rsconnect-jupyter' not in line]) + if 'rsconnect' not in line]) return { 'filename': 'requirements.txt', From 567c6cf2ae8db6c5271a5a13de1ab3e208e981d4 Mon Sep 17 00:00:00 2001 From: Michael Marchetti Date: Fri, 25 Jan 2019 11:17:50 -0500 Subject: [PATCH 6/8] filter out rsconnect --- rsconnect_jupyter/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsconnect_jupyter/environment.py b/rsconnect_jupyter/environment.py index 16f8ae0d..f8c15aac 100644 --- a/rsconnect_jupyter/environment.py +++ b/rsconnect_jupyter/environment.py @@ -110,7 +110,7 @@ def pip_freeze(): raise EnvironmentException('Error during pip freeze: %s' % msg) pip_stdout = '\n'.join([line for line in pip_stdout.split('\n') - if 'rsconnect-jupyter' not in line]) + if 'rsconnect' not in line]) return { 'filename': 'requirements.txt', From b2effcd9bdf8a7a17640a73396789fe613fbfa19 Mon Sep 17 00:00:00 2001 From: Michael Marchetti Date: Mon, 28 Jan 2019 13:56:34 -0500 Subject: [PATCH 7/8] update docstring --- rsconnect_jupyter/environment.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/rsconnect_jupyter/environment.py b/rsconnect_jupyter/environment.py index f8c15aac..a2aaf566 100644 --- a/rsconnect_jupyter/environment.py +++ b/rsconnect_jupyter/environment.py @@ -18,9 +18,7 @@ class EnvironmentException(Exception): def detect_environment(): """Determine the python dependencies in the environment. - If requirements.txt exists in the notebook directory, - its contents will be used. Otherwise, the results - of `pip freeze` will be used. + `pip freeze` will be used to introspect the environment. Returns a dictionary containing the package spec filename and contents if successful, or a dictionary containing 'error' From 808cea60f3c15de161fa57b3625b0e31d15d6a1a Mon Sep 17 00:00:00 2001 From: Michael Marchetti Date: Tue, 29 Jan 2019 14:50:40 -0500 Subject: [PATCH 8/8] udpate docs --- docs/guide/README.md | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/docs/guide/README.md b/docs/guide/README.md index 9059fb50..9e6838be 100644 --- a/docs/guide/README.md +++ b/docs/guide/README.md @@ -131,16 +131,9 @@ kernel environment; that is, the environment where the `ipykernel` package is installed. In most cases that will be the same as the notebook server environment where `jupyter` is installed. -If there is a `requirements.txt` file in the same directory as the notebook -file, its contents will be used. This allows you to directly control which -packages will be installed on the RStudio Connect server before the notebook is -rendered. If you use this option, you must ensure that all necessary packages -are listed in the `requirements.txt` file. - -If there isn't a requirements file, the command `pip freeze` will be used to -inspect the environment. The output of `pip freeze` lists all packages currently -installed, as well as their versions, which enables RStudio Connect to recreate -the same environment. +The command `pip freeze` will be used to inspect the environment. The output +of `pip freeze` lists all packages currently installed, as well as their +versions, which enables RStudio Connect to recreate the same environment. ### Handling conflicts