From fd5f41759640299f31a5447f39d038ffcfaff417 Mon Sep 17 00:00:00 2001 From: Samuel Pangestu Date: Thu, 2 Jan 2025 12:21:35 -0800 Subject: [PATCH 1/4] Persisting CE settings **Description** persisting settings and mergin settings --- .../dirs/usr/local/bin/merge-settings-util.py | 23 +++++++++++++++ .../v2/dirs/usr/local/bin/start-code-editor | 28 +++++++++++++++++-- 2 files changed, 48 insertions(+), 3 deletions(-) create mode 100644 template/v2/dirs/usr/local/bin/merge-settings-util.py diff --git a/template/v2/dirs/usr/local/bin/merge-settings-util.py b/template/v2/dirs/usr/local/bin/merge-settings-util.py new file mode 100644 index 00000000..6ad58b76 --- /dev/null +++ b/template/v2/dirs/usr/local/bin/merge-settings-util.py @@ -0,0 +1,23 @@ +import json +from sys import argv + + +# merges json files file1 and file2, overwriting any settings that already exist in file1 +# todo: add error handling and logging +def main(): + file1, file2 = argv[1], argv[2] + # Read JSON data from files + with open(file1, 'r') as f1, open(file2, 'r') as f2: + data1 = json.load(f1) + data2 = json.load(f2) + + # Merge the data (simple update) + merged_data = {**data1, **data2} + + # Write the merged data to a new file + with open(file2, 'w') as f: + json.dump(merged_data, f) + + +if __name__ == "__main__": + main() diff --git a/template/v2/dirs/usr/local/bin/start-code-editor b/template/v2/dirs/usr/local/bin/start-code-editor index bc97106c..715bed18 100755 --- a/template/v2/dirs/usr/local/bin/start-code-editor +++ b/template/v2/dirs/usr/local/bin/start-code-editor @@ -1,6 +1,26 @@ #!/bin/bash set -e +override_settings() { + # create a new settings file with preset defaults or merge the defaults into the existing settings file + local persistent_settings_folder="/home/sagemaker-user/code_editor_machine_settings/" + local default_settings_folder="/home/sagemaker-user/sagemaker-code-editor-server-data/" + local settings_file_path_suffix="data/Machine/settings.json" + local persistent_machine_settings_file="${persistent_settings_folder}/${settings_file_path_suffix}" + local default_machine_settings_file="${default_settings_folder}/${settings_file_path_suffix}" + + if [ ! -f "$persistent_machine_settings_file" ]; then + # copy settings file to EBS if it doesn't exist in EBS + mkdir -p "$persistent_settings_folder" + cp "$default_machine_settings_file" "$persistent_machine_settings_file" + echo "Created persistent settings file with default settings at $persistent_machine_settings_file" + else + # if it does exist then merge settings + echo "File already exists: ${persistent_machine_settings_file}. Merging default settings with existing settings." + python3 /usr/local/bin/merge-settings-util.py "$default_machine_settings_file" "$persistent_machine_settings_file" + fi +} + eval "$(micromamba shell hook --shell=bash)" # Activate conda environment 'base', which is the default environment for sagemaker-distribution @@ -8,12 +28,14 @@ micromamba activate base # Start code-editor server if [ -n "$SAGEMAKER_APP_TYPE_LOWERCASE" ]; then - # SAGEMAKER_APP_TYPE is set, indicating the server is running within a SageMaker - # app. Configure the base url to be `//default`. + # SAGEMAKER_APP_TYPE is set, indicating the server is running within a SageMaker app. + override_settings + + # Configure the base url to be `//default`. sagemaker-code-editor --host 0.0.0.0 --port 8888 \ --without-connection-token \ --base-path "/$SAGEMAKER_APP_TYPE_LOWERCASE/default" \ - --server-data-dir /opt/amazon/sagemaker/sagemaker-code-editor-server-data \ + --server-data-dir /home/sagemaker-user/sagemaker-code-editor-server-data \ --extensions-dir /opt/amazon/sagemaker/sagemaker-code-editor-server-data/extensions \ --user-data-dir /opt/amazon/sagemaker/sagemaker-code-editor-user-data else From b48594246cffae0b0eca157c8fa5d087e5b9e496 Mon Sep 17 00:00:00 2001 From: Samuel Pangestu Date: Fri, 3 Jan 2025 19:14:50 -0800 Subject: [PATCH 2/4] Redirect CE web server error logs to stdout For debugging purposes --- .../v2/dirs/etc/supervisor/conf.d/supervisord-code-editor.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/template/v2/dirs/etc/supervisor/conf.d/supervisord-code-editor.conf b/template/v2/dirs/etc/supervisor/conf.d/supervisord-code-editor.conf index cac5669b..58e187d0 100644 --- a/template/v2/dirs/etc/supervisor/conf.d/supervisord-code-editor.conf +++ b/template/v2/dirs/etc/supervisor/conf.d/supervisord-code-editor.conf @@ -7,5 +7,6 @@ command=start-code-editor autostart=true autorestart=true stdout_logfile=/dev/fd/1 ; Redirect web server logs to stdout +stderr_logfile=/dev/fd/1 stdout_logfile_maxbytes = 0 ; Fix: https://github.com/Supervisor/supervisor/issues/935 stderr_logfile_maxbytes = 0 ; Fix: https://github.com/Supervisor/supervisor/issues/935 From 2bf77404bd1076f40cafd0e3dbf37d0af39b11c1 Mon Sep 17 00:00:00 2001 From: Samuel Pangestu Date: Tue, 7 Jan 2025 13:51:17 -0800 Subject: [PATCH 3/4] Fixes --- .../dirs/usr/local/bin/merge-settings-util.py | 7 ++-- .../v2/dirs/usr/local/bin/start-code-editor | 38 ++++++++++++++----- 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/template/v2/dirs/usr/local/bin/merge-settings-util.py b/template/v2/dirs/usr/local/bin/merge-settings-util.py index 6ad58b76..8b93cae1 100644 --- a/template/v2/dirs/usr/local/bin/merge-settings-util.py +++ b/template/v2/dirs/usr/local/bin/merge-settings-util.py @@ -2,12 +2,11 @@ from sys import argv -# merges json files file1 and file2, overwriting any settings that already exist in file1 -# todo: add error handling and logging +# merges json files file1 and file2, keys in file2 overwriting any keys that already exist in file1 def main(): file1, file2 = argv[1], argv[2] # Read JSON data from files - with open(file1, 'r') as f1, open(file2, 'r') as f2: + with open(file1, "r") as f1, open(file2, "r") as f2: data1 = json.load(f1) data2 = json.load(f2) @@ -15,7 +14,7 @@ def main(): merged_data = {**data1, **data2} # Write the merged data to a new file - with open(file2, 'w') as f: + with open(file1, "w") as f: json.dump(merged_data, f) diff --git a/template/v2/dirs/usr/local/bin/start-code-editor b/template/v2/dirs/usr/local/bin/start-code-editor index 715bed18..9d37c2a9 100755 --- a/template/v2/dirs/usr/local/bin/start-code-editor +++ b/template/v2/dirs/usr/local/bin/start-code-editor @@ -1,23 +1,41 @@ #!/bin/bash set -e -override_settings() { +EFS_MOUNT_POINT="/opt/amazon/sagemaker" +EBS_MOUNT_POINT="/home/sagemaker-user" + +persistent_settings_folder="${EBS_MOUNT_POINT}/sagemaker-code-editor-server-data" +default_settings_folder="${EFS_MOUNT_POINT}/sagemaker-code-editor-server-data" + +override_machine_settings() { # create a new settings file with preset defaults or merge the defaults into the existing settings file - local persistent_settings_folder="/home/sagemaker-user/code_editor_machine_settings/" - local default_settings_folder="/home/sagemaker-user/sagemaker-code-editor-server-data/" - local settings_file_path_suffix="data/Machine/settings.json" + local settings_relative_path="data/Machine" + local settings_file_path_suffix="${settings_relative_path}/settings.json" local persistent_machine_settings_file="${persistent_settings_folder}/${settings_file_path_suffix}" local default_machine_settings_file="${default_settings_folder}/${settings_file_path_suffix}" if [ ! -f "$persistent_machine_settings_file" ]; then # copy settings file to EBS if it doesn't exist in EBS - mkdir -p "$persistent_settings_folder" + mkdir -p "${persistent_settings_folder}/${settings_relative_path}" cp "$default_machine_settings_file" "$persistent_machine_settings_file" echo "Created persistent settings file with default settings at $persistent_machine_settings_file" else # if it does exist then merge settings echo "File already exists: ${persistent_machine_settings_file}. Merging default settings with existing settings." - python3 /usr/local/bin/merge-settings-util.py "$default_machine_settings_file" "$persistent_machine_settings_file" + python3 /usr/local/bin/merge-settings-util.py "$persistent_machine_settings_file" "$default_machine_settings_file" + fi +} + +copy_user_settings() { + local settings_relative_path="data/User" + local settings_file_path_suffix="${settings_relative_path}/settings.json" + local persistent_user_settings_file="${persistent_settings_folder}/${settings_file_path_suffix}" + local default_user_settings_file="${default_settings_folder}/${settings_file_path_suffix}" + if [ ! -f "$persistent_user_settings_file" ]; then + # copy user settings file to EBS if it doesn't exist in EBS + mkdir -p "${persistent_settings_folder}/${settings_relative_path}" + cp "$default_user_settings_file" "$persistent_user_settings_file" + echo "Created persistent settings file with default settings at $persistent_user_settings_file" fi } @@ -29,13 +47,13 @@ micromamba activate base # Start code-editor server if [ -n "$SAGEMAKER_APP_TYPE_LOWERCASE" ]; then # SAGEMAKER_APP_TYPE is set, indicating the server is running within a SageMaker app. - override_settings - + override_machine_settings + copy_user_settings # Configure the base url to be `//default`. sagemaker-code-editor --host 0.0.0.0 --port 8888 \ --without-connection-token \ --base-path "/$SAGEMAKER_APP_TYPE_LOWERCASE/default" \ - --server-data-dir /home/sagemaker-user/sagemaker-code-editor-server-data \ + --server-data-dir $persistent_settings_folder \ --extensions-dir /opt/amazon/sagemaker/sagemaker-code-editor-server-data/extensions \ --user-data-dir /opt/amazon/sagemaker/sagemaker-code-editor-user-data else @@ -44,4 +62,4 @@ else --server-data-dir /opt/amazon/sagemaker/sagemaker-code-editor-server-data \ --extension-dir /opt/amazon/sagemaker/sagemaker-code-editor-server-data/extensions \ --user-data-dir /opt/amazon/sagemaker/sagemaker-code-editor-user-data -fi +fi \ No newline at end of file From e32e524ff4f5d57e0019b15471680ee694389c02 Mon Sep 17 00:00:00 2001 From: Samuel Pangestu Date: Tue, 14 Jan 2025 17:03:13 -0800 Subject: [PATCH 4/4] Support CodeEditor persistent settings in v3 **Description** Add CodeEditor persistent settings to v3 template --- .../conf.d/supervisord-code-editor.conf | 1 + .../dirs/usr/local/bin/merge-settings-util.py | 22 +++++++++ .../v3/dirs/usr/local/bin/start-code-editor | 46 +++++++++++++++++-- 3 files changed, 66 insertions(+), 3 deletions(-) create mode 100644 template/v3/dirs/usr/local/bin/merge-settings-util.py diff --git a/template/v3/dirs/etc/supervisor/conf.d/supervisord-code-editor.conf b/template/v3/dirs/etc/supervisor/conf.d/supervisord-code-editor.conf index cac5669b..58e187d0 100644 --- a/template/v3/dirs/etc/supervisor/conf.d/supervisord-code-editor.conf +++ b/template/v3/dirs/etc/supervisor/conf.d/supervisord-code-editor.conf @@ -7,5 +7,6 @@ command=start-code-editor autostart=true autorestart=true stdout_logfile=/dev/fd/1 ; Redirect web server logs to stdout +stderr_logfile=/dev/fd/1 stdout_logfile_maxbytes = 0 ; Fix: https://github.com/Supervisor/supervisor/issues/935 stderr_logfile_maxbytes = 0 ; Fix: https://github.com/Supervisor/supervisor/issues/935 diff --git a/template/v3/dirs/usr/local/bin/merge-settings-util.py b/template/v3/dirs/usr/local/bin/merge-settings-util.py new file mode 100644 index 00000000..8b93cae1 --- /dev/null +++ b/template/v3/dirs/usr/local/bin/merge-settings-util.py @@ -0,0 +1,22 @@ +import json +from sys import argv + + +# merges json files file1 and file2, keys in file2 overwriting any keys that already exist in file1 +def main(): + file1, file2 = argv[1], argv[2] + # Read JSON data from files + with open(file1, "r") as f1, open(file2, "r") as f2: + data1 = json.load(f1) + data2 = json.load(f2) + + # Merge the data (simple update) + merged_data = {**data1, **data2} + + # Write the merged data to a new file + with open(file1, "w") as f: + json.dump(merged_data, f) + + +if __name__ == "__main__": + main() diff --git a/template/v3/dirs/usr/local/bin/start-code-editor b/template/v3/dirs/usr/local/bin/start-code-editor index bc97106c..d331ce4f 100755 --- a/template/v3/dirs/usr/local/bin/start-code-editor +++ b/template/v3/dirs/usr/local/bin/start-code-editor @@ -1,6 +1,44 @@ #!/bin/bash set -e +EFS_MOUNT_POINT="/opt/amazon/sagemaker" +EBS_MOUNT_POINT="/home/sagemaker-user" + +persistent_settings_folder="${EBS_MOUNT_POINT}/sagemaker-code-editor-server-data" +default_settings_folder="${EFS_MOUNT_POINT}/sagemaker-code-editor-server-data" + +override_machine_settings() { + # create a new settings file with preset defaults or merge the defaults into the existing settings file + local settings_relative_path="data/Machine" + local settings_file_path_suffix="${settings_relative_path}/settings.json" + local persistent_machine_settings_file="${persistent_settings_folder}/${settings_file_path_suffix}" + local default_machine_settings_file="${default_settings_folder}/${settings_file_path_suffix}" + + if [ ! -f "$persistent_machine_settings_file" ]; then + # copy settings file to EBS if it doesn't exist in EBS + mkdir -p "${persistent_settings_folder}/${settings_relative_path}" + cp "$default_machine_settings_file" "$persistent_machine_settings_file" + echo "Created persistent settings file with default settings at $persistent_machine_settings_file" + else + # if it does exist then merge settings + echo "File already exists: ${persistent_machine_settings_file}. Merging default settings with existing settings." + python3 /usr/local/bin/merge-settings-util.py "$persistent_machine_settings_file" "$default_machine_settings_file" + fi +} + +copy_user_settings() { + local settings_relative_path="data/User" + local settings_file_path_suffix="${settings_relative_path}/settings.json" + local persistent_user_settings_file="${persistent_settings_folder}/${settings_file_path_suffix}" + local default_user_settings_file="${default_settings_folder}/${settings_file_path_suffix}" + if [ ! -f "$persistent_user_settings_file" ]; then + # copy user settings file to EBS if it doesn't exist in EBS + mkdir -p "${persistent_settings_folder}/${settings_relative_path}" + cp "$default_user_settings_file" "$persistent_user_settings_file" + echo "Created persistent settings file with default settings at $persistent_user_settings_file" + fi +} + eval "$(micromamba shell hook --shell=bash)" # Activate conda environment 'base', which is the default environment for sagemaker-distribution @@ -8,12 +46,14 @@ micromamba activate base # Start code-editor server if [ -n "$SAGEMAKER_APP_TYPE_LOWERCASE" ]; then - # SAGEMAKER_APP_TYPE is set, indicating the server is running within a SageMaker - # app. Configure the base url to be `//default`. + # SAGEMAKER_APP_TYPE is set, indicating the server is running within a SageMaker app. + override_machine_settings + copy_user_settings + # Configure the base url to be `//default`. sagemaker-code-editor --host 0.0.0.0 --port 8888 \ --without-connection-token \ --base-path "/$SAGEMAKER_APP_TYPE_LOWERCASE/default" \ - --server-data-dir /opt/amazon/sagemaker/sagemaker-code-editor-server-data \ + --server-data-dir $persistent_settings_folder \ --extensions-dir /opt/amazon/sagemaker/sagemaker-code-editor-server-data/extensions \ --user-data-dir /opt/amazon/sagemaker/sagemaker-code-editor-user-data else