From de29d7a7dc9fcdcee69ddd042dc8ddb3dd0df997 Mon Sep 17 00:00:00 2001 From: Waleed Date: Thu, 19 Nov 2020 15:32:42 +0200 Subject: [PATCH 1/7] remove ipv6 check for farms selection (#1707) only ipv4 is required for 3bot --- jumpscale/packages/marketplace/chats/gitea.py | 1 - jumpscale/packages/marketplace/chats/mattermost.py | 1 - jumpscale/packages/marketplace/chats/meetings.py | 1 - jumpscale/packages/marketplace/chats/publisher.py | 1 - jumpscale/packages/marketplace/chats/taiga.py | 1 - jumpscale/packages/threebot_deployer/chats/threebot.py | 3 +-- jumpscale/sals/marketplace/apps_chatflow.py | 10 +++------- 7 files changed, 4 insertions(+), 14 deletions(-) diff --git a/jumpscale/packages/marketplace/chats/gitea.py b/jumpscale/packages/marketplace/chats/gitea.py index cb2f7b84e4..312eebddea 100644 --- a/jumpscale/packages/marketplace/chats/gitea.py +++ b/jumpscale/packages/marketplace/chats/gitea.py @@ -71,7 +71,6 @@ def _deploy(self): env=var_dict, interactive=False, entrypoint="/start_gitea.sh", - public_ipv6=True, disk_size=self.container_resources["sru"] * 1024, secret_env={"POSTGRES_PASSWORD": self.database_password}, solution_uuid=self.solution_id, diff --git a/jumpscale/packages/marketplace/chats/mattermost.py b/jumpscale/packages/marketplace/chats/mattermost.py index 7b89e5cae0..1cfa367f60 100644 --- a/jumpscale/packages/marketplace/chats/mattermost.py +++ b/jumpscale/packages/marketplace/chats/mattermost.py @@ -98,7 +98,6 @@ def _deploy(self): interactive=False, entrypoint="/start_mattermost.sh", volumes=volume_config, - public_ipv6=True, solution_uuid=self.solution_id, **self.solution_metadata, ) diff --git a/jumpscale/packages/marketplace/chats/meetings.py b/jumpscale/packages/marketplace/chats/meetings.py index 5e359d6b10..4d7d782d67 100644 --- a/jumpscale/packages/marketplace/chats/meetings.py +++ b/jumpscale/packages/marketplace/chats/meetings.py @@ -69,7 +69,6 @@ def _deploy(self): disk_size=self.resources["sru"] * 1024, interactive=False, entrypoint="/entrypoint.sh", - public_ipv6=True, solution_uuid=self.solution_id, **self.solution_metadata, ) diff --git a/jumpscale/packages/marketplace/chats/publisher.py b/jumpscale/packages/marketplace/chats/publisher.py index 89595c84e5..258eadb23b 100644 --- a/jumpscale/packages/marketplace/chats/publisher.py +++ b/jumpscale/packages/marketplace/chats/publisher.py @@ -145,7 +145,6 @@ def _deploy(self): secret_env=secret_env, interactive=False, solution_uuid=self.solution_id, - public_ipv6=True, **self.solution_metadata, ) ) diff --git a/jumpscale/packages/marketplace/chats/taiga.py b/jumpscale/packages/marketplace/chats/taiga.py index 3b98ba50a9..036372f9b1 100644 --- a/jumpscale/packages/marketplace/chats/taiga.py +++ b/jumpscale/packages/marketplace/chats/taiga.py @@ -86,7 +86,6 @@ def _deploy(self): env=var_dict, interactive=False, entrypoint="/start_taiga.sh", - public_ipv6=True, secret_env={ "EMAIL_HOST_PASSWORD": self.EMAIL_HOST_PASSWORD, "PRIVATE_KEY": private_key, diff --git a/jumpscale/packages/threebot_deployer/chats/threebot.py b/jumpscale/packages/threebot_deployer/chats/threebot.py index 7e84dc7cb0..8af551cae4 100644 --- a/jumpscale/packages/threebot_deployer/chats/threebot.py +++ b/jumpscale/packages/threebot_deployer/chats/threebot.py @@ -67,7 +67,6 @@ def _threebot_start(self): self.query = {"cru": 2, "mru": 2, "sru": 2.25} self.container_resources = {"cru": 1, "mru": 1, "sru": 2} self.expiration = 60 * 60 # 60 minutes for 3bot - self.ip_version = "IPv6" self.retries = 3 self.allow_custom_domain = False self.custom_domain = False @@ -232,7 +231,7 @@ def _select_farm(self): @chatflow_step(title="Deployment location policy") def choose_location(self): - self._get_available_farms() + self._get_available_farms(only_one=False) self.farms_by_continent = deployer.group_farms_by_continent(self.available_farms) choices = ["Automatic", "Farm", "Specific node"] if self.farms_by_continent: diff --git a/jumpscale/sals/marketplace/apps_chatflow.py b/jumpscale/sals/marketplace/apps_chatflow.py index 6db389f96c..ae94f94a3c 100644 --- a/jumpscale/sals/marketplace/apps_chatflow.py +++ b/jumpscale/sals/marketplace/apps_chatflow.py @@ -33,7 +33,6 @@ def _init_solution(self): self.username = self.user_info()["username"] self.solution_metadata["owner"] = self.username self.threebot_name = j.data.text.removesuffix(self.username, ".3bot") - self.ip_version = "IPv6" self.expiration = 60 * 60 * 3 # expiration 3 hours self.retries = 3 self.custom_domain = False @@ -159,7 +158,7 @@ def _get_pool(self): return self.pool_id def _select_node(self): - self.selected_node = deployer.schedule_container(self.pool_id, ip_version=self.ip_version, **self.query) + self.selected_node = deployer.schedule_container(self.pool_id, **self.query) @chatflow_step(title="New Expiration") def set_expiration(self): @@ -439,12 +438,9 @@ def _get_available_farms(self, only_one=True, identity_name=None): for farm in farms: farm_name = farm.name available_ipv4, _, _, _, _ = deployer.check_farm_capacity( - farm_name, currencies=[self.currency], ip_version="IPv4" + farm_name, currencies=[self.currency], ip_version="IPv4", **self.query ) - available_ipv6, _, _, _, _ = deployer.check_farm_capacity( - farm_name, currencies=[self.currency], ip_version="IPv6", **self.query - ) - if available_ipv4 and available_ipv6: + if available_ipv4: self.available_farms.append(farm) if only_one: return From 29173f54bd5b1844c1c7954e48917a576aab4697 Mon Sep 17 00:00:00 2001 From: Maged Motawea Date: Sun, 22 Nov 2020 14:05:28 +0200 Subject: [PATCH 2/7] fix ssh key cleaning (#1718) --- jumpscale/packages/tfgrid_solutions/chats/gollum.py | 2 +- jumpscale/packages/tfgrid_solutions/chats/kubernetes.py | 2 +- jumpscale/packages/tfgrid_solutions/chats/minio.py | 2 +- jumpscale/packages/tfgrid_solutions/chats/monitoring.py | 2 +- jumpscale/packages/tfgrid_solutions/chats/ubuntu.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/jumpscale/packages/tfgrid_solutions/chats/gollum.py b/jumpscale/packages/tfgrid_solutions/chats/gollum.py index 1d89284cec..f77ff2ece5 100644 --- a/jumpscale/packages/tfgrid_solutions/chats/gollum.py +++ b/jumpscale/packages/tfgrid_solutions/chats/gollum.py @@ -72,7 +72,7 @@ def gollum_network(self): def public_key_get(self): self.public_key = self.upload_file( """Please upload your public SSH key to be able to access the depolyed container via ssh""", required=True - ).split("\n")[0] + ).strip() @chatflow_step(title="Email") def gollum_email(self): diff --git a/jumpscale/packages/tfgrid_solutions/chats/kubernetes.py b/jumpscale/packages/tfgrid_solutions/chats/kubernetes.py index a419b449a1..41ba5961fe 100644 --- a/jumpscale/packages/tfgrid_solutions/chats/kubernetes.py +++ b/jumpscale/packages/tfgrid_solutions/chats/kubernetes.py @@ -74,7 +74,7 @@ def public_key_get(self): """Please upload your public SSH key to be able to access the depolyed container via ssh Note: please use keys compatible with Dropbear server eg: RSA""", required=True, - ).split("\n") + ).splitlines() self.cluster_secret = self.string_ask("Please add the cluster secret", default="secret", required=True) diff --git a/jumpscale/packages/tfgrid_solutions/chats/minio.py b/jumpscale/packages/tfgrid_solutions/chats/minio.py index e8e22e5121..05603cba2d 100644 --- a/jumpscale/packages/tfgrid_solutions/chats/minio.py +++ b/jumpscale/packages/tfgrid_solutions/chats/minio.py @@ -174,7 +174,7 @@ def public_key(self): Just upload the file with the key. (Optional)""" ) if public_key_file: - self.public_ssh_key = public_key_file.split("\n")[0] + self.public_ssh_key = public_key_file.strip() else: self.public_ssh_key = "" diff --git a/jumpscale/packages/tfgrid_solutions/chats/monitoring.py b/jumpscale/packages/tfgrid_solutions/chats/monitoring.py index 5829169cf8..bac849e12c 100644 --- a/jumpscale/packages/tfgrid_solutions/chats/monitoring.py +++ b/jumpscale/packages/tfgrid_solutions/chats/monitoring.py @@ -63,7 +63,7 @@ def public_key_get(self): Just upload the file with the key. Note: please use keys compatible with Dropbear server eg: rsa """, required=True, - ).split("\n")[0] + ).strip() @chatflow_step(title="Prometheus container resources") def prometheus_container_resources(self): diff --git a/jumpscale/packages/tfgrid_solutions/chats/ubuntu.py b/jumpscale/packages/tfgrid_solutions/chats/ubuntu.py index 5a4bd8d49d..5470469497 100644 --- a/jumpscale/packages/tfgrid_solutions/chats/ubuntu.py +++ b/jumpscale/packages/tfgrid_solutions/chats/ubuntu.py @@ -91,7 +91,7 @@ def container_logs(self): def public_key_get(self): self.public_key = self.upload_file( """Please upload your public SSH key to be able to access the depolyed container via ssh""", required=True, - ).split("\n")[0] + ).strip() @chatflow_step(title="Global IPv6 Address") def ipv6_config(self): From 85c97f2e858c67bd92102c6b0ed01b859f6d14d6 Mon Sep 17 00:00:00 2001 From: Maged Motawea Date: Sun, 22 Nov 2020 14:23:37 +0200 Subject: [PATCH 3/7] check test cert before certificate limit (#1719) --- jumpscale/sals/marketplace/apps_chatflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jumpscale/sals/marketplace/apps_chatflow.py b/jumpscale/sals/marketplace/apps_chatflow.py index ae94f94a3c..6da9fbe6e2 100644 --- a/jumpscale/sals/marketplace/apps_chatflow.py +++ b/jumpscale/sals/marketplace/apps_chatflow.py @@ -316,7 +316,7 @@ def _get_domain(self): else: deployer.unblock_managed_domain(domain) try: - if j.sals.crtsh.has_reached_limit(domain): + if not j.core.config.get("TEST_CERT") and j.sals.crtsh.has_reached_limit(domain): continue except requests.exceptions.HTTPError: is_http_failure = True From bbd140da33af239a5fd582c9c090805c96484fa6 Mon Sep 17 00:00:00 2001 From: Waleed Date: Tue, 24 Nov 2020 15:29:52 +0200 Subject: [PATCH 4/7] fix wg locations and threebot deployer image build (#1724) --- jumpscale/install/Dockerfile | 11 ++++++----- jumpscale/sals/reservation_chatflow/deployer.py | 6 +++--- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/jumpscale/install/Dockerfile b/jumpscale/install/Dockerfile index c0e04fa816..10824ad7c4 100644 --- a/jumpscale/install/Dockerfile +++ b/jumpscale/install/Dockerfile @@ -1,13 +1,11 @@ FROM threefoldtech/phusion:20.04 ARG BRANCH -RUN apt-get update && apt-get install curl wget git python3-pip python3-venv redis-server tmux nginx restic musl musl-tools -y -RUN curl https://sh.rustup.rs -sSf | sh -s -- -y && cp ~/.cargo/bin/* /usr/local/bin/ +RUN apt-get update && apt-get install curl wget git python3-pip python3-venv redis-server tmux nginx restic -y + RUN pip3 install poetry RUN mkdir -p /sandbox/code/github/threefoldtech -RUN git clone https://github.com/threefoldtech/zinit.git /sandbox/code/github/threefoldtech/zinit -b master -WORKDIR /sandbox/code/github/threefoldtech/zinit -RUN make && cp target/x86_64-unknown-linux-musl/release/zinit /sbin/zinit + RUN git clone https://github.com/threefoldtech/js-sdk.git /sandbox/code/github/threefoldtech/js-sdk -b $BRANCH WORKDIR /sandbox/code/github/threefoldtech/js-sdk RUN poetry config virtualenvs.create false && poetry install @@ -15,6 +13,9 @@ RUN poetry shell RUN /etc/init.d/redis-server start RUN python3 jumpscale/install/codeserver-install.py +RUN wget https://github.com/threefoldtech/zinit/releases/download/v0.1/zinit -O /sbin/zinit \ + && chmod +x /sbin/zinit + COPY rootfs / ENTRYPOINT [ "zinit", "init" ] diff --git a/jumpscale/sals/reservation_chatflow/deployer.py b/jumpscale/sals/reservation_chatflow/deployer.py index 4ed85c1305..ee83901a52 100644 --- a/jumpscale/sals/reservation_chatflow/deployer.py +++ b/jumpscale/sals/reservation_chatflow/deployer.py @@ -695,10 +695,10 @@ def deploy_network(self, name, access_node, ip_range, ip_version, pool_id, ident wg_quick = j.sals.zos.get(identity_name).network.add_access( network, access_node.node_id, str(next(node_subnets)), ipv4=use_ipv4 ) - network_config["wg"] = wg_quick - j.sals.fs.mkdir(f"{j.core.dirs.CFGDIR}/wireguard/") - j.sals.fs.write_file(f"{j.core.dirs.CFGDIR}/{name}.conf", f"{wg_quick}") + wg_dir = j.sals.fs.join_paths(j.core.dirs.CFGDIR, "wireguard") + j.sals.fs.mkdirs(wg_dir) + j.sals.fs.write_file(j.sals.fs.join_paths(wg_dir, f"{identity_name}_{name}.conf"), wg_quick) ids = [] parent_id = None From 5b4949df44efd592ee626c0b53eb29575cb91e77 Mon Sep 17 00:00:00 2001 From: Waleed Date: Tue, 24 Nov 2020 16:54:11 +0200 Subject: [PATCH 5/7] bump version b13 (#1731) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index bd6c8ceb37..997c1f4500 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ name = "js-sdk" packages = [{ include = "jumpscale" }] -version = "11.1a1" +version = "11.0b13" description = "SDK for threefold grid" authors = ["xmonader "] license = "Apache-2.0" From 1a38cb53fdcb878c1bc040a714845a63218bb675 Mon Sep 17 00:00:00 2001 From: Christophe de Carvalho Date: Tue, 24 Nov 2020 15:56:14 +0100 Subject: [PATCH 6/7] add support for k8s sizes required for VDC (#1729) --- jumpscale/clients/explorer/models.py | 9 ++++++--- jumpscale/sals/zos/kubernetes.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/jumpscale/clients/explorer/models.py b/jumpscale/clients/explorer/models.py index c829763710..9e3e8e55bb 100644 --- a/jumpscale/clients/explorer/models.py +++ b/jumpscale/clients/explorer/models.py @@ -420,9 +420,12 @@ def resource_units(self): 9: {"cru": 8, "mru": 32, "sru": 100}, 10: {"cru": 8, "mru": 32, "sru": 200}, 11: {"cru": 8, "mru": 32, "sru": 800}, - 12: {"cru": 1, "mru": 64, "sru": 200}, - 13: {"cru": 1, "mru": 64, "sru": 400}, - 14: {"cru": 1, "mru": 64, "sru": 800}, + 12: {"cru": 16, "mru": 64, "sru": 200}, + 13: {"cru": 16, "mru": 64, "sru": 400}, + 14: {"cru": 16, "mru": 64, "sru": 800}, + 15: {"cru": 1, "mru": 2, "sru": 25}, + 16: {"cru": 2, "mru": 4, "sru": 50}, + 17: {"cru": 4, "mru": 8, "sru": 50}, } resource_units = ResourceUnitAmount() diff --git a/jumpscale/sals/zos/kubernetes.py b/jumpscale/sals/zos/kubernetes.py index 9c3e69ed61..eb4f45b376 100644 --- a/jumpscale/sals/zos/kubernetes.py +++ b/jumpscale/sals/zos/kubernetes.py @@ -42,7 +42,7 @@ def add_master( Input: if size is not supported """ - if size not in range(1, 15): + if size not in range(1, 18): raise Input(f"VM size {size} is not supported") master = K8s() From c42f76866274d2eb1c26098cbe4249bd5d8dc38b Mon Sep 17 00:00:00 2001 From: Mostafa Hassan <31102305+mhost39@users.noreply.github.com> Date: Tue, 24 Nov 2020 16:58:05 +0200 Subject: [PATCH 7/7] add typo-ci file to ignore python files (#1726) --- .typo-ci.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .typo-ci.yml diff --git a/.typo-ci.yml b/.typo-ci.yml new file mode 100644 index 0000000000..93e63f26dd --- /dev/null +++ b/.typo-ci.yml @@ -0,0 +1,20 @@ +# This is a sample .typo-ci.yml file, it's used to configure how Typo CI will behave. +# Add it to the root of your project and push it to github. + +# What language dictionaries should it use? Currently Typo CI supports: +# en + +dictionaries: + - en + +# Any files/folders we should ignore? +excluded_files: + - "*.py" + - ".typo-ci.yml" + +# Any typos we should ignore? +excluded_words: + - typoci + +# Would you like filenames to also be spellchecked? +spellcheck_filenames: true