diff --git a/.github/workflows/js-sdk.yml b/.github/workflows/js-sdk.yml index fa9ffc6e62..b087469e1f 100644 --- a/.github/workflows/js-sdk.yml +++ b/.github/workflows/js-sdk.yml @@ -17,7 +17,7 @@ jobs: - name: Gathering deps run: | sudo apt-get update - sudo apt-get install -y git python3-pip python3-venv python3-setuptools tmux redis + sudo apt-get install -y git python3-pip python3-venv python3-setuptools tmux redis nginx sudo pip3 install poetry sudo poetry run pip3 install pytest - name: Install diff --git a/docs/architecture/decisions/0008-add-kwargs-to-3bot-start.md b/docs/architecture/decisions/0008-add-kwargs-to-3bot-start.md new file mode 100644 index 0000000000..38a7f6ae3c --- /dev/null +++ b/docs/architecture/decisions/0008-add-kwargs-to-3bot-start.md @@ -0,0 +1,19 @@ +# 8. add_kwargs_to_3bot_start + +Date: 2020-09-27 + +## Status + +Accepted + +## Context + +Adding packages with kwargs has some limitations and hence kwargs are needed every time start is called not only once when adding package. + +## Decision + +Add kwargs passed to the package instance that will be saved locally, and can be retrieved everytime the threebot server restarts and starts the package. + +## Consequences + +Any package that is added with kwargs will save them and hence with every restart of the package, they are reloaded and used in the install of the package. diff --git a/docs/wiki/tutorials/add_marketplace_chatflow.md b/docs/wiki/tutorials/add_marketplace_chatflow.md index 5910d9d45d..3207bea428 100644 --- a/docs/wiki/tutorials/add_marketplace_chatflow.md +++ b/docs/wiki/tutorials/add_marketplace_chatflow.md @@ -70,12 +70,34 @@ - And for returning your solution count should append your `{SOLUTION_TYPE}` in the `count_dict` in `count_solutions` method in the same module ## Add app in frontend -- In the frontend, you just need to add your app object in `apps` dict in `packages/marketplace/frontend/App.vue` - ```js - { - name: "App Name in frontend", - type: "{SOLUTION_TYPE}", - path: "/{your_app_name}", - meta: { icon: "app_icon" }, - } - ``` +- In the frontend, you just need to add your app object as below in `apps` dict under the section you want to list your app in `packages/marketplace/frontend/data.js` + - If you need to add another section, just create new one in the `SECTIONS` object with the same structure: + ```js + "SECTION NAME": { + titleToolTip: "Tooltip shown on hovering on section title in the frontend", + apps: { + // list your applications objects as below structure + "App Name": { + name: "App Name in frontend", + type: "{SOLUTION_TYPE}", // defined in the previous steps + image: "./assets/appImage.png", // add your app image in the assets dir + disable: false, // make it true if you want to hide your app in the marketplace frontend + helpLink: "https://now10.threefold.io/docs", // link to application manual + description: "Description of your application" + }, + }, + }, + ``` + - If you just need to add your application in an existing section, add a new app object with below structure in the section object you want to list in: + ```js + { + "App Name": { + name: "App Name in frontend", + type: "{SOLUTION_TYPE}", // defined in the previous steps + image: "./assets/appImage.png", // add your app image in the assets dir + disable: false, // make it true if you want to hide your app in the marketplace frontend + helpLink: "https://now10.threefold.io/docs", // link to application manual + description: "Description of your application" + }, + } + ``` diff --git a/examplescripts/minio.py b/examplescripts/minio.py index 409f22df67..37d6ffc840 100644 --- a/examplescripts/minio.py +++ b/examplescripts/minio.py @@ -18,7 +18,7 @@ PASSWORD = "supersecurepassowrd" network_name = str(uuid.uuid4()) print(f"network name: {network_name}") -BAD_NODES = set([]) +BAD_NODES = set(["A7FmQZ72h7FzjkJMGXmzLDFyfyxzitDZYuernGG97nv7"]) UP_FOR = 60 * 20 # number of seconds @@ -55,6 +55,21 @@ def wait_workload(wid): workload = zos.workloads.get(wid) +def wait_zdb_workloads(zdb_wids): + # Looks like the workload_id can be set before the namespace + for wid in zdb_wids: + workload = zos.workloads.get(wid) + data = j.data.serializers.json.loads(workload.info.result.data_json) + if workload.info.result.message: + x = workload.info.result.message + raise Exception(f"Failed to initialize ZDB: {x}") + elif data.get("IP") or data.get("IPs"): + return + else: + sleep(1) + continue + + def wait_pools(pools): for pool in pools: while pool.cus == 0: @@ -286,6 +301,7 @@ def pick_minio_nodes(nodes): backup_vol_id = deploy_volume(minio_backup_node.node_id, backup_pool) zdb_wids = [x.id for x in zdb_workloads] wait_workloads(zdb_wids) +wait_zdb_workloads(zdb_wids) wait_workload(tlog_workload.id) wait_workload(master_vol_id) wait_workload(backup_vol_id) @@ -325,6 +341,7 @@ def pick_minio_nodes(nodes): zdb_new_workloads = deploy_zdbs(zdb_later_nodes, zdb_new_pools) zdb_new_wids = [x.id for x in zdb_new_workloads] wait_workloads(zdb_new_wids) +wait_zdb_workloads(zdb_new_wids) new_namespace_config = get_namespace_config(zdb_new_workloads) print("Removing three backup storages") diff --git a/jumpscale/clients/explorer/explorer.py b/jumpscale/clients/explorer/explorer.py index 31085d45a9..2978cd2fa1 100644 --- a/jumpscale/clients/explorer/explorer.py +++ b/jumpscale/clients/explorer/explorer.py @@ -18,6 +18,15 @@ from .users import Users from .workloads import Workloads +from jumpscale.loader import j + + +def log_request(r, *args, **kwargs): + if j.config.get("EXPLORER_LOGS"): + j.logger.debug( + f"Request {r.request.url} method: {r.request.method} body: {r.request.body} headers: {r.request.headers}" + ) + class Explorer(Client): url = fields.String() @@ -30,7 +39,7 @@ def __init__(self, url=None, identity_name=None, **kwargs): else: self._loaded_identity = identity.get_identity() self._session = requests.Session() - self._session.hooks = dict(response=raise_for_status) + self._session.hooks = dict(response=[log_request, raise_for_status]) secret = self._loaded_identity.nacl.signing_key.encode(Base64Encoder) auth = HTTPSignatureAuth( diff --git a/jumpscale/clients/explorer/models.py b/jumpscale/clients/explorer/models.py index 255dce0b5a..abd1adb7b0 100644 --- a/jumpscale/clients/explorer/models.py +++ b/jumpscale/clients/explorer/models.py @@ -42,6 +42,7 @@ class Location(Base): def __str__(self): return ",".join([x for x in [self.continent, self.country, self.city] if x]) + class Farm(Base): id = fields.Integer() threebot_id = fields.Integer() @@ -55,7 +56,6 @@ class Farm(Base): def __str__(self): return " - ".join([x for x in [self.name, str(self.location)] if x]) - class WorkloadsAmount(Base): @@ -250,9 +250,14 @@ class ContainerNetworkConnection(Base): class ContainerLogsRedis(Base): + # deprecated, please use secret_stdout instead stdout = fields.String(default="") + # deprecated, please use secret_stderr instead stderr = fields.String(default="") + secret_stdout = fields.String(default="") + secret_stderr = fields.String(default="") + class ContainerLogs(Base): type = fields.String(default="") diff --git a/jumpscale/clients/sendgrid/sendgrid.py b/jumpscale/clients/sendgrid/sendgrid.py index a9fe7d764c..22b6e3df43 100644 --- a/jumpscale/clients/sendgrid/sendgrid.py +++ b/jumpscale/clients/sendgrid/sendgrid.py @@ -12,8 +12,8 @@ class SendGridClient(Client): apikey = fields.String() - def __init__(self): - super().__init__() + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) def build_attachment(self, filepath, typ="application/pdf"): """ diff --git a/jumpscale/clients/stellar/stellar.py b/jumpscale/clients/stellar/stellar.py index 142c578185..c32dd980ef 100644 --- a/jumpscale/clients/stellar/stellar.py +++ b/jumpscale/clients/stellar/stellar.py @@ -387,7 +387,7 @@ def transfer( if asset != "XLM": assetStr = asset.split(":") if len(assetStr) != 2: - raise Exception("Wrong asset format should be in format 'assetcode:issuer'") + raise Exception(f"Wrong asset format should be in format 'assetcode:issuer', but received {assetStr}") asset = assetStr[0] issuer = assetStr[1] diff --git a/jumpscale/core/identity/__init__.py b/jumpscale/core/identity/__init__.py index f105ffa8b8..8b86693a16 100644 --- a/jumpscale/core/identity/__init__.py +++ b/jumpscale/core/identity/__init__.py @@ -12,9 +12,9 @@ from jumpscale.sals.nettools import get_default_ip_config DEFAULT_EXPLORER_URLS = { - "mainnet": "https://explorer.grid.tf/api/v1/", - "testnet": "https://explorer.testnet.grid.tf/api/v1/", - "devnet": "https://explorer.devnet.grid.tf/api/v1/", + "mainnet": "https://explorer.grid.tf/api/v1", + "testnet": "https://explorer.testnet.grid.tf/api/v1", + "devnet": "https://explorer.devnet.grid.tf/api/v1", } EXPLORER_URLS = js_config.set_default("explorer_api_urls", DEFAULT_EXPLORER_URLS) @@ -54,6 +54,7 @@ def __init__( Raises: Input: when params are missing """ self._explorer = None + explorer_url = explorer_url.rstrip("/") super().__init__( tname=tname, email=email, words=words, explorer_url=explorer_url, _tid=_tid, admins=admins, *args, **kwargs, ) @@ -108,6 +109,7 @@ def explorer(self): js_config.set("has_migrated_explorer_url", True) if self.explorer_url: + self.explorer_url = self.explorer_url.rstrip("/") self._explorer = ex_factory.get_by_url_and_identity(self.explorer_url, identity_name=self.instance_name) else: self._explorer = ex_factory.get_default() @@ -169,6 +171,11 @@ def register(self, host=None): self.save() return tid + def set_default(self): + from jumpscale.loader import j + + return j.core.identity.set_default(self.instance_name) + def get_identity(): return IdentityFactory(Identity).me diff --git a/jumpscale/packages/admin/actors/admin.py b/jumpscale/packages/admin/actors/admin.py index d8a7c2e648..0719166105 100644 --- a/jumpscale/packages/admin/actors/admin.py +++ b/jumpscale/packages/admin/actors/admin.py @@ -116,17 +116,24 @@ def delete_identity(self, identity_instance_name: str) -> str: def get_developer_options(self) -> str: test_cert = j.core.config.set_default("TEST_CERT", False) over_provision = j.core.config.set_default("OVER_PROVISIONING", False) - return j.data.serializers.json.dumps({"data": {"test_cert": test_cert, "over_provision": over_provision}}) + explorer_logs = j.core.config.set_default("EXPLORER_LOGS", False) + return j.data.serializers.json.dumps( + {"data": {"test_cert": test_cert, "over_provision": over_provision, "explorer_logs": explorer_logs}} + ) @actor_method - def set_developer_options(self, test_cert: bool, over_provision: bool) -> str: + def set_developer_options(self, test_cert: bool, over_provision: bool, explorer_logs: bool) -> str: j.core.config.set("TEST_CERT", test_cert) j.core.config.set("OVER_PROVISIONING", over_provision) - return j.data.serializers.json.dumps({"data": {"test_cert": test_cert, "over_provision": over_provision}}) + j.core.config.set("EXPLORER_LOGS", explorer_logs) + return j.data.serializers.json.dumps( + {"data": {"test_cert": test_cert, "over_provision": over_provision, "explorer_logs": explorer_logs}} + ) @actor_method def clear_blocked_nodes(self) -> str: - j.sals.reservation_chatflow.reservation_chatflow.clear_blocked_nodes() + j.sals.reservation_chatflow.reservation_chatflow.clear_blocked_nodes() return j.data.serializers.json.dumps({"data": "blocked nodes got cleared successfully."}) + Actor = Admin diff --git a/jumpscale/packages/admin/frontend/api.js b/jumpscale/packages/admin/frontend/api.js index cadf8dd532..9235de1bec 100644 --- a/jumpscale/packages/admin/frontend/api.js +++ b/jumpscale/packages/admin/frontend/api.js @@ -168,12 +168,12 @@ const apiClient = { url: `${baseURL}/admin/get_developer_options` }) }, - setDeveloperOptions: (testCert, overProvision) => { + setDeveloperOptions: (testCert, overProvision, explorerLogs) => { return axios({ url: `${baseURL}/admin/set_developer_options`, method: "post", headers: { 'Content-Type': 'application/json' }, - data: { test_cert: testCert, over_provision: overProvision } + data: { test_cert: testCert, over_provision: overProvision, explorer_logs: explorerLogs } }) }, clearBlockedNodes: () => { diff --git a/jumpscale/packages/admin/frontend/components/settings/Settings.vue b/jumpscale/packages/admin/frontend/components/settings/Settings.vue index c9c3c8ed55..26b48a28fb 100644 --- a/jumpscale/packages/admin/frontend/components/settings/Settings.vue +++ b/jumpscale/packages/admin/frontend/components/settings/Settings.vue @@ -6,7 +6,11 @@