Merge branch 'release/11.0b7'

.github/workflows/js-sdk.yml

@@ -17,7 +17,7 @@ jobs:
       - name: Gathering deps
         run: |
           sudo apt-get update
-          sudo apt-get install -y git python3-pip python3-venv python3-setuptools tmux redis
+          sudo apt-get install -y git python3-pip python3-venv python3-setuptools tmux redis nginx
           sudo pip3 install poetry
           sudo poetry run pip3 install pytest
       - name: Install

docs/architecture/decisions/0008-add-kwargs-to-3bot-start.md

@@ -0,0 +1,19 @@
+# 8. add_kwargs_to_3bot_start
+
+Date: 2020-09-27
+
+## Status
+
+Accepted
+
+## Context
+
+Adding packages with kwargs has some limitations and hence kwargs are needed every time start is called not only once when adding package.
+
+## Decision
+
+Add kwargs passed to the package instance that will be saved locally, and can be retrieved everytime the threebot server restarts and starts the package.
+
+## Consequences
+
+Any package that is added with kwargs will save them and hence with every restart of the package, they are reloaded and used in the install of the package.

docs/wiki/tutorials/add_marketplace_chatflow.md

@@ -70,12 +70,34 @@
 - And for returning your solution count should append your `{SOLUTION_TYPE}` in the `count_dict` in `count_solutions` method in the same module
 
 ## Add app in frontend
-- In the frontend, you just need to add your app object in `apps` dict in `packages/marketplace/frontend/App.vue`
-    ```js
-    {
-        name: "App Name in frontend",
-        type: "{SOLUTION_TYPE}",
-        path: "/{your_app_name}",
-        meta: { icon: "app_icon" },
-    }
-    ```
+- In the frontend, you just need to add your app object as below in `apps` dict under the section you want to list your app in `packages/marketplace/frontend/data.js`
+    - If you need to add another section, just create new one in the `SECTIONS` object with the same structure:
+        ```js
+        "SECTION NAME": {
+            titleToolTip: "Tooltip shown on hovering on section title in the frontend",
+            apps: {
+                // list your applications objects as below structure
+                "App Name": {
+                    name: "App Name in frontend",
+                    type: "{SOLUTION_TYPE}", // defined in the previous steps
+                    image: "./assets/appImage.png", // add your app image in the assets dir
+                    disable: false, // make it true if you want to hide your app in the marketplace frontend
+                    helpLink: "https://now10.threefold.io/docs", // link to application manual
+                    description: "Description of your application"
+                },
+            },
+        },
+        ```
+    - If you just need to add your application in an existing section, add a new app object with below structure in the section object you want to list in:
+        ```js
+        {
+            "App Name": {
+                name: "App Name in frontend",
+                type: "{SOLUTION_TYPE}", // defined in the previous steps
+                image: "./assets/appImage.png", // add your app image in the assets dir
+                disable: false, // make it true if you want to hide your app in the marketplace frontend
+                helpLink: "https://now10.threefold.io/docs", // link to application manual
+                description: "Description of your application"
+            },
+        }
+        ```

examplescripts/minio.py

@@ -18,7 +18,7 @@
 PASSWORD = "supersecurepassowrd"
 network_name = str(uuid.uuid4())
 print(f"network name: {network_name}")
-BAD_NODES = set([])
+BAD_NODES = set(["A7FmQZ72h7FzjkJMGXmzLDFyfyxzitDZYuernGG97nv7"])
 UP_FOR = 60 * 20  # number of seconds
 
 
@@ -55,6 +55,21 @@ def wait_workload(wid):
         workload = zos.workloads.get(wid)
 
 
+def wait_zdb_workloads(zdb_wids):
+    # Looks like the workload_id can be set before the namespace
+    for wid in zdb_wids:
+        workload = zos.workloads.get(wid)
+        data = j.data.serializers.json.loads(workload.info.result.data_json)
+        if workload.info.result.message:
+            x = workload.info.result.message
+            raise Exception(f"Failed to initialize ZDB: {x}")
+        elif data.get("IP") or data.get("IPs"):
+            return
+        else:
+            sleep(1)
+            continue
+
+
 def wait_pools(pools):
     for pool in pools:
         while pool.cus == 0:
@@ -286,6 +301,7 @@ def pick_minio_nodes(nodes):
 backup_vol_id = deploy_volume(minio_backup_node.node_id, backup_pool)
 zdb_wids = [x.id for x in zdb_workloads]
 wait_workloads(zdb_wids)
+wait_zdb_workloads(zdb_wids)
 wait_workload(tlog_workload.id)
 wait_workload(master_vol_id)
 wait_workload(backup_vol_id)
@@ -325,6 +341,7 @@ def pick_minio_nodes(nodes):
 zdb_new_workloads = deploy_zdbs(zdb_later_nodes, zdb_new_pools)
 zdb_new_wids = [x.id for x in zdb_new_workloads]
 wait_workloads(zdb_new_wids)
+wait_zdb_workloads(zdb_new_wids)
 new_namespace_config = get_namespace_config(zdb_new_workloads)
 
 print("Removing three backup storages")

jumpscale/clients/explorer/explorer.py

@@ -18,6 +18,15 @@
 from .users import Users
 from .workloads import Workloads
 
+from jumpscale.loader import j
+
+
+def log_request(r, *args, **kwargs):
+    if j.config.get("EXPLORER_LOGS"):
+        j.logger.debug(
+            f"Request {r.request.url} method: {r.request.method} body: {r.request.body} headers: {r.request.headers}"
+        )
+
 
 class Explorer(Client):
     url = fields.String()
@@ -30,7 +39,7 @@ def __init__(self, url=None, identity_name=None, **kwargs):
         else:
             self._loaded_identity = identity.get_identity()
         self._session = requests.Session()
-        self._session.hooks = dict(response=raise_for_status)
+        self._session.hooks = dict(response=[log_request, raise_for_status])
 
         secret = self._loaded_identity.nacl.signing_key.encode(Base64Encoder)
         auth = HTTPSignatureAuth(

jumpscale/clients/explorer/models.py

@@ -42,6 +42,7 @@ class Location(Base):
     def __str__(self):
         return ",".join([x for x in [self.continent, self.country, self.city] if x])
 
+
 class Farm(Base):
     id = fields.Integer()
     threebot_id = fields.Integer()
@@ -55,7 +56,6 @@ class Farm(Base):
 
     def __str__(self):
         return " - ".join([x for x in [self.name, str(self.location)] if x])
-
 
 
 class WorkloadsAmount(Base):
@@ -250,9 +250,14 @@ class ContainerNetworkConnection(Base):
 
 
 class ContainerLogsRedis(Base):
+    # deprecated, please use secret_stdout instead
     stdout = fields.String(default="")
+    # deprecated, please use secret_stderr instead
     stderr = fields.String(default="")
 
+    secret_stdout = fields.String(default="")
+    secret_stderr = fields.String(default="")
+
 
 class ContainerLogs(Base):
     type = fields.String(default="")

jumpscale/clients/sendgrid/sendgrid.py

@@ -12,8 +12,8 @@
 class SendGridClient(Client):
     apikey = fields.String()
 
-    def __init__(self):
-        super().__init__()
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
 
     def build_attachment(self, filepath, typ="application/pdf"):
         """

jumpscale/clients/stellar/stellar.py

@@ -387,7 +387,7 @@ def transfer(
         if asset != "XLM":
             assetStr = asset.split(":")
             if len(assetStr) != 2:
-                raise Exception("Wrong asset format should be in format 'assetcode:issuer'")
+                raise Exception(f"Wrong asset format should be in format 'assetcode:issuer', but received {assetStr}")
             asset = assetStr[0]
             issuer = assetStr[1]
 

jumpscale/core/identity/__init__.py

@@ -12,9 +12,9 @@
 from jumpscale.sals.nettools import get_default_ip_config
 
 DEFAULT_EXPLORER_URLS = {
-    "mainnet": "https://explorer.grid.tf/api/v1/",
-    "testnet": "https://explorer.testnet.grid.tf/api/v1/",
-    "devnet": "https://explorer.devnet.grid.tf/api/v1/",
+    "mainnet": "https://explorer.grid.tf/api/v1",
+    "testnet": "https://explorer.testnet.grid.tf/api/v1",
+    "devnet": "https://explorer.devnet.grid.tf/api/v1",
 }
 
 EXPLORER_URLS = js_config.set_default("explorer_api_urls", DEFAULT_EXPLORER_URLS)
@@ -54,6 +54,7 @@ def __init__(
         Raises: Input: when params are missing
         """
         self._explorer = None
+        explorer_url = explorer_url.rstrip("/")
         super().__init__(
             tname=tname, email=email, words=words, explorer_url=explorer_url, _tid=_tid, admins=admins, *args, **kwargs,
         )
@@ -108,6 +109,7 @@ def explorer(self):
                     js_config.set("has_migrated_explorer_url", True)
 
             if self.explorer_url:
+                self.explorer_url = self.explorer_url.rstrip("/")
                 self._explorer = ex_factory.get_by_url_and_identity(self.explorer_url, identity_name=self.instance_name)
             else:
                 self._explorer = ex_factory.get_default()
@@ -169,6 +171,11 @@ def register(self, host=None):
         self.save()
         return tid
 
+    def set_default(self):
+        from jumpscale.loader import j
+
+        return j.core.identity.set_default(self.instance_name)
+
 
 def get_identity():
     return IdentityFactory(Identity).me

jumpscale/packages/admin/actors/admin.py

@@ -116,17 +116,24 @@ def delete_identity(self, identity_instance_name: str) -> str:
     def get_developer_options(self) -> str:
         test_cert = j.core.config.set_default("TEST_CERT", False)
         over_provision = j.core.config.set_default("OVER_PROVISIONING", False)
-        return j.data.serializers.json.dumps({"data": {"test_cert": test_cert, "over_provision": over_provision}})
+        explorer_logs = j.core.config.set_default("EXPLORER_LOGS", False)
+        return j.data.serializers.json.dumps(
+            {"data": {"test_cert": test_cert, "over_provision": over_provision, "explorer_logs": explorer_logs}}
+        )
 
     @actor_method
-    def set_developer_options(self, test_cert: bool, over_provision: bool) -> str:
+    def set_developer_options(self, test_cert: bool, over_provision: bool, explorer_logs: bool) -> str:
         j.core.config.set("TEST_CERT", test_cert)
         j.core.config.set("OVER_PROVISIONING", over_provision)
-        return j.data.serializers.json.dumps({"data": {"test_cert": test_cert, "over_provision": over_provision}})
+        j.core.config.set("EXPLORER_LOGS", explorer_logs)
+        return j.data.serializers.json.dumps(
+            {"data": {"test_cert": test_cert, "over_provision": over_provision, "explorer_logs": explorer_logs}}
+        )
 
     @actor_method
     def clear_blocked_nodes(self) -> str:
-        j.sals.reservation_chatflow.reservation_chatflow.clear_blocked_nodes() 
+        j.sals.reservation_chatflow.reservation_chatflow.clear_blocked_nodes()
         return j.data.serializers.json.dumps({"data": "blocked nodes got cleared successfully."})
 
+
 Actor = Admin

jumpscale/packages/admin/frontend/api.js

@@ -168,12 +168,12 @@ const apiClient = {
                 url: `${baseURL}/admin/get_developer_options`
             })
         },
-        setDeveloperOptions: (testCert, overProvision) => {
+        setDeveloperOptions: (testCert, overProvision, explorerLogs) => {
             return axios({
                 url: `${baseURL}/admin/set_developer_options`,
                 method: "post",
                 headers: { 'Content-Type': 'application/json' },
-                data: { test_cert: testCert, over_provision: overProvision }
+                data: { test_cert: testCert, over_provision: overProvision, explorer_logs: explorerLogs }
             })
         },
         clearBlockedNodes: () => {

jumpscale/packages/admin/frontend/components/settings/Settings.vue

@@ -6,7 +6,11 @@
       <template #default>
         <v-row align="start" justify="start">
           <v-col class="mt-0 pt-0" cols="12" md="4">
-            <base-section title="Admins" icon="mdi-account-lock" :loading="loading.admins">
+            <base-section
+              title="Admins"
+              icon="mdi-account-lock"
+              :loading="loading.admins"
+            >
               <template #actions>
                 <v-btn text @click.stop="dialogs.addAdmin = true">
                   <v-icon left>mdi-plus</v-icon>Add
@@ -24,7 +28,8 @@
                 label
                 close
                 close-icon="mdi-close-circle-outline"
-              >{{ admin }}</v-chip>
+                >{{ admin }}</v-chip
+              >
             </base-section>
           </v-col>
           <v-col class="mt-0 pt-0" cols="12" md="4">
@@ -46,7 +51,8 @@
                 :key="i"
                 :color="getColor(identity.instance_name)"
                 @click="openIdentity(identity.instance_name)"
-              >{{identity.instance_name}}</v-chip>
+                >{{ identity.instance_name }}</v-chip
+              >
             </base-section>
           </v-col>
           <v-col class="mt-0 pt-0" cols="12" md="3">
@@ -70,23 +76,42 @@
                 :label="`Allow over provisioning`"
                 @click.stop="setDeveloperOptions()"
               ></v-switch>
+              <v-switch
+                hide-details
+                class="my-2 pl-2"
+                v-model="explorerLogs"
+                :label="`Enable explorer logs`"
+                @click.stop="setDeveloperOptions()"
+              ></v-switch>
               <v-btn
                 hide-details
                 class="my-2 ml-2"
                 small
                 color="success"
                 @click="clearBlockedNodes()"
-              >Clear blocked nodes</v-btn>
+                >Clear blocked nodes</v-btn
+              >
             </base-section>
           </v-col>
         </v-row>
       </template>
     </base-component>
 
     <add-admin v-model="dialogs.addAdmin" @done="listAdmins"></add-admin>
-    <remove-admin v-model="dialogs.removeAdmin" :name="selectedAdmin" @done="listAdmins"></remove-admin>
-    <identity-info v-model="dialogs.identityInfo" :name="selectedIdentity" @done="listIdentities"></identity-info>
-    <add-identity v-model="dialogs.addIdentity" @done="listIdentities"></add-identity>
+    <remove-admin
+      v-model="dialogs.removeAdmin"
+      :name="selectedAdmin"
+      @done="listAdmins"
+    ></remove-admin>
+    <identity-info
+      v-model="dialogs.identityInfo"
+      :name="selectedIdentity"
+      @done="listIdentities"
+    ></identity-info>
+    <add-identity
+      v-model="dialogs.addIdentity"
+      @done="listIdentities"
+    ></add-identity>
   </div>
 </template>
 
@@ -118,6 +143,7 @@ module.exports = {
       identities: [],
       testCert: false,
       overProvision: false,
+      explorerLogs: false,
     };
   },
   methods: {
@@ -188,14 +214,19 @@ module.exports = {
           let developerOptions = JSON.parse(response.data).data;
           this.testCert = developerOptions["test_cert"];
           this.overProvision = developerOptions["over_provision"];
+          this.explorerLogs = developerOptions["explorer_logs"];
         })
         .finally(() => {
           this.loading.developerOptions = false;
         });
     },
     setDeveloperOptions() {
       this.$api.admins
-        .setDeveloperOptions(this.testCert, this.overProvision)
+        .setDeveloperOptions(
+          this.testCert,
+          this.overProvision,
+          this.explorerLogs
+        )
         .then((response) => {
           this.alert("Developer options updated", "success");
         })