From b19b71356da8e357a46ffba493c8a12c8e3987f5 Mon Sep 17 00:00:00 2001 From: Yaliang Wu Date: Thu, 1 Jun 2023 14:01:51 -0700 Subject: [PATCH 1/2] fix security IT failure caused by weak password (#951) Signed-off-by: Yaliang Wu --- .../ml/rest/MLModelGroupRestIT.java | 50 +++++++------------ .../opensearch/ml/rest/SecureMLRestIT.java | 38 ++++++-------- 2 files changed, 34 insertions(+), 54 deletions(-) diff --git a/plugin/src/test/java/org/opensearch/ml/rest/MLModelGroupRestIT.java b/plugin/src/test/java/org/opensearch/ml/rest/MLModelGroupRestIT.java index 849ff71b2b..3c6e1e5327 100644 --- a/plugin/src/test/java/org/opensearch/ml/rest/MLModelGroupRestIT.java +++ b/plugin/src/test/java/org/opensearch/ml/rest/MLModelGroupRestIT.java @@ -58,6 +58,7 @@ public class MLModelGroupRestIT extends MLCommonsRestTestCase { public ExpectedException exceptionRule = ExpectedException.none(); private String modelGroupId; + private String password = "IntegTest@MLModelGroupRestIT123"; @Before public void setup() throws IOException { @@ -77,56 +78,43 @@ public void setup() throws IOException { } createSearchRole(indexSearchAccessRole, "*"); - createUser(mlNoAccessUser, mlNoAccessUser, ImmutableList.of(opensearchBackendRole)); - mlNoAccessClient = new SecureRestClientBuilder( - getClusterHosts().toArray(new HttpHost[0]), - isHttps(), - mlNoAccessUser, - mlNoAccessUser - ).setSocketTimeout(60000).build(); + createUser(mlNoAccessUser, password, ImmutableList.of(opensearchBackendRole)); + mlNoAccessClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), mlNoAccessUser, password) + .setSocketTimeout(60000) + .build(); - createUser(mlReadOnlyUser, mlReadOnlyUser, ImmutableList.of(opensearchBackendRole)); - mlReadOnlyClient = new SecureRestClientBuilder( - getClusterHosts().toArray(new HttpHost[0]), - isHttps(), - mlReadOnlyUser, - mlReadOnlyUser - ).setSocketTimeout(60000).build(); + createUser(mlReadOnlyUser, password, ImmutableList.of(opensearchBackendRole)); + mlReadOnlyClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), mlReadOnlyUser, password) + .setSocketTimeout(60000) + .build(); - createUser(mlFullAccessNoIndexAccessUser, mlFullAccessNoIndexAccessUser, ImmutableList.of(opensearchBackendRole)); + createUser(mlFullAccessNoIndexAccessUser, password, ImmutableList.of(opensearchBackendRole)); mlFullAccessNoIndexAccessClient = new SecureRestClientBuilder( getClusterHosts().toArray(new HttpHost[0]), isHttps(), mlFullAccessNoIndexAccessUser, - mlFullAccessNoIndexAccessUser + password ).setSocketTimeout(60000).build(); - createUser(mlFullAccessUser, mlFullAccessUser, ImmutableList.of(opensearchBackendRole)); - mlFullAccessClient = new SecureRestClientBuilder( - getClusterHosts().toArray(new HttpHost[0]), - isHttps(), - mlFullAccessUser, - mlFullAccessUser - ).setSocketTimeout(60000).build(); + createUser(mlFullAccessUser, password, ImmutableList.of(opensearchBackendRole)); + mlFullAccessClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), mlFullAccessUser, password) + .setSocketTimeout(60000) + .build(); - createUser(mlNonAdminFullAccessWithoutBackendRoleUser, mlNonAdminFullAccessWithoutBackendRoleUser, ImmutableList.of()); + createUser(mlNonAdminFullAccessWithoutBackendRoleUser, password, ImmutableList.of()); mlNonAdminFullAccessWithoutBackendRoleClient = new SecureRestClientBuilder( getClusterHosts().toArray(new HttpHost[0]), isHttps(), mlNonAdminFullAccessWithoutBackendRoleUser, - mlNonAdminFullAccessWithoutBackendRoleUser + password ).setSocketTimeout(60000).build(); - createUser( - mlNonOwnerFullAccessWithBackendRoleUser, - mlNonOwnerFullAccessWithBackendRoleUser, - ImmutableList.of(opensearchBackendRole) - ); + createUser(mlNonOwnerFullAccessWithBackendRoleUser, password, ImmutableList.of(opensearchBackendRole)); mlNonOwnerFullAccessWithBackendRoleClient = new SecureRestClientBuilder( getClusterHosts().toArray(new HttpHost[0]), isHttps(), mlNonOwnerFullAccessWithBackendRoleUser, - mlNonOwnerFullAccessWithBackendRoleUser + password ).setSocketTimeout(60000).build(); createRoleMapping("ml_read_access", ImmutableList.of(mlReadOnlyUser)); diff --git a/plugin/src/test/java/org/opensearch/ml/rest/SecureMLRestIT.java b/plugin/src/test/java/org/opensearch/ml/rest/SecureMLRestIT.java index 3ea69b7cae..7d3b85ee95 100644 --- a/plugin/src/test/java/org/opensearch/ml/rest/SecureMLRestIT.java +++ b/plugin/src/test/java/org/opensearch/ml/rest/SecureMLRestIT.java @@ -58,6 +58,7 @@ public class SecureMLRestIT extends MLCommonsRestTestCase { public ExpectedException exceptionRule = ExpectedException.none(); private String modelGroupId; + private String password = "IntegTest@SecureMLRestIT123"; @Before public void setup() throws IOException, ParseException { @@ -77,37 +78,28 @@ public void setup() throws IOException, ParseException { } createSearchRole(indexSearchAccessRole, "*"); - createUser(mlNoAccessUser, mlNoAccessUser, new ArrayList<>(Arrays.asList(opensearchBackendRole))); - mlNoAccessClient = new SecureRestClientBuilder( - getClusterHosts().toArray(new HttpHost[0]), - isHttps(), - mlNoAccessUser, - mlNoAccessUser - ).setSocketTimeout(60000).build(); + createUser(mlNoAccessUser, password, new ArrayList<>(Arrays.asList(opensearchBackendRole))); + mlNoAccessClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), mlNoAccessUser, password) + .setSocketTimeout(60000) + .build(); - createUser(mlReadOnlyUser, mlReadOnlyUser, new ArrayList<>(Arrays.asList(opensearchBackendRole))); - mlReadOnlyClient = new SecureRestClientBuilder( - getClusterHosts().toArray(new HttpHost[0]), - isHttps(), - mlReadOnlyUser, - mlReadOnlyUser - ).setSocketTimeout(60000).build(); + createUser(mlReadOnlyUser, password, new ArrayList<>(Arrays.asList(opensearchBackendRole))); + mlReadOnlyClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), mlReadOnlyUser, password) + .setSocketTimeout(60000) + .build(); - createUser(mlFullAccessNoIndexAccessUser, mlFullAccessNoIndexAccessUser, new ArrayList<>(Arrays.asList(opensearchBackendRole))); + createUser(mlFullAccessNoIndexAccessUser, password, new ArrayList<>(Arrays.asList(opensearchBackendRole))); mlFullAccessNoIndexAccessClient = new SecureRestClientBuilder( getClusterHosts().toArray(new HttpHost[0]), isHttps(), mlFullAccessNoIndexAccessUser, - mlFullAccessNoIndexAccessUser + password ).setSocketTimeout(60000).build(); - createUser(mlFullAccessUser, mlFullAccessUser, new ArrayList<>(Arrays.asList(opensearchBackendRole))); - mlFullAccessClient = new SecureRestClientBuilder( - getClusterHosts().toArray(new HttpHost[0]), - isHttps(), - mlFullAccessUser, - mlFullAccessUser - ).setSocketTimeout(60000).build(); + createUser(mlFullAccessUser, password, new ArrayList<>(Arrays.asList(opensearchBackendRole))); + mlFullAccessClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), mlFullAccessUser, password) + .setSocketTimeout(60000) + .build(); createRoleMapping("ml_read_access", new ArrayList<>(Arrays.asList(mlReadOnlyUser))); createRoleMapping("ml_full_access", new ArrayList<>(Arrays.asList(mlFullAccessNoIndexAccessUser, mlFullAccessUser))); From f7d0ddd897858b480cd56a07f7a88a60db23ccca Mon Sep 17 00:00:00 2001 From: zane-neo Date: Mon, 28 Aug 2023 10:06:16 +0800 Subject: [PATCH 2/2] Fix pre-trained model metadata parse exception Signed-off-by: zane-neo --- .../java/org/opensearch/ml/engine/ModelHelper.java | 14 ++++---------- .../algorithms/text_embedding/ModelHelperTest.java | 2 +- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/ml-algorithms/src/main/java/org/opensearch/ml/engine/ModelHelper.java b/ml-algorithms/src/main/java/org/opensearch/ml/engine/ModelHelper.java index b5a9034225..28c9dc20d5 100644 --- a/ml-algorithms/src/main/java/org/opensearch/ml/engine/ModelHelper.java +++ b/ml-algorithms/src/main/java/org/opensearch/ml/engine/ModelHelper.java @@ -150,16 +150,10 @@ public boolean isModelAllowed(MLRegisterModelInput registerModelInput, List mode String version = registerModelInput.getVersion(); MLModelFormat modelFormat = registerModelInput.getModelFormat(); for (Object meta: modelMetaList) { - Map metaMap = (Map) meta; - String name = (String) metaMap.get("name"); - Map versions = (Map) metaMap.get("versions"); - Object versionObj = versions.get(version); - if (versionObj == null) return false; - Map versionMap = (Map) versionObj; - Object formatObj = versionMap.get("format"); - if (formatObj == null) return false; - List formats = (List) formatObj; - if (name.equals(modelName) && versions.containsKey(version.toLowerCase(Locale.ROOT)) && formats.contains(modelFormat.toString().toLowerCase(Locale.ROOT))) { + String name = (String) ((Map)meta).get("name"); + List versions = (List) ((Map)meta).get("version"); + List formats = (List) ((Map)meta).get("format"); + if (name.equals(modelName) && versions.contains(version.toLowerCase(Locale.ROOT)) && formats.contains(modelFormat.toString().toLowerCase(Locale.ROOT))) { return true; } } diff --git a/ml-algorithms/src/test/java/org/opensearch/ml/engine/algorithms/text_embedding/ModelHelperTest.java b/ml-algorithms/src/test/java/org/opensearch/ml/engine/algorithms/text_embedding/ModelHelperTest.java index bfd2eab36e..7418dc3aba 100644 --- a/ml-algorithms/src/test/java/org/opensearch/ml/engine/algorithms/text_embedding/ModelHelperTest.java +++ b/ml-algorithms/src/test/java/org/opensearch/ml/engine/algorithms/text_embedding/ModelHelperTest.java @@ -184,7 +184,7 @@ public void testDownloadPrebuiltModelMetaList() throws PrivilegedActionException .modelNodeIds(new String[]{"node_id1"}) .build(); List modelMetaList = modelHelper.downloadPrebuiltModelMetaList(taskId, registerModelInput); - assertEquals("huggingface/sentence-transformers/all-MiniLM-L12-v2", ((Map)modelMetaList.get(0)).get("name")); + assertEquals("huggingface/sentence-transformers/all-distilroberta-v1", ((Map)modelMetaList.get(0)).get("name")); } @Test