diff --git a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java index 2998e6934..a8ed2f15b 100644 --- a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java +++ b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java @@ -114,15 +114,17 @@ public void indexIocs(List iocs) throws IOException { initFeedIndex(newActiveIndex, ActionListener.wrap( r -> { + // reset the store configs + if (saTifSourceConfig.getIocStoreConfig() instanceof DefaultIocStoreConfig) { + ((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocToIndexDetails().clear(); + } + + // recreate the store configs saTifSourceConfig.getIocTypes().forEach(type -> { - IOCType iocType = new IOCType(type); if (saTifSourceConfig.getIocStoreConfig() instanceof DefaultIocStoreConfig) { - List listOfIocToIndexDetails = - ((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocToIndexDetails(); - listOfIocToIndexDetails.removeIf(iocToIndexDetails -> iocToIndexDetails.getIocType() == iocType); DefaultIocStoreConfig.IocToIndexDetails iocToIndexDetails = - new DefaultIocStoreConfig.IocToIndexDetails(iocType, iocIndexPattern, newActiveIndex); - listOfIocToIndexDetails.add(iocToIndexDetails); + new DefaultIocStoreConfig.IocToIndexDetails(new IOCType(type), iocIndexPattern, newActiveIndex); + ((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocToIndexDetails().add(iocToIndexDetails); } }); bulkIndexIocs(iocs, newActiveIndex); diff --git a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java index 0b7da3d70..98a645e29 100644 --- a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java +++ b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java @@ -123,6 +123,7 @@ public void downloadAndIndexIOCs(SATIFSourceConfig saTifSourceConfig, ActionList } catch (Exception e) { log.error("Failed to download IOCs.", e); listener.onFailure(e); + return; } try { diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java index eb0c32393..9c8f81025 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java @@ -39,6 +39,7 @@ import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; import java.time.Instant; import java.util.ArrayList; @@ -46,6 +47,7 @@ import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.SortedMap; @@ -158,7 +160,7 @@ public void createIocAndTIFSourceConfig( )); }, e -> { - log.error("Failed to download and save IOCs for source config [{}]", indexSaTifSourceConfigResponse.getId()); + log.error("Failed to download and save IOCs for threat intel source config [{}]", indexSaTifSourceConfigResponse.getId()); saTifSourceConfigService.deleteTIFSourceConfig(indexSaTifSourceConfigResponse, ActionListener.wrap( deleteResponse -> { log.debug("Successfully deleted threat intel source config [{}]", indexSaTifSourceConfigResponse.getId()); @@ -204,7 +206,7 @@ public void downloadAndSaveIOCs(SATIFSourceConfig saTifSourceConfig, if (saTifSourceConfig.getIocTypes().contains(stix2IOC.getType().toString())) { validStix2IocList.add(stix2IOC); } else { - log.error("{} is not a supported Ioc type for tif source config {}. Skipping IOC {}: of type {} value {}", + log.error("{} is not a supported Ioc type for threat intel source config {}. Skipping IOC {}: of type {} value {}", stix2IOC.getType().toString(), saTifSourceConfig.getId(), stix2IOC.getId(), stix2IOC.getType().toString(), stix2IOC.getValue() ); @@ -212,7 +214,7 @@ public void downloadAndSaveIOCs(SATIFSourceConfig saTifSourceConfig, } if (validStix2IocList.isEmpty()) { log.error("No supported IOCs to index"); - actionListener.onFailure(new OpenSearchStatusException("No compatible Iocs were uploaded for config " + saTifSourceConfig.getName(), RestStatus.BAD_REQUEST)); + actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException("No compatible Iocs were uploaded for threat intel source config " + saTifSourceConfig.getName(), RestStatus.BAD_REQUEST))); return; } stix2IOCFetchService.onlyIndexIocs(saTifSourceConfig, validStix2IocList, actionListener); @@ -274,14 +276,18 @@ public void updateIocAndTIFSourceConfig( saTifSourceConfigService.getTIFSourceConfig(saTifSourceConfigDto.getId(), ActionListener.wrap( retrievedSaTifSourceConfig -> { if (TIFJobState.AVAILABLE.equals(retrievedSaTifSourceConfig.getState()) == false && TIFJobState.REFRESH_FAILED.equals(retrievedSaTifSourceConfig.getState()) == false) { - log.error("Invalid TIF job state. Expecting {} or {} but received {}", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, retrievedSaTifSourceConfig.getState()); - listener.onFailure(new OpenSearchException("Invalid TIF job state. Expecting {} or {} but received {}", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, retrievedSaTifSourceConfig.getState())); + log.error("Invalid threat intel source config state. Expecting {} or {} but received {}", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, retrievedSaTifSourceConfig.getState()); + listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException( + String.format(Locale.getDefault(), "Invalid threat intel source config state. Expecting %s or %s but received %s", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, retrievedSaTifSourceConfig.getState()), + RestStatus.BAD_REQUEST))); return; } if (false == saTifSourceConfigDto.getType().equals(retrievedSaTifSourceConfig.getType())) { - log.error("Unable to update source config, type cannot change from {} to {}", retrievedSaTifSourceConfig.getType(), saTifSourceConfigDto.getType()); - listener.onFailure(new OpenSearchException("Unable to update source config, type cannot change from {} to {}", retrievedSaTifSourceConfig.getType(), saTifSourceConfigDto.getType())); + log.error("Unable to update threat intel source config, type cannot change from {} to {}", retrievedSaTifSourceConfig.getType(), saTifSourceConfigDto.getType()); + listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException( + String.format(Locale.getDefault(), "Unable to update threat intel source config, type cannot change from %s to %s", retrievedSaTifSourceConfig.getType(), saTifSourceConfigDto.getType()), + RestStatus.BAD_REQUEST))); return; } @@ -345,7 +351,6 @@ private void storeAndDeleteIocIndices(List stix2IOCList, ActionListene saTifSourceConfigService.getClusterState(ActionListener.wrap( clusterStateResponse -> { - List iocTypes = updatedSaTifSourceConfig.getIocTypes(); IocStoreConfig iocStoreConfig = updatedSaTifSourceConfig.getIocStoreConfig(); Set activeIndices = new HashSet<>(); Set indicesToDelete = new HashSet<>(); @@ -388,11 +393,13 @@ private void storeAndDeleteIocIndices(List stix2IOCList, ActionListene ), iocIndexPatterns.toArray(new String[0])); }, e -> { - log.error("Failed to download and save IOCs for source config [{}]", updatedSaTifSourceConfig.getId()); + log.error("Failed to download and save IOCs for threat intel source config [{}]", updatedSaTifSourceConfig.getId(), e); markSourceConfigAsAction(updatedSaTifSourceConfig, TIFJobState.REFRESH_FAILED, ActionListener.wrap( r -> { log.info("Set threat intel source config as REFRESH_FAILED for [{}]", updatedSaTifSourceConfig.getId()); - listener.onFailure(new OpenSearchException("Set threat intel source config as REFRESH_FAILED for [{}]", updatedSaTifSourceConfig.getId())); + listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchException( + String.format(Locale.getDefault(), "Failed to download and save IOCs for threat intel source config [%s]. Set source config as REFRESH_FAILED", updatedSaTifSourceConfig.getId()), + e))); }, ex -> { log.error("Failed to set threat intel source config as REFRESH_FAILED for [{}]", updatedSaTifSourceConfig.getId()); listener.onFailure(ex); @@ -423,14 +430,18 @@ public void refreshTIFSourceConfig( saTifSourceConfigService.getTIFSourceConfig(saTifSourceConfigId, ActionListener.wrap( saTifSourceConfig -> { if (saTifSourceConfig.getType() == IOC_UPLOAD) { - log.error("Unable to refresh source config [{}] with a source type of [{}]", saTifSourceConfig.getId(), IOC_UPLOAD); - listener.onFailure(new OpenSearchException("Unable to refresh source config [{}] with a source type of [{}]", saTifSourceConfig.getId(), IOC_UPLOAD)); + log.error("Unable to refresh threat intel source config [{}] with a source type of [{}]", saTifSourceConfig.getId(), IOC_UPLOAD); + listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException( + String.format(Locale.getDefault(), "Unable to refresh threat intel source config [%s] with a source type of [%s]", saTifSourceConfig.getId(), IOC_UPLOAD), + RestStatus.BAD_REQUEST))); return; } if (TIFJobState.AVAILABLE.equals(saTifSourceConfig.getState()) == false && TIFJobState.REFRESH_FAILED.equals(saTifSourceConfig.getState()) == false) { - log.error("Invalid TIF job state. Expecting {} or {} but received {}", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, saTifSourceConfig.getState()); - listener.onFailure(new OpenSearchException("Invalid TIF job state. Expecting {} or {} but received {}", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, saTifSourceConfig.getState())); + log.error("Invalid threat intel source config state. Expecting {} or {} but received {}", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, saTifSourceConfig.getState()); + listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException( + String.format(Locale.getDefault(), "Invalid threat intel source config state. Expecting %s or %s but received %s", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, saTifSourceConfig.getState()), + RestStatus.BAD_REQUEST))); return; } @@ -488,14 +499,16 @@ private void downloadAndSaveIocsToRefresh(ActionListener l )); }, downloadAndSaveIocsError -> { // Update source config as refresh failed - log.error("Failed to download and save IOCs for threat intel source config [{}]", updatedSourceConfig.getId()); + log.error("Failed to download and save IOCs for threat intel source config [{}]", updatedSourceConfig.getId(), downloadAndSaveIocsError); markSourceConfigAsAction(updatedSourceConfig, TIFJobState.REFRESH_FAILED, ActionListener.wrap( r -> { - log.debug("Set threat intel source config as REFRESH_FAILED for [{}]", updatedSourceConfig.getId()); - listener.onFailure(new OpenSearchException("Set threat intel source config as REFRESH_FAILED for [{}]", updatedSourceConfig.getId())); - }, e -> { + log.info("Set threat intel source config as REFRESH_FAILED for [{}]", updatedSourceConfig.getId()); + listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchException( + String.format(Locale.getDefault(), "Failed to download and save IOCs for threat intel source config [%s]. Set source config as REFRESH_FAILED", updatedSourceConfig.getId()), + downloadAndSaveIocsError))); + }, ex -> { log.error("Failed to set threat intel source config as REFRESH_FAILED for [{}]", updatedSourceConfig.getId()); - listener.onFailure(e); + listener.onFailure(ex); } )); })); @@ -528,7 +541,7 @@ public void deleteTIFSourceConfig( }, e -> { log.error("Failed to get threat intel source config for [{}]", saTifSourceConfigId); if (e instanceof IndexNotFoundException) { - listener.onFailure(new OpenSearchException("Threat intel source config [{}] not found", saTifSourceConfigId)); + listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(),"Threat intel source config [%s] not found.", saTifSourceConfigId), RestStatus.NOT_FOUND))); } else { listener.onFailure(e); } @@ -708,7 +721,7 @@ private void deleteAllIocsAndSourceConfig(String saTifSourceConfigId, ActionList } )); }, e -> { - log.error("Failed to delete IOC indices for source config [{}]", updateSaTifSourceConfigResponse.getId()); + log.error("Failed to delete IOC indices for threat intel source config [{}]", updateSaTifSourceConfigResponse.getId()); listener.onFailure(e); } )); diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java index 14aa45bb1..b3f47e303 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java @@ -7,7 +7,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.OpenSearchException; import org.opensearch.OpenSearchStatusException; import org.opensearch.ResourceAlreadyExistsException; import org.opensearch.action.StepListener; @@ -226,7 +225,7 @@ public void getTIFSourceConfig( client.get(getRequest, ActionListener.wrap( getResponse -> { if (!getResponse.isExists()) { - actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException("Threat intel source config not found.", RestStatus.NOT_FOUND))); + actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(),"Threat intel source config [%s] not found.", tifSourceConfigId), RestStatus.NOT_FOUND))); return; } SATIFSourceConfig saTifSourceConfig = null; @@ -238,7 +237,7 @@ public void getTIFSourceConfig( saTifSourceConfig = SATIFSourceConfig.docParse(xcp, getResponse.getId(), getResponse.getVersion()); } if (saTifSourceConfig == null) { - actionListener.onFailure(new OpenSearchException("No threat intel source config exists [{}]", tifSourceConfigId)); + actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(),"No threat intel source config exists [%s]", tifSourceConfigId), RestStatus.BAD_REQUEST))); } else { log.debug("Threat intel source config with id [{}] fetched", getResponse.getId()); actionListener.onResponse(saTifSourceConfig); @@ -258,7 +257,7 @@ public void searchTIFSourceConfigs( // Check to make sure the job index exists if (clusterService.state().metadata().hasIndex(SecurityAnalyticsPlugin.JOB_INDEX_NAME) == false) { - actionListener.onFailure(new OpenSearchException("Threat intel source config index does not exist")); + actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException("Threat intel source config index does not exist", RestStatus.BAD_REQUEST))); return; } @@ -350,7 +349,7 @@ public void deleteTIFSourceConfig( ) { // check to make sure the job index exists if (clusterService.state().metadata().hasIndex(SecurityAnalyticsPlugin.JOB_INDEX_NAME) == false) { - actionListener.onFailure(new OpenSearchException("Threat intel source config index does not exist")); + actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException("Threat intel source config index does not exist", RestStatus.BAD_REQUEST))); return; } @@ -410,7 +409,7 @@ private void deleteIocIndex(Set indicesToDelete, Boolean backgroundJob, if (!response.isAcknowledged()) { log.error("Could not delete one or more IOC indices: " + index); if (backgroundJob == false) { - listener.onFailure(new OpenSearchException("Could not delete one or more IOC indices: " + index)); + listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(), "Could not delete one or more IOC indices: " + index), RestStatus.INTERNAL_SERVER_ERROR))); } } else { log.debug("Successfully deleted one or more IOC indices:" + index); diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java index 3aa5f739a..3989f38f3 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java @@ -8,10 +8,14 @@ package org.opensearch.securityanalytics.resthandler; +import org.apache.hc.core5.http.io.entity.StringEntity; +import org.apache.hc.core5.http.message.BasicHeader; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.junit.Assert; import org.opensearch.client.Response; +import org.opensearch.client.ResponseException; +import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchHit; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; @@ -19,7 +23,6 @@ import org.opensearch.securityanalytics.action.ListIOCsActionResponse; import org.opensearch.securityanalytics.commons.model.IOCType; import org.opensearch.securityanalytics.model.STIX2IOCDto; -import org.opensearch.securityanalytics.services.STIX2IOCFeedStore; import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; @@ -32,6 +35,7 @@ import java.util.Map; import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.JOB_INDEX_NAME; +import static org.opensearch.securityanalytics.services.STIX2IOCFeedStore.IOC_ALL_INDEX_PATTERN; import static org.opensearch.securityanalytics.services.STIX2IOCFeedStore.getAllIocIndexPatternById; public class SourceConfigWithoutS3RestApiIT extends SecurityAnalyticsRestTestCase { @@ -82,7 +86,7 @@ public void testCreateIocUploadSourceConfig() throws IOException { ); Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); - Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + Assert.assertEquals(RestStatus.CREATED, restStatus(response)); Map responseBody = asMap(response); String createdId = responseBody.get("_id").toString(); @@ -106,9 +110,9 @@ public void testCreateIocUploadSourceConfig() throws IOException { hits = executeSearch(indexName, request); Assert.assertEquals(iocs.size(), hits.size()); -// Retrieve all IOCs + // Retrieve all IOCs Response iocResponse = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(), Collections.emptyMap(), null); - Assert.assertEquals(200, iocResponse.getStatusLine().getStatusCode()); + Assert.assertEquals(RestStatus.OK, restStatus(iocResponse)); Map respMap = asMap(iocResponse); // Evaluate response @@ -117,9 +121,9 @@ public void testCreateIocUploadSourceConfig() throws IOException { List> iocHits = (List>) respMap.get(ListIOCsActionResponse.HITS_FIELD); assertTrue(iocs.size() < iocHits.size()); -// Retrieve all IOCs by feed Ids + // Retrieve all IOCs by feed Ids iocResponse = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(), Map.of("feed_ids", createdId + ",random"), null); - Assert.assertEquals(200, iocResponse.getStatusLine().getStatusCode()); + Assert.assertEquals(RestStatus.OK, restStatus(iocResponse)); respMap = asMap(iocResponse); // Evaluate response @@ -134,7 +138,7 @@ public void testCreateIocUploadSourceConfig() throws IOException { String.format("%s,%s", IOCType.IPV4_TYPE, IOCType.DOMAIN_NAME_TYPE) ); iocResponse = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(), params, null); - Assert.assertEquals(200, iocResponse.getStatusLine().getStatusCode()); + Assert.assertEquals(RestStatus.OK, restStatus(iocResponse)); respMap = asMap(iocResponse); // Evaluate response @@ -145,4 +149,437 @@ public void testCreateIocUploadSourceConfig() throws IOException { assertTrue(iocs.size() < iocHits.size()); } + public void testCreateIocUploadSourceConfigIncorrectIocTypes() throws IOException { + // Attempt to create ioc upload source config with no correct ioc types + String feedName = "test_ioc_upload"; + String feedFormat = "STIX"; + SourceConfigType sourceConfigType = SourceConfigType.IOC_UPLOAD; + + List iocs = List.of(new STIX2IOCDto( + "id", + "name", + new IOCType(IOCType.IPV4_TYPE), + "value", + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L)); + + IocUploadSource iocUploadSource = new IocUploadSource(null, iocs); + Boolean enabled = false; + List iocTypes = List.of(IOCType.DOMAIN_NAME_TYPE); + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + null, + iocUploadSource, + null, + null, + null, + null, + null, + null, + null, + enabled, + iocTypes, true + ); + + try { + makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + } catch (ResponseException ex) { + Assert.assertEquals(RestStatus.BAD_REQUEST, restStatus(ex.getResponse())); + } + } + + public void testUpdateIocUploadSourceConfig() throws IOException, InterruptedException { + // Create source config with IPV4 IOCs + String feedName = "test_update"; + String feedFormat = "STIX"; + SourceConfigType sourceConfigType = SourceConfigType.IOC_UPLOAD; + + List iocs = List.of(new STIX2IOCDto( + "1", + "ioc", + new IOCType(IOCType.IPV4_TYPE), + "value", + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L)); + + IocUploadSource iocUploadSource = new IocUploadSource(null, iocs); + Boolean enabled = false; + List iocTypes = List.of("ipv4-addr"); + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + null, + iocUploadSource, + null, + null, + null, + null, + null, + null, + null, + enabled, + iocTypes, true + ); + + // create source config with ipv4 ioc type + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.CREATED, restStatus(response)); + Map responseBody = asMap(response); + + String createdId = responseBody.get("_id").toString(); + Assert.assertNotEquals("response is missing Id", SATIFSourceConfigDto.NO_ID, createdId); + + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, createdId), response.getHeader("Location")); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(JOB_INDEX_NAME, request); + Assert.assertEquals(1, hits.size()); + + // ensure same number of iocs got indexed + String indexName = getAllIocIndexPatternById(createdId); + hits = executeSearch(indexName, request); + Assert.assertEquals(iocs.size(), hits.size()); + + // Retrieve all IOCs by feed Ids + Response iocResponse = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(), Map.of("feed_ids", createdId + ",random"), null); + Assert.assertEquals(RestStatus.OK, restStatus(iocResponse)); + Map respMap = asMap(iocResponse); + + // Evaluate response + int totalHits = (int) respMap.get(ListIOCsActionResponse.TOTAL_HITS_FIELD); + assertEquals(iocs.size(), totalHits); + + List> iocHits = (List>) respMap.get(ListIOCsActionResponse.HITS_FIELD); + assertEquals(iocs.size(), iocHits.size()); + + // update source config to contain only hashes as an ioc type + iocs = List.of(new STIX2IOCDto( + "2", + "ioc", + new IOCType(IOCType.HASHES_TYPE), + "value", + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L), + new STIX2IOCDto( + "3", + "ioc", + new IOCType(IOCType.DOMAIN_NAME_TYPE), + "value", + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L)); + + iocUploadSource = new IocUploadSource(null, iocs); + iocTypes = List.of("hashes"); + saTifSourceConfigDto = new SATIFSourceConfigDto( + saTifSourceConfigDto.getId(), + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + null, + iocUploadSource, + null, + null, + null, + null, + null, + null, + null, + enabled, + iocTypes, true + ); + + // update source config with hashes ioc type + response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI +"/" + createdId, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.OK, restStatus(response)); + + // Retrieve all IOCs by feed Ids + iocResponse = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(), Map.of("feed_ids", createdId + ",random"), null); + Assert.assertEquals(RestStatus.OK, restStatus(iocResponse)); + respMap = asMap(iocResponse); + + // Evaluate response - there should only be 1 ioc indexed according to the ioc type + totalHits = (int) respMap.get(ListIOCsActionResponse.TOTAL_HITS_FIELD); + assertEquals(1, totalHits); + + iocHits = (List>) respMap.get(ListIOCsActionResponse.HITS_FIELD); + assertEquals(1, iocHits.size()); + Thread.sleep(10000); + } + + public void testDeleteIocUploadSourceConfigAndAllIocs() throws IOException { + String feedName = "test_ioc_upload"; + String feedFormat = "STIX"; + SourceConfigType sourceConfigType = SourceConfigType.IOC_UPLOAD; + + List iocs = List.of(new STIX2IOCDto( + "id", + "name", + new IOCType(IOCType.IPV4_TYPE), + "value", + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L)); + + IocUploadSource iocUploadSource = new IocUploadSource(null, iocs); + Boolean enabled = false; + List iocTypes = List.of(IOCType.IPV4_TYPE); + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + null, + iocUploadSource, + null, + null, + null, + null, + null, + null, + null, + enabled, + iocTypes, true + ); + + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.CREATED, restStatus(response)); + Map responseBody = asMap(response); + + String createdId = responseBody.get("_id").toString(); + Assert.assertNotEquals("response is missing Id", SATIFSourceConfigDto.NO_ID, createdId); + + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, createdId), response.getHeader("Location")); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(JOB_INDEX_NAME, request); + Assert.assertEquals(1, hits.size()); + + // Delete source config + response = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI + "/" + createdId, Collections.emptyMap(), null); + Assert.assertEquals(RestStatus.OK, restStatus(response)); + responseBody = asMap(response); + + String id = responseBody.get("_id").toString(); + assertEquals(id, createdId); + + // ensure all source configs are deleted + hits = executeSearch(JOB_INDEX_NAME, request); + Assert.assertEquals(0, hits.size()); + + // ensure all iocs are deleted + hits = executeSearch(IOC_ALL_INDEX_PATTERN, request); + Assert.assertEquals(0, hits.size()); + } + + public void testRefreshIocUploadSourceConfigFailure() throws IOException { + String feedName = "test_ioc_upload"; + String feedFormat = "STIX"; + SourceConfigType sourceConfigType = SourceConfigType.IOC_UPLOAD; + + List iocs = List.of(new STIX2IOCDto( + "id", + "name", + new IOCType(IOCType.IPV4_TYPE), + "value", + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L)); + + IocUploadSource iocUploadSource = new IocUploadSource(null, iocs); + Boolean enabled = false; + List iocTypes = List.of(IOCType.IPV4_TYPE); + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + null, + iocUploadSource, + null, + null, + null, + null, + null, + null, + null, + enabled, + iocTypes, true + ); + + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.CREATED, restStatus(response)); + Map responseBody = asMap(response); + + String createdId = responseBody.get("_id").toString(); + Assert.assertNotEquals("response is missing Id", SATIFSourceConfigDto.NO_ID, createdId); + + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, createdId), response.getHeader("Location")); + + + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(JOB_INDEX_NAME, request); + Assert.assertEquals(1, hits.size()); + + // Try to execute refresh api + try { + makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI + "/" + createdId + "/_refresh", Collections.emptyMap(), null); + } catch (ResponseException ex) { + Assert.assertEquals(RestStatus.BAD_REQUEST, restStatus(ex.getResponse())); + } + } + + public void testSearchIocUploadSourceConfig() throws IOException { + String feedName = "test_ioc_upload"; + String feedFormat = "STIX"; + SourceConfigType sourceConfigType = SourceConfigType.IOC_UPLOAD; + + List iocs = List.of(new STIX2IOCDto( + "id", + "name", + new IOCType(IOCType.IPV4_TYPE), + "value", + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L)); + + IocUploadSource iocUploadSource = new IocUploadSource(null, iocs); + Boolean enabled = false; + List iocTypes = List.of(IOCType.IPV4_TYPE); + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + null, + iocUploadSource, + null, + null, + null, + null, + null, + null, + null, + enabled, + iocTypes, true + ); + + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.CREATED, restStatus(response)); + Map responseBody = asMap(response); + + String createdId = responseBody.get("_id").toString(); + Assert.assertNotEquals("response is missing Id", SATIFSourceConfigDto.NO_ID, createdId); + + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, createdId), response.getHeader("Location")); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(JOB_INDEX_NAME, request); + Assert.assertEquals(1, hits.size()); + + // Search all source configs + Response sourceConfigResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI + "/_search", Collections.emptyMap(), new StringEntity(request), new BasicHeader("Content-type", "application/json")); + Assert.assertEquals(RestStatus.OK, restStatus(sourceConfigResponse)); + Map respMap = asMap(sourceConfigResponse); + + // Expected value is 2 - one ioc upload source config and one default source config + Assert.assertEquals(2, ((Map) ((Map) respMap.get("hits")).get("total")).get("value")); + } + }