Merge pull request #3060 from wazuh/backport_merged_optimization

Backport merged files generation optimization to 4.3 - Integration tests

CHANGELOG.md

@@ -11,6 +11,7 @@ Release report: TBD
 
 - Fix GCloud IT - test_max_messages error ([#3006](https://github.com/wazuh/wazuh-qa/pull/3006)) \- (Framework + Tests)
 - Fix Remoted IT - test_agent_communication ([#3088](https://github.com/wazuh/wazuh-qa/pull/3088)) \- (Framework)
+- Add Remoted IT - test_multi_groups ([#3060](https://github.com/wazuh/wazuh-qa/pull/3060)) \- (Framework + Tests)
 
 ## [4.3.5] - 29-06-2022
 

deps/wazuh_testing/wazuh_testing/tools/configuration.py

@@ -14,6 +14,7 @@
 from wazuh_testing import global_parameters, logger
 from wazuh_testing.tools import WAZUH_PATH, GEN_OSSEC, WAZUH_CONF, PREFIX, WAZUH_LOCAL_INTERNAL_OPTIONS
 from wazuh_testing import global_parameters, logger
+from wazuh_testing.tools import file
 
 
 # customize _serialize_xml to avoid lexicographical order in XML attributes
@@ -642,3 +643,29 @@ def set_local_internal_options_dict(dict_local_internal_options):
         for option_name, option_value in dict_local_internal_options.items():
             local_internal_configuration_string = f"{str(option_name)}={str(option_value)}\n"
             local_internal_option_file.write(local_internal_configuration_string)
+
+
+def get_test_cases_data(data_file_path):
+    """Load a test case template file and get its data.
+
+    Template example file: tests/integration/test_remoted/test_multi_groups/data/test_cases/case_file_actions.yaml
+
+    Args:
+        data_file_path (str): Test case template file path.
+
+    Returns:
+        (list(dict), list(dict), list(str)): Configurations, metadata and test case names.
+    """
+    test_cases_data = file.read_yaml(data_file_path)
+    configuration_parameters = []
+    configuration_metadata = []
+    test_cases_ids = []
+
+    for test_case in test_cases_data:
+        configuration_parameters.append(test_case['configuration_parameters'])
+        metadata_parameters = {'name': test_case['name'], 'description': test_case['description']}
+        metadata_parameters.update(test_case['metadata'])
+        configuration_metadata.append(metadata_parameters)
+        test_cases_ids.append(test_case['name'])
+
+    return configuration_parameters, configuration_metadata, test_cases_ids

deps/wazuh_testing/wazuh_testing/tools/wazuh_manager.py

@@ -1,7 +1,9 @@
 import os
 import subprocess
+import requests
 from wazuh_testing.tools import WAZUH_PATH
 from wazuh_testing.wazuh_db import query_wdb
+from wazuh_testing.api import get_api_details_dict
 
 
 def list_agents_ids():
@@ -12,7 +14,7 @@ def list_agents_ids():
 
 
 def remove_agents(agents_id, remove_type='wazuhdb'):
-    if remove_type not in ['wazuhdb', 'manage_agents']:
+    if remove_type not in ['wazuhdb', 'manage_agents', 'api']:
         raise ValueError("Invalid type of agent removal: %s" % remove_type)
 
     if agents_id:
@@ -22,6 +24,18 @@ def remove_agents(agents_id, remove_type='wazuhdb'):
                                 stderr=subprocess.STDOUT)
             elif remove_type == 'wazuhdb':
                 result = query_wdb(f"global delete-agent {str(agent_id)}")
+        if remove_type == 'api':
+            api_details = get_api_details_dict()
+            payload = {
+                'agents_list': agents_id,
+                'status': 'all',
+                'older_than': '0s'
+            }
+            url = f"{api_details['base_url']}/agents"
+            response = requests.delete(url, headers=api_details['auth_headers'], params=payload, verify=False)
+            response_data = response.json()
+            if response.status_code != 200:
+                raise RuntimeError(f"Error deleting an agent: {response_data}")
 
 
 def remove_all_agents(remove_type):

tests/integration/test_remoted/test_multi_groups/data/test_cases/case_file_actions.yaml

@@ -0,0 +1,14 @@
+- name: check_that_the_new_file_is_added
+  description: When a file is created in the group directory, the merged.mg file should be updated and the new file must
+               appears inside it.
+  configuration_parameters:
+    ACTION: create
+  metadata:
+    action: create
+- name: check_that_the_old_file_is_deleted
+  description: When a file is deleted in the group directory, the merged.mg file should be updated without the file in
+               it.
+  configuration_parameters:
+    ACTION: delete
+  metadata:
+    action: delete

tests/integration/test_remoted/test_multi_groups/test_merged_mg_file_content.py

@@ -0,0 +1,216 @@
+'''
+copyright: Copyright (C) 2015-2022, Wazuh Inc.
+           Created by Wazuh, Inc. <[email protected]>.
+           This program is free software; you can redistribute it and/or modify it under the terms of GPLv2
+
+type: integration
+
+brief: Whenever a file is created or deleted from the group directory the merged.mg file must be updated. This test
+       checks the content of the merged.mg file that wazuh-remoted compiles for multi-groups.
+
+tier: 0
+
+modules:
+    - remoted
+
+components:
+    - manager
+
+daemons:
+    - wazuh-remoted
+
+os_platform:
+    - linux
+
+os_version:
+    - Arch Linux
+    - Amazon Linux 2
+    - Amazon Linux 1
+    - CentOS 8
+    - CentOS 7
+    - Debian Buster
+    - Red Hat 8
+    - Ubuntu Focal
+    - Ubuntu Bionic
+
+references:
+    - https://documentation.wazuh.com/current/user-manual/reference/daemons/wazuh-remoted.html
+
+tags:
+    - remoted
+'''
+import hashlib
+import os
+import re
+import pytest
+import requests
+from time import sleep
+
+from wazuh_testing.api import get_api_details_dict
+from wazuh_testing.remote import DEFAULT_TESTING_GROUP_NAME, new_agent_group, \
+                                  remove_agent_group
+from wazuh_testing.tools import REMOTE_DAEMON, WAZUH_PATH, configuration
+from wazuh_testing.tools.file import delete_file
+from wazuh_testing.tools.services import check_daemon_status, control_service
+from wazuh_testing.tools.wazuh_manager import remove_agents
+
+# Marks
+pytestmarks = [pytest.mark.server]
+
+# Variables
+agent_name = 'testing_agent'
+agent_ip = 'any'
+groups_folder = os.path.join(WAZUH_PATH, 'queue', 'agent-groups')
+default_group_name = 'default'
+groups_list = [default_group_name, DEFAULT_TESTING_GROUP_NAME]
+mg_name = hashlib.sha256(','.join(groups_list).encode()).hexdigest()[:8]
+mg_folder_path = os.path.join(WAZUH_PATH, 'var', 'multigroups', mg_name)
+merged_mg_file = os.path.join(mg_folder_path, 'merged.mg')
+shared_folder_path = os.path.join(WAZUH_PATH, 'etc', 'shared')
+shared_file_name = 'testing_file'
+shared_file_path = os.path.join(shared_folder_path, DEFAULT_TESTING_GROUP_NAME, shared_file_name)
+response_data = None
+elapsed_time = 2
+wait_time = 1
+expected_line = f"!0 {shared_file_name}"
+
+# Configuration
+local_internal_options = {'remoted.shared_reload': f"{wait_time}"}
+test_data_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data')
+test_cases_path = os.path.join(test_data_path, 'test_cases')
+tcases_data = os.path.join(test_cases_path, 'case_file_actions.yaml')
+configuration_parameters, configuration_metadata, test_case_ids = configuration.get_test_cases_data(tcases_data)
+
+
+# Fixtures
+@pytest.fixture(scope="module")
+def restart_remoted():
+    """Restart the wazuh-remoted daemon."""
+
+    control_service('restart', daemon=REMOTE_DAEMON)
+    check_daemon_status(target_daemon=REMOTE_DAEMON)
+
+    yield
+
+    control_service('stop', daemon=REMOTE_DAEMON)
+    check_daemon_status(target_daemon=REMOTE_DAEMON, running_condition=False)
+
+
+@pytest.fixture(scope='function')
+def prepare_environment(request, register_agent):
+    '''
+    Configure a custom environment for testing:
+    1) Register a new agent using the API.
+    2) Create a new group using the agent_groups tool.
+    3) Assign the new group to the agent.
+    4) After the execution of the test, remove the group created.
+    '''
+
+    new_agent_group()
+
+    agent_id = getattr(request.module, 'response_data')['id']
+
+    # Delete this in 4.4, we have to use the agent_groups script to assign the groups
+    with open(f"{WAZUH_PATH}/queue/agent-groups/{agent_id}", 'w') as agent_group_file:
+        agent_group_file.write(f"default,{DEFAULT_TESTING_GROUP_NAME}")
+
+    yield
+
+    remove_agent_group(DEFAULT_TESTING_GROUP_NAME)
+    delete_file(os.path.join(groups_folder, agent_id))
+
+
+@pytest.fixture(scope='function')
+def register_agent(request):
+    """Register an agent via API."""
+
+    api_details = get_api_details_dict()
+    data = {
+        'name': agent_name,
+        'ip': agent_ip
+    }
+    url = f"{api_details['base_url']}/agents"
+    response = requests.post(url=url, headers=api_details['auth_headers'], json=data, verify=False)
+    response_data = response.json()
+
+    if response.status_code != 200:
+        raise RuntimeError(f"Error registering an agent: {response_data}")
+
+    setattr(request.module, 'response_data', response_data['data'])
+
+    yield
+
+    registered_agent = getattr(request.module, 'response_data')['id']
+    remove_agents(agents_id=[registered_agent], remove_type='api')
+
+
+def manipulate_file(action, file_path):
+    if action == 'create':
+        f = open(file_path, "w")
+        f.close()
+    else:
+        delete_file(file_path)
+
+
+@pytest.mark.tier(level=0)
+@pytest.mark.parametrize('metadata', configuration_metadata, ids=test_case_ids)
+@pytest.mark.filterwarnings('ignore::urllib3.exceptions.InsecureRequestWarning')
+def test_merged_mg_file_content(metadata, configure_local_internal_options_module, restart_remoted,
+                                prepare_environment):
+    '''
+    description: Check the content of the merged.mg file that wazuh-remoted compiles for multi-groups.
+
+    wazuh_min_version: 4.3.6
+
+    tier: 0
+
+    test_phases:
+        - Set a custom Wazuh configuration.
+        - Restart wazuh-remoted.
+        - Register an agent, create a group and assign it to the agent.
+        - Execute an action (create/delete file) inside the group folder.
+        - Verify that the action (create/delete file) was applied in the multigroup folder.
+
+    parameters:
+        - metadata:
+            type: dict
+            brief: Metadata containing the action to execute.
+        - configure_local_internal_options_module:
+            type: fixture
+            brief: Fixture to configure the local internal options file.
+        - restart_remoted:
+            type: fixture
+            brief: Restart the wazuh-remoted daemon.
+        - prepare_environment:
+            type: fixture
+            brief: Configure a custom environment for testing.
+
+    assertions:
+        - Verify that the file exists or not in the multigroups directory.
+        - Verify that the file is or is not in the merged.mg file
+
+    input_description: Different test cases defined in a YAML file.
+
+    expected_output:
+        - r'^!0 testing_file$'
+    '''
+    action = metadata['action']
+    match_expected_line = None
+
+    manipulate_file(action, shared_file_path)
+    sleep(wait_time + elapsed_time)
+
+    file_exists = os.path.exists(os.path.join(mg_folder_path, shared_file_name))
+    if os.path.exists(merged_mg_file):
+        with open(merged_mg_file, 'r') as merged_file:
+            merged_file_lines = merged_file.readlines()
+            match_regex = re.compile(rf"^{expected_line}$")
+            match_expected_line = list(filter(match_regex.match, merged_file_lines))
+    else:
+        raise FileNotFoundError(f"The file: {merged_mg_file} was not created.")
+
+    expected_conditions = [True, [expected_line + '\n']] if action == 'create' else [False, []]
+
+    assert file_exists == expected_conditions[0], f"The file was not {action}d in the multigroups directory.\n"
+    assert match_expected_line == expected_conditions[1], f"The file is \
+                                                          {'not' if action == 'created' else ''} in {merged_mg_file}."