diff --git a/.github/workflows/molecule.yml b/.github/workflows/molecule.yml new file mode 100644 index 00000000..ac17650d --- /dev/null +++ b/.github/workflows/molecule.yml @@ -0,0 +1,57 @@ +--- +name: Molecule Tests + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +jobs: + # lint: + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # + # - uses: actions/setup-python@v5 + # with: + # python-version: "3.12" + # cache: pip + + # - name: Install dependencies + # run: pip install -r requirements.txt + + # - name: Run ansible-lint + # run: ansible-lint + # + molecule: + runs-on: ubuntu-latest + # needs: lint + strategy: + fail-fast: false + matrix: + scenario: + - default + - security + - custom-config + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + + - name: Install dependencies + run: pip install -r requirements.txt + + - name: Install Ansible collections + run: ansible-galaxy collection install -r requirements.yml + + - name: Run Molecule tests + run: molecule test -s ${{ matrix.scenario }} + env: + ANSIBLE_FORCE_COLOR: "true" + PY_COLORS: "1" diff --git a/.gitignore b/.gitignore index ff930462..917cd8d6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -.kitchen/ license*.json *.pyc .vendor diff --git a/.kitchen.yml b/.kitchen.yml deleted file mode 100644 index 15af7cfd..00000000 --- a/.kitchen.yml +++ /dev/null @@ -1,195 +0,0 @@ ---- -driver: - name: docker - -transport: - max_ssh_sessions: 6 - -provisioner: - name: ansible_playbook - hosts: localhost - roles_path: ../ - require_ansible_repo: true - require_ansible_omnibus: false - require_ansible_source: false - http_proxy: <%= ENV['HTTP_PROXY'] %> - https_proxy: <%= ENV['HTTPS_PROXY'] %> - no_proxy: localhost,127.0.0.1 - ignore_extensions_from_root: [".git",".idea",".kitchen.yml"] - ignore_paths_from_root: [".git",".idea",".kitchen"] - <% if ENV['VERSION'] %> - attributes: - extra_vars: - es_major_version: "<%= ENV['VERSION'] %>" - <% if ENV['VERSION'] == '6.x' %> - es_version: '6.8.23' - <% end %> - <% end %> - -platforms: - - name: ubuntu-14.04 - driver_config: - image: ubuntu:14.04 - privileged: true - provision_command: - - apt-get update -q && apt-get install -y -q software-properties-common && add-apt-repository -y ppa:ansible/ansible && add-apt-repository -y ppa:openjdk-r/ppa - - apt-get update -q && apt-get -y -q install ansible openjdk-8-jre python-jmespath - - locale-gen en_US.UTF-8 && localedef -i en_US -c -f UTF-8 en_US.UTF-8 - use_sudo: false - volume: - - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json - - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers - - name: ubuntu-16.04 - driver_config: - image: ubuntu:16.04 - privileged: true - provision_command: - - apt-get update -q && apt-get install -y -q iproute locales software-properties-common && add-apt-repository -y ppa:ansible/ansible - - apt-get update -q && apt-get install -y -q ansible python-jmespath - - locale-gen en_US.UTF-8 && localedef -i en_US -c -f UTF-8 en_US.UTF-8 - use_sudo: false - volume: - - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json - - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers - run_command: "/sbin/init" - - name: ubuntu-18.04 - driver_config: - image: ubuntu:18.04 - privileged: true - provision_command: - - apt-get update -q && apt-get install -y -q ansible iproute2 python-jmespath - use_sudo: false - volume: - - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json - - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers - run_command: "/sbin/init" - - name: ubuntu-20.04 - driver_config: - image: ubuntu:20.04 - privileged: true - provision_command: - - apt-get update -q && apt-get install -y -q ansible gpg iproute2 python3-jmespath - use_sudo: false - volume: - - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json - - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers - run_command: "/sbin/init" - - name: debian-8 - driver_config: - image: debian:8 - privileged: true - provision_command: - - apt-get update -q && apt-get install -y -q gnupg2 python-jmespath - - echo "deb http://archive.debian.org/debian jessie-backports main" > /etc/apt/sources.list.d/jessie-backports.list - - echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf - - apt-get update && apt-get -y install -t jessie-backports openjdk-8-jre-headless - - echo "deb http://ppa.launchpad.net/ansible/ansible/ubuntu trusty main" > /etc/apt/sources.list.d/ansible.list - - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 93C4A3FD7BB9C367 - - apt-get update -q && apt-get install -y -q ansible - volume: - - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json - - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers - use_sudo: false - run_command: "/sbin/init" - - name: debian-9 - driver_config: - image: debian:9 - privileged: true - provision_command: - - apt-get update -q && apt-get install -y -q gnupg2 python-jmespath systemd-sysv - - echo "deb http://ppa.launchpad.net/ansible/ansible/ubuntu trusty main" > /etc/apt/sources.list.d/ansible.list - - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 93C4A3FD7BB9C367 - - apt-get update -q && apt-get install -y -q ansible - volume: - - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json - - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers - use_sudo: false - run_command: "/sbin/init" - - name: debian-10 - driver_config: - image: debian:10 - privileged: true - provision_command: - - apt-get update -q && apt-get install -y -q gnupg2 python-jmespath systemd-sysv - - echo "deb http://ppa.launchpad.net/ansible/ansible/ubuntu trusty main" > /etc/apt/sources.list.d/ansible.list - - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 93C4A3FD7BB9C367 - - apt-get update -q && apt-get install -y -q ansible - volume: - - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json - - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers - use_sudo: false - run_command: "/sbin/init" - - name: centos-7 - driver_config: - image: centos:7 - provision_command: - - yum -y install epel-release - - yum -y install ansible iproute python2-jmespath - volume: - - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json - - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers - run_command: "/usr/sbin/init" - privileged: true - use_sudo: false - - name: centos-8 - driver_config: - image: centos:8 - provision_command: - - yum -y install epel-release - - yum -y install ansible iproute python3-jmespath - volume: - - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json - - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers - run_command: "/usr/sbin/init" - privileged: true - use_sudo: false - - name: amazonlinux-2 - driver_config: - image: amazonlinux:2 - provision_command: - - yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm - - yum -y install ansible iproute python2-jmespath - volume: - - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json - - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers - run_command: "/usr/sbin/init" - privileged: true - use_sudo: false - -suites: - - name: custom-config - provisioner: - idempotency_test: true - playbook: test/integration/custom-config.yml - - name: default - provisioner: - playbook: test/integration/default.yml - idempotency_test: true - - name: license - provisioner: - playbook: test/integration/license.yml - idempotency_test: true - - name: oss - provisioner: - idempotency_test: true - playbook: test/integration/oss.yml - - name: oss-to-default-upgrade - provisioner: - playbook: test/integration/oss-to-default-upgrade.yml - idempotency_test: false - - name: oss-upgrade - provisioner: - playbook: test/integration/oss-upgrade.yml - idempotency_test: false - - name: security - provisioner: - playbook: test/integration/security.yml - idempotency_test: true - - name: trial - provisioner: - playbook: test/integration/trial.yml - idempotency_test: false # es_xpack_trial is not idempotent currently - - name: upgrade - provisioner: - playbook: test/integration/upgrade.yml - idempotency_test: false diff --git a/.ruby-version b/.ruby-version deleted file mode 100644 index 0aec50e6..00000000 --- a/.ruby-version +++ /dev/null @@ -1 +0,0 @@ -3.1.4 diff --git a/Gemfile b/Gemfile deleted file mode 100644 index bf42881a..00000000 --- a/Gemfile +++ /dev/null @@ -1,6 +0,0 @@ -source 'https://rubygems.org' - -gem 'test-kitchen' -gem 'kitchen-docker' -gem 'kitchen-ansible' -gem 'net-ssh' diff --git a/Gemfile.lock b/Gemfile.lock deleted file mode 100644 index 1a0822a4..00000000 --- a/Gemfile.lock +++ /dev/null @@ -1,131 +0,0 @@ -GEM - remote: https://rubygems.org/ - specs: - base64 (0.3.0) - bcrypt_pbkdf (1.1.2) - benchmark (0.5.0) - bigdecimal (4.0.1) - builder (3.3.0) - cgi (0.5.1) - chef-gyoku (1.5.0) - builder (>= 2.1.2) - rexml (~> 3.4) - chef-utils (19.1.164) - concurrent-ruby - chef-winrm (2.5.0) - builder (>= 2.1.2) - chef-gyoku (~> 1.5) - erubi (~> 1.8) - gssapi (~> 1.2) - httpclient (~> 2.2, >= 2.2.0.2) - logging (>= 1.6.1, < 3.0) - nori (~> 2.7) - rexml (>= 3.4.2, < 4.0) - rubyntlm (~> 0.6.0, >= 0.6.3) - chef-winrm-elevated (1.2.5) - chef-winrm (>= 2.3.11) - chef-winrm-fs (>= 1.3.7) - erubi (~> 1.8) - chef-winrm-fs (1.4.2) - benchmark (~> 0.5.0) - chef-winrm (~> 2.4) - csv (~> 3.3) - erubi (>= 1.7) - logging (>= 1.6.1, < 3.0) - rubyzip (~> 2.0) - concurrent-ruby (1.3.6) - csv (3.3.5) - date (3.5.1) - ed25519 (1.4.0) - erb (4.0.4) - cgi (>= 0.3.3) - erubi (1.13.1) - ffi (1.17.3) - gssapi (1.3.1) - ffi (>= 1.0.1) - httpclient (2.9.0) - mutex_m - io-console (0.8.2) - irb (1.17.0) - pp (>= 0.6.0) - prism (>= 1.3.0) - rdoc (>= 4.0.0) - reline (>= 0.4.2) - kitchen-ansible (0.58.0) - net-ssh (>= 3) - test-kitchen (>= 1.4) - kitchen-docker (3.0.0) - test-kitchen (>= 1.0.0) - little-plugger (1.1.4) - logger (1.7.0) - logging (2.4.0) - little-plugger (~> 1.1) - multi_json (~> 1.14) - mixlib-install (3.16.0) - mixlib-shellout - mixlib-versioning - thor - mixlib-shellout (3.4.10) - chef-utils - mixlib-versioning (1.2.12) - multi_json (1.19.1) - mutex_m (0.3.0) - net-scp (4.1.0) - net-ssh (>= 2.6.5, < 8.0.0) - net-ssh (7.3.2) - net-ssh-gateway (2.0.0) - net-ssh (>= 4.0.0) - nori (2.7.1) - bigdecimal - ostruct (0.6.3) - pp (0.6.3) - prettyprint - prettyprint (0.2.0) - prism (1.9.0) - psych (5.3.1) - date - stringio - rdoc (7.2.0) - erb - psych (>= 4.0.0) - tsort - reline (0.6.3) - io-console (~> 0.5) - rexml (3.4.4) - rubyntlm (0.6.5) - base64 - rubyzip (2.4.1) - stringio (3.2.0) - syslog (0.4.0) - logger - test-kitchen (4.0.0) - bcrypt_pbkdf (~> 1.0) - chef-utils (>= 16.4.35) - chef-winrm (>= 2.5.0, < 3.0) - chef-winrm-elevated (>= 1.0, < 2.0) - chef-winrm-fs (>= 1.0, < 2.0) - csv (~> 3.3) - ed25519 (~> 1.3) - irb (~> 1.15) - mixlib-install (~> 3.6) - mixlib-shellout (>= 1.2, < 4.0) - net-scp (>= 1.1, < 5.0) - net-ssh (>= 2.9, < 8.0) - net-ssh-gateway (>= 1.2, < 3.0) - ostruct (~> 0.6) - syslog (~> 0.3) - thor (>= 0.19, < 2.0) - thor (1.5.0) - tsort (0.2.0) - -PLATFORMS - ruby - -DEPENDENCIES - kitchen-ansible - kitchen-docker - net-ssh - test-kitchen - -BUNDLED WITH - 2.4.15 diff --git a/Makefile b/Makefile deleted file mode 100644 index 0de81fb1..00000000 --- a/Makefile +++ /dev/null @@ -1,32 +0,0 @@ -default: build - -SHELL:=/bin/bash -eux -export VERSION := 7.x -PATTERN := default-ubuntu-1804 - -.PHONY: converge verify test login destroy list - -setup: - bundle install - docker ps - -converge: - bundle exec kitchen converge $(PATTERN) - -verify: - bundle exec kitchen verify $(PATTERN) - -test: - bundle exec kitchen test $(PATTERN) --destroy=always - -login: - bundle exec kitchen login $(PATTERN) - -destroy: - bundle exec kitchen destroy $(PATTERN) - -destroy-all: - bundle exec kitchen destroy - -list: - bundle exec kitchen list diff --git a/README.md b/README.md index 05e762ea..30570edc 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ You can now override the configuration files with your own versions by using the ## Testing -This playbook uses [Kitchen](https://kitchen.ci/) for CI and local testing. +This playbook uses [Molecule](https://molecule.readthedocs.io/) for CI and local testing. ### Requirements diff --git a/ansible.cfg b/ansible.cfg index 0440d489..613d83b1 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -1 +1,2 @@ [defaults] +roles_path = ../ diff --git a/filter_plugins/custom.py b/filter_plugins/custom.py index ad13ab18..19818496 100644 --- a/filter_plugins/custom.py +++ b/filter_plugins/custom.py @@ -2,7 +2,6 @@ import re import os.path -from six import string_types def modify_list(values=[], pattern="", replacement="", ignorecase=False): @@ -16,7 +15,7 @@ def modify_list(values=[], pattern="", replacement="", ignorecase=False): def append_to_list(values=[], suffix=""): - if isinstance(values, string_types): + if isinstance(values, str): values = values.split(",") return [str(value + suffix) for value in values] diff --git a/meta/main.yml b/meta/main.yml index fa8e0bd7..0c001586 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -2,9 +2,11 @@ allow_duplicates: true galaxy_info: - author: Robin Clarke, Jakob Reiter, Dale McDiarmid - description: Elasticsearch for Linux - company: "Elastic.co" + author: hostinger/observability + description: Fork of elastic.elasticsearch role from Ansible Galaxy, with customizations for Hostinger's use case. + namespace: hostinger + role_name: elasticsearch + company: "Hostinger" license: "license (Apache)" min_ansible_version: 2.5.0 platforms: diff --git a/molecule/custom-config/converge.yml b/molecule/custom-config/converge.yml new file mode 100644 index 00000000..69fc8405 --- /dev/null +++ b/molecule/custom-config/converge.yml @@ -0,0 +1,31 @@ +--- +- name: Converge + hosts: all + become: true + roles: + - role: ansible-elasticsearch + vars: + es_version: "8.19.10" + es_version_lock: true + es_bootstrap: true + es_config: + discovery.type: single-node + xpack.security.enabled: false + xpack.security.http.ssl.enabled: false + xpack.security.transport.ssl.enabled: false + network.host: "0.0.0.0" + http.port: 9200 + node.roles: [master, data, ingest] + cluster.name: test-cluster + node.attr.rack_id: test-rack + cluster.routing.allocation.awareness.attributes: rack_id + es_config_log4j2: "{{ lookup('env', 'MOLECULE_SCENARIO_DIRECTORY') }}/files/log4j2.properties" + es_plugins: + - plugin: repository-s3 + es_keystore_entries: + - key: s3.client.hostinger_s3.access_key + value: test-access-key + state: present + - key: s3.client.hostinger_s3.secret_key + value: test-secret-key + state: present diff --git a/molecule/custom-config/files/log4j2.properties b/molecule/custom-config/files/log4j2.properties new file mode 100644 index 00000000..4c97828c --- /dev/null +++ b/molecule/custom-config/files/log4j2.properties @@ -0,0 +1,132 @@ +status = error + +appender.console.type = Console +appender.console.name = console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%consoleException%n + +######## Server JSON ############################ +appender.rolling.type = RollingFile +appender.rolling.name = rolling +appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_server.json +appender.rolling.layout.type = ECSJsonLayout +appender.rolling.layout.dataset = elasticsearch.server + +appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.json.gz +appender.rolling.policies.type = Policies +appender.rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.rolling.policies.time.interval = 1 +appender.rolling.policies.time.modulate = true +appender.rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.rolling.policies.size.size = 128MB +appender.rolling.strategy.type = DefaultRolloverStrategy +appender.rolling.strategy.max = 30 +appender.rolling.strategy.action.type = Delete +appender.rolling.strategy.action.basepath = ${sys:es.logs.base_path} +appender.rolling.strategy.action.condition.type = IfFileName +appender.rolling.strategy.action.condition.glob = ${sys:es.logs.cluster_name}-* +appender.rolling.strategy.action.condition.nested_condition.type = IfLastModified +appender.rolling.strategy.action.condition.nested_condition.age = 30D +################################################ +######## Server - old style pattern ########### +appender.rolling_old.type = RollingFile +appender.rolling_old.name = rolling_old +appender.rolling_old.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log +appender.rolling_old.layout.type = PatternLayout +appender.rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n + +appender.rolling_old.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.log.gz +appender.rolling_old.policies.type = Policies +appender.rolling_old.policies.time.type = TimeBasedTriggeringPolicy +appender.rolling_old.policies.time.interval = 1 +appender.rolling_old.policies.time.modulate = true +appender.rolling_old.policies.size.type = SizeBasedTriggeringPolicy +appender.rolling_old.policies.size.size = 128MB +appender.rolling_old.strategy.type = DefaultRolloverStrategy +appender.rolling_old.strategy.max = 30 +appender.rolling_old.strategy.action.type = Delete +appender.rolling_old.strategy.action.basepath = ${sys:es.logs.base_path} +appender.rolling_old.strategy.action.condition.type = IfFileName +appender.rolling_old.strategy.action.condition.glob = ${sys:es.logs.cluster_name}-* +appender.rolling_old.strategy.action.condition.nested_condition.type = IfLastModified +appender.rolling_old.strategy.action.condition.nested_condition.age = 30D +################################################ + +rootLogger.level = info +rootLogger.appenderRef.console.ref = console +rootLogger.appenderRef.rolling.ref = rolling +rootLogger.appenderRef.rolling_old.ref = rolling_old + +######## Deprecation JSON ####################### +appender.deprecation_rolling.type = RollingFile +appender.deprecation_rolling.name = deprecation_rolling +appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.json +appender.deprecation_rolling.layout.type = ECSJsonLayout +appender.deprecation_rolling.layout.dataset = deprecation.elasticsearch +appender.deprecation_rolling.filter.rate_limit.type = RateLimitingFilter + +appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.json.gz +appender.deprecation_rolling.policies.type = Policies +appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.deprecation_rolling.policies.size.size = 1GB +appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy +appender.deprecation_rolling.strategy.max = 4 + +appender.header_warning.type = HeaderWarningAppender +appender.header_warning.name = header_warning +################################################# + +logger.deprecation.name = org.elasticsearch.deprecation +logger.deprecation.level = WARN +logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling +logger.deprecation.appenderRef.header_warning.ref = header_warning +logger.deprecation.additivity = false + +######## Search slowlog JSON #################### +appender.index_search_slowlog_rolling.type = RollingFile +appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling +appender.index_search_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog.json +appender.index_search_slowlog_rolling.layout.type = ECSJsonLayout +appender.index_search_slowlog_rolling.layout.dataset = elasticsearch.index_search_slowlog + +appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog-%i.json.gz +appender.index_search_slowlog_rolling.policies.type = Policies +appender.index_search_slowlog_rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.index_search_slowlog_rolling.policies.size.size = 1GB +appender.index_search_slowlog_rolling.strategy.type = DefaultRolloverStrategy +appender.index_search_slowlog_rolling.strategy.max = 4 +################################################# + +logger.index_search_slowlog_rolling.name = index.search.slowlog +logger.index_search_slowlog_rolling.level = trace +logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling +logger.index_search_slowlog_rolling.additivity = false + +######## Indexing slowlog JSON ################## +appender.index_indexing_slowlog_rolling.type = RollingFile +appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling +appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog.json +appender.index_indexing_slowlog_rolling.layout.type = ECSJsonLayout +appender.index_indexing_slowlog_rolling.layout.dataset = elasticsearch.index_indexing_slowlog + +appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog-%i.json.gz +appender.index_indexing_slowlog_rolling.policies.type = Policies +appender.index_indexing_slowlog_rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.index_indexing_slowlog_rolling.policies.size.size = 1GB +appender.index_indexing_slowlog_rolling.strategy.type = DefaultRolloverStrategy +appender.index_indexing_slowlog_rolling.strategy.max = 4 +################################################# + +logger.index_indexing_slowlog.name = index.indexing.slowlog.index +logger.index_indexing_slowlog.level = trace +logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling +logger.index_indexing_slowlog.additivity = false + +logger.com_amazonaws.name = com.amazonaws +logger.com_amazonaws.level = warn + +logger.com_amazonaws_jmx_SdkMBeanRegistrySupport.name = com.amazonaws.jmx.SdkMBeanRegistrySupport +logger.com_amazonaws_jmx_SdkMBeanRegistrySupport.level = error + +logger.com_amazonaws_metrics_AwsSdkMetrics.name = com.amazonaws.metrics.AwsSdkMetrics +logger.com_amazonaws_metrics_AwsSdkMetrics.level = error diff --git a/molecule/custom-config/molecule.yml b/molecule/custom-config/molecule.yml new file mode 100644 index 00000000..ec2ac84a --- /dev/null +++ b/molecule/custom-config/molecule.yml @@ -0,0 +1,44 @@ +--- +driver: + name: docker + +platforms: + - name: elasticsearch + image: geerlingguy/docker-ubuntu2204-ansible:latest + pre_build_image: true + command: "" + privileged: true + cgroupns_mode: host + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + ulimits: + - "nofile:65536:65536" + - "memlock:-1:-1" + +provisioner: + name: ansible + env: + ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY}/.." + ANSIBLE_FORCE_COLOR: "true" + PY_COLORS: "1" + config_options: + defaults: + stdout_callback: default + result_format: yaml + +verifier: + name: ansible + +scenario: + test_sequence: + - dependency + - cleanup + - destroy + - syntax + - create + - prepare + - converge + - side_effect + - verify + - cleanup + - destroy diff --git a/molecule/custom-config/prepare.yml b/molecule/custom-config/prepare.yml new file mode 100644 index 00000000..f74eaad3 --- /dev/null +++ b/molecule/custom-config/prepare.yml @@ -0,0 +1,14 @@ +--- +- name: Prepare + hosts: all + become: true + tasks: + - name: Update apt cache + ansible.builtin.apt: + update_cache: true + cache_valid_time: 3600 + + - name: Install gnupg for apt key management + ansible.builtin.apt: + name: gnupg + state: present diff --git a/molecule/custom-config/verify.yml b/molecule/custom-config/verify.yml new file mode 100644 index 00000000..d57a2c05 --- /dev/null +++ b/molecule/custom-config/verify.yml @@ -0,0 +1,60 @@ +--- +- name: Verify + hosts: all + become: true + tasks: + - name: Gather service facts + ansible.builtin.service_facts: + + - name: Assert Elasticsearch is running + ansible.builtin.assert: + that: + - "'elasticsearch.service' in ansible_facts.services" + - "ansible_facts.services['elasticsearch.service'].state == 'running'" + + - name: Wait for Elasticsearch API + ansible.builtin.uri: + url: "http://localhost:9200" + status_code: 200 + register: api_response + until: api_response.status == 200 + retries: 30 + delay: 10 + + - name: Assert correct version + ansible.builtin.assert: + that: + - "api_response.json.version.number == '8.19.10'" + fail_msg: "Expected 8.19.10, got {{ api_response.json.version.number }}" + + - name: Read deployed log4j2.properties + ansible.builtin.slurp: + src: /etc/elasticsearch/log4j2.properties + register: log4j2_content + + - name: Assert custom log4j2 with ECS JSON layout is deployed + ansible.builtin.assert: + that: + - "'ECSJsonLayout' in (log4j2_content.content | b64decode)" + + - name: Check repository-s3 module directory exists + ansible.builtin.stat: + path: /usr/share/elasticsearch/modules/repository-s3 + register: module_dir + + - name: Assert repository-s3 is available (bundled as module in ES 8.x) + ansible.builtin.assert: + that: + - module_dir.stat.exists + - module_dir.stat.isdir + + - name: Read elasticsearch.yml + ansible.builtin.slurp: + src: /etc/elasticsearch/elasticsearch.yml + register: es_config_content + + - name: Assert cluster name and rack_id in config + ansible.builtin.assert: + that: + - "'test-cluster' in (es_config_content.content | b64decode)" + - "'test-rack' in (es_config_content.content | b64decode)" diff --git a/molecule/default/converge.yml b/molecule/default/converge.yml new file mode 100644 index 00000000..75ede618 --- /dev/null +++ b/molecule/default/converge.yml @@ -0,0 +1,17 @@ +--- +- name: Converge + hosts: all + become: true + roles: + - role: ansible-elasticsearch + vars: + es_version: "8.19.10" + es_version_lock: true + es_bootstrap: true + es_config: + discovery.type: single-node + xpack.security.enabled: false + xpack.security.http.ssl.enabled: false + xpack.security.transport.ssl.enabled: false + network.host: "0.0.0.0" + http.port: 9200 diff --git a/molecule/default/molecule.yml b/molecule/default/molecule.yml new file mode 100644 index 00000000..c91796d2 --- /dev/null +++ b/molecule/default/molecule.yml @@ -0,0 +1,30 @@ +--- +driver: + name: docker + +platforms: + - name: elasticsearch + image: geerlingguy/docker-ubuntu2204-ansible:latest + pre_build_image: true + command: "" + privileged: true + cgroupns_mode: host + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + ulimits: + - "nofile:65536:65536" + - "memlock:-1:-1" + +provisioner: + name: ansible + env: + ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY}/.." + ANSIBLE_FORCE_COLOR: "true" + PY_COLORS: "1" + config_options: + defaults: + stdout_callback: default + result_format: yaml + +verifier: + name: ansible diff --git a/molecule/default/prepare.yml b/molecule/default/prepare.yml new file mode 100644 index 00000000..f74eaad3 --- /dev/null +++ b/molecule/default/prepare.yml @@ -0,0 +1,14 @@ +--- +- name: Prepare + hosts: all + become: true + tasks: + - name: Update apt cache + ansible.builtin.apt: + update_cache: true + cache_valid_time: 3600 + + - name: Install gnupg for apt key management + ansible.builtin.apt: + name: gnupg + state: present diff --git a/molecule/default/verify.yml b/molecule/default/verify.yml new file mode 100644 index 00000000..91229312 --- /dev/null +++ b/molecule/default/verify.yml @@ -0,0 +1,37 @@ +--- +- name: Verify + hosts: all + become: true + tasks: + - name: Gather service facts + ansible.builtin.service_facts: + + - name: Assert Elasticsearch is running + ansible.builtin.assert: + that: + - "'elasticsearch.service' in ansible_facts.services" + - "ansible_facts.services['elasticsearch.service'].state == 'running'" + + - name: Wait for Elasticsearch API + ansible.builtin.uri: + url: "http://localhost:9200" + status_code: 200 + register: api_response + until: api_response.status == 200 + retries: 30 + delay: 10 + + - name: Assert correct version + ansible.builtin.assert: + that: + - "api_response.json.version.number == '8.19.10'" + fail_msg: "Expected 8.19.10, got {{ api_response.json.version.number }}" + + - name: Assert elasticsearch config file exists + ansible.builtin.stat: + path: /etc/elasticsearch/elasticsearch.yml + register: config_stat + + - name: Verify config file present + ansible.builtin.assert: + that: config_stat.stat.exists diff --git a/molecule/security/converge.yml b/molecule/security/converge.yml new file mode 100644 index 00000000..c940259f --- /dev/null +++ b/molecule/security/converge.yml @@ -0,0 +1,24 @@ +--- +- name: Converge + hosts: all + become: true + roles: + - role: ansible-elasticsearch + vars: + es_version: "8.19.10" + es_version_lock: true + es_api_basic_auth_username: elastic + es_api_basic_auth_password: "changeme" + es_enable_http_ssl: true + es_enable_transport_ssl: true + es_ssl_key: /etc/ssl/private/test.key + es_ssl_certificate: /etc/ssl/certs/test.crt + es_ssl_key_password: false + es_ssl_remote_src: true + es_validate_certs: true + es_bootstrap: true + es_config: + discovery.type: single-node + network.host: "0.0.0.0" + http.port: 9200 + transport.port: 9300 diff --git a/molecule/security/molecule.yml b/molecule/security/molecule.yml new file mode 100644 index 00000000..945f6f4f --- /dev/null +++ b/molecule/security/molecule.yml @@ -0,0 +1,33 @@ +--- +driver: + name: docker + +platforms: + - name: elasticsearch + image: geerlingguy/docker-ubuntu2204-ansible:latest + pre_build_image: true + command: "" + privileged: true + cgroupns_mode: host + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + ulimits: + - "nofile:65536:65536" + - "memlock:-1:-1" + +provisioner: + name: ansible + env: + ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY}/.." + ANSIBLE_FORCE_COLOR: "true" + PY_COLORS: "1" + config_options: + defaults: + stdout_callback: default + result_format: yaml + + playbooks: + prepare: prepare.yml + +verifier: + name: ansible diff --git a/molecule/security/prepare.yml b/molecule/security/prepare.yml new file mode 100644 index 00000000..d6dbffb4 --- /dev/null +++ b/molecule/security/prepare.yml @@ -0,0 +1,49 @@ +--- +- name: Prepare SSL certificates + hosts: all + become: true + tasks: + - name: Update apt cache + ansible.builtin.apt: + update_cache: true + cache_valid_time: 3600 + + - name: Install gnupg for apt key management + ansible.builtin.apt: + name: gnupg + state: present + + - name: Ensure openssl is installed + ansible.builtin.package: + name: openssl + state: present + + - name: Ensure SSL directories exist with least privilege + ansible.builtin.file: + path: "{{ item.path }}" + state: directory + mode: "{{ item.mode }}" + loop: + - { path: "/etc/ssl/private", mode: "0750" } + - { path: "/etc/ssl/certs", mode: "0755" } + + - name: Generate self-signed certificate and key + ansible.builtin.command: + cmd: > + openssl req -x509 -nodes -newkey rsa:2048 + -keyout /etc/ssl/private/test.key + -out /etc/ssl/certs/test.crt + -days 365 + -subj "/CN=localhost" + -addext "subjectAltName=IP:127.0.0.1,DNS:localhost" + creates: /etc/ssl/certs/test.crt + + - name: Set private key permissions + ansible.builtin.file: + path: /etc/ssl/private/test.key + mode: "0640" + + - name: Set certificate permissions + ansible.builtin.file: + path: /etc/ssl/certs/test.crt + mode: "0644" diff --git a/molecule/security/verify.yml b/molecule/security/verify.yml new file mode 100644 index 00000000..adafb1b3 --- /dev/null +++ b/molecule/security/verify.yml @@ -0,0 +1,40 @@ +--- +- name: Verify + hosts: all + become: true + tasks: + - name: Gather service facts + ansible.builtin.service_facts: + + - name: Assert Elasticsearch is running + ansible.builtin.assert: + that: + - "'elasticsearch.service' in ansible_facts.services" + - "ansible_facts.services['elasticsearch.service'].state == 'running'" + + - name: Wait for Elasticsearch HTTPS API + ansible.builtin.uri: + url: "https://localhost:9200" + url_username: elastic + url_password: "changeme" + validate_certs: false + status_code: 200 + register: api_response + until: api_response.status == 200 + retries: 30 + delay: 10 + + - name: Assert correct version + ansible.builtin.assert: + that: + - "api_response.json.version.number == '8.19.10'" + fail_msg: "Expected 8.19.10, got {{ api_response.json.version.number }}" + + - name: Assert TLS certificate file exists + ansible.builtin.stat: + path: /etc/ssl/certs/test.crt + register: cert_stat + + - name: Verify SSL certificate in place + ansible.builtin.assert: + that: cert_stat.stat.exists diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..0f1220ae --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +molecule>=6.0 +molecule-plugins[docker]>=23.0 +ansible>=9.0 +ansible-lint>=6.0 diff --git a/requirements.yml b/requirements.yml new file mode 100644 index 00000000..86c7754a --- /dev/null +++ b/requirements.yml @@ -0,0 +1,4 @@ +collections: + - name: https://github.com/ansible-collections/community.elastic.git + type: git + version: 1.1.3 diff --git a/tasks/compatibility-variables.yml b/tasks/compatibility-variables.yml index fc9af088..4f34711f 100644 --- a/tasks/compatibility-variables.yml +++ b/tasks/compatibility-variables.yml @@ -9,7 +9,8 @@ when: es_enable_xpack is defined and not es_enable_xpack block: - name: Set fact oss_version when using es_enable_xpack - set_fact: oss_version=true + ansible.builtin.set_fact: + oss_version: true - name: Warn about deprecated es_enable_xpack variable debug: msg: "WARNING: es_enable_xpack variable is now deprecated. You should use oss_version instead" diff --git a/tasks/elasticsearch-Debian.yml b/tasks/elasticsearch-Debian.yml index 4590da14..cb2227a5 100644 --- a/tasks/elasticsearch-Debian.yml +++ b/tasks/elasticsearch-Debian.yml @@ -51,7 +51,7 @@ url: '{{ es_apt_key }}' id: '{{ es_apt_key_id }}' state: present - when: es_add_repository and es_apt_key | string + when: es_add_repository and es_apt_key | length > 0 - name: Debian - Add elasticsearch repository apt_repository: @@ -79,7 +79,7 @@ dpkg_selections: name: "{{ es_package_name }}" selection: "install" - when: not es_version_lock or (installed_es_version.stdout and installed_es_version.stdout != es_version) + when: not es_version_lock or (installed_es_version.stdout | length > 0 and installed_es_version.stdout != es_version) - name: Debian - Ensure elasticsearch is installed become: yes diff --git a/tasks/elasticsearch-plugins.yml b/tasks/elasticsearch-plugins.yml index d96ac96e..2c7972ce 100644 --- a/tasks/elasticsearch-plugins.yml +++ b/tasks/elasticsearch-plugins.yml @@ -3,14 +3,17 @@ # es_plugins_reinstall will be set to true if elasticsearch_install_from_repo.changed or elasticsearch_install_from_package.changed # i.e. we have changed ES version(or we have clean installation of ES), or if no plugins listed. Otherwise it is false and requires explicitly setting. - name: set fact es_plugins_reinstall to true - set_fact: es_plugins_reinstall=true + ansible.builtin.set_fact: + es_plugins_reinstall: true when: (((debian_elasticsearch_install_from_repo is defined and debian_elasticsearch_install_from_repo.changed) or (redhat_elasticsearch_install_from_repo is defined and redhat_elasticsearch_install_from_repo.changed)) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed)) or es_plugins is not defined or es_plugins is none - name: set fact list_command - set_fact: list_command="" + ansible.builtin.set_fact: + list_command: "" #If we are reinstalling all plugins, e.g. to a version change, we need to remove all plugins (inc. x-pack) to install any plugins. Otherwise we don't consider x-pack so the role stays idempotent. - name: set fact list_command check for x-pack - set_fact: list_command="| grep -vE 'x-pack'" + ansible.builtin.set_fact: + list_command: "| grep -vE 'x-pack'" when: not es_plugins_reinstall - name: remove x-pack plugin directory when it isn't a plugin @@ -38,17 +41,17 @@ #if the plugins listed are different than those requested, we remove those installed but not listed in the config - name: set fact plugins_to_remove to delete plugins installed but not listed in es_plugins - set_fact: plugins_to_remove="{{ installed_plugins.stdout_lines | difference(es_plugins | json_query('[*].plugin')) | default([]) }}" + set_fact: plugins_to_remove="{{ installed_plugins.stdout_lines | difference(es_plugins | map(attribute='plugin') | list) | default([]) }}" when: not es_plugins_reinstall #if es_plugins_reinstall is set to true we (re)install ALL plugins - name: set fact plugins_to_install to es_plugins - set_fact: plugins_to_install="{{ es_plugins | json_query('[*].plugin') | default([]) }}" + set_fact: plugins_to_install="{{ es_plugins | map(attribute='plugin') | list | default([]) }}" when: es_plugins_reinstall #if the plugins listed are different than those requested, we install those not installed but listed in the config - name: set fact to plugins_to_install to those in es_config but not installed - set_fact: plugins_to_install="{{ es_plugins | json_query('[*].plugin') | difference(installed_plugins.stdout_lines) | default([]) }}" + set_fact: plugins_to_install="{{ es_plugins | map(attribute='plugin') | list | difference(installed_plugins.stdout_lines) | default([]) }}" when: not es_plugins_reinstall # This removes any currently installed plugins (to prevent errors when reinstalling) @@ -65,7 +68,7 @@ - name: Install elasticsearch plugins become: yes - command: "{{es_home}}/bin/elasticsearch-plugin install {{ item.url | default(item.plugin) }} --batch --silent" + command: "{{es_home}}/bin/elasticsearch-plugin install {{ item.url | default(item.plugin) }} --batch" register: plugin_installed changed_when: plugin_installed.rc|default(0) == 0 with_items: "{{ es_plugins }}" diff --git a/tasks/elasticsearch-ssl.yml b/tasks/elasticsearch-ssl.yml index cb2d8019..303ab1bf 100644 --- a/tasks/elasticsearch-ssl.yml +++ b/tasks/elasticsearch-ssl.yml @@ -1,10 +1,12 @@ --- - name: set fact es_same_keystore - set_fact: es_same_keystore=false + ansible.builtin.set_fact: + es_same_keystore: false - name: set fact es_same_keystore if stores match - set_fact: es_same_keystore=true + ansible.builtin.set_fact: + es_same_keystore: true when: es_ssl_keystore == es_ssl_truststore - name: Ensure certificate directory exists @@ -15,7 +17,7 @@ owner: root group: "{{ es_group }}" mode: "750" - when: es_ssl_upload + when: es_ssl_upload | bool - name: Upload SSL/TLS keystore become: yes @@ -26,7 +28,7 @@ group: "{{ es_group }}" mode: "640" remote_src: "{{ es_ssl_remote_src }}" - when: es_ssl_upload and es_ssl_keystore and es_ssl_truststore + when: es_ssl_upload | bool and es_ssl_keystore | length > 0 and es_ssl_truststore | length > 0 notify: restart elasticsearch register: copy_keystore @@ -39,7 +41,7 @@ group: "{{ es_group }}" mode: "640" remote_src: "{{ es_ssl_remote_src }}" - when: es_ssl_upload and es_ssl_keystore and es_ssl_truststore + when: es_ssl_upload | bool and es_ssl_keystore | length > 0 and es_ssl_truststore | length > 0 notify: restart elasticsearch register: copy_truststore @@ -55,7 +57,7 @@ with_items: - "{{ es_ssl_key }}" - "{{ es_ssl_certificate }}" - when: es_ssl_upload and es_ssl_key and es_ssl_certificate + when: es_ssl_upload | bool and es_ssl_key | length > 0 and es_ssl_certificate | length > 0 #Restart if these change notify: restart elasticsearch register: copy_certificates @@ -71,13 +73,13 @@ remote_src: "{{ es_ssl_remote_src }}" #Restart if this changes notify: restart elasticsearch - when: es_ssl_upload and (es_ssl_certificate_authority is defined) and (es_ssl_certificate_authority|length > 0) + when: es_ssl_upload | bool and (es_ssl_certificate_authority is defined) and (es_ssl_certificate_authority | length > 0) - name: Set keystore password become: yes shell: echo "{{ es_ssl_keystore_password }}" | {{ es_home }}/bin/elasticsearch-keystore add -x -f 'xpack.security.{{ item }}.ssl.keystore.secure_password' no_log: True - when: es_ssl_keystore_password and (copy_keystore.changed or (es_same_keystore and copy_truststore.changed)) + when: es_ssl_keystore_password | length > 0 and (copy_keystore.changed or (es_same_keystore and copy_truststore.changed)) with_items: - http - transport @@ -86,7 +88,7 @@ become: yes shell: echo "{{ es_ssl_truststore_password }}" | {{ es_home }}/bin/elasticsearch-keystore add -x -f 'xpack.security.{{ item }}.ssl.truststore.secure_password' no_log: True - when: es_ssl_truststore_password and (copy_truststore.changed or (es_same_keystore and copy_keystore.changed)) + when: es_ssl_truststore_password | length > 0 and (copy_truststore.changed or (es_same_keystore and copy_keystore.changed)) with_items: - http - transport @@ -94,7 +96,7 @@ - name: Remove keystore password become: yes shell: "{{ es_home }}/bin/elasticsearch-keystore remove 'xpack.security.{{ item }}.ssl.keystore.secure_password'" - when: es_ssl_keystore_password == "" and (copy_keystore.changed or (es_same_keystore and copy_truststore.changed)) + when: es_ssl_keystore_password | length == 0 and (copy_keystore.changed or (es_same_keystore and copy_truststore.changed)) ignore_errors: yes with_items: - http @@ -103,7 +105,7 @@ - name: Remove truststore password become: yes shell: "{{ es_home }}/bin/elasticsearch-keystore remove 'xpack.security.{{ item }}.ssl.truststore.secure_password'" - when: es_ssl_truststore_password == "" and (copy_truststore.changed or (es_same_keystore and copy_keystore.changed)) + when: es_ssl_truststore_password | length == 0 and (copy_truststore.changed or (es_same_keystore and copy_keystore.changed)) ignore_errors: yes with_items: - http @@ -113,7 +115,7 @@ become: yes shell: echo "{{ es_ssl_key_password }}" | {{ es_home }}/bin/elasticsearch-keystore add -x -f 'xpack.security.{{ item }}.ssl.secure_key_passphrase' no_log: True - when: es_ssl_key_password and copy_certificates.changed + when: es_ssl_key_password is string and es_ssl_key_password | length > 0 and copy_certificates.changed with_items: - http - transport @@ -121,7 +123,7 @@ - name: Remove key password become: yes shell: "{{ es_home }}/bin/elasticsearch-keystore remove 'xpack.security.{{ item }}.ssl.secure_key_passphrase'" - when: es_ssl_key_password == "" and copy_certificates.changed + when: (es_ssl_key_password is sameas false or (es_ssl_key_password is string and es_ssl_key_password | length == 0)) and copy_certificates.changed ignore_errors: yes with_items: - http diff --git a/tasks/main.yml b/tasks/main.yml index e2a6893a..198fd501 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -84,10 +84,12 @@ when: es_restarted is defined and es_restarted.changed and es_start_service - name: set fact manage_native_realm to false - ansible.builtin.set_fact: manage_native_realm=false + ansible.builtin.set_fact: + manage_native_realm: false - name: set fact manage_native_realm to true - ansible.builtin.set_fact: manage_native_realm=true + ansible.builtin.set_fact: + manage_native_realm: true when: - es_start_service - not oss_version diff --git a/tasks/xpack/security/elasticsearch-security-file.yml b/tasks/xpack/security/elasticsearch-security-file.yml index b169b9ee..3c04d969 100644 --- a/tasks/xpack/security/elasticsearch-security-file.yml +++ b/tasks/xpack/security/elasticsearch-security-file.yml @@ -1,7 +1,9 @@ --- -- set_fact: manage_file_users=false +- ansible.builtin.set_fact: + manage_file_users: false -- set_fact: manage_file_users=true +- ansible.builtin.set_fact: + manage_file_users: true when: es_users is defined and es_users.file is defined and es_users.file.keys() | list | length > 0 # Users migration from elasticsearch < 6.3 versions diff --git a/tasks/xpack/security/elasticsearch-security-native.yml b/tasks/xpack/security/elasticsearch-security-native.yml index f8502640..5957336f 100644 --- a/tasks/xpack/security/elasticsearch-security-native.yml +++ b/tasks/xpack/security/elasticsearch-security-native.yml @@ -1,19 +1,24 @@ --- - name: set fact change_api_password to false - set_fact: change_api_password=false + ansible.builtin.set_fact: + change_api_password: false - name: set fact manage_native_users to false - set_fact: manage_native_users=false + ansible.builtin.set_fact: + manage_native_users: false - name: set fact manage_native_users to true - set_fact: manage_native_users=true + ansible.builtin.set_fact: + manage_native_users: true when: es_users is defined and es_users.native is defined and es_users.native.keys() | list | length > 0 - name: set fact manage_native_role to false - set_fact: manage_native_roles=false + ansible.builtin.set_fact: + manage_native_roles: false - name: set fact manage_native_roles to true - set_fact: manage_native_roles=true + ansible.builtin.set_fact: + manage_native_roles: true when: es_roles is defined and es_roles.native is defined and es_roles.native.keys() | list | length > 0 #If the node has just has security installed it maybe either stopped or started 1. if stopped, we need to start to load native realms 2. if started, we need to restart to load @@ -47,7 +52,8 @@ when: manage_native_users - name: set fact change_api_password to true - set_fact: change_api_password=true + ansible.builtin.set_fact: + change_api_password: true when: manage_native_users and es_api_basic_auth_username in native_users and native_users[es_api_basic_auth_username].password is defined - name: Update API User Password diff --git a/test/integration/custom-config.yml b/test/integration/custom-config.yml deleted file mode 100644 index e6aad179..00000000 --- a/test/integration/custom-config.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -- name: Setup Elasticsearch with custom config files - hosts: localhost - post_tasks: - - ansible.builtin.include_tasks: elasticsearch/test/integration/debug.yml - roles: - - elasticsearch - vars: - es_config_default: "test/integration/files/custom_config/elasticsearch" - es_config_log4j2: "test/integration/files/custom_config/log4j2.properties" - es_config_jvm: "test/integration/files/custom_config/jvm.options" diff --git a/test/integration/custom-config/custom_config.yml b/test/integration/custom-config/custom_config.yml deleted file mode 100644 index a3c37e19..00000000 --- a/test/integration/custom-config/custom_config.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -- host: test-kitchen diff --git a/test/integration/custom-config/serverspec/default_spec.rb b/test/integration/custom-config/serverspec/default_spec.rb deleted file mode 100644 index edcd5935..00000000 --- a/test/integration/custom-config/serverspec/default_spec.rb +++ /dev/null @@ -1,9 +0,0 @@ -require 'custom_config_spec' -require 'shared_spec' -require 'json' -vars = JSON.parse(File.read('/tmp/vars.json')) - -describe 'Custom Config Tests' do - include_examples 'custom_config::init', vars - include_examples 'shared::init', vars -end diff --git a/test/integration/debug.yml b/test/integration/debug.yml deleted file mode 100644 index 81b08b11..00000000 --- a/test/integration/debug.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: Dump all variables to a file - changed_when: False - copy: - content: '{{ vars | to_nice_json }} ' - dest: '/tmp/vars.json' diff --git a/test/integration/default.yml b/test/integration/default.yml deleted file mode 100644 index 93ab2e3b..00000000 --- a/test/integration/default.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- name: Setup Elasticsearch using default configuration - hosts: localhost - post_tasks: - - ansible.builtin.include_tasks: elasticsearch/test/integration/debug.yml - roles: - - elasticsearch diff --git a/test/integration/default/default.yml b/test/integration/default/default.yml deleted file mode 100644 index a3c37e19..00000000 --- a/test/integration/default/default.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -- host: test-kitchen diff --git a/test/integration/default/serverspec/default_spec.rb b/test/integration/default/serverspec/default_spec.rb deleted file mode 100644 index 248a3397..00000000 --- a/test/integration/default/serverspec/default_spec.rb +++ /dev/null @@ -1,7 +0,0 @@ -require 'shared_spec' -require 'json' -vars = JSON.parse(File.read('/tmp/vars.json')) - -describe 'default tests' do - include_examples 'shared::init', vars -end diff --git a/test/integration/files/certs/keystore-password-ca.p12 b/test/integration/files/certs/keystore-password-ca.p12 deleted file mode 100644 index e8331cf9..00000000 Binary files a/test/integration/files/certs/keystore-password-ca.p12 and /dev/null differ diff --git a/test/integration/files/certs/keystore-password.p12 b/test/integration/files/certs/keystore-password.p12 deleted file mode 100644 index 6a58eaac..00000000 Binary files a/test/integration/files/certs/keystore-password.p12 and /dev/null differ diff --git a/test/integration/files/certs/shared-store-no-password-ca.p12 b/test/integration/files/certs/shared-store-no-password-ca.p12 deleted file mode 100644 index 9b76934b..00000000 Binary files a/test/integration/files/certs/shared-store-no-password-ca.p12 and /dev/null differ diff --git a/test/integration/files/certs/shared-store-no-password.p12 b/test/integration/files/certs/shared-store-no-password.p12 deleted file mode 100644 index c675ae21..00000000 Binary files a/test/integration/files/certs/shared-store-no-password.p12 and /dev/null differ diff --git a/test/integration/files/certs/truststore-password-ca.p12 b/test/integration/files/certs/truststore-password-ca.p12 deleted file mode 100644 index a1662a83..00000000 Binary files a/test/integration/files/certs/truststore-password-ca.p12 and /dev/null differ diff --git a/test/integration/files/certs/truststore-password.p12 b/test/integration/files/certs/truststore-password.p12 deleted file mode 100644 index 25d9aa4c..00000000 Binary files a/test/integration/files/certs/truststore-password.p12 and /dev/null differ diff --git a/test/integration/files/custom_config/elasticsearch b/test/integration/files/custom_config/elasticsearch deleted file mode 100644 index 36540c8a..00000000 --- a/test/integration/files/custom_config/elasticsearch +++ /dev/null @@ -1,7 +0,0 @@ -# Elasticsearch CUSTOM FILE - -ES_HOME=/usr/share/elasticsearch -ES_PATH_CONF=/etc/elasticsearch -PID_DIR=/var/run/elasticsearch -ES_JAVA_OPTS= -ES_STARTUP_SLEEP_TIME=5 diff --git a/test/integration/files/custom_config/jvm.options b/test/integration/files/custom_config/jvm.options deleted file mode 100644 index 2c6049ee..00000000 --- a/test/integration/files/custom_config/jvm.options +++ /dev/null @@ -1,31 +0,0 @@ -## JVM configuration CUSTOM FILE - --Xms2g --Xmx2g --Des.networkaddress.cache.ttl=60 --Des.networkaddress.cache.negative.ttl=10 --XX:+AlwaysPreTouch --Xss1m --Djava.awt.headless=true --Dfile.encoding=UTF-8 --Djna.nosys=true --XX:-OmitStackTraceInFastThrow --Dio.netty.noUnsafe=true --Dio.netty.noKeySetOptimization=true --Dio.netty.recycler.maxCapacityPerThread=0 --Dlog4j.shutdownHookEnabled=false --Dlog4j2.disable.jmx=true --Djava.io.tmpdir=${ES_TMPDIR} --XX:+HeapDumpOnOutOfMemoryError --XX:HeapDumpPath=${heap.dump.path} --XX:ErrorFile=/var/log/elasticsearch/hs_err_pid%p.log -8:-XX:+PrintGCDetails -8:-XX:+PrintGCDateStamps -8:-XX:+PrintTenuringDistribution -8:-XX:+PrintGCApplicationStoppedTime -8:-Xloggc:/var/log/elasticsearch/gc.log -8:-XX:+UseGCLogFileRotation -8:-XX:NumberOfGCLogFiles=32 -8:-XX:GCLogFileSize=64m -9-:-Xlog:gc*,gc+age=trace,safepoint:file=/var/log/elasticsearch/gc.log:utctime,pid,tags:filecount=32,filesize=64m -9-:-Djava.locale.providers=COMPAT diff --git a/test/integration/files/custom_config/log4j2.properties b/test/integration/files/custom_config/log4j2.properties deleted file mode 100644 index 1573120e..00000000 --- a/test/integration/files/custom_config/log4j2.properties +++ /dev/null @@ -1,86 +0,0 @@ -{% raw %} -# Log4j CUSTOM FILE - -status = error -logger.action.name = org.elasticsearch.action -logger.action.level = debug -appender.rolling.type = Console -appender.rolling.name = rolling -appender.rolling.layout.type = ESJsonLayout -appender.rolling.layout.type_name = server -rootLogger.level = info -rootLogger.appenderRef.rolling.ref = rolling -appender.deprecation_rolling.type = Console -appender.deprecation_rolling.name = deprecation_rolling -appender.deprecation_rolling.layout.type = ESJsonLayout -appender.deprecation_rolling.layout.type_name = deprecation -appender.deprecation_rolling.layout.esmessagefields=x-opaque-id -logger.deprecation.name = org.elasticsearch.deprecation -logger.deprecation.level = warn -logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling -logger.deprecation.additivity = false -appender.index_search_slowlog_rolling.type = Console -appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling -appender.index_search_slowlog_rolling.layout.type = ESJsonLayout -appender.index_search_slowlog_rolling.layout.type_name = index_search_slowlog -appender.index_search_slowlog_rolling.layout.esmessagefields=message,took,took_millis,total_hits,stats,search_type,total_shards,source,id -logger.index_search_slowlog_rolling.name = index.search.slowlog -logger.index_search_slowlog_rolling.level = trace -logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling -logger.index_search_slowlog_rolling.additivity = false -appender.index_indexing_slowlog_rolling.type = Console -appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling -appender.index_indexing_slowlog_rolling.layout.type = ESJsonLayout -appender.index_indexing_slowlog_rolling.layout.type_name = index_indexing_slowlog -appender.index_indexing_slowlog_rolling.layout.esmessagefields=message,took,took_millis,doc_type,id,routing,source -logger.index_indexing_slowlog.name = index.indexing.slowlog.index -logger.index_indexing_slowlog.level = trace -logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling -logger.index_indexing_slowlog.additivity = false -appender.audit_rolling.type = Console -appender.audit_rolling.name = audit_rolling -appender.audit_rolling.layout.type = PatternLayout -appender.audit_rolling.layout.pattern = {\ -"type": "audit", \ -"timestamp":"%d{yyyy-MM-dd'T'HH:mm:ss,SSSZ}"\ -%varsNotEmpty{, "node.name":"%enc{%map{node.name}}{JSON}"}\ -%varsNotEmpty{, "node.id":"%enc{%map{node.id}}{JSON}"}\ -%varsNotEmpty{, "host.name":"%enc{%map{host.name}}{JSON}"}\ -%varsNotEmpty{, "host.ip":"%enc{%map{host.ip}}{JSON}"}\ -%varsNotEmpty{, "event.type":"%enc{%map{event.type}}{JSON}"}\ -%varsNotEmpty{, "event.action":"%enc{%map{event.action}}{JSON}"}\ -%varsNotEmpty{, "user.name":"%enc{%map{user.name}}{JSON}"}\ -%varsNotEmpty{, "user.run_by.name":"%enc{%map{user.run_by.name}}{JSON}"}\ -%varsNotEmpty{, "user.run_as.name":"%enc{%map{user.run_as.name}}{JSON}"}\ -%varsNotEmpty{, "user.realm":"%enc{%map{user.realm}}{JSON}"}\ -%varsNotEmpty{, "user.run_by.realm":"%enc{%map{user.run_by.realm}}{JSON}"}\ -%varsNotEmpty{, "user.run_as.realm":"%enc{%map{user.run_as.realm}}{JSON}"}\ -%varsNotEmpty{, "user.roles":%map{user.roles}}\ -%varsNotEmpty{, "origin.type":"%enc{%map{origin.type}}{JSON}"}\ -%varsNotEmpty{, "origin.address":"%enc{%map{origin.address}}{JSON}"}\ -%varsNotEmpty{, "realm":"%enc{%map{realm}}{JSON}"}\ -%varsNotEmpty{, "url.path":"%enc{%map{url.path}}{JSON}"}\ -%varsNotEmpty{, "url.query":"%enc{%map{url.query}}{JSON}"}\ -%varsNotEmpty{, "request.method":"%enc{%map{request.method}}{JSON}"}\ -%varsNotEmpty{, "request.body":"%enc{%map{request.body}}{JSON}"}\ -%varsNotEmpty{, "request.id":"%enc{%map{request.id}}{JSON}"}\ -%varsNotEmpty{, "action":"%enc{%map{action}}{JSON}"}\ -%varsNotEmpty{, "request.name":"%enc{%map{request.name}}{JSON}"}\ -%varsNotEmpty{, "indices":%map{indices}}\ -%varsNotEmpty{, "opaque_id":"%enc{%map{opaque_id}}{JSON}"}\ -%varsNotEmpty{, "x_forwarded_for":"%enc{%map{x_forwarded_for}}{JSON}"}\ -%varsNotEmpty{, "transport.profile":"%enc{%map{transport.profile}}{JSON}"}\ -%varsNotEmpty{, "rule":"%enc{%map{rule}}{JSON}"}\ -%varsNotEmpty{, "event.category":"%enc{%map{event.category}}{JSON}"}\ -}%n -logger.xpack_security_audit_logfile.name = org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail -logger.xpack_security_audit_logfile.level = info -logger.xpack_security_audit_logfile.appenderRef.audit_rolling.ref = audit_rolling -logger.xpack_security_audit_logfile.additivity = false -logger.xmlsig.name = org.apache.xml.security.signature.XMLSignature -logger.xmlsig.level = error -logger.samlxml_decrypt.name = org.opensaml.xmlsec.encryption.support.Decrypter -logger.samlxml_decrypt.level = fatal -logger.saml2_decrypt.name = org.opensaml.saml.saml2.encryption.Decrypter -logger.saml2_decrypt.level = fatal -{% endraw %} diff --git a/test/integration/files/templates-6.x/basic.json b/test/integration/files/templates-6.x/basic.json deleted file mode 100644 index 3a3871d9..00000000 --- a/test/integration/files/templates-6.x/basic.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "template" : "te*", - "settings" : { - "number_of_shards" : 1 - }, - "mappings" : { - "type1" : { - "_source" : { "enabled" : false } - } - } -} \ No newline at end of file diff --git a/test/integration/files/templates-7.x/basic.json b/test/integration/files/templates-7.x/basic.json deleted file mode 100644 index 31796dae..00000000 --- a/test/integration/files/templates-7.x/basic.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "index_patterns" : "te*", - "settings" : { - "number_of_shards" : 1 - }, - "mappings" : { - "_source" : { "enabled" : false } - } -} \ No newline at end of file diff --git a/test/integration/helpers/serverspec/Gemfile b/test/integration/helpers/serverspec/Gemfile deleted file mode 100644 index e6aafc0b..00000000 --- a/test/integration/helpers/serverspec/Gemfile +++ /dev/null @@ -1,3 +0,0 @@ -source 'https://rubygems.org' - -gem 'rspec-retry' diff --git a/test/integration/helpers/serverspec/custom_config_spec.rb b/test/integration/helpers/serverspec/custom_config_spec.rb deleted file mode 100644 index 96255103..00000000 --- a/test/integration/helpers/serverspec/custom_config_spec.rb +++ /dev/null @@ -1,20 +0,0 @@ -require 'spec_helper' -require 'shared_spec' - -shared_examples 'custom_config::init' do |vars| - describe file("/etc/elasticsearch/log4j2.properties") do - it { should be_file } - it { should be_owned_by 'root' } - it { should contain 'Log4j CUSTOM FILE' } - end - describe file("/etc/elasticsearch/jvm.options") do - it { should be_file } - it { should be_owned_by 'root' } - it { should contain 'JVM configuration CUSTOM FILE' } - end - describe file($family['defaults_path']) do - it { should be_file } - it { should be_owned_by 'root' } - it { should contain 'Elasticsearch CUSTOM FILE' } - end -end diff --git a/test/integration/helpers/serverspec/license_spec.rb b/test/integration/helpers/serverspec/license_spec.rb deleted file mode 100644 index 4045a63d..00000000 --- a/test/integration/helpers/serverspec/license_spec.rb +++ /dev/null @@ -1,27 +0,0 @@ -require 'spec_helper' -require 'json' -require 'pathname' -vars = JSON.parse(File.read('/tmp/vars.json')) - -es_api_url = "#{vars['es_api_scheme']}://localhost:#{vars['es_api_port']}" -username = vars['es_api_basic_auth_username'] -password = vars['es_api_basic_auth_password'] -es_keystore = Pathname.new(vars['es_ssl_keystore']).basename.to_s -es_truststore = Pathname.new(vars['es_ssl_truststore']).basename.to_s - -if vars['es_major_version'] == '7.x' - es_license_api = "_license" -else - es_license_api = "_xpack/license" -end - -shared_examples 'license::init' do |vars| - describe 'License check' do - result = curl_json("#{es_api_url}/#{es_license_api}", username=username, password=password) - it 'should list the license issued by Elastic' do - expect(result['license']['status']).to eq('active') - expect(result['license']['type']).to eq('trial') - expect(result['license']['issued_to']).to eq('Elastic - INTERNAL (non-production environments)') - end - end -end diff --git a/test/integration/helpers/serverspec/security_spec.rb b/test/integration/helpers/serverspec/security_spec.rb deleted file mode 100644 index 94108984..00000000 --- a/test/integration/helpers/serverspec/security_spec.rb +++ /dev/null @@ -1,88 +0,0 @@ -require 'spec_helper' -require 'json' -require 'pathname' -vars = JSON.parse(File.read('/tmp/vars.json')) - -es_api_url = "#{vars['es_api_scheme']}://localhost:#{vars['es_api_port']}" -username = vars['es_api_basic_auth_username'] -password = vars['es_api_basic_auth_password'] -es_keystore = Pathname.new(vars['es_ssl_keystore']).basename.to_s -es_truststore = Pathname.new(vars['es_ssl_truststore']).basename.to_s - -if vars['es_major_version'] == '7.x' - es_security_api = "_security" -else - es_security_api = "_xpack/security" -end - -shared_examples 'security::init' do |vars| - #Test users file, users_roles and roles.yml - describe file("/etc/elasticsearch/users_roles") do - it { should be_owned_by 'root' } - it { should contain 'admin:es_admin' } - it { should contain 'power_user:testUser' } - end - - describe file("/etc/elasticsearch/users") do - it { should be_owned_by 'root' } - it { should contain 'testUser:' } - it { should contain 'es_admin:' } - end - - describe 'security roles' do - it 'should list the security roles' do - roles = curl_json("#{es_api_url}/#{es_security_api}/role", username='es_admin', password='changeMeAgain') - expect(roles.key?('superuser')) - end - end - - describe file("/etc/elasticsearch/elasticsearch.yml") do - if vars['es_major_version'] == '7.x' - it { should contain 'security.authc.realms.file.file1.order: 0' } - else - it { should contain 'security.authc.realms.file1.order: 0' } - it { should contain 'security.authc.realms.file1.type: file' } - end - it { should contain 'xpack.security.transport.ssl.enabled: true' } - it { should contain 'xpack.security.http.ssl.enabled: true' } - it { should contain es_keystore } - it { should contain es_truststore } - end - - #Test contents of role_mapping.yml - describe file("/etc/elasticsearch/role_mapping.yml") do - it { should be_owned_by 'root' } - it { should contain 'power_user:' } - it { should contain 'user:' } - end - - #check accounts are correct i.e. we can auth and they have the correct roles - describe 'security users' do - result = curl_json("#{es_api_url}/#{es_security_api}/user", username=username, password=password) - it 'should have the elastic user' do - expect(result['elastic']['username']).to eq('elastic') - expect(result['elastic']['roles']).to eq(['superuser']) - expect(result['elastic']['enabled']).to eq(true) - end - it 'should have the kibana user' do - expect(result['kibana']['username']).to eq('kibana') - expect(result['kibana']['roles']).to eq(['kibana_system']) - expect(result['kibana']['enabled']).to eq(true) - end - it 'should have the logstash user' do - expect(result['logstash_system']['username']).to eq('logstash_system') - expect(result['logstash_system']['roles']).to eq(['logstash_system']) - expect(result['logstash_system']['enabled']).to eq(true) - end - end - - describe 'SSL certificate check' do - certificates = curl_json("#{es_api_url}/_ssl/certificates", username=username, password=password) - it 'should list the keystore file' do - expect(certificates.any? { |cert| cert['path'].include? es_keystore }).to be true - end - it 'should list the truststore file' do - expect(certificates.any? { |cert| cert['path'].include? es_truststore }).to be true - end - end -end diff --git a/test/integration/helpers/serverspec/shared_spec.rb b/test/integration/helpers/serverspec/shared_spec.rb deleted file mode 100644 index 598d7e21..00000000 --- a/test/integration/helpers/serverspec/shared_spec.rb +++ /dev/null @@ -1,157 +0,0 @@ -require 'spec_helper' -require 'json' -vars = JSON.parse(File.read('/tmp/vars.json')) - -$families = { - 'Debian' => { - 'shell' => '/bin/false', - 'password' => '*', - 'defaults_path' => '/etc/default/elasticsearch' - }, - 'RedHat' => { - 'shell' => '/sbin/nologin', - 'password' => '!!', - 'defaults_path' => '/etc/sysconfig/elasticsearch' - } -} - -$family = $families[vars['ansible_os_family']] - -es_api_url = "#{vars['es_api_scheme']}://localhost:#{vars['es_api_port']}" -username = vars['es_api_basic_auth_username'] -password = vars['es_api_basic_auth_password'] - -# Sample of default features status -features = { - 'monitoring' => { - 'enabled' => 'true', - 'available' => 'true' - }, - 'ml' => { - 'enabled' => 'true', - 'available' => 'false' - }, - 'sql' => { - 'enabled' => 'true', - 'available' => 'true' - } -} - -shared_examples 'shared::init' do |vars| - describe 'version check' do - it 'should be reported as version '+vars['es_version'] do - expect(curl_json(es_api_url, username=username, password=password)['version']['number']).to eq(vars['es_version']) - end - end - describe 'xpack checks' do - if not vars['oss_version'] - it 'should be be running the basic version' do - expect(curl_json("#{es_api_url}/_xpack", username=username, password=password)['tagline']).to eq('You know, for X') - end - it 'xpack should be activated' do - expect(curl_json("#{es_api_url}/_license", username=username, password=password)['license']['status']).to eq('active') - end - end - end - describe user(vars['es_user']) do - it { should exist } - it { should belong_to_group vars['es_group'] } - it { should have_uid vars['es_user_id'] } if vars.key?('es_user_id') - - it { should have_login_shell $family['shell'] } - - its(:encrypted_password) { should eq($family['password']) } - end - - describe package(vars['es_package_name']) do - it { should be_installed } - end - - describe service("elasticsearch") do - it { should be_running } - end - - describe port(vars['es_api_port']) do - it { should be_listening.with('tcp') } - end - - if vars['es_templates'] - describe file('/etc/elasticsearch/templates') do - it { should be_directory } - it { should be_owned_by 'root' } - end - describe file('/etc/elasticsearch/templates/basic.json') do - it { should be_file } - it { should be_owned_by 'root' } - end - #This is possibly subject to format changes in the response across versions so may fail in the future - describe 'Template Contents Correct' do - it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do - template = curl_json("#{es_api_url}/_template/basic", username=username, password=password) - expect(template.key?('basic')) - expect(template['basic']['settings']['index']['number_of_shards']).to eq("1") - if vars['es_major_version'] == '7.x' - expect(template['basic']['mappings']['_source']['enabled']).to eq(false) - else - expect(template['basic']['mappings']['type1']['_source']['enabled']).to eq(false) - end - end - end - end - - describe file($family['defaults_path']) do - its(:content) { should match '' } - end - - if vars.key?('es_plugins') - vars['es_plugins'].each do |plugin| - name = plugin['plugin'] - describe file('/usr/share/elasticsearch/plugins/'+name) do - it { should be_directory } - it { should be_owned_by 'root' } - end - it 'should be installed and the right version' do - plugins = curl_json("#{es_api_url}/_nodes/plugins", username=username, password=password) - version = nil - _node, data = plugins['nodes'].first - data['plugins'].each do |p| - version = p['version'] if p['name'] == name - end - expect(version).to eql(vars['es_version']) - end - end - end - describe file("/etc/elasticsearch/elasticsearch.yml") do - it { should be_owned_by 'root' } - it { should contain "node.name: localhost" } - it { should contain 'cluster.name: elasticsearch' } - it { should_not contain "path.conf: /etc/elasticsearch" } - its(:content) { should match "path.data: #{vars['es_data_dirs'].join(',')}" } - its(:content) { should match "path.logs: /var/log/elasticsearch" } - end - - if vars['es_use_repository'] - if vars['ansible_os_family'] == 'RedHat' - describe file("/etc/yum.repos.d/elasticsearch-#{vars['es_repo_name']}.repo") do - it { should exist } - end - describe yumrepo("elasticsearch-#{vars['es_repo_name']}") do - it { should exist } - it { should be_enabled } - end - describe file("/etc/yum.repos.d/elasticsearch-#{vars['es_other_repo_name']}.repo") do - it { should_not exist } - end - describe yumrepo("elasticsearch-#{vars['es_other_repo_name']}") do - it { should_not exist } - it { should_not be_enabled } - end - end - if vars['ansible_os_family'] == 'Debian' - describe command('apt-cache policy') do - its(:stdout) { should match /elastic.co.*\/#{Regexp.quote(vars['es_repo_name'])}\//} - its(:stdout) { should_not match /elastic.co.*\/#{Regexp.quote(vars['es_other_repo_name'])}\//} - end - end - end -end diff --git a/test/integration/helpers/serverspec/spec_helper.rb b/test/integration/helpers/serverspec/spec_helper.rb deleted file mode 100644 index 2f18ef3d..00000000 --- a/test/integration/helpers/serverspec/spec_helper.rb +++ /dev/null @@ -1,31 +0,0 @@ -require 'serverspec' -require 'net/http' -require 'json' - -set :backend, :exec - -require 'rspec/retry' - -RSpec.configure do |config| - # show retry status in spec process - config.verbose_retry = true - # show exception that triggers a retry if verbose_retry is set to true - config.display_try_failure_messages = true -end - -def curl_json(uri, username=nil, password=nil) - uri = URI(uri) - req = Net::HTTP::Get.new(uri) - if username && password - req.basic_auth username, password - end - res = Net::HTTP.start( - uri.hostname, - uri.port, - :use_ssl => uri.scheme == 'https', - :verify_mode => OpenSSL::SSL::VERIFY_NONE - ) {|http| - http.request(req) - } - return JSON.parse(res.body) -end diff --git a/test/integration/license.yml b/test/integration/license.yml deleted file mode 100644 index a9ac3b38..00000000 --- a/test/integration/license.yml +++ /dev/null @@ -1,57 +0,0 @@ ---- -- name: Elasticsearch Xpack HTTP different keystore and truststore with password - hosts: localhost - post_tasks: - - ansible.builtin.include_tasks: elasticsearch/test/integration/debug.yml - roles: - - elasticsearch - vars: - es_config_6x: - xpack.security.authc.realms.file1.order: 0 - xpack.security.authc.realms.file1.type: file - es_config_7x: - xpack.security.authc.realms.file.file1.order: 0 - es_config: "{{ es_config_7x if es_major_version == '7.x' else es_config_6x }}" - es_xpack_license: "{{ lookup('file', '/tmp/license.json') }}" - es_api_basic_auth_username: elastic - es_api_basic_auth_password: changeme - es_api_sleep: 5 - es_enable_http_ssl: true - es_enable_transport_ssl: true - es_ssl_keystore: "test/integration/files/certs/keystore-password.p12" - es_ssl_truststore: "test/integration/files/certs/truststore-password.p12" - es_ssl_keystore_password: password1 - es_ssl_truststore_password: password2 - es_validate_certs: no - es_users: - file: - es_admin: - password: changeMe - roles: - - admin - testUser: - password: changeMeAlso! - roles: - - power_user - - user - es_roles: - file: - admin: - cluster: - - all - indices: - - names: '*' - privileges: - - all - power_user: - cluster: - - monitor - indices: - - names: '*' - privileges: - - all - user: - indices: - - names: '*' - privileges: - - read diff --git a/test/integration/license/license.yml b/test/integration/license/license.yml deleted file mode 100644 index a3c37e19..00000000 --- a/test/integration/license/license.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -- host: test-kitchen diff --git a/test/integration/license/serverspec/default_spec.rb b/test/integration/license/serverspec/default_spec.rb deleted file mode 100644 index aabba223..00000000 --- a/test/integration/license/serverspec/default_spec.rb +++ /dev/null @@ -1,9 +0,0 @@ -require 'license_spec' -require 'shared_spec' -require 'json' -vars = JSON.parse(File.read('/tmp/vars.json')) - -describe 'license tests' do - include_examples 'shared::init', vars - include_examples 'license::init', vars -end diff --git a/test/integration/oss-to-default-upgrade.yml b/test/integration/oss-to-default-upgrade.yml deleted file mode 100644 index 51f1c484..00000000 --- a/test/integration/oss-to-default-upgrade.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -- name: Deploy Elasticsearch OSS version - hosts: localhost - post_tasks: - - ansible.builtin.include_tasks: elasticsearch/test/integration/debug.yml - roles: - - elasticsearch - vars: - es_version: "{{ '7.0.0' if es_major_version == '7.x' else '6.7.1' }}" # This is set to an older version than the current default to force an upgrade - oss_version: true - -- name: Upgrade to Elasticsearch default version - hosts: localhost - post_tasks: - - ansible.builtin.include_tasks: elasticsearch/test/integration/debug.yml - roles: - - elasticsearch - vars: - oss_version: false diff --git a/test/integration/oss-to-default-upgrade/oss-to-default-upgrade.yml b/test/integration/oss-to-default-upgrade/oss-to-default-upgrade.yml deleted file mode 100644 index a3c37e19..00000000 --- a/test/integration/oss-to-default-upgrade/oss-to-default-upgrade.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -- host: test-kitchen diff --git a/test/integration/oss-to-default-upgrade/serverspec/default_spec.rb b/test/integration/oss-to-default-upgrade/serverspec/default_spec.rb deleted file mode 100644 index 08e2a2df..00000000 --- a/test/integration/oss-to-default-upgrade/serverspec/default_spec.rb +++ /dev/null @@ -1,7 +0,0 @@ -require 'shared_spec' -require 'json' -vars = JSON.parse(File.read('/tmp/vars.json')) - -describe 'oss to default upgrade tests' do - include_examples 'shared::init', vars -end diff --git a/test/integration/oss-upgrade.yml b/test/integration/oss-upgrade.yml deleted file mode 100644 index 4e89dab1..00000000 --- a/test/integration/oss-upgrade.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -- name: Deploy Elasticsearch OSS previous version - hosts: localhost - post_tasks: - - ansible.builtin.include_tasks: elasticsearch/test/integration/debug.yml - roles: - - elasticsearch - vars: - es_version: "{{ '7.0.0' if es_major_version == '7.x' else '6.7.1' }}" # This is set to an older version than the current default to force an upgrade - oss_version: true - -- name: Deploy Elasticsearch OSS latest version - hosts: localhost - post_tasks: - - ansible.builtin.include_tasks: elasticsearch/test/integration/debug.yml - roles: - - elasticsearch - vars: - oss_version: true diff --git a/test/integration/oss-upgrade/oss.yml b/test/integration/oss-upgrade/oss.yml deleted file mode 100644 index a3c37e19..00000000 --- a/test/integration/oss-upgrade/oss.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -- host: test-kitchen diff --git a/test/integration/oss-upgrade/serverspec/default_spec.rb b/test/integration/oss-upgrade/serverspec/default_spec.rb deleted file mode 100644 index cd365924..00000000 --- a/test/integration/oss-upgrade/serverspec/default_spec.rb +++ /dev/null @@ -1,7 +0,0 @@ -require 'shared_spec' -require 'json' -vars = JSON.parse(File.read('/tmp/vars.json')) - -describe 'oss upgrade tests' do - include_examples 'shared::init', vars -end diff --git a/test/integration/oss.yml b/test/integration/oss.yml deleted file mode 100644 index 9b3ebb4c..00000000 --- a/test/integration/oss.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -- name: Setup Elasticsearch OSS 6.x - hosts: localhost - post_tasks: - - ansible.builtin.include_tasks: elasticsearch/test/integration/debug.yml - roles: - - elasticsearch - vars: - oss_version: true - -#Do not add tests here. This test is run twice and confirms idempotency. diff --git a/test/integration/oss/oss.yml b/test/integration/oss/oss.yml deleted file mode 100644 index a3c37e19..00000000 --- a/test/integration/oss/oss.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -- host: test-kitchen diff --git a/test/integration/oss/serverspec/default_spec.rb b/test/integration/oss/serverspec/default_spec.rb deleted file mode 100644 index b32dc04b..00000000 --- a/test/integration/oss/serverspec/default_spec.rb +++ /dev/null @@ -1,7 +0,0 @@ -require 'shared_spec' -require 'json' -vars = JSON.parse(File.read('/tmp/vars.json')) - -describe 'oss tests' do - include_examples 'shared::init', vars -end diff --git a/test/integration/security.yml b/test/integration/security.yml deleted file mode 100644 index 54bc511b..00000000 --- a/test/integration/security.yml +++ /dev/null @@ -1,56 +0,0 @@ ---- -- name: Elasticsearch Xpack HTTP different keystore and truststore with password - hosts: localhost - post_tasks: - - ansible.builtin.include_tasks: elasticsearch/test/integration/debug.yml - roles: - - elasticsearch - vars: - es_config_6x: - xpack.security.authc.realms.file1.order: 0 - xpack.security.authc.realms.file1.type: file - es_config_7x: - xpack.security.authc.realms.file.file1.order: 0 - es_config: "{{ es_config_7x if es_major_version == '7.x' else es_config_6x }}" - es_api_basic_auth_username: elastic - es_api_basic_auth_password: changeme - es_api_sleep: 5 - es_enable_http_ssl: true - es_enable_transport_ssl: true - es_ssl_keystore: "test/integration/files/certs/keystore-password.p12" - es_ssl_truststore: "test/integration/files/certs/truststore-password.p12" - es_ssl_keystore_password: password1 - es_ssl_truststore_password: password2 - es_validate_certs: no - es_users: - file: - es_admin: - password: changeMe - roles: - - admin - testUser: - password: changeMeAlso! - roles: - - power_user - - user - es_roles: - file: - admin: - cluster: - - all - indices: - - names: '*' - privileges: - - all - power_user: - cluster: - - monitor - indices: - - names: '*' - privileges: - - all - user: - indices: - - names: '*' - privileges: - - read diff --git a/test/integration/security/security.yml b/test/integration/security/security.yml deleted file mode 100644 index a3c37e19..00000000 --- a/test/integration/security/security.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -- host: test-kitchen diff --git a/test/integration/security/serverspec/default_spec.rb b/test/integration/security/serverspec/default_spec.rb deleted file mode 100644 index 25696e8f..00000000 --- a/test/integration/security/serverspec/default_spec.rb +++ /dev/null @@ -1,9 +0,0 @@ -require 'security_spec' -require 'shared_spec' -require 'json' -vars = JSON.parse(File.read('/tmp/vars.json')) - -describe 'security tests' do - include_examples 'shared::init', vars - include_examples 'security::init', vars -end diff --git a/test/integration/trial.yml b/test/integration/trial.yml deleted file mode 100644 index d7a0593f..00000000 --- a/test/integration/trial.yml +++ /dev/null @@ -1,106 +0,0 @@ ---- -- name: Setup Elasticsearch with security enabled and a trial license - hosts: localhost - post_tasks: - - ansible.builtin.include_tasks: elasticsearch/test/integration/debug.yml - roles: - - elasticsearch - vars: - es_config_6x: - xpack.security.authc.realms.file1.order: 0 - xpack.security.authc.realms.file1.type: file - xpack.security.authc.realms.native1.order: 1 - xpack.security.authc.realms.native1.type: native - es_config_7x: - xpack.security.authc.realms.file.file1.order: 0 - xpack.security.authc.realms.native.native1.order: 1 - es_config: "{{ es_config_7x if es_major_version == '7.x' else es_config_6x }}" - es_xpack_trial: true - es_api_basic_auth_username: elastic - es_api_basic_auth_password: changeme - es_api_sleep: 5 - es_enable_http_ssl: false - es_enable_transport_ssl: true - es_ssl_keystore: "test/integration/files/certs/keystore-password.p12" - es_ssl_truststore: "test/integration/files/certs/truststore-password.p12" - es_ssl_keystore_password: password1 - es_ssl_truststore_password: password2 - es_validate_certs: no - es_role_mapping: - power_user: - - "cn=admins,dc=example,dc=com" - user: - - "cn=users,dc=example,dc=com" - - "cn=admins,dc=example,dc=com" - es_users: - native: - kibana4_server: - password: changeMe - roles: - - kibana4_server - logstash_system: - #this should be successfully modified - password: aNewLogstashPassword - #this will be ignored - roles: - - kibana4_server - elastic: - password: elasticChanged - file: - es_admin: - password: changeMe - roles: - - admin - testUser: - password: changeMeAlso! - roles: - - power_user - - user - es_roles: - file: - admin: - cluster: - - all - indices: - - names: '*' - privileges: - - all - power_user: - cluster: - - monitor - indices: - - names: '*' - privileges: - - all - user: - indices: - - names: '*' - privileges: - - read - kibana4_server: - cluster: - - monitor - indices: - - names: '.kibana' - privileges: - - all - native: - logstash: - cluster: - - manage_index_templates - indices: - - names: 'logstash-*' - privileges: - - write - - delete - - create_index - #this will be ignored - its reserved - logstash_system: - cluster: - - manage_index_templates - indices: - - names: 'logstash-*' - privileges: - - write - - delete - - create_index diff --git a/test/integration/trial/serverspec/default_spec.rb b/test/integration/trial/serverspec/default_spec.rb deleted file mode 100644 index 9700f5bd..00000000 --- a/test/integration/trial/serverspec/default_spec.rb +++ /dev/null @@ -1,7 +0,0 @@ -require 'shared_spec' -require 'json' -vars = JSON.parse(File.read('/tmp/vars.json')) - -describe 'trial tests' do - include_examples 'shared::init', vars -end diff --git a/test/integration/trial/trial.yml b/test/integration/trial/trial.yml deleted file mode 100644 index a3c37e19..00000000 --- a/test/integration/trial/trial.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -- host: test-kitchen diff --git a/test/integration/upgrade.yml b/test/integration/upgrade.yml deleted file mode 100644 index c590db21..00000000 --- a/test/integration/upgrade.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -- name: Deploy Elasticsearch previous version - hosts: localhost - post_tasks: - - ansible.builtin.include_tasks: elasticsearch/test/integration/debug.yml - roles: - - elasticsearch - vars: - es_version: "{{ '7.0.0' if es_major_version == '7.x' else '6.7.1' }}" - -- name: Deploy Elasticsearch latest version - hosts: localhost - post_tasks: - - ansible.builtin.include_tasks: elasticsearch/test/integration/debug.yml - roles: - - elasticsearch diff --git a/test/integration/upgrade/serverspec/default_spec.rb b/test/integration/upgrade/serverspec/default_spec.rb deleted file mode 100644 index 29eacb20..00000000 --- a/test/integration/upgrade/serverspec/default_spec.rb +++ /dev/null @@ -1,7 +0,0 @@ -require 'shared_spec' -require 'json' -vars = JSON.parse(File.read('/tmp/vars.json')) - -describe 'upgrade tests' do - include_examples 'shared::init', vars -end diff --git a/test/integration/upgrade/upgrade.yml b/test/integration/upgrade/upgrade.yml deleted file mode 100644 index a3c37e19..00000000 --- a/test/integration/upgrade/upgrade.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -- host: test-kitchen diff --git a/test/matrix-6x.yml b/test/matrix-6x.yml deleted file mode 100644 index d76a5cca..00000000 --- a/test/matrix-6x.yml +++ /dev/null @@ -1,19 +0,0 @@ -OS: - - ubuntu-1604 - - ubuntu-1804 - - ubuntu-2004 - - debian-8 - - debian-9 - - debian-10 - - centos-7 - - amazonlinux-2 -TEST_TYPE: - - custom-config - - default - - license - - oss - - oss-to-default-upgrade - - oss-upgrade - - security - - trial - - upgrade diff --git a/test/matrix.yml b/test/matrix.yml deleted file mode 100644 index 1955cf59..00000000 --- a/test/matrix.yml +++ /dev/null @@ -1,17 +0,0 @@ -OS: - - ubuntu-1604 - - ubuntu-1804 - - ubuntu-2004 - - debian-8 - - debian-9 - - debian-10 - - centos-7 - - amazonlinux-2 -TEST_TYPE: - - custom-config - - default - - license - - oss-to-default-upgrade - - security - - trial - - upgrade