diff --git a/.gitignore b/.gitignore index 5797d25..c5bc8e9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .idea app/php/vendor +/volumes/logstash/logs/**/* /var/ \ No newline at end of file diff --git a/README.md b/README.md index 13a688e..0bf01c6 100644 --- a/README.md +++ b/README.md @@ -2,23 +2,35 @@ ![Build Elastic Stack](https://github.com/CodelyTV/elastic-stack-example/workflows/Build%20Elastic%20Stack/badge.svg) ## How to run the Stack - To initialize all the needed services. ``` docker-compose up -d ``` -## How to run PHP sample app +## How to query the Stack directly +Https and authentication have been disabled for the sake of simplicity. + +``` +curl --location --request GET 'http://localhost:9200/' \ +--header 'Content-Type: application/json' \ +--data-raw '' +``` -Install needed PHP dependencies + +## How to run PHP sample app +No local setup is eneded: the php docker-compose service can be used. Access it and install the needed PHP dependencies ``` -cd app && composer install +docker-compose exec php ``` -Execute the PHP sample application +inside the docker install the dependencies: +``` +composer install +``` +Execute the PHP sample application within the docker: ``` -php app/php/app.php -a fo1 -b bar +php app.php -a fo1 -b bar ``` diff --git a/app/php/app.php b/app/php/app.php index c4c68df..42a7df2 100644 --- a/app/php/app.php +++ b/app/php/app.php @@ -5,7 +5,6 @@ use Monolog\Logger; use Elastica\Client; use Monolog\Formatter\JsonFormatter; -use Monolog\Handler\ElasticSearchHandler; use Monolog\Handler\ErrorLogHandler; use Monolog\Handler\RotatingFileHandler; @@ -16,23 +15,27 @@ $stdoutHandler = new ErrorLogHandler(); $formatter = new JsonFormatter(); $stdoutHandler->setFormatter($formatter); -$log->pushHandler($stdoutHandler); // File Handler -$fileHandler = new RotatingFileHandler('../var/logs/app.log', 0, Logger::DEBUG); +$fileHandler = new RotatingFileHandler(__DIR__.'/logs/app.log', 0, \Monolog\Level::Debug); $formatter = new JsonFormatter(); $fileHandler->setFormatter($formatter); -$log->pushHandler($fileHandler); // Elasticsearch Handler $elasticaClient = new Client( [ - 'host' => 'localhost', - 'port' => 9200 + 'host' => 'elasticsearch', + 'port' => 9200, + 'transport' => 'http', + 'hosts' => ['elasticsearch:9200'], ] ); -$elasticsearchHandler = new ElasticSearchHandler($elasticaClient); +$elasticsearchHandler = new \Monolog\Handler\ElasticaHandler($elasticaClient, ['index' => 'codelytv', 'type' => 'record']); + +// Register Handlers +$log->pushHandler($fileHandler); +$log->pushHandler($stdoutHandler); $log->pushHandler($elasticsearchHandler); // My Application @@ -40,7 +43,7 @@ # App Servidor A if ($options['a'] === 'warning') { - $log->warn('Esto es un Warning', ['Servidor' => 'Servidor A']); + $log->warning('Esto es un nuevo Warning', ['Servidor' => 'Servidor A']); } else { $log->info('Esto es un Info', ['Servidor' => 'Servidor A']); } diff --git a/app/php/composer.json b/app/php/composer.json index 3acb60a..344f3c3 100644 --- a/app/php/composer.json +++ b/app/php/composer.json @@ -1,6 +1,11 @@ { "require": { - "monolog/monolog": "^1.23", - "ruflin/elastica": "^6.0" + "ruflin/elastica": "^8.1", + "monolog/monolog": "^3.9.0" + }, + "config": { + "allow-plugins": { + "php-http/discovery": true + } } } diff --git a/app/php/composer.lock b/app/php/composer.lock deleted file mode 100644 index fee8313..0000000 --- a/app/php/composer.lock +++ /dev/null @@ -1,405 +0,0 @@ -{ - "_readme": [ - "This file locks the dependencies of your project to a known state", - "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", - "This file is @generated automatically" - ], - "content-hash": "503e47eb01fdc2b28493f8e2bf37a872", - "packages": [ - { - "name": "elasticsearch/elasticsearch", - "version": "v6.0.1", - "source": { - "type": "git", - "url": "https://github.com/elastic/elasticsearch-php.git", - "reference": "b8e3bc9d1fc54d6a18692df0b74956efe7fe241a" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/elastic/elasticsearch-php/zipball/b8e3bc9d1fc54d6a18692df0b74956efe7fe241a", - "reference": "b8e3bc9d1fc54d6a18692df0b74956efe7fe241a", - "shasum": "" - }, - "require": { - "ext-json": ">=1.3.7", - "guzzlehttp/ringphp": "~1.0", - "php": "^7.0", - "psr/log": "~1.0" - }, - "require-dev": { - "cpliakas/git-wrapper": "~1.0", - "doctrine/inflector": "^1.1", - "mockery/mockery": "0.9.4", - "phpstan/phpstan-shim": "0.8.3", - "phpunit/phpunit": "6.3.0", - "squizlabs/php_codesniffer": "3.0.2", - "symfony/finder": "^2.8", - "symfony/yaml": "^2.8" - }, - "suggest": { - "ext-curl": "*", - "monolog/monolog": "Allows for client-level logging and tracing" - }, - "type": "library", - "autoload": { - "psr-4": { - "Elasticsearch\\": "src/Elasticsearch/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "Apache-2.0" - ], - "authors": [ - { - "name": "Zachary Tong" - } - ], - "description": "PHP Client for Elasticsearch", - "keywords": [ - "client", - "elasticsearch", - "search" - ], - "time": "2017-12-05T14:15:58+00:00" - }, - { - "name": "guzzlehttp/ringphp", - "version": "1.1.1", - "source": { - "type": "git", - "url": "https://github.com/guzzle/RingPHP.git", - "reference": "5e2a174052995663dd68e6b5ad838afd47dd615b" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/guzzle/RingPHP/zipball/5e2a174052995663dd68e6b5ad838afd47dd615b", - "reference": "5e2a174052995663dd68e6b5ad838afd47dd615b", - "shasum": "" - }, - "require": { - "guzzlehttp/streams": "~3.0", - "php": ">=5.4.0", - "react/promise": "~2.0" - }, - "require-dev": { - "ext-curl": "*", - "phpunit/phpunit": "~4.0" - }, - "suggest": { - "ext-curl": "Guzzle will use specific adapters if cURL is present" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "1.1-dev" - } - }, - "autoload": { - "psr-4": { - "GuzzleHttp\\Ring\\": "src/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Michael Dowling", - "email": "mtdowling@gmail.com", - "homepage": "https://github.com/mtdowling" - } - ], - "description": "Provides a simple API and specification that abstracts away the details of HTTP into a single PHP function.", - "time": "2018-07-31T13:22:33+00:00" - }, - { - "name": "guzzlehttp/streams", - "version": "3.0.0", - "source": { - "type": "git", - "url": "https://github.com/guzzle/streams.git", - "reference": "47aaa48e27dae43d39fc1cea0ccf0d84ac1a2ba5" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/guzzle/streams/zipball/47aaa48e27dae43d39fc1cea0ccf0d84ac1a2ba5", - "reference": "47aaa48e27dae43d39fc1cea0ccf0d84ac1a2ba5", - "shasum": "" - }, - "require": { - "php": ">=5.4.0" - }, - "require-dev": { - "phpunit/phpunit": "~4.0" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "3.0-dev" - } - }, - "autoload": { - "psr-4": { - "GuzzleHttp\\Stream\\": "src/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Michael Dowling", - "email": "mtdowling@gmail.com", - "homepage": "https://github.com/mtdowling" - } - ], - "description": "Provides a simple abstraction over streams of data", - "homepage": "http://guzzlephp.org/", - "keywords": [ - "Guzzle", - "stream" - ], - "time": "2014-10-12T19:18:40+00:00" - }, - { - "name": "monolog/monolog", - "version": "1.23.0", - "source": { - "type": "git", - "url": "https://github.com/Seldaek/monolog.git", - "reference": "fd8c787753b3a2ad11bc60c063cff1358a32a3b4" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/fd8c787753b3a2ad11bc60c063cff1358a32a3b4", - "reference": "fd8c787753b3a2ad11bc60c063cff1358a32a3b4", - "shasum": "" - }, - "require": { - "php": ">=5.3.0", - "psr/log": "~1.0" - }, - "provide": { - "psr/log-implementation": "1.0.0" - }, - "require-dev": { - "aws/aws-sdk-php": "^2.4.9 || ^3.0", - "doctrine/couchdb": "~1.0@dev", - "graylog2/gelf-php": "~1.0", - "jakub-onderka/php-parallel-lint": "0.9", - "php-amqplib/php-amqplib": "~2.4", - "php-console/php-console": "^3.1.3", - "phpunit/phpunit": "~4.5", - "phpunit/phpunit-mock-objects": "2.3.0", - "ruflin/elastica": ">=0.90 <3.0", - "sentry/sentry": "^0.13", - "swiftmailer/swiftmailer": "^5.3|^6.0" - }, - "suggest": { - "aws/aws-sdk-php": "Allow sending log messages to AWS services like DynamoDB", - "doctrine/couchdb": "Allow sending log messages to a CouchDB server", - "ext-amqp": "Allow sending log messages to an AMQP server (1.0+ required)", - "ext-mongo": "Allow sending log messages to a MongoDB server", - "graylog2/gelf-php": "Allow sending log messages to a GrayLog2 server", - "mongodb/mongodb": "Allow sending log messages to a MongoDB server via PHP Driver", - "php-amqplib/php-amqplib": "Allow sending log messages to an AMQP server using php-amqplib", - "php-console/php-console": "Allow sending log messages to Google Chrome", - "rollbar/rollbar": "Allow sending log messages to Rollbar", - "ruflin/elastica": "Allow sending log messages to an Elastic Search server", - "sentry/sentry": "Allow sending log messages to a Sentry server" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "2.0.x-dev" - } - }, - "autoload": { - "psr-4": { - "Monolog\\": "src/Monolog" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Jordi Boggiano", - "email": "j.boggiano@seld.be", - "homepage": "http://seld.be" - } - ], - "description": "Sends your logs to files, sockets, inboxes, databases and various web services", - "homepage": "http://github.com/Seldaek/monolog", - "keywords": [ - "log", - "logging", - "psr-3" - ], - "time": "2017-06-19T01:22:40+00:00" - }, - { - "name": "psr/log", - "version": "1.0.2", - "source": { - "type": "git", - "url": "https://github.com/php-fig/log.git", - "reference": "4ebe3a8bf773a19edfe0a84b6585ba3d401b724d" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/php-fig/log/zipball/4ebe3a8bf773a19edfe0a84b6585ba3d401b724d", - "reference": "4ebe3a8bf773a19edfe0a84b6585ba3d401b724d", - "shasum": "" - }, - "require": { - "php": ">=5.3.0" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "1.0.x-dev" - } - }, - "autoload": { - "psr-4": { - "Psr\\Log\\": "Psr/Log/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "PHP-FIG", - "homepage": "http://www.php-fig.org/" - } - ], - "description": "Common interface for logging libraries", - "homepage": "https://github.com/php-fig/log", - "keywords": [ - "log", - "psr", - "psr-3" - ], - "time": "2016-10-10T12:19:37+00:00" - }, - { - "name": "react/promise", - "version": "v2.7.0", - "source": { - "type": "git", - "url": "https://github.com/reactphp/promise.git", - "reference": "f4edc2581617431aea50430749db55cc3fc031b3" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/reactphp/promise/zipball/f4edc2581617431aea50430749db55cc3fc031b3", - "reference": "f4edc2581617431aea50430749db55cc3fc031b3", - "shasum": "" - }, - "require": { - "php": ">=5.4.0" - }, - "require-dev": { - "phpunit/phpunit": "~4.8" - }, - "type": "library", - "autoload": { - "psr-4": { - "React\\Promise\\": "src/" - }, - "files": [ - "src/functions_include.php" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Jan Sorgalla", - "email": "jsorgalla@gmail.com" - } - ], - "description": "A lightweight implementation of CommonJS Promises/A for PHP", - "keywords": [ - "promise", - "promises" - ], - "time": "2018-06-13T15:59:06+00:00" - }, - { - "name": "ruflin/elastica", - "version": "6.0.2", - "source": { - "type": "git", - "url": "https://github.com/ruflin/Elastica.git", - "reference": "8a6a3b2e71fdf7c96ad92fd2622bcc5904696fcf" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/ruflin/Elastica/zipball/8a6a3b2e71fdf7c96ad92fd2622bcc5904696fcf", - "reference": "8a6a3b2e71fdf7c96ad92fd2622bcc5904696fcf", - "shasum": "" - }, - "require": { - "elasticsearch/elasticsearch": "^6.0", - "php": "^7.0", - "psr/log": "~1.0" - }, - "require-dev": { - "aws/aws-sdk-php": "~3.0", - "guzzlehttp/guzzle": "~6.0" - }, - "suggest": { - "aws/aws-sdk-php": "Allow using IAM authentication with Amazon ElasticSearch Service", - "egeloen/http-adapter": "Allow using httpadapter transport", - "guzzlehttp/guzzle": "Allow using guzzle 6 as the http transport", - "monolog/monolog": "Logging request" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "6.0.x-dev" - } - }, - "autoload": { - "psr-4": { - "Elastica\\": "lib/Elastica/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Nicolas Ruflin", - "homepage": "http://ruflin.com/" - } - ], - "description": "Elasticsearch Client", - "homepage": "http://elastica.io/", - "keywords": [ - "client", - "search" - ], - "time": "2018-05-29T08:31:52+00:00" - } - ], - "packages-dev": [], - "aliases": [], - "minimum-stability": "stable", - "stability-flags": [], - "prefer-stable": false, - "prefer-lowest": false, - "platform": [], - "platform-dev": [] -} diff --git a/docker-compose.yml b/docker-compose.yml index 984246a..2eb4b48 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,54 +1,62 @@ -version: '3.7' - services: - elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.5.0 + image: docker.elastic.co/elasticsearch/elasticsearch:8.1.0 container_name: elasticsearch - ports: ['9200:9200'] - environment: - - "ELASTIC_PASSWORD=elastic" - - "bootstrap.memory_lock=true" - - "ES_JAVA_OPTS=-Xms1g -Xmx1g" - - "discovery.type=single-node" - - "TAKE_FILE_OWNERSHIP=1" + ports: [ '9200:9200' ] + env_file: + - path: ./env/.env + required: true + - path: ./env/.elastic.env + required: false ulimits: memlock: soft: -1 hard: -1 volumes: - - './var/elasticsearch:/usr/share/elasticsearch/data' + - './volumes/elasticsearch:/usr/share/elasticsearch/data' + + php: + image: composer:2.8.8 + command: tail -f /dev/null + volumes: + - './app/php:/app' + - './volumes/php/vendor:/app/vendor' + - './volumes/logstash/logs:/app/logs' + depends_on: + - elasticsearch logstash: - image: docker.elastic.co/logstash/logstash:7.5.0 + image: docker.elastic.co/logstash/logstash:8.1.0 container_name: logstash - environment: - - "ELASTIC_PASSWORD=elastic" - ports: ['9600:9600'] - depends_on: ['elasticsearch'] + env_file: + - path: ./env/.env + required: true + ports: [ '9600:9600' ] + depends_on: [ 'elasticsearch' ] volumes: - - './etc/logstash/pipeline:/usr/share/logstash/pipeline' - - './var/logs:/logs' + - './volumes/logstash/pipeline:/usr/share/logstash/pipeline:rw' + - './volumes/logstash/logs:/logs:rw' filebeat: - image: docker.elastic.co/beats/filebeat:7.5.0 + image: docker.elastic.co/beats/filebeat:8.1.0 container_name: filebeat - environment: - - "ELASTIC_PASSWORD=elastic" + env_file: + - path: ./env/.env + required: true user: root volumes: - - './var/filebeat/log/:/var/log/:ro' - - './var/filebeat/docker.sock:/var/run/docker.sock' - - './var/logs/:/var/lib/docker/containers/logs:ro' - - './etc/filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml' - command: ["--strict.perms=false"] + - './volumes/filebeat/log:/var/log/:ro' + - './volumes/filebeat/docker.sock:/var/run/docker.sock' + - './volumes/filebeat/logs/:/var/lib/docker/containers/logs:ro' + - './volumes/filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml' + command: [ "--strict.perms=false" ] kibana: - image: docker.elastic.co/kibana/kibana:7.5.0 + image: docker.elastic.co/kibana/kibana:8.1.0 container_name: kibana - environment: - - "ELASTIC_PASSWORD=elastic" - ports: ['5601:5601'] + env_file: + - path: ./env/.env + required: true + ports: [ '5601:5601' ] depends_on: - - elasticsearch - + - elasticsearch \ No newline at end of file diff --git a/env/.elastic.env b/env/.elastic.env new file mode 100644 index 0000000..5b71927 --- /dev/null +++ b/env/.elastic.env @@ -0,0 +1,8 @@ +bootstrap.memory_lock=true +ES_JAVA_OPTS="-Xms1g -Xmx1g" +discovery.type="single-node" +TAKE_FILE_OWNERSHIP=1 +xpack.security.enabled=false +xpack.security.transport.ssl.enabled=false +xpack.security.http.ssl.enabled=false +cluster.routing.allocation.disk.threshold_enabled=false diff --git a/env/.env b/env/.env new file mode 100644 index 0000000..0a03f61 --- /dev/null +++ b/env/.env @@ -0,0 +1 @@ +ELASTIC_PASSWORD="password" diff --git a/etc/filebeat/filebeat.yml b/etc/filebeat/filebeat.yml deleted file mode 100644 index faa496b..0000000 --- a/etc/filebeat/filebeat.yml +++ /dev/null @@ -1,19 +0,0 @@ -logging.level: info -logging.to_files: false - -filebeat.inputs: - - type: log - enabled: true - paths: - - '/var/lib/docker/containers/logs/*.log' - json.keys_under_root: true - - processors: - - decode_json_fields: - fields: ['message'] - target: json - -output.elasticsearch: - hosts: ["elasticsearch:9200"] - username: elastic - password: changeme diff --git a/volumes/.gitkeep b/volumes/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/volumes/filebeat/filebeat.yml b/volumes/filebeat/filebeat.yml new file mode 100644 index 0000000..9eb5953 --- /dev/null +++ b/volumes/filebeat/filebeat.yml @@ -0,0 +1,19 @@ +logging.level: info +logging.to_files: false + +filebeat.inputs: + - type: log + enabled: true + paths: + - '/var/lib/docker/containers/logs/*.log' + json.keys_under_root: true + + processors: + - decode_json_fields: + fields: [ 'message' ] + target: json + +output.elasticsearch: + hosts: [ "elasticsearch:9200" ] + username: elastic + password: changeme diff --git a/volumes/filebeat/log/.gitkeep b/volumes/filebeat/log/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/volumes/filebeat/logs/.gitkeep b/volumes/filebeat/logs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/volumes/logstash/logs/.gitkeep b/volumes/logstash/logs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/etc/logstash/pipeline/logstash.conf b/volumes/logstash/pipeline/logstash.conf similarity index 100% rename from etc/logstash/pipeline/logstash.conf rename to volumes/logstash/pipeline/logstash.conf diff --git a/volumes/php/vendor/.gitkeep b/volumes/php/vendor/.gitkeep new file mode 100644 index 0000000..e69de29