init

Author: hkulekci

.gitignore

@@ -28,3 +28,5 @@ node_modules
 
 # Debug log from npm
 npm-debug.log
+
+.env

README.md

@@ -0,0 +1,80 @@
+## Node Elasticsearch Example
+
+## Database Structure and Initialization
+
+Please check `data/sample_data.sql` for our sample data.
+
+
+## Elastic Integration
+
+First of all create your `products` index and `product` type.
+
+```
+DELETE products
+
+PUT products
+{
+  "settings": {
+    "number_of_shards": 1,
+    "number_of_replicas": 0
+  }
+}
+
+
+PUT products/product/_mapping
+{
+  "properties": {
+    "name": {
+      "type": "text"
+    },
+    "description": {
+      "type": "text"
+    },
+    "quantity": {
+      "type": "long"
+    },
+    "price": {
+      "type": "double"
+    },
+    "created_at": {
+      "type": "date"
+    },
+    "updated_at": {
+      "type": "date"
+    },
+    "categories": {
+      "type": "object",
+      "properties": {
+        "id": {
+          "type": "long"
+        },
+        "name": {
+          "type": "text"
+        }
+      }
+    }
+  }
+}
+```
+
+Then check `data/logstash.conf` file and also last part of the 
+`data/sample_data.sql`. There is a procedure in there. 
+
+```
+DROP PROCEDURE fetchDataForElastic;
+
+DELIMITER //
+CREATE PROCEDURE fetchDataForElastic
+(IN currentdate Datetime)
+BEGIN
+  SELECT
+    p.*,
+    
+    CAST( (CONCAT ('[', GROUP_CONCAT(CONCAT('{"id":', c.id, ', "name":"',c.name,'"}')), ']'))  AS JSON) categories
+    FROM products p LEFT JOIN product_category pc ON pc.product_id = p.id LEFT JOIN categories c ON c.id = pc.category_id
+  WHERE p.updated_at > currentdate GROUP BY p.id;
+END //
+DELIMITER ;
+```
+
+At the end, run logstash. `docker-compose up logstash`

app.js

@@ -0,0 +1,58 @@
+var express = require('express');
+var path = require('path');
+var favicon = require('serve-favicon');
+var logger = require('morgan');
+var cookieParser = require('cookie-parser');
+var bodyParser = require('body-parser');
+var lessMiddleware = require('less-middleware');
+
+// .env file configuration
+require('dotenv').config();
+
+
+var index = require('./routes/index');
+var products = require('./routes/products');
+var categories = require('./routes/categories');
+var search = require('./routes/search');
+var graphs = require('./routes/graphs');
+
+var app = express();
+
+// view engine setup
+app.set('views', path.join(__dirname, 'views'));
+app.set('view engine', 'twig');
+
+// uncomment after placing your favicon in /public
+//app.use(favicon(path.join(__dirname, 'public', 'favicon.ico')));
+app.use(logger('dev'));
+app.use(bodyParser.json());
+app.use(bodyParser.urlencoded({ extended: false }));
+app.use(cookieParser());
+app.use(lessMiddleware(path.join(__dirname, 'public')));
+app.use(express.static(path.join(__dirname, 'public')));
+
+app.use('/', index);
+app.use('/product', products);
+app.use('/category', categories);
+app.use('/search', search);
+app.use('/graphs', graphs);
+
+// catch 404 and forward to error handler
+app.use(function(req, res, next) {
+  var err = new Error('Not Found');
+  err.status = 404;
+  next(err);
+});
+
+// error handler
+app.use(function(err, req, res, next) {
+  // set locals, only providing error in development
+  res.locals.message = err.message;
+  res.locals.error = req.app.get('env') === 'development' ? err : {};
+
+  // render the error page
+  res.status(err.status || 500);
+  res.render('error');
+});
+
+module.exports = app;

bin/www

@@ -0,0 +1,90 @@
+#!/usr/bin/env node
+
+/**
+ * Module dependencies.
+ */
+
+var app = require('../app');
+var debug = require('debug')('node-es-example:server');
+var http = require('http');
+
+/**
+ * Get port from environment and store in Express.
+ */
+
+var port = normalizePort(process.env.PORT || '3000');
+app.set('port', port);
+
+/**
+ * Create HTTP server.
+ */
+
+var server = http.createServer(app);
+
+/**
+ * Listen on provided port, on all network interfaces.
+ */
+
+server.listen(port);
+server.on('error', onError);
+server.on('listening', onListening);
+
+/**
+ * Normalize a port into a number, string, or false.
+ */
+
+function normalizePort(val) {
+  var port = parseInt(val, 10);
+
+  if (isNaN(port)) {
+    // named pipe
+    return val;
+  }
+
+  if (port >= 0) {
+    // port number
+    return port;
+  }
+
+  return false;
+}
+
+/**
+ * Event listener for HTTP server "error" event.
+ */
+
+function onError(error) {
+  if (error.syscall !== 'listen') {
+    throw error;
+  }
+
+  var bind = typeof port === 'string'
+    ? 'Pipe ' + port
+    : 'Port ' + port;
+
+  // handle specific listen errors with friendly messages
+  switch (error.code) {
+    case 'EACCES':
+      console.error(bind + ' requires elevated privileges');
+      process.exit(1);
+      break;
+    case 'EADDRINUSE':
+      console.error(bind + ' is already in use');
+      process.exit(1);
+      break;
+    default:
+      throw error;
+  }
+}
+
+/**
+ * Event listener for HTTP server "listening" event.
+ */
+
+function onListening() {
+  var addr = server.address();
+  var bind = typeof addr === 'string'
+    ? 'pipe ' + addr
+    : 'port ' + addr.port;
+  debug('Listening on ' + bind);
+}

data/logstash.conf

@@ -0,0 +1,34 @@
+input {
+  jdbc {
+    jdbc_driver_library => "mysql-connector-java-5.1.36-bin.jar"
+    jdbc_driver_class => "com.mysql.jdbc.Driver"
+    jdbc_connection_string => "jdbc:mysql://127.0.0.1:3306/node_es_example"
+    jdbc_user => "root"
+    jdbc_password => ""
+    schedule => "* * * * *"
+    statement => "CALL fetchDataForElastic(:sql_last_value);"
+  }
+}
+
+filter {
+  json {
+    source => "categories"
+    target => "categories"
+  }
+  mutate { remove_field => [ "@version", "@timestamp" ] }
+}
+
+output {
+  stdout
+  {
+    codec => rubydebug
+  }
+
+  elasticsearch
+  {
+    hosts => ["elasticsearch:9200"]
+    index => "products"
+    document_type => "product"
+    document_id => "%{id}"
+  }
+}