Adding Rake-based toolchain to bootstrap, build, and control NGINX

.gitignore

@@ -0,0 +1,2 @@
+build
+vendor

Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gem "rake"

Gemfile.lock

@@ -0,0 +1,13 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    rake (11.1.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  rake
+
+BUNDLED WITH
+   1.11.2

README.md

@@ -8,79 +8,93 @@ Table of Contents
 
 * [Nginx Kafka Module](#nginx-kafka-module)
 * [Installation](#installation)
-    * [Dependency Installation](#dependency-installation)
+    * [Bootstrapping](#bootstrapping)
     * [Compilation](#compilation)
-* [Nginx Configuration](#nginx-configuration)
+* [NGINX Configuration](#nginx-configuration)
+* [Starting and Stopping NGINX](#starting-and-stopping-nginx)
 * [Example of Usage](#example-of-usage)
 * [Report Bugs](#report-bugs)
 * [Copyright & License](#copyright--license)
 
 Installation
 ====
 
-Dependency Installation
+This module has a Rake-based toolchain to simplify quickly getting the
+module running in an NGINX instance. If you do not have
+[Bundler](http://bundler.io/) installed, get it now with `gem install
+bundler`.  Next, inside your clone of the repository do `bundle
+install`.
+
+At this point, you can see the commands available to you:
+```
+$ rake -T
+rake bootstrap        # Bootstraps the local development environment
+rake bootstrap:clean  # Removes vendor code
+rake nginx:compile    # Recompiles NGINX
+rake nginx:start      # Starts NGINX
+rake nginx:stop       # Stops NGINX
+```
+
+Bootstrapping
 ----
 
-Install [librdkafka](https://github.com/edenhill/librdkafka)
+Doing `rake bootstrap` will download, build, and install NGINX and librdkafka
+for you.
 
-    git clone https://github.com/edenhill/librdkafka
-    cd librdkafka
-    ./configure
-    make
-    sudo make install
+Doing `rake bootstrap:clean` removes source and installs of NGINX and
+librdkafka.
+
+[Back to TOC](#table-of-contents)
 
 Compilation
 ----
 
-Compile this module into nginx
-
-    git clone https://github.com/brg-liuwei/ngx_kafka_module
-
-    # cd /path/to/nginx
-    ./configure --add-module=/path/to/ngx_kafka_module
-
-    make
-
-    sudo make install
-    # or, use `sudo make upgrade` instead of `sudo make install`
+Compile this module into NGINX by doing `rake nginx:compile`.
 
 [Back to TOC](#table-of-contents)
 
-Nginx Configuration
+NGINX Configuration
 ====
 
-Add the code to nginx conf file as follows
-
-    http {
-
-        # some other configs
+The [nginx.conf](nginx.conf) file demonstrates how to configure the module, and
+that file is used in the NGINX that was built in the previous step.
 
-        kafka;
-
-        kafka_broker_list 127.0.0.1:9092 127.0.0.1:9093; # host:port ...
+[Back to TOC](#table-of-contents)
 
-        server {
+Starting and Stopping NGINX
+====
 
-            # some other configs
+Once it has been built, you can start and stop NGINX with the following
+commands.
+```
+$ rake nginx:start
+$ rake nginx:stop
+```
 
-            location = /your/path/topic0 {
-                kafka_topic your_topic0;
-            }
+Example of Usage
+====
 
-            location = /your/path/topic1 {
-                kafka_topic your_topic1;
-            }
-        }
-    }
+All that's left now to see the module in action is to get Kafka running and
+make POST requests to the server.
 
+Follow the instructions in the [Kafka Quickstart](http://kafka.apache.org/documentation.html#quickstart)
+to get Zookeeper and Kafka running. Then create a topic named "test". Finally,
+you will want to start a consumer so you can see the messages being written to
+the topic. This is also shown in the Kafka Quickstart and looks something like:
+```
+$ kafka-console-consumer --zookeeper localhost:2181 --topic test --from-beginning
+```
 
-[Back to TOC](#table-of-contents)
+At this point, you should have a running NGINX that was built with the Kafka
+module and a running Kafka instance that you are watching. Now it's time to
+make requests!
 
-Example of Usage
-====
+```
+$ curl localhost:8080/publish -d "it's working"
+```
 
-    curl localhost/your/path/topic0 -d "message send to kafka topic0"
-    curl localhost/your/path/topic1 -d "message send to kafka topic1"
+If all goes as planned, you will see "it's working" printed in the terminal
+that is running the `kafka-console-consumer` command.
 
 [Back to TOC](#table-of-contents)
 

Rakefile

@@ -0,0 +1,32 @@
+require 'rake'
+
+namespace :nginx do
+  desc "Starts NGINX"
+  task :start do
+    `build/nginx/sbin/nginx`
+    sleep 1
+  end
+
+  desc "Stops NGINX"
+  task :stop do
+    `build/nginx/sbin/nginx -s stop`
+  end
+
+  desc "Recompiles NGINX"
+  task :compile do
+    sh "PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$PWD/build/librdkafka/lib/pkgconfig scripts/compile"
+  end
+end
+
+desc "Bootstraps the local development environment"
+task :bootstrap do
+  sh "scripts/bootstrap"
+end
+
+namespace :bootstrap do
+  desc "Removes vendor code"
+  task :clean do
+    sh "scripts/bootstrap clean"
+  end
+
+end

config

@@ -2,4 +2,5 @@ ngx_addon_name=ngx_http_kafka_module
 HTTP_MODULES="$HTTP_MODULES ngx_http_kafka_module"
 NGX_ADDON_SRCS="$NGX_ADDON_SRCS $ngx_addon_dir/ngx_http_kafka_module.c"
 
-CORE_LIBS="$CORE_LIBS -lrdkafka -lz -lpthread" 
+CORE_LIBS="$CORE_LIBS `pkg-config --libs rdkafka` -lz -lpthread"
+CFLAGS="$CFLAGS `pkg-config --cflags rdkafka`"

nginx.conf

@@ -0,0 +1,16 @@
+events {
+  worker_connections 1024;
+}
+
+http {
+  kafka;
+  kafka_broker_list 127.0.0.1:9092;
+
+  server {
+    listen 8080;
+
+    location = /publish {
+      kafka_topic test;
+    }
+  }
+}

ngx_http_kafka_module.c

@@ -130,9 +130,7 @@ ngx_int_t ngx_str_equal(ngx_str_t *s1, ngx_str_t *s2)
 char *ngx_http_kafka_main_conf_broker_add(ngx_http_kafka_main_conf_t *cf,
         ngx_str_t *broker)
 {
-    ngx_str_t   *value, *new_broker;
-
-    value = cf->broker_list->elts;
+    ngx_str_t   *new_broker;
 
     new_broker = ngx_array_push(cf->broker_list);
     if (new_broker == NULL) {
@@ -193,20 +191,17 @@ char *ngx_http_set_kafka(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
 char *ngx_http_set_kafka_broker_list(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
 {
     ngx_uint_t  i;
-    ngx_str_t  *value, *broker;
+    ngx_str_t  *value;
 
     ngx_http_kafka_main_conf_t *main_conf;
-    
+
     main_conf = conf;
     value = cf->args->elts;
 
     for (i = 1; i < cf->args->nelts; ++i) {
-        broker = ngx_array_push(main_conf->broker_list);
-        if (broker == NULL) {
+        if (ngx_http_kafka_main_conf_broker_add(main_conf, &value[i]) == NGX_CONF_ERROR) {
             return NGX_CONF_ERROR;
         }
-
-        *broker = value[i];
     }
 
     return NGX_OK;
@@ -422,5 +417,3 @@ void ngx_str_helper(ngx_str_t *str, ngx_str_op op)
             ngx_abort();
     }
 }
-
-

scripts/bootstrap

@@ -0,0 +1,78 @@
+#!/bin/bash
+
+set -o nounset
+set -o errexit
+
+DIR=$(pwd)
+BUILDDIR=$DIR/build
+SCRIPTDIR=$DIR/scripts
+NGINX_DIR=nginx
+LIBRDKAFKA_VERSION=0.9.0.99
+
+. $SCRIPTDIR/vars
+
+clean () {
+    rm -rf build vendor
+}
+
+setup_local_directories () {
+    if [ ! -d $BUILDDIR ]; then
+        mkdir $BUILDDIR > /dev/null 2>&1
+        mkdir $BUILDDIR/$NGINX_DIR > /dev/null 2>&1
+    fi
+
+    if [ ! -d "vendor" ]; then
+        mkdir vendor > /dev/null 2>&1
+    fi
+}
+
+install_nginx () {
+    if [ ! -d "vendor/nginx-$NGINX_VERSION" ]; then
+        pushd vendor > /dev/null 2>&1
+        curl -s -L -O "http://nginx.org/download/nginx-$NGINX_VERSION.tar.gz"
+        tar xzf "nginx-$NGINX_VERSION.tar.gz"
+        pushd "nginx-$NGINX_VERSION" > /dev/null 2>&1
+        ./configure                           \
+            --with-debug                      \
+            --prefix=$(pwd)/../../build/nginx \
+            --conf-path=conf/nginx.conf       \
+            --error-log-path=logs/error.log   \
+            --http-log-path=logs/access.log
+        make -j4
+        make install
+        popd > /dev/null 2>&1
+        popd > /dev/null 2>&1
+        ln -sf $(pwd)/nginx.conf $(pwd)/build/nginx/conf/nginx.conf
+    else
+        printf "NGINX already installed\n"
+    fi
+}
+
+install_librdkafka () {
+    if [ ! -d "vendor/librdkafka-$LIBRDKAFKA_VERSION" ]; then
+        pushd vendor > /dev/null 2>&1
+        curl -s -L -o "librdkafka-$LIBRDKAFKA_VERSION.tar.gz" "https://github.com/edenhill/librdkafka/archive/$LIBRDKAFKA_VERSION.tar.gz"
+        tar xzf "librdkafka-$LIBRDKAFKA_VERSION.tar.gz"
+        pushd "librdkafka-$LIBRDKAFKA_VERSION" > /dev/null 2>&1
+        ./configure                                \
+            --prefix=$(pwd)/../../build/librdkafka
+        make -j4
+        make install
+        popd > /dev/null 2>&1
+        popd > /dev/null 2>&1
+    else
+        printf "librdkafka already installed\n"
+    fi
+}
+
+if [[ "$#" -eq 1 ]]; then
+    if [[ "$1" == "clean" ]]; then
+        clean
+    else
+        echo "clean is the only option"
+    fi
+else
+    setup_local_directories
+    install_nginx
+    install_librdkafka
+fi

scripts/compile

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+SCRIPTDIR=$(pwd)/scripts
+echo $SCRIPTDIR/vars
+. $SCRIPTDIR/vars
+
+pushd "vendor"
+pushd "nginx-$NGINX_VERSION"
+CFLAGS="-g -O0" ./configure           \
+    --with-debug                      \
+    --prefix=$(pwd)/../../build/nginx \
+    --conf-path=conf/nginx.conf       \
+    --error-log-path=logs/error.log   \
+    --http-log-path=logs/access.log   \
+    --add-module=../../
+make -j4
+make install
+popd
+popd

scripts/vars

@@ -0,0 +1 @@
+NGINX_VERSION=1.9.15