Skip to content

Commit

Permalink
Merge branch 'release/v1.2.0'
Browse files Browse the repository at this point in the history
  • Loading branch information
oalam committed Sep 4, 2019
2 parents 60a1012 + 4d296ea commit 99e4172
Show file tree
Hide file tree
Showing 1,498 changed files with 538,860 additions and 6,812 deletions.
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
**/.idea/*

# generated documentation
logisland-core/logisland-framework/logisland-resources/src/main/resources/docs/
logisland-resources/src/main/resources/docs/

# Eclipse
**/.project
Expand Down
9 changes: 7 additions & 2 deletions .travis.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
sudo: required
dist: trusty

dist: trusty
#services:
# - docker

Expand Down Expand Up @@ -31,12 +33,14 @@ script:
# Build
# use travis_wait so it does not time_out after 10 minutes without output (unfortunately that seems to not work)
# use -q so there's not too much output for travis (4Mb max)
- mvn clean install -Pfull -q
- travis_wait mvn clean install -Pintegration-tests -q
# build assembly (there is currently missing jars in assembly when using mvn clean install...)
- mvn clean package -DskipTests -q
# Integrations tests
- sudo logisland-docker/src/integration-test/run-all-test.sh

jdk:
- oraclejdk8
- openjdk8

install:
- sudo sysctl -w vm.max_map_count=262144
Expand All @@ -45,6 +49,7 @@ notifications:
email:
- [email protected]
- [email protected]
- [email protected]

webhooks:
urls:
Expand Down
37 changes: 18 additions & 19 deletions README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -53,9 +53,9 @@ to build from the source just clone source and package with maven (logisland req
git clone https://github.com/Hurence/logisland.git
cd logisland
mvn clean install -Pfull
mvn clean package
the final package is available at `logisland-assembly/target/logisland-1.1.2-bin.tar.gz`
the final package is available at `logisland-assembly/target/logisland-1.2.0-bin.tar.gz`

You can also download the `latest release build <https://github.com/Hurence/logisland/releases>`_

Expand Down Expand Up @@ -88,9 +88,9 @@ But you should choose the Spark version that is compatible with your environment
curl -s http://d3kbcqa49mib13.cloudfront.net/spark-<spark-version>-bin-hadoop<hadoop-version>.tgz | tar -xz -C /usr/local/
export SPARK_HOME=/usr/local/spark-<spark-version>-bin-hadoop<hadoop-version>
# install Logisland 1.1.2
# install Logisland 1.2.0
curl -s https://github.com/Hurence/logisland/releases/download/v1.0.0-RC2/logisland-1.0.0-RC2-bin.tar.gz | tar -xz -C /usr/local/
cd /usr/local/logisland-1.1.2
cd /usr/local/logisland-1.2.0
# launch a logisland job
bin/logisland.sh --conf conf/index-apache-logs.yml
Expand Down Expand Up @@ -119,9 +119,9 @@ Launching logisland streaming apps is just easy as unarchiving logisland distrib

.. code-block:: sh
# install Logisland 1.1.2
curl -s https://github.com/Hurence/logisland/releases/download/v0.10.0/logisland-1.1.2-bin-hdp2.5.tar.gz | tar -xz -C /usr/local/
cd /usr/local/logisland-1.1.2
# install Logisland 1.2.0
curl -s https://github.com/Hurence/logisland/releases/download/v0.10.0/logisland-1.2.0-bin-hdp2.5.tar.gz | tar -xz -C /usr/local/
cd /usr/local/logisland-1.2.0
bin/logisland.sh --conf conf/index-apache-logs.yml
Expand All @@ -142,7 +142,7 @@ The first part is the `ProcessingEngine` configuration (here a Spark streaming e

.. code-block:: yaml
version: 1.1.2
version: 1.2.0
documentation: LogIsland job config file
engine:
component: com.hurence.logisland.engine.spark.KafkaStreamProcessingEngine
Expand Down Expand Up @@ -180,13 +180,12 @@ Then comes a list of `ControllerService` which are the shared components that in

.. code-block:: yaml
- controllerService: elasticsearch_service
component: com.hurence.logisland.service.elasticsearch.Elasticsearch_2_3_3_ClientService
- controllerService: datastore_service
component: com.hurence.logisland.service.elasticsearch.Elasticsearch_6_6_2_ClientService
type: service
documentation: elasticsearch service
configuration:
hosts: sandbox:9300
cluster.name: elasticsearch
hosts: sandbox:9200
batch.size: 5000
Then comes a list of `RecordStream`, each of them route the input batch of `Record` through a pipeline of `Processor`
Expand All @@ -213,7 +212,7 @@ to the output topic
kafka.topic.default.replicationFactor: 1
Then come the configurations of all the `Processor` pipeline. Each Record will go through these components.
Here we first parse raw apache logs and then we add those records to Elasticsearch. Pleas note that the ES processor makes
Here we first parse raw apache logs and then we add those records to Elasticsearch. Please note that the datastore processor makes
use of the previously defined ControllerService.

.. code-block:: yaml
Expand All @@ -230,16 +229,16 @@ use of the previously defined ControllerService.
value.fields: src_ip,identd,user,record_time,http_method,http_query,http_version,http_status,bytes_out
- processor: es_publisher
component: com.hurence.logisland.processor.elasticsearch.BulkAddElasticsearch
component: com.hurence.logisland.processor.datastore.BulkPut
type: processor
documentation: a processor that indexes processed events in elasticsearch
configuration:
elasticsearch.client.service: elasticsearch_service
default.index: logisland
datastore.client.service: datastore_service
default.collection: logisland
default.type: event
timebased.index: yesterday
es.index.field: search_index
es.type.field: record_type
timebased.collection: yesterday
collection.field: search_index
type.field: record_type
Expand Down
95 changes: 95 additions & 0 deletions bump_version.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
#!/usr/bin/env bash

# Usage: ./bump_version.sh <major|minor|patch> - Increments the relevant version part by one.
#
# Usage 2: ./bump_version.sh <version-from> <version-to>
# e.g: ./bump_version.sh 1.1.1 2.0

set -e

# Define which files to update and the pattern to look for
# $1 Current version
# $2 New version
function bump_files() {

for i in `find . -name "pom.xml"` ; do
bump $i "<version>$current_version<" "<version>$new_version<"
done
for i in `find . -name "*.yml"` ; do
bump $i "version: $current_version" "version: $new_version"
done
for i in `find . -name "*.rst"` ; do
bump $i "$current_version" "$new_version"
done
}

function bump() {
echo -n "Updating $1..."
tmp_file=$(mktemp)
rm -f "$tmp_file"
sed -i '' "s/$2/$3/1w $tmp_file" $1
if [ -s "$tmp_file" ]; then
echo "Done"
else
echo "Nothing to change"
fi
rm -f "$tmp_file"
}

function confirm() {
read -r -p "$@ [Y/n]: " confirm

case "$confirm" in
[Nn][Oo]|[Nn])
echo "Aborting."
exit
;;
esac
}

if [ "$1" == "" ]; then
echo >&2 "No 'from' version set. Aborting."
exit 1
fi

if [ "$1" == "major" ] || [ "$1" == "minor" ] || [ "$1" == "patch" ]; then
current_version=$(grep -Po '(?<="version": ")[^"]*' package.json)

IFS='.' read -a version_parts <<< "$current_version"

major=${version_parts[0]}
minor=${version_parts[1]}
patch=${version_parts[2]}

case "$1" in
"major")
major=$((major + 1))
minor=0
patch=0
;;
"minor")
minor=$((minor + 1))
patch=0
;;
"patch")
patch=$((patch + 1))
;;
esac
new_version="$major.$minor.$patch"
else
if [ "$2" == "" ]; then
echo >&2 "No 'to' version set. Aborting."
exit 1
fi
current_version="$1"
new_version="$2"
fi

if ! [[ "$new_version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
echo >&2 "'to' version doesn't look like a valid semver version tag (e.g: 1.2.3). Aborting."
exit 1
fi

confirm "Bump version number from $current_version to $new_version?"

bump_files "$current_version" "$new_version"
Loading

0 comments on commit 99e4172

Please sign in to comment.