From c5b74538db7ff312db70dd8c42eb5f323cf0a5fc Mon Sep 17 00:00:00 2001 From: Pablo Murillo Date: Tue, 19 Dec 2023 17:55:45 +0000 Subject: [PATCH] remove opentsdb and hbase --- .env.example | 1 - .../topology.properties.tmpl | 5 - .../docker-compose/docker-compose.tmpl | 53 +----- confd/templates/makefile/makefile.tmpl | 4 +- .../opentsdb-topology/opentsdb-topology.tmpl | 10 -- confd/vars/main.yaml | 2 - docker/hbase/Dockerfile | 35 ---- docker/hbase/hbase-conf/hbase-env.sh | 137 -------------- docker/hbase/hbase-conf/hbase-site.xml | 72 -------- docker/hbase/hbase-conf/start-hbase | 6 - docker/opentsdb/Dockerfile | 33 ---- docker/opentsdb/app/create_tables.py | 57 ------ docker/opentsdb/app/start-opentsdb | 6 - docker/opentsdb/app/wait-for-it.sh | 169 ------------------ docker/opentsdb/conf/opentsdb.conf | 71 -------- src-java/opentsdb-topology/README.md | 9 +- 16 files changed, 7 insertions(+), 663 deletions(-) delete mode 100644 docker/hbase/Dockerfile delete mode 100644 docker/hbase/hbase-conf/hbase-env.sh delete mode 100644 docker/hbase/hbase-conf/hbase-site.xml delete mode 100755 docker/hbase/hbase-conf/start-hbase delete mode 100644 docker/opentsdb/Dockerfile delete mode 100644 docker/opentsdb/app/create_tables.py delete mode 100644 docker/opentsdb/app/start-opentsdb delete mode 100644 docker/opentsdb/app/wait-for-it.sh delete mode 100644 docker/opentsdb/conf/opentsdb.conf diff --git a/.env.example b/.env.example index ec0a5dcaf02..4f53d825a0e 100644 --- a/.env.example +++ b/.env.example @@ -7,7 +7,6 @@ LAB_API_MEM_LIMIT=512m NEO4J_MEM_LIMIT=2g ZOOKEEPER_MEM_LIMIT=2g KAFKA_MEM_LIMIT=2g -HBASE_MEM_LIMIT=2g FL_1_MEM_LIMIT=2g FL_2_MEM_LIMIT=2g FL_STATS_MEM_LIMIT=2g diff --git a/confd/templates/base-storm-topology/topology.properties.tmpl b/confd/templates/base-storm-topology/topology.properties.tmpl index 4a6fc2972cc..a0386aadacc 100644 --- a/confd/templates/base-storm-topology/topology.properties.tmpl +++ b/confd/templates/base-storm-topology/topology.properties.tmpl @@ -54,11 +54,6 @@ zookeeper.reconnect_delay={{ getv "/kilda_zookeeper_reconnect_delay_ms"}} persistence.implementation.default = {{ getv "/kilda_persistence_default_implementation" }} persistence.implementation.area.history = {{ getv "/kilda_persistence_history_implementation" }} -{{if getv "/kilda_opentsdb_hosts"}} -opentsdb.target.opentsdb = http://{{ getv "/kilda_opentsdb_hosts" }}:{{ getv "/kilda_opentsdb_port" }} -{{else}} -opentsdb.target.opentsdb = -{{end}} {{if getv "/kilda_victoriametrics_host"}} opentsdb.target.victoriametrics = http://{{ getv "/kilda_victoriametrics_host" }}:{{ getv "/kilda_victoriametrics_write_port" }}{{ getv "/kilda_victoriametrics_path" }} {{else}} diff --git a/confd/templates/docker-compose/docker-compose.tmpl b/confd/templates/docker-compose/docker-compose.tmpl index fae1ded4d98..909595e1d8c 100644 --- a/confd/templates/docker-compose/docker-compose.tmpl +++ b/confd/templates/docker-compose/docker-compose.tmpl @@ -271,27 +271,6 @@ services: environment: - WFM_TOPOLOGIES_MODE - hbase: - container_name: hbase - hostname: hbase.pendev - image: kilda/hbase:latest - command: /opt/hbase/bin/start-hbase - volumes: - - hbase_data:/data/hbase - depends_on: - zookeeper: - condition: service_healthy - healthcheck: - test: ["CMD-SHELL", "jps | grep --silent HMaster"] - interval: 30s - timeout: 10s - retries: 3 - networks: - default: - aliases: - - hbase.pendev - mem_limit: ${HBASE_MEM_LIMIT:-2g} - storm-nimbus: container_name: storm-nimbus hostname: nimbus.pendev @@ -300,10 +279,6 @@ services: depends_on: zookeeper: condition: service_healthy -{{if not (exists "/no_opentsdb")}} - opentsdb: - condition: service_started -{{end}} {{if not (exists "/no_victoriametrics")}} victoriametrics: condition: service_started @@ -358,10 +333,6 @@ services: logstash: condition: service_healthy {{end}} -{{if not (exists "/no_opentsdb")}} - opentsdb: - condition: service_started -{{end}} {{if not (exists "/no_victoriametrics")}} victoriametrics: condition: service_started @@ -558,25 +529,6 @@ services: - grpc-speaker.pendev mem_limit: ${GRPC_MEM_LIMIT:-2g} {{end}} -{{if not (exists "/no_opentsdb")}} - opentsdb: - container_name: opentsdb - hostname: opentsdb.pendev - image: kilda/opentsdb:latest - command: /app/wait-for-it.sh -t 120 -h hbase.pendev -p 9090 -- /app/start-opentsdb - depends_on: - zookeeper: - condition: service_healthy - hbase: - condition: service_healthy - ports: - - "4242:4242" - networks: - default: - aliases: - - opentsdb.pendev - mem_limit: ${OTSDB_MEM_LIMIT:-2g} -{{end}} {{if not (exists "/no_victoriametrics")}} victoriametrics: container_name: victoriametrics @@ -606,9 +558,7 @@ services: depends_on: northbound: condition: service_started -{{if not (exists "/no_opentsdb")}} opentsdb: - condition: service_started -{{end}}{{if not (exists "/no_victoriametrics")}} victoriametrics: +{{if not (exists "/no_victoriametrics")}} victoriametrics: condition: service_started {{end}} networks: default: @@ -809,7 +759,6 @@ volumes: zookeeper_data: kafka_data: app_server_data: - hbase_data: odb1_data: sql_data: {{if not (exists "/single_orientdb")}} odb2_data: diff --git a/confd/templates/makefile/makefile.tmpl b/confd/templates/makefile/makefile.tmpl index 7d7024d4e72..580ff2e51cd 100644 --- a/confd/templates/makefile/makefile.tmpl +++ b/confd/templates/makefile/makefile.tmpl @@ -8,10 +8,8 @@ build-base: {{if not (exists "/no_grpc_stub")}}build-grpc-stub {{end}}build-lock docker build -t kilda/base-ubuntu:latest docker/base/kilda-base-ubuntu/ docker build -t kilda/zookeeper:latest docker/zookeeper docker build -t kilda/kafka:latest docker/kafka - docker build -t kilda/hbase:latest docker/hbase docker build -t kilda/storm:latest docker/storm -{{if not (exists "/no_opentsdb")}} docker build -t kilda/opentsdb:latest docker/opentsdb -{{end}}{{if not (exists "/no_logstash")}} docker build -t kilda/logstash:latest docker/logstash +{{if not (exists "/no_logstash")}} docker build -t kilda/logstash:latest docker/logstash {{end}}{{if not (exists "/no_elasticsearch")}} docker build -t kilda/elasticsearch:latest docker/elasticsearch {{end}} $(MAKE) -C src-python/lab-service find-python-requirements docker build -t kilda/base-lab-service:latest docker/base/kilda-base-lab-service/ {{if (exists "/ovs_vxlan")}} --build-arg=OVS_VERSION=kilda.v2.15.1.3{{end}} diff --git a/confd/templates/opentsdb-topology/opentsdb-topology.tmpl b/confd/templates/opentsdb-topology/opentsdb-topology.tmpl index 576dafbcdfa..3ff635a8aac 100644 --- a/confd/templates/opentsdb-topology/opentsdb-topology.tmpl +++ b/confd/templates/opentsdb-topology/opentsdb-topology.tmpl @@ -9,8 +9,6 @@ config: # spout definitions spouts: - - id: "input.opentsdb" - parallelism: {{ getv "/kilda_opentsdb_num_spouts" }} - id: "input.victoriametrics" parallelism: {{ getv "/kilda_opentsdb_num_spouts" }} - id: "zookeeper.spout" @@ -18,19 +16,11 @@ spouts: # bolt definitions bolts: - - id: "DatapointParseBolt.opentsdb" - parallelism: {{ getv "/kilda_opentsdb_num_datapointparserbolt" }} - numTasks: {{ getv "/kilda_opentsdb_workers_datapointparserbolt" }} - id: "DatapointParseBolt.victoriametrics" parallelism: {{ getv "/kilda_opentsdb_num_datapointparserbolt" }} numTasks: {{ getv "/kilda_opentsdb_workers_datapointparserbolt" }} - - id: "OpenTsdbFilterBolt.opentsdb" - parallelism: {{ getv "/kilda_opentsdb_num_opentsdbfilterbolt" }} - id: "OpenTsdbFilterBolt.victoriametrics" parallelism: {{ getv "/kilda_opentsdb_num_opentsdbfilterbolt" }} - - id: "output.opentsdb" - parallelism: {{ getv "/kilda_opentsdb_num_output_bolt" }} - numTasks: {{ getv "/kilda_opentsdb_tasks_output_bolt" }} - id: "output.victoriametrics" parallelism: {{ getv "/kilda_opentsdb_num_output_bolt" }} numTasks: {{ getv "/kilda_opentsdb_tasks_output_bolt" }} diff --git a/confd/vars/main.yaml b/confd/vars/main.yaml index 871838444e1..af367d6db6b 100644 --- a/confd/vars/main.yaml +++ b/confd/vars/main.yaml @@ -18,8 +18,6 @@ kilda_zookeeper_state_root: "kilda" kilda_zookeeper_reconnect_delay_ms: 100 kilda_zookeeper_reconnect_delay_ms_server42: 10000 kilda_zookeeper_reconnect_delay_ms_speaker: 1000 -kilda_opentsdb_hosts: "opentsdb.pendev" -kilda_opentsdb_port: "4242" kilda_victoriametrics_host: "victoriametrics.pendev" kilda_victoriametrics_write_port: "4242" kilda_victoriametrics_read_port: "8428" diff --git a/docker/hbase/Dockerfile b/docker/hbase/Dockerfile deleted file mode 100644 index 1bd77de317c..00000000000 --- a/docker/hbase/Dockerfile +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2017 Telstra Open Source -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -ARG base_image=kilda/base-ubuntu -FROM ${base_image} - -ENV PACKAGE hbase-1.2.4 - -WORKDIR /tmp/ - -RUN wget -q https://archive.apache.org/dist/hbase/1.2.4/${PACKAGE}-bin.tar.gz \ - && wget -q https://archive.apache.org/dist/hbase/1.2.4/${PACKAGE}-bin.tar.gz.md5 \ - && sed 's/\ //g' ${PACKAGE}-bin.tar.gz.md5 > $PACKAGE.tmp.md5 \ - && awk -F ":" '{print $2 " " $1}' $PACKAGE.tmp.md5 > ${PACKAGE}-bin.tar.gz.md5 \ - && md5sum -c ${PACKAGE}-bin.tar.gz.md5 \ - && tar -xzf ${PACKAGE}-bin.tar.gz --directory /opt/ \ - && ln -s /opt/$PACKAGE /opt/hbase \ - && rm -rfv /tmp/* - -WORKDIR /opt/hbase/ - -COPY hbase-conf/hbase-env.sh hbase-conf/hbase-site.xml /opt/hbase/conf/ -COPY hbase-conf/start-hbase /opt/hbase/bin/start-hbase diff --git a/docker/hbase/hbase-conf/hbase-env.sh b/docker/hbase/hbase-conf/hbase-env.sh deleted file mode 100644 index 171a5ecbb9c..00000000000 --- a/docker/hbase/hbase-conf/hbase-env.sh +++ /dev/null @@ -1,137 +0,0 @@ -# -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -# Set environment variables here. - -# This script sets variables multiple times over the course of starting an hbase process, -# so try to keep things idempotent unless you want to take an even deeper look -# into the startup scripts (bin/hbase, etc.) - -# The java implementation to use. Java 1.7+ required. -export JAVA_HOME=/usr/ - -# Extra Java CLASSPATH elements. Optional. -# export HBASE_CLASSPATH= - -# The maximum amount of heap to use. Default is left to JVM default. -# export HBASE_HEAPSIZE=1G - -# Uncomment below if you intend to use off heap cache. For example, to allocate 8G of -# offheap, set the value to "8G". -# export HBASE_OFFHEAPSIZE=1G - -# Extra Java runtime options. -# Below are what we set by default. May only work with SUN JVM. -# For more on why as well as other possible settings, -# see http://wiki.apache.org/hadoop/PerformanceTuning -export HBASE_OPTS="-XX:+UseConcMarkSweepGC -XX:+PrintFlagsFinal -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap" - -# Configure PermSize. Only needed in JDK7. You can safely remove it for JDK8+ -#export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -XX:PermSize=128m -XX:MaxPermSize=128m" -#export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -XX:PermSize=128m -XX:MaxPermSize=128m" - -# Uncomment one of the below three options to enable java garbage collection logging for the server-side processes. - -# This enables basic gc logging to the .out file. -# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps" - -# This enables basic gc logging to its own file. -# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR . -# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:" - -# This enables basic GC logging to its own file with automatic log rolling. Only applies to jdk 1.6.0_34+ and 1.7.0_2+. -# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR . -# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc: -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=1 -XX:GCLogFileSize=512M" - -# Uncomment one of the below three options to enable java garbage collection logging for the client processes. - -# This enables basic gc logging to the .out file. -# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps" - -# This enables basic gc logging to its own file. -# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR . -# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:" - -# This enables basic GC logging to its own file with automatic log rolling. Only applies to jdk 1.6.0_34+ and 1.7.0_2+. -# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR . -# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc: -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=1 -XX:GCLogFileSize=512M" - -# See the package documentation for org.apache.hadoop.hbase.io.hfile for other configurations -# needed setting up off-heap block caching. - -# Uncomment and adjust to enable JMX exporting -# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. -# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html -# NOTE: HBase provides an alternative JMX implementation to fix the random ports issue, please see JMX -# section in HBase Reference Guide for instructions. - -# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" -# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10101" -# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10102" -# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103" -# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104" -# export HBASE_REST_OPTS="$HBASE_REST_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10105" - -# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. -# export HBASE_REGIONSERVERS=${HBASE_HOME}/conf/regionservers - -# Uncomment and adjust to keep all the Region Server pages mapped to be memory resident -#HBASE_REGIONSERVER_MLOCK=true -#HBASE_REGIONSERVER_UID="hbase" - -# File naming hosts on which backup HMaster will run. $HBASE_HOME/conf/backup-masters by default. -# export HBASE_BACKUP_MASTERS=${HBASE_HOME}/conf/backup-masters - -# Extra ssh options. Empty by default. -# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR" - -# Where log files are stored. $HBASE_HOME/logs by default. -# export HBASE_LOG_DIR=${HBASE_HOME}/logs - -# Enable remote JDWP debugging of major HBase processes. Meant for Core Developers -# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8070" -# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8071" -# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8072" -# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8073" - -# A string representing this instance of hbase. $USER by default. -# export HBASE_IDENT_STRING=$USER - -# The scheduling priority for daemon processes. See 'man nice'. -# export HBASE_NICENESS=10 - -# The directory where pid files are stored. /tmp by default. -# export HBASE_PID_DIR=/var/hadoop/pids - -# Seconds to sleep between slave commands. Unset by default. This -# can be useful in large clusters, where, e.g., slave rsyncs can -# otherwise arrive faster than the master can service them. -# export HBASE_SLAVE_SLEEP=0.1 - -# Tell HBase whether it should manage it's own instance of Zookeeper or not. -export HBASE_MANAGES_ZK=false - -# The default log rolling policy is RFA, where the log file is rolled as per the size defined for the -# RFA appender. Please refer to the log4j.properties file to see more details on this appender. -# In case one needs to do log rolling on a date change, one should set the environment property -# HBASE_ROOT_LOGGER to ",DRFA". -# For example: -# HBASE_ROOT_LOGGER=INFO,DRFA -# The reason for changing default to RFA is to avoid the boundary case of filling out disk space as -# DRFA doesn't put any cap on the log size. Please refer to HBase-5655 for more context. diff --git a/docker/hbase/hbase-conf/hbase-site.xml b/docker/hbase/hbase-conf/hbase-site.xml deleted file mode 100644 index c04ab64f877..00000000000 --- a/docker/hbase/hbase-conf/hbase-site.xml +++ /dev/null @@ -1,72 +0,0 @@ - - - - - - hbase.cluster.distributed - true - - - hbase.rootdir - file:///data/hbase - - - hbase.master.port - 60000 - - - hbase.master.info.port - 60010 - - - hbase.regionserver.port - 60020 - - - hbase.regionserver.info.port - 60030 - - - hbase.regionserver.ipc.address - 0.0.0.0 - - - hbase.rest.port - 8070 - - - hbase.rest.info.port - 8090 - - - hbase.zookeeper.quorum - zookeeper.pendev:2181 - - - hbase.thrift.port - 9070 - - - hbase.thrift.info.port - 9080 - - diff --git a/docker/hbase/hbase-conf/start-hbase b/docker/hbase/hbase-conf/start-hbase deleted file mode 100755 index 7035875be1b..00000000000 --- a/docker/hbase/hbase-conf/start-hbase +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -/opt/hbase/bin/hbase-daemon.sh start rest -/opt/hbase/bin/hbase-daemon.sh start thrift -/opt/hbase/bin/hbase regionserver start > logregion.log 2>&1 & -/opt/hbase/bin/hbase master start --localRegionServers=0 diff --git a/docker/opentsdb/Dockerfile b/docker/opentsdb/Dockerfile deleted file mode 100644 index d0817c82585..00000000000 --- a/docker/opentsdb/Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2017 Telstra Open Source -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -ARG base_image=kilda/hbase -FROM ${base_image} - -WORKDIR /tmp/ - -COPY app /app - -RUN wget -q https://github.com/OpenTSDB/opentsdb/releases/download/v2.3.0/opentsdb-2.3.0_all.deb \ - && apt-get update -q \ - && apt-get install -yq --no-install-recommends \ - gnuplot \ - gcc \ - && python3 -m pip install happybase==1.2.0 \ - && dpkg -i opentsdb-2.3.0_all.deb \ - && chmod 777 /app/* \ - && rm -rfv /var/lib/apt/lists/* /tmp/* /var/tmp/* - -COPY conf/opentsdb.conf /etc/opentsdb/opentsdb.conf diff --git a/docker/opentsdb/app/create_tables.py b/docker/opentsdb/app/create_tables.py deleted file mode 100644 index d83db8b931e..00000000000 --- a/docker/opentsdb/app/create_tables.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/python -# Copyright 2017 Telstra Open Source -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -import happybase -import sys - -host = 'hbase.pendev' -port = 9090 - -new_tables = ['tsdb', 'tsdb-uid', 'tsdb-tree', 'tsdb-meta'] - -connection = happybase.Connection(host=host, port=port) -existing_tables = connection.tables() - - -def create_table(table): - if (table == 'tsdb') or (table == 'tsdb-tree'): - families = {'t': dict(max_versions=1, compression='none', - bloom_filter_type='ROW')} - elif table == 'tsdb-uid': - families = {'id': dict(compression='none', bloom_filter_type='ROW'), - 'name': dict(compression='none', bloom_filter_type='ROW')} - elif table == 'tsdb-meta': - families = {'name': dict(compression='none', bloom_filter_type='ROW')} - else: - sys.exit("Unknown table {} was requested.".format(table)) - - print("Creating %s" % table) - connection.create_table(table, families) - if bytes(table, 'utf-8') not in connection.tables(): - sys.exit("Could not create {}".format(table)) - - -for table in new_tables[:]: - if table in existing_tables: - print("%s exist" % table) - new_tables.remove(table) - -if len(new_tables) > 0: - for table in new_tables: - create_table(table) -else: - print("All OpenTSDB tables already created") diff --git a/docker/opentsdb/app/start-opentsdb b/docker/opentsdb/app/start-opentsdb deleted file mode 100644 index 0dcca9bc857..00000000000 --- a/docker/opentsdb/app/start-opentsdb +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -echo "checking tables in hbase" -python3 /app/create_tables.py - -echo "starting opentsdb" -/usr/share/opentsdb/bin/tsdb tsd --port=4242 --staticroot=/usr/share/opentsdb/static --cachedir=/tmp --auto-metric diff --git a/docker/opentsdb/app/wait-for-it.sh b/docker/opentsdb/app/wait-for-it.sh deleted file mode 100644 index 3251e3ea017..00000000000 --- a/docker/opentsdb/app/wait-for-it.sh +++ /dev/null @@ -1,169 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2017 Telstra Open Source -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Use this script to test if a given TCP host/port are available - -cmdname=$(basename $0) - -echoerr() { if [[ $QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } - -usage() -{ - cat << USAGE >&2 -Usage: - $cmdname host:port [-s] [-t timeout] [-- command args] - -h HOST | --host=HOST Host or IP under test - -p PORT | --port=PORT TCP port under test - Alternatively, you specify the host and port as host:port - -s | --strict Only execute subcommand if the test succeeds - -q | --quiet Don't output any status messages - -t TIMEOUT | --timeout=TIMEOUT - Timeout in seconds, zero for no timeout - -- COMMAND ARGS Execute command with args after the test finishes -USAGE - exit 1 -} -wait_for() -{ - if [[ $TIMEOUT -gt 0 ]]; then - echoerr "$cmdname: waiting $TIMEOUT seconds for $HOST:$PORT" - else - echoerr "$cmdname: waiting for $HOST:$PORT without a timeout" - fi - start_ts=$(date +%s) - while : - do - (echo > /dev/tcp/$HOST/$PORT) >/dev/null 2>&1 - result=$? - if [[ $result -eq 0 ]]; then - end_ts=$(date +%s) - echoerr "$cmdname: $HOST:$PORT is available after $((end_ts - start_ts)) seconds" - break - fi - sleep 1 - done - return $result -} -wait_for_wrapper() -{ - # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 - if [[ $QUIET -eq 1 ]]; then - timeout $TIMEOUT $0 --quiet --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & - else - timeout $TIMEOUT $0 --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & - fi - PID=$! - trap "kill -INT -$PID" INT - wait $PID - RESULT=$? - if [[ $RESULT -ne 0 ]]; then - echoerr "$cmdname: timeout occurred after waiting $TIMEOUT seconds for $HOST:$PORT" - fi - return $RESULT -} -# process arguments -while [[ $# -gt 0 ]] -do - case "$1" in - *:* ) - hostport=(${1//:/ }) - HOST=${hostport[0]} - PORT=${hostport[1]} - shift 1 - ;; - --child) - CHILD=1 - shift 1 - ;; - -q | --quiet) - QUIET=1 - shift 1 - ;; - -s | --strict) - STRICT=1 - shift 1 - ;; - -h) - HOST="$2" - if [[ $HOST == "" ]]; then break; fi - shift 2 - ;; - --host=*) - HOST="${1#*=}" - shift 1 - ;; - -p) - PORT="$2" - if [[ $PORT == "" ]]; then break; fi - shift 2 - ;; - --port=*) - PORT="${1#*=}" - shift 1 - ;; - -t) - TIMEOUT="$2" - if [[ $TIMEOUT == "" ]]; then break; fi - shift 2 - ;; - --timeout=*) - TIMEOUT="${1#*=}" - shift 1 - ;; - --) - shift - CLI="$@" - break - ;; - --help) - usage - ;; - *) - echoerr "Unknown argument: $1" - usage - ;; - esac -done -if [[ "$HOST" == "" || "$PORT" == "" ]]; then - echoerr "Error: you need to provide a host and port to test." - usage -fi -TIMEOUT=${TIMEOUT:-15} -STRICT=${STRICT:-0} -CHILD=${CHILD:-0} -QUIET=${QUIET:-0} -if [[ $CHILD -gt 0 ]]; then - wait_for - RESULT=$? - exit $RESULT -else - if [[ $TIMEOUT -gt 0 ]]; then - wait_for_wrapper - RESULT=$? - else - wait_for - RESULT=$? - fi -fi -if [[ $CLI != "" ]]; then - if [[ $RESULT -ne 0 && $STRICT -eq 1 ]]; then - echoerr "$cmdname: strict mode, refusing to execute subprocess" - exit $RESULT - fi - exec $CLI -else - exit $RESULT -fi diff --git a/docker/opentsdb/conf/opentsdb.conf b/docker/opentsdb/conf/opentsdb.conf deleted file mode 100644 index 15fd1041add..00000000000 --- a/docker/opentsdb/conf/opentsdb.conf +++ /dev/null @@ -1,71 +0,0 @@ -# --------- NETWORK ---------- -# The TCP port TSD should use for communications -# *** REQUIRED *** -tsd.network.port = 4242 - -# The IPv4 network address to bind to, defaults to all addresses -# tsd.network.bind = 0.0.0.0 - -# Disable Nagel's algorithm. Default is True -#tsd.network.tcp_no_delay = true - -# Determines whether or not to send keepalive packets to peers, default -# is True -#tsd.network.keep_alive = true - -# Determines if the same socket should be used for new connections, default -# is True -#tsd.network.reuse_address = true - -# Number of worker threads dedicated to Netty, defaults to # of CPUs * 2 -#tsd.network.worker_threads = 8 - -# Whether or not to use NIO or tradditional blocking IO, defaults to True -#tsd.network.async_io = true - -# ----------- HTTP ----------- -# The location of static files for the HTTP GUI interface. -# *** REQUIRED *** -tsd.http.staticroot = /usr/share/opentsdb/static/ - -# Where TSD should write it's cache files to -# *** REQUIRED *** -tsd.http.cachedir = /tmp/opentsdb - -# --------- CORE ---------- -# Whether or not to automatically create UIDs for new metric types, default -# is False -tsd.core.auto_create_metrics = true -tsd.core.auto_create_tagks = true -tsd.core.auto_create_tagvs = true - -# Full path to a directory containing plugins for OpenTSDB -tsd.core.plugin_path = /usr/share/opentsdb/plugins - -# --------- STORAGE ---------- -# Whether or not to enable data compaction in HBase, default is True -#tsd.storage.enable_compaction = true - -tsd.storage.fix_duplicates=true - -# How often, in milliseconds, to flush the data point queue to storage, -# default is 1,000 -# tsd.storage.flush_interval = 1000 - -# Name of the HBase table where data points are stored, default is "tsdb" -#tsd.storage.hbase.data_table = tsdb - -# Name of the HBase table where UID information is stored, default is "tsdb-uid"#tsd.storage.hbase.uid_table = tsdb-uid - -# Path under which the znode for the -ROOT- region is located, default is "/hbase" -#tsd.storage.hbase.zk_basedir = /hbase_unsecure - -# A comma separated list of Zookeeper hosts to connect to, with or without -# port specifiers, default is "localhost" -tsd.storage.hbase.zk_quorum = zookeeper.pendev:2181 - -# Whether or not to enable incoming chunk support for the HTTP RPC -tsd.http.request.enable_chunked = true -tsd.http.request.max_chunk = 16384 - -tsd.storage.max_tags = 12 diff --git a/src-java/opentsdb-topology/README.md b/src-java/opentsdb-topology/README.md index a6ab50e8d26..fe78355aa51 100644 --- a/src-java/opentsdb-topology/README.md +++ b/src-java/opentsdb-topology/README.md @@ -1,9 +1,10 @@ # WFM (WorkFlow Manager) - OpenTSDB Topology -This sub-project holds the Storm topology "OpenTSDB" that is used to submit -the statistics from OpenKilda to OpenTSDB. +This sub-project holds the Storm topology "OpenTSDB" that is used to submit +the statistics from OpenKilda to OpenTSDB or another TSDB with writting OpenTSDB +capabilities like Victoria Metrics. -## Deployment, Configuration, Testing and Debugging tips +## Deployment, Configuration, Testing and Debugging tips The guidelines, recommendations and commands from `base-topology/README.md` -are applicable to this topology. \ No newline at end of file +are applicable to this topology. \ No newline at end of file