From ddf7b2cc5b2328247084885ed9315178969c4095 Mon Sep 17 00:00:00 2001 From: Maxim Pogozhiy Date: Tue, 8 Feb 2022 13:16:06 +0700 Subject: [PATCH] Add sphinx compose and fix issue #16 --- .github/workflows/build.yml | 27 ++++++++----- Dockerfile.compose | 16 ++++++++ docker-compose-3.4.yml | 20 ++++++++++ go.mod | 6 +-- go.sum | 27 +------------ main.go | 77 +++++++++++++++++++++++++++++-------- tests/sphinx/Dockerfile | 48 +++++++++++++++++++++++ tests/sphinx/docs.xml | 24 ++++++++++++ tests/sphinx/run.sh | 3 ++ tests/sphinx/sphinx.conf | 39 +++++++++++++++++++ 10 files changed, 233 insertions(+), 54 deletions(-) create mode 100644 Dockerfile.compose create mode 100644 docker-compose-3.4.yml create mode 100644 tests/sphinx/Dockerfile create mode 100644 tests/sphinx/docs.xml create mode 100644 tests/sphinx/run.sh create mode 100644 tests/sphinx/sphinx.conf diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d803012..fbd639d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,18 +1,11 @@ name: "build-and-test" -on: - push: - branches: - - master - pull_request: - branches: - - master +on: [push] jobs: - build-and-tests: + build: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Set up Go uses: actions/setup-go@v2 with: @@ -38,3 +31,19 @@ jobs: - name: Go build run: go build . + +# tests: +# needs: build +# runs-on: ubuntu-latest +# strategy: +# matrix: +# sphinx-version: [4.5.4, 2.4.3, 4.4.0] +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: Go mod download +# run: go mod download +# +# - name: Test +# run: go test ./... diff --git a/Dockerfile.compose b/Dockerfile.compose new file mode 100644 index 0000000..31bf37e --- /dev/null +++ b/Dockerfile.compose @@ -0,0 +1,16 @@ +FROM golang:alpine as build + +RUN apk add git + +WORKDIR /app +COPY go.mod go.sum /app/ +RUN go mod download +COPY . . +RUN apk add alpine-sdk +RUN go build . + +FROM alpine:3.15 +RUN apk --no-cache add ca-certificates +COPY --from=build /app/sphinx_exporter /bin/ +EXPOSE 9247 +ENTRYPOINT ["/bin/sphinx_exporter"] diff --git a/docker-compose-3.4.yml b/docker-compose-3.4.yml new file mode 100644 index 0000000..87cdf8b --- /dev/null +++ b/docker-compose-3.4.yml @@ -0,0 +1,20 @@ +version: "3.9" +services: +# exporter: +# container_name: exporter +# command: +# - "--sphinx.address=sphinx" +# - "--sphinx.port=3306" +# build: +# context: . +# dockerfile: Dockerfile.compose +# ports: +# - "9247:9247" + sphinx: + container_name: sphinx + build: + context: tests/sphinx + ports: + - "3306:3306" + + diff --git a/go.mod b/go.mod index f7a9bb9..02f94f6 100644 --- a/go.mod +++ b/go.mod @@ -14,17 +14,15 @@ require ( gopkg.in/alecthomas/kingpin.v2 v2.2.6 ) +require github.com/Masterminds/semver v1.5.0 + require ( - github.com/BurntSushi/toml v0.4.1 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.1 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/sirupsen/logrus v1.6.0 // indirect - golang.org/x/mod v0.5.1 // indirect - golang.org/x/tools v0.1.8 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/protobuf v1.26.0-rc.1 // indirect - honnef.co/go/tools v0.2.2 // indirect ) diff --git a/go.sum b/go.sum index 70f52ba..cfa6dba 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,6 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw= -github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -100,28 +99,19 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38= -golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -132,24 +122,13 @@ golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 h1:JWgyZ1qgdTaF3N3oxC+MdTV7qvEEgHo3otj+HB5CM7Q= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.8 h1:P1HhGGuLW4aAclzjtmJdf0mJOjVUZUzOTqkAkWL+l6w= -golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -172,5 +151,3 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -honnef.co/go/tools v0.2.2 h1:MNh1AVMyVX23VUHE2O27jm6lNj3vjO5DexS4A1xvnzk= -honnef.co/go/tools v0.2.2/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY= diff --git a/main.go b/main.go index 27ab290..d0226f2 100644 --- a/main.go +++ b/main.go @@ -2,16 +2,18 @@ package main import ( "database/sql" + "github.com/Masterminds/semver" + "github.com/prometheus/common/version" "math" "net/http" "strconv" + "strings" "time" _ "github.com/go-sql-driver/mysql" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/common/log" - "github.com/prometheus/common/version" "gopkg.in/alecthomas/kingpin.v2" ) @@ -25,7 +27,8 @@ var ( // Exporter collects metrics from a searchd server. type Exporter struct { - sphinx string + sphinx string + version *semver.Version up *prometheus.Desc uptime *prometheus.Desc @@ -79,8 +82,36 @@ type Exporter struct { func NewExporter(server string, port string, timeout time.Duration) *Exporter { c := "@tcp(" + server + ":" + port + ")/" + db, err := sql.Open("mysql", c) + if err != nil { + log.Error(err) + } + defer db.Close() + + rows, err := db.Query("show variables") + if err != nil { + log.Fatal(err) + } + + variables := make(map[string]string) + + for rows.Next() { + var variableName, variableValue string + err = rows.Scan(&variableName, &variableValue) + if err != nil { + log.Error(err) + } + variables[variableName] = variableValue + } + + version, err := semver.NewVersion(strings.Split(variables["version"], " ")[0]) + if err != nil { + log.Fatal(err) + } + return &Exporter{ - sphinx: c, + sphinx: c, + version: version, up: prometheus.NewDesc( prometheus.BuildFQName(namespace, "", "up"), "Could the searchd server be reached.", @@ -595,20 +626,34 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) { threads := make(map[string][]string) for threads_rows.Next() { - var ( - tid, name, proto, state, host, connID, time, workTime, workTimeCPU, - thdEfficiency, jobsDone, lastJobTook, inIdle, info string - ) - - err = threads_rows.Scan( - &tid, &name, &proto, &state, &host, &connID, &time, &workTime, &workTimeCPU, - &thdEfficiency, &jobsDone, &lastJobTook, &inIdle, &info, - ) - if err != nil { - log.Error(err) - return + if e.version.Major() >= 4 { + var ( + tid, name, proto, state, host, connID, time, workTime, workTimeCPU, + thdEfficiency, jobsDone, lastJobTook, inIdle, info string + ) + + err = threads_rows.Scan( + &tid, &name, &proto, &state, &host, &connID, &time, &workTime, &workTimeCPU, + &thdEfficiency, &jobsDone, &lastJobTook, &inIdle, &info, + ) + if err != nil { + log.Error(err) + return + } + threads[state] = append(threads[state], tid) + } else { + var ( + tid, proto, state, time, info string + ) + err = threads_rows.Scan( + &tid, &proto, &state, &state, &time, &info, + ) + if err != nil { + log.Error(err) + return + } + threads[state] = append(threads[state], tid) } - threads[state] = append(threads[state], tid) } for threads_state, threads_times := range threads { diff --git a/tests/sphinx/Dockerfile b/tests/sphinx/Dockerfile new file mode 100644 index 0000000..e63462d --- /dev/null +++ b/tests/sphinx/Dockerfile @@ -0,0 +1,48 @@ +# Dockerfile for Sphinx SE +# https://hub.docker.com/_/alpine/ +FROM alpine:3.14 + +# https://sphinxsearch.com/blog/ +ENV SPHINX_VERSION 3.4.1-efbcc65 + +# install dependencies +RUN apk add --no-cache mariadb-connector-c-dev \ + postgresql-dev \ + wget + +# set up and expose directories +RUN mkdir -pv /opt/sphinx/log /opt/sphinx/index +VOLUME /opt/sphinx/index + +# http://sphinxsearch.com/downloads/sphinx-3.3.1-b72d67b-linux-amd64-musl.tar.gz +RUN wget http://sphinxsearch.com/files/sphinx-${SPHINX_VERSION}-linux-amd64-musl.tar.gz -O /tmp/sphinxsearch.tar.gz \ + && cd /opt/sphinx && tar -xf /tmp/sphinxsearch.tar.gz \ + && rm /tmp/sphinxsearch.tar.gz + +# point to sphinx binaries +ENV PATH "${PATH}:/opt/sphinx/sphinx-3.4.1/bin" +RUN indexer -v + +# redirect logs to stdout +RUN ln -sv /dev/stdout /opt/sphinx/log/query.log \ + && ln -sv /dev/stdout /opt/sphinx/log/searchd.log + +# expose TCP port +EXPOSE 3306 + +RUN mkdir -p /opt/sphinx/conf +RUN mkdir -p /opt/sphinx/tests + +COPY docs.xml /opt/sphinx/tests/ +COPY sphinx.conf /opt/sphinx/conf/ + +# allow custom config file to be passed +ARG SPHINX_CONFIG_FILE=/opt/sphinx/conf/sphinx.conf +ENV SPHINX_CONFIG_FILE ${SPHINX_CONFIG_FILE} + +# prepare a start script +RUN echo "exec searchd --nodetach --config \${SPHINX_CONFIG_FILE}" > /opt/sphinx/start.sh + +RUN /opt/sphinx/sphinx-3.4.1/bin/indexer --all --config "/opt/sphinx/conf/sphinx.conf" + +CMD sh /opt/sphinx/start.sh diff --git a/tests/sphinx/docs.xml b/tests/sphinx/docs.xml new file mode 100644 index 0000000..2c208a4 --- /dev/null +++ b/tests/sphinx/docs.xml @@ -0,0 +1,24 @@ + + + + + + + + + + + this is the main content entry +must be handled properly by xml parser lib]]> + 1012325463 + note how field/attr tags can be + in randomized order + some undeclared element + + + another subject + here comes another document, and i am given to understand, + that in-document field order must not matter, sir + 1012325467 + + diff --git a/tests/sphinx/run.sh b/tests/sphinx/run.sh new file mode 100644 index 0000000..93dc10e --- /dev/null +++ b/tests/sphinx/run.sh @@ -0,0 +1,3 @@ +#!/bin/sh +/opt/sphinx/sphinx-3.4.1/bin/indexer --all --config "/opt/sphinx/conf/sphinx.conf" +/opt/sphinx/start.sh diff --git a/tests/sphinx/sphinx.conf b/tests/sphinx/sphinx.conf new file mode 100644 index 0000000..72b7730 --- /dev/null +++ b/tests/sphinx/sphinx.conf @@ -0,0 +1,39 @@ +source xml +{ + type = xmlpipe2 + xmlpipe_fixup_utf8 = 1 + xmlpipe_command = cat /opt/sphinx/tests/docs.xml +} + +index test_index +{ + source = xml + path = /opt/sphinx/index/test_index + + # @see http://sphinxsearch.com/docs/2.0.1/conf-blend-chars.html + blend_chars = - + + # CALL SUGGEST + min_infix_len = 3 + + # wsparcie dla polskich znaków + # @see http://sphinxsearch.com/wiki/doku.php?id=charset_tables#polish + charset_table = 0..9, A..Z->a..z, a..z, U+0143->U+0144, U+0104->U+0105, U+0106->U+0107, U+0118->U+0119, U+0141->U+0142, U+00D3->U+00F3, U+015A->U+015B, U+0179->U+017A, U+017B->U+017C, U+0105, U+0107, U+0119, U+0142, U+00F3, U+015B, U+017A, U+017C, U+0144 +} + +indexer +{ + mem_limit = 256M +} + +searchd +{ + listen = 3306:mysql41 + log = /opt/sphinx/log/searchd.log + query_log = /opt/sphinx/log/query.log + query_log_format = sphinxql + pid_file = /opt/sphinx/searchd.pid + + # binlogs + binlog_path = # disable logging +}