From 1fe269d629ae180393739144c74089606ebaa007 Mon Sep 17 00:00:00 2001 From: Prometheus-collab Date: Tue, 17 Sep 2024 16:04:49 +0800 Subject: [PATCH 1/4] Providing a Common MQTT Mapper Signed-off-by: Prometheus-collab --- .../kubeedge-v1.17.0/mqtt-mapper/Dockerfile | 20 + .../mqtt-mapper/Dockerfile_nostream | 20 + .../mqtt-mapper/Dockerfile_stream | 35 ++ mappers/kubeedge-v1.17.0/mqtt-mapper/Makefile | 34 ++ .../kubeedge-v1.17.0/mqtt-mapper/cmd/main.go | 60 +++ .../kubeedge-v1.17.0/mqtt-mapper/config.yaml | 9 + .../data/dbmethod/influxdb2/client.go | 76 +++ .../data/dbmethod/influxdb2/handler.go | 73 +++ .../mqtt-mapper/data/dbmethod/mysql/client.go | 108 ++++ .../data/dbmethod/mysql/handler.go | 73 +++ .../mqtt-mapper/data/dbmethod/redis/client.go | 119 +++++ .../data/dbmethod/redis/handler.go | 74 +++ .../data/dbmethod/tdengine/client.go | 156 ++++++ .../data/dbmethod/tdengine/handler.go | 74 +++ .../mqtt-mapper/data/publish/http/client.go | 73 +++ .../mqtt-mapper/data/publish/mqtt/client.go | 63 +++ .../mqtt-mapper/data/stream/handler.go | 67 +++ .../mqtt-mapper/data/stream/img.go | 243 +++++++++ .../mqtt-mapper/data/stream/video.go | 142 +++++ .../mqtt-mapper/device/device.go | 493 ++++++++++++++++++ .../mqtt-mapper/device/devicestatus.go | 69 +++ .../mqtt-mapper/device/devicetwin.go | 108 ++++ .../mqtt-mapper/driver/devicetype.go | 80 +++ .../mqtt-mapper/driver/driver.go | 375 +++++++++++++ .../mqtt-mapper/driver/messagequeue.go | 60 +++ mappers/kubeedge-v1.17.0/mqtt-mapper/go.mod | 41 ++ mappers/kubeedge-v1.17.0/mqtt-mapper/go.sum | 105 ++++ .../mqtt-mapper/hack/make-rules/mapper.sh | 159 ++++++ .../mqtt-mapper/resource/configmap.yaml | 15 + .../mqtt-mapper/resource/deployment.yaml | 51 ++ 30 files changed, 3075 insertions(+) create mode 100644 mappers/kubeedge-v1.17.0/mqtt-mapper/Dockerfile create mode 100644 mappers/kubeedge-v1.17.0/mqtt-mapper/Dockerfile_nostream create mode 100644 mappers/kubeedge-v1.17.0/mqtt-mapper/Dockerfile_stream create mode 100644 mappers/kubeedge-v1.17.0/mqtt-mapper/Makefile create mode 100644 mappers/kubeedge-v1.17.0/mqtt-mapper/cmd/main.go create mode 100644 mappers/kubeedge-v1.17.0/mqtt-mapper/config.yaml create mode 100644 mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/influxdb2/client.go create mode 100644 mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/influxdb2/handler.go create mode 100644 mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/mysql/client.go create mode 100644 mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/mysql/handler.go create mode 100644 mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/redis/client.go create mode 100644 mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/redis/handler.go create mode 100644 mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/tdengine/client.go create mode 100644 mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/tdengine/handler.go create mode 100644 mappers/kubeedge-v1.17.0/mqtt-mapper/data/publish/http/client.go create mode 100644 mappers/kubeedge-v1.17.0/mqtt-mapper/data/publish/mqtt/client.go create mode 100644 mappers/kubeedge-v1.17.0/mqtt-mapper/data/stream/handler.go create mode 100644 mappers/kubeedge-v1.17.0/mqtt-mapper/data/stream/img.go create mode 100644 mappers/kubeedge-v1.17.0/mqtt-mapper/data/stream/video.go create mode 100644 mappers/kubeedge-v1.17.0/mqtt-mapper/device/device.go create mode 100644 mappers/kubeedge-v1.17.0/mqtt-mapper/device/devicestatus.go create mode 100644 mappers/kubeedge-v1.17.0/mqtt-mapper/device/devicetwin.go create mode 100644 mappers/kubeedge-v1.17.0/mqtt-mapper/driver/devicetype.go create mode 100644 mappers/kubeedge-v1.17.0/mqtt-mapper/driver/driver.go create mode 100644 mappers/kubeedge-v1.17.0/mqtt-mapper/driver/messagequeue.go create mode 100644 mappers/kubeedge-v1.17.0/mqtt-mapper/go.mod create mode 100644 mappers/kubeedge-v1.17.0/mqtt-mapper/go.sum create mode 100644 mappers/kubeedge-v1.17.0/mqtt-mapper/hack/make-rules/mapper.sh create mode 100644 mappers/kubeedge-v1.17.0/mqtt-mapper/resource/configmap.yaml create mode 100644 mappers/kubeedge-v1.17.0/mqtt-mapper/resource/deployment.yaml diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/Dockerfile b/mappers/kubeedge-v1.17.0/mqtt-mapper/Dockerfile new file mode 100644 index 00000000..8feae090 --- /dev/null +++ b/mappers/kubeedge-v1.17.0/mqtt-mapper/Dockerfile @@ -0,0 +1,20 @@ +FROM golang:1.20.10-alpine3.18 AS builder + +WORKDIR /build + +ENV GO111MODULE=on \ + GOPROXY=https://goproxy.cn,direct + +COPY . . + +RUN CGO_ENABLED=0 GOOS=linux go build -o main cmd/main.go + + +FROM ubuntu:18.04 + +RUN mkdir -p kubeedge + +COPY --from=builder /build/main kubeedge/ +COPY ./config.yaml kubeedge/ + +WORKDIR kubeedge diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/Dockerfile_nostream b/mappers/kubeedge-v1.17.0/mqtt-mapper/Dockerfile_nostream new file mode 100644 index 00000000..d694feaf --- /dev/null +++ b/mappers/kubeedge-v1.17.0/mqtt-mapper/Dockerfile_nostream @@ -0,0 +1,20 @@ +FROM golang:1.21.11-alpine3.19 AS builder + +WORKDIR /build + +ENV GO111MODULE=on \ + GOPROXY=https://goproxy.cn,direct + +COPY . . + +RUN CGO_ENABLED=0 GOOS=linux go build -o main cmd/main.go + + +FROM ubuntu:18.04 + +RUN mkdir -p kubeedge + +COPY --from=builder /build/main kubeedge/ +COPY ./config.yaml kubeedge/ + +WORKDIR kubeedge \ No newline at end of file diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/Dockerfile_stream b/mappers/kubeedge-v1.17.0/mqtt-mapper/Dockerfile_stream new file mode 100644 index 00000000..f0793b0f --- /dev/null +++ b/mappers/kubeedge-v1.17.0/mqtt-mapper/Dockerfile_stream @@ -0,0 +1,35 @@ +FROM golang:1.21.11-bullseye AS builder + +WORKDIR /build + +ENV GO111MODULE=on \ + GOPROXY=https://goproxy.cn,direct + +COPY . . + +RUN apt-get update && \ + apt-get install -y bzip2 curl upx-ucl gcc-aarch64-linux-gnu libc6-dev-arm64-cross gcc-arm-linux-gnueabi libc6-dev-armel-cross libva-dev libva-drm2 libx11-dev libvdpau-dev libxext-dev libsdl1.2-dev libxcb1-dev libxau-dev libxdmcp-dev yasm + +RUN curl -sLO https://ffmpeg.org/releases/ffmpeg-4.1.6.tar.bz2 && \ + tar -jx --strip-components=1 -f ffmpeg-4.1.6.tar.bz2 && \ + ./configure && make && \ + make install + +RUN GOOS=linux go build -o main cmd/main.go + +FROM ubuntu:18.04 + +RUN mkdir -p kubeedge + +RUN apt-get update && \ + apt-get install -y bzip2 curl upx-ucl gcc-aarch64-linux-gnu libc6-dev-arm64-cross gcc-arm-linux-gnueabi libc6-dev-armel-cross libva-dev libva-drm2 libx11-dev libvdpau-dev libxext-dev libsdl1.2-dev libxcb1-dev libxau-dev libxdmcp-dev yasm + +RUN curl -sLO https://ffmpeg.org/releases/ffmpeg-4.1.6.tar.bz2 && \ + tar -jx --strip-components=1 -f ffmpeg-4.1.6.tar.bz2 && \ + ./configure && make && \ + make install + +COPY --from=builder /build/main kubeedge/ +COPY ./config.yaml kubeedge/ + +WORKDIR kubeedge \ No newline at end of file diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/Makefile b/mappers/kubeedge-v1.17.0/mqtt-mapper/Makefile new file mode 100644 index 00000000..4dbacfe2 --- /dev/null +++ b/mappers/kubeedge-v1.17.0/mqtt-mapper/Makefile @@ -0,0 +1,34 @@ +SHELL := /bin/bash + +curr_dir := $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST))))) +rest_args := $(wordlist 2, $(words $(MAKECMDGOALS)), $(MAKECMDGOALS)) +$(eval $(rest_args):;@:) + +help: + # + # Usage: + # make generate : generate a mapper based on a template. + # make mapper {mapper-name} : execute mapper building process. + # + # Actions: + # - mod, m : download code dependencies. + # - lint, l : verify code via go fmt and `golangci-lint`. + # - build, b : compile code. + # - package, p : package docker image. + # - clean, c : clean output binary. + # + # Parameters: + # ARM : true or undefined + # ARM64 : true or undefined + # + # Example: + # - make mapper modbus ARM64=true : execute `build` "modbus" mapper for ARM64. + # - make mapper modbus test : execute `test` "modbus" mapper. + @echo + +make_rules := $(shell ls $(curr_dir)/hack/make-rules | sed 's/.sh//g') +$(make_rules): + @$(curr_dir)/hack/make-rules/$@.sh $(rest_args) + +.DEFAULT_GOAL := help +.PHONY: $(make_rules) build test package \ No newline at end of file diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/cmd/main.go b/mappers/kubeedge-v1.17.0/mqtt-mapper/cmd/main.go new file mode 100644 index 00000000..5f01ae1c --- /dev/null +++ b/mappers/kubeedge-v1.17.0/mqtt-mapper/cmd/main.go @@ -0,0 +1,60 @@ +package main + +import ( + "errors" + + "k8s.io/klog/v2" + + "github.com/kubeedge/kubeedge-v1.17.0/device" + "github.com/kubeedge/mapper-framework/pkg/common" + "github.com/kubeedge/mapper-framework/pkg/config" + "github.com/kubeedge/mapper-framework/pkg/grpcclient" + "github.com/kubeedge/mapper-framework/pkg/grpcserver" + "github.com/kubeedge/mapper-framework/pkg/httpserver" +) + +func main() { + var err error + var c *config.Config + + klog.InitFlags(nil) + defer klog.Flush() + + if c, err = config.Parse(); err != nil { + klog.Fatal(err) + } + klog.Infof("config: %+v", c) + + klog.Infoln("Mapper will register to edgecore") + deviceList, deviceModelList, err := grpcclient.RegisterMapper(true) + if err != nil { + klog.Fatal(err) + } + klog.Infoln("Mapper register finished") + + panel := device.NewDevPanel() + err = panel.DevInit(deviceList, deviceModelList) + if err != nil && !errors.Is(err, device.ErrEmptyData) { + klog.Fatal(err) + } + klog.Infoln("devInit finished") + go panel.DevStart() + + // start http server + httpServer := httpserver.NewRestServer(panel, c.Common.HTTPPort) + go httpServer.StartServer() + + // start grpc server + grpcServer := grpcserver.NewServer( + grpcserver.Config{ + SockPath: c.GrpcServer.SocketPath, + Protocol: common.ProtocolCustomized, + }, + panel, + ) + defer grpcServer.Stop() + if err = grpcServer.Start(); err != nil { + klog.Fatal(err) + } + +} diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/config.yaml b/mappers/kubeedge-v1.17.0/mqtt-mapper/config.yaml new file mode 100644 index 00000000..d456caad --- /dev/null +++ b/mappers/kubeedge-v1.17.0/mqtt-mapper/config.yaml @@ -0,0 +1,9 @@ +grpc_server: + socket_path: /etc/kubeedge/kubeedge-v1.17.0.sock +common: + name: Kubeedge-V1.17.0-mapper + version: v1.13.0 + api_version: v1.0.0 + protocol: # TODO add your protocol name + address: 127.0.0.1 + edgecore_sock: /etc/kubeedge/dmi.sock diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/influxdb2/client.go b/mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/influxdb2/client.go new file mode 100644 index 00000000..9da4232b --- /dev/null +++ b/mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/influxdb2/client.go @@ -0,0 +1,76 @@ +package influxdb2 + +import ( + "context" + "encoding/json" + "os" + "time" + + "k8s.io/klog/v2" + + influxdb2 "github.com/influxdata/influxdb-client-go/v2" + "github.com/kubeedge/mapper-framework/pkg/common" +) + +type DataBaseConfig struct { + Influxdb2ClientConfig *Influxdb2ClientConfig `json:"influxdb2ClientConfig,omitempty"` + Influxdb2DataConfig *Influxdb2DataConfig `json:"influxdb2DataConfig,omitempty"` +} + +type Influxdb2ClientConfig struct { + Url string `json:"url,omitempty"` + Org string `json:"org,omitempty"` + Bucket string `json:"bucket,omitempty"` +} + +type Influxdb2DataConfig struct { + Measurement string `json:"measurement,omitempty"` + Tag map[string]string `json:"tag,omitempty"` + FieldKey string `json:"fieldKey,omitempty"` +} + +func NewDataBaseClient(clientConfig json.RawMessage, dataConfig json.RawMessage) (*DataBaseConfig, error) { + // parse influx database config data + influxdb2ClientConfig := new(Influxdb2ClientConfig) + influxdb2DataConfig := new(Influxdb2DataConfig) + err := json.Unmarshal(clientConfig, influxdb2ClientConfig) + if err != nil { + return nil, err + } + err = json.Unmarshal(dataConfig, influxdb2DataConfig) + if err != nil { + return nil, err + } + return &DataBaseConfig{ + Influxdb2ClientConfig: influxdb2ClientConfig, + Influxdb2DataConfig: influxdb2DataConfig, + }, nil +} + +func (d *DataBaseConfig) InitDbClient() influxdb2.Client { + var usrtoken string + usrtoken = os.Getenv("TOKEN") + client := influxdb2.NewClient(d.Influxdb2ClientConfig.Url, usrtoken) + + return client +} + +func (d *DataBaseConfig) CloseSession(client influxdb2.Client) { + client.Close() +} + +func (d *DataBaseConfig) AddData(data *common.DataModel, client influxdb2.Client) error { + // write device data to influx database + writeAPI := client.WriteAPIBlocking(d.Influxdb2ClientConfig.Org, d.Influxdb2ClientConfig.Bucket) + p := influxdb2.NewPoint(d.Influxdb2DataConfig.Measurement, + d.Influxdb2DataConfig.Tag, + map[string]interface{}{d.Influxdb2DataConfig.FieldKey: data.Value}, + time.Now()) + // write point immediately + err := writeAPI.WritePoint(context.Background(), p) + if err != nil { + klog.V(4).Info("Exit AddData") + return err + } + return nil +} diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/influxdb2/handler.go b/mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/influxdb2/handler.go new file mode 100644 index 00000000..2569294e --- /dev/null +++ b/mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/influxdb2/handler.go @@ -0,0 +1,73 @@ +/* +Copyright 2023 The KubeEdge Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package influxdb2 + +import ( + "context" + "time" + + "k8s.io/klog/v2" + + "github.com/kubeedge/kubeedge-v1.17.0/driver" + "github.com/kubeedge/mapper-framework/pkg/common" +) + +func DataHandler(ctx context.Context, twin *common.Twin, client *driver.CustomizedClient, visitorConfig *driver.VisitorConfig, dataModel *common.DataModel) { + dbConfig, err := NewDataBaseClient(twin.Property.PushMethod.DBMethod.DBConfig.Influxdb2ClientConfig, twin.Property.PushMethod.DBMethod.DBConfig.Influxdb2DataConfig) + if err != nil { + klog.Errorf("new database client error: %v", err) + return + } + dbClient := dbConfig.InitDbClient() + if err != nil { + klog.Errorf("init database client err: %v", err) + return + } + reportCycle := time.Millisecond * time.Duration(twin.Property.ReportCycle) + if reportCycle == 0 { + reportCycle = common.DefaultReportCycle + } + ticker := time.NewTicker(reportCycle) + go func() { + for { + select { + case <-ticker.C: + deviceData, err := client.GetDeviceData(visitorConfig) + if err != nil { + klog.Errorf("publish error: %v", err) + continue + } + sData, err := common.ConvertToString(deviceData) + if err != nil { + klog.Errorf("Failed to convert publish method data : %v", err) + continue + } + dataModel.SetValue(sData) + dataModel.SetTimeStamp() + + err = dbConfig.AddData(dataModel, dbClient) + if err != nil { + klog.Errorf("influx database add data error: %v", err) + return + } + case <-ctx.Done(): + dbConfig.CloseSession(dbClient) + return + } + } + }() +} diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/mysql/client.go b/mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/mysql/client.go new file mode 100644 index 00000000..e2e9a259 --- /dev/null +++ b/mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/mysql/client.go @@ -0,0 +1,108 @@ +/* +Copyright 2024 The KubeEdge Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mysql + +import ( + "database/sql" + "encoding/json" + "fmt" + "os" + "time" + + _ "github.com/go-sql-driver/mysql" + "k8s.io/klog/v2" + + "github.com/kubeedge/mapper-framework/pkg/common" +) + +var ( + DB *sql.DB +) + +type DataBaseConfig struct { + MySQLClientConfig *MySQLClientConfig `json:"mysqlClientConfig"` +} + +type MySQLClientConfig struct { + Addr string `json:"addr,omitempty"` + Database string `json:"database,omitempty"` + UserName string `json:"userName,omitempty"` +} + +func NewDataBaseClient(config json.RawMessage) (*DataBaseConfig, error) { + configdata := new(MySQLClientConfig) + err := json.Unmarshal(config, configdata) + if err != nil { + return nil, err + } + return &DataBaseConfig{ + MySQLClientConfig: configdata, + }, nil +} + +func (d *DataBaseConfig) InitDbClient() error { + password := os.Getenv("PASSWORD") + usrName := d.MySQLClientConfig.UserName + addr := d.MySQLClientConfig.Addr + dataBase := d.MySQLClientConfig.Database + dataSourceName := fmt.Sprintf("%s:%s@tcp(%s)/%s", usrName, password, addr, dataBase) + var err error + DB, err = sql.Open("mysql", dataSourceName) + if err != nil { + return fmt.Errorf("connection to %s of mysql faild with err:%v", dataBase, err) + } + + return nil +} + +func (d *DataBaseConfig) CloseSession() { + err := DB.Close() + if err != nil { + klog.Errorf("close mysql failed with err:%v", err) + } +} + +func (d *DataBaseConfig) AddData(data *common.DataModel) error { + deviceName := data.DeviceName + propertyName := data.PropertyName + namespace := data.Namespace + tableName := namespace + "/" + deviceName + "/" + propertyName + datatime := time.Unix(data.TimeStamp/1e3, 0).Format("2006-01-02 15:04:05") + + createTable := fmt.Sprintf("CREATE TABLE IF NOT EXISTS `%s` (id INT AUTO_INCREMENT PRIMARY KEY, ts DATETIME NOT NULL,field TEXT)", tableName) + _, err := DB.Exec(createTable) + if err != nil { + return fmt.Errorf("create tabe into mysql failed with err:%v", err) + } + + stmt, err := DB.Prepare(fmt.Sprintf("INSERT INTO `%s` (ts,field) VALUES (?,?)", tableName)) + if err != nil { + return fmt.Errorf("prepare parament failed with err:%v", err) + } + defer func(stmt *sql.Stmt) { + err := stmt.Close() + if err != nil { + klog.Errorf("close mysql's statement failed with err:%v", err) + } + }(stmt) + _, err = stmt.Exec(datatime, data.Value) + if err != nil { + return fmt.Errorf("insert data into msyql failed with err:%v", err) + } + + return nil +} diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/mysql/handler.go b/mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/mysql/handler.go new file mode 100644 index 00000000..cb4c8e01 --- /dev/null +++ b/mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/mysql/handler.go @@ -0,0 +1,73 @@ +/* +Copyright 2024 The KubeEdge Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mysql + +import ( + "context" + "time" + + "k8s.io/klog/v2" + + "github.com/kubeedge/kubeedge-v1.17.0/driver" + "github.com/kubeedge/mapper-framework/pkg/common" +) + +func DataHandler(ctx context.Context, twin *common.Twin, client *driver.CustomizedClient, visitorConfig *driver.VisitorConfig, dataModel *common.DataModel) { + dbConfig, err := NewDataBaseClient(twin.Property.PushMethod.DBMethod.DBConfig.MySQLClientConfig) + if err != nil { + klog.Errorf("new database client error: %v", err) + return + } + err = dbConfig.InitDbClient() + if err != nil { + klog.Errorf("init redis database client err: %v", err) + return + } + reportCycle := time.Duration(twin.Property.ReportCycle) + if reportCycle == 0 { + reportCycle = common.DefaultReportCycle + } + ticker := time.NewTicker(reportCycle) + go func() { + for { + select { + case <-ticker.C: + deviceData, err := client.GetDeviceData(visitorConfig) + if err != nil { + klog.Errorf("publish error: %v", err) + continue + } + sData, err := common.ConvertToString(deviceData) + if err != nil { + klog.Errorf("Failed to convert publish method data : %v", err) + continue + } + dataModel.SetValue(sData) + dataModel.SetTimeStamp() + + err = dbConfig.AddData(dataModel) + if err != nil { + klog.Errorf("mysql database add data error: %v", err) + return + } + case <-ctx.Done(): + dbConfig.CloseSession() + return + } + } + }() +} diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/redis/client.go b/mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/redis/client.go new file mode 100644 index 00000000..41fbbc3f --- /dev/null +++ b/mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/redis/client.go @@ -0,0 +1,119 @@ +package redis + +import ( + "context" + "encoding/json" + "errors" + "os" + "strconv" + + "github.com/go-redis/redis/v8" + "k8s.io/klog/v2" + + "github.com/kubeedge/mapper-framework/pkg/common" +) + +var ( + RedisCli *redis.Client +) + +type DataBaseConfig struct { + RedisClientConfig *RedisClientConfig +} + +type RedisClientConfig struct { + Addr string `json:"addr,omitempty"` + DB int `json:"db,omitempty"` + PoolSize int `json:"poolSize,omitempty"` + MinIdleConns int `json:"minIdleConns,omitempty"` +} + +func NewDataBaseClient(config json.RawMessage) (*DataBaseConfig, error) { + configdata := new(RedisClientConfig) + err := json.Unmarshal(config, configdata) + if err != nil { + return nil, err + } + return &DataBaseConfig{RedisClientConfig: configdata}, nil +} + +func (d *DataBaseConfig) InitDbClient() error { + var password string + password = os.Getenv("PASSWORD") + RedisCli = redis.NewClient(&redis.Options{ + Addr: d.RedisClientConfig.Addr, + Password: password, + DB: d.RedisClientConfig.DB, + PoolSize: d.RedisClientConfig.PoolSize, + MinIdleConns: d.RedisClientConfig.MinIdleConns, + }) + pong, err := RedisCli.Ping(context.Background()).Result() + if err != nil { + klog.Errorf("init redis database failed, err = %v", err) + return err + } + klog.V(1).Infof("init redis database successfully, with return cmd %s", pong) + return nil +} + +func (d *DataBaseConfig) CloseSession() { + err := RedisCli.Close() + if err != nil { + klog.V(4).Info("close database failed") + } +} + +func (d *DataBaseConfig) AddData(data *common.DataModel) error { + ctx := context.Background() + // The key to construct the ordered set, here DeviceName is used as the key + klog.V(1).Infof("deviceName:%s", data.DeviceName) + // Check if the current ordered set exists + deviceData := "TimeStamp: " + strconv.FormatInt(data.TimeStamp, 10) + " PropertyName: " + data.PropertyName + " data: " + data.Value + // Add data to ordered set. If the ordered set does not exist, it will be created. + _, err := RedisCli.ZAdd(ctx, data.DeviceName, &redis.Z{ + Score: float64(data.TimeStamp), + Member: deviceData, + }).Result() + if err != nil { + klog.V(4).Info("Exit AddData") + return err + } + return nil +} + +func (d *DataBaseConfig) GetDataByDeviceName(deviceName string) ([]*common.DataModel, error) { + ctx := context.Background() + + dataJSON, err := RedisCli.ZRevRange(ctx, deviceName, 0, -1).Result() + if err != nil { + klog.V(4).Infof("fail query data for deviceName,err:%v", err) + } + + var dataModels []*common.DataModel + + for _, jsonStr := range dataJSON { + var data common.DataModel + if err := json.Unmarshal([]byte(jsonStr), &data); err != nil { + klog.V(4).Infof("Error unMarshaling data: %v\n", err) + continue + } + + dataModels = append(dataModels, &data) + } + return dataModels, nil +} + +func (d *DataBaseConfig) GetPropertyDataByDeviceName(deviceName string, propertyData string) ([]*common.DataModel, error) { + //TODO implement me + return nil, errors.New("implement me") +} + +func (d *DataBaseConfig) GetDataByTimeRange(start int64, end int64) ([]*common.DataModel, error) { + //TODO implement me + return nil, errors.New("implement me") +} + +func (d *DataBaseConfig) DeleteDataByTimeRange(start int64, end int64) ([]*common.DataModel, error) { + //TODO implement me + return nil, errors.New("implement me") +} diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/redis/handler.go b/mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/redis/handler.go new file mode 100644 index 00000000..249dab65 --- /dev/null +++ b/mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/redis/handler.go @@ -0,0 +1,74 @@ +/* +Copyright 2023 The KubeEdge Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package redis + +import ( + "context" + "time" + + "k8s.io/klog/v2" + + "github.com/kubeedge/kubeedge-v1.17.0/driver" + "github.com/kubeedge/mapper-framework/pkg/common" +) + +func DataHandler(ctx context.Context, twin *common.Twin, client *driver.CustomizedClient, visitorConfig *driver.VisitorConfig, dataModel *common.DataModel) { + dbConfig, err := NewDataBaseClient(twin.Property.PushMethod.DBMethod.DBConfig.RedisClientConfig) + if err != nil { + klog.Errorf("new database client error: %v", err) + return + } + err = dbConfig.InitDbClient() + if err != nil { + klog.Errorf("init redis database client err: %v", err) + return + } + reportCycle := time.Millisecond * time.Duration(twin.Property.ReportCycle) + if reportCycle == 0 { + reportCycle = common.DefaultReportCycle + } + ticker := time.NewTicker(reportCycle) + go func() { + for { + select { + case <-ticker.C: + deviceData, err := client.GetDeviceData(visitorConfig) + if err != nil { + klog.Errorf("publish error: %v", err) + continue + } + sData, err := common.ConvertToString(deviceData) + if err != nil { + klog.Errorf("Failed to convert publish method data : %v", err) + continue + } + dataModel.SetValue(sData) + dataModel.SetTimeStamp() + + err = dbConfig.AddData(dataModel) + if err != nil { + klog.Errorf("redis database add data error: %v", err) + return + } + case <-ctx.Done(): + dbConfig.CloseSession() + return + } + } + }() + +} diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/tdengine/client.go b/mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/tdengine/client.go new file mode 100644 index 00000000..17faf6b1 --- /dev/null +++ b/mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/tdengine/client.go @@ -0,0 +1,156 @@ +package tdengine + +import ( + "database/sql" + "encoding/json" + "errors" + "fmt" + "os" + "strings" + "time" + + _ "github.com/taosdata/driver-go/v3/taosRestful" + "k8s.io/klog/v2" + + "github.com/kubeedge/mapper-framework/pkg/common" +) + +var ( + DB *sql.DB +) + +type DataBaseConfig struct { + TDEngineClientConfig *TDEngineClientConfig `json:"config,omitempty"` +} +type TDEngineClientConfig struct { + Addr string `json:"addr,omitempty"` + DBName string `json:"dbName,omitempty"` +} + +func NewDataBaseClient(config json.RawMessage) (*DataBaseConfig, error) { + configdata := new(TDEngineClientConfig) + err := json.Unmarshal(config, configdata) + if err != nil { + return nil, err + } + return &DataBaseConfig{ + TDEngineClientConfig: configdata, + }, nil +} +func (d *DataBaseConfig) InitDbClient() error { + username := os.Getenv("USERNAME") + password := os.Getenv("PASSWORD") + dsn := fmt.Sprintf("%s:%s@http(%s)/%s", username, password, d.TDEngineClientConfig.Addr, d.TDEngineClientConfig.DBName) + var err error + DB, err = sql.Open("taosRestful", dsn) + if err != nil { + klog.Errorf("init TDEngine db fail, err= %v:", err) + } + klog.V(1).Infof("init TDEngine database successfully") + return nil +} + +func (d *DataBaseConfig) CloseSessio() { + err := DB.Close() + if err != nil { + klog.Errorf("close TDEngine failed") + } +} + +func (d *DataBaseConfig) AddData(data *common.DataModel) error { + + legal_table := strings.Replace(data.DeviceName, "-", "_", -1) + legal_tag := strings.Replace(data.PropertyName, "-", "_", -1) + + stable_name := fmt.Sprintf("SHOW STABLES LIKE '%s'", legal_table) + stabel := fmt.Sprintf("CREATE STABLE %s (ts timestamp, devicename binary(64), propertyname binary(64), data binary(64),type binary(64)) TAGS (localtion binary(64));", legal_table) + + datatime := time.Unix(data.TimeStamp/1e3, 0).Format("2006-01-02 15:04:05") + insertSQL := fmt.Sprintf("INSERT INTO %s USING %s TAGS ('%s') VALUES('%v','%s', '%s', '%s', '%s');", + legal_tag, legal_table, legal_tag, datatime, data.DeviceName, data.PropertyName, data.Value, data.Type) + + rows, _ := DB.Query(stable_name) + defer rows.Close() + + if err := rows.Err(); err != nil { + klog.Errorf("query stable failed：%v", err) + } + + switch rows.Next() { + case false: + _, err := DB.Exec(stabel) + if err != nil { + klog.Errorf("create stable failed %v\n", err) + } + _, err = DB.Exec(insertSQL) + if err != nil { + klog.Errorf("failed add data to TdEngine:%v", err) + } + case true: + _, err := DB.Exec(insertSQL) + if err != nil { + klog.Errorf("failed add data to TdEngine:%v", err) + } + default: + klog.Infoln("failed add data to TdEngine") + } + + return nil +} +func (d *DataBaseConfig) GetDataByDeviceName(deviceName string) ([]*common.DataModel, error) { + querySql := fmt.Sprintf("SELECT ts, devicename, propertyname, data, type FROM %s", deviceName) + rows, err := DB.Query(querySql) + if err != nil { + return nil, err + } + defer rows.Close() + var dataModel []*common.DataModel + for rows.Next() { + var data common.DataModel + var ts time.Time + err := rows.Scan(&ts, &data.DeviceName, &data.PropertyName, &data.Value, &data.Type) + if err != nil { + klog.Errorf(" data scan error: %v\n", err) + //fmt.Printf("scan error:\n", err) + return nil, err + } + data.TimeStamp = ts.Unix() + dataModel = append(dataModel, &data) + } + return dataModel, nil +} +func (d *DataBaseConfig) GetPropertyDataByDeviceName(deviceName string, propertyData string) ([]*common.DataModel, error) { + //TODO implement me + panic("implement me") +} +func (d *DataBaseConfig) GetDataByTimeRange(deviceName string, start int64, end int64) ([]*common.DataModel, error) { + + legal_table := strings.Replace(deviceName, "-", "_", -1) + startTime := time.Unix(start, 0).UTC().Format("2006-01-02 15:04:05") + endTime := time.Unix(end, 0).UTC().Format("2006-01-02 15:04:05") + //Query data within a specified time range + querySQL := fmt.Sprintf("SELECT ts, devicename, propertyname, data, type FROM %s WHERE ts >= '%s' AND ts <= '%s'", legal_table, startTime, endTime) + fmt.Println(querySQL) + rows, err := DB.Query(querySQL) + if err != nil { + return nil, err + } + defer rows.Close() + + var dataModels []*common.DataModel + for rows.Next() { + var data common.DataModel + var ts time.Time + err := rows.Scan(&ts, &data.DeviceName, &data.PropertyName, &data.Value, &data.Type) + if err != nil { + klog.V(4).Infof("data scan failed：%v", err) + continue + } + dataModels = append(dataModels, &data) + } + return dataModels, nil +} +func (d *DataBaseConfig) DeleteDataByTimeRange(start int64, end int64) ([]*common.DataModel, error) { + //TODO implement me + return nil, errors.New("implement me") +} diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/tdengine/handler.go b/mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/tdengine/handler.go new file mode 100644 index 00000000..e5feef4c --- /dev/null +++ b/mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/tdengine/handler.go @@ -0,0 +1,74 @@ +/* +Copyright 2023 The KubeEdge Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tdengine + +import ( + "context" + "time" + + "k8s.io/klog/v2" + + "github.com/kubeedge/kubeedge-v1.17.0/driver" + "github.com/kubeedge/mapper-framework/pkg/common" +) + +func DataHandler(ctx context.Context, twin *common.Twin, client *driver.CustomizedClient, visitorConfig *driver.VisitorConfig, dataModel *common.DataModel) { + dbConfig, err := NewDataBaseClient(twin.Property.PushMethod.DBMethod.DBConfig.TDEngineClientConfig) + if err != nil { + klog.Errorf("new database client error: %v", err) + return + } + err = dbConfig.InitDbClient() + if err != nil { + klog.Errorf("init database client err: %v", err) + return + } + reportCycle := time.Millisecond * time.Duration(twin.Property.ReportCycle) + if reportCycle == 0 { + reportCycle = common.DefaultReportCycle + } + ticker := time.NewTicker(reportCycle) + go func() { + for { + select { + case <-ticker.C: + deviceData, err := client.GetDeviceData(visitorConfig) + if err != nil { + klog.Errorf("publish error: %v", err) + continue + } + sData, err := common.ConvertToString(deviceData) + if err != nil { + klog.Errorf("Failed to convert publish method data : %v", err) + continue + } + dataModel.SetValue(sData) + dataModel.SetTimeStamp() + + err = dbConfig.AddData(dataModel) + if err != nil { + klog.Errorf("tdengine database add data error: %v", err) + return + } + case <-ctx.Done(): + dbConfig.CloseSessio() + return + } + } + }() + +} diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/data/publish/http/client.go b/mappers/kubeedge-v1.17.0/mqtt-mapper/data/publish/http/client.go new file mode 100644 index 00000000..1ff484a9 --- /dev/null +++ b/mappers/kubeedge-v1.17.0/mqtt-mapper/data/publish/http/client.go @@ -0,0 +1,73 @@ +package http + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" + + "k8s.io/klog/v2" + + "github.com/kubeedge/mapper-framework/pkg/common" + "github.com/kubeedge/mapper-framework/pkg/global" +) + +type PushMethod struct { + HTTP *HTTPConfig `json:"http"` +} + +type HTTPConfig struct { + HostName string `json:"hostName,omitempty"` + Port int `json:"port,omitempty"` + RequestPath string `json:"requestPath,omitempty"` + Timeout int `json:"timeout,omitempty"` +} + +func NewDataPanel(config json.RawMessage) (global.DataPanel, error) { + httpConfig := new(HTTPConfig) + err := json.Unmarshal(config, httpConfig) + if err != nil { + return nil, err + } + return &PushMethod{ + HTTP: httpConfig, + }, nil +} + +func (pm *PushMethod) InitPushMethod() error { + klog.V(1).Info("Init HTTP") + return nil +} + +func (pm *PushMethod) Push(data *common.DataModel) { + klog.V(2).Info("Publish device data by HTTP") + + targetUrl := pm.HTTP.HostName + ":" + strconv.Itoa(pm.HTTP.Port) + pm.HTTP.RequestPath + payload := data.PropertyName + "=" + data.Value + formatTimeStr := time.Unix(data.TimeStamp/1e3, 0).Format("2006-01-02 15:04:05") + currentTime := "&time" + "=" + formatTimeStr + payload += currentTime + + klog.V(3).Infof("Publish %v to %s", payload, targetUrl) + + resp, err := http.Post(targetUrl, + "application/x-www-form-urlencoded", + strings.NewReader(payload)) + + if err != nil { + fmt.Println(err) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + // handle error + klog.Errorf("Publish device data by HTTP failed, err = %v", err) + return + } + klog.V(1).Info("############### Message published. ###############") + klog.V(3).Infof("HTTP reviced %s", string(body)) + +} diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/data/publish/mqtt/client.go b/mappers/kubeedge-v1.17.0/mqtt-mapper/data/publish/mqtt/client.go new file mode 100644 index 00000000..ed8fe554 --- /dev/null +++ b/mappers/kubeedge-v1.17.0/mqtt-mapper/data/publish/mqtt/client.go @@ -0,0 +1,63 @@ +package mqtt + +import ( + "encoding/json" + "fmt" + "os" + "time" + + mqtt "github.com/eclipse/paho.mqtt.golang" + "k8s.io/klog/v2" + + "github.com/kubeedge/mapper-framework/pkg/common" + "github.com/kubeedge/mapper-framework/pkg/global" +) + +type PushMethod struct { + MQTT *MQTTConfig `json:"http"` +} + +type MQTTConfig struct { + Address string `json:"address,omitempty"` + Topic string `json:"topic,omitempty"` + QoS int `json:"qos,omitempty"` + Retained bool `json:"retained,omitempty"` +} + +func NewDataPanel(config json.RawMessage) (global.DataPanel, error) { + mqttConfig := new(MQTTConfig) + err := json.Unmarshal(config, mqttConfig) + if err != nil { + return nil, err + } + return &PushMethod{ + MQTT: mqttConfig, + }, nil +} + +func (pm *PushMethod) InitPushMethod() error { + klog.V(1).Info("Init MQTT") + return nil +} + +func (pm *PushMethod) Push(data *common.DataModel) { + klog.V(1).Infof("Publish %v to %s on topic: %s, Qos: %d, Retained: %v", + data.Value, pm.MQTT.Address, pm.MQTT.Topic, pm.MQTT.QoS, pm.MQTT.Retained) + + opts := mqtt.NewClientOptions().AddBroker(pm.MQTT.Address) + client := mqtt.NewClient(opts) + + if token := client.Connect(); token.Wait() && token.Error() != nil { + fmt.Println(token.Error()) + os.Exit(1) + } + formatTimeStr := time.Unix(data.TimeStamp/1e3, 0).Format("2006-01-02 15:04:05") + str_time := "time is " + formatTimeStr + " " + str_publish := str_time + pm.MQTT.Topic + ": " + data.Value + + token := client.Publish(pm.MQTT.Topic, byte(pm.MQTT.QoS), pm.MQTT.Retained, str_publish) + token.Wait() + + client.Disconnect(250) + klog.V(2).Info("############### Message published. ###############") +} diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/data/stream/handler.go b/mappers/kubeedge-v1.17.0/mqtt-mapper/data/stream/handler.go new file mode 100644 index 00000000..9b8a2fdc --- /dev/null +++ b/mappers/kubeedge-v1.17.0/mqtt-mapper/data/stream/handler.go @@ -0,0 +1,67 @@ +/* +Copyright 2024 The KubeEdge Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stream + +import ( + "encoding/json" + "fmt" + + "k8s.io/klog/v2" + + "github.com/kubeedge/kubeedge-v1.17.0/driver" + "github.com/kubeedge/mapper-framework/pkg/common" +) + +type StreamConfig struct { + Format string `json:"format"` + OutputDir string `json:"outputDir"` + FrameCount int `json:"frameCount"` + FrameInterval int `json:"frameInterval"` + VideoNum int `json:"videoNum"` +} + +func StreamHandler(twin *common.Twin, client *driver.CustomizedClient, visitorConfig *driver.VisitorConfig) error { + // Get RTSP URI from camera device + streamURI, err := client.GetDeviceData(visitorConfig) + if err != nil { + return err + } + + // parse streamConfig data from device visitorConfig + var streamConfig StreamConfig + visitorConfigData, err := json.Marshal(visitorConfig.VisitorConfigData) + err = json.Unmarshal(visitorConfigData, &streamConfig) + if err != nil { + return fmt.Errorf("Unmarshal streamConfigs error: %v", err) + } + + switch twin.PropertyName { + // Currently, the function of saving frames and saving videos is built-in according to the configuration. + // Other functions can be expanded here. + case common.SaveFrame: + err = SaveFrame(streamURI.(string), streamConfig.OutputDir, streamConfig.Format, streamConfig.FrameCount, streamConfig.FrameInterval) + case common.SaveVideo: + err = SaveVideo(streamURI.(string), streamConfig.OutputDir, streamConfig.Format, streamConfig.FrameCount, streamConfig.VideoNum) + default: + err = fmt.Errorf("cannot find the processing method for the corresponding Property %s of the stream data", twin.PropertyName) + } + if err != nil { + return err + } + klog.V(2).Infof("Successfully processed streaming data by %s", twin.PropertyName) + return nil +} diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/data/stream/img.go b/mappers/kubeedge-v1.17.0/mqtt-mapper/data/stream/img.go new file mode 100644 index 00000000..d7d05d00 --- /dev/null +++ b/mappers/kubeedge-v1.17.0/mqtt-mapper/data/stream/img.go @@ -0,0 +1,243 @@ +/* +Copyright 2024 The KubeEdge Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stream + +import ( + "errors" + "fmt" + "time" + "unsafe" + + "github.com/sailorvii/goav/avcodec" + "github.com/sailorvii/goav/avformat" + "github.com/sailorvii/goav/avutil" + "github.com/sailorvii/goav/swscale" + "k8s.io/klog/v2" +) + +// GenFileName generate file name with current time. Formate f. +func GenFileName(dir string, format string) string { + return fmt.Sprintf("%s/f%s.%s", dir, time.Now().Format(time.RFC3339Nano), format) +} + +func save(frame *avutil.Frame, width int, height int, dir string, format string) error { + // Save video frames to picture file + outputFile := GenFileName(dir, format) + var outputFmtCtx *avformat.Context + avformat.AvAllocOutputContext2(&outputFmtCtx, nil, nil, &outputFile) + if outputFmtCtx == nil { + return errors.New("Could not create output context") + } + defer outputFmtCtx.AvformatFreeContext() + + ofmt := avformat.AvGuessFormat("", outputFile, "") + outputFmtCtx.SetOformat(ofmt) + + avIOContext, err := avformat.AvIOOpen(outputFile, avformat.AVIO_FLAG_WRITE) + if err != nil { + return fmt.Errorf("Could not open output file '%s'", outputFile) + } + outputFmtCtx.SetPb(avIOContext) + + outStream := outputFmtCtx.AvformatNewStream(nil) + if outStream == nil { + return errors.New("Failed allocating output stream") + } + + // Set the frame format + pCodecCtx := outStream.Codec() + pCodecCtx.SetCodecId(ofmt.GetVideoCodec()) + pCodecCtx.SetCodecType(avformat.AVMEDIA_TYPE_VIDEO) + pCodecCtx.SetPixelFormat(avcodec.AV_PIX_FMT_YUVJ420P) + pCodecCtx.SetWidth(width) + pCodecCtx.SetHeight(height) + pCodecCtx.SetTimeBase(1, 25) + outputFmtCtx.AvDumpFormat(0, outputFile, 1) + + // Get video codec + pCodec := avcodec.AvcodecFindEncoder(pCodecCtx.CodecId()) + if pCodec == nil { + return errors.New("Codec not found.") + } + defer pCodecCtx.AvcodecClose() + + // open video codec + cctx := avcodec.Context(*pCodecCtx) + defer cctx.AvcodecClose() + if cctx.AvcodecOpen2(pCodec, nil) < 0 { + return errors.New("Could not open codec.") + } + + outputFmtCtx.AvformatWriteHeader(nil) + ySize := width * height + + // Write media data to media files + var packet avcodec.Packet + packet.AvNewPacket(ySize * 3) + defer packet.AvPacketUnref() + var gotPicture int + if cctx.AvcodecEncodeVideo2(&packet, frame, &gotPicture) < 0 { + return errors.New("Encode Error") + } + if gotPicture == 1 { + packet.SetStreamIndex(outStream.Index()) + outputFmtCtx.AvWriteFrame(&packet) + } + + outputFmtCtx.AvWriteTrailer() + if outputFmtCtx.Oformat().GetFlags()&avformat.AVFMT_NOFILE == 0 { + if err = outputFmtCtx.Pb().Close(); err != nil { + return fmt.Errorf("close output fmt context failed: %v", err) + } + } + return nil +} + +// SaveFrame save frame. +func SaveFrame(input string, outDir string, format string, frameCount int, frameInterval int) error { + // Open video file + avformat.AvDictSet(&avformat.Dict, "rtsp_transport", "tcp", 0) + avformat.AvDictSet(&avformat.Dict, "max_delay", "5000000", 0) + + pFormatContext := avformat.AvformatAllocContext() + if avformat.AvformatOpenInput(&pFormatContext, input, nil, &avformat.Dict) != 0 { + return fmt.Errorf("Unable to open file %s", input) + } + // Retrieve stream information + if pFormatContext.AvformatFindStreamInfo(nil) < 0 { + return errors.New("Couldn't find stream information") + } + // Dump information about file onto standard error + pFormatContext.AvDumpFormat(0, input, 0) + // Find the first video stream + streamIndex := -1 + for i := 0; i < int(pFormatContext.NbStreams()); i++ { + if pFormatContext.Streams()[i].CodecParameters().AvCodecGetType() == avformat.AVMEDIA_TYPE_VIDEO { + streamIndex = i + break + } + } + if streamIndex == -1 { + return errors.New("couldn't find video stream") + } + // Get a pointer to the codec context for the video stream + pCodecCtxOrig := pFormatContext.Streams()[streamIndex].Codec() + // Find the decoder for the video stream + pCodec := avcodec.AvcodecFindDecoder(pCodecCtxOrig.CodecId()) + if pCodec == nil { + return errors.New("unsupported codec") + } + // Copy context + pCodecCtx := pCodec.AvcodecAllocContext3() + if pCodecCtx.AvcodecCopyContext((*avcodec.Context)(unsafe.Pointer(pCodecCtxOrig))) != 0 { + return errors.New("couldn't copy codec context") + } + + // Open codec + if pCodecCtx.AvcodecOpen2(pCodec, nil) < 0 { + return errors.New("could not open codec") + } + + // Allocate video frame + pFrame := avutil.AvFrameAlloc() + + // Allocate an AVFrame structure + pFrameRGB := avutil.AvFrameAlloc() + if pFrameRGB == nil { + return errors.New("unable to allocate RGB Frame") + } + // Determine required buffer size and allocate buffer + numBytes := uintptr(avcodec.AvpictureGetSize(avcodec.AV_PIX_FMT_YUVJ420P, pCodecCtx.Width(), + pCodecCtx.Height())) + buffer := avutil.AvMalloc(numBytes) + + // Assign appropriate parts of buffer to image planes in pFrameRGB + // Note that pFrameRGB is an AVFrame, but AVFrame is a superset + // of AVPicture + avp := (*avcodec.Picture)(unsafe.Pointer(pFrameRGB)) + avp.AvpictureFill((*uint8)(buffer), avcodec.AV_PIX_FMT_YUVJ420P, pCodecCtx.Width(), pCodecCtx.Height()) + + // initialize SWS context for software scaling + swsCtx := swscale.SwsGetcontext( + pCodecCtx.Width(), + pCodecCtx.Height(), + (swscale.PixelFormat)(pCodecCtx.PixFmt()), + pCodecCtx.Width(), + pCodecCtx.Height(), + avcodec.AV_PIX_FMT_YUVJ420P, + avcodec.SWS_BICUBIC, + nil, + nil, + nil, + ) + frameNum := 0 + failureNum := 0 + failureCount := 5 * frameCount + packet := avcodec.AvPacketAlloc() + // Start capturing and saving video frames + for { + if failureNum >= failureCount { + klog.Error("the number of failed attempts to save frames has reached the upper limit") + return errors.New("the number of failed attempts to save frames has reached the upper limit") + } + + if pFormatContext.AvReadFrame(packet) < 0 { + klog.Error("Read frame failed") + time.Sleep(time.Second) + continue + } + + // Is this a packet from the video stream? + if packet.StreamIndex() != streamIndex { + failureNum++ + continue + } + + // Decode video frame + response := pCodecCtx.AvcodecSendPacket(packet) + if response < 0 { + klog.Errorf("Error while sending a packet to the decoder: %s", avutil.ErrorFromCode(response)) + failureNum++ + continue + } + response = pCodecCtx.AvcodecReceiveFrame((*avutil.Frame)(unsafe.Pointer(pFrame))) + if response == avutil.AvErrorEAGAIN || response == avutil.AvErrorEOF { + failureNum++ + continue + } else if response < 0 { + klog.Errorf("Error while receiving a frame from the decoder: %s", avutil.ErrorFromCode(response)) + failureNum++ + continue + } + // Convert the image from its native format to RGB + swscale.SwsScale2(swsCtx, avutil.Data(pFrame), + avutil.Linesize(pFrame), 0, pCodecCtx.Height(), + avutil.Data(pFrameRGB), avutil.Linesize(pFrameRGB)) + + // Save the frame to disk + err := save(pFrameRGB, pCodecCtx.Width(), pCodecCtx.Height(), outDir, format) + if err != nil { + klog.Error(err) + continue + } + frameNum++ + if frameNum >= frameCount { + return nil + } + time.Sleep(time.Nanosecond * time.Duration(frameInterval)) + } +} diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/data/stream/video.go b/mappers/kubeedge-v1.17.0/mqtt-mapper/data/stream/video.go new file mode 100644 index 00000000..7954e215 --- /dev/null +++ b/mappers/kubeedge-v1.17.0/mqtt-mapper/data/stream/video.go @@ -0,0 +1,142 @@ +/* +Copyright 2024 The KubeEdge Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stream + +import ( + "errors" + "fmt" + + "github.com/sailorvii/goav/avcodec" + "github.com/sailorvii/goav/avformat" + "github.com/sailorvii/goav/avutil" + "k8s.io/klog/v2" +) + +// SaveVideo save video. +func SaveVideo(inputFile string, outDir string, format string, frameCount int, videoNum int) error { + var fragmentedMp4Options int + //initialize input file with Context + var inputFmtCtx *avformat.Context + + avformat.AvDictSet(&avformat.Dict, "rtsp_transport", "tcp", 0) + avformat.AvDictSet(&avformat.Dict, "max_delay", "5000000", 0) + + if avformat.AvformatOpenInput(&inputFmtCtx, inputFile, nil, &avformat.Dict) < 0 { + return fmt.Errorf("could not open input file '%s", inputFile) + } + defer inputFmtCtx.AvformatFreeContext() + //read stream information + + if inputFmtCtx.AvformatFindStreamInfo(nil) < 0 { + return errors.New("failed to retrieve input stream information") + } + + //initialize streamMapping + streamMappingSize := int(inputFmtCtx.NbStreams()) + streamMapping := make([]int, streamMappingSize) + var streamIndex int + + validTypeMap := map[avcodec.MediaType]int{ + avformat.AVMEDIA_TYPE_VIDEO: 1, + avformat.AVMEDIA_TYPE_AUDIO: 1, + avformat.AVMEDIA_TYPE_SUBTITLE: 1, + } + var inCodecParam *avcodec.AvCodecParameters + defer inCodecParam.AvCodecParametersFree() + + var outputFmtCtx *avformat.Context + outputFile := GenFileName(outDir, format) + avformat.AvAllocOutputContext2(&outputFmtCtx, nil, nil, &outputFile) + if outputFmtCtx == nil { + return errors.New("Could not create output context") + } + defer outputFmtCtx.AvformatFreeContext() + + for index, inStream := range inputFmtCtx.Streams() { + inCodecParam = inStream.CodecParameters() + inCodecType := inCodecParam.AvCodecGetType() + + if validTypeMap[inCodecType] == 0 { + streamMapping[index] = -1 + continue + } + streamMapping[index] = streamIndex + streamIndex++ + outStream := outputFmtCtx.AvformatNewStream(nil) + if outStream == nil { + return errors.New("Failed allocating output stream") + } + if inCodecParam.AvCodecParametersCopyTo(outStream.CodecParameters()) < 0 { + return errors.New("Failed to copy codec parameters") + } + } + + // initialize opts + var opts *avutil.Dictionary + defer opts.AvDictFree() + if fragmentedMp4Options != 0 { + opts.AvDictSet("movflags", "frag_keyframe+empty_moov+default_base_moof", 0) + } + var packet avcodec.Packet + defer packet.AvPacketUnref() + + // Capture a set number of video segments + for idx := 0; idx < videoNum; idx++ { + outputFile = GenFileName(outDir, format) + // initialize output file with Context + outputFmtCtx.AvDumpFormat(0, outputFile, 1) + if outputFmtCtx.Oformat().GetFlags()&avformat.AVFMT_NOFILE == 0 { + avIOContext, err := avformat.AvIOOpen(outputFile, avformat.AVIO_FLAG_WRITE) + if err != nil { + return fmt.Errorf("could not open output file '%s'", outputFile) + } + outputFmtCtx.SetPb(avIOContext) + } + + if outputFmtCtx.AvformatWriteHeader(&opts) < 0 { + return errors.New("Error occurred when opening output file") + } + // Capture and generate video according to the set number of frames + for i := 1; i < frameCount; i++ { + if inputFmtCtx.AvReadFrame(&packet) < 0 { + return errors.New("read frame failed") + } + index := packet.StreamIndex() + inputStream := inputFmtCtx.Streams()[index] + if index >= streamMappingSize || streamMapping[index] < 0 { + continue + } + packet.SetStreamIndex(streamMapping[index]) + outputStream := outputFmtCtx.Streams()[index] + packet.AvPacketRescaleTs(inputStream.TimeBase(), outputStream.TimeBase()) + packet.SetPos(-1) + if outputFmtCtx.AvInterleavedWriteFrame(&packet) < 0 { + klog.Error("Error muxing packet") + continue + } + } + + outputFmtCtx.AvWriteTrailer() + if outputFmtCtx.Oformat().GetFlags()&avformat.AVFMT_NOFILE == 0 { + if outputFmtCtx.Pb().Close() != nil { + klog.Error("Error close output context") + return errors.New("error close output context") + } + } + } + return nil +} diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/device/device.go b/mappers/kubeedge-v1.17.0/mqtt-mapper/device/device.go new file mode 100644 index 00000000..6df39da1 --- /dev/null +++ b/mappers/kubeedge-v1.17.0/mqtt-mapper/device/device.go @@ -0,0 +1,493 @@ +package device + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/signal" + "strings" + "sync" + "time" + + "k8s.io/klog/v2" + + dbInflux "github.com/kubeedge/kubeedge-v1.17.0/data/dbmethod/influxdb2" + dbMysql "github.com/kubeedge/kubeedge-v1.17.0/data/dbmethod/mysql" + dbRedis "github.com/kubeedge/kubeedge-v1.17.0/data/dbmethod/redis" + dbTdengine "github.com/kubeedge/kubeedge-v1.17.0/data/dbmethod/tdengine" + httpMethod "github.com/kubeedge/kubeedge-v1.17.0/data/publish/http" + mqttMethod "github.com/kubeedge/kubeedge-v1.17.0/data/publish/mqtt" + "github.com/kubeedge/kubeedge-v1.17.0/data/stream" + "github.com/kubeedge/kubeedge-v1.17.0/driver" + dmiapi "github.com/kubeedge/kubeedge/pkg/apis/dmi/v1beta1" + "github.com/kubeedge/mapper-framework/pkg/common" + "github.com/kubeedge/mapper-framework/pkg/global" + "github.com/kubeedge/mapper-framework/pkg/util/parse" +) + +type DevPanel struct { + deviceMuxs map[string]context.CancelFunc + devices map[string]*driver.CustomizedDev + models map[string]common.DeviceModel + wg sync.WaitGroup + serviceMutex sync.Mutex + quitChan chan os.Signal +} + +var ( + devPanel *DevPanel + once sync.Once +) + +var ErrEmptyData = errors.New("device or device model list is empty") + +// NewDevPanel init and return devPanel +func NewDevPanel() *DevPanel { + once.Do(func() { + devPanel = &DevPanel{ + deviceMuxs: make(map[string]context.CancelFunc), + devices: make(map[string]*driver.CustomizedDev), + models: make(map[string]common.DeviceModel), + wg: sync.WaitGroup{}, + serviceMutex: sync.Mutex{}, + quitChan: make(chan os.Signal), + } + }) + return devPanel +} + +// DevStart start all devices. +func (d *DevPanel) DevStart() { + for id, dev := range d.devices { + klog.V(4).Info("Dev: ", id, dev) + ctx, cancel := context.WithCancel(context.Background()) + d.deviceMuxs[id] = cancel + d.wg.Add(1) + go d.start(ctx, dev) + } + signal.Notify(d.quitChan, os.Interrupt) + go func() { + <-d.quitChan + for id, device := range d.devices { + err := device.CustomizedClient.StopDevice() + if err != nil { + klog.Errorf("Service has stopped but failed to stop %s:%v", id, err) + } + } + klog.V(1).Info("Exit mapper") + os.Exit(1) + }() + d.wg.Wait() +} + +// start the device +func (d *DevPanel) start(ctx context.Context, dev *driver.CustomizedDev) { + defer d.wg.Done() + + var protocolConfig driver.ProtocolConfig + if err := json.Unmarshal(dev.Instance.PProtocol.ConfigData, &protocolConfig); err != nil { + klog.Errorf("Unmarshal ProtocolConfigs error: %v", err) + return + } + client, err := driver.NewClient(protocolConfig) + if err != nil { + klog.Errorf("Init dev %s error: %v", dev.Instance.Name, err) + return + } + dev.CustomizedClient = client + err = dev.CustomizedClient.InitDevice() + if err != nil { + klog.Errorf("Init device %s error: %v", dev.Instance.ID, err) + return + } + go dataHandler(ctx, dev) + <-ctx.Done() +} + +// dataHandler initialize the timer to handle data plane and devicetwin. +func dataHandler(ctx context.Context, dev *driver.CustomizedDev) { + for _, twin := range dev.Instance.Twins { + twin.Property.PProperty.DataType = strings.ToLower(twin.Property.PProperty.DataType) + var visitorConfig driver.VisitorConfig + + err := json.Unmarshal(twin.Property.Visitors, &visitorConfig) + visitorConfig.VisitorConfigData.DataType = strings.ToLower(visitorConfig.VisitorConfigData.DataType) + if err != nil { + klog.Errorf("Unmarshal VisitorConfig error: %v", err) + continue + } + err = setVisitor(&visitorConfig, &twin, dev) + if err != nil { + klog.Error(err) + continue + } + + // If the device property type is streaming, it will directly enter the streaming data processing function, + // such as saving frames or saving videos, and will no longer push it to the user database and application. + // If there are other needs for stream data processing, users can add functions in the mapper/data/stream directory. + if twin.Property.PProperty.DataType == "stream" { + err = stream.StreamHandler(&twin, dev.CustomizedClient, &visitorConfig) + if err != nil { + klog.Errorf("processed streaming data by %s Error: %v", twin.PropertyName, err) + } + continue + } + + // handle twin + twinData := &TwinData{ + DeviceName: dev.Instance.Name, + DeviceNamespace: dev.Instance.Namespace, + Client: dev.CustomizedClient, + Name: twin.PropertyName, + Type: twin.ObservedDesired.Metadata.Type, + ObservedDesired: twin.ObservedDesired, + VisitorConfig: &visitorConfig, + Topic: fmt.Sprintf(common.TopicTwinUpdate, dev.Instance.ID), + CollectCycle: time.Millisecond * time.Duration(twin.Property.CollectCycle), + ReportToCloud: twin.Property.ReportToCloud, + } + go twinData.Run(ctx) + + //handle status + getStates := &DeviceStates{Client: dev.CustomizedClient, DeviceName: dev.Instance.Name, + DeviceNamespace: dev.Instance.Namespace} + go getStates.Run(ctx) + + // handle push method + if twin.Property.PushMethod.MethodConfig != nil && twin.Property.PushMethod.MethodName != "" { + dataModel := common.NewDataModel(dev.Instance.Name, twin.Property.PropertyName, dev.Instance.Namespace, common.WithType(twin.ObservedDesired.Metadata.Type)) + pushHandler(ctx, &twin, dev.CustomizedClient, &visitorConfig, dataModel) + } + // handle database + if twin.Property.PushMethod.DBMethod.DBMethodName != "" { + dataModel := common.NewDataModel(dev.Instance.Name, twin.Property.PropertyName, dev.Instance.Namespace, common.WithType(twin.ObservedDesired.Metadata.Type)) + dbHandler(ctx, &twin, dev.CustomizedClient, &visitorConfig, dataModel) + switch twin.Property.PushMethod.DBMethod.DBMethodName { + // TODO add more database + case "influx": + dbInflux.DataHandler(ctx, &twin, dev.CustomizedClient, &visitorConfig, dataModel) + case "redis": + dbRedis.DataHandler(ctx, &twin, dev.CustomizedClient, &visitorConfig, dataModel) + case "tdengine": + dbTdengine.DataHandler(ctx, &twin, dev.CustomizedClient, &visitorConfig, dataModel) + case "mysql": + dbMysql.DataHandler(ctx, &twin, dev.CustomizedClient, &visitorConfig, dataModel) + } + } + } +} + +// pushHandler start data panel work +func pushHandler(ctx context.Context, twin *common.Twin, client *driver.CustomizedClient, visitorConfig *driver.VisitorConfig, dataModel *common.DataModel) { + var dataPanel global.DataPanel + var err error + // initialization dataPanel + switch twin.Property.PushMethod.MethodName { + case "http": + dataPanel, err = httpMethod.NewDataPanel(twin.Property.PushMethod.MethodConfig) + case "mqtt": + dataPanel, err = mqttMethod.NewDataPanel(twin.Property.PushMethod.MethodConfig) + default: + err = errors.New("custom protocols are not currently supported when push data") + } + if err != nil { + klog.Errorf("new data panel error: %v", err) + return + } + // initialization PushMethod + err = dataPanel.InitPushMethod() + if err != nil { + klog.Errorf("init publish method err: %v", err) + return + } + reportCycle := time.Millisecond * time.Duration(twin.Property.ReportCycle) + if reportCycle == 0 { + reportCycle = common.DefaultReportCycle + } + ticker := time.NewTicker(reportCycle) + go func() { + for { + select { + case <-ticker.C: + deviceData, err := client.GetDeviceData(visitorConfig) + if err != nil { + klog.Errorf("publish error: %v", err) + continue + } + sData, err := common.ConvertToString(deviceData) + if err != nil { + klog.Errorf("Failed to convert publish method data : %v", err) + continue + } + dataModel.SetValue(sData) + dataModel.SetTimeStamp() + dataPanel.Push(dataModel) + case <-ctx.Done(): + return + } + } + }() +} + +// dbHandler start db client to save data +func dbHandler(ctx context.Context, twin *common.Twin, client *driver.CustomizedClient, visitorConfig *driver.VisitorConfig, dataModel *common.DataModel) { + switch twin.Property.PushMethod.DBMethod.DBMethodName { + // TODO add more database + case "influx": + dbInflux.DataHandler(ctx, twin, client, visitorConfig, dataModel) + + case "redis": + dbRedis.DataHandler(ctx, twin, client, visitorConfig, dataModel) + + case "tdengine": + dbTdengine.DataHandler(ctx, twin, client, visitorConfig, dataModel) + + case "mysql": + dbMysql.DataHandler(ctx, twin, client, visitorConfig, dataModel) + } +} + +// setVisitor check if visitor property is readonly, if not then set it. +func setVisitor(visitorConfig *driver.VisitorConfig, twin *common.Twin, dev *driver.CustomizedDev) error { + if twin.Property.PProperty.AccessMode == "ReadOnly" { + klog.V(3).Infof("%s twin readonly property: %s", dev.Instance.Name, twin.PropertyName) + return nil + } + klog.V(2).Infof("Convert type: %s, value: %s ", twin.Property.PProperty.DataType, twin.ObservedDesired.Value) + value, err := common.Convert(twin.Property.PProperty.DataType, twin.ObservedDesired.Value) + if err != nil { + klog.Errorf("Failed to convert value as %s : %v", twin.Property.PProperty.DataType, err) + return err + } + err = dev.CustomizedClient.SetDeviceData(value, visitorConfig) + if err != nil { + return fmt.Errorf("%s set device data error: %v", twin.PropertyName, err) + } + return nil +} + +// DevInit initialize the device +func (d *DevPanel) DevInit(deviceList []*dmiapi.Device, deviceModelList []*dmiapi.DeviceModel) error { + if len(deviceList) == 0 || len(deviceModelList) == 0 { + return ErrEmptyData + } + + for i := range deviceModelList { + model := deviceModelList[i] + cur := parse.GetDeviceModelFromGrpc(model) + d.models[model.Name] = cur + } + + for i := range deviceList { + device := deviceList[i] + commonModel := d.models[device.Spec.DeviceModelReference] + protocol, err := parse.BuildProtocolFromGrpc(device) + if err != nil { + return err + } + instance, err := parse.GetDeviceFromGrpc(device, &commonModel) + if err != nil { + return err + } + instance.PProtocol = protocol + + cur := new(driver.CustomizedDev) + cur.Instance = *instance + d.devices[instance.ID] = cur + } + + return nil +} + +// UpdateDev stop old device, then update and start new device +func (d *DevPanel) UpdateDev(model *common.DeviceModel, device *common.DeviceInstance) { + d.serviceMutex.Lock() + defer d.serviceMutex.Unlock() + + if oldDevice, ok := d.devices[device.ID]; ok { + err := d.stopDev(oldDevice, device.ID) + if err != nil { + klog.Error(err) + } + } + // start new device + d.devices[device.ID] = new(driver.CustomizedDev) + d.devices[device.ID].Instance = *device + d.models[model.ID] = *model + + ctx, cancelFunc := context.WithCancel(context.Background()) + d.deviceMuxs[device.ID] = cancelFunc + d.wg.Add(1) + go d.start(ctx, d.devices[device.ID]) +} + +// UpdateDevTwins update device's twins +func (d *DevPanel) UpdateDevTwins(deviceID string, twins []common.Twin) error { + d.serviceMutex.Lock() + defer d.serviceMutex.Unlock() + dev, ok := d.devices[deviceID] + if !ok { + return fmt.Errorf("device %s not found", deviceID) + } + dev.Instance.Twins = twins + model := d.models[dev.Instance.Model] + d.UpdateDev(&model, &dev.Instance) + + return nil +} + +// DealDeviceTwinGet get device's twin data +func (d *DevPanel) DealDeviceTwinGet(deviceID string, twinName string) (interface{}, error) { + d.serviceMutex.Lock() + defer d.serviceMutex.Unlock() + dev, ok := d.devices[deviceID] + if !ok { + return nil, fmt.Errorf("not found device %s", deviceID) + } + var res []parse.TwinResultResponse + for _, twin := range dev.Instance.Twins { + if twinName != "" && twin.PropertyName != twinName { + continue + } + payload, err := getTwinData(deviceID, twin, d.devices[deviceID]) + if err != nil { + return nil, err + } + item := parse.TwinResultResponse{ + PropertyName: twinName, + Payload: payload, + } + res = append(res, item) + } + return json.Marshal(res) +} + +// getTwinData get twin +func getTwinData(deviceID string, twin common.Twin, dev *driver.CustomizedDev) ([]byte, error) { + var visitorConfig driver.VisitorConfig + err := json.Unmarshal(twin.Property.Visitors, &visitorConfig) + if err != nil { + return nil, err + } + err = setVisitor(&visitorConfig, &twin, dev) + if err != nil { + return nil, err + } + twinData := &TwinData{ + DeviceName: deviceID, + Client: dev.CustomizedClient, + Name: twin.PropertyName, + Type: twin.ObservedDesired.Metadata.Type, + VisitorConfig: &visitorConfig, + Topic: fmt.Sprintf(common.TopicTwinUpdate, deviceID), + } + return twinData.GetPayLoad() +} + +// GetDevice get device instance +func (d *DevPanel) GetDevice(deviceID string) (interface{}, error) { + d.serviceMutex.Lock() + defer d.serviceMutex.Unlock() + found, ok := d.devices[deviceID] + if !ok || found == nil { + return nil, fmt.Errorf("device %s not found", deviceID) + } + + // get the latest reported twin value + for i, twin := range found.Instance.Twins { + payload, err := getTwinData(deviceID, twin, found) + if err != nil { + return nil, err + } + found.Instance.Twins[i].Reported.Value = string(payload) + } + return found, nil +} + +// RemoveDevice remove device instance +func (d *DevPanel) RemoveDevice(deviceID string) error { + d.serviceMutex.Lock() + defer d.serviceMutex.Unlock() + dev := d.devices[deviceID] + delete(d.devices, deviceID) + err := d.stopDev(dev, deviceID) + if err != nil { + return err + } + return nil +} + +// stopDev stop device and goroutine +func (d *DevPanel) stopDev(dev *driver.CustomizedDev, id string) error { + cancelFunc, ok := d.deviceMuxs[id] + if !ok { + return fmt.Errorf("can not find device %s from device muxs", id) + } + + err := dev.CustomizedClient.StopDevice() + if err != nil { + klog.Errorf("stop device %s error: %v", id, err) + } + cancelFunc() + return nil +} + +// GetModel if the model exists, return device model +func (d *DevPanel) GetModel(modelID string) (common.DeviceModel, error) { + d.serviceMutex.Lock() + defer d.serviceMutex.Unlock() + if model, ok := d.models[modelID]; ok { + return model, nil + } + return common.DeviceModel{}, fmt.Errorf("deviceModel %s not found", modelID) +} + +// UpdateModel update device model +func (d *DevPanel) UpdateModel(model *common.DeviceModel) { + d.serviceMutex.Lock() + d.models[model.ID] = *model + d.serviceMutex.Unlock() +} + +// RemoveModel remove device model +func (d *DevPanel) RemoveModel(modelID string) { + d.serviceMutex.Lock() + delete(d.models, modelID) + d.serviceMutex.Unlock() +} + +// GetTwinResult Get twin's value and data type +func (d *DevPanel) GetTwinResult(deviceID string, twinName string) (string, string, error) { + d.serviceMutex.Lock() + defer d.serviceMutex.Unlock() + dev, ok := d.devices[deviceID] + if !ok { + return "", "", fmt.Errorf("not found device %s", deviceID) + } + var res string + var dataType string + for _, twin := range dev.Instance.Twins { + if twinName != "" && twin.PropertyName != twinName { + continue + } + var visitorConfig driver.VisitorConfig + err := json.Unmarshal(twin.Property.Visitors, &visitorConfig) + if err != nil { + return "", "", err + } + err = setVisitor(&visitorConfig, &twin, dev) + + data, err := dev.CustomizedClient.GetDeviceData(&visitorConfig) + if err != nil { + return "", "", fmt.Errorf("get device data failed: %v", err) + } + res, err = common.ConvertToString(data) + if err != nil { + return "", "", err + } + dataType = twin.Property.PProperty.DataType + } + return res, dataType, nil +} diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/device/devicestatus.go b/mappers/kubeedge-v1.17.0/mqtt-mapper/device/devicestatus.go new file mode 100644 index 00000000..a00160f9 --- /dev/null +++ b/mappers/kubeedge-v1.17.0/mqtt-mapper/device/devicestatus.go @@ -0,0 +1,69 @@ +/* +Copyright 2024 The KubeEdge Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package device + +import ( + "context" + "log" + "time" + + "k8s.io/klog/v2" + + "github.com/kubeedge/kubeedge-v1.17.0/driver" + dmiapi "github.com/kubeedge/kubeedge/pkg/apis/dmi/v1beta1" + "github.com/kubeedge/mapper-framework/pkg/grpcclient" +) + +// DeviceStates is structure for getting device states. +type DeviceStates struct { + Client *driver.CustomizedClient + DeviceName string + DeviceNamespace string +} + +// Run timer function. +func (deviceStates *DeviceStates) PushStatesToEdgeCore() { + states, error := deviceStates.Client.GetDeviceStates() + if error != nil { + klog.Errorf("GetDeviceStates failed: %v", error) + return + } + + statesRequest := &dmiapi.ReportDeviceStatesRequest{ + DeviceName: deviceStates.DeviceName, + State: states, + DeviceNamespace: deviceStates.DeviceNamespace, + } + + log.Printf("send statesRequest", statesRequest.DeviceName, statesRequest.State) + if err := grpcclient.ReportDeviceStates(statesRequest); err != nil { + klog.Errorf("fail to report device states of %s with err: %+v", deviceStates.DeviceName, err) + } +} + +func (deviceStates *DeviceStates) Run(ctx context.Context) { + // TODO setting states reportCycle + ticker := time.NewTicker(2 * time.Second) + for { + select { + case <-ticker.C: + deviceStates.PushStatesToEdgeCore() + case <-ctx.Done(): + return + } + } +} diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/device/devicetwin.go b/mappers/kubeedge-v1.17.0/mqtt-mapper/device/devicetwin.go new file mode 100644 index 00000000..12dd38d3 --- /dev/null +++ b/mappers/kubeedge-v1.17.0/mqtt-mapper/device/devicetwin.go @@ -0,0 +1,108 @@ +package device + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "k8s.io/klog/v2" + + "github.com/kubeedge/kubeedge-v1.17.0/driver" + dmiapi "github.com/kubeedge/kubeedge/pkg/apis/dmi/v1beta1" + "github.com/kubeedge/mapper-framework/pkg/common" + "github.com/kubeedge/mapper-framework/pkg/grpcclient" + "github.com/kubeedge/mapper-framework/pkg/util/parse" +) + +type TwinData struct { + DeviceName string + DeviceNamespace string + Client *driver.CustomizedClient + Name string + Type string + ObservedDesired common.TwinProperty + VisitorConfig *driver.VisitorConfig + Topic string + Results interface{} + CollectCycle time.Duration + ReportToCloud bool +} + +func (td *TwinData) GetPayLoad() ([]byte, error) { + var err error + td.VisitorConfig.VisitorConfigData.DataType = strings.ToLower(td.VisitorConfig.VisitorConfigData.DataType) + td.Results, err = td.Client.GetDeviceData(td.VisitorConfig) + if err != nil { + return nil, fmt.Errorf("get device data failed: %v", err) + } + sData, err := common.ConvertToString(td.Results) + if err != nil { + klog.Errorf("Failed to convert %s %s value as string : %v", td.DeviceName, td.Name, err) + return nil, err + } + if len(sData) > 30 { + klog.V(4).Infof("Get %s : %s ,value is %s......", td.DeviceName, td.Name, sData[:30]) + } else { + klog.V(4).Infof("Get %s : %s ,value is %s", td.DeviceName, td.Name, sData) + } + var payload []byte + if strings.Contains(td.Topic, "$hw") { + if payload, err = common.CreateMessageTwinUpdate(td.Name, td.Type, sData, td.ObservedDesired.Value); err != nil { + return nil, fmt.Errorf("create message twin update failed: %v", err) + } + } else { + if payload, err = common.CreateMessageData(td.Name, td.Type, sData); err != nil { + return nil, fmt.Errorf("create message data failed: %v", err) + } + } + return payload, nil +} + +func (td *TwinData) PushToEdgeCore() { + payload, err := td.GetPayLoad() + if err != nil { + klog.Errorf("twindata %s unmarshal failed, err: %s", td.Name, err) + return + } + + var msg common.DeviceTwinUpdate + if err = json.Unmarshal(payload, &msg); err != nil { + klog.Errorf("twindata %s unmarshal failed, err: %s", td.Name, err) + return + } + + twins := parse.ConvMsgTwinToGrpc(msg.Twin) + + var rdsr = &dmiapi.ReportDeviceStatusRequest{ + DeviceName: td.DeviceName, + DeviceNamespace: td.DeviceNamespace, + ReportedDevice: &dmiapi.DeviceStatus{ + Twins: twins, + //State: "OK", + }, + } + + if err := grpcclient.ReportDeviceStatus(rdsr); err != nil { + klog.Errorf("fail to report device status of %s with err: %+v", rdsr.DeviceName, err) + } +} + +func (td *TwinData) Run(ctx context.Context) { + if !td.ReportToCloud { + return + } + if td.CollectCycle == 0 { + td.CollectCycle = common.DefaultCollectCycle + } + ticker := time.NewTicker(td.CollectCycle) + for { + select { + case <-ticker.C: + td.PushToEdgeCore() + case <-ctx.Done(): + return + } + } +} diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/driver/devicetype.go b/mappers/kubeedge-v1.17.0/mqtt-mapper/driver/devicetype.go new file mode 100644 index 00000000..ffd999c8 --- /dev/null +++ b/mappers/kubeedge-v1.17.0/mqtt-mapper/driver/devicetype.go @@ -0,0 +1,80 @@ +package driver + +import ( + "sync" + mqtt "github.com/eclipse/paho.mqtt.golang" + "github.com/kubeedge/mapper-framework/pkg/common" +) + +// CustomizedDev is the customized device configuration and client information. +type CustomizedDev struct { + CustomizedClient *CustomizedClient + Instance common.DeviceInstance +} + +type CustomizedClient struct { + // TODO add some variables to help you better implement device drivers + deviceMutex sync.Mutex + ProtocolConfig + MessageQueue MessageQueue +} + +type ProtocolConfig struct { + ProtocolName string `json:"protocolName"` + ConfigData `json:"configData"` +} + +type ConfigData struct { + // MQTT protocol config data + ClientID string `json:"clientID"` // MQTT Client ID + Topic string `json:"topic"` // Client need to specify a topic when publishing or subsribing. + Message string `json:"message"` // Content of the message +} + + +type VisitorConfig struct { + ProtocolName string `json:"protocolName"` + VisitorConfigData `json:"configData"` +} + +type VisitorConfigData struct { + DataType string `json:"dataType"` + + ClientID string `json:"clientID"` // MQTT Client ID + DeviceInfo string `json:"deviceInfo"` // Device information, such as device identification or other important information. + OperationInfo OperationInfoType `json:"operationInfo"` // Operation information, such as adding, deleting, modifying and so on. + SerializedFormat SerializedFormatType `json:"fileType"` // Supported formats: json, xml and yaml. + ParsedMessage interface{} `json:"parsedMessage"` // The parsed message +} + +// OperationInfoType defines the enumeration values for device operation. +type OperationInfoType uint + +const ( + FULLTEXTMODIFY OperationInfoType = iota // full text revision + PATHMODIFY // path revision + VALUEMODIFY // value revision +) + +// SerializedFormatType defines the enumeration values for serialized types. +type SerializedFormatType uint + +const ( + JSON SerializedFormatType = iota // json + YAML // yaml + XML // xml +) + +// MessageQueue defines generic message queue operations and contains three methods: +// Publish is used to publish a message to the specified topic, the type of the message is interface{} in order to support multiple message formats. +// Subscribe subscribes to the specified topic, return the received message. +// Unsubscribe unsubscribes to the specified topic. +type MessageQueue interface { + Publish(topic string, message interface{}) error + Subscribe(topic string) (interface{}, error) + Unsubscribe(topic string) error +} + +type MqttMessageQueue struct { + client mqtt.Client +} diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/driver/driver.go b/mappers/kubeedge-v1.17.0/mqtt-mapper/driver/driver.go new file mode 100644 index 00000000..bebae9bd --- /dev/null +++ b/mappers/kubeedge-v1.17.0/mqtt-mapper/driver/driver.go @@ -0,0 +1,375 @@ +package driver + +import ( + "encoding/json" + "encoding/xml" + "errors" + "fmt" + "strings" + "sync" + "gopkg.in/yaml.v3" +) + +func NewClientWithMessageQueue(protocol ProtocolConfig, queue MessageQueue) (*CustomizedClient, error) { + return &CustomizedClient{ + ProtocolConfig: protocol, + deviceMutex: sync.Mutex{}, + MessageQueue: queue, + }, nil +} + +// Thread-safe access to ProtocolConfig. +func (client *CustomizedClient) GetProtocolConfig() ProtocolConfig { + client.deviceMutex.Lock() + defer client.deviceMutex.Unlock() + + return client.ProtocolConfig +} + +// Thread-safe setting of ProtocolConfig. +func (client *CustomizedClient) SetProtocolConfig(config ProtocolConfig) { + client.deviceMutex.Lock() + defer client.deviceMutex.Unlock() + + client.ProtocolConfig = config +} + +// Thread-safe publishing of messages to MQTT. +func (client *CustomizedClient) SafePublish(topic string, message interface{}) error { + client.deviceMutex.Lock() + defer client.deviceMutex.Unlock() + + if client.MessageQueue == nil { + return errors.New("message queue is not initialized") + } + + // Calls the Publish method of the message queue + err := client.MessageQueue.Publish(topic, message) + if err != nil { + return fmt.Errorf("failed to publish message: %v", err) + } + return nil +} + +// Thread-safe subscription to MQTT topics +func (client *CustomizedClient) SafeSubscribe(topic string) (interface{}, error) { + client.deviceMutex.Lock() + defer client.deviceMutex.Unlock() + + if client.MessageQueue == nil { + return nil, errors.New("message queue is not initialized") + } + + // Call the Subscribe method of the message queue + msg, err := client.MessageQueue.Subscribe(topic) + if err != nil { + return nil, fmt.Errorf("failed to subscribe to topic: %v", err) + } + return msg, nil +} + +// Thread-Safe Unsubscription to MQTT Topics +func (client *CustomizedClient) SafeUnsubscribe(topic string) error { + client.deviceMutex.Lock() + defer client.deviceMutex.Unlock() + + if client.MessageQueue == nil { + return errors.New("message queue is not initialized") + } + + // Call the Unsubscribe method of the message queue + err := client.MessageQueue.Unsubscribe(topic) + if err != nil { + return fmt.Errorf("failed to unsubscribe from topic: %v", err) + } + return nil +} + +/* --------------------------------------------------------------------------------------- */ +// The function NewConfigData is a constructor for ConfigData to initialize the structure. +// It returns the ConfigData instance and an error value to handle the validity of the passed parameters. +func NewConfigData(clientID, topic, message string) (*ConfigData, error) { + if clientID == "" { + return nil, errors.New("clientID cannot be empty") + } + if topic == "" { + return nil, errors.New("topic cannot be empty") + } + if message == "" { + message = "default message" + } + + return &ConfigData{ + ClientID: clientID, + Topic: topic, + Message: message, + }, nil +} + +// The function GetClientID returns the value of the ClientID field and error. +func (c *ConfigData) GetClientID() (string, error) { + if c.ClientID == "" { + return "", errors.New("clientID is empty") + } + return c.ClientID, nil +} + +// The function GetTopic returns the value of the Topic field and error. +func (c *ConfigData) GetTopic() (string, error) { + if c.Topic == "" { + return "", errors.New("topic is empty") + } + return c.Topic, nil +} + +// GetMessage returns the value of the Message field and error. +func (c *ConfigData) GetMessage() (string, error) { + if c.Message == "" { + return "", errors.New("message is empty") + } + return c.Message, nil +} + +// OperationInfoType and SerializedFormatType mappings +var operationTypeMap = map[string]OperationInfoType{ + "fulltextmodify": FULLTEXTMODIFY, + "pathmodify": PATHMODIFY, + "valuemodify": VALUEMODIFY, +} + +var serializedFormatMap = map[string]SerializedFormatType{ + "json": JSON, + "yaml": YAML, + "xml": XML, +} + +// The function SplitTopic splits the Topic into three parts and returns each. +// OperationInfoType(fulltextmodify: 0, pathmodify: 1, valuemodify: 2) +// SerializedFormatType(json: 0, yaml: 1, xml: 2) +func (c *ConfigData) SplitTopic() (string, OperationInfoType, SerializedFormatType, error) { + if c.Topic == "" { + return "", 0, 0, errors.New("topic is empty") + } + + parts := strings.Split(c.Topic, "/") + + if len(parts) < 3 { + return "", 0, 0, errors.New("topic format is invalid, must have at least three parts") + } + + deviceInfo := strings.Join(parts[:len(parts)-2], "/") + + // Get operation type from map + operationType, exists := operationTypeMap[parts[len(parts)-2]] + if !exists { + return "", 0, 0, errors.New("invalid operation type") + } + + // Get serialized format from map + serializedFormat, exists := serializedFormatMap[parts[len(parts)-1]] + if !exists { + return "", 0, 0, errors.New("invalid serialized format") + } + + return deviceInfo, operationType, serializedFormat, nil +} + +// The function ParseMessage parses the Message field according to the incoming type. +// parseType(0: json, 1: yaml, 2: xml) +// The value interface{} represents the parsed structure. +func (c *ConfigData) ParseMessage(parseType SerializedFormatType) (interface{}, error) { + if c.Message == "" { + return nil, errors.New("message is empty") + } + + switch parseType { + case JSON: // json + return c.parseJSON() + + case YAML: // yaml + convertedMessage, err := convertYAMLToJSON(c.Message) + if err != nil { + return nil, err + } + c.Message = convertedMessage + return c.parseJSON() + + case XML: // xml + convertedMessage, err := convertXMLToJSON(c.Message) + if err != nil { + return nil, err + } + c.Message = convertedMessage + return c.parseJSON() + + default: + return nil, errors.New("unsupported parse type") + } +} + +// The function parseJSON parses the Message field of the ConfigData (assumed to be a JSON string). +func (c *ConfigData) parseJSON() (interface{}, error) { + if c.Message == "" { + return nil, errors.New("message is empty") + } + + var result interface{} + err := json.Unmarshal([]byte(c.Message), &result) + if err != nil { + return nil, err + } + return result, nil +} + +// The function ValidateMessage checks if the message content is valid. +func (c *ConfigData) ValidateMessage() error { + if c.Message == "" { + return errors.New("message is empty") + } + + // Example: Check if the message is valid JSON (you can expand for other formats) + var temp map[string]interface{} + if err := json.Unmarshal([]byte(c.Message), &temp); err != nil { + return errors.New("message is not valid JSON") + } + + return nil +} + +// NewVisitorConfigData creates a new instance of VisitorConfigData using ConfigData pointer and the result of SplitTopic. +func (c *ConfigData) NewVisitorConfigData() (*VisitorConfigData, error) { + // get ClientID + clientID, err := c.GetClientID() + if err != nil { + return nil, err + } + + // get DeviceInfo, OperationInfo and SerializedFormat + deviceInfo, operationInfo, serializedFormat, err := c.SplitTopic() + if err != nil { + return nil, err + } + + // get ParsedMessage + parsedMessage, err := c.ParseMessage(serializedFormat) + if err != nil { + return nil, err + } + + // create + return &VisitorConfigData{ + DataType: "string", + ClientID: clientID, + DeviceInfo: deviceInfo, + OperationInfo: operationInfo, + SerializedFormat: serializedFormat, + ParsedMessage: parsedMessage, + }, nil +} + +/* --------------------------------------------------------------------------------------- */ +// The function ConvertYAMLToJSON converts a YAML string to a JSON string. +func convertYAMLToJSON(yamlString string) (string, error) { + // Converting a YAML string to a generic map object + var yamlData map[string]interface{} + err := yaml.Unmarshal([]byte(yamlString), &yamlData) + if err != nil { + return "", err + } + + // Convert a map object to a JSON string + jsonData, err := json.Marshal(yamlData) + if err != nil { + return "", err + } + + return string(jsonData), nil +} + +// The function convertXMLToJSON converts an XML string to a JSON string. +func convertXMLToJSON(xmlString string) (string, error) { + xmlData, err := convertXMLToMap(xmlString) + if err != nil { + return "", err + } + + jsonData, err := mapToJSON(xmlData) + if err != nil { + return "", err + } + + return jsonData, nil +} + +// The function ConvertXMLToMap converts XML string to map[string]interface{}. +func convertXMLToMap(xmlString string) (map[string]interface{}, error) { + // Wrap the XML content with + wrappedXML := wrapXMLWithRoot(xmlString) + + var node Node + err := xml.Unmarshal([]byte(wrappedXML), &node) + if err != nil { + return nil, err + } + return nodeToMap(node), nil +} + +// The function WrapXMLWithRoot wraps XML strings in tags. +func wrapXMLWithRoot(xmlString string) string { + // Remove the XML declaration if it exists + if strings.HasPrefix(xmlString, "") + if end != -1 { + xmlString = xmlString[end+2:] + } + } + + // Wrap the remaining XML content with + wrappedXML := "" + xmlString + "" + return wrappedXML +} + +// Node structure +type Node struct { + XMLName xml.Name + Content string `xml:",chardata"` + Nodes []Node `xml:",any"` + Attr []xml.Attr `xml:"-"` +} + +// The function nodeToMap recursively converts XML nodes to map[string]interface{}. +func nodeToMap(node Node) map[string]interface{} { + result := make(map[string]interface{}) + + if len(node.Nodes) == 0 { + // Leaf node + return map[string]interface{}{node.XMLName.Local: node.Content} + } + + for _, child := range node.Nodes { + childMap := nodeToMap(child) + if existing, found := result[child.XMLName.Local]; found { + switch v := existing.(type) { + case []interface{}: + result[child.XMLName.Local] = append(v, childMap[child.XMLName.Local]) + default: + result[child.XMLName.Local] = []interface{}{v, childMap[child.XMLName.Local]} + } + } else { + result[child.XMLName.Local] = childMap[child.XMLName.Local] + } + } + + return map[string]interface{}{node.XMLName.Local: result} +} + +// The function MapToJSON converts map[string]interface{} to JSON string. +func mapToJSON(data map[string]interface{}) (string, error) { + jsonData, err := json.Marshal(data) + if err != nil { + return "", err + } + return string(jsonData), nil +} + +/* --------------------------------------------------------------------------------------- */ \ No newline at end of file diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/driver/messagequeue.go b/mappers/kubeedge-v1.17.0/mqtt-mapper/driver/messagequeue.go new file mode 100644 index 00000000..98fb22d4 --- /dev/null +++ b/mappers/kubeedge-v1.17.0/mqtt-mapper/driver/messagequeue.go @@ -0,0 +1,60 @@ +package driver + +import ( + "fmt" + "time" + + mqtt "github.com/eclipse/paho.mqtt.golang" +) + +// NewMqttMessageQueueWithAuth creates and initializes an MqttMessageQueue instance with authentication. +func NewMqttMessageQueueWithAuth(broker string, clientID string, username string, password string) (*MqttMessageQueue, error) { + opts := mqtt.NewClientOptions(). + AddBroker(broker). + SetClientID(clientID). + SetUsername(username). + SetPassword(password). + SetConnectTimeout(10 * time.Second) + + client := mqtt.NewClient(opts) + if token := client.Connect(); token.Wait() && token.Error() != nil { + return nil, token.Error() + } + + return &MqttMessageQueue{ + client: client, + }, nil +} + + +// Publish is used to publish a message to the specified topic, the type of the message is interface{} in order to support multiple message formats. +func (mq *MqttMessageQueue) Publish(topic string, message interface{}) error { + payload, ok := message.(string) + if !ok { + return fmt.Errorf("message must be a string") + } + token := mq.client.Publish(topic, 0, false, payload) + token.Wait() + return token.Error() +} + +// Subscribe subscribes to the specified topic, return the received message. +func (mq *MqttMessageQueue) Subscribe(topic string) (interface{}, error) { + ch := make(chan mqtt.Message) + token := mq.client.Subscribe(topic, 0, func(client mqtt.Client, msg mqtt.Message) { + ch <- msg + }) + token.Wait() + if token.Error() != nil { + return nil, token.Error() + } + message := <-ch + return string(message.Payload()), nil +} + +// Unsubscribe unsubscribes to the specified topic. +func (mq *MqttMessageQueue) Unsubscribe(topic string) error { + token := mq.client.Unsubscribe(topic) + token.Wait() + return token.Error() +} diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/go.mod b/mappers/kubeedge-v1.17.0/mqtt-mapper/go.mod new file mode 100644 index 00000000..9c9a739a --- /dev/null +++ b/mappers/kubeedge-v1.17.0/mqtt-mapper/go.mod @@ -0,0 +1,41 @@ +module github.com/kubeedge/kubeedge-v1.17.0 + +go 1.21 + +require ( + github.com/eclipse/paho.mqtt.golang v1.2.0 + github.com/go-redis/redis/v8 v8.11.5 + github.com/go-sql-driver/mysql v1.7.1 + github.com/influxdata/influxdb-client-go/v2 v2.13.0 + github.com/kubeedge/kubeedge v1.18.0 + github.com/kubeedge/mapper-framework v1.17.1-0.20240727071908-23ae39c11809 + github.com/sailorvii/goav v0.1.4 + github.com/taosdata/driver-go/v3 v3.5.1 + k8s.io/klog/v2 v2.110.1 +) + +require ( + github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect + github.com/avast/retry-go v3.0.0+incompatible // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/mux v1.8.0 // indirect + github.com/gorilla/websocket v1.5.0 // indirect + github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/oapi-codegen/runtime v1.0.0 // indirect + github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace // indirect + golang.org/x/net v0.23.0 // indirect + golang.org/x/sys v0.19.0 // indirect + golang.org/x/text v0.14.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de // indirect + google.golang.org/grpc v1.63.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect +) diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/go.sum b/mappers/kubeedge-v1.17.0/mqtt-mapper/go.sum new file mode 100644 index 00000000..e19f9139 --- /dev/null +++ b/mappers/kubeedge-v1.17.0/mqtt-mapper/go.sum @@ -0,0 +1,105 @@ +github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= +github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= +github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= +github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= +github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= +github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/eclipse/paho.mqtt.golang v1.2.0 h1:1F8mhG9+aO5/xpdtFkW4SxOJB67ukuDC3t2y2qayIX0= +github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= +github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= +github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= +github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gosuri/uilive v0.0.0-20170323041506-ac356e6e42cd/go.mod h1:qkLSc0A5EXSP6B04TrN4oQoxqFI7A8XvoXSlJi8cwk8= +github.com/gosuri/uiprogress v0.0.0-20170224063937-d0567a9d84a1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0= +github.com/influxdata/influxdb-client-go/v2 v2.13.0 h1:ioBbLmR5NMbAjP4UVA5r9b5xGjpABD7j65pI8kFphDM= +github.com/influxdata/influxdb-client-go/v2 v2.13.0/go.mod h1:k+spCbt9hcvqvUiz0sr5D8LolXHqAAOfPw9v/RIRHl4= +github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 h1:W9WBk7wlPfJLvMCdtV4zPulc4uCPrlywQOmbFOhgQNU= +github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kubeedge/kubeedge v1.18.0 h1:v2RFM9c/iGSkQ0Jd4TIco1GnKFZPmdinI9P7v87VaH8= +github.com/kubeedge/kubeedge v1.18.0/go.mod h1:lw/MuITfLLSzWa5OTrzSckayd0frVuj/wbdGyvPM7xw= +github.com/kubeedge/mapper-framework v1.17.1-0.20240727071908-23ae39c11809 h1:LnToqAGc9C58zP6AW60uq+1eDrE3tUVIlpASmO/0Q94= +github.com/kubeedge/mapper-framework v1.17.1-0.20240727071908-23ae39c11809/go.mod h1:opmle2heQfoWs1HM6TtEqCrSDSQ8pstMCFsEvIl4EFY= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/oapi-codegen/runtime v1.0.0 h1:P4rqFX5fMFWqRzY9M/3YF9+aPSPPB06IzP2P7oOxrWo= +github.com/oapi-codegen/runtime v1.0.0/go.mod h1:LmCUMQuPB4M/nLXilQXhHw+BLZdDb18B34OO356yJ/A= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg= +github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sailorvii/goav v0.1.4 h1:4FwbikqIxx26dcHlZ8195WSPQSWbNnvRvTSgRTPgh2w= +github.com/sailorvii/goav v0.1.4/go.mod h1:upppsyLr1RLWDZ0+U3RYYGTv9NVwCjz14j/zzxRM018= +github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace h1:9PNP1jnUjRhfmGMlkXHjYPishpcw4jpSt/V/xYY3FMA= +github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/taosdata/driver-go/v3 v3.5.1 h1:ln8gLJ6HR6gHU6dodmOa9utUjPUpAcdIplh6arFO26Q= +github.com/taosdata/driver-go/v3 v3.5.1/go.mod h1:H2vo/At+rOPY1aMzUV9P49SVX7NlXb3LAbKw+MCLrmU= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de h1:cZGRis4/ot9uVm639a+rHCUaG0JJHEsdyzSQTMX+suY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:H4O17MA/PE9BsGx3w+a+W2VOLLD1Qf7oJneAoU6WktY= +google.golang.org/grpc v1.63.0 h1:WjKe+dnvABXyPJMD7KDNLxtoGk5tgk+YFWN6cBWjZE8= +google.golang.org/grpc v1.63.0/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= +k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/hack/make-rules/mapper.sh b/mappers/kubeedge-v1.17.0/mqtt-mapper/hack/make-rules/mapper.sh new file mode 100644 index 00000000..4cba4250 --- /dev/null +++ b/mappers/kubeedge-v1.17.0/mqtt-mapper/hack/make-rules/mapper.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +CURR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)" +ROOT_DIR="$(cd "${CURR_DIR}/../.." && pwd -P)" +source "${ROOT_DIR}/hack/lib/init.sh" + +mkdir -p "${CURR_DIR}/bin" +mkdir -p "${CURR_DIR}/dist" + +function mod() { + [[ "${2:-}" != "only" ]] + local mapper="${1}" + + # the mapper is sharing the vendor with root + pushd "${ROOT_DIR}" >/dev/null || exist 1 + echo "downloading dependencies for mapper ${mapper}..." + + if [[ "$(go env GO111MODULE)" == "off" ]]; then + echo "go mod has been disabled by GO111MODULE=off" + else + echo "tidying" + go mod tidy + echo "vending" + go mod vendor + fi + + echo "...done" + popd >/dev/null || return +} + +function lint() { + [[ "${2:-}" != "only" ]] && mod "$@" + local mapper="${1}" + + echo "fmt and linting mapper ${mapper}..." + + gofmt -s -w "${CURR_DIR}/" + golangci-lint run "${CURR_DIR}/..." + + echo "...done" +} + +function build() { + [[ "${2:-}" != "only" ]] && lint "$@" + local mapper="${1}" + + local flags=" -w -s " + local ext_flags=" -extldflags '-static' " + local os="${OS:-$(go env GOOS)}" + local arch="${ARCH:-$(go env GOARCH)}" + + local platform + if [[ "${ARM:-false}" == "true" ]]; then + echo "crossed packaging for linux/arm" + platform=("linux/arm") + elif [[ "${ARM64:-false}" == "true" ]]; then + echo "crossed packaging for linux/arm64" + platform=("linux/arm64") + else + local os="${OS:-$(go env GOOS)}" + local arch="${ARCH:-$(go env GOARCH)}" + platform=("${os}/${arch}") + fi + + echo "building ${platform}" + + local os_arch + IFS="/" read -r -a os_arch <<<"${platform}" + local os=${os_arch[0]} + local arch=${os_arch[1]} + GOOS=${os} GOARCH=${arch} CGO_ENABLED=0 go build \ + -ldflags "${flags} ${ext_flags}" \ + -o "${CURR_DIR}/bin/${mapper}_${os}_${arch}" \ + "${CURR_DIR}/cmd/main.go" + + cp ${CURR_DIR}/bin/${mapper}_${os}_${arch} ${CURR_DIR}/bin/${mapper} + echo "...done" +} + +function package() { + [[ "${2:-}" != "only" ]] && build "$@" + local mapper="${1}" + + echo "packaging mapper ${mapper}..." + + local image_name="${mapper}-mapper" + local tag=v1.0 + + local platform + if [[ "${ARM:-false}" == "true" ]]; then + echo "crossed packaging for linux/arm" + platform=("linux/arm") + elif [[ "${ARM64:-false}" == "true" ]]; then + echo "crossed packaging for linux/arm64" + platform=("linux/arm64") + else + local os="${OS:-$(go env GOOS)}" + local arch="${ARCH:-$(go env GOARCH)}" + platform=("${os}/${arch}") + fi + + pushd "${CURR_DIR}" >/dev/null 2>&1 + if [[ "${platform}" =~ darwin/* ]]; then + echo "package into Darwin OS image is unavailable, please use CROSS=true env to containerize multiple arch images or use OS=linux ARCH=amd64 env to containerize linux/amd64 image" + fi + + local image_tag="${image_name}:${tag}-${platform////-}" + echo "packaging ${image_tag}" + sudo docker build \ + --platform "${platform}" \ + -t "${image_tag}" . + popd >/dev/null 2>&1 + + echo "...done" +} + +function clean() { + local mapper="${1}" + + echo "cleanup mapper ${mapper}..." + + rm -rf "${CURR_DIR}/bin/*" + + echo "...done" +} + +function entry() { + local mapper="${1:-}" + shift 1 + + local stages="${1:-build}" + shift $(($# > 0 ? 1 : 0)) + + IFS="," read -r -a stages <<<"${stages}" + local commands=$* + if [[ ${#stages[@]} -ne 1 ]]; then + commands="only" + fi + + for stage in "${stages[@]}"; do + echo "# make mapper ${mapper} ${stage} ${commands}" + case ${stage} in + m | mod) mod "${mapper}" "${commands}" ;; + l | lint) lint "${mapper}" "${commands}" ;; + b | build) build "${mapper}" "${commands}" ;; + p | pkg | package) package "${mapper}" "${commands}" ;; + t | test) test "${mapper}" "${commands}" ;; + c | clean) clean "${mapper}" "${commands}" ;; + *) echo "unknown action '${stage}', select from mod,lint,build,test,clean" ;; + esac + done +} + +echo $@ +entry "$@" \ No newline at end of file diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/resource/configmap.yaml b/mappers/kubeedge-v1.17.0/mqtt-mapper/resource/configmap.yaml new file mode 100644 index 00000000..cddc6279 --- /dev/null +++ b/mappers/kubeedge-v1.17.0/mqtt-mapper/resource/configmap.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: cm-mapper +data: + configData: | + grpc_server: + socket_path: /etc/kubeedge/kubeedge-v1.17.0.sock + common: + name: Kubeedge-V1.17.0-mapper + version: v1.13.0 + api_version: v1.0.0 + protocol: # TODO add your protocol name + address: 127.0.0.1 + edgecore_sock: /etc/kubeedge/dmi.sock diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/resource/deployment.yaml b/mappers/kubeedge-v1.17.0/mqtt-mapper/resource/deployment.yaml new file mode 100644 index 00000000..4df3db54 --- /dev/null +++ b/mappers/kubeedge-v1.17.0/mqtt-mapper/resource/deployment.yaml @@ -0,0 +1,51 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mapper-test + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app: demo + template: + metadata: + labels: + app: demo + spec: + nodeName: # replace with your edge node name + containers: + - name: demo + volumeMounts: # Required, mapper need to communicate with grpcclient and get the config + - name: test-volume + mountPath: /etc/kubeedge + - name: config + mountPath: /tmp + env: # Not Required, this field is used to mount the user database key + - name: TOKEN + valueFrom: + secretKeyRef: + name: mysecret + key: token + image: # Replace with your mapper image name + imagePullPolicy: IfNotPresent + resources: + limits: + cpu: 300m + memory: 500Mi + requests: + cpu: 100m + memory: 100Mi + command: [ "/bin/sh","-c" ] + args: [ "/kubeedge/main --config-file /tmp/config.yaml --v 4" ] + volumes: + - name: test-volume + hostPath: + path: /etc/kubeedge + type: Directory + - name: config + configMap: + name: cm-mapper + items: + - key: configData + path: config.yaml From 95c0447615ea1435999c7da59a7aa4c263dfeb42 Mon Sep 17 00:00:00 2001 From: Prometheus-collab Date: Wed, 18 Sep 2024 14:53:16 +0800 Subject: [PATCH 2/4] Providing a Common MQTT Mapper Signed-off-by: Prometheus-collab --- .../mqtt-mapper/driver/devicetype.go | 14 +-- .../mqtt-mapper/driver/driver.go | 114 ++++++++++++++++-- 2 files changed, 112 insertions(+), 16 deletions(-) diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/driver/devicetype.go b/mappers/kubeedge-v1.17.0/mqtt-mapper/driver/devicetype.go index ffd999c8..dc714464 100644 --- a/mappers/kubeedge-v1.17.0/mqtt-mapper/driver/devicetype.go +++ b/mappers/kubeedge-v1.17.0/mqtt-mapper/driver/devicetype.go @@ -40,20 +40,18 @@ type VisitorConfig struct { type VisitorConfigData struct { DataType string `json:"dataType"` - ClientID string `json:"clientID"` // MQTT Client ID - DeviceInfo string `json:"deviceInfo"` // Device information, such as device identification or other important information. - OperationInfo OperationInfoType `json:"operationInfo"` // Operation information, such as adding, deleting, modifying and so on. - SerializedFormat SerializedFormatType `json:"fileType"` // Supported formats: json, xml and yaml. - ParsedMessage interface{} `json:"parsedMessage"` // The parsed message + ClientID string `json:"clientID"` // MQTT Client ID + DeviceInfo string `json:"deviceInfo"` // Device information, such as device identification or other important information. + OperationInfo OperationInfoType `json:"operationInfo"` // Operation information, such as adding, deleting, modifying and so on. + SerializedFormat SerializedFormatType `json:"fileType"` // Supported formats: json, xml and yaml. + ParsedMessage map[string]interface{} `json:"parsedMessage"` // The parsed message } // OperationInfoType defines the enumeration values for device operation. type OperationInfoType uint const ( - FULLTEXTMODIFY OperationInfoType = iota // full text revision - PATHMODIFY // path revision - VALUEMODIFY // value revision + UPDATE OperationInfoType = iota // revision ) // SerializedFormatType defines the enumeration values for serialized types. diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/driver/driver.go b/mappers/kubeedge-v1.17.0/mqtt-mapper/driver/driver.go index bebae9bd..a6e10f81 100644 --- a/mappers/kubeedge-v1.17.0/mqtt-mapper/driver/driver.go +++ b/mappers/kubeedge-v1.17.0/mqtt-mapper/driver/driver.go @@ -5,8 +5,10 @@ import ( "encoding/xml" "errors" "fmt" + "reflect" "strings" "sync" + "gopkg.in/yaml.v3" ) @@ -132,9 +134,7 @@ func (c *ConfigData) GetMessage() (string, error) { // OperationInfoType and SerializedFormatType mappings var operationTypeMap = map[string]OperationInfoType{ - "fulltextmodify": FULLTEXTMODIFY, - "pathmodify": PATHMODIFY, - "valuemodify": VALUEMODIFY, + "update": UPDATE, } var serializedFormatMap = map[string]SerializedFormatType{ @@ -177,7 +177,7 @@ func (c *ConfigData) SplitTopic() (string, OperationInfoType, SerializedFormatTy // The function ParseMessage parses the Message field according to the incoming type. // parseType(0: json, 1: yaml, 2: xml) // The value interface{} represents the parsed structure. -func (c *ConfigData) ParseMessage(parseType SerializedFormatType) (interface{}, error) { +func (c *ConfigData) ParseMessage(parseType SerializedFormatType) (map[string]interface{}, error) { if c.Message == "" { return nil, errors.New("message is empty") } @@ -200,7 +200,15 @@ func (c *ConfigData) ParseMessage(parseType SerializedFormatType) (interface{}, return nil, err } c.Message = convertedMessage - return c.parseJSON() + originalMap, err := c.parseJSON() + var mp map[string]interface{} + for _, value := range originalMap { + if nestedMap, ok := value.(map[string]interface{}); ok { + mp = nestedMap + break + } + } + return mp, err default: return nil, errors.New("unsupported parse type") @@ -208,12 +216,12 @@ func (c *ConfigData) ParseMessage(parseType SerializedFormatType) (interface{}, } // The function parseJSON parses the Message field of the ConfigData (assumed to be a JSON string). -func (c *ConfigData) parseJSON() (interface{}, error) { +func (c *ConfigData) parseJSON() (map[string]interface{}, error) { if c.Message == "" { return nil, errors.New("message is empty") } - var result interface{} + var result map[string]interface{} err := json.Unmarshal([]byte(c.Message), &result) if err != nil { return nil, err @@ -267,6 +275,95 @@ func (c *ConfigData) NewVisitorConfigData() (*VisitorConfigData, error) { }, nil } +/* --------------------------------------------------------------------------------------- */ +func (v *VisitorConfigData) ModifyVisitorConfigData(destDataConfig interface{}) error { + destValue := reflect.ValueOf(destDataConfig) + if destValue.Kind() != reflect.Ptr || destValue.Elem().Kind() != reflect.Struct { + return errors.New("destDataConfig must be a pointer to a struct") + } + + destValue = destValue.Elem() + + var tagName string + switch v.SerializedFormat { + case JSON: + tagName = "json" + case YAML: + tagName = "yaml" + case XML: + tagName = "xml" + default: + return errors.New("unknown serialized format") + } + + // Update the destination struct using JSON tag + if err := updateStructFields(destValue, v.ParsedMessage, tagName); err != nil { + return err + } + + return nil +} + +// updateStructFields recursively updates struct fields from the given map using specified tag type +func updateStructFields(structValue reflect.Value, data map[string]interface{}, tagName string) error { + structType := structValue.Type() + + for i := 0; i < structValue.NumField(); i++ { + field := structValue.Field(i) + fieldType := structType.Field(i) + tagValue := fieldType.Tag.Get(tagName) + + if tagValue == "" { + // Skip fields without the specified tag + continue + } + + // Get the corresponding value from the map + value, exists := data[tagValue] + if !exists { + continue + } + + // Update the field based on its kind + if field.Kind() == reflect.Struct { + // Recursively update nested structs + nestedData, ok := value.(map[string]interface{}) + if !ok { + return fmt.Errorf("type mismatch for nested field %s", tagValue) + } + if err := updateStructFields(field, nestedData, tagName); err != nil { + return err + } + } else if field.Kind() == reflect.Slice { + // Handle slices if necessary + sliceData, ok := value.([]interface{}) + if !ok { + return fmt.Errorf("type mismatch for slice field %s", tagValue) + } + newSlice := reflect.MakeSlice(field.Type(), len(sliceData), len(sliceData)) + for j, item := range sliceData { + itemValue := reflect.ValueOf(item) + if newSlice.Index(j).Kind() == itemValue.Kind() { + newSlice.Index(j).Set(itemValue) + } else { + return fmt.Errorf("type mismatch for slice item in field %s", tagValue) + } + } + field.Set(newSlice) + } else { + // Set the field value + fieldValue := reflect.ValueOf(value) + if field.Type() == fieldValue.Type() { + field.Set(fieldValue) + } else { + return fmt.Errorf("type mismatch for field %s", tagValue) + } + } + } + return nil +} + + /* --------------------------------------------------------------------------------------- */ // The function ConvertYAMLToJSON converts a YAML string to a JSON string. func convertYAMLToJSON(yamlString string) (string, error) { @@ -325,7 +422,8 @@ func wrapXMLWithRoot(xmlString string) string { } // Wrap the remaining XML content with - wrappedXML := "" + xmlString + "" + // wrappedXML := "" + xmlString + "" + wrappedXML := xmlString return wrappedXML } From 8101572581c06092e2c0488e8c264346b9ed06c3 Mon Sep 17 00:00:00 2001 From: Prometheus-collab Date: Tue, 24 Sep 2024 19:31:08 +0800 Subject: [PATCH 3/4] Providing a Common MQTT Mapper Signed-off-by: Prometheus-collab --- .../mqtt-mapper/driver/devicetype.go | 78 ------- .../mqtt-mapper/driver/messagequeue.go | 60 ----- .../mqtt-mapper/Dockerfile | 0 .../mqtt-mapper/Dockerfile_nostream | 0 .../mqtt-mapper/Dockerfile_stream | 0 .../mqtt-mapper/Makefile | 0 .../mqtt-mapper/cmd/main.go | 0 .../mqtt-mapper/config.yaml | 0 .../data/dbmethod/influxdb2/client.go | 0 .../data/dbmethod/influxdb2/handler.go | 0 .../mqtt-mapper/data/dbmethod/mysql/client.go | 0 .../data/dbmethod/mysql/handler.go | 0 .../mqtt-mapper/data/dbmethod/redis/client.go | 0 .../data/dbmethod/redis/handler.go | 0 .../data/dbmethod/tdengine/client.go | 0 .../data/dbmethod/tdengine/handler.go | 0 .../mqtt-mapper/data/publish/http/client.go | 0 .../mqtt-mapper/data/publish/mqtt/client.go | 0 .../mqtt-mapper/data/stream/handler.go | 0 .../mqtt-mapper/data/stream/img.go | 0 .../mqtt-mapper/data/stream/video.go | 0 .../mqtt-mapper/device/device.go | 0 .../mqtt-mapper/device/devicestatus.go | 0 .../mqtt-mapper/device/devicetwin.go | 0 mappers/mqtt-mapper/driver/devicetype.go | 72 ++++++ .../mqtt-mapper/driver/driver.go | 212 ++++++++---------- .../{kubeedge-v1.17.0 => }/mqtt-mapper/go.mod | 0 .../{kubeedge-v1.17.0 => }/mqtt-mapper/go.sum | 0 .../mqtt-mapper/hack/make-rules/mapper.sh | 0 .../mqtt-mapper/resource/configmap.yaml | 0 .../mqtt-mapper/resource/deployment.yaml | 0 31 files changed, 163 insertions(+), 259 deletions(-) delete mode 100644 mappers/kubeedge-v1.17.0/mqtt-mapper/driver/devicetype.go delete mode 100644 mappers/kubeedge-v1.17.0/mqtt-mapper/driver/messagequeue.go rename mappers/{kubeedge-v1.17.0 => }/mqtt-mapper/Dockerfile (100%) rename mappers/{kubeedge-v1.17.0 => }/mqtt-mapper/Dockerfile_nostream (100%) rename mappers/{kubeedge-v1.17.0 => }/mqtt-mapper/Dockerfile_stream (100%) rename mappers/{kubeedge-v1.17.0 => }/mqtt-mapper/Makefile (100%) rename mappers/{kubeedge-v1.17.0 => }/mqtt-mapper/cmd/main.go (100%) rename mappers/{kubeedge-v1.17.0 => }/mqtt-mapper/config.yaml (100%) rename mappers/{kubeedge-v1.17.0 => }/mqtt-mapper/data/dbmethod/influxdb2/client.go (100%) rename mappers/{kubeedge-v1.17.0 => }/mqtt-mapper/data/dbmethod/influxdb2/handler.go (100%) rename mappers/{kubeedge-v1.17.0 => }/mqtt-mapper/data/dbmethod/mysql/client.go (100%) rename mappers/{kubeedge-v1.17.0 => }/mqtt-mapper/data/dbmethod/mysql/handler.go (100%) rename mappers/{kubeedge-v1.17.0 => }/mqtt-mapper/data/dbmethod/redis/client.go (100%) rename mappers/{kubeedge-v1.17.0 => }/mqtt-mapper/data/dbmethod/redis/handler.go (100%) rename mappers/{kubeedge-v1.17.0 => }/mqtt-mapper/data/dbmethod/tdengine/client.go (100%) rename mappers/{kubeedge-v1.17.0 => }/mqtt-mapper/data/dbmethod/tdengine/handler.go (100%) rename mappers/{kubeedge-v1.17.0 => }/mqtt-mapper/data/publish/http/client.go (100%) rename mappers/{kubeedge-v1.17.0 => }/mqtt-mapper/data/publish/mqtt/client.go (100%) rename mappers/{kubeedge-v1.17.0 => }/mqtt-mapper/data/stream/handler.go (100%) rename mappers/{kubeedge-v1.17.0 => }/mqtt-mapper/data/stream/img.go (100%) rename mappers/{kubeedge-v1.17.0 => }/mqtt-mapper/data/stream/video.go (100%) rename mappers/{kubeedge-v1.17.0 => }/mqtt-mapper/device/device.go (100%) rename mappers/{kubeedge-v1.17.0 => }/mqtt-mapper/device/devicestatus.go (100%) rename mappers/{kubeedge-v1.17.0 => }/mqtt-mapper/device/devicetwin.go (100%) create mode 100644 mappers/mqtt-mapper/driver/devicetype.go rename mappers/{kubeedge-v1.17.0 => }/mqtt-mapper/driver/driver.go (72%) rename mappers/{kubeedge-v1.17.0 => }/mqtt-mapper/go.mod (100%) rename mappers/{kubeedge-v1.17.0 => }/mqtt-mapper/go.sum (100%) rename mappers/{kubeedge-v1.17.0 => }/mqtt-mapper/hack/make-rules/mapper.sh (100%) rename mappers/{kubeedge-v1.17.0 => }/mqtt-mapper/resource/configmap.yaml (100%) rename mappers/{kubeedge-v1.17.0 => }/mqtt-mapper/resource/deployment.yaml (100%) diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/driver/devicetype.go b/mappers/kubeedge-v1.17.0/mqtt-mapper/driver/devicetype.go deleted file mode 100644 index dc714464..00000000 --- a/mappers/kubeedge-v1.17.0/mqtt-mapper/driver/devicetype.go +++ /dev/null @@ -1,78 +0,0 @@ -package driver - -import ( - "sync" - mqtt "github.com/eclipse/paho.mqtt.golang" - "github.com/kubeedge/mapper-framework/pkg/common" -) - -// CustomizedDev is the customized device configuration and client information. -type CustomizedDev struct { - CustomizedClient *CustomizedClient - Instance common.DeviceInstance -} - -type CustomizedClient struct { - // TODO add some variables to help you better implement device drivers - deviceMutex sync.Mutex - ProtocolConfig - MessageQueue MessageQueue -} - -type ProtocolConfig struct { - ProtocolName string `json:"protocolName"` - ConfigData `json:"configData"` -} - -type ConfigData struct { - // MQTT protocol config data - ClientID string `json:"clientID"` // MQTT Client ID - Topic string `json:"topic"` // Client need to specify a topic when publishing or subsribing. - Message string `json:"message"` // Content of the message -} - - -type VisitorConfig struct { - ProtocolName string `json:"protocolName"` - VisitorConfigData `json:"configData"` -} - -type VisitorConfigData struct { - DataType string `json:"dataType"` - - ClientID string `json:"clientID"` // MQTT Client ID - DeviceInfo string `json:"deviceInfo"` // Device information, such as device identification or other important information. - OperationInfo OperationInfoType `json:"operationInfo"` // Operation information, such as adding, deleting, modifying and so on. - SerializedFormat SerializedFormatType `json:"fileType"` // Supported formats: json, xml and yaml. - ParsedMessage map[string]interface{} `json:"parsedMessage"` // The parsed message -} - -// OperationInfoType defines the enumeration values for device operation. -type OperationInfoType uint - -const ( - UPDATE OperationInfoType = iota // revision -) - -// SerializedFormatType defines the enumeration values for serialized types. -type SerializedFormatType uint - -const ( - JSON SerializedFormatType = iota // json - YAML // yaml - XML // xml -) - -// MessageQueue defines generic message queue operations and contains three methods: -// Publish is used to publish a message to the specified topic, the type of the message is interface{} in order to support multiple message formats. -// Subscribe subscribes to the specified topic, return the received message. -// Unsubscribe unsubscribes to the specified topic. -type MessageQueue interface { - Publish(topic string, message interface{}) error - Subscribe(topic string) (interface{}, error) - Unsubscribe(topic string) error -} - -type MqttMessageQueue struct { - client mqtt.Client -} diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/driver/messagequeue.go b/mappers/kubeedge-v1.17.0/mqtt-mapper/driver/messagequeue.go deleted file mode 100644 index 98fb22d4..00000000 --- a/mappers/kubeedge-v1.17.0/mqtt-mapper/driver/messagequeue.go +++ /dev/null @@ -1,60 +0,0 @@ -package driver - -import ( - "fmt" - "time" - - mqtt "github.com/eclipse/paho.mqtt.golang" -) - -// NewMqttMessageQueueWithAuth creates and initializes an MqttMessageQueue instance with authentication. -func NewMqttMessageQueueWithAuth(broker string, clientID string, username string, password string) (*MqttMessageQueue, error) { - opts := mqtt.NewClientOptions(). - AddBroker(broker). - SetClientID(clientID). - SetUsername(username). - SetPassword(password). - SetConnectTimeout(10 * time.Second) - - client := mqtt.NewClient(opts) - if token := client.Connect(); token.Wait() && token.Error() != nil { - return nil, token.Error() - } - - return &MqttMessageQueue{ - client: client, - }, nil -} - - -// Publish is used to publish a message to the specified topic, the type of the message is interface{} in order to support multiple message formats. -func (mq *MqttMessageQueue) Publish(topic string, message interface{}) error { - payload, ok := message.(string) - if !ok { - return fmt.Errorf("message must be a string") - } - token := mq.client.Publish(topic, 0, false, payload) - token.Wait() - return token.Error() -} - -// Subscribe subscribes to the specified topic, return the received message. -func (mq *MqttMessageQueue) Subscribe(topic string) (interface{}, error) { - ch := make(chan mqtt.Message) - token := mq.client.Subscribe(topic, 0, func(client mqtt.Client, msg mqtt.Message) { - ch <- msg - }) - token.Wait() - if token.Error() != nil { - return nil, token.Error() - } - message := <-ch - return string(message.Payload()), nil -} - -// Unsubscribe unsubscribes to the specified topic. -func (mq *MqttMessageQueue) Unsubscribe(topic string) error { - token := mq.client.Unsubscribe(topic) - token.Wait() - return token.Error() -} diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/Dockerfile b/mappers/mqtt-mapper/Dockerfile similarity index 100% rename from mappers/kubeedge-v1.17.0/mqtt-mapper/Dockerfile rename to mappers/mqtt-mapper/Dockerfile diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/Dockerfile_nostream b/mappers/mqtt-mapper/Dockerfile_nostream similarity index 100% rename from mappers/kubeedge-v1.17.0/mqtt-mapper/Dockerfile_nostream rename to mappers/mqtt-mapper/Dockerfile_nostream diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/Dockerfile_stream b/mappers/mqtt-mapper/Dockerfile_stream similarity index 100% rename from mappers/kubeedge-v1.17.0/mqtt-mapper/Dockerfile_stream rename to mappers/mqtt-mapper/Dockerfile_stream diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/Makefile b/mappers/mqtt-mapper/Makefile similarity index 100% rename from mappers/kubeedge-v1.17.0/mqtt-mapper/Makefile rename to mappers/mqtt-mapper/Makefile diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/cmd/main.go b/mappers/mqtt-mapper/cmd/main.go similarity index 100% rename from mappers/kubeedge-v1.17.0/mqtt-mapper/cmd/main.go rename to mappers/mqtt-mapper/cmd/main.go diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/config.yaml b/mappers/mqtt-mapper/config.yaml similarity index 100% rename from mappers/kubeedge-v1.17.0/mqtt-mapper/config.yaml rename to mappers/mqtt-mapper/config.yaml diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/influxdb2/client.go b/mappers/mqtt-mapper/data/dbmethod/influxdb2/client.go similarity index 100% rename from mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/influxdb2/client.go rename to mappers/mqtt-mapper/data/dbmethod/influxdb2/client.go diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/influxdb2/handler.go b/mappers/mqtt-mapper/data/dbmethod/influxdb2/handler.go similarity index 100% rename from mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/influxdb2/handler.go rename to mappers/mqtt-mapper/data/dbmethod/influxdb2/handler.go diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/mysql/client.go b/mappers/mqtt-mapper/data/dbmethod/mysql/client.go similarity index 100% rename from mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/mysql/client.go rename to mappers/mqtt-mapper/data/dbmethod/mysql/client.go diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/mysql/handler.go b/mappers/mqtt-mapper/data/dbmethod/mysql/handler.go similarity index 100% rename from mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/mysql/handler.go rename to mappers/mqtt-mapper/data/dbmethod/mysql/handler.go diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/redis/client.go b/mappers/mqtt-mapper/data/dbmethod/redis/client.go similarity index 100% rename from mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/redis/client.go rename to mappers/mqtt-mapper/data/dbmethod/redis/client.go diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/redis/handler.go b/mappers/mqtt-mapper/data/dbmethod/redis/handler.go similarity index 100% rename from mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/redis/handler.go rename to mappers/mqtt-mapper/data/dbmethod/redis/handler.go diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/tdengine/client.go b/mappers/mqtt-mapper/data/dbmethod/tdengine/client.go similarity index 100% rename from mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/tdengine/client.go rename to mappers/mqtt-mapper/data/dbmethod/tdengine/client.go diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/tdengine/handler.go b/mappers/mqtt-mapper/data/dbmethod/tdengine/handler.go similarity index 100% rename from mappers/kubeedge-v1.17.0/mqtt-mapper/data/dbmethod/tdengine/handler.go rename to mappers/mqtt-mapper/data/dbmethod/tdengine/handler.go diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/data/publish/http/client.go b/mappers/mqtt-mapper/data/publish/http/client.go similarity index 100% rename from mappers/kubeedge-v1.17.0/mqtt-mapper/data/publish/http/client.go rename to mappers/mqtt-mapper/data/publish/http/client.go diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/data/publish/mqtt/client.go b/mappers/mqtt-mapper/data/publish/mqtt/client.go similarity index 100% rename from mappers/kubeedge-v1.17.0/mqtt-mapper/data/publish/mqtt/client.go rename to mappers/mqtt-mapper/data/publish/mqtt/client.go diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/data/stream/handler.go b/mappers/mqtt-mapper/data/stream/handler.go similarity index 100% rename from mappers/kubeedge-v1.17.0/mqtt-mapper/data/stream/handler.go rename to mappers/mqtt-mapper/data/stream/handler.go diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/data/stream/img.go b/mappers/mqtt-mapper/data/stream/img.go similarity index 100% rename from mappers/kubeedge-v1.17.0/mqtt-mapper/data/stream/img.go rename to mappers/mqtt-mapper/data/stream/img.go diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/data/stream/video.go b/mappers/mqtt-mapper/data/stream/video.go similarity index 100% rename from mappers/kubeedge-v1.17.0/mqtt-mapper/data/stream/video.go rename to mappers/mqtt-mapper/data/stream/video.go diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/device/device.go b/mappers/mqtt-mapper/device/device.go similarity index 100% rename from mappers/kubeedge-v1.17.0/mqtt-mapper/device/device.go rename to mappers/mqtt-mapper/device/device.go diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/device/devicestatus.go b/mappers/mqtt-mapper/device/devicestatus.go similarity index 100% rename from mappers/kubeedge-v1.17.0/mqtt-mapper/device/devicestatus.go rename to mappers/mqtt-mapper/device/devicestatus.go diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/device/devicetwin.go b/mappers/mqtt-mapper/device/devicetwin.go similarity index 100% rename from mappers/kubeedge-v1.17.0/mqtt-mapper/device/devicetwin.go rename to mappers/mqtt-mapper/device/devicetwin.go diff --git a/mappers/mqtt-mapper/driver/devicetype.go b/mappers/mqtt-mapper/driver/devicetype.go new file mode 100644 index 00000000..0d01e426 --- /dev/null +++ b/mappers/mqtt-mapper/driver/devicetype.go @@ -0,0 +1,72 @@ +package driver + +import ( + "sync" + "time" + + "github.com/kubeedge/mapper-framework/pkg/common" +) + +// CustomizedDev is the customized device configuration and client information. +type CustomizedDev struct { + CustomizedClient *CustomizedClient + Instance common.DeviceInstance +} + +type CustomizedClient struct { + // TODO add some variables to help you better implement device drivers + deviceMutex sync.Mutex + ProtocolConfig + DeviceInfo string `json:"deviceInfo"` + ParsedDeviceInfo map[string]interface{} `json:"parsedDeviceInfo"` +} + +type ProtocolConfig struct { + ProtocolName string `json:"protocolName"` + ConfigData `json:"configData"` +} + +type ConfigData struct { + // MQTT protocol config data + ClientID string `json:"clientID"` // MQTT Client ID + BrokerURL string `json:"brokerURL"` // MQTT Broker URL + Topic string `json:"topic"` // Topic for publishing or subscribing + Message string `json:"message"` // Content of the message + Username string `json:"username"` // Username for MQTT broker authentication + Password string `json:"password"` // Password for MQTT broker authentication + ConnectionTTL time.Duration `json:"connectionTTL"` // Connection timeout duration + LastMessage time.Time `json:"lastMessage"` // Timestamp of the last received message + IsData bool `json:"isData"` // Indicates if there is valid data +} + +type VisitorConfig struct { + ProtocolName string `json:"protocolName"` + VisitorConfigData `json:"configData"` +} + +type VisitorConfigData struct { + DataType string `json:"dataType"` + + ClientID string `json:"clientID"` // MQTT Client ID + DeviceInfo string `json:"deviceInfo"` // Device information, such as device identification or other important information. + OperationInfo OperationInfoType `json:"operationInfo"` // Operation information, such as adding, deleting, modifying and so on. + SerializedFormat SerializedFormatType `json:"fileType"` // Supported formats: json, xml and yaml. + ParsedMessage map[string]interface{} `json:"parsedMessage"` // The parsed message +} + +// OperationInfoType defines the enumeration values for device operation. +type OperationInfoType uint + +const ( + UPDATE OperationInfoType = iota // revision +) + +// SerializedFormatType defines the enumeration values for serialized types. +type SerializedFormatType uint + +const ( + JSON SerializedFormatType = iota // json + YAML // yaml + XML // xml + JSONPATH +) diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/driver/driver.go b/mappers/mqtt-mapper/driver/driver.go similarity index 72% rename from mappers/kubeedge-v1.17.0/mqtt-mapper/driver/driver.go rename to mappers/mqtt-mapper/driver/driver.go index a6e10f81..078a14de 100644 --- a/mappers/kubeedge-v1.17.0/mqtt-mapper/driver/driver.go +++ b/mappers/mqtt-mapper/driver/driver.go @@ -6,85 +6,46 @@ import ( "errors" "fmt" "reflect" + "strconv" "strings" "sync" + "github.com/kubeedge/mapper-framework/pkg/common" "gopkg.in/yaml.v3" ) -func NewClientWithMessageQueue(protocol ProtocolConfig, queue MessageQueue) (*CustomizedClient, error) { - return &CustomizedClient{ - ProtocolConfig: protocol, - deviceMutex: sync.Mutex{}, - MessageQueue: queue, - }, nil +func NewClient(protocol ProtocolConfig) (*CustomizedClient, error) { + client := &CustomizedClient{ + ProtocolConfig: protocol, + deviceMutex: sync.Mutex{}, + DeviceInfo: "", + ParsedDeviceInfo: make(map[string]interface{}), + } + return client, nil } -// Thread-safe access to ProtocolConfig. -func (client *CustomizedClient) GetProtocolConfig() ProtocolConfig { - client.deviceMutex.Lock() - defer client.deviceMutex.Unlock() - - return client.ProtocolConfig +func (c *CustomizedClient) InitDevice() error { + configData := &c.ProtocolConfig.ConfigData + _, _, format, _ := configData.SplitTopic() + c.DeviceInfo = c.ProtocolConfig.ConfigData.Message + c.ParsedDeviceInfo, _ = c.ParseMessage(format) + return nil } -// Thread-safe setting of ProtocolConfig. -func (client *CustomizedClient) SetProtocolConfig(config ProtocolConfig) { - client.deviceMutex.Lock() - defer client.deviceMutex.Unlock() +func (c *CustomizedClient) GetDeviceData(visitor *VisitorConfig) (interface{}, error) { - client.ProtocolConfig = config + return nil, nil } -// Thread-safe publishing of messages to MQTT. -func (client *CustomizedClient) SafePublish(topic string, message interface{}) error { - client.deviceMutex.Lock() - defer client.deviceMutex.Unlock() - - if client.MessageQueue == nil { - return errors.New("message queue is not initialized") - } - - // Calls the Publish method of the message queue - err := client.MessageQueue.Publish(topic, message) - if err != nil { - return fmt.Errorf("failed to publish message: %v", err) - } +func (c *CustomizedClient) SetDeviceData(data interface{}, visitor *VisitorConfig) error { + vPointer := visitor.VisitorConfigData + vPointer.ModifyVisitorConfigData(c.ParsedDeviceInfo) return nil } -// Thread-safe subscription to MQTT topics -func (client *CustomizedClient) SafeSubscribe(topic string) (interface{}, error) { - client.deviceMutex.Lock() - defer client.deviceMutex.Unlock() - - if client.MessageQueue == nil { - return nil, errors.New("message queue is not initialized") - } - - // Call the Subscribe method of the message queue - msg, err := client.MessageQueue.Subscribe(topic) - if err != nil { - return nil, fmt.Errorf("failed to subscribe to topic: %v", err) - } - return msg, nil -} - -// Thread-Safe Unsubscription to MQTT Topics -func (client *CustomizedClient) SafeUnsubscribe(topic string) error { - client.deviceMutex.Lock() - defer client.deviceMutex.Unlock() - - if client.MessageQueue == nil { - return errors.New("message queue is not initialized") - } - - // Call the Unsubscribe method of the message queue - err := client.MessageQueue.Unsubscribe(topic) - if err != nil { - return fmt.Errorf("failed to unsubscribe from topic: %v", err) - } - return nil +func (c *CustomizedClient) GetDeviceStates() (string, error) { + // TODO: GetDeviceStates + return common.DeviceStatusOK, nil } /* --------------------------------------------------------------------------------------- */ @@ -195,12 +156,12 @@ func (c *ConfigData) ParseMessage(parseType SerializedFormatType) (map[string]in return c.parseJSON() case XML: // xml - convertedMessage, err := convertXMLToJSON(c.Message) + convertedMessage, err := convertXMLToMap(c.Message) if err != nil { return nil, err } - c.Message = convertedMessage - originalMap, err := c.parseJSON() + // c.Message = convertedMessage + originalMap := convertedMessage var mp map[string]interface{} for _, value := range originalMap { if nestedMap, ok := value.(map[string]interface{}); ok { @@ -246,33 +207,33 @@ func (c *ConfigData) ValidateMessage() error { // NewVisitorConfigData creates a new instance of VisitorConfigData using ConfigData pointer and the result of SplitTopic. func (c *ConfigData) NewVisitorConfigData() (*VisitorConfigData, error) { - // get ClientID - clientID, err := c.GetClientID() - if err != nil { - return nil, err - } - - // get DeviceInfo, OperationInfo and SerializedFormat - deviceInfo, operationInfo, serializedFormat, err := c.SplitTopic() - if err != nil { - return nil, err - } - - // get ParsedMessage - parsedMessage, err := c.ParseMessage(serializedFormat) - if err != nil { - return nil, err - } - - // create - return &VisitorConfigData{ - DataType: "string", - ClientID: clientID, - DeviceInfo: deviceInfo, - OperationInfo: operationInfo, - SerializedFormat: serializedFormat, - ParsedMessage: parsedMessage, - }, nil + // get ClientID + clientID, err := c.GetClientID() + if err != nil { + return nil, err + } + + // get DeviceInfo, OperationInfo and SerializedFormat + deviceInfo, operationInfo, serializedFormat, err := c.SplitTopic() + if err != nil { + return nil, err + } + + // get ParsedMessage + parsedMessage, err := c.ParseMessage(serializedFormat) + if err != nil { + return nil, err + } + + // create + return &VisitorConfigData{ + DataType: "string", + ClientID: clientID, + DeviceInfo: deviceInfo, + OperationInfo: operationInfo, + SerializedFormat: serializedFormat, + ParsedMessage: parsedMessage, + }, nil } /* --------------------------------------------------------------------------------------- */ @@ -313,20 +274,26 @@ func updateStructFields(structValue reflect.Value, data map[string]interface{}, fieldType := structType.Field(i) tagValue := fieldType.Tag.Get(tagName) - if tagValue == "" { - // Skip fields without the specified tag - continue + var value interface{} + var exists bool + + if tagValue != "" { + // Attempt to get value using tag + value, exists = data[tagValue] + } + + if !exists { + // Fallback to field name if tag is not found + tagValue = fieldType.Name + value, exists = data[tagValue] } - // Get the corresponding value from the map - value, exists := data[tagValue] if !exists { continue } // Update the field based on its kind if field.Kind() == reflect.Struct { - // Recursively update nested structs nestedData, ok := value.(map[string]interface{}) if !ok { return fmt.Errorf("type mismatch for nested field %s", tagValue) @@ -335,7 +302,6 @@ func updateStructFields(structValue reflect.Value, data map[string]interface{}, return err } } else if field.Kind() == reflect.Slice { - // Handle slices if necessary sliceData, ok := value.([]interface{}) if !ok { return fmt.Errorf("type mismatch for slice field %s", tagValue) @@ -351,7 +317,6 @@ func updateStructFields(structValue reflect.Value, data map[string]interface{}, } field.Set(newSlice) } else { - // Set the field value fieldValue := reflect.ValueOf(value) if field.Type() == fieldValue.Type() { field.Set(fieldValue) @@ -363,7 +328,6 @@ func updateStructFields(structValue reflect.Value, data map[string]interface{}, return nil } - /* --------------------------------------------------------------------------------------- */ // The function ConvertYAMLToJSON converts a YAML string to a JSON string. func convertYAMLToJSON(yamlString string) (string, error) { @@ -383,21 +347,6 @@ func convertYAMLToJSON(yamlString string) (string, error) { return string(jsonData), nil } -// The function convertXMLToJSON converts an XML string to a JSON string. -func convertXMLToJSON(xmlString string) (string, error) { - xmlData, err := convertXMLToMap(xmlString) - if err != nil { - return "", err - } - - jsonData, err := mapToJSON(xmlData) - if err != nil { - return "", err - } - - return jsonData, nil -} - // The function ConvertXMLToMap converts XML string to map[string]interface{}. func convertXMLToMap(xmlString string) (map[string]interface{}, error) { // Wrap the XML content with @@ -422,7 +371,6 @@ func wrapXMLWithRoot(xmlString string) string { } // Wrap the remaining XML content with - // wrappedXML := "" + xmlString + "" wrappedXML := xmlString return wrappedXML } @@ -435,15 +383,29 @@ type Node struct { Attr []xml.Attr `xml:"-"` } +// convertValue attempts to convert string content to appropriate type. +func convertValue(content string) interface{} { + if f, err := strconv.ParseFloat(content, 64); err == nil { + return f + } else if i, err := strconv.Atoi(content); err == nil { + return i + } else if b, err := strconv.ParseBool(content); err == nil { + return b + } else { + return content + } +} + // The function nodeToMap recursively converts XML nodes to map[string]interface{}. func nodeToMap(node Node) map[string]interface{} { result := make(map[string]interface{}) + // If the node has no children, it is a leaf node, apply type conversion. if len(node.Nodes) == 0 { - // Leaf node - return map[string]interface{}{node.XMLName.Local: node.Content} + return map[string]interface{}{node.XMLName.Local: convertValue(strings.TrimSpace(node.Content))} } + // Process child nodes recursively. for _, child := range node.Nodes { childMap := nodeToMap(child) if existing, found := result[child.XMLName.Local]; found { @@ -470,4 +432,12 @@ func mapToJSON(data map[string]interface{}) (string, error) { return string(jsonData), nil } -/* --------------------------------------------------------------------------------------- */ \ No newline at end of file +func StructToJSON(v interface{}) (string, error) { + jsonData, err := json.MarshalIndent(v, "", " ") + if err != nil { + return "", err + } + return string(jsonData), nil +} + +/* --------------------------------------------------------------------------------------- */ diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/go.mod b/mappers/mqtt-mapper/go.mod similarity index 100% rename from mappers/kubeedge-v1.17.0/mqtt-mapper/go.mod rename to mappers/mqtt-mapper/go.mod diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/go.sum b/mappers/mqtt-mapper/go.sum similarity index 100% rename from mappers/kubeedge-v1.17.0/mqtt-mapper/go.sum rename to mappers/mqtt-mapper/go.sum diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/hack/make-rules/mapper.sh b/mappers/mqtt-mapper/hack/make-rules/mapper.sh similarity index 100% rename from mappers/kubeedge-v1.17.0/mqtt-mapper/hack/make-rules/mapper.sh rename to mappers/mqtt-mapper/hack/make-rules/mapper.sh diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/resource/configmap.yaml b/mappers/mqtt-mapper/resource/configmap.yaml similarity index 100% rename from mappers/kubeedge-v1.17.0/mqtt-mapper/resource/configmap.yaml rename to mappers/mqtt-mapper/resource/configmap.yaml diff --git a/mappers/kubeedge-v1.17.0/mqtt-mapper/resource/deployment.yaml b/mappers/mqtt-mapper/resource/deployment.yaml similarity index 100% rename from mappers/kubeedge-v1.17.0/mqtt-mapper/resource/deployment.yaml rename to mappers/mqtt-mapper/resource/deployment.yaml From ed98057deaad725a3d05b3c1d09efbb606bc4347 Mon Sep 17 00:00:00 2001 From: Prometheus-collab Date: Thu, 26 Sep 2024 17:49:31 +0800 Subject: [PATCH 4/4] Providing a Common MQTT Mapper Signed-off-by: Prometheus-collab --- mappers/mqtt-mapper/config.yaml | 4 +- mappers/mqtt-mapper/driver/devicetype.go | 11 +- mappers/mqtt-mapper/driver/driver.go | 441 ++++++++++++------ .../resource/mqttdevice-instance.yaml | 49 ++ .../resource/mqttdevice-model.yaml | 15 + 5 files changed, 381 insertions(+), 139 deletions(-) create mode 100644 mappers/mqtt-mapper/resource/mqttdevice-instance.yaml create mode 100644 mappers/mqtt-mapper/resource/mqttdevice-model.yaml diff --git a/mappers/mqtt-mapper/config.yaml b/mappers/mqtt-mapper/config.yaml index d456caad..43b00da0 100644 --- a/mappers/mqtt-mapper/config.yaml +++ b/mappers/mqtt-mapper/config.yaml @@ -1,9 +1,9 @@ grpc_server: socket_path: /etc/kubeedge/kubeedge-v1.17.0.sock common: - name: Kubeedge-V1.17.0-mapper + name: github.com/kubeedge/mqtt-mapper version: v1.13.0 api_version: v1.0.0 - protocol: # TODO add your protocol name + protocol: mqtt address: 127.0.0.1 edgecore_sock: /etc/kubeedge/dmi.sock diff --git a/mappers/mqtt-mapper/driver/devicetype.go b/mappers/mqtt-mapper/driver/devicetype.go index 0d01e426..9925b5d3 100644 --- a/mappers/mqtt-mapper/driver/devicetype.go +++ b/mappers/mqtt-mapper/driver/devicetype.go @@ -17,8 +17,8 @@ type CustomizedClient struct { // TODO add some variables to help you better implement device drivers deviceMutex sync.Mutex ProtocolConfig - DeviceInfo string `json:"deviceInfo"` - ParsedDeviceInfo map[string]interface{} `json:"parsedDeviceInfo"` + TempMessage string `json:"tempMessage"` + DeviceConfigData interface{} `json:"deviceConfigData"` } type ProtocolConfig struct { @@ -36,7 +36,6 @@ type ConfigData struct { Password string `json:"password"` // Password for MQTT broker authentication ConnectionTTL time.Duration `json:"connectionTTL"` // Connection timeout duration LastMessage time.Time `json:"lastMessage"` // Timestamp of the last received message - IsData bool `json:"isData"` // Indicates if there is valid data } type VisitorConfig struct { @@ -58,7 +57,10 @@ type VisitorConfigData struct { type OperationInfoType uint const ( - UPDATE OperationInfoType = iota // revision + DEVICEINfO OperationInfoType = iota // set global device config data + UPDATE // update the device config data + SETSINGLEVALUE // find the most related setting value and update + GETSINGLEVALUE // find the most related setting value ) // SerializedFormatType defines the enumeration values for serialized types. @@ -68,5 +70,4 @@ const ( JSON SerializedFormatType = iota // json YAML // yaml XML // xml - JSONPATH ) diff --git a/mappers/mqtt-mapper/driver/driver.go b/mappers/mqtt-mapper/driver/driver.go index 078a14de..9187cc7b 100644 --- a/mappers/mqtt-mapper/driver/driver.go +++ b/mappers/mqtt-mapper/driver/driver.go @@ -5,67 +5,125 @@ import ( "encoding/xml" "errors" "fmt" + "gopkg.in/yaml.v3" "reflect" "strconv" "strings" "sync" + "time" "github.com/kubeedge/mapper-framework/pkg/common" - "gopkg.in/yaml.v3" ) func NewClient(protocol ProtocolConfig) (*CustomizedClient, error) { client := &CustomizedClient{ - ProtocolConfig: protocol, - deviceMutex: sync.Mutex{}, - DeviceInfo: "", - ParsedDeviceInfo: make(map[string]interface{}), + ProtocolConfig: protocol, + deviceMutex: sync.Mutex{}, + TempMessage: "", + DeviceConfigData: nil, } return client, nil } func (c *CustomizedClient) InitDevice() error { configData := &c.ProtocolConfig.ConfigData - _, _, format, _ := configData.SplitTopic() - c.DeviceInfo = c.ProtocolConfig.ConfigData.Message - c.ParsedDeviceInfo, _ = c.ParseMessage(format) + _, operationInfo, _, err := configData.SplitTopic() + if operationInfo != DEVICEINfO { + return errors.New("This is not a device config.") + } + if err != nil { + return err + } + c.TempMessage = configData.Message return nil } func (c *CustomizedClient) GetDeviceData(visitor *VisitorConfig) (interface{}, error) { + configData := &c.ProtocolConfig.ConfigData + _, operationInfo, _, err := configData.SplitTopic() + if operationInfo != DEVICEINfO { + return nil, errors.New("This is not a device config.") + } + if err != nil { + return nil, err + } + visitor.ProcessOperation(c.DeviceConfigData) + return c.DeviceConfigData, nil +} - return nil, nil +func (c *CustomizedClient) SetDeviceData(visitor *VisitorConfig) error { + configData := &c.ProtocolConfig.ConfigData + _, operationInfo, _, err := configData.SplitTopic() + if operationInfo == DEVICEINfO { + return errors.New("This is a device config, not to set device data.") + } + if err != nil { + return err + } + visitor.ProcessOperation(c.DeviceConfigData) + return nil } -func (c *CustomizedClient) SetDeviceData(data interface{}, visitor *VisitorConfig) error { - vPointer := visitor.VisitorConfigData - vPointer.ModifyVisitorConfigData(c.ParsedDeviceInfo) +func (c *CustomizedClient) StopDevice() error { + updateFieldsByTag(c.DeviceConfigData, map[string]interface{}{ + "status": common.DeviceStatusDisCONN, + "Status": common.DeviceStatusDisCONN, + }, "json") + updateFieldsByTag(c.DeviceConfigData, map[string]interface{}{ + "status": common.DeviceStatusDisCONN, + "Status": common.DeviceStatusDisCONN, + }, "yaml") + updateFieldsByTag(c.DeviceConfigData, map[string]interface{}{ + "status": common.DeviceStatusDisCONN, + "Status": common.DeviceStatusDisCONN, + }, "xml") return nil } -func (c *CustomizedClient) GetDeviceStates() (string, error) { - // TODO: GetDeviceStates - return common.DeviceStatusOK, nil +func (c *CustomizedClient) GetDeviceStates(visitor *VisitorConfig) (string, error) { + res, err := visitor.getFieldByTag(c.DeviceConfigData) + if err != nil { + return common.DeviceStatusOK, nil + } + return res, nil + } /* --------------------------------------------------------------------------------------- */ // The function NewConfigData is a constructor for ConfigData to initialize the structure. // It returns the ConfigData instance and an error value to handle the validity of the passed parameters. -func NewConfigData(clientID, topic, message string) (*ConfigData, error) { +func NewConfigData(clientID, brokerURL, topic, message, username, password string, connectionTTL time.Duration) (*ConfigData, error) { if clientID == "" { return nil, errors.New("clientID cannot be empty") } + if brokerURL == "" { + return nil, errors.New("borkerURL cannot be empty") + } if topic == "" { return nil, errors.New("topic cannot be empty") } if message == "" { - message = "default message" + return nil, errors.New("message cannot be empty") + } + if username == "" { + username = "defaultUser" + } + if password == "" { + password = "defaultPass" + } + if connectionTTL == 0 { + connectionTTL = 30 * time.Second // default timeout of 30 seconds } return &ConfigData{ - ClientID: clientID, - Topic: topic, - Message: message, + ClientID: clientID, + BrokerURL: brokerURL, + Topic: topic, + Message: message, + Username: username, + Password: password, + ConnectionTTL: connectionTTL, + LastMessage: time.Now(), // set last message time to current time }, nil } @@ -96,6 +154,9 @@ func (c *ConfigData) GetMessage() (string, error) { // OperationInfoType and SerializedFormatType mappings var operationTypeMap = map[string]OperationInfoType{ "update": UPDATE, + "deviceinfo": DEVICEINfO, + "setsinglevalue" : SETSINGLEVALUE, + "getsinglevalue" : GETSINGLEVALUE, } var serializedFormatMap = map[string]SerializedFormatType{ @@ -145,31 +206,13 @@ func (c *ConfigData) ParseMessage(parseType SerializedFormatType) (map[string]in switch parseType { case JSON: // json - return c.parseJSON() + return c.jsonParse() case YAML: // yaml - convertedMessage, err := convertYAMLToJSON(c.Message) - if err != nil { - return nil, err - } - c.Message = convertedMessage - return c.parseJSON() + return c.yamlParse() case XML: // xml - convertedMessage, err := convertXMLToMap(c.Message) - if err != nil { - return nil, err - } - // c.Message = convertedMessage - originalMap := convertedMessage - var mp map[string]interface{} - for _, value := range originalMap { - if nestedMap, ok := value.(map[string]interface{}); ok { - mp = nestedMap - break - } - } - return mp, err + return c.xmlParse() default: return nil, errors.New("unsupported parse type") @@ -177,67 +220,123 @@ func (c *ConfigData) ParseMessage(parseType SerializedFormatType) (map[string]in } // The function parseJSON parses the Message field of the ConfigData (assumed to be a JSON string). -func (c *ConfigData) parseJSON() (map[string]interface{}, error) { +func (c *ConfigData) jsonParse() (map[string]interface{}, error) { if c.Message == "" { return nil, errors.New("message is empty") } - var result map[string]interface{} - err := json.Unmarshal([]byte(c.Message), &result) + var jsonMsg map[string]interface{} + err := json.Unmarshal([]byte(c.Message), &jsonMsg) if err != nil { return nil, err } - return result, nil + return jsonMsg, nil } -// The function ValidateMessage checks if the message content is valid. -func (c *ConfigData) ValidateMessage() error { +// The function parseYAML parses the Message field of the ConfigData (assumed to be a YAML string). +func (c *ConfigData)yamlParse() (map[string]interface{}, error) { if c.Message == "" { - return errors.New("message is empty") + return nil, errors.New("message is empty") } - // Example: Check if the message is valid JSON (you can expand for other formats) - var temp map[string]interface{} - if err := json.Unmarshal([]byte(c.Message), &temp); err != nil { - return errors.New("message is not valid JSON") + var yamlMsg map[string]interface{} + err := yaml.Unmarshal([]byte(c.Message), &yamlMsg) + if err != nil { + return nil, err } - - return nil + return yamlMsg, nil } -// NewVisitorConfigData creates a new instance of VisitorConfigData using ConfigData pointer and the result of SplitTopic. -func (c *ConfigData) NewVisitorConfigData() (*VisitorConfigData, error) { - // get ClientID - clientID, err := c.GetClientID() - if err != nil { - return nil, err +// The function xmlParse parses the Message field of the ConfigData (assumed to be a XML string). +func (c *ConfigData)xmlParse() (map[string]interface{}, error) { + msg := c.Message + if strings.HasPrefix(msg, "") + if end != -1 { + msg = msg[end+2:] + } } - // get DeviceInfo, OperationInfo and SerializedFormat - deviceInfo, operationInfo, serializedFormat, err := c.SplitTopic() + var node Node + err := xml.Unmarshal([]byte(msg), &node) if err != nil { return nil, err } - // get ParsedMessage - parsedMessage, err := c.ParseMessage(serializedFormat) - if err != nil { - return nil, err + xmlMsg := nodeToMap(node) + var mp map[string]interface{} + for _, value := range xmlMsg { + if nestedMap, ok := value.(map[string]interface{}); ok { + mp = nestedMap + break + } } + return mp, err +} - // create - return &VisitorConfigData{ - DataType: "string", - ClientID: clientID, - DeviceInfo: deviceInfo, - OperationInfo: operationInfo, - SerializedFormat: serializedFormat, - ParsedMessage: parsedMessage, +// NewVisitorConfig creates a new instance of VisitorConfig using ConfigData pointer and the result of SplitTopic. +func (c *ConfigData) NewVisitorConfig() (*VisitorConfig, error) { + // get ClientID + clientID, err := c.GetClientID() + if err != nil { + return nil, err + } + + // get DeviceInfo, OperationInfo and SerializedFormat + deviceInfo, operationInfo, serializedFormat, err := c.SplitTopic() + if err != nil { + return nil, err + } + + // get ParsedMessage + parsedMessage, err := c.ParseMessage(serializedFormat) + if err != nil { + return nil, err + } + + // create + return &VisitorConfig{ + ProtocolName: "mqtt", + VisitorConfigData: VisitorConfigData{ + DataType: "DefaultDataType", + ClientID: clientID, + DeviceInfo: deviceInfo, + OperationInfo: operationInfo, + SerializedFormat: serializedFormat, + ParsedMessage: parsedMessage, + }, }, nil } /* --------------------------------------------------------------------------------------- */ -func (v *VisitorConfigData) ModifyVisitorConfigData(destDataConfig interface{}) error { +// The function ParseMessage parses the Message field according to the incoming type. +// parseType(0: json, 1: yaml, 2: xml) +// The value interface{} represents the parsed structure. +func (v *VisitorConfig) ProcessOperation(deviceConfigData interface{}) error { + if v.VisitorConfigData.ParsedMessage == nil { + return errors.New("visitor message is empty") + } + + if deviceConfigData == nil { + return errors.New("device message is empty") + } + + switch v.VisitorConfigData.OperationInfo { + case DEVICEINfO: // device config data + v.updateFullConfig(deviceConfigData) + return nil + case UPDATE: // update the full text according the visitor config and the tag (json, yaml, xml) + v.updateFullConfig(deviceConfigData) + return nil + case SETSINGLEVALUE: // update the single value according the visitor config and the tag (json, yaml, xml) + v.updateFieldsByTag(deviceConfigData) + return nil + default: + return errors.New("unsupported operation type") + } +} + +func (v *VisitorConfig) updateFullConfig(destDataConfig interface{}) error { destValue := reflect.ValueOf(destDataConfig) if destValue.Kind() != reflect.Ptr || destValue.Elem().Kind() != reflect.Struct { return errors.New("destDataConfig must be a pointer to a struct") @@ -246,7 +345,7 @@ func (v *VisitorConfigData) ModifyVisitorConfigData(destDataConfig interface{}) destValue = destValue.Elem() var tagName string - switch v.SerializedFormat { + switch v.VisitorConfigData.SerializedFormat { case JSON: tagName = "json" case YAML: @@ -258,13 +357,37 @@ func (v *VisitorConfigData) ModifyVisitorConfigData(destDataConfig interface{}) } // Update the destination struct using JSON tag - if err := updateStructFields(destValue, v.ParsedMessage, tagName); err != nil { + if err := updateStructFields(destValue, v.VisitorConfigData.ParsedMessage, tagName); err != nil { return err } return nil } +func (v *VisitorConfig)updateFieldsByTag(destDataConfig interface{}) error { + vv := reflect.ValueOf(destDataConfig).Elem() + + var tagName string + switch v.VisitorConfigData.SerializedFormat { + case JSON: + tagName = "json" + case YAML: + tagName = "yaml" + case XML: + tagName = "xml" + default: + return errors.New("unknown serialized format") + } + + for key, value := range v.VisitorConfigData.ParsedMessage { + if err := setFieldByTag(vv, key, value, tagName); err != nil { + return err + } + } + return nil +} + +/* --------------------------------------------------------------------------------------- */ // updateStructFields recursively updates struct fields from the given map using specified tag type func updateStructFields(structValue reflect.Value, data map[string]interface{}, tagName string) error { structType := structValue.Type() @@ -328,53 +451,6 @@ func updateStructFields(structValue reflect.Value, data map[string]interface{}, return nil } -/* --------------------------------------------------------------------------------------- */ -// The function ConvertYAMLToJSON converts a YAML string to a JSON string. -func convertYAMLToJSON(yamlString string) (string, error) { - // Converting a YAML string to a generic map object - var yamlData map[string]interface{} - err := yaml.Unmarshal([]byte(yamlString), &yamlData) - if err != nil { - return "", err - } - - // Convert a map object to a JSON string - jsonData, err := json.Marshal(yamlData) - if err != nil { - return "", err - } - - return string(jsonData), nil -} - -// The function ConvertXMLToMap converts XML string to map[string]interface{}. -func convertXMLToMap(xmlString string) (map[string]interface{}, error) { - // Wrap the XML content with - wrappedXML := wrapXMLWithRoot(xmlString) - - var node Node - err := xml.Unmarshal([]byte(wrappedXML), &node) - if err != nil { - return nil, err - } - return nodeToMap(node), nil -} - -// The function WrapXMLWithRoot wraps XML strings in tags. -func wrapXMLWithRoot(xmlString string) string { - // Remove the XML declaration if it exists - if strings.HasPrefix(xmlString, "") - if end != -1 { - xmlString = xmlString[end+2:] - } - } - - // Wrap the remaining XML content with - wrappedXML := xmlString - return wrappedXML -} - // Node structure type Node struct { XMLName xml.Name @@ -396,31 +472,74 @@ func convertValue(content string) interface{} { } } +// Convert XML attributes to map entries +func attrsToMap(attrs []xml.Attr) map[string]interface{} { + attrMap := make(map[string]interface{}) + for _, attr := range attrs { + attrMap[attr.Name.Local] = attr.Value + } + return attrMap +} + // The function nodeToMap recursively converts XML nodes to map[string]interface{}. func nodeToMap(node Node) map[string]interface{} { - result := make(map[string]interface{}) + xmlMsg := make(map[string]interface{}) + + // Process attributes + if len(node.Attr) > 0 { + xmlMsg["attributes"] = attrsToMap(node.Attr) + } // If the node has no children, it is a leaf node, apply type conversion. if len(node.Nodes) == 0 { - return map[string]interface{}{node.XMLName.Local: convertValue(strings.TrimSpace(node.Content))} + xmlMsg[node.XMLName.Local] = convertValue(strings.TrimSpace(node.Content)) + return xmlMsg } // Process child nodes recursively. + children := make(map[string]interface{}) for _, child := range node.Nodes { childMap := nodeToMap(child) - if existing, found := result[child.XMLName.Local]; found { + if existing, found := children[child.XMLName.Local]; found { switch v := existing.(type) { case []interface{}: - result[child.XMLName.Local] = append(v, childMap[child.XMLName.Local]) + children[child.XMLName.Local] = append(v, childMap[child.XMLName.Local]) default: - result[child.XMLName.Local] = []interface{}{v, childMap[child.XMLName.Local]} + children[child.XMLName.Local] = []interface{}{v, childMap[child.XMLName.Local]} } } else { - result[child.XMLName.Local] = childMap[child.XMLName.Local] + children[child.XMLName.Local] = childMap[child.XMLName.Local] } } - return map[string]interface{}{node.XMLName.Local: result} + xmlMsg[node.XMLName.Local] = children + return xmlMsg +} + +func setFieldByTag(v reflect.Value, key string, value interface{}, tagName string) error { + if v.Kind() == reflect.Pointer { + v = v.Elem() + } + for i := 0; i < v.NumField(); i++ { + field := v.Type().Field(i) + fieldVal := v.Field(i) + + if field.Tag.Get(tagName) == key { + val := reflect.ValueOf(value) + if fieldVal.Type() != val.Type() { + return fmt.Errorf("type mismatch: cannot assign %s to %s", val.Type(), fieldVal.Type()) + } + fieldVal.Set(val) + return nil + } + + if fieldVal.Kind() == reflect.Struct { + if err := setFieldByTag(fieldVal, key, value, tagName); err == nil { + return nil + } + } + } + return fmt.Errorf("no such field with tag: %s", key) } // The function MapToJSON converts map[string]interface{} to JSON string. @@ -440,4 +559,62 @@ func StructToJSON(v interface{}) (string, error) { return string(jsonData), nil } -/* --------------------------------------------------------------------------------------- */ +func updateFieldsByTag(s interface{}, updates map[string]interface{}, tagName string) error { + v := reflect.ValueOf(s).Elem() + for key, value := range updates { + if err := setFieldByTag(v, key, value, tagName); err != nil { + return err + } + } + return nil +} + +func (v * VisitorConfig)getFieldByTag(s interface{}) (string, error) { + vv := reflect.ValueOf(s).Elem() + + var tagName string + switch v.VisitorConfigData.SerializedFormat { + case JSON: + tagName = "json" + case YAML: + tagName = "yaml" + case XML: + tagName = "xml" + default: + return "", errors.New("unknown serialized format") + } + + res, err := findFieldByTag(vv, "status", tagName) + if err != nil { + res, err = findFieldByTag(vv, "Status", tagName) + if err != nil { + return "", err + } else { + return res, nil + } + } else { + return res, nil + } +} + +func findFieldByTag(v reflect.Value, key string, tagName string) (string, error) { + if v.Kind() == reflect.Pointer { + v = v.Elem() + } + for i := 0; i < v.NumField(); i++ { + field := v.Type().Field(i) + fieldVal := v.Field(i) + + if field.Tag.Get(tagName) == key { + return fieldVal.String(), nil + } + + if fieldVal.Kind() == reflect.Struct { + if value, err := findFieldByTag(fieldVal, key, tagName); err == nil { + return value, nil + } + } + } + return "", fmt.Errorf("no such field with tag: %s", key) +} +/* --------------------------------------------------------------------------------------- */ \ No newline at end of file diff --git a/mappers/mqtt-mapper/resource/mqttdevice-instance.yaml b/mappers/mqtt-mapper/resource/mqttdevice-instance.yaml new file mode 100644 index 00000000..8bd91b8e --- /dev/null +++ b/mappers/mqtt-mapper/resource/mqttdevice-instance.yaml @@ -0,0 +1,49 @@ +apiVersion: devices.kubeedge.io/v1beta1 + kind: Device + metadata: + name: beta1-device + spec: + deviceModelRef: + name: temperture-model + nodeName: k8s-worker1 + properties: + - name: temperature + collectCycle: 10000000000 # The frequency of reporting data to the cloud, once every 10 seconds + reportCycle: 10000000000 # The frequency of data push to user applications or databases, once every 10 seconds + reportToCloud: true + desired: + value: "30" + pushMethod: + mqtt: + address: tcp://101.133.150.110:1883 + topic: temperture/update/json + qos: 0 + retained: false + dbMethod: + influxdb2: + influxdb2ClientConfig: + url: http://127.0.0.1:8086 + org: test-org + bucket: test-bucket + influxdb2DataConfig: + measurement: temperture_stats + tag: + unit: temperature + fieldKey: temperture_value + visitors: + protocolName: mqtt + configData: + topic: "sensor/data" + qos: 1 + retain: false + clientId: "temperture_client" + username: "user" + password: "pass" + cleanSession: true + keepAlive: 60 + + protocol: + protocolName: mqtt + configData: + ip: 101.133.150.110 + port: 1883 \ No newline at end of file diff --git a/mappers/mqtt-mapper/resource/mqttdevice-model.yaml b/mappers/mqtt-mapper/resource/mqttdevice-model.yaml new file mode 100644 index 00000000..5a601acf --- /dev/null +++ b/mappers/mqtt-mapper/resource/mqttdevice-model.yaml @@ -0,0 +1,15 @@ + apiVersion: devices.kubeedge.io/v1beta1 + kind: DeviceModel + metadata: + name: temperture-model + namespace: default + spec: + properties: + - name: temperture + description: Temperture sensor model + type: INT + accessMode: ReadWrite + maximum: "100" + minimum: "1" + unit: "Celsius" + protocol: mqtt \ No newline at end of file