Skip to content

Commit

Permalink
add mysql/etcd query support (#624)
Browse files Browse the repository at this point in the history
* feat: add data query service

---------

Co-authored-by: rick <[email protected]>
  • Loading branch information
LinuxSuRen and LinuxSuRen authored Feb 23, 2025
1 parent 4a38bd2 commit cc5a463
Show file tree
Hide file tree
Showing 29 changed files with 2,657 additions and 1,906 deletions.
10 changes: 9 additions & 1 deletion .gitattributes
Original file line number Diff line number Diff line change
@@ -1 +1,9 @@
* text eol=lf
*.go text eol=lf
*.vue text eol=lf
*.js text eol=lf
*.css text eol=lf
*.html text eol=lf
*.md text eol=lf
*.yaml text eol=lf
*.json text eol=lf
*.yml text eol=lf
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -19,3 +19,4 @@ console/atest-desktop/node_modules
console/atest-desktop/atest
console/atest-desktop/atest.exe
console/atest-desktop/coverage
atest-store-git
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ This is an awesome API testing tool. 🚀
* Pre and post handle with the API request
* Run in server mode, and provide the [gRPC](pkg/server/server.proto) and HTTP endpoint
* [VS Code extension](https://github.com/LinuxSuRen/vscode-api-testing) support
* Simple Database query support
* Multiple storage backends supported(Local, ORM Database, S3, Git, Etcd, etc.)
* [HTTP API record](https://github.com/LinuxSuRen/atest-ext-collector)
* Install in multiple use cases(CLI, Container, Native-Service, [Operator](https://github.com/LinuxSuRen/atest-operator), Helm, etc.)
Expand Down
Binary file removed atest-store-git
Binary file not shown.
12 changes: 8 additions & 4 deletions cmd/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ func createServerCmd(execer fakeruntime.Execer, httpServer server.HTTPServer) (c
flags.StringArrayVarP(&opt.mockConfig, "mock-config", "", nil, "The mock config files")
flags.StringVarP(&opt.mockPrefix, "mock-prefix", "", "/mock", "The mock server API prefix")
flags.StringVarP(&opt.extensionRegistry, "extension-registry", "", "docker.io", "The extension registry URL")
flags.DurationVarP(&opt.downloadTimeout, "download-timeout", "", time.Second*10, "The timeout of extension download")
flags.DurationVarP(&opt.downloadTimeout, "download-timeout", "", time.Minute, "The timeout of extension download")

// gc related flags
flags.IntVarP(&opt.gcPercent, "gc-percent", "", 100, "The GC percent of Go")
Expand Down Expand Up @@ -288,6 +288,7 @@ func (o *serverOption) runE(cmd *cobra.Command, args []string) (err error) {
}
server.RegisterRunnerServer(s, remoteServer)
server.RegisterMockServer(s, mockServerController)
server.RegisterDataServerServer(s, remoteServer.(server.DataServerServer))
serverLogger.Info("gRPC server listening at", "addr", lis.Addr())
s.Serve(lis)
}()
Expand Down Expand Up @@ -322,11 +323,14 @@ func (o *serverOption) runE(cmd *cobra.Command, args []string) (err error) {
}
err = errors.Join(
server.RegisterRunnerHandlerFromEndpoint(ctx, mux, gRPCServerAddr, []grpc.DialOption{grpc.WithTransportCredentials(creds)}),
server.RegisterMockHandlerFromEndpoint(ctx, mux, gRPCServerAddr, []grpc.DialOption{grpc.WithTransportCredentials(creds)}))
server.RegisterMockHandlerFromEndpoint(ctx, mux, gRPCServerAddr, []grpc.DialOption{grpc.WithTransportCredentials(creds)}),
server.RegisterDataServerHandlerFromEndpoint(ctx, mux, gRPCServerAddr, []grpc.DialOption{grpc.WithTransportCredentials(creds)}))
} else {
dialOption := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}
err = errors.Join(
server.RegisterRunnerHandlerFromEndpoint(ctx, mux, gRPCServerAddr, []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}),
server.RegisterMockHandlerFromEndpoint(ctx, mux, gRPCServerAddr, []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}))
server.RegisterRunnerHandlerFromEndpoint(ctx, mux, gRPCServerAddr, dialOption),
server.RegisterMockHandlerFromEndpoint(ctx, mux, gRPCServerAddr, dialOption),
server.RegisterDataServerHandlerFromEndpoint(ctx, mux, gRPCServerAddr, dialOption))
}

if err == nil {
Expand Down
7 changes: 7 additions & 0 deletions console/atest-ui/src/App.vue
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
Share,
ArrowDown,
Guide,
DataAnalysis
} from '@element-plus/icons-vue'
import { ref, watch } from 'vue'
import { API } from './views/net'
Expand All @@ -17,6 +18,7 @@ import MockManager from './views/MockManager.vue'
import StoreManager from './views/StoreManager.vue'
import SecretManager from './views/SecretManager.vue'
import WelcomePage from './views/WelcomePage.vue'
import DataManager from './views/DataManager.vue'
import { useI18n } from 'vue-i18n'
const { t, locale: i18nLocale } = useI18n()
Expand Down Expand Up @@ -114,6 +116,10 @@ const toHistoryPanel = ({ ID: selectID, panelName: historyPanelName }) => {
<el-icon><Guide /></el-icon>
<template #title>{{ t('title.mock' )}}</template>
</el-menu-item>
<el-menu-item index="data" test-id="data-menu">
<el-icon><DataAnalysis /></el-icon>
<template #title>{{ t('title.data' )}}</template>
</el-menu-item>
<el-menu-item index="secret">
<el-icon><document /></el-icon>
<template #title>{{ t('title.secrets') }}</template>
Expand Down Expand Up @@ -142,6 +148,7 @@ const toHistoryPanel = ({ ID: selectID, panelName: historyPanelName }) => {
</div>
<TestingPanel v-if="panelName === 'testing'" @toHistoryPanel="toHistoryPanel"/>
<TestingHistoryPanel v-else-if="panelName === 'history'" :ID="ID"/>
<DataManager v-else-if="panelName === 'data'" />
<MockManager v-else-if="panelName === 'mock'" />
<StoreManager v-else-if="panelName === 'store'" />
<SecretManager v-else-if="panelName === 'secret'" />
Expand Down
3 changes: 2 additions & 1 deletion console/atest-ui/src/locales/en.json
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,8 @@
"functionQuery": "Functions Query",
"output": "Output",
"proxy": "Proxy",
"secure": "Secure"
"secure": "Secure",
"data": "Data"
},
"tip": {
"filter": "Filter Keyword",
Expand Down
3 changes: 2 additions & 1 deletion console/atest-ui/src/locales/zh.json
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,8 @@
"functionQuery": "函数查询",
"output": "输出",
"proxy": "代理",
"secure": "安全"
"secure": "安全",
"data": "数据"
},
"tip": {
"filter": "过滤",
Expand Down
132 changes: 132 additions & 0 deletions console/atest-ui/src/views/DataManager.vue
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
<script setup lang="ts">
import { ref, watch } from 'vue'
import { API } from './net'
import { ElMessage } from 'element-plus'
const stores = ref([])
const kind = ref('')
const store = ref('')
const sqlQuery = ref('')
const queryResult = ref([])
const columns = ref([])
const queryTip = ref('')
watch(store, (s) => {
stores.value.forEach((e: Store) => {
if (e.name === s) {
kind.value = e.kind.name
return
}
})
})
watch(kind, (k) => {
switch (k) {
case 'atest-store-orm':
sqlQuery.value = 'show tables'
queryTip.value = 'Enter SQL query'
break;
case 'atest-store-etcd':
sqlQuery.value = ''
queryTip.value = 'Enter key'
break;
}
})
API.GetStores((data) => {
stores.value = data.data
}, (e) => {
ElMessage({
showClose: true,
message: e.message,
type: 'error'
});
})
const ormDataHandler = (data) => {
const result = []
const cols = new Set()
data.items.forEach(e => {
const obj = {}
e.data.forEach(item => {
obj[item.key] = item.value
cols.add(item.key)
})
result.push(obj)
})
queryResult.value = result
columns.value = Array.from(cols).sort((a, b) => {
if (a === 'id') return -1;
if (b === 'id') return 1;
return a.localeCompare(b);
})
}
const keyValueDataHandler = (data) => {
queryResult.value = []
data.data.forEach(e => {
const obj = {}
obj['key'] = e.key
obj['value'] = e.value
queryResult.value.push(obj)
columns.value = ['key', 'value']
})
}
const executeQuery = async () => {
API.DataQuery(store.value, kind.value, sqlQuery.value, (data) => {
switch (kind.value) {
case 'atest-store-orm':
ormDataHandler(data)
break;
case 'atest-store-etcd':
keyValueDataHandler(data)
break;
default:
ElMessage({
showClose: true,
message: 'Unsupported store kind',
type: 'error'
});
}
}, (e) => {
ElMessage({
showClose: true,
message: e.message,
type: 'error'
});
})
}
</script>

<template>
<div>
<el-form @submit.prevent="executeQuery">
<el-row :gutter="10">
<el-col :span="2">
<el-form-item>
<el-select v-model="store" placeholder="Select store">
<el-option v-for="item in stores" :key="item.name" :label="item.name"
:value="item.name" :disabled="!item.ready" :kind="item.kind.name"></el-option>
</el-select>
</el-form-item>
</el-col>
<el-col :span="18">
<el-form-item>
<el-input v-model="sqlQuery" :placeholder="queryTip" @keyup.enter="executeQuery"></el-input>
</el-form-item>
</el-col>
<el-col :span="2">
<el-form-item>
<el-button type="primary" @click="executeQuery">Execute</el-button>
</el-form-item>
</el-col>
</el-row>
</el-form>
<el-table :data="queryResult">
<el-table-column v-for="col in columns" :key="col" :prop="col" :label="col"></el-table-column>
</el-table>
</div>
</template>
25 changes: 24 additions & 1 deletion console/atest-ui/src/views/net.ts
Original file line number Diff line number Diff line change
Expand Up @@ -773,6 +773,29 @@ var SBOM = (callback: (d: any) => void) => {
.then(callback)
}

var DataQuery = (store: string, kind: string, query: string, callback: (d: any) => void, errHandler: (d: any) => void) => {
const queryObj = {}
switch (kind) {
case 'atest-store-orm':
queryObj['sql'] = query;
break;
case 'atest-store-etcd':
queryObj['key'] = query;
break;
}
const requestOptions = {
method: 'POST',
headers: {
'X-Store-Name': store
},
body: JSON.stringify(queryObj)
}
fetch(`/api/v1/data/query`, requestOptions)
.then(DefaultResponseProcess)
.then(callback)
.catch(errHandler)
}

export const API = {
DefaultResponseProcess,
GetVersion,
Expand All @@ -785,6 +808,6 @@ export const API = {
FunctionsQuery,
GetSecrets, DeleteSecret, CreateOrUpdateSecret,
GetSuggestedAPIs,
ReloadMockServer, GetMockConfig, SBOM,
ReloadMockServer, GetMockConfig, SBOM, DataQuery,
getToken
}
68 changes: 34 additions & 34 deletions extensions/README.md
Original file line number Diff line number Diff line change
@@ -1,34 +1,34 @@
Ports in extensions:

| Type | Name | Port |
|------|------|------|
| Store | [orm](https://github.com/LinuxSuRen/atest-ext-store-orm) | 4071 |
| Store | [s3](https://github.com/LinuxSuRen/atest-ext-store-s3) | 4072 |
| Store | [etcd](https://github.com/LinuxSuRen/atest-ext-store-etcd) | 4073 |
| Store | [git](https://github.com/LinuxSuRen/atest-ext-store-git) | 4074 |
| Store | [mongodb](https://github.com/LinuxSuRen/atest-ext-store-mongodb) | 4075 |
| Monitor | [docker-monitor](https://github.com/LinuxSuRen/atest-ext-monitor-docker) | |
| Agent | [collector](https://github.com/LinuxSuRen/atest-ext-collector) | |
| Secret | [Vault](https://github.com/LinuxSuRen/api-testing-vault-extension) | |

## Contribute a new extension

* First, create a repository. And please keep the same naming convertion.
* Second, implement the `Loader` gRPC service which defined by [this proto](../pkg/testing/remote/loader.proto).
* Finally, add the extension's name into function [SupportedExtensions](../console/atest-ui/src/views/store.ts).

## Naming conventions
Please follow the following conventions if you want to add a new store extension:

`store-xxx`

`xxx` should be a type of a backend storage.

## Test

First, build and copy the binary file into the system path. You can run the following
command in the root directory of this project:

```shell
make build-ext-etcd copy-ext
```
Ports in extensions:

| Type | Name | Port |
|------|------|------|
| Store | [orm](https://github.com/LinuxSuRen/atest-ext-store-orm) | 4071 |
| Store | [s3](https://github.com/LinuxSuRen/atest-ext-store-s3) | 4072 |
| Store | [etcd](https://github.com/LinuxSuRen/atest-ext-store-etcd) | 4073 |
| Store | [git](https://github.com/LinuxSuRen/atest-ext-store-git) | 4074 |
| Store | [mongodb](https://github.com/LinuxSuRen/atest-ext-store-mongodb) | 4075 |
| Monitor | [docker-monitor](https://github.com/LinuxSuRen/atest-ext-monitor-docker) | |
| Agent | [collector](https://github.com/LinuxSuRen/atest-ext-collector) | |
| Secret | [Vault](https://github.com/LinuxSuRen/api-testing-vault-extension) | |

## Contribute a new extension

* First, create a repository. And please keep the same naming convertion.
* Second, implement the `Loader` gRPC service which defined by [this proto](../pkg/testing/remote/loader.proto).
* Finally, add the extension's name into function [SupportedExtensions](../console/atest-ui/src/views/store.ts).

## Naming conventions
Please follow the following conventions if you want to add a new store extension:

`store-xxx`

`xxx` should be a type of a backend storage.

## Test

First, build and copy the binary file into the system path. You can run the following
command in the root directory of this project:

```shell
make build-ext-etcd copy-ext
```
5 changes: 5 additions & 0 deletions pkg/mock/in_memory.go
Original file line number Diff line number Diff line change
Expand Up @@ -453,10 +453,15 @@ func runWebhook(ctx context.Context, objCtx interface{}, wh *Webhook) (err error
req.Header.Set(k, v)
}

memLogger.Info("send webhook request", "api", api)
resp, err := client.Do(req)
if err != nil {
err = fmt.Errorf("error when sending webhook")
} else {
if resp.StatusCode != http.StatusOK {
memLogger.Info("unexpected status", "code", resp.StatusCode)
}

data, _ := io.ReadAll(resp.Body)
memLogger.V(7).Info("received from webhook", "code", resp.StatusCode, "response", string(data))
}
Expand Down
Loading

0 comments on commit cc5a463

Please sign in to comment.