Skip to content

Commit 6811c5e

Browse files
medclmedcl
committed
feat: support search templates in orm (#643)
Reviewed-on: https://git.infini.ltd/infini/framework/pulls/643
1 parent 3c29763 commit 6811c5e

File tree

6 files changed

+167
-37
lines changed

6 files changed

+167
-37
lines changed

‎core/elastic/api.go

+13-3
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ type API interface {
2929
TemplateAPI
3030
ReplicationAPI
3131
SecurityAPI
32+
ScriptAPI
3233

3334
InitDefaultTemplate(templateName, indexPrefix string)
3435

@@ -116,9 +117,10 @@ type API interface {
116117
}
117118

118119
type TemplateAPI interface {
119-
TemplateExists(templateName string) (bool, error)
120-
PutTemplate(templateName string, template []byte) ([]byte, error)
121-
GetTemplate(templateName string) (map[string]interface{}, error)
120+
TemplateExists(scriptName string) (bool, error)
121+
PutTemplate(scriptName string, template []byte) ([]byte, error)
122+
GetTemplate(scriptName string) (map[string]interface{}, error)
123+
SearchByTemplate(indexName,scriptName string,params map[string]interface{}) (*SearchResponse, error)
122124
}
123125

124126
type MappingAPI interface {
@@ -132,6 +134,14 @@ type ScrollAPI interface {
132134
ClearScroll(scrollId string) error
133135
}
134136

137+
type ScriptAPI interface {
138+
ScriptExists(scriptName string)(bool,error)
139+
PutScript(scriptName string, script []byte)([]byte,error)
140+
SearchByTemplate(indexName,scriptName string,params map[string]interface{}) (*SearchResponse, error)
141+
//GetScript(scriptName string)([]byte,error)
142+
//DeleteScript(scriptName string)([]byte,error)
143+
}
144+
135145
type ReplicationAPI interface {
136146
StartReplication(followIndex string, body []byte) error
137147
StopReplication(indexName string, body []byte) error

‎core/orm/orm.go

+36-28
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@ type Context struct {
3333
}
3434

3535
type ORM interface {
36-
3736
RegisterSchemaWithIndexName(t interface{}, indexName string) error
3837

3938
GetIndexName(o interface{}) string
@@ -64,10 +63,11 @@ type ORMObjectBase struct {
6463
Created *time.Time `json:"created,omitempty" elastic_mapping:"created: { type: date }"`
6564
Updated *time.Time `json:"updated,omitempty" elastic_mapping:"updated: { type: date }"`
6665
}
67-
func ( obj *ORMObjectBase) GetID() string{
66+
67+
func (obj *ORMObjectBase) GetID() string {
6868
return obj.ID
6969
}
70-
func ( obj *ORMObjectBase) SetID( ID string){
70+
func (obj *ORMObjectBase) SetID(ID string) {
7171
obj.ID = ID
7272
}
7373

@@ -87,15 +87,21 @@ const ASC SortType = "asc"
8787
const DESC SortType = "desc"
8888

8989
type Query struct {
90-
Sort *[]Sort
91-
QueryArgs *[]util.KV
92-
From int
93-
CollapseField string
94-
Size int
95-
Conds []*Cond
96-
RawQuery []byte
97-
WildcardIndex bool
98-
IndexName string
90+
Sort *[]Sort
91+
QueryArgs *[]util.KV
92+
From int
93+
CollapseField string
94+
Size int
95+
Conds []*Cond
96+
RawQuery []byte
97+
TemplatedQuery *TemplatedQuery
98+
WildcardIndex bool
99+
IndexName string
100+
}
101+
102+
type TemplatedQuery struct {
103+
TemplateID string `json:"id"`
104+
Parameters map[string]interface{} `json:"params"`
99105
}
100106

101107
func (q *Query) Collapse(field string) *Query {
@@ -302,18 +308,18 @@ func getFieldStringValue(rValue reflect.Value, fieldName string) (bool, string)
302308
return false, ""
303309
}
304310

305-
func existsNonNullField(rValue reflect.Value, fieldName string) (bool) {
311+
func existsNonNullField(rValue reflect.Value, fieldName string) bool {
306312

307313
if rValue.Kind() == reflect.Ptr {
308314
rValue = reflect.Indirect(rValue)
309315
}
310316

311317
f := rValue.FieldByName(fieldName)
312-
if f.Kind()==reflect.Ptr{
318+
if f.Kind() == reflect.Ptr {
313319
return !f.IsNil()
314320
}
315321

316-
if f.IsValid(){
322+
if f.IsValid() {
317323
return true
318324
}
319325
return false
@@ -382,10 +388,10 @@ func Save(ctx *Context, o interface{}) error {
382388
return errors.New("id was not found")
383389
}
384390

385-
createdExists:= existsNonNullField(rValue, "Created")
391+
createdExists := existsNonNullField(rValue, "Created")
386392
t := time.Now()
387393
setFieldValue(rValue, "Updated", &t)
388-
if !createdExists{
394+
if !createdExists {
389395
setFieldValue(rValue, "Created", &t)
390396
}
391397

@@ -437,24 +443,24 @@ func GroupBy(o interface{}, selectField, groupField, haveQuery string, haveValue
437443
return getHandler().GroupBy(o, selectField, groupField, haveQuery, haveValue)
438444
}
439445

440-
var registeredSchemas=[]util.KeyValue{}
446+
var registeredSchemas = []util.KeyValue{}
441447

442-
func MustRegisterSchemaWithIndexName(t interface{}, index string){
443-
err := RegisterSchemaWithIndexName(t,index)
448+
func MustRegisterSchemaWithIndexName(t interface{}, index string) {
449+
err := RegisterSchemaWithIndexName(t, index)
444450
if err != nil {
445451
panic(err)
446452
}
447453
}
448454

449455
func RegisterSchemaWithIndexName(t interface{}, index string) error {
450-
registeredSchemas=append(registeredSchemas,util.KeyValue{Key: index, Payload: t})
456+
registeredSchemas = append(registeredSchemas, util.KeyValue{Key: index, Payload: t})
451457
return nil
452458
}
453459

454-
func InitSchema() error{
455-
for _,v:=range registeredSchemas{
456-
err:=getHandler().RegisterSchemaWithIndexName(v.Payload, v.Key)
457-
if err!=nil{
460+
func InitSchema() error {
461+
for _, v := range registeredSchemas {
462+
err := getHandler().RegisterSchemaWithIndexName(v.Payload, v.Key)
463+
if err != nil {
458464
return err
459465
}
460466
}
@@ -490,7 +496,9 @@ func Register(name string, h ORM) {
490496
}
491497

492498
type ProtectedFilterKeyType string
499+
493500
const ProtectedFilterKey ProtectedFilterKeyType = "FILTER_PROTECTED"
501+
494502
//FilterFieldsByProtected filter struct fields by tag protected recursively,
495503
//returns a filtered fields map
496504
func FilterFieldsByProtected(obj interface{}, protected bool) map[string]interface{} {
@@ -523,13 +531,13 @@ func FilterFieldsByProtected(obj interface{}, protected bool) map[string]interfa
523531
if strings.ToLower(tagVal) != "true" && protected {
524532
delete(mapObj, jsonName)
525533
continue
526-
}else if strings.ToLower(tagVal) == "true" && !protected {
534+
} else if strings.ToLower(tagVal) == "true" && !protected {
527535
delete(mapObj, jsonName)
528536
continue
529537
}
530-
if fieldType.Type.Kind() == reflect.Struct || (fieldType.Type.Kind() == reflect.Ptr && fieldType.Type.Elem().Kind() == reflect.Struct){
538+
if fieldType.Type.Kind() == reflect.Struct || (fieldType.Type.Kind() == reflect.Ptr && fieldType.Type.Elem().Kind() == reflect.Struct) {
531539
mapObj[jsonName] = FilterFieldsByProtected(v.Field(i).Interface(), protected)
532540
}
533541
}
534542
return mapObj
535-
}
543+
}

‎modules/elastic/adapter/elasticsearch/v0.go

+13
Original file line numberDiff line numberDiff line change
@@ -1871,4 +1871,17 @@ func (c *ESAPIV0) Flush(indexName string) ([]byte, error) {
18711871
return nil, fmt.Errorf(string(resp.Body))
18721872
}
18731873
return resp.Body, nil
1874+
}
1875+
1876+
1877+
func (c *ESAPIV0) ScriptExists(scriptName string)(bool,error){
1878+
panic("not implemented")
1879+
}
1880+
1881+
func (c *ESAPIV0) PutScript(scriptName string, script []byte)([]byte,error){
1882+
panic("not implemented")
1883+
}
1884+
1885+
func (c *ESAPIV0)SearchByTemplate(indexName,scriptName string,params map[string]interface{}) (*elastic.SearchResponse, error) {
1886+
panic("not implemented")
18741887
}

‎modules/elastic/adapter/elasticsearch/v7.go

+72
Original file line numberDiff line numberDiff line change
@@ -315,3 +315,75 @@ func (c *ESAPIV7) UpdateMapping(indexName string, docType string, mappings []byt
315315

316316
return resp.Body, err
317317
}
318+
319+
320+
func (c *ESAPIV7) ScriptExists(scriptName string)(bool,error){
321+
if scriptName == "" {
322+
return false,errors.New("invalid script name")
323+
}
324+
325+
url := fmt.Sprintf("/_scripts/%s", scriptName)
326+
url = c.GetEndpoint() + url
327+
resp, err := c.Request(nil, util.Verb_POST, url, nil)
328+
if err != nil {
329+
return false, err
330+
}
331+
if resp.StatusCode != http.StatusOK {
332+
return false, fmt.Errorf(string(resp.Body))
333+
}
334+
return true, nil
335+
}
336+
337+
func (c *ESAPIV7) PutScript(scriptName string, script []byte)([]byte,error){
338+
if scriptName == "" {
339+
return nil,errors.New("invalid script name")
340+
}
341+
342+
url := fmt.Sprintf("/_scripts/%s", scriptName)
343+
url = c.GetEndpoint() + url
344+
resp, err := c.Request(nil, util.Verb_POST, url, script)
345+
if err != nil {
346+
return nil, err
347+
}
348+
if resp.StatusCode != http.StatusOK {
349+
return nil, fmt.Errorf(string(resp.Body))
350+
}
351+
return resp.Body, nil
352+
}
353+
354+
func (c *ESAPIV7)SearchByTemplate(indexName,scriptName string,params map[string]interface{}) (*elastic.SearchResponse, error) {
355+
356+
if indexName == "" {
357+
return nil,errors.New("invalid index name")
358+
}
359+
if scriptName == "" {
360+
return nil,errors.New("invalid script name")
361+
}
362+
363+
url := fmt.Sprintf("/%s/_search/template", indexName)
364+
url = c.GetEndpoint() + url
365+
body:=util.MapStr{
366+
"id": scriptName,
367+
"params":params,
368+
}
369+
370+
resp, err := c.Request(nil, util.Verb_GET, url, util.MustToJSONBytes(body))
371+
esResp := &elastic.SearchResponse{}
372+
373+
if resp != nil {
374+
esResp.StatusCode = resp.StatusCode
375+
esResp.RawResult = resp
376+
esResp.ErrorObject = err
377+
}
378+
379+
if err != nil {
380+
return nil, err
381+
}
382+
383+
err = json.Unmarshal(resp.Body, esResp)
384+
if err != nil {
385+
return esResp, err
386+
}
387+
388+
return esResp, nil
389+
}

‎modules/elastic/common/config.go

+5-3
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,11 @@ type ORMConfig struct {
2424
OverrideExistsTemplate bool `config:"override_exists_template"`
2525
TemplateName string `config:"template_name"` //default template name
2626

27-
InitSchema bool `config:"init_schema"`
28-
IndexPrefix string `config:"index_prefix"`
29-
Templates map[string]string `config:"templates"` //template_name -> template_content
27+
InitSchema bool `config:"init_schema"`
28+
IndexPrefix string `config:"index_prefix"`
29+
30+
IndexTemplates map[string]string `config:"index_templates"` //template_name -> template_content
31+
SearchTemplates map[string]string `config:"search_templates"` //template_name -> template_content
3032
}
3133

3234
type StoreConfig struct {

‎modules/elastic/orm.go

+28-3
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,9 @@ func InitTemplate(force bool) {
3535
client.InitDefaultTemplate(moduleConfig.ORMConfig.TemplateName, moduleConfig.ORMConfig.IndexPrefix)
3636
}
3737

38-
if moduleConfig.ORMConfig.Templates!=nil&&len(moduleConfig.ORMConfig.Templates)>0{
39-
for k,v:= range moduleConfig.ORMConfig.Templates{
38+
//index templates
39+
if moduleConfig.ORMConfig.IndexTemplates!=nil&&len(moduleConfig.ORMConfig.IndexTemplates)>0{
40+
for k,v:= range moduleConfig.ORMConfig.IndexTemplates{
4041
var skip=false
4142
if !moduleConfig.ORMConfig.OverrideExistsTemplate{
4243
exists,err:= client.TemplateExists(k)
@@ -56,6 +57,29 @@ func InitTemplate(force bool) {
5657
}
5758
}
5859
}
60+
61+
//search templates
62+
if moduleConfig.ORMConfig.SearchTemplates!=nil&&len(moduleConfig.ORMConfig.SearchTemplates)>0{
63+
for k,v:= range moduleConfig.ORMConfig.SearchTemplates{
64+
var skip=false
65+
if !moduleConfig.ORMConfig.OverrideExistsTemplate{
66+
exists,err:= client.ScriptExists(k)
67+
if err!=nil{
68+
panic(err)
69+
}
70+
skip=exists
71+
}
72+
if !skip{
73+
v,err:=client.PutScript(k,[]byte(v))
74+
if err!=nil{
75+
if v!=nil{
76+
log.Error(string(v))
77+
}
78+
panic(err)
79+
}
80+
}
81+
}
82+
}
5983
}
6084
templateInited = true
6185
}
@@ -245,6 +269,8 @@ func (handler *ElasticORM) Search(t interface{}, q *api.Query) (error, api.Resul
245269

246270
if len(q.RawQuery) > 0 {
247271
searchResponse, err = handler.Client.QueryDSL(nil, indexName, q.QueryArgs, q.RawQuery)
272+
}else if q.TemplatedQuery!=nil{
273+
searchResponse, err =handler.Client.SearchByTemplate(indexName,q.TemplatedQuery.TemplateID,q.TemplatedQuery.Parameters)
248274
} else {
249275

250276
if q.Conds != nil && len(q.Conds) > 0 {
@@ -279,7 +305,6 @@ func (handler *ElasticORM) Search(t interface{}, q *api.Query) (error, api.Resul
279305
}
280306

281307
searchResponse, err = handler.Client.Search(indexName, &request)
282-
283308
}
284309

285310
if err != nil {

0 commit comments

Comments
 (0)