Skip to content

Commit a7295dc

Browse files
shifucunshifucun
authored
[V2 API] Handle observation in node API (#1093)
Observation is not stored as regular nodes in BigTable but only in BigQuery. In V1, we only handle this in triples API but not in property and property-value API. This PR fixes that. It also prepares to migrate the client to use property + property-values API instead of triples API.
1 parent c7e5e0a commit a7295dc

File tree

6 files changed

+154
-26
lines changed

6 files changed

+154
-26
lines changed

‎internal/server/handler_v2.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,7 @@ func (s *Server) V2Node(
107107
return v2pv.API(
108108
ctx,
109109
s.store,
110+
s.metadata,
110111
in.GetNodes(),
111112
[]string{arc.SingleProp},
112113
direction,
@@ -140,6 +141,7 @@ func (s *Server) V2Node(
140141
return v2pv.API(
141142
ctx,
142143
s.store,
144+
s.metadata,
143145
in.GetNodes(),
144146
arc.BracketProps,
145147
direction,

‎internal/server/v2/properties/api.go‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ package properties
1717

1818
import (
1919
"context"
20+
"strings"
2021

2122
pbv2 "github.com/datacommonsorg/mixer/internal/proto/v2"
2223
nodewrapper "github.com/datacommonsorg/mixer/internal/server/node"
@@ -45,8 +46,25 @@ func API(
4546
}
4647
} else if direction == util.DirectionOut {
4748
for node, d := range data {
49+
var properties []string
50+
if strings.HasPrefix(node, "dc/o") {
51+
properties = []string{
52+
"observationAbout",
53+
"variableMeasured",
54+
"value",
55+
"observationDate",
56+
"observationPeriod",
57+
"measurementMethod",
58+
"unit",
59+
"scalingFactor",
60+
"samplePopulation",
61+
"location",
62+
}
63+
} else {
64+
properties = d.GetOutLabels()
65+
}
4866
res.Data[node] = &pbv2.LinkedGraph{
49-
Properties: d.GetOutLabels(),
67+
Properties: properties,
5068
}
5169
}
5270
}

‎internal/server/v2/propertyvalues/api.go‎

Lines changed: 61 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121

2222
pb "github.com/datacommonsorg/mixer/internal/proto"
2323
pbv2 "github.com/datacommonsorg/mixer/internal/proto/v2"
24+
"github.com/datacommonsorg/mixer/internal/server/node"
2425
"github.com/datacommonsorg/mixer/internal/server/placein"
2526
"github.com/datacommonsorg/mixer/internal/server/resource"
2627
"github.com/datacommonsorg/mixer/internal/server/statvar"
@@ -36,39 +37,80 @@ import (
3637
func API(
3738
ctx context.Context,
3839
store *store.Store,
40+
metadata *resource.Metadata,
3941
nodes []string,
4042
properties []string,
4143
direction string,
4244
limit int,
4345
reqToken string,
4446
) (*pbv2.NodeResponse, error) {
45-
data, pi, err := v1pv.Fetch(
46-
ctx,
47-
store,
48-
nodes,
49-
properties,
50-
limit,
51-
reqToken,
52-
direction,
53-
)
54-
if err != nil {
55-
return nil, err
47+
obsNodes := []string{}
48+
regularNodes := []string{}
49+
for _, n := range nodes {
50+
if strings.HasPrefix(n, "dc/o") {
51+
obsNodes = append(obsNodes, n)
52+
} else {
53+
regularNodes = append(regularNodes, n)
54+
}
5655
}
5756
res := &pbv2.NodeResponse{Data: map[string]*pbv2.LinkedGraph{}}
58-
for _, node := range nodes {
59-
res.Data[node] = &pbv2.LinkedGraph{Arcs: map[string]*pbv2.Nodes{}}
60-
for _, property := range properties {
61-
res.Data[node].Arcs[property] = &pbv2.Nodes{
62-
Nodes: v1pv.MergeTypedNodes(data[node][property]),
57+
58+
if len(obsNodes) > 0 {
59+
propertySet := map[string]struct{}{}
60+
for _, p := range properties {
61+
propertySet[p] = struct{}{}
62+
}
63+
tripleResp, err := node.GetObsTriples(ctx, store, metadata, obsNodes)
64+
if err != nil {
65+
return nil, err
66+
}
67+
for dcid, tripleList := range tripleResp {
68+
res.Data[dcid] = &pbv2.LinkedGraph{Arcs: map[string]*pbv2.Nodes{}}
69+
for _, t := range tripleList {
70+
if _, ok := propertySet[t.Predicate]; ok {
71+
res.Data[dcid].Arcs[t.Predicate] = &pbv2.Nodes{
72+
Nodes: []*pb.EntityInfo{
73+
{
74+
Name: t.ObjectName,
75+
Value: t.ObjectValue,
76+
Types: t.ObjectTypes,
77+
Dcid: t.ObjectId,
78+
ProvenanceId: t.ProvenanceId,
79+
},
80+
},
81+
}
82+
}
6383
}
6484
}
6585
}
66-
if pi != nil {
67-
respToken, err := util.EncodeProto(pi)
86+
if len(regularNodes) > 0 {
87+
data, pi, err := v1pv.Fetch(
88+
ctx,
89+
store,
90+
regularNodes,
91+
properties,
92+
limit,
93+
reqToken,
94+
direction,
95+
)
6896
if err != nil {
6997
return nil, err
7098
}
71-
res.NextToken = respToken
99+
for _, n := range regularNodes {
100+
res.Data[n] = &pbv2.LinkedGraph{Arcs: map[string]*pbv2.Nodes{}}
101+
for _, property := range properties {
102+
res.Data[n].Arcs[property] = &pbv2.Nodes{
103+
Nodes: v1pv.MergeTypedNodes(data[n][property]),
104+
}
105+
}
106+
}
107+
if pi != nil {
108+
respToken, err := util.EncodeProto(pi)
109+
if err != nil {
110+
return nil, err
111+
}
112+
res.NextToken = respToken
113+
}
72114
}
73115
return res, nil
74116
}
@@ -87,7 +129,6 @@ func LinkedPropertyValues(
87129
if !ok {
88130
return nil, status.Errorf(codes.InvalidArgument, "must provide typeOf filters")
89131
}
90-
91132
if linkedProperty == "containedInPlace" && direction == util.DirectionIn {
92133
data, err := placein.GetPlacesIn(
93134
ctx,

‎internal/server/v2/propertyvalues/golden/simple/obs.json‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
{
2+
"data": {
3+
"dc/o/vs51dzghn79eg": {
4+
"arcs": {
5+
"measurementMethod": {
6+
"nodes": [
7+
{
8+
"name": "CensusACS5yrSurvey",
9+
"dcid": "CensusACS5yrSurvey",
10+
"provenanceId": "dc/base/CensusACS5YearSurvey"
11+
}
12+
]
13+
},
14+
"observationAbout": {
15+
"nodes": [
16+
{
17+
"name": "Mountain View",
18+
"dcid": "geoId/0649670",
19+
"provenanceId": "dc/base/CensusACS5YearSurvey"
20+
}
21+
]
22+
},
23+
"observationDate": {
24+
"nodes": [
25+
{
26+
"provenanceId": "dc/base/CensusACS5YearSurvey",
27+
"value": "2016"
28+
}
29+
]
30+
},
31+
"unit": {
32+
"nodes": [
33+
{
34+
"name": "Year",
35+
"dcid": "Year",
36+
"provenanceId": "dc/base/CensusACS5YearSurvey"
37+
}
38+
]
39+
},
40+
"value": {
41+
"nodes": [
42+
{
43+
"provenanceId": "dc/base/CensusACS5YearSurvey",
44+
"value": "35"
45+
}
46+
]
47+
},
48+
"variableMeasured": {
49+
"nodes": [
50+
{
51+
"name": "Median Age",
52+
"dcid": "Median_Age_Person",
53+
"provenanceId": "dc/base/CensusACS5YearSurvey"
54+
}
55+
]
56+
}
57+
}
58+
}
59+
}
60+
}

‎internal/server/v2/propertyvalues/golden/simple_property_values_test.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,13 @@ func TestSimplePropertyValues(t *testing.T) {
9898
500,
9999
"H4sIAAAAAAAA/5ySXUoDMRSFnTitMf4NeZpdWH3wyQd1rFhQGRhdQEwu7YVrApm00gW4Pdck2jfp3EAXcD6+czjqVsk5hJk7n1ypygafDHpwM9+SsaCOGvD9sn+Nxia9p4UstJBCC7mvhSx1KUdVoa5ZRNlgWm/Jjn+z9aF6YNPVlOADfDJx3dlFCDRo8cJy6v+ce+xTRKbVDctTjzhfZIzuWMLxMzpHkGFkLNrwCbEl44d78PuetBFXJg1rHMhRJeqvQk35Nu3yndCymKL+FqrJ6Pxh3hLS9tdsOmUg3WUDRE+wArqY7DjMafYmmwOPM8OcdWCDd9n//gAAAP//AQAA//+e6uq5iQMAAA==",
100100
},
101+
{
102+
"obs.json",
103+
[]string{"dc/o/vs51dzghn79eg"},
104+
"->[observationAbout, variableMeasured, value, observationDate, observationPeriod, measurementMethod, unit]",
105+
0,
106+
"",
107+
},
101108
} {
102109
req := &pbv2.NodeRequest{
103110
Nodes: c.nodes,

‎proto/service/mixer.proto‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -366,15 +366,15 @@ service Mixer {
366366
};
367367
}
368368

369-
// V2 [TODO]
369+
// V2 [NA]
370370
rpc Triples(datacommons.v1.TriplesRequest)
371371
returns (datacommons.v1.TriplesResponse) {
372372
option (google.api.http) = {
373373
get : "/v1/triples/{direction}/{node=**}"
374374
};
375375
}
376376

377-
// V2 [TODO]
377+
// V2 [NA]
378378
rpc BulkTriples(datacommons.v1.BulkTriplesRequest)
379379
returns (datacommons.v1.BulkTriplesResponse) {
380380
option (google.api.http) = {
@@ -545,14 +545,14 @@ service Mixer {
545545
};
546546
}
547547

548-
// V2 [TODO]
548+
// V2 [NA]
549549
rpc BioPage(datacommons.v1.BioPageRequest) returns (GraphNodes) {
550550
option (google.api.http) = {
551551
get : "/v1/internal/page/bio/{node=**}"
552552
};
553553
}
554554

555-
// V2 [TODO]
555+
// V2 [NA]
556556
rpc PlacePage(datacommons.v1.PlacePageRequest)
557557
returns (datacommons.v1.PlacePageResponse) {
558558
option (google.api.http) = {
@@ -561,7 +561,7 @@ service Mixer {
561561
};
562562
}
563563

564-
// V2 [TODO]
564+
// V2 [DONE]
565565
rpc VariableAncestors(datacommons.v1.VariableAncestorsRequest)
566566
returns (datacommons.v1.VariableAncestorsResponse) {
567567
option (google.api.http) = {

0 commit comments

Comments
 (0)