Skip to content

Commit 8dd8d5c

Browse files
authored
Merge pull request #131 from GoogleCloudPlatform/huyhg/dep
2 parents 3f0a175 + 25ec20c commit 8dd8d5c

19 files changed

+786
-235
lines changed

Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ test/py: $(PYTHON_TEST_DIRS)
2929
$(PYTHON_TEST_DIRS): %: .build/testing/py
3030
$(info === Running tests in directory $@ ===)
3131
@docker run --rm \
32-
-v $(PWD):/data \
32+
-v $(PWD):/data:ro \
3333
testing/py \
3434
python2 -m unittest discover -s "/data/$@" -p "*_test.py"
3535

app.Makefile

Lines changed: 12 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,8 @@ endef
2424

2525
# Combines APP_PARAMETERS and APP_TEST_PARAMETERS.
2626
define combined_parameters
27-
$(shell echo '$(APP_PARAMETERS)' '$(APP_TEST_PARAMETERS)'
28-
| docker run -i --entrypoint=usr/bin/jq --rm $(APP_DEPLOYER_IMAGE) -s '.[0] * .[1]')
27+
$(shell echo '$(APP_PARAMETERS)' '$(APP_TEST_PARAMETERS)' \
28+
| docker run -i --entrypoint=/usr/bin/jq --rm $(APP_DEPLOYER_IMAGE) -s '.[0] * .[1]')
2929
endef
3030

3131

@@ -46,27 +46,23 @@ app/build:: ;
4646

4747
# Installs the application into target namespace on the cluster.
4848
.PHONY: app/install
49-
app/install: app/build \
50-
.build/var/MARKETPLACE_TOOLS_PATH \
51-
.build/var/APP_DEPLOYER_IMAGE \
52-
.build/var/APP_PARAMETERS
49+
app/install:: app/build \
50+
.build/var/MARKETPLACE_TOOLS_PATH \
51+
.build/var/APP_DEPLOYER_IMAGE \
52+
.build/var/APP_PARAMETERS
5353
$(MARKETPLACE_TOOLS_PATH)/scripts/start.sh \
54-
--namespace='$(call namespace_parameter)' \
55-
--name='$(call name_parameter)' \
5654
--deployer='$(APP_DEPLOYER_IMAGE)' \
5755
--parameters='$(APP_PARAMETERS)'
5856

5957

6058
# Installs the application into target namespace on the cluster.
6159
.PHONY: app/install-test
62-
app/install-test: app/build \
63-
.build/var/MARKETPLACE_TOOLS_PATH \
64-
.build/var/APP_DEPLOYER_IMAGE \
65-
.build/var/APP_PARAMETERS \
66-
.build/var/APP_TEST_PARAMETERS
60+
app/install-test:: app/build \
61+
.build/var/MARKETPLACE_TOOLS_PATH \
62+
.build/var/APP_DEPLOYER_IMAGE \
63+
.build/var/APP_PARAMETERS \
64+
.build/var/APP_TEST_PARAMETERS
6765
$(MARKETPLACE_TOOLS_PATH)/scripts/start.sh \
68-
--namespace='$(call namespace_parameter)' \
69-
--name='$(call name_parameter)' \
7066
--deployer='$(APP_DEPLOYER_IMAGE)' \
7167
--parameters='$(call combined_parameters)' \
7268
--entrypoint='/bin/deploy_with_tests.sh'
@@ -89,9 +85,8 @@ app/verify: app/build \
8985
.build/var/APP_DEPLOYER_IMAGE \
9086
.build/var/APP_PARAMETERS \
9187
.build/var/APP_TEST_PARAMETERS
92-
$(MARKETPLACE_TOOLS_PATH)/marketplace/driver/driver.sh \
88+
$(MARKETPLACE_TOOLS_PATH)/scripts/driver/driver.sh \
9389
--deployer='$(APP_DEPLOYER_IMAGE)' \
94-
--marketplace_tools='$(MARKETPLACE_TOOLS_PATH)' \
9590
--parameters='$(call combined_parameters)'
9691

9792

marketplace/deployer_util/config_env.py

Lines changed: 3 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -19,38 +19,22 @@
1919
from argparse import ArgumentParser
2020

2121
import config_helper
22+
import schema_values_common
2223

2324
_PROG_HELP = """
2425
Runs a specified command within an environment with env variables
2526
setup from the config parameters.
2627
"""
2728

28-
CODEC_UTF8 = 'UTF-8'
29-
CODEC_ASCII = 'ASCII'
30-
3129

3230
def main():
3331
parser = ArgumentParser(description=_PROG_HELP)
34-
parser.add_argument('--values_dir',
35-
help='Where the value files should be read from',
36-
default='/data/final_values')
37-
parser.add_argument('--encoding',
38-
help='Encoding of the value files',
39-
choices=[CODEC_UTF8, CODEC_ASCII], default='UTF-8')
40-
parser.add_argument('--schema_file', help='Path to the schema file',
41-
default='/data/schema.yaml')
42-
parser.add_argument('--schema_file_encoding',
43-
help='Encoding of the schema file',
44-
choices=[CODEC_UTF8, CODEC_ASCII], default=CODEC_UTF8)
32+
schema_values_common.add_to_argument_parser(parser)
4533
parser.add_argument('command', help='Command to run')
4634
parser.add_argument('arguments', nargs='*', help='Arguments to the command')
4735
args = parser.parse_args()
4836

49-
schema = config_helper.Schema.load_yaml_file(args.schema_file,
50-
args.schema_file_encoding)
51-
values = config_helper.read_values_to_dict(args.values_dir,
52-
args.encoding,
53-
schema)
37+
values = schema_values_common.load_values(args)
5438
# Convert values to strings to pass to subprocess.
5539
values = {k: str(v) for k, v in values.iteritems()}
5640

marketplace/deployer_util/config_helper.py

Lines changed: 100 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,16 @@
1818
import io
1919
import os
2020
import re
21+
import sys
2122

2223
import yaml
2324

2425
NAME_RE = re.compile(r'[a-zA-z0-9_\.]+$')
2526

2627
XGOOGLE = 'x-google-marketplace'
2728
XTYPE_PASSWORD = 'GENERATED_PASSWORD'
29+
XTYPE_SERVICE_ACCOUNT = 'SERVICE_ACCOUNT'
30+
XTYPE_STORAGE_CLASS = 'STORAGE_CLASS'
2831

2932

3033
class InvalidName(Exception):
@@ -39,7 +42,21 @@ class InvalidSchema(Exception):
3942
pass
4043

4144

42-
def read_values_to_dict(values_dir, codec, schema):
45+
def load_values(values_file,
46+
values_dir,
47+
values_dir_encoding,
48+
schema):
49+
if values_file == '-':
50+
return yaml.safe_load(sys.stdin.read())
51+
if values_file and os.path.isfile(values_file):
52+
with open(values_file, 'r') as f:
53+
return yaml.safe_load(f.read())
54+
return _read_values_to_dict(values_dir,
55+
values_dir_encoding,
56+
schema)
57+
58+
59+
def _read_values_to_dict(values_dir, codec, schema):
4360
"""Returns a dict constructed from files in values_dir."""
4461
files = [f for f in os.listdir(values_dir)
4562
if os.path.isfile(os.path.join(values_dir, f))]
@@ -63,8 +80,8 @@ class Schema:
6380
"""Wrapper class providing convenient access to a JSON schema."""
6481

6582
@staticmethod
66-
def load_yaml_file(filepath, encoding='utf_8'):
67-
with io.open(filepath, 'r', encoding=encoding) as f:
83+
def load_yaml_file(filepath):
84+
with io.open(filepath, 'r') as f:
6885
d = yaml.load(f)
6986
return Schema(d)
7087

@@ -75,10 +92,16 @@ def load_yaml(yaml_str):
7592
def __init__(self, dictionary):
7693
self._required = dictionary.get('required', [])
7794
self._properties = {
78-
k: SchemaProperty(k, v)
95+
k: SchemaProperty(k, v, k in self._required)
7996
for k, v in dictionary.get('properties', {}).iteritems()
8097
}
8198

99+
bad_required_names = [x for x in self._required
100+
if x not in self._properties]
101+
if bad_required_names:
102+
raise InvalidSchema('Undefined property names found in required: {}'
103+
.format(', '.join(bad_required_names)))
104+
82105
@property
83106
def required(self):
84107
return self._required
@@ -87,16 +110,23 @@ def required(self):
87110
def properties(self):
88111
return self._properties
89112

113+
def properties_matching(self, definition):
114+
return [v for k, v in self._properties.iteritems()
115+
if v.matches_definition(definition)]
116+
90117

91118
class SchemaProperty:
92119
"""Wrapper class providing convenient access to a JSON schema property."""
93120

94-
def __init__(self, name, dictionary):
121+
def __init__(self, name, dictionary, required):
95122
self._name = name
96123
self._d = dictionary
124+
self._required = required
97125
self._default = dictionary.get('default', None)
98126
self._x = dictionary.get(XGOOGLE, None)
99127
self._password = None
128+
self._service_account = None
129+
self._storage_class = None
100130

101131
if not NAME_RE.match(name):
102132
raise InvalidSchema('Invalid property name: {}'.format(name))
@@ -130,11 +160,22 @@ def __init__(self, name, dictionary):
130160
'base64': d.get('base64', True),
131161
}
132162
self._password = SchemaXPassword(**spec)
163+
elif xt == XTYPE_SERVICE_ACCOUNT:
164+
d = self._x.get('serviceAccount', {})
165+
self._service_account = SchemaXServiceAccount(d)
166+
elif xt == XTYPE_STORAGE_CLASS:
167+
d = self._x.get('storageClass', {})
168+
self._storage_class = SchemaXStorageClass(d)
169+
133170

134171
@property
135172
def name(self):
136173
return self._name
137174

175+
@property
176+
def required(self):
177+
return self._required
178+
138179
@property
139180
def default(self):
140181
return self._default
@@ -154,6 +195,14 @@ def xtype(self):
154195
def password(self):
155196
return self._password
156197

198+
@property
199+
def service_account(self):
200+
return self._service_account
201+
202+
@property
203+
def storage_class(self):
204+
return self._storage_class
205+
157206
def str_to_type(self, str_val):
158207
if self._type == bool:
159208
if str_val in {'true', 'True', 'yes', 'Yes'}:
@@ -200,3 +249,49 @@ def __eq__(self, other):
200249
['length',
201250
'include_symbols',
202251
'base64'])
252+
253+
254+
class SchemaXServiceAccount:
255+
"""Wrapper class providing convenient access to SERVICE_ACCOUNT property."""
256+
257+
def __init__(self, dictionary):
258+
self._roles = dictionary.get('roles', [])
259+
260+
def custom_role_rules(self):
261+
"""Returns a list of rules for custom Roles."""
262+
return [role.get('rules', [])
263+
for role in self._roles
264+
if role['type'] == 'Role'
265+
and role['rulesType'] == 'CUSTOM']
266+
267+
def custom_cluster_role_rules(self):
268+
"""Returns a list of rules for custom ClusterRoles."""
269+
return [role.get('rules', [])
270+
for role in self._roles
271+
if role['type'] == 'ClusterRole'
272+
and role['rulesType'] == 'CUSTOM']
273+
274+
def predefined_roles(self):
275+
"""Returns a list of predefined Roles."""
276+
return [role.get('rulesFromRoleName')
277+
for role in self._roles
278+
if role['type'] == 'Role'
279+
and role['rulesType'] == 'PREDEFINED']
280+
281+
def predefined_cluster_roles(self):
282+
"""Returns a list of predefined ClusterRoles."""
283+
return [role.get('rulesFromRoleName')
284+
for role in self._roles
285+
if role['type'] == 'ClusterRole'
286+
and role['rulesType'] == 'PREDEFINED']
287+
288+
289+
class SchemaXStorageClass:
290+
"""Wrapper class providing convenient access to STORAGE_CLASS property"""
291+
292+
def __init__(self, dictionary):
293+
self._type = dictionary['type']
294+
295+
@property
296+
def ssd(self):
297+
return self._type == 'SSD'

0 commit comments

Comments
 (0)