Skip to content

Commit 33a5039

Browse files
cognifloydcognifloyd
authored
Merge pull request #5868: Add pants-plugins/pack_metadata to inform pants about our packs
2 parents 5373574 + a73225c commit 33a5039

File tree

11 files changed

+372
-2
lines changed

11 files changed

+372
-2
lines changed

‎CHANGELOG.rst

+1-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ Added
1414
working on StackStorm, improve our security posture, and improve CI reliability thanks in part
1515
to pants' use of PEX lockfiles. This is not a user-facing addition.
1616
#5778 #5789 #5817 #5795 #5830 #5833 #5834 #5841 #5840 #5838 #5842 #5837 #5849 #5850
17-
#5846 #5853 #5848 #5847 #5858 #5857 #5860
17+
#5846 #5853 #5848 #5847 #5858 #5857 #5860 #5868
1818
Contributed by @cognifloyd
1919

2020
* Added a joint index to solve the problem of slow mongo queries for scheduled executions. #5805

‎pants-plugins/README.md

+23
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,9 @@ The plugins here add custom goals or other logic into pants.
88

99
To see available goals, do "./pants help goals" and "./pants help $goal".
1010

11+
These StackStorm-specific plugins might be useful in other StackStorm-related repos.
12+
- `pack_metadata`
13+
1114
These StackStorm-specific plugins are probably only useful for the st2 repo.
1215
- `api_spec`
1316
- `sample_conf`
@@ -26,6 +29,26 @@ This plugin also wires up pants so that the `lint` goal runs additional
2629
api spec validation on `st2common/st2common/openapi.yaml` with something
2730
like `./pants lint st2common/st2common/openapi.yaml`.
2831

32+
### `pack_metadata` plugin
33+
34+
This plugin adds two new targets to pants:
35+
- `pack_metadata`
36+
- `pack_metadata_in_git_submodule`
37+
38+
These targets include all StackStorm pack metadata files in a pack.
39+
Pack metadata includes top-level files (`pack.yaml`, `<pack>.yaml.example`,
40+
`config.schema.yaml`, and `icon.png`) and metadata (`*.yaml`, `*.yml`)
41+
for actions, action-aliases, policies, rules, and sensors.
42+
43+
This plugin also wires up the `tailor` goal, so that it will add a
44+
`pack_metadata(name="metadata")` target wherever it finds a `pack.yaml` file.
45+
46+
One of the packs in this repo is in a git submodule to test our handling
47+
of git submodules (`st2tests/st2tests/fixtures/packs/test_content_version`).
48+
If it is not checked out, then some of the tests will fail.
49+
If it is not checked out, `pack_metadata_in_git_submodule` handles providing
50+
a helpful, instructive error message as early as possible.
51+
2952
### `sample_conf` plugin
3053

3154
This plugin wires up pants to make sure `conf/st2.conf.sample` gets

‎pants-plugins/pack_metadata/BUILD

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
python_sources()
2+
3+
python_tests(
4+
name="tests",
5+
)

‎pants-plugins/pack_metadata/__init__.py

Whitespace-only changes.

‎pants-plugins/pack_metadata/register.py

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Copyright 2023 The StackStorm Authors.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
from pack_metadata import tailor
15+
from pack_metadata.target_types import PackMetadata, PackMetadataInGitSubmodule
16+
17+
18+
def rules():
19+
return tailor.rules()
20+
21+
22+
def target_types():
23+
return [PackMetadata, PackMetadataInGitSubmodule]

‎pants-plugins/pack_metadata/tailor.py

+62
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
# Copyright 2023 The StackStorm Authors.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
import os
15+
from dataclasses import dataclass
16+
17+
from pants.core.goals.tailor import (
18+
AllOwnedSources,
19+
PutativeTarget,
20+
PutativeTargets,
21+
PutativeTargetsRequest,
22+
)
23+
from pants.engine.fs import PathGlobs, Paths
24+
from pants.engine.rules import collect_rules, Get, rule, UnionRule
25+
from pants.util.logging import LogLevel
26+
27+
from pack_metadata.target_types import PackMetadata
28+
29+
30+
@dataclass(frozen=True)
31+
class PutativePackMetadataTargetsRequest(PutativeTargetsRequest):
32+
pass
33+
34+
35+
@rule(
36+
desc="Find pack (config, action, alias, sensor, icon, etc) metadata files.",
37+
level=LogLevel.DEBUG,
38+
)
39+
async def find_putative_targets(
40+
_: PutativePackMetadataTargetsRequest, all_owned_sources: AllOwnedSources
41+
) -> PutativeTargets:
42+
all_pack_yaml_files = await Get(Paths, PathGlobs(["**/pack.yaml"]))
43+
44+
unowned_pack_yaml_files = set(all_pack_yaml_files.files) - set(all_owned_sources)
45+
unowned_pack_dirs = [os.path.dirname(p) for p in unowned_pack_yaml_files]
46+
47+
name = "metadata"
48+
return PutativeTargets(
49+
[
50+
PutativeTarget.for_target_type(
51+
PackMetadata, dirname, name, ("pack.yaml",), kwargs={"name": name}
52+
)
53+
for dirname in unowned_pack_dirs
54+
]
55+
)
56+
57+
58+
def rules():
59+
return [
60+
*collect_rules(),
61+
UnionRule(PutativeTargetsRequest, PutativePackMetadataTargetsRequest),
62+
]

‎pants-plugins/pack_metadata/tailor_test.py

+107
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
# Copyright 2023 The StackStorm Authors.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
from __future__ import annotations
15+
16+
import pytest
17+
18+
from pants.core.goals.tailor import (
19+
AllOwnedSources,
20+
PutativeTarget,
21+
PutativeTargets,
22+
)
23+
from pants.testutil.rule_runner import QueryRule, RuleRunner
24+
25+
from .tailor import (
26+
PutativePackMetadataTargetsRequest,
27+
rules as pack_metadata_rules,
28+
)
29+
from .target_types import PackMetadata, PackMetadataInGitSubmodule
30+
31+
32+
@pytest.fixture
33+
def rule_runner() -> RuleRunner:
34+
return RuleRunner(
35+
rules=[
36+
*pack_metadata_rules(),
37+
QueryRule(
38+
PutativeTargets, (PutativePackMetadataTargetsRequest, AllOwnedSources)
39+
),
40+
],
41+
target_types=[PackMetadata, PackMetadataInGitSubmodule],
42+
)
43+
44+
45+
def test_find_putative_targets(rule_runner: RuleRunner) -> None:
46+
rule_runner.write_files(
47+
{
48+
"packs/already_owned/pack.yaml": "---\nname: already_owned\n",
49+
"packs/already_owned/actions/action.yaml": "---\nname: action\n",
50+
"packs/foo/pack.yaml": "---\nname: foo\n",
51+
"packs/foo/actions/action.yaml": "---\nname: action\n",
52+
"packs/bar/pack.yaml": "---\nname: bar\n",
53+
"packs/bar/sensors/sensor.yaml": "---\nname: sensor\n",
54+
"other/deep/baz/pack.yaml": "---\nname: baz\n",
55+
}
56+
)
57+
pts = rule_runner.request(
58+
PutativeTargets,
59+
[
60+
PutativePackMetadataTargetsRequest(
61+
(
62+
"packs",
63+
"packs/already_owned",
64+
"packs/already_owned/actions",
65+
"packs/foo",
66+
"packs/foo/actions",
67+
"packs/bar",
68+
"packs/bar/sensors",
69+
"other/deep/baz",
70+
)
71+
),
72+
AllOwnedSources(
73+
[
74+
"packs/already_owned/pack.yaml",
75+
"packs/already_owned/actions/action.yaml",
76+
]
77+
),
78+
],
79+
)
80+
assert (
81+
PutativeTargets(
82+
[
83+
PutativeTarget.for_target_type(
84+
PackMetadata,
85+
path="packs/foo",
86+
name="metadata",
87+
triggering_sources=["pack.yaml"],
88+
kwargs={"name": "metadata"},
89+
),
90+
PutativeTarget.for_target_type(
91+
PackMetadata,
92+
path="packs/bar",
93+
name="metadata",
94+
triggering_sources=["pack.yaml"],
95+
kwargs={"name": "metadata"},
96+
),
97+
PutativeTarget.for_target_type(
98+
PackMetadata,
99+
path="other/deep/baz",
100+
name="metadata",
101+
triggering_sources=["pack.yaml"],
102+
kwargs={"name": "metadata"},
103+
),
104+
]
105+
)
106+
== pts
107+
)

‎pants-plugins/pack_metadata/target_types.py

+79
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
# Copyright 2023 The StackStorm Authors.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
from typing import Sequence
15+
16+
from pants.engine.target import COMMON_TARGET_FIELDS, Dependencies
17+
from pants.core.target_types import (
18+
ResourcesGeneratingSourcesField,
19+
ResourcesGeneratorTarget,
20+
)
21+
22+
23+
class UnmatchedGlobsError(Exception):
24+
"""Error thrown when a required set of globs didn't match."""
25+
26+
27+
class PackMetadataSourcesField(ResourcesGeneratingSourcesField):
28+
required = False
29+
default = (
30+
# metadata does not include any python, shell, or other sources.
31+
"pack.yaml",
32+
"config.schema.yaml",
33+
"*.yaml.example",
34+
"**/*.yaml",
35+
"**/*.yml",
36+
"icon.png", # used in st2web ui
37+
# "requirements*.txt", # including this causes target conflicts
38+
# "README.md",
39+
# "HISTORY.md",
40+
)
41+
42+
43+
class PackMetadataInGitSubmoduleSources(PackMetadataSourcesField):
44+
required = True
45+
46+
def validate_resolved_files(self, files: Sequence[str]) -> None:
47+
if not files:
48+
raise UnmatchedGlobsError(
49+
# see: st2tests.fixturesloader.GIT_SUBMODULES_NOT_CHECKED_OUT_ERROR
50+
"One or more git submodules is not checked out. Make sure to run "
51+
'"git submodule update --init --recursive"'
52+
"in the repository root directory to check out all the submodules."
53+
)
54+
super().validate_resolved_files(files)
55+
56+
57+
class PackMetadata(ResourcesGeneratorTarget):
58+
alias = "pack_metadata"
59+
core_fields = (*COMMON_TARGET_FIELDS, Dependencies, PackMetadataSourcesField)
60+
help = (
61+
"Loose pack metadata files.\n\n"
62+
"Pack metadata includes top-level files (pack.yaml, <pack>.yaml.example, "
63+
"config.schema.yaml, and icon.png) and metadata for actions, "
64+
"action-aliases, policies, rules, and sensors."
65+
)
66+
67+
68+
class PackMetadataInGitSubmodule(PackMetadata):
69+
alias = "pack_metadata_in_git_submodule"
70+
core_fields = (
71+
*COMMON_TARGET_FIELDS,
72+
Dependencies,
73+
PackMetadataInGitSubmoduleSources,
74+
)
75+
help = PackMetadata.help + (
76+
"\npack_metadata_in_git_submodule variant errors if the sources field "
77+
"has unmatched globs. It prints instructions on how to checkout git "
78+
"submodules."
79+
)

‎pants-plugins/pack_metadata/target_types_test.py

+68
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
# Copyright 2023 The StackStorm Authors.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
from __future__ import annotations
15+
16+
import pytest
17+
18+
from pants.engine.addresses import Address
19+
from pants.engine.internals.scheduler import ExecutionError
20+
from pants.testutil.rule_runner import RuleRunner
21+
22+
from .target_types import (
23+
PackMetadata,
24+
# PackMetadataSourcesField,
25+
PackMetadataInGitSubmodule,
26+
# PackMetadataInGitSubmoduleSources,
27+
UnmatchedGlobsError,
28+
)
29+
30+
31+
@pytest.fixture
32+
def rule_runner() -> RuleRunner:
33+
return RuleRunner(
34+
rules=[],
35+
target_types=[PackMetadata, PackMetadataInGitSubmodule],
36+
)
37+
38+
39+
GIT_SUBMODULE_BUILD_FILE = """
40+
pack_metadata_in_git_submodule(
41+
name="metadata",
42+
sources=["./submodule_dir/pack.yaml"],
43+
)
44+
"""
45+
46+
47+
def test_git_submodule_sources_missing(rule_runner: RuleRunner) -> None:
48+
rule_runner.write_files(
49+
{
50+
"packs/BUILD": GIT_SUBMODULE_BUILD_FILE,
51+
}
52+
)
53+
with pytest.raises(ExecutionError) as e:
54+
_ = rule_runner.get_target(Address("packs", target_name="metadata"))
55+
exc = e.value.wrapped_exceptions[0]
56+
assert isinstance(exc, UnmatchedGlobsError)
57+
assert "One or more git submodules is not checked out" in str(exc)
58+
59+
60+
def test_git_submodule_sources_present(rule_runner: RuleRunner) -> None:
61+
rule_runner.write_files(
62+
{
63+
"packs/BUILD": GIT_SUBMODULE_BUILD_FILE,
64+
"packs/submodule_dir/pack.yaml": "---\nname: foobar\n",
65+
}
66+
)
67+
# basically: this asserts that it does not raise UnmatchedGlobsError
68+
_ = rule_runner.get_target(Address("packs", target_name="metadata"))

0 commit comments

Comments
 (0)