Skip to content

Commit

Permalink
Merge pull request #2197 from atlanhq/gov-1179
Browse files Browse the repository at this point in the history 
[stag] GOV-1179 Support Resources, Terms linking in Purpose metadata policy
  • Loading branch information
@nikhilbonte21
nikhilbonte21 authored Aug 7, 2023
2 parents 851bec6 + 14a2cc5 commit cb399cd
Show file tree
Hide file tree
Showing 10 changed files with 257 additions and 31 deletions.
0 ...c/templates/policy_cache_transformer.json → ...tes/policy_cache_transformer_persona.json
View file
File renamed without changes.
68 changes: 68 additions & 0 deletions addons/static/templates/policy_cache_transformer_purpose.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
{
"entity-update": [
{
"policyType": "ACCESS",
"policyResourceCategory": "TAG",
"resources": [
"tag:{tag}"
],
"actions": ["entity-update"]
},
{
"policyServiceName": "atlas",
"policyType": "ACCESS",
"policyResourceCategory": "RELATIONSHIP",
"resources": [
"end-one-entity-classification:{tag}",
"end-one-entity:*",
"end-one-entity-type:*",

"end-two-entity:*",
"end-two-entity-type:Readme",
"end-two-entity-type:Link",
"end-two-entity-classification:*",

"relationship-type:*"
],
"actions": ["add-relationship", "remove-relationship"]
}
],
"purpose-add-terms": [
{
"policyServiceName": "atlas",
"policyType": "ACCESS",
"policyResourceCategory": "RELATIONSHIP",
"resources": [
"relationship-type:*",

"end-one-entity-classification:*",
"end-one-entity:*",
"end-one-entity-type:AtlasGlossaryTerm",

"end-two-entity:*",
"end-two-entity-type:*",
"end-two-entity-classification:{tag}"
],
"actions": ["add-relationship"]
}
],
"purpose-remove-terms": [
{
"policyServiceName": "atlas",
"policyType": "ACCESS",
"policyResourceCategory": "RELATIONSHIP",
"resources": [
"relationship-type:*",

"end-one-entity-classification:*",
"end-one-entity:*",
"end-one-entity-type:AtlasGlossaryTerm",

"end-two-entity:*",
"end-two-entity-type:*",
"end-two-entity-classification:{tag}"
],
"actions": ["remove-relationship"]
}
]
}
22 changes: 14 additions & 8 deletions ...mmon/src/main/java/org/apache/atlas/policytransformer/AbstractCachePolicyTransformer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,23 +21,29 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import static org.apache.atlas.policytransformer.CacheTransformerTemplateHelper.RESOURCE_POLICY_TRANSFORMER;
import java.util.HashMap;
import java.util.Map;

public abstract class AbstractCachePolicyTransformer implements CachePolicyTransformer {

private static final Logger LOG = LoggerFactory.getLogger(AbstractCachePolicyTransformer.class);

public static final String PLACEHOLDER_ENTITY = "{entity}";
public static final String PLACEHOLDER_ENTITY_TYPE = "{entity-type}";
public static final String PLACEHOLDER_TAG = "{tag}";

public PolicyTransformerTemplate templates;
private static Map<String, PolicyTransformerTemplate> TEMPLATES = new HashMap<>();

public AbstractCachePolicyTransformer() throws AtlasBaseException {
try {
templates = CacheTransformerTemplateHelper.getTemplate();
} catch (AtlasBaseException e) {
LOG.error("Failed to load template for policies: {}", RESOURCE_POLICY_TRANSFORMER);
throw e;
public PolicyTransformerTemplate getTemplate(String fileSuffix) throws AtlasBaseException {
if (!TEMPLATES.containsKey(fileSuffix)) {
try {
TEMPLATES.put(fileSuffix, CacheTransformerTemplateHelper.getTemplate(fileSuffix));
} catch (AtlasBaseException e) {
LOG.error("Failed to load template for policies: {}", fileSuffix);
throw e;
}
}

return TEMPLATES.get(fileSuffix);
}
}
29 changes: 23 additions & 6 deletions ...s-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ public class CachePolicyTransformerImpl {
private EntityGraphRetriever entityRetriever;

private PersonaCachePolicyTransformer personaTransformer;
private PurposeCachePolicyTransformer purposeTransformer;

private AtlasEntityHeader service;

Expand All @@ -119,6 +120,7 @@ public CachePolicyTransformerImpl(AtlasTypeRegistry typeRegistry) throws AtlasBa
this.entityRetriever = new EntityGraphRetriever(graph, typeRegistry);

personaTransformer = new PersonaCachePolicyTransformer(entityRetriever);
purposeTransformer = new PurposeCachePolicyTransformer(entityRetriever);

try {
this.discoveryService = new EntityDiscoveryService(typeRegistry, graph, null, null, null, null);
Expand Down Expand Up @@ -146,26 +148,27 @@ public ServicePolicies getPolicies(String serviceName, String pluginId, Long las
servicePolicies.setPolicyUpdateTime(new Date());

if (service != null) {
List<RangerPolicy> policies = getServicePolicies(service);
List<RangerPolicy> allPolicies = getServicePolicies(service);
servicePolicies.setServiceName(serviceName);
servicePolicies.setPolicies(policies);
//servicePolicies.setPolicies(policies);
servicePolicies.setServiceId(service.getGuid());

String serviceDefName = String.format(RESOURCE_SERVICE_DEF_PATTERN, serviceName);
servicePolicies.setServiceDef(getResourceAsObject(serviceDefName, RangerServiceDef.class));


//Process tag based policies
String tagServiceName = (String) service.getAttribute("tagService");
String tagServiceName = (String) service.getAttribute(ATTR_SERVICE_TAG_SERVICE);
if (StringUtils.isNotEmpty(tagServiceName)) {
AtlasEntityHeader tagService = getServiceEntity(tagServiceName);

if (tagService != null) {
policies = getServicePolicies(tagService);
allPolicies.addAll(getServicePolicies(tagService));

TagPolicies tagPolicies = new TagPolicies();

tagPolicies.setServiceName(tagServiceName);
tagPolicies.setPolicies(policies);
//tagPolicies.setPolicies(atlasTagPolicies);
tagPolicies.setPolicyUpdateTime(new Date());
tagPolicies.setServiceId(tagService.getGuid());
tagPolicies.setPolicyVersion(-1L);
Expand All @@ -177,6 +180,16 @@ public ServicePolicies getPolicies(String serviceName, String pluginId, Long las
}
}

AtlasPerfMetrics.MetricRecorder recorder1 = RequestContext.get().startMetricRecord("filterPolicies");
//filter out policies based on serviceName
List<RangerPolicy> policiesA = allPolicies.stream().filter(x -> serviceName.equals(x.getService())).collect(Collectors.toList());
List<RangerPolicy> policiesB = allPolicies.stream().filter(x -> tagServiceName.equals(x.getService())).collect(Collectors.toList());

servicePolicies.setPolicies(policiesA);
servicePolicies.getTagPolicies().setPolicies(policiesB);

RequestContext.get().endMetricRecord(recorder1);

if (LOG.isDebugEnabled()) {
LOG.debug("Found {} policies", servicePolicies.getPolicies().size());
}
Expand Down Expand Up @@ -224,7 +237,11 @@ private List<RangerPolicy> transformAtlasPoliciesToRangerPolicies(List<AtlasEnti
}

} else if (POLICY_CATEGORY_PURPOSE.equals(policyCategory)) {
rangerPolicies.add(toRangerPolicy(atlasPolicy, serviceType));
List<AtlasEntityHeader> transformedAtlasPolicies = purposeTransformer.transform(atlasPolicy);

for (AtlasEntityHeader transformedPolicy : transformedAtlasPolicies) {
rangerPolicies.add(toRangerPolicy(transformedPolicy, serviceType));
}

} else {
rangerPolicies.add(toRangerPolicy(atlasPolicy, serviceType));
Expand Down
21 changes: 8 additions & 13 deletions ...mmon/src/main/java/org/apache/atlas/policytransformer/CacheTransformerTemplateHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,27 +28,22 @@
public class CacheTransformerTemplateHelper {
private static final Logger LOG = LoggerFactory.getLogger(CacheTransformerTemplateHelper.class);

static final String RESOURCE_POLICY_TRANSFORMER = "templates/policy_cache_transformer.json";
static final String RESOURCE_POLICY_TRANSFORMER = "templates/policy_cache_transformer_%s.json";

private static PolicyTransformerTemplate templates;

public static PolicyTransformerTemplate getTemplate() throws AtlasBaseException {
if (templates == null) {
loadTemplate();
}

return templates;
}

private static void loadTemplate() {
public static PolicyTransformerTemplate getTemplate(String fileSuffix) throws AtlasBaseException {
PolicyTransformerTemplate templates;
String jsonTemplate = null;
String fileName = String.format(RESOURCE_POLICY_TRANSFORMER, fileSuffix);

try {
jsonTemplate = getStaticFileAsString(RESOURCE_POLICY_TRANSFORMER);
jsonTemplate = getStaticFileAsString(fileName);
} catch (IOException e) {
LOG.error("Failed to load template for policies: {}", RESOURCE_POLICY_TRANSFORMER);
throw new AtlasBaseException(e);
}
templates = new PolicyTransformerTemplate();
templates.fromJsonString(jsonTemplate);

return templates;
}
}
7 changes: 5 additions & 2 deletions ...ommon/src/main/java/org/apache/atlas/policytransformer/PersonaCachePolicyTransformer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,13 @@
public class PersonaCachePolicyTransformer extends AbstractCachePolicyTransformer {
private static final Logger LOG = LoggerFactory.getLogger(PersonaCachePolicyTransformer.class);

private final static String TEMPLATE_SUFFIX = "persona";

private EntityGraphRetriever entityRetriever = null;
private PolicyTransformerTemplate personaTemplate;

public PersonaCachePolicyTransformer(EntityGraphRetriever entityRetriever) throws AtlasBaseException {
super();
personaTemplate = getTemplate(TEMPLATE_SUFFIX);
this.entityRetriever = entityRetriever;
}

Expand All @@ -69,7 +72,7 @@ public List<AtlasEntityHeader> transform(AtlasEntityHeader atlasPolicy) {

int index = 0;
for (String atlasAction : atlasActions) {
List<PolicyTransformerTemplate.TemplatePolicy> currentTemplates = templates.getTemplate(atlasAction);
List<PolicyTransformerTemplate.TemplatePolicy> currentTemplates = personaTemplate.getTemplate(atlasAction);

if (CollectionUtils.isEmpty(currentTemplates)) {
LOG.warn("PolicyTransformerImpl: Skipping unknown action {} while transforming policy {}", atlasAction, atlasPolicy.getGuid());
Expand Down
17 changes: 15 additions & 2 deletions ...ts-common/src/main/java/org/apache/atlas/policytransformer/PolicyTransformerTemplate.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,16 @@
package org.apache.atlas.policytransformer;

import org.apache.atlas.type.AtlasType;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class PolicyTransformerTemplate {
private static final Logger LOG = LoggerFactory.getLogger(PolicyTransformerTemplate.class);
Expand All @@ -39,8 +42,8 @@ public List<TemplatePolicy> getTemplate(String action) {
return actionToPoliciesMap.get(action);
}

public Map<String, List<TemplatePolicy>> getTemplates() {
return actionToPoliciesMap;
public Set<String> getTemplateActions() {
return new HashSet<>(actionToPoliciesMap.keySet());
}

public void fromJsonString(String json) {
Expand All @@ -58,6 +61,7 @@ public void fromJsonString(String json) {
templatePolicy.setResources((List<String>) policy.get("resources"));
templatePolicy.setPolicyType((String) policy.get("policyType"));
templatePolicy.setPolicyResourceCategory((String) policy.get("policyResourceCategory"));
templatePolicy.setPolicyServiceName((String) policy.get("policyServiceName"));

policies.add(templatePolicy);
}
Expand All @@ -67,11 +71,20 @@ public void fromJsonString(String json) {
}

class TemplatePolicy {
private String policyServiceName;
private String policyType;
private List<String> resources;
private List<String> actions;
private String policyResourceCategory;

public String getPolicyServiceName() {
return policyServiceName;
}

public void setPolicyServiceName(String policyServiceName) {
this.policyServiceName = policyServiceName;
}

public String getPolicyType() {
return policyType;
}
Expand Down
Loading

0 comments on commit cb399cd

Please sign in to comment.