Skip to content

Commit

Permalink
Merge pull request #3819 from atlanhq/dg1869dev
Browse files Browse the repository at this point in the history 
DG-1869 Evaluates API cache optimisation.
  • Loading branch information
@hr2904
hr2904 authored Jan 15, 2025
2 parents f15fcb2 + 0313b3a commit 396fb19
Show file tree
Hide file tree
Showing 4 changed files with 136 additions and 96 deletions.
9 changes: 9 additions & 0 deletions repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -374,4 +374,13 @@ EntityMutationResponse deleteByUniqueAttributes(List<AtlasObjectId> objectIds)
void unlinkBusinessPolicy(String policyId, Set<String> unlinkGuids) throws AtlasBaseException;

void moveBusinessPolicies(Set<String> policyId, String assetId, String type) throws AtlasBaseException;

/**
*
* @param entities
* @throws AtlasBaseException
*
* For evaluations of policies
*/
List<AtlasEvaluatePolicyResponse> evaluatePolicies(List<AtlasEvaluatePolicyRequest> entities) throws AtlasBaseException;
}
111 changes: 111 additions & 0 deletions repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,10 +92,13 @@
import javax.inject.Inject;
import java.io.InputStream;
import java.util.*;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;

import static java.lang.Boolean.FALSE;
import static org.apache.atlas.AtlasConfiguration.STORE_DIFFERENTIAL_AUDITS;
import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST;
import static org.apache.atlas.authorize.AtlasPrivilege.*;
import static org.apache.atlas.bulkimport.BulkImportResponse.ImportStatus.FAILED;
import static org.apache.atlas.model.instance.AtlasEntity.Status.ACTIVE;
import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.*;
Expand Down Expand Up @@ -758,6 +761,114 @@ public EntityMutationResponse deleteByUniqueAttributes(AtlasEntityType entityTyp
return ret;
}

private AtlasEntityHeader getAtlasEntityHeader(String entityGuid, String entityId, String entityType) throws AtlasBaseException {
// Metric logs
AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("getAtlasEntityHeader");
AtlasEntityHeader entityHeader = null;
String cacheKey = generateCacheKey(entityGuid, entityId, entityType);
entityHeader = RequestContext.get().getCachedEntityHeader(cacheKey);
if(Objects.nonNull(entityHeader)){
return entityHeader;
}
if (StringUtils.isNotEmpty(entityGuid)) {
AtlasEntityWithExtInfo ret = getByIdWithoutAuthorization(entityGuid);
entityHeader = new AtlasEntityHeader(ret.getEntity());
} else if (StringUtils.isNotEmpty(entityId) && StringUtils.isNotEmpty(entityType)) {
try {
entityHeader = getAtlasEntityHeaderWithoutAuthorization(null, entityId, entityType);
} catch (AtlasBaseException abe) {
if (abe.getAtlasErrorCode() == AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND) {
Map<String, Object> attributes = new HashMap<>();
attributes.put(QUALIFIED_NAME, entityId);
entityHeader = new AtlasEntityHeader(entityType, attributes);
}
}
} else {
throw new AtlasBaseException(BAD_REQUEST, "requires entityGuid or typeName and qualifiedName for entity authorization");
}
RequestContext.get().setEntityHeaderCache(cacheKey, entityHeader);
RequestContext.get().endMetricRecord(metric);
return entityHeader;
}

@Override
public List<AtlasEvaluatePolicyResponse> evaluatePolicies(List<AtlasEvaluatePolicyRequest> entities) throws AtlasBaseException {
List<AtlasEvaluatePolicyResponse> response = new ArrayList<>();
HashMap<String, AtlasEntityHeader> atlasEntityHeaderCache = new HashMap<>();
for (AtlasEvaluatePolicyRequest entity : entities) {
String action = entity.getAction();

if (action == null) {
throw new AtlasBaseException(BAD_REQUEST, "action is null");
}
AtlasEntityHeader entityHeader = null;

if (ENTITY_READ.name().equals(action) || ENTITY_CREATE.name().equals(action) || ENTITY_UPDATE.name().equals(action)
|| ENTITY_DELETE.name().equals(action) || ENTITY_UPDATE_BUSINESS_METADATA.name().equals(action)) {

try {
entityHeader = getAtlasEntityHeader(entity.getEntityGuid(), entity.getEntityId(), entity.getTypeName());
AtlasEntityAccessRequest.AtlasEntityAccessRequestBuilder requestBuilder = new AtlasEntityAccessRequest.AtlasEntityAccessRequestBuilder(typeRegistry, AtlasPrivilege.valueOf(entity.getAction()), entityHeader);
if (entity.getBusinessMetadata() != null) {
requestBuilder.setBusinessMetadata(entity.getBusinessMetadata());
}

AtlasEntityAccessRequest entityAccessRequest = requestBuilder.build();

AtlasAuthorizationUtils.verifyAccess(entityAccessRequest, entity.getAction() + "guid=" + entity.getEntityGuid());
response.add(new AtlasEvaluatePolicyResponse(entity.getTypeName(), entity.getEntityGuid(), entity.getAction(), entity.getEntityId(), true, null , entity.getBusinessMetadata()));
} catch (AtlasBaseException e) {
AtlasErrorCode code = e.getAtlasErrorCode();
String errorCode = code.getErrorCode();
response.add(new AtlasEvaluatePolicyResponse(entity.getTypeName(), entity.getEntityGuid(), entity.getAction(), entity.getEntityId(), false, errorCode, entity.getBusinessMetadata()));
}

} else if (ENTITY_REMOVE_CLASSIFICATION.name().equals(action) || ENTITY_ADD_CLASSIFICATION.name().equals(action) || ENTITY_UPDATE_CLASSIFICATION.name().equals(action)) {

if (entity.getClassification() == null) {
throw new AtlasBaseException(BAD_REQUEST, "classification needed for " + action + " authorization");
}
try {
entityHeader = getAtlasEntityHeader(entity.getEntityGuid(), entity.getEntityId(), entity.getTypeName());

AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.valueOf(entity.getAction()), entityHeader, new AtlasClassification(entity.getClassification())));
response.add(new AtlasEvaluatePolicyResponse(entity.getTypeName(), entity.getEntityGuid(), entity.getAction(), entity.getEntityId(), entity.getClassification(), true, null));

} catch (AtlasBaseException e) {
AtlasErrorCode code = e.getAtlasErrorCode();
String errorCode = code.getErrorCode();
response.add(new AtlasEvaluatePolicyResponse(entity.getTypeName(), entity.getEntityGuid(), entity.getAction(), entity.getEntityId(), entity.getClassification(), false, errorCode));
}

} else if (RELATIONSHIP_ADD.name().equals(action) || RELATIONSHIP_REMOVE.name().equals(action) || RELATIONSHIP_UPDATE.name().equals(action)) {

if (entity.getRelationShipTypeName() == null) {
throw new AtlasBaseException(BAD_REQUEST, "RelationShip TypeName needed for " + action + " authorization");
}

try {
AtlasEntityHeader end1Entity = getAtlasEntityHeader(entity.getEntityGuidEnd1(), entity.getEntityIdEnd1(), entity.getEntityTypeEnd1());

AtlasEntityHeader end2Entity = getAtlasEntityHeader(entity.getEntityGuidEnd2(), entity.getEntityIdEnd2(), entity.getEntityTypeEnd2());

AtlasAuthorizationUtils.verifyAccess(new AtlasRelationshipAccessRequest(typeRegistry, AtlasPrivilege.valueOf(action), entity.getRelationShipTypeName(), end1Entity, end2Entity));
response.add(new AtlasEvaluatePolicyResponse(action, entity.getRelationShipTypeName(), entity.getEntityTypeEnd1(), entity.getEntityGuidEnd1(), entity.getEntityIdEnd1(), entity.getEntityTypeEnd2(), entity.getEntityGuidEnd2(), entity.getEntityIdEnd2(), true, null));
} catch (AtlasBaseException e) {
AtlasErrorCode code = e.getAtlasErrorCode();
String errorCode = code.getErrorCode();
response.add(new AtlasEvaluatePolicyResponse(action, entity.getRelationShipTypeName(), entity.getEntityTypeEnd1(), entity.getEntityGuidEnd1(), entity.getEntityIdEnd1(), entity.getEntityTypeEnd2(), entity.getEntityGuidEnd2(), entity.getEntityIdEnd2(), false, errorCode));
}
}
}
return response;
}

private String generateCacheKey(String guid, String id, String typeName) {
return (guid != null ? guid : "") + "|" + (id != null ? id : "") + "|" + (typeName != null ? typeName : "");
}



@Override
@GraphTransaction
public EntityMutationResponse deleteByUniqueAttributes(List<AtlasObjectId> objectIds) throws AtlasBaseException {
Expand Down
21 changes: 15 additions & 6 deletions server-api/src/main/java/org/apache/atlas/RequestContext.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ public class RequestContext {
private final Set<String> deletedEdgesIds = new HashSet<>();
private final Set<String> processGuidIds = new HashSet<>();

private Map<String, String> evaluateEntityHeaderCache = null;
private final AtlasPerfMetrics metrics = isMetricsEnabled ? new AtlasPerfMetrics() : null;
private final List<AtlasPerfMetrics.Metric> applicationMetrics = new ArrayList<>();
private List<EntityGuidPair> entityGuidInRequest = null;
Expand Down Expand Up @@ -567,17 +568,17 @@ public void cacheDifferentialEntity(AtlasEntity entity) {
}
}

public void setEntityHeaderCache(AtlasEntityHeader headerCache){
if(headerCache != null && headerCache.getGuid() != null){
entityHeaderCache.put(headerCache.getGuid(), headerCache);
public void setEntityHeaderCache(String cacheKey, AtlasEntityHeader headerCache){
if(headerCache != null && StringUtils.isNotEmpty(cacheKey)){
entityHeaderCache.put(cacheKey, headerCache);
}
}

public AtlasEntityHeader getCachedEntityHeader(String guid){
if(guid == null){
public AtlasEntityHeader getCachedEntityHeader(String cacheKey){
if(cacheKey == null){
return null;
}
return entityHeaderCache.get(guid);
return entityHeaderCache.getOrDefault(cacheKey,null);
}

public AtlasEntity getDifferentialEntity(String guid) {
Expand Down Expand Up @@ -753,6 +754,14 @@ public void setClientOrigin(String clientOrigin) {
this.clientOrigin = StringUtils.isEmpty(this.clientOrigin) ? "other" :clientOrigin;
}

public Map<String, String> getEvaluateEntityHeaderCache() {
return evaluateEntityHeaderCache;
}

public void setEvaluateEntityHeaderCache(Map<String, String> evaluateEntityHeaderCache) {
this.evaluateEntityHeaderCache = evaluateEntityHeaderCache;
}

public class EntityGuidPair {
private final Object entity;
private final String guid;
Expand Down
91 changes: 1 addition & 90 deletions webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,74 +164,7 @@ public List<AtlasEvaluatePolicyResponse> evaluatePolicies(List<AtlasEvaluatePoli
if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityREST.evaluatePolicies()");
}

for (int i = 0; i < entities.size(); i++) {

String action = entities.get(i).getAction();

if (action == null) {
throw new AtlasBaseException(BAD_REQUEST, "action is null");
}

if (ENTITY_READ.name().equals(action) || ENTITY_CREATE.name().equals(action) || ENTITY_UPDATE.name().equals(action)
|| ENTITY_DELETE.name().equals(action) || ENTITY_UPDATE_BUSINESS_METADATA.name().equals(action)) {

try {
AtlasEntityHeader entityHeader = getAtlasEntityHeader(entities.get(i).getEntityGuid(), entities.get(i).getEntityId(),entities.get(i).getTypeName());

AtlasEntityAccessRequest.AtlasEntityAccessRequestBuilder requestBuilder = new AtlasEntityAccessRequest.AtlasEntityAccessRequestBuilder(typeRegistry, AtlasPrivilege.valueOf(entities.get(i).getAction()), entityHeader);
if (entities.get(i).getBusinessMetadata() != null) {
requestBuilder.setBusinessMetadata(entities.get(i).getBusinessMetadata());
}

AtlasEntityAccessRequest entityAccessRequest = requestBuilder.build();

AtlasAuthorizationUtils.verifyAccess(entityAccessRequest, entities.get(i).getAction() + "guid=" + entities.get(i).getEntityGuid());
response.add(new AtlasEvaluatePolicyResponse(entities.get(i).getTypeName(), entities.get(i).getEntityGuid(), entities.get(i).getAction(), entities.get(i).getEntityId(), true, null , entities.get(i).getBusinessMetadata()));
} catch (AtlasBaseException e) {
AtlasErrorCode code = e.getAtlasErrorCode();
String errorCode = code.getErrorCode();
response.add(new AtlasEvaluatePolicyResponse(entities.get(i).getTypeName(), entities.get(i).getEntityGuid(), entities.get(i).getAction(), entities.get(i).getEntityId(), false, errorCode, entities.get(i).getBusinessMetadata()));
}

} else if (ENTITY_REMOVE_CLASSIFICATION.name().equals(action) || ENTITY_ADD_CLASSIFICATION.name().equals(action) || ENTITY_UPDATE_CLASSIFICATION.name().equals(action)) {

if (entities.get(i).getClassification() == null) {
throw new AtlasBaseException(BAD_REQUEST, "classification needed for " + action + " authorization");
}
try {
AtlasEntityHeader entityHeader = getAtlasEntityHeader(entities.get(i).getEntityGuid(), entities.get(i).getEntityId(),entities.get(i).getTypeName());

AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.valueOf(entities.get(i).getAction()), entityHeader, new AtlasClassification(entities.get(i).getClassification())));
response.add(new AtlasEvaluatePolicyResponse(entities.get(i).getTypeName(), entities.get(i).getEntityGuid(), entities.get(i).getAction(), entities.get(i).getEntityId(), entities.get(i).getClassification(), true, null));

} catch (AtlasBaseException e) {
AtlasErrorCode code = e.getAtlasErrorCode();
String errorCode = code.getErrorCode();
response.add(new AtlasEvaluatePolicyResponse(entities.get(i).getTypeName(), entities.get(i).getEntityGuid(), entities.get(i).getAction(), entities.get(i).getEntityId(), entities.get(i).getClassification(), false, errorCode));
}

} else if (RELATIONSHIP_ADD.name().equals(action) || RELATIONSHIP_REMOVE.name().equals(action) || RELATIONSHIP_UPDATE.name().equals(action)) {

if (entities.get(i).getRelationShipTypeName() == null) {
throw new AtlasBaseException(BAD_REQUEST, "RelationShip TypeName needed for " + action + " authorization");
}

try {
AtlasEntityHeader end1Entity = getAtlasEntityHeader(entities.get(i).getEntityGuidEnd1(), entities.get(i).getEntityIdEnd1(), entities.get(i).getEntityTypeEnd1());

AtlasEntityHeader end2Entity = getAtlasEntityHeader(entities.get(i).getEntityGuidEnd2(), entities.get(i).getEntityIdEnd2(), entities.get(i).getEntityTypeEnd2());

AtlasAuthorizationUtils.verifyAccess(new AtlasRelationshipAccessRequest(typeRegistry, AtlasPrivilege.valueOf(action), entities.get(i).getRelationShipTypeName(), end1Entity, end2Entity));
response.add(new AtlasEvaluatePolicyResponse(action, entities.get(i).getRelationShipTypeName(), entities.get(i).getEntityTypeEnd1(), entities.get(i).getEntityGuidEnd1(), entities.get(i).getEntityIdEnd1(), entities.get(i).getEntityTypeEnd2(), entities.get(i).getEntityGuidEnd2(), entities.get(i).getEntityIdEnd2(), true, null));
} catch (AtlasBaseException e) {
AtlasErrorCode code = e.getAtlasErrorCode();
String errorCode = code.getErrorCode();
response.add(new AtlasEvaluatePolicyResponse(action, entities.get(i).getRelationShipTypeName(), entities.get(i).getEntityTypeEnd1(), entities.get(i).getEntityGuidEnd1(), entities.get(i).getEntityIdEnd1(), entities.get(i).getEntityTypeEnd2(), entities.get(i).getEntityGuidEnd2(), entities.get(i).getEntityIdEnd2(), false, errorCode));
}
}

}
response = entitiesStore.evaluatePolicies(entities);
} finally {
AtlasPerfTracer.log(perf);
}
Expand Down Expand Up @@ -263,28 +196,6 @@ public List<AtlasAccessorResponse> getAccessors(List<AtlasAccessorRequest> atlas
return ret;
}

private AtlasEntityHeader getAtlasEntityHeader(String entityGuid, String entityId, String entityType) throws AtlasBaseException {
AtlasEntityHeader entityHeader = null;

if (StringUtils.isNotEmpty(entityGuid)) {
AtlasEntityWithExtInfo ret = entitiesStore.getByIdWithoutAuthorization(entityGuid);
entityHeader = new AtlasEntityHeader(ret.getEntity());
} else if (StringUtils.isNotEmpty(entityId) && StringUtils.isNotEmpty(entityType)) {
try {
entityHeader = entitiesStore.getAtlasEntityHeaderWithoutAuthorization(null, entityId, entityType);
} catch (AtlasBaseException abe) {
if (abe.getAtlasErrorCode() == AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND) {
Map<String, Object> attributes = new HashMap<>();
attributes.put(QUALIFIED_NAME, entityId);
entityHeader = new AtlasEntityHeader(entityType, attributes);
}
}
} else {
throw new AtlasBaseException(BAD_REQUEST, "requires entityGuid or typeName and qualifiedName for entity authorization");
}
return entityHeader;
}


/**
* Get entity header given its GUID.
Expand Down

0 comments on commit 396fb19

Please sign in to comment.