Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: atlanhq/dbt-action
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: v1-alpha
Choose a base ref
...
head repository: atlanhq/dbt-action
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: staging
Choose a head ref
Loading
Showing with 27,087 additions and 8,432 deletions.
  1. +2 −2 .github/workflows/test-action.yml
  2. +64 −0 adapters/api/create-resource.js
  3. +149 −0 adapters/api/get-asset.js
  4. +37 −0 adapters/api/get-classifications.js
  5. +127 −0 adapters/api/get-downstream-assets.js
  6. +2 −1 {src → adapters}/api/index.js
  7. +38 −0 adapters/api/segment.js
  8. BIN {src → adapters}/assets/atlan-logo.png
  9. BIN {src → adapters}/assets/certification_deprecated.png
  10. BIN {src → adapters}/assets/certification_drafted.png
  11. BIN {src → adapters}/assets/certification_verified.png
  12. BIN {src → adapters}/assets/connector_airflow.png
  13. BIN {src → adapters}/assets/connector_athena.png
  14. BIN {src → adapters}/assets/connector_aws-s3.png
  15. BIN {src → adapters}/assets/connector_azure-datalake.png
  16. BIN {src → adapters}/assets/connector_bigquery.png
  17. BIN {src → adapters}/assets/connector_databricks.png
  18. BIN {src → adapters}/assets/connector_dbt.png
  19. BIN {src → adapters}/assets/connector_gcp.png
  20. BIN {src → adapters}/assets/connector_glue.png
  21. BIN {src → adapters}/assets/connector_grafana.png
  22. BIN {src → adapters}/assets/connector_looker.png
  23. BIN {src → adapters}/assets/connector_mocks.png
  24. BIN {src → adapters}/assets/connector_mysql.png
  25. BIN {src → adapters}/assets/connector_oracle.png
  26. BIN {src → adapters}/assets/connector_postgres.png
  27. BIN {src → adapters}/assets/connector_powerbi.png
  28. BIN {src → adapters}/assets/connector_presto.png
  29. BIN {src → adapters}/assets/connector_python.png
  30. BIN {src → adapters}/assets/connector_r.png
  31. BIN {src → adapters}/assets/connector_redash.png
  32. BIN {src → adapters}/assets/connector_redshift.png
  33. BIN {src → adapters}/assets/connector_sisense.png
  34. BIN {src → adapters}/assets/connector_snowflake.png
  35. BIN {src → adapters}/assets/connector_tableau.png
  36. +10 −0 adapters/gateway.js
  37. +16 −0 adapters/index.js
  38. +54 −0 adapters/integrations/contract/contract.js
  39. +1,043 −0 adapters/integrations/github-integration.js
  40. +1,061 −0 adapters/integrations/gitlab-integration.js
  41. +65 −0 adapters/logger/logger.js
  42. +32 −0 adapters/templates/atlan.js
  43. +104 −0 adapters/templates/github-integration.js
  44. +104 −0 adapters/templates/gitlab-integration.js
  45. +24 −0 adapters/utils/auth.js
  46. +9 −0 adapters/utils/create-comment.js
  47. +84 −0 adapters/utils/get-environment-variables.js
  48. 0 {src → adapters}/utils/get-image-url.js
  49. 0 {src → adapters}/utils/hosted-images.js
  50. +12 −0 adapters/utils/index.js
  51. +23,729 −7,435 dist/index.js
  52. +313 −0 package-lock.json
  53. +8 −1 package.json
  54. +0 −62 src/api/create-resource.js
  55. +0 −114 src/api/get-asset.js
  56. +0 −34 src/api/get-classifications.js
  57. +0 −106 src/api/get-downstream-assets.js
  58. +0 −56 src/api/segment.js
  59. +0 −44 src/index.js
  60. +0 −2 src/main/index.js
  61. +0 −105 src/main/print-downstream-assets.js
  62. +0 −107 src/main/set-resource-on-asset.js
  63. +0 −55 src/utils/auth.js
  64. +0 −182 src/utils/create-comment.js
  65. +0 −82 src/utils/file-system.js
  66. +0 −21 src/utils/get-environment-variables.js
  67. +0 −23 src/utils/index.js
4 changes: 2 additions & 2 deletions .github/workflows/test-action.yml
Original file line number Diff line number Diff line change
@@ -2,7 +2,7 @@ name: Test Action

on:
pull_request:
types: [ opened, edited, synchronize, reopened, closed ]
types: [opened, edited, synchronize, reopened, closed]

jobs:
get-downstream-assets:
@@ -28,4 +28,4 @@ jobs:
main: DBT-DEMO-PROD
beta: Wide World Importers PE1
test-action: Wide World Importers PE1
IGNORE_MODEL_ALIAS_MATCHING: true
IGNORE_MODEL_ALIAS_MATCHING: true
64 changes: 64 additions & 0 deletions adapters/api/create-resource.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
import { v4 as uuidv4 } from "uuid";
import fetch from "node-fetch";
import stringify from "json-stringify-safe";
import {
ATLAN_INSTANCE_URL,
ATLAN_API_TOKEN,
} from "../utils/get-environment-variables.js";

export default async function createResource(
guid,
name,
link,
sendSegmentEventOfIntegration
) {
var myHeaders = {
Authorization: `Bearer ${ATLAN_API_TOKEN}`,
"Content-Type": "application/json",
};

var raw = stringify({
entities: [
{
typeName: "Link",
attributes: {
qualifiedName: uuidv4(),
name,
link,
tenantId: "default",
},
relationshipAttributes: {
asset: {
guid,
},
},
},
],
});

var requestOptions = {
method: "POST",
headers: myHeaders,
body: raw,
};

var response = await fetch(
`${ATLAN_INSTANCE_URL}/api/meta/entity/bulk`,
requestOptions
)
.then((e) => e.json())
.catch((err) => {
console.log(err);
sendSegmentEventOfIntegration({
action: "dbt_ci_action_failure",
properties: {
reason: "failed_to_create_resource",
asset_name: name, // This should change
msg: err,
},
});
});

if (response?.errorCode) return null;
return response;
}
149 changes: 149 additions & 0 deletions adapters/api/get-asset.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
import fetch from "node-fetch";
import stringify from "json-stringify-safe";
import {
getErrorModelNotFound,
getErrorDoesNotMaterialize,
} from "../templates/atlan.js";
import {
ATLAN_INSTANCE_URL,
ATLAN_API_TOKEN,
} from "../utils/get-environment-variables.js";

export default async function getAsset({
name,
sendSegmentEventOfIntegration,
environment,
integration,
}) {
var myHeaders = {
Authorization: `Bearer ${ATLAN_API_TOKEN}`,
"Content-Type": "application/json",
};

var raw = stringify({
dsl: {
from: 0,
size: 21,
query: {
bool: {
must: [
{
match: {
__state: "ACTIVE",
},
},
{
match: {
"__typeName.keyword": "DbtModel",
},
},
{
match: {
"name.keyword": name,
},
},
...(environment
? [
{
term: {
"assetDbtEnvironmentName.keyword": environment,
},
},
]
: []),
],
},
},
},
attributes: [
"name",
"description",
"userDescription",
"sourceURL",
"qualifiedName",
"connectorName",
"certificateStatus",
"certificateUpdatedBy",
"certificateUpdatedAt",
"ownerUsers",
"ownerGroups",
"classificationNames",
"meanings",
"dbtModelSqlAssets",
],
relationAttributes: [
"name",
"description",
"assetDbtProjectName",
"assetDbtEnvironmentName",
"connectorName",
"certificateStatus",
],
});

var requestOptions = {
method: "POST",
headers: myHeaders,
body: raw,
};

var response = await fetch(
`${ATLAN_INSTANCE_URL}/api/meta/search/indexsearch#findAssetByExactName`,
requestOptions
)
.then((e) => e.json())
.catch((err) => {
sendSegmentEventOfIntegration({
action: "dbt_ci_action_failure",
properties: {
reason: "failed_to_get_asset",
asset_name: name,
msg: err,
},
});
});

if (!response?.entities?.length) {
return {
error: getErrorModelNotFound(name),
};
}

if (Array.isArray(response.entities)) {
response.entities.sort((entityA, entityB) => {
const hasDbtModelSqlAssetsA =
entityA.attributes.dbtModelSqlAssets &&
entityA.attributes.dbtModelSqlAssets.length > 0;
const hasDbtModelSqlAssetsB =
entityB.attributes.dbtModelSqlAssets &&
entityB.attributes.dbtModelSqlAssets.length > 0;

if (hasDbtModelSqlAssetsA && !hasDbtModelSqlAssetsB) {
return -1; // entityA comes before entityB
} else if (!hasDbtModelSqlAssetsA && hasDbtModelSqlAssetsB) {
return 1; // entityB comes before entityA
}

// Primary sorting criterion: Latest createTime comes first
if (entityA.createTime > entityB.createTime) {
return -1;
} else if (entityA.createTime < entityB.createTime) {
return 1;
}

return 0; // No difference in sorting for these two entities
});
}

if (!response?.entities[0]?.attributes?.dbtModelSqlAssets?.length > 0)
return {
error: getErrorDoesNotMaterialize(
name,
ATLAN_INSTANCE_URL,
response,
integration
),
};

return response.entities[0];
}
37 changes: 37 additions & 0 deletions adapters/api/get-classifications.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
import fetch from "node-fetch";
import {
ATLAN_INSTANCE_URL,
ATLAN_API_TOKEN,
} from "../utils/get-environment-variables.js";

export default async function getClassifications({
sendSegmentEventOfIntegration,
}) {
var myHeaders = {
Authorization: `Bearer ${ATLAN_API_TOKEN}`,
"Content-Type": "application/json",
};

var requestOptions = {
method: "GET",
headers: myHeaders,
redirect: "follow",
};

var response = await fetch(
`${ATLAN_INSTANCE_URL}/api/meta/types/typedefs?type=classification`,
requestOptions
)
.then((e) => e.json())
.catch((err) => {
sendSegmentEventOfIntegration({
action: "dbt_ci_action_failure",
properties: {
reason: "failed_to_get_classifications",
msg: err,
},
});
});

return response?.classificationDefs;
}
127 changes: 127 additions & 0 deletions adapters/api/get-downstream-assets.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
import fetch from "node-fetch";
import {
getConnectorImage,
getCertificationImage,
getImageURL,
} from "../utils/index.js";
import stringify from "json-stringify-safe";
import {
ATLAN_INSTANCE_URL,
ATLAN_API_TOKEN,
} from "../utils/get-environment-variables.js";

const ASSETS_LIMIT = 100;

export default async function getDownstreamAssets(
asset,
guid,
totalModifiedFiles,
sendSegmentEventOfIntegration,
integration
) {
var myHeaders = {
authorization: `Bearer ${ATLAN_API_TOKEN}`,
"content-type": "application/json",
};

var raw = stringify({
guid: guid,
size: Math.max(Math.ceil(ASSETS_LIMIT / totalModifiedFiles), 1),
from: 0,
depth: 21,
direction: "OUTPUT",
entityFilters: {
condition: "AND",
criterion: [
{
attributeName: "__typeName",
operator: "not_contains",
attributeValue: "Process",
},
{
attributeName: "__state",
operator: "eq",
attributeValue: "ACTIVE",
},
],
},
attributes: [
"name",
"description",
"userDescription",
"sourceURL",
"qualifiedName",
"connectorName",
"certificateStatus",
"certificateUpdatedBy",
"certificateUpdatedAt",
"ownerUsers",
"ownerGroups",
"classificationNames",
"meanings",
],
excludeMeanings: false,
excludeClassifications: false,
});

var requestOptions = {
method: "POST",
headers: myHeaders,
body: raw,
};

var handleError = (err) => {
const comment = `### ${getConnectorImage(
asset.attributes.connectorName
)} [${asset.displayText}](${ATLAN_INSTANCE_URL}/assets/${
asset.guid
}/overview?utm_source=dbt_${integration}_action) ${
asset.attributes?.certificateStatus
? getCertificationImage(asset.attributes.certificateStatus)
: ""
}
_Failed to fetch impacted assets._
${getImageURL(
"atlan-logo",
15,
15
)} [View lineage in Atlan](${ATLAN_INSTANCE_URL}/assets/${
asset.guid
}/lineage/overview?utm_source=dbt_${integration}_action)`;

sendSegmentEventOfIntegration({
action: "dbt_ci_action_failure",
properties: {
reason: "failed_to_fetch_lineage",
asset_guid: asset.guid,
asset_name: asset.name,
asset_typeName: asset.typeName,
msg: err,
},
});

return comment;
};

var response = await fetch(
`${ATLAN_INSTANCE_URL}/api/meta/lineage/list`,
requestOptions
)
.then((e) => {
if (e.status === 200) {
return e.json();
} else {
throw e;
}
})
.catch((err) => {
return {
error: handleError(err),
};
});
if (response.error) return response;

return response;
}
Loading