Skip to content

Import landscape.cncf.io dataset, publish to Neo4j Aura #852

Import landscape.cncf.io dataset, publish to Neo4j Aura

Import landscape.cncf.io dataset, publish to Neo4j Aura #852

Workflow file for this run

name: Import landscape.cncf.io dataset, publish to Neo4j Aura
on:
schedule:
- cron: 0 0 * * *
workflow_dispatch: {}
push:
branches:
- main
paths:
- .github/workflows/flat.yml
- clean-landscape-json.ts
jobs:
scheduled:
runs-on: ubuntu-latest
steps:
- name: Setup deno
uses: denoland/setup-deno@main
with:
deno-version: v1.10.x
- name: Check out repo
uses: actions/checkout@v2
- name: Fetch data
uses: githubocto/flat@v3
with:
http_url: https://landscape.cncf.io/data/items.json
downloaded_filename: landscape-items.json
postprocess: ./clean-landscape-json.ts
- name: Import to Neo4j Aura
uses: johnymontana/[email protected]
with:
neo4j-user: ${{secrets.NEO4J_AURA_USER}}
neo4j-password: ${{secrets.NEO4J_AURA_PASSWORD}}
neo4j-uri: ${{secrets.NEO4J_AURA_URI}}
filename: 'landscape-items-clean.json'
cypher-query: >
UNWIND $value AS value
MERGE (c:Card {name: value.name})
SET c += value {
.bestPracticeBadgeId,
.bestPracticePercentage,
.commitsThisYear,
.contributorsCount,
.crunchbase,
.description,
.homepage_url,
.isSubsidiaryProject,
.logo,
.oss,
.repo_url,
.stars,
.twitter,
.license,
.member
}
WITH c, value
MERGE(org:Organization { name: value.organization })
MERGE(c)-[:IN_ORGANIZATION]->(org)
MERGE(p:Path {name: value.path})
MERGE(c)-[:IN_PATH]->(p)
MERGE(cat:Category {name: value.category})
MERGE(c)-[:IN_CATEGORY]->(cat)
MERGE(hq:Headquarters { name: value.headquarters })
MERGE(c)-[:BASED_IN]->(hq)
WITH c, value
UNWIND value.industries as industry
MERGE (i:Industry {name: industry})
MERGE (c)-[:IN_INDUSTRY]->(i)
WITH c, value
UNWIND value.github_data.languages as language
MERGE (l:Language {name: language.name})
MERGE (c)-[:USES_LANGUAGE]->(l)
WITH c, value
UNWIND value.repos as repo
MERGE (r:Repo {url: repo.url}) // TODO: should parse out an org/repo id for this
MERGE (c)-[:OWNS_REPO]->(r)
RETURN c.name, c.homepage_url, c.description
- name: Assign OSS Licenses to Projects
uses: johnymontana/[email protected]
with:
neo4j-user: ${{secrets.NEO4J_AURA_USER}}
neo4j-password: ${{secrets.NEO4J_AURA_PASSWORD}}
neo4j-uri: ${{secrets.NEO4J_AURA_URI}}
filename: 'landscape-items-clean.json'
cypher-query: >
MATCH(card:Card)
WHERE card.license IS NOT NULL
MERGE(proj:Project {name: card.name})
MERGE(ossl:License { name: card.license })
MERGE (c)-[:IS_PROJECT]->(proj)
MERGE (p)-[:HAS_LICENSE]->(ossl)
RETURN p.name, ossl.name
- name: Assign Membership Types to Members
uses: johnymontana/[email protected]
with:
neo4j-user: ${{secrets.NEO4J_AURA_USER}}
neo4j-password: ${{secrets.NEO4J_AURA_PASSWORD}}
neo4j-uri: ${{secrets.NEO4J_AURA_URI}}
filename: 'landscape-items-clean.json'
cypher-query: >
MATCH(c:Card)
WHERE c.member IS NOT NULL
MERGE(m:Membership {name: c.member})
MERGE(c)-[:IS_MEMBERTYPE]->(m)
RETURN c.name, m.name