Merge pull request #1 from cncf/my-docker

local docker for iterative dev
cncf · Apr 25, 2022 · c1ad9bd · c1ad9bd
2 parents d561c6b + fd0e89d
commit c1ad9bd
Show file tree

Hide file tree

Showing 2 changed files with 310 additions and 0 deletions.
diff --git a/apps/local-docker/load-clean-landscape.cypher b/apps/local-docker/load-clean-landscape.cypher
@@ -0,0 +1,154 @@
+CREATE CONSTRAINT IF NOT EXISTS FOR (n:Card) REQUIRE n.name IS UNIQUE;
+CREATE CONSTRAINT IF NOT EXISTS FOR (n:Category) REQUIRE n.name IS UNIQUE;
+CREATE CONSTRAINT IF NOT EXISTS FOR (n:Headquarters) REQUIRE n.name IS UNIQUE;
+CREATE CONSTRAINT IF NOT EXISTS FOR (n:Industry) REQUIRE n.name IS UNIQUE;
+CREATE CONSTRAINT IF NOT EXISTS FOR (n:Language) REQUIRE n.name IS UNIQUE;
+CREATE CONSTRAINT IF NOT EXISTS FOR (n:License) REQUIRE n.name IS UNIQUE;
+CREATE CONSTRAINT IF NOT EXISTS FOR (n:Org) REQUIRE n.name IS UNIQUE;
+CREATE CONSTRAINT IF NOT EXISTS FOR (n:Path) REQUIRE n.name IS UNIQUE;
+CREATE CONSTRAINT IF NOT EXISTS FOR (n:Project) REQUIRE n.name IS UNIQUE;
+CREATE CONSTRAINT IF NOT EXISTS FOR (n:Relation) REQUIRE n.relation IS UNIQUE;
+CREATE CONSTRAINT IF NOT EXISTS FOR (n:Repo) REQUIRE n.name IS UNIQUE;
+
+CREATE INDEX IF NOT EXISTS FOR (n:Card) ON (n.name);
+CREATE INDEX IF NOT EXISTS FOR (n:Card) ON (n.license);
+CREATE INDEX IF NOT EXISTS FOR (n:Card) ON (n.member);
+CREATE INDEX IF NOT EXISTS FOR (n:Category) ON (n.name);
+CREATE INDEX IF NOT EXISTS FOR (n:Headquarters) ON (n.name);
+CREATE INDEX IF NOT EXISTS FOR (n:Industry) ON (n.name);
+CREATE INDEX IF NOT EXISTS FOR (n:Language) ON (n.name);
+CREATE INDEX IF NOT EXISTS FOR (n:License) ON (n.name);
+CREATE INDEX IF NOT EXISTS FOR (n:Org) ON (n.name);
+CREATE INDEX IF NOT EXISTS FOR (n:Path) ON (n.name);
+CREATE INDEX IF NOT EXISTS FOR (n:Project) ON (n.name);
+CREATE INDEX IF NOT EXISTS FOR (n:Relation) ON (n.name);
+CREATE INDEX IF NOT EXISTS FOR (n:Repo) ON (n.name);
+
+// 
+
+
+// load the json, which is a simple array, one entry per card.  
+// "YIELD" is the iterator pattern.
+CALL apoc.load.json("file:///landscape-items-clean.json")
+YIELD value
+
+// //WITH apoc.map.clean(value, [], [""]) as cleanedItems
+// WITH $keysToKeepInCards + $keysBecomingNodes as keysToKeep 
+// WITH apoc.map.fromLists($keysToKeep, apoc.map.values(value, $keysToKeep)) as allTheCards
+// RETURN allTheCards
+
+MERGE (c:Card {name: value.name})
+SET c += value { 
+    .bestPracticeBadgeId, 
+    .bestPracticePercentage, 
+    .commitsThisYear, 
+    .contributorsCount, 
+    .crunchbase, 
+    .description, 
+    .homepage_url, 
+    .isSubsidiaryProject, 
+    .logo,
+    .oss,
+    .repo_url, 
+    .stars, 
+    .twitter,
+    .license,
+    .member
+}  
+WITH c, value
+
+MERGE(org:Organization { name: value.organization })
+MERGE(c)-[:IN_ORGANIZATION]->(org)
+
+MERGE(p:Path {name: value.path})
+MERGE(c)-[:IN_PATH]->(p)
+
+MERGE(cat:Category {name: value.category})
+MERGE(c)-[:IN_CATEGORY]->(cat)
+
+MERGE(hq:Headquarters { name: value.headquarters })
+MERGE(c)-[:BASED_IN]->(hq)
+
+WITH c, value
+UNWIND value.industries as industry
+MERGE (i:Industry {name: industry})
+MERGE (c)-[:IN_INDUSTRY]->(i)
+
+WITH c, value
+UNWIND value.github_data.languages as language
+MERGE (l:Language {name: language.name})
+MERGE (c)-[:USES_LANGUAGE]->(l)
+
+WITH c, value
+UNWIND value.repos as repo
+MERGE (r:Repo {url: repo.url}) // TODO: should parse out an org/repo id for this
+MERGE (c)-[:OWNS_REPO]->(r)
+
+;
+
+MATCH(card:Card)
+WHERE card.license IS NOT NULL
+MERGE(proj:Project {name: card.name})
+MERGE(ossl:License { name: card.license })
+MERGE (c)-[:IS_PROJECT]->(proj)
+MERGE (p)-[:HAS_LICENSE]->(ossl)
+
+;
+
+MATCH(c:Card)
+WHERE c.member IS NOT NULL
+MERGE(m:Membership {name: c.member})
+MERGE(c)-[:IS_MEMBERTYPE]->(m)
+
+return count(*)
+
+
+////////////////////////////////
+
+// these become properties to keep in Card nodes.
+// :param keysToKeepInCards => ['bestPracticeBadgeId',
+//                              'bestPracticePercentage',
+//                              'commitsThisYear',
+//                              'contributorsCount',
+//                              'crunchbase',
+//                              'description',
+//                              'homepage_url',
+//                              'isSubsidiaryProject',
+//                              'logo',
+//                              'oss',
+//                              'repo_url',
+//                              'stars',
+//                              'twitter'];
+
+// :param keysBecomingNodes => ['organization',
+//                              'path',
+//                              'category',
+//                              'headquarters',
+//                              'license',
+//                              'member']
+// CALL apoc.load.json("file:///landscape-items-clean.json")
+// YIELD value
+// WITH ["bestPracticeBadgeId",
+//       "bestPracticePercentage",
+//       "commitsThisYear",
+//       "contributorsCount",
+//       "crunchbase",
+//       "description",
+//       "homepage_url",
+//       "isSubsidiaryProject",
+//       "logo",
+//       "oss",
+//       "repo_url",
+//       "stars",
+//       "twitter"] as keysToKeepInCards, 
+//      ["organization",
+//       "path",
+//       "category",
+//       "headquarters",
+//       "license",
+//       "member"] as keysBecomingNodes, value
+// call apoc.merge.node("Card", {name :coalesce(value.name), apoc.create.uuid())},       
+// WITH apoc.map.clean(keysToKeepInCards + keysBecomingNodes, [], [""]) as keysToKeep, value
+// WITH apoc.map.fromLists(keysToKeep, apoc.map.values(value, keysToKeep)) as allTheCards
+// RETURN allTheCards
+
diff --git a/apps/local-docker/local-docker-neo.sh b/apps/local-docker/local-docker-neo.sh
@@ -0,0 +1,156 @@
+#!/usr/bin/env bash
+
+# provided as a handy way to quickly iterate using only docker.
+
+function docker-ps-pretty() 
+{ 
+    local DOCKER_PS_PRETTY="table {{.ID}}\t{{.Names}}\t{{.State}}\t{{.Status}}\t{{.RunningFor}}\t{{.Image}}"
+    echo; 
+    docker ps --all --format "$DOCKER_PS_PRETTY"
+}
+
+function tree-pretty()
+{
+    tree -apugshDC $1
+}
+
+function warn-user()
+{
+    echo "*** Press ENTER when ready or ctrl-C to halt. $1"
+    read
+}
+
+
+# arg1 --> container name
+function docker-stop-rm-if-exists()
+{
+    local CONTAINER="$1"
+    echo "*** docker: stopping and removing $CONTAINER"
+    docker ps -q --all --filter "name=$CONTAINER" | grep -q . && docker stop $CONTAINER && docker rm -fv $CONTAINER
+}
+
+###
+### Begin Festivities
+###
+
+#set -x
+
+#
+# Variables
+#
+LANDSCAPE_ROOT=$HOME/landscape
+LANDSCAPE_BASE=$LANDSCAPE_ROOT/neo4j
+CONTAINER_NAME=landscape-graph
+NEO4J_USERNAME=neo4j
+NEO4J_PASSWORD=landscape
+
+echo "*** $CONTAINER_NAME: Using $LANDSCAPE_BASE for docker mount points (/data, /logs, /import, /plugins)"
+echo ""
+warn-user "THIS WILL OBLITERATE ANYTHING EXISTING IN $LANDSCAPE_BASE + stop/rm docker container"
+
+docker-ps-pretty
+docker-stop-rm-if-exists "$CONTAINER_NAME"
+docker-ps-pretty
+
+# nuke anything left in mounted dirs
+sudo chown $USER:$USER -R ~/landscape/neo4j
+rm -rf $LANDSCAPE_BASE
+tree-pretty $LANDSCAPE_ROOT
+
+# create mount points
+mkdir -p $LANDSCAPE_BASE/data
+mkdir -p $LANDSCAPE_BASE/logs
+mkdir -p $LANDSCAPE_BASE/import
+mkdir -p $LANDSCAPE_BASE/plugins
+
+# copy cleaned up landscape json --> import directory
+export LANDSCAPE_GIT_REPO_ROOT=$(git rev-parse --show-toplevel)
+cp -fv $LANDSCAPE_GIT_REPO_ROOT/landscape-items-clean.json $LANDSCAPE_BASE/import
+tree-pretty "$LANDSCAPE_BASE"
+
+echo "presently running in docker:"
+docker-ps-pretty
+
+warn-user "...last chance beforoe docker run ... neo4j"
+echo "*** LAUNCHING NEO4J @ $LANDSCAPE_BASE"
+
+######################################################################
+# handy extras
+#     --env NEO4J_AUTH=none \
+# --env NEO4JLABS_PLUGINS='["apoc", "graph-data-science", "bloom"]'
+#    --user="$(id -u):$(id -g)" \
+######################################################################
+
+# run neo!
+docker run \
+    --name $CONTAINER_NAME \
+    -p7474:7474 -p7687:7687 \
+    -d \
+    -v $LANDSCAPE_BASE/data:/data \
+    -v $LANDSCAPE_BASE/logs:/logs \
+    -v $LANDSCAPE_BASE/import:/var/lib/neo4j/import \
+    -v $LANDSCAPE_BASE/plugins:/plugins \
+    --env NEO4J_AUTH=$NEO4J_USERNAME/$NEO4J_PASSWORD \
+    --env NEO4J_apoc_import_file=enabled \
+    --env NEO4JLABS_PLUGINS='["apoc"]' \
+    --env NEO4J_dbms_logs_query_enabled=false \
+    --env NEO4J_ACCEPT_LICENSE_AGREEMENT=yes \
+    --env NEO4J_apoc_import_file_enabled=true \
+    --env NEO4J_apoc_export_file_enabled=true \
+    --env NEO4J_dbms_security_procedures_unrestricted=apoc.\\\*,gds.\\\* \
+    --env NEO4J_dbms_logs_debug_level=INFO \
+    neo4j:latest
+
+docker-ps-pretty
+
+# launch Cypher Shell in container, execute cypher
+until echo 'match (n) return count (n);' | docker exec --interactive landscape-graph cypher-shell -u $NEO4J_USERNAME -p $NEO4J_PASSWORD; \
+do \
+    echo "waiting 1s for bolt to appear"; \
+    sleep 1; \
+done
+echo "*** neo4j online!"
+
+# launch Cypher Shell in container, execute cypher
+cat load-clean-landscape.cypher \
+| docker exec --interactive landscape-graph cypher-shell -u $NEO4J_USERNAME -p $NEO4J_PASSWORD
+
+echo ""
+echo "open http://localhost:7474 ($NEO4J_USERNAME/$NEO4J_PASSWORD)"
+echo ""
+
+#
+# TODO Nuke this
+#
+# docker run \
+#   --user $(id -u):$(id -g) \
+#   --name "${CONTAINER}" \
+#   --detach \
+#   --restart unless-stopped \
+#   --publish ${NEO_HTTPS}:${NEO_HTTPS} \
+#   --publish ${NEO_HTTP}:${NEO_HTTP} \
+#   --publish ${NEO_BOLT}:${NEO_BOLT} \
+#   --env NEO4JLABS_PLUGINS="${NEO_PLUGINS}" \
+#   --env NEO4J_dbms_connector_https_listen__address=0.0.0.0:${NEO_HTTPS} \
+#   --env NEO4J_dbms_connector_http_listen__address=0.0.0.0:${NEO_HTTP} \
+#   --env NEO4J_dbms_connector_bolt_listen__address=0.0.0.0:${NEO_BOLT} \
+#   --env NEO4J_dbms_connector_https_advertised__address=${NEO_HOST}:${NEO_HTTPS} \
+#   --env NEO4J_dbms_connector_http_advertised__address=${NEO_HOST}:${NEO_HTTP} \
+#   --env NEO4J_dbms_connector_bolt_advertised__address=${NEO_HOST}:${NEO_BOLT} \
+#   --volume ${NEO_GRAPH_DIR}/data:/data \
+#   --volume ${NEO_GRAPH_DIR}/import:/import \
+#   --volume ${NEO_GRAPH_DIR}/plugins:/plugins \
+#   --volume ${NEO_GRAPH_DIR}/conf:/var/lib/neo4j/conf \
+#   --volume ${NEO_GRAPH_DIR}/logs:/logs \
+#   --env NEO4J_dbms_logs_query_enabled=false \
+#   --env NEO4J_dbms_connector_bolt_advertised__address=${NEO_HOST}:${NEO_BOLT} \
+#   --env NEO4J_dbms_active__database="${CONTAINER}" \
+#   --env NEO4J_ACCEPT_LICENSE_AGREEMENT=yes \
+#   --env NEO4J_apoc_import_file_enabled=true \
+#   --env NEO4J_apoc_export_file_enabled=true \
+#   --env NEO4J_dbms_security_procedures_unrestricted=apoc.\\\*,gds.\\\* \
+#   --env NEO4J_browser_remote__content__hostname__whitelist="${NEO_WHITELIST}" \
+#   --env NEO4J_browser_post__connect__cmd="${NEO_PLAY}" \
+#   --env NEO4J_dbms_memory_heap_initial__size=${NEO4J_dbms_memory_heap_initial__size} \
+#   --env NEO4J_dbms_memory_heap_max__size=${NEO4J_dbms_memory_heap_max__size} \
+#   --env NEO4J_dbms_memory_pagecache_size=${NEO4J_dbms_memory_pagecache_size} \