From 1cd327791715a33eda182a72e883057e3762d321 Mon Sep 17 00:00:00 2001
From: Matt Young <halcyondude@gmail.com>
Date: Mon, 18 Apr 2022 03:03:43 -0400
Subject: [PATCH 1/2] local docker for iterative dev

Close #45
---
 apps/local-docker/compose-landscape-graph.yml |  33 ++++
 apps/local-docker/load-clean-landscape.cypher | 154 ++++++++++++++++++
 apps/local-docker/local-docker-neo.sh         | 144 ++++++++++++++++
 3 files changed, 331 insertions(+)
 create mode 100644 apps/local-docker/compose-landscape-graph.yml
 create mode 100644 apps/local-docker/load-clean-landscape.cypher
 create mode 100755 apps/local-docker/local-docker-neo.sh

diff --git a/apps/local-docker/compose-landscape-graph.yml b/apps/local-docker/compose-landscape-graph.yml
new file mode 100644
index 0000000..8570cc9
--- /dev/null
+++ b/apps/local-docker/compose-landscape-graph.yml
@@ -0,0 +1,33 @@
+# docker-compose.yml
+version: "3.9"
+networks:
+  neo4j_user:
+services:
+  neo4j:
+    image: neo4j:latest
+    ports:
+      - 7474:7474 # http://0.0.0.0:7474/
+      - 7687:7687 # bolt
+    expose:
+      - 7474
+      - 7687
+    volumes:
+      - neo4j_data:/data
+    networks:
+      - neo4j_user
+    healthcheck:
+        test: wget http://localhost:7474 || exit 1
+        interval: 1s
+        timeout: 10s
+        retries: 20
+        start_period: 3s
+  app:
+    build: .
+    depends_on:
+      neo4j:
+        condition: service_healthy
+    networks:
+      - neo4j_user
+
+volumes:
+  neo4j_data:
\ No newline at end of file
diff --git a/apps/local-docker/load-clean-landscape.cypher b/apps/local-docker/load-clean-landscape.cypher
new file mode 100644
index 0000000..095776c
--- /dev/null
+++ b/apps/local-docker/load-clean-landscape.cypher
@@ -0,0 +1,154 @@
+CREATE CONSTRAINT IF NOT EXISTS FOR (n:Card) REQUIRE n.name IS UNIQUE;
+CREATE CONSTRAINT IF NOT EXISTS FOR (n:Category) REQUIRE n.name IS UNIQUE;
+CREATE CONSTRAINT IF NOT EXISTS FOR (n:Headquarters) REQUIRE n.name IS UNIQUE;
+CREATE CONSTRAINT IF NOT EXISTS FOR (n:Industry) REQUIRE n.name IS UNIQUE;
+CREATE CONSTRAINT IF NOT EXISTS FOR (n:Language) REQUIRE n.name IS UNIQUE;
+CREATE CONSTRAINT IF NOT EXISTS FOR (n:License) REQUIRE n.name IS UNIQUE;
+CREATE CONSTRAINT IF NOT EXISTS FOR (n:Org) REQUIRE n.name IS UNIQUE;
+CREATE CONSTRAINT IF NOT EXISTS FOR (n:Path) REQUIRE n.name IS UNIQUE;
+CREATE CONSTRAINT IF NOT EXISTS FOR (n:Project) REQUIRE n.name IS UNIQUE;
+CREATE CONSTRAINT IF NOT EXISTS FOR (n:Relation) REQUIRE n.relation IS UNIQUE;
+CREATE CONSTRAINT IF NOT EXISTS FOR (n:Repo) REQUIRE n.name IS UNIQUE;
+
+CREATE INDEX IF NOT EXISTS FOR (n:Card) ON (n.name);
+CREATE INDEX IF NOT EXISTS FOR (n:Card) ON (n.license);
+CREATE INDEX IF NOT EXISTS FOR (n:Card) ON (n.member);
+CREATE INDEX IF NOT EXISTS FOR (n:Category) ON (n.name);
+CREATE INDEX IF NOT EXISTS FOR (n:Headquarters) ON (n.name);
+CREATE INDEX IF NOT EXISTS FOR (n:Industry) ON (n.name);
+CREATE INDEX IF NOT EXISTS FOR (n:Language) ON (n.name);
+CREATE INDEX IF NOT EXISTS FOR (n:License) ON (n.name);
+CREATE INDEX IF NOT EXISTS FOR (n:Org) ON (n.name);
+CREATE INDEX IF NOT EXISTS FOR (n:Path) ON (n.name);
+CREATE INDEX IF NOT EXISTS FOR (n:Project) ON (n.name);
+CREATE INDEX IF NOT EXISTS FOR (n:Relation) ON (n.name);
+CREATE INDEX IF NOT EXISTS FOR (n:Repo) ON (n.name);
+
+// 
+
+
+// load the json, which is a simple array, one entry per card.  
+// "YIELD" is the iterator pattern.
+CALL apoc.load.json("file:///landscape-items-clean.json")
+YIELD value
+
+// //WITH apoc.map.clean(value, [], [""]) as cleanedItems
+// WITH $keysToKeepInCards + $keysBecomingNodes as keysToKeep 
+// WITH apoc.map.fromLists($keysToKeep, apoc.map.values(value, $keysToKeep)) as allTheCards
+// RETURN allTheCards
+
+MERGE (c:Card {name: value.name})
+SET c += value { 
+    .bestPracticeBadgeId, 
+    .bestPracticePercentage, 
+    .commitsThisYear, 
+    .contributorsCount, 
+    .crunchbase, 
+    .description, 
+    .homepage_url, 
+    .isSubsidiaryProject, 
+    .logo,
+    .oss,
+    .repo_url, 
+    .stars, 
+    .twitter,
+    .license,
+    .member
+}  
+WITH c, value
+
+MERGE(org:Organization { name: value.organization })
+MERGE(c)-[:IN_ORGANIZATION]->(org)
+
+MERGE(p:Path {name: value.path})
+MERGE(c)-[:IN_PATH]->(p)
+
+MERGE(cat:Category {name: value.category})
+MERGE(c)-[:IN_CATEGORY]->(cat)
+
+MERGE(hq:Headquarters { name: value.headquarters })
+MERGE(c)-[:BASED_IN]->(hq)
+
+WITH c, value
+UNWIND value.industries as industry
+MERGE (i:Industry {name: industry})
+MERGE (c)-[:IN_INDUSTRY]->(i)
+
+WITH c, value
+UNWIND value.github_data.languages as language
+MERGE (l:Language {name: language.name})
+MERGE (c)-[:USES_LANGUAGE]->(l)
+
+WITH c, value
+UNWIND value.repos as repo
+MERGE (r:Repo {url: repo.url}) // TODO: should parse out an org/repo id for this
+MERGE (c)-[:OWNS_REPO]->(r)
+
+;
+
+MATCH(card:Card)
+WHERE card.license IS NOT NULL
+MERGE(proj:Project {name: card.name})
+MERGE(ossl:License { name: card.license })
+MERGE (c)-[:IS_PROJECT]->(proj)
+MERGE (p)-[:HAS_LICENSE]->(ossl)
+
+;
+
+MATCH(c:Card)
+WHERE c.member IS NOT NULL
+MERGE(m:Membership {name: c.member})
+MERGE(c)-[:IS_MEMBERTYPE]->(m)
+
+return count(*)
+
+
+////////////////////////////////
+
+// these become properties to keep in Card nodes.
+// :param keysToKeepInCards => ['bestPracticeBadgeId',
+//                              'bestPracticePercentage',
+//                              'commitsThisYear',
+//                              'contributorsCount',
+//                              'crunchbase',
+//                              'description',
+//                              'homepage_url',
+//                              'isSubsidiaryProject',
+//                              'logo',
+//                              'oss',
+//                              'repo_url',
+//                              'stars',
+//                              'twitter'];
+
+// :param keysBecomingNodes => ['organization',
+//                              'path',
+//                              'category',
+//                              'headquarters',
+//                              'license',
+//                              'member']
+// CALL apoc.load.json("file:///landscape-items-clean.json")
+// YIELD value
+// WITH ["bestPracticeBadgeId",
+//       "bestPracticePercentage",
+//       "commitsThisYear",
+//       "contributorsCount",
+//       "crunchbase",
+//       "description",
+//       "homepage_url",
+//       "isSubsidiaryProject",
+//       "logo",
+//       "oss",
+//       "repo_url",
+//       "stars",
+//       "twitter"] as keysToKeepInCards, 
+//      ["organization",
+//       "path",
+//       "category",
+//       "headquarters",
+//       "license",
+//       "member"] as keysBecomingNodes, value
+// call apoc.merge.node("Card", {name :coalesce(value.name), apoc.create.uuid())},       
+// WITH apoc.map.clean(keysToKeepInCards + keysBecomingNodes, [], [""]) as keysToKeep, value
+// WITH apoc.map.fromLists(keysToKeep, apoc.map.values(value, keysToKeep)) as allTheCards
+// RETURN allTheCards
+
diff --git a/apps/local-docker/local-docker-neo.sh b/apps/local-docker/local-docker-neo.sh
new file mode 100755
index 0000000..0743e53
--- /dev/null
+++ b/apps/local-docker/local-docker-neo.sh
@@ -0,0 +1,144 @@
+#!/usr/bin/env bash
+
+# provided as a handy way to quickly iterate using only docker.
+
+function docker-ps-pretty() 
+{ 
+    local DOCKER_PS_PRETTY="table {{.ID}}\t{{.Names}}\t{{.State}}\t{{.Status}}\t{{.RunningFor}}\t{{.Image}}"
+    echo; 
+    docker ps --all --format "$DOCKER_PS_PRETTY"
+}
+
+function tree-pretty()
+{
+    tree -apugshDC $1
+}
+
+function warn-user()
+{
+    echo "*** Press ENTER when ready or ctrl-C to halt. $1"
+    read
+}
+
+
+# arg1 --> container name
+function docker-stop-rm-if-exists()
+{
+    local CONTAINER="$1"
+    echo "*** docker: stopping and removing $CONTAINER"
+    docker ps -q --all --filter "name=$CONTAINER" | grep -q . && docker stop $CONTAINER && docker rm -fv $CONTAINER
+}
+
+###
+### Begin Festivities
+###
+
+#set -x
+
+#
+# Variables
+#
+LANDSCAPE_BASE=$HOME/landscape/neo4j
+CONTAINER_NAME=landscape-graph
+NEO4J_USERNAME=neo4j
+NEO4J_PASSWORD=landscape
+
+echo "*** $CONTAINER_NAME: Using $LANDSCAPE_BASE for docker mount points (/data, /logs, /import, /plugins)"
+echo ""
+warn-user "THIS WILL OBLITERATE ANYTHING EXISTING IN $LANDSCAPE_BASE + stop/rm docker container"
+
+docker-ps-pretty
+docker-stop-rm-if-exists "$CONTAINER_NAME"
+docker-ps-pretty
+
+# nuke anything left in mounted dirs
+rm -rf $LANDSCAPE_BASE
+
+# create mount points
+mkdir -p $LANDSCAPE_BASE/data
+mkdir -p $LANDSCAPE_BASE/logs
+mkdir -p $LANDSCAPE_BASE/import
+mkdir -p $LANDSCAPE_BASE/plugins
+
+# copy cleaned up landscape json --> import directory
+cp -fv ../../landscape-items-clean.json $LANDSCAPE_BASE/import
+tree-pretty "$LANDSCAPE_BASE"
+
+echo "presently running in docker:"
+docker-ps-pretty
+
+warn-user "...last chance beforoe docker run ... neo4j"
+echo "*** LAUNCHING NEO4J @ $LANDSCAPE_BASE"
+
+######################################################################
+# handy extras
+#     --env NEO4J_AUTH=none \
+# --env NEO4JLABS_PLUGINS='["apoc", "graph-data-science", "bloom"]'
+#    --user="$(id -u):$(id -g)" \
+######################################################################
+
+# run neo!
+docker run \
+    --name $CONTAINER_NAME \
+    -p7474:7474 -p7687:7687 \
+    -d \
+    -v $LANDSCAPE_BASE/data:/data \
+    -v $LANDSCAPE_BASE/logs:/logs \
+    -v $LANDSCAPE_BASE/import:/var/lib/neo4j/import \
+    -v $LANDSCAPE_BASE/plugins:/plugins \
+    --env NEO4J_AUTH=$NEO4J_USERNAME/$NEO4J_PASSWORD \
+    --env NEO4J_apoc_import_file=enabled \
+    --env NEO4JLABS_PLUGINS='["apoc"]' \
+    --env NEO4J_dbms_logs_query_enabled=false \
+    --env NEO4J_ACCEPT_LICENSE_AGREEMENT=yes \
+    --env NEO4J_apoc_import_file_enabled=true \
+    --env NEO4J_apoc_export_file_enabled=true \
+    --env NEO4J_dbms_security_procedures_unrestricted=apoc.\\\*,gds.\\\* \
+    neo4j:latest
+
+docker-ps-pretty
+
+# launch Cypher Shell in container, execute cypher
+until echo 'match (n) return count (n);' | docker exec --interactive landscape-graph cypher-shell -u $NEO4J_USERNAME -p $NEO4J_PASSWORD; \
+do \
+    echo "waiting 1s for bolt to appear"; \
+    sleep 1; \
+done
+echo '*** neo4j online!'
+
+# launch Cypher Shell in container, execute cypher
+cat load-clean-landscape.cypher \
+| docker exec --interactive landscape-graph cypher-shell -u $NEO4J_USERNAME -p $NEO4J_PASSWORD
+
+# docker run \
+#   --user $(id -u):$(id -g) \
+#   --name "${CONTAINER}" \
+#   --detach \
+#   --restart unless-stopped \
+#   --publish ${NEO_HTTPS}:${NEO_HTTPS} \
+#   --publish ${NEO_HTTP}:${NEO_HTTP} \
+#   --publish ${NEO_BOLT}:${NEO_BOLT} \
+#   --env NEO4JLABS_PLUGINS="${NEO_PLUGINS}" \
+#   --env NEO4J_dbms_connector_https_listen__address=0.0.0.0:${NEO_HTTPS} \
+#   --env NEO4J_dbms_connector_http_listen__address=0.0.0.0:${NEO_HTTP} \
+#   --env NEO4J_dbms_connector_bolt_listen__address=0.0.0.0:${NEO_BOLT} \
+#   --env NEO4J_dbms_connector_https_advertised__address=${NEO_HOST}:${NEO_HTTPS} \
+#   --env NEO4J_dbms_connector_http_advertised__address=${NEO_HOST}:${NEO_HTTP} \
+#   --env NEO4J_dbms_connector_bolt_advertised__address=${NEO_HOST}:${NEO_BOLT} \
+#   --volume ${NEO_GRAPH_DIR}/data:/data \
+#   --volume ${NEO_GRAPH_DIR}/import:/import \
+#   --volume ${NEO_GRAPH_DIR}/plugins:/plugins \
+#   --volume ${NEO_GRAPH_DIR}/conf:/var/lib/neo4j/conf \
+#   --volume ${NEO_GRAPH_DIR}/logs:/logs \
+#   --env NEO4J_dbms_logs_query_enabled=false \
+#   --env NEO4J_dbms_connector_bolt_advertised__address=${NEO_HOST}:${NEO_BOLT} \
+#   --env NEO4J_dbms_active__database="${CONTAINER}" \
+#   --env NEO4J_ACCEPT_LICENSE_AGREEMENT=yes \
+#   --env NEO4J_apoc_import_file_enabled=true \
+#   --env NEO4J_apoc_export_file_enabled=true \
+#   --env NEO4J_dbms_security_procedures_unrestricted=apoc.\\\*,gds.\\\* \
+#   --env NEO4J_browser_remote__content__hostname__whitelist="${NEO_WHITELIST}" \
+#   --env NEO4J_browser_post__connect__cmd="${NEO_PLAY}" \
+#   --env NEO4J_dbms_memory_heap_initial__size=${NEO4J_dbms_memory_heap_initial__size} \
+#   --env NEO4J_dbms_memory_heap_max__size=${NEO4J_dbms_memory_heap_max__size} \
+#   --env NEO4J_dbms_memory_pagecache_size=${NEO4J_dbms_memory_pagecache_size} \
\ No newline at end of file

From fd0e89dd929103a1407821556ad3b6d5e4ba4d8f Mon Sep 17 00:00:00 2001
From: Matt Young <halcyondude@gmail.com>
Date: Mon, 25 Apr 2022 12:51:59 -0400
Subject: [PATCH 2/2] script nit, don't rely on relative path

---
 apps/local-docker/compose-landscape-graph.yml | 33 -------------------
 apps/local-docker/local-docker-neo.sh         | 18 ++++++++--
 2 files changed, 15 insertions(+), 36 deletions(-)
 delete mode 100644 apps/local-docker/compose-landscape-graph.yml

diff --git a/apps/local-docker/compose-landscape-graph.yml b/apps/local-docker/compose-landscape-graph.yml
deleted file mode 100644
index 8570cc9..0000000
--- a/apps/local-docker/compose-landscape-graph.yml
+++ /dev/null
@@ -1,33 +0,0 @@
-# docker-compose.yml
-version: "3.9"
-networks:
-  neo4j_user:
-services:
-  neo4j:
-    image: neo4j:latest
-    ports:
-      - 7474:7474 # http://0.0.0.0:7474/
-      - 7687:7687 # bolt
-    expose:
-      - 7474
-      - 7687
-    volumes:
-      - neo4j_data:/data
-    networks:
-      - neo4j_user
-    healthcheck:
-        test: wget http://localhost:7474 || exit 1
-        interval: 1s
-        timeout: 10s
-        retries: 20
-        start_period: 3s
-  app:
-    build: .
-    depends_on:
-      neo4j:
-        condition: service_healthy
-    networks:
-      - neo4j_user
-
-volumes:
-  neo4j_data:
\ No newline at end of file
diff --git a/apps/local-docker/local-docker-neo.sh b/apps/local-docker/local-docker-neo.sh
index 0743e53..7e5978a 100755
--- a/apps/local-docker/local-docker-neo.sh
+++ b/apps/local-docker/local-docker-neo.sh
@@ -38,7 +38,8 @@ function docker-stop-rm-if-exists()
 #
 # Variables
 #
-LANDSCAPE_BASE=$HOME/landscape/neo4j
+LANDSCAPE_ROOT=$HOME/landscape
+LANDSCAPE_BASE=$LANDSCAPE_ROOT/neo4j
 CONTAINER_NAME=landscape-graph
 NEO4J_USERNAME=neo4j
 NEO4J_PASSWORD=landscape
@@ -52,7 +53,9 @@ docker-stop-rm-if-exists "$CONTAINER_NAME"
 docker-ps-pretty
 
 # nuke anything left in mounted dirs
+sudo chown $USER:$USER -R ~/landscape/neo4j
 rm -rf $LANDSCAPE_BASE
+tree-pretty $LANDSCAPE_ROOT
 
 # create mount points
 mkdir -p $LANDSCAPE_BASE/data
@@ -61,7 +64,8 @@ mkdir -p $LANDSCAPE_BASE/import
 mkdir -p $LANDSCAPE_BASE/plugins
 
 # copy cleaned up landscape json --> import directory
-cp -fv ../../landscape-items-clean.json $LANDSCAPE_BASE/import
+export LANDSCAPE_GIT_REPO_ROOT=$(git rev-parse --show-toplevel)
+cp -fv $LANDSCAPE_GIT_REPO_ROOT/landscape-items-clean.json $LANDSCAPE_BASE/import
 tree-pretty "$LANDSCAPE_BASE"
 
 echo "presently running in docker:"
@@ -94,6 +98,7 @@ docker run \
     --env NEO4J_apoc_import_file_enabled=true \
     --env NEO4J_apoc_export_file_enabled=true \
     --env NEO4J_dbms_security_procedures_unrestricted=apoc.\\\*,gds.\\\* \
+    --env NEO4J_dbms_logs_debug_level=INFO \
     neo4j:latest
 
 docker-ps-pretty
@@ -104,12 +109,19 @@ do \
     echo "waiting 1s for bolt to appear"; \
     sleep 1; \
 done
-echo '*** neo4j online!'
+echo "*** neo4j online!"
 
 # launch Cypher Shell in container, execute cypher
 cat load-clean-landscape.cypher \
 | docker exec --interactive landscape-graph cypher-shell -u $NEO4J_USERNAME -p $NEO4J_PASSWORD
 
+echo ""
+echo "open http://localhost:7474 ($NEO4J_USERNAME/$NEO4J_PASSWORD)"
+echo ""
+
+#
+# TODO Nuke this
+#
 # docker run \
 #   --user $(id -u):$(id -g) \
 #   --name "${CONTAINER}" \