From a2a0819b835bb3f42f75674fd9ba64b626d2d230 Mon Sep 17 00:00:00 2001
From: Josh Kneubuhl <jkneubuh@us.ibm.com>
Date: Mon, 25 Jul 2022 07:52:06 -0400
Subject: [PATCH 1/5] work in progress prom-operator

Signed-off-by: Josh Kneubuhl <jkneubuh@us.ibm.com>
---
 config/prometheus/ingress-grafana.yaml    | 27 +++++++++++++++++
 config/prometheus/ingress-prometheus.yaml | 27 +++++++++++++++++
 config/prometheus/kustomization.yaml      | 29 +++++++++++++++++-
 sample-network/scripts/cluster.sh         | 37 +++++++++++++++++++++++
 4 files changed, 119 insertions(+), 1 deletion(-)
 create mode 100644 config/prometheus/ingress-grafana.yaml
 create mode 100644 config/prometheus/ingress-prometheus.yaml

diff --git a/config/prometheus/ingress-grafana.yaml b/config/prometheus/ingress-grafana.yaml
new file mode 100644
index 00000000..3d75a5e2
--- /dev/null
+++ b/config/prometheus/ingress-grafana.yaml
@@ -0,0 +1,27 @@
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s
+    nginx.ingress.kubernetes.io/ssl-passthrough: "false"
+  labels:
+    app: grafana
+  name: grafana
+  namespace: monitoring
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: grafana.localho.st
+      http:
+        paths:
+          - backend:
+              service:
+                name: grafana
+                port:
+                  name: http
+            path: /
+            pathType: ImplementationSpecific
+  tls:
+    - hosts:
+        - prometheus.localho.st
diff --git a/config/prometheus/ingress-prometheus.yaml b/config/prometheus/ingress-prometheus.yaml
new file mode 100644
index 00000000..377f0f5d
--- /dev/null
+++ b/config/prometheus/ingress-prometheus.yaml
@@ -0,0 +1,27 @@
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s
+    nginx.ingress.kubernetes.io/ssl-passthrough: "false"
+  labels:
+    app: prometheus-k8s
+  name: prometheus-k8s
+  namespace: monitoring
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: prometheus.localho.st
+      http:
+        paths:
+          - backend:
+              service:
+                name: prometheus-k8s
+                port:
+                  name: web
+            path: /
+            pathType: ImplementationSpecific
+  tls:
+    - hosts:
+        - prometheus.localho.st
diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml
index ed137168..d887c6df 100644
--- a/config/prometheus/kustomization.yaml
+++ b/config/prometheus/kustomization.yaml
@@ -1,2 +1,29 @@
+#
+# Copyright contributors to the Hyperledger Fabric Operator project
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# 	  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
 resources:
-- monitor.yaml
+  - https://github.com/prometheus-operator/kube-prometheus.git/manifests/setup/0alertmanagerConfigCustomResourceDefinition.yaml
+
+#patchesStrategicMerge:
+#  - xyz.yaml
+
+#resources:
+#- monitor.yaml
+
diff --git a/sample-network/scripts/cluster.sh b/sample-network/scripts/cluster.sh
index af464a55..a0fc2370 100644
--- a/sample-network/scripts/cluster.sh
+++ b/sample-network/scripts/cluster.sh
@@ -101,6 +101,8 @@ function cluster_init() {
     pull_docker_images
     kind_load_images
   fi
+
+  apply_prometheus
 }
 
 function apply_fabric_crds() {
@@ -151,6 +153,40 @@ function wait_for_nginx_ingress() {
   pop_fn
 }
 
+# https://prometheus-operator.dev/docs/prologue/quick-start/#deploy-kube-prometheus
+function apply_prometheus() {
+  push_fn "Applying prometheus operator"
+
+  set -x
+
+  git clone https://github.com/prometheus-operator/kube-prometheus.git $TEMP_DIR/kube-prometheus || true
+
+  # Create the namespace and CRDs, and then wait for them to be availble before creating the remaining resources
+  kubectl create -f $TEMP_DIR/kube-prometheus/manifests/setup || true
+
+  # Wait until the "servicemonitors" CRD is created. The message "No resources found" means success in this context.
+  until kubectl get servicemonitors --all-namespaces ; do date; sleep 1; echo ""; done
+
+  kubectl create -f $TEMP_DIR/kube-prometheus/manifests/ || true
+
+  # create ingress for prometheus and grafana at localho.st
+  kubectl apply -f ../config/prometheus/ingress-prometheus.yaml
+  kubectl apply -f ../config/prometheus/ingress-grafana.yaml    # TODO: need to set grafana server.ini root_url
+
+  pop_fn
+}
+
+function delete_prometheus() {
+  push_fn "Deleting prometheus operator"
+
+  kubectl delete --ignore-not-found=true -f $TEMP_DIR/kube-prometheus/manifests/setup
+  kubectl delete --ignore-not-found=true -f $TEMP_DIR/kube-prometheus/manifests/
+
+  pop_fn
+}
+
+
+
 # Allow pods running in kubernetes to access services at the ingress domain *.localho.st.
 #
 # This function identifies the CLUSTER-IP for the ingress controller and overrides the coredns
@@ -207,6 +243,7 @@ EOF
 function cluster_clean() {
   delete_fabric_crds
   delete_nginx_ingress
+  delete_prometheus
 }
 
 

From abdaad69fcdc722bdce991a20108b63a98307112 Mon Sep 17 00:00:00 2001
From: Josh Kneubuhl <jkneubuh@us.ibm.com>
Date: Mon, 25 Jul 2022 08:58:56 -0400
Subject: [PATCH 2/5] wip integration with prometheus

Signed-off-by: Josh Kneubuhl <jkneubuh@us.ibm.com>
---
 config/prometheus/kustomization.yaml          | 28 +------------------
 sample-network/config/peers/org1-peer1.yaml   |  2 ++
 .../config}/prometheus/ingress-grafana.yaml   |  4 +--
 .../prometheus/ingress-prometheus.yaml        |  4 +--
 .../config/prometheus/kustomization.yaml      | 25 +++++++++++++++++
 .../prometheus/servicemonitor-org1-peer1.yaml | 16 +++++++++++
 sample-network/network                        |  1 +
 sample-network/scripts/cluster.sh             | 19 ++++++++-----
 8 files changed, 61 insertions(+), 38 deletions(-)
 rename {config => sample-network/config}/prometheus/ingress-grafana.yaml (87%)
 rename {config => sample-network/config}/prometheus/ingress-prometheus.yaml (87%)
 create mode 100644 sample-network/config/prometheus/kustomization.yaml
 create mode 100644 sample-network/config/prometheus/servicemonitor-org1-peer1.yaml

diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml
index d887c6df..a6a04ac7 100644
--- a/config/prometheus/kustomization.yaml
+++ b/config/prometheus/kustomization.yaml
@@ -1,29 +1,3 @@
-#
-# Copyright contributors to the Hyperledger Fabric Operator project
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-# 	  http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
 resources:
-  - https://github.com/prometheus-operator/kube-prometheus.git/manifests/setup/0alertmanagerConfigCustomResourceDefinition.yaml
-
-#patchesStrategicMerge:
-#  - xyz.yaml
-
-#resources:
-#- monitor.yaml
+- monitor.yaml
 
diff --git a/sample-network/config/peers/org1-peer1.yaml b/sample-network/config/peers/org1-peer1.yaml
index ce07ca30..8906692d 100644
--- a/sample-network/config/peers/org1-peer1.yaml
+++ b/sample-network/config/peers/org1-peer1.yaml
@@ -33,6 +33,8 @@ spec:
     peer:
       keepalive:
         minInterval: 61s
+    metrics:
+      provider: prometheus
   customNames:
     pvc: {}
   images:
diff --git a/config/prometheus/ingress-grafana.yaml b/sample-network/config/prometheus/ingress-grafana.yaml
similarity index 87%
rename from config/prometheus/ingress-grafana.yaml
rename to sample-network/config/prometheus/ingress-grafana.yaml
index 3d75a5e2..becb3eab 100644
--- a/config/prometheus/ingress-grafana.yaml
+++ b/sample-network/config/prometheus/ingress-grafana.yaml
@@ -12,7 +12,7 @@ metadata:
 spec:
   ingressClassName: nginx
   rules:
-    - host: grafana.localho.st
+    - host: grafana.${INGRESS_DOMAIN}
       http:
         paths:
           - backend:
@@ -24,4 +24,4 @@ spec:
             pathType: ImplementationSpecific
   tls:
     - hosts:
-        - prometheus.localho.st
+        - prometheus.${INGRESS_DOMAIN}
diff --git a/config/prometheus/ingress-prometheus.yaml b/sample-network/config/prometheus/ingress-prometheus.yaml
similarity index 87%
rename from config/prometheus/ingress-prometheus.yaml
rename to sample-network/config/prometheus/ingress-prometheus.yaml
index 377f0f5d..2aef1748 100644
--- a/config/prometheus/ingress-prometheus.yaml
+++ b/sample-network/config/prometheus/ingress-prometheus.yaml
@@ -12,7 +12,7 @@ metadata:
 spec:
   ingressClassName: nginx
   rules:
-    - host: prometheus.localho.st
+    - host: prometheus.${INGRESS_DOMAIN}
       http:
         paths:
           - backend:
@@ -24,4 +24,4 @@ spec:
             pathType: ImplementationSpecific
   tls:
     - hosts:
-        - prometheus.localho.st
+        - prometheus.${INGRESS_DOMAIN}
diff --git a/sample-network/config/prometheus/kustomization.yaml b/sample-network/config/prometheus/kustomization.yaml
new file mode 100644
index 00000000..23ad0942
--- /dev/null
+++ b/sample-network/config/prometheus/kustomization.yaml
@@ -0,0 +1,25 @@
+#
+# Copyright contributors to the Hyperledger Fabric Operator project
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# 	  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - servicemonitor-org1-peer1.yaml
+#  - servicemonitor-org1-peer2.yaml
+#  - servicemonitor-org2-peer1.yaml
+#  - servicemonitor-org2-peer2.yaml
diff --git a/sample-network/config/prometheus/servicemonitor-org1-peer1.yaml b/sample-network/config/prometheus/servicemonitor-org1-peer1.yaml
new file mode 100644
index 00000000..ea5c4336
--- /dev/null
+++ b/sample-network/config/prometheus/servicemonitor-org1-peer1.yaml
@@ -0,0 +1,16 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: org1-peer1
+  namespace: test-network
+  labels:
+    release: mypro  #Prometheus label
+spec:
+  namespaceSelector: #service namespace
+    matchNames:
+      - test-network
+  selector:
+    matchLabels:
+      app: org1-peer1
+  endpoints:
+    - port: operations
\ No newline at end of file
diff --git a/sample-network/network b/sample-network/network
index 933dc5f5..e8383bfe 100755
--- a/sample-network/network
+++ b/sample-network/network
@@ -48,6 +48,7 @@ context CLUSTER_NAME              $CLUSTER_RUNTIME
 context KUBE_DNS_DOMAIN           ${NS}.svc.cluster.local
 context INGRESS_DOMAIN            localho.st
 context COREDNS_DOMAIN_OVERRIDE   true
+context PROMETHEUS_MONITORING     false
 context LOG_FILE                  network.log
 context DEBUG_FILE                network-debug.log
 context LOG_ERROR_LINES           1
diff --git a/sample-network/scripts/cluster.sh b/sample-network/scripts/cluster.sh
index a0fc2370..91aafa18 100644
--- a/sample-network/scripts/cluster.sh
+++ b/sample-network/scripts/cluster.sh
@@ -102,7 +102,9 @@ function cluster_init() {
     kind_load_images
   fi
 
-  apply_prometheus
+  if [ "${PROMETHEUS_MONITORING}" == true ]; then
+    apply_prometheus
+  fi
 }
 
 function apply_fabric_crds() {
@@ -169,9 +171,11 @@ function apply_prometheus() {
 
   kubectl create -f $TEMP_DIR/kube-prometheus/manifests/ || true
 
-  # create ingress for prometheus and grafana at localho.st
-  kubectl apply -f ../config/prometheus/ingress-prometheus.yaml
-  kubectl apply -f ../config/prometheus/ingress-grafana.yaml    # TODO: need to set grafana server.ini root_url
+  # create ingress for prometheus and grafana at $INGRESS_DOMAIN
+  cat config/prometheus/ingress-prometheus.yaml | envsubst | kubectl apply -f -
+  cat config/prometheus/ingress-grafana.yaml | envsubst | kubectl apply -f -
+
+  # TODO: override grafana root_url in server.ini (secret)
 
   pop_fn
 }
@@ -185,8 +189,6 @@ function delete_prometheus() {
   pop_fn
 }
 
-
-
 # Allow pods running in kubernetes to access services at the ingress domain *.localho.st.
 #
 # This function identifies the CLUSTER-IP for the ingress controller and overrides the coredns
@@ -243,7 +245,10 @@ EOF
 function cluster_clean() {
   delete_fabric_crds
   delete_nginx_ingress
-  delete_prometheus
+
+  if [ "${PROMETHEUS_MONITORING}" == true ]; then
+    delete_prometheus
+  fi
 }
 
 

From e82f0a210bef4586fe025967390b6df87ae2a999 Mon Sep 17 00:00:00 2001
From: Josh Kneubuhl <jkneubuh@us.ibm.com>
Date: Wed, 27 Jul 2022 13:53:34 -0400
Subject: [PATCH 3/5] Rough WIP for prometheus-operator ServiceMonitors via TLS

Signed-off-by: Josh Kneubuhl <jkneubuh@us.ibm.com>
---
 .../config/orderers/org0-orderers.yaml        | 12 +++----
 sample-network/config/peers/org1-peer1.yaml   |  6 ++--
 sample-network/config/peers/org1-peer2.yaml   |  6 ++--
 sample-network/config/peers/org2-peer1.yaml   |  6 ++--
 sample-network/config/peers/org2-peer2.yaml   |  6 ++--
 .../config/prometheus/kustomization.yaml      |  2 +-
 .../prometheus/servicemonitor-org1-peer1.yaml | 31 +++++++++++++----
 .../prometheus/servicemonitor-org1-peer2.yaml | 33 +++++++++++++++++++
 .../config/rbac/hlf-operator-rolebinding.yaml |  2 +-
 sample-network/scripts/test_network.sh        |  2 +-
 10 files changed, 79 insertions(+), 27 deletions(-)
 create mode 100644 sample-network/config/prometheus/servicemonitor-org1-peer2.yaml

diff --git a/sample-network/config/orderers/org0-orderers.yaml b/sample-network/config/orderers/org0-orderers.yaml
index 3251afc1..088d12bc 100644
--- a/sample-network/config/orderers/org0-orderers.yaml
+++ b/sample-network/config/orderers/org0-orderers.yaml
@@ -43,7 +43,7 @@ spec:
     - enrollment:
         component:
           caname: ca
-          cahost: test-network-org0-ca-ca.${INGRESS_DOMAIN}
+          cahost: ${NS}-org0-ca-ca.${INGRESS_DOMAIN}
           caport: "443"
           catls:
             cacert: "${ORG0_CA_CERT}"
@@ -51,7 +51,7 @@ spec:
           enrollsecret: "orderer1pw"
         tls:
           caname: tlsca
-          cahost: test-network-org0-ca-ca.${INGRESS_DOMAIN}
+          cahost: ${NS}-org0-ca-ca.${INGRESS_DOMAIN}
           caport: "443"
           catls:
             cacert: "${ORG0_CA_CERT}"
@@ -64,7 +64,7 @@ spec:
     - enrollment:
         component:
           caname: ca
-          cahost: test-network-org0-ca-ca.${INGRESS_DOMAIN}
+          cahost: ${NS}-org0-ca-ca.${INGRESS_DOMAIN}
           caport: "443"
           catls:
             cacert: "${ORG0_CA_CERT}"
@@ -72,7 +72,7 @@ spec:
           enrollsecret: "orderer2pw"
         tls:
           caname: tlsca
-          cahost: test-network-org0-ca-ca.${INGRESS_DOMAIN}
+          cahost: ${NS}-org0-ca-ca.${INGRESS_DOMAIN}
           caport: "443"
           catls:
             cacert: "${ORG0_CA_CERT}"
@@ -85,7 +85,7 @@ spec:
     - enrollment:
         component:
           caname: ca
-          cahost: test-network-org0-ca-ca.${INGRESS_DOMAIN}
+          cahost: ${NS}-org0-ca-ca.${INGRESS_DOMAIN}
           caport: "443"
           catls:
             cacert: "${ORG0_CA_CERT}"
@@ -93,7 +93,7 @@ spec:
           enrollsecret: "orderer3pw"
         tls:
           caname: tlsca
-          cahost: test-network-org0-ca-ca.${INGRESS_DOMAIN}
+          cahost: ${NS}-org0-ca-ca.${INGRESS_DOMAIN}
           caport: "443"
           catls:
             cacert: "${ORG0_CA_CERT}"
diff --git a/sample-network/config/peers/org1-peer1.yaml b/sample-network/config/peers/org1-peer1.yaml
index 8906692d..513d16c6 100644
--- a/sample-network/config/peers/org1-peer1.yaml
+++ b/sample-network/config/peers/org1-peer1.yaml
@@ -23,7 +23,7 @@ metadata:
 spec:
   version: "${FABRIC_VERSION}"
   domain: "${INGRESS_DOMAIN}"
-  peerExternalEndpoint: "test-network-org1-peer1-peer.${INGRESS_DOMAIN}:443"
+  peerExternalEndpoint: "${NS}-org1-peer1-peer.${INGRESS_DOMAIN}:443"
   license:
     accept: true
   action:
@@ -50,7 +50,7 @@ spec:
     enrollment:
       component:
         caname: ca
-        cahost: "test-network-org1-ca-ca.${INGRESS_DOMAIN}"
+        cahost: "${NS}-org1-ca-ca.${INGRESS_DOMAIN}"
         caport: "443"
         catls:
           cacert: "${ORG1_CA_CERT}"
@@ -58,7 +58,7 @@ spec:
         enrollsecret: "peer1pw"
       tls:
         caname: tlsca
-        cahost: "test-network-org1-ca-ca.${INGRESS_DOMAIN}"
+        cahost: "${NS}-org1-ca-ca.${INGRESS_DOMAIN}"
         caport: "443"
         catls:
           cacert: "${ORG1_CA_CERT}"
diff --git a/sample-network/config/peers/org1-peer2.yaml b/sample-network/config/peers/org1-peer2.yaml
index 0cbb0ce1..e8e64bed 100644
--- a/sample-network/config/peers/org1-peer2.yaml
+++ b/sample-network/config/peers/org1-peer2.yaml
@@ -23,7 +23,7 @@ metadata:
 spec:
   version: "${FABRIC_VERSION}"
   domain: "${INGRESS_DOMAIN}"
-  peerExternalEndpoint: "test-network-org1-peer2-peer.${INGRESS_DOMAIN}:443"
+  peerExternalEndpoint: "${NS}-org1-peer2-peer.${INGRESS_DOMAIN}:443"
   license:
     accept: true
   action:
@@ -48,7 +48,7 @@ spec:
     enrollment:
       component:
         caname: ca
-        cahost: "test-network-org1-ca-ca.${INGRESS_DOMAIN}"
+        cahost: "${NS}-org1-ca-ca.${INGRESS_DOMAIN}"
         caport: "443"
         catls:
           cacert: "${ORG1_CA_CERT}"
@@ -56,7 +56,7 @@ spec:
         enrollsecret: "peer2pw"
       tls:
         caname: tlsca
-        cahost: "test-network-org1-ca-ca.${INGRESS_DOMAIN}"
+        cahost: "${NS}-org1-ca-ca.${INGRESS_DOMAIN}"
         caport: "443"
         catls:
           cacert: "${ORG1_CA_CERT}"
diff --git a/sample-network/config/peers/org2-peer1.yaml b/sample-network/config/peers/org2-peer1.yaml
index 8e93282e..b9fef501 100644
--- a/sample-network/config/peers/org2-peer1.yaml
+++ b/sample-network/config/peers/org2-peer1.yaml
@@ -23,7 +23,7 @@ metadata:
 spec:
   version: "${FABRIC_VERSION}"
   domain: "${INGRESS_DOMAIN}"
-  peerExternalEndpoint: "test-network-org2-peer1-peer.${INGRESS_DOMAIN}:443"
+  peerExternalEndpoint: "${NS}-org2-peer1-peer.${INGRESS_DOMAIN}:443"
   license:
     accept: true
   action:
@@ -48,7 +48,7 @@ spec:
     enrollment:
       component:
         caname: ca
-        cahost: "test-network-org2-ca-ca.${INGRESS_DOMAIN}"
+        cahost: "${NS}-org2-ca-ca.${INGRESS_DOMAIN}"
         caport: "443"
         catls:
           cacert: "${ORG2_CA_CERT}"
@@ -56,7 +56,7 @@ spec:
         enrollsecret: "peer1pw"
       tls:
         caname: tlsca
-        cahost: "test-network-org2-ca-ca.${INGRESS_DOMAIN}"
+        cahost: "${NS}-org2-ca-ca.${INGRESS_DOMAIN}"
         caport: "443"
         catls:
           cacert: "${ORG2_CA_CERT}"
diff --git a/sample-network/config/peers/org2-peer2.yaml b/sample-network/config/peers/org2-peer2.yaml
index c5f00a5c..3321b46d 100644
--- a/sample-network/config/peers/org2-peer2.yaml
+++ b/sample-network/config/peers/org2-peer2.yaml
@@ -23,7 +23,7 @@ metadata:
 spec:
   version: "${FABRIC_VERSION}"
   domain: "${INGRESS_DOMAIN}"
-  peerExternalEndpoint: "test-network-org2-peer2-peer.${INGRESS_DOMAIN}:443"
+  peerExternalEndpoint: "${NS}-org2-peer2-peer.${INGRESS_DOMAIN}:443"
   license:
     accept: true
   action:
@@ -48,7 +48,7 @@ spec:
     enrollment:
       component:
         caname: ca
-        cahost: "test-network-org2-ca-ca.${INGRESS_DOMAIN}"
+        cahost: "${NS}-org2-ca-ca.${INGRESS_DOMAIN}"
         caport: "443"
         catls:
           cacert: "${ORG2_CA_CERT}"
@@ -56,7 +56,7 @@ spec:
         enrollsecret: "peer2pw"
       tls:
         caname: tlsca
-        cahost: "test-network-org2-ca-ca.${INGRESS_DOMAIN}"
+        cahost: "${NS}-org2-ca-ca.${INGRESS_DOMAIN}"
         caport: "443"
         catls:
           cacert: "${ORG2_CA_CERT}"
diff --git a/sample-network/config/prometheus/kustomization.yaml b/sample-network/config/prometheus/kustomization.yaml
index 23ad0942..f642c3f6 100644
--- a/sample-network/config/prometheus/kustomization.yaml
+++ b/sample-network/config/prometheus/kustomization.yaml
@@ -20,6 +20,6 @@ kind: Kustomization
 
 resources:
   - servicemonitor-org1-peer1.yaml
-#  - servicemonitor-org1-peer2.yaml
+  - servicemonitor-org1-peer2.yaml
 #  - servicemonitor-org2-peer1.yaml
 #  - servicemonitor-org2-peer2.yaml
diff --git a/sample-network/config/prometheus/servicemonitor-org1-peer1.yaml b/sample-network/config/prometheus/servicemonitor-org1-peer1.yaml
index ea5c4336..eba8e6fd 100644
--- a/sample-network/config/prometheus/servicemonitor-org1-peer1.yaml
+++ b/sample-network/config/prometheus/servicemonitor-org1-peer1.yaml
@@ -2,15 +2,34 @@ apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
   name: org1-peer1
-  namespace: test-network
-  labels:
-    release: mypro  #Prometheus label
+  namespace: monitoring
 spec:
-  namespaceSelector: #service namespace
+  namespaceSelector:
+    # todo: use ${NS} and establish RBAC for visibility from system:serviceaccount:monitoring:prometheus-k8s
     matchNames:
-      - test-network
+      - default
   selector:
     matchLabels:
       app: org1-peer1
   endpoints:
-    - port: operations
\ No newline at end of file
+    - port: operations
+      scheme: https
+      # todo: add kustomization target to create org1 monitoring secret from
+      # cacert.pem = temp/enrollments/org1/users/org1admin/tls/cacerts/default-org1-ca-ca-localho-st-443-tlsca.pem
+      # cert.pem = temp/enrollments/org1/users/org1admin/tls/signcerts/cert.pem
+      # key.pem = temp/enrollments/org1/users/org1admin/tls/keystore/key.pem
+      tlsConfig:
+        serverName: org1-peer1
+        ca:
+          secret:
+            name: org1-servicemonitor-tls-secret
+            key: cacert.pem
+        cert:
+          secret:
+            name: org1-servicemonitor-tls-secret
+            key: cert.pem
+        keySecret:
+          name: org1-servicemonitor-tls-secret
+          key: key.pem
+
+
diff --git a/sample-network/config/prometheus/servicemonitor-org1-peer2.yaml b/sample-network/config/prometheus/servicemonitor-org1-peer2.yaml
new file mode 100644
index 00000000..c454b44a
--- /dev/null
+++ b/sample-network/config/prometheus/servicemonitor-org1-peer2.yaml
@@ -0,0 +1,33 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: org1-peer2
+  namespace: monitoring
+spec:
+  namespaceSelector:
+    # todo: use ${NS} and establish RBAC for visibility from system:serviceaccount:monitoring:prometheus-k8s
+    matchNames:
+      - default
+  selector:
+    matchLabels:
+      app: org1-peer2
+  endpoints:
+    - port: operations
+      scheme: https
+      # todo: add kustomization target to create org1 monitoring secret from
+      # cacert.pem = temp/enrollments/org1/users/org1admin/tls/cacerts/default-org1-ca-ca-localho-st-443-tlsca.pem
+      # cert.pem = temp/enrollments/org1/users/org1admin/tls/signcerts/cert.pem
+      # key.pem = temp/enrollments/org1/users/org1admin/tls/keystore/key.pem
+      tlsConfig:
+        serverName: org1-peer2
+        ca:
+          secret:
+            name: org1-servicemonitor-tls-secret
+            key: cacert.pem
+        cert:
+          secret:
+            name: org1-servicemonitor-tls-secret
+            key: cert.pem
+        keySecret:
+          name: org1-servicemonitor-tls-secret
+          key: key.pem
diff --git a/sample-network/config/rbac/hlf-operator-rolebinding.yaml b/sample-network/config/rbac/hlf-operator-rolebinding.yaml
index 04f948e8..afec3c5a 100644
--- a/sample-network/config/rbac/hlf-operator-rolebinding.yaml
+++ b/sample-network/config/rbac/hlf-operator-rolebinding.yaml
@@ -16,4 +16,4 @@
 # limitations under the License.
 #
 ---
-kubectl -n $NS create rolebinding hlf-operator-rolebinding --clusterrole=hlf-operator-role --group=system:serviceaccounts:test-network --
+kubectl -n $NS create rolebinding hlf-operator-rolebinding --clusterrole=hlf-operator-role --group=system:serviceaccounts:$NS --
diff --git a/sample-network/scripts/test_network.sh b/sample-network/scripts/test_network.sh
index 20f0c10f..fa9f194b 100644
--- a/sample-network/scripts/test_network.sh
+++ b/sample-network/scripts/test_network.sh
@@ -56,7 +56,7 @@ function init_namespace() {
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: test-network
+  name: ${NS}
 EOF
 
   pop_fn

From bbcd250faeaa8004597aa7fa0158d2067fe3d804 Mon Sep 17 00:00:00 2001
From: Josh Kneubuhl <jkneubuh@us.ibm.com>
Date: Thu, 28 Jul 2022 08:53:18 -0400
Subject: [PATCH 4/5] add sample prometheus service monitors for the sample
 network

Signed-off-by: Josh Kneubuhl <jkneubuh@us.ibm.com>
---
 sample-network/config/cas/kustomization.yaml  |  2 +
 .../config/console/kustomization.yaml         |  2 +
 .../config/manager/kustomization.yaml         |  2 +
 .../config/orderers/kustomization.yaml        |  2 +
 .../config/peers/kustomization.yaml           |  2 +
 .../config/prometheus/kustomization.yaml      |  9 ++++-
 .../servicemonitor-org0-orderersnode1.yaml    | 40 +++++++++++++++++++
 .../servicemonitor-org0-orderersnode2.yaml    | 40 +++++++++++++++++++
 .../servicemonitor-org0-orderersnode3.yaml    | 40 +++++++++++++++++++
 .../prometheus/servicemonitor-org1-peer1.yaml | 19 +++++----
 .../prometheus/servicemonitor-org1-peer2.yaml | 21 ++++++----
 .../prometheus/servicemonitor-org2-peer1.yaml | 40 +++++++++++++++++++
 .../prometheus/servicemonitor-org2-peer2.yaml | 40 +++++++++++++++++++
 sample-network/config/rbac/kustomization.yaml |  2 +
 sample-network/scripts/channel.sh             |  5 +--
 sample-network/scripts/test_network.sh        | 34 ++++++++++++++++
 sample-network/scripts/utils.sh               |  4 +-
 17 files changed, 282 insertions(+), 22 deletions(-)
 create mode 100644 sample-network/config/prometheus/servicemonitor-org0-orderersnode1.yaml
 create mode 100644 sample-network/config/prometheus/servicemonitor-org0-orderersnode2.yaml
 create mode 100644 sample-network/config/prometheus/servicemonitor-org0-orderersnode3.yaml
 create mode 100644 sample-network/config/prometheus/servicemonitor-org2-peer1.yaml
 create mode 100644 sample-network/config/prometheus/servicemonitor-org2-peer2.yaml

diff --git a/sample-network/config/cas/kustomization.yaml b/sample-network/config/cas/kustomization.yaml
index 0d354cf4..3135920c 100644
--- a/sample-network/config/cas/kustomization.yaml
+++ b/sample-network/config/cas/kustomization.yaml
@@ -19,6 +19,8 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
+namespace: ${NS}
+
 resources:
   - org0-ca.yaml
   - org1-ca.yaml
diff --git a/sample-network/config/console/kustomization.yaml b/sample-network/config/console/kustomization.yaml
index 1f7ec5bd..140b32ec 100644
--- a/sample-network/config/console/kustomization.yaml
+++ b/sample-network/config/console/kustomization.yaml
@@ -1,5 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
+namespace: ${NS}
+
 resources:
   - hlf-operations-console.yaml
diff --git a/sample-network/config/manager/kustomization.yaml b/sample-network/config/manager/kustomization.yaml
index 9bc57c5e..34cb2444 100644
--- a/sample-network/config/manager/kustomization.yaml
+++ b/sample-network/config/manager/kustomization.yaml
@@ -18,5 +18,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
+namespace: ${NS}
+
 resources:
   - hlf-operator-manager.yaml
diff --git a/sample-network/config/orderers/kustomization.yaml b/sample-network/config/orderers/kustomization.yaml
index 91c9041d..75fcf393 100644
--- a/sample-network/config/orderers/kustomization.yaml
+++ b/sample-network/config/orderers/kustomization.yaml
@@ -19,6 +19,8 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
+namespace: ${NS}
+
 resources:
   - org0-orderers.yaml
 
diff --git a/sample-network/config/peers/kustomization.yaml b/sample-network/config/peers/kustomization.yaml
index a962324c..6d91ba75 100644
--- a/sample-network/config/peers/kustomization.yaml
+++ b/sample-network/config/peers/kustomization.yaml
@@ -19,6 +19,8 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
+namespace: ${NS}
+
 resources:
   - org1-peer1.yaml
   - org1-peer2.yaml
diff --git a/sample-network/config/prometheus/kustomization.yaml b/sample-network/config/prometheus/kustomization.yaml
index f642c3f6..720467a8 100644
--- a/sample-network/config/prometheus/kustomization.yaml
+++ b/sample-network/config/prometheus/kustomization.yaml
@@ -18,8 +18,13 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
+namespace: monitoring
+
 resources:
+  - servicemonitor-org0-orderersnode1.yaml
+  - servicemonitor-org0-orderersnode2.yaml
+  - servicemonitor-org0-orderersnode3.yaml
   - servicemonitor-org1-peer1.yaml
   - servicemonitor-org1-peer2.yaml
-#  - servicemonitor-org2-peer1.yaml
-#  - servicemonitor-org2-peer2.yaml
+  - servicemonitor-org2-peer1.yaml
+  - servicemonitor-org2-peer2.yaml
diff --git a/sample-network/config/prometheus/servicemonitor-org0-orderersnode1.yaml b/sample-network/config/prometheus/servicemonitor-org0-orderersnode1.yaml
new file mode 100644
index 00000000..e71c5c34
--- /dev/null
+++ b/sample-network/config/prometheus/servicemonitor-org0-orderersnode1.yaml
@@ -0,0 +1,40 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: org0-orderersnode1
+  namespace: monitoring
+spec:
+  namespaceSelector:
+    # todo: establish RBAC for visibility from system:serviceaccount:monitoring:prometheus-k8s
+    matchNames:
+      - ${NS}
+  selector:
+    matchLabels:
+      app: org0-orderersnode1
+  endpoints:
+    - port: operations
+      scheme: https
+
+      # The operations /metrics endpoint is secured by mTLS, and must be presented with a client certificate
+      # and key signed by the org CA.  In this example we will use the org admin TLS enrollment to connect
+      # to the endpoint. See https://cloud.ibm.com/docs/blockchain?topic=blockchain-ibp-monitoring-prometheus
+      # for an example of creating a custom org enrollment for external monitors via ingress.
+      #
+      # tlsca-signcert.pem : temp/cas/$org-ca/tlsca-signcert.pem
+      # cert.pem : temp/enrollments/$org/users/org0admin/tls/signcerts/cert.pem
+      # key.pem : temp/enrollments/$org/users/org0admin/tls/keystore/key.pem
+      tlsConfig:
+        serverName: org0-orderersnode1
+        ca:
+          secret:
+            name: org0-servicemonitor-tls-secret
+            key: tlsca-signcert.pem
+        cert:
+          secret:
+            name: org0-servicemonitor-tls-secret
+            key: cert.pem
+        keySecret:
+          name: org0-servicemonitor-tls-secret
+          key: key.pem
+
+
diff --git a/sample-network/config/prometheus/servicemonitor-org0-orderersnode2.yaml b/sample-network/config/prometheus/servicemonitor-org0-orderersnode2.yaml
new file mode 100644
index 00000000..1d07cd12
--- /dev/null
+++ b/sample-network/config/prometheus/servicemonitor-org0-orderersnode2.yaml
@@ -0,0 +1,40 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: org0-orderersnode2
+  namespace: monitoring
+spec:
+  namespaceSelector:
+    # todo: establish RBAC for visibility from system:serviceaccount:monitoring:prometheus-k8s
+    matchNames:
+      - ${NS}
+  selector:
+    matchLabels:
+      app: org0-orderersnode2
+  endpoints:
+    - port: operations
+      scheme: https
+
+      # The operations /metrics endpoint is secured by mTLS, and must be presented with a client certificate
+      # and key signed by the org CA.  In this example we will use the org admin TLS enrollment to connect
+      # to the endpoint. See https://cloud.ibm.com/docs/blockchain?topic=blockchain-ibp-monitoring-prometheus
+      # for an example of creating a custom org enrollment for external monitors via ingress.
+      #
+      # tlsca-signcert.pem : temp/cas/$org-ca/tlsca-signcert.pem
+      # cert.pem : temp/enrollments/$org/users/org0admin/tls/signcerts/cert.pem
+      # key.pem : temp/enrollments/$org/users/org0admin/tls/keystore/key.pem
+      tlsConfig:
+        serverName: org0-orderersnode2
+        ca:
+          secret:
+            name: org0-servicemonitor-tls-secret
+            key: tlsca-signcert.pem
+        cert:
+          secret:
+            name: org0-servicemonitor-tls-secret
+            key: cert.pem
+        keySecret:
+          name: org0-servicemonitor-tls-secret
+          key: key.pem
+
+
diff --git a/sample-network/config/prometheus/servicemonitor-org0-orderersnode3.yaml b/sample-network/config/prometheus/servicemonitor-org0-orderersnode3.yaml
new file mode 100644
index 00000000..ce27f605
--- /dev/null
+++ b/sample-network/config/prometheus/servicemonitor-org0-orderersnode3.yaml
@@ -0,0 +1,40 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: org0-orderersnode3
+  namespace: monitoring
+spec:
+  namespaceSelector:
+    # todo: establish RBAC for visibility from system:serviceaccount:monitoring:prometheus-k8s
+    matchNames:
+      - ${NS}
+  selector:
+    matchLabels:
+      app: org0-orderersnode3
+  endpoints:
+    - port: operations
+      scheme: https
+
+      # The operations /metrics endpoint is secured by mTLS, and must be presented with a client certificate
+      # and key signed by the org CA.  In this example we will use the org admin TLS enrollment to connect
+      # to the endpoint. See https://cloud.ibm.com/docs/blockchain?topic=blockchain-ibp-monitoring-prometheus
+      # for an example of creating a custom org enrollment for external monitors via ingress.
+      #
+      # tlsca-signcert.pem : temp/cas/$org-ca/tlsca-signcert.pem
+      # cert.pem : temp/enrollments/$org/users/org0admin/tls/signcerts/cert.pem
+      # key.pem : temp/enrollments/$org/users/org0admin/tls/keystore/key.pem
+      tlsConfig:
+        serverName: org0-orderersnode3
+        ca:
+          secret:
+            name: org0-servicemonitor-tls-secret
+            key: tlsca-signcert.pem
+        cert:
+          secret:
+            name: org0-servicemonitor-tls-secret
+            key: cert.pem
+        keySecret:
+          name: org0-servicemonitor-tls-secret
+          key: key.pem
+
+
diff --git a/sample-network/config/prometheus/servicemonitor-org1-peer1.yaml b/sample-network/config/prometheus/servicemonitor-org1-peer1.yaml
index eba8e6fd..346c692a 100644
--- a/sample-network/config/prometheus/servicemonitor-org1-peer1.yaml
+++ b/sample-network/config/prometheus/servicemonitor-org1-peer1.yaml
@@ -5,25 +5,30 @@ metadata:
   namespace: monitoring
 spec:
   namespaceSelector:
-    # todo: use ${NS} and establish RBAC for visibility from system:serviceaccount:monitoring:prometheus-k8s
+    # todo: establish RBAC for visibility from system:serviceaccount:monitoring:prometheus-k8s
     matchNames:
-      - default
+      - ${NS}
   selector:
     matchLabels:
       app: org1-peer1
   endpoints:
     - port: operations
       scheme: https
-      # todo: add kustomization target to create org1 monitoring secret from
-      # cacert.pem = temp/enrollments/org1/users/org1admin/tls/cacerts/default-org1-ca-ca-localho-st-443-tlsca.pem
-      # cert.pem = temp/enrollments/org1/users/org1admin/tls/signcerts/cert.pem
-      # key.pem = temp/enrollments/org1/users/org1admin/tls/keystore/key.pem
+
+      # The operations /metrics endpoint is secured by mTLS, and must be presented with a client certificate
+      # and key signed by the org CA.  In this example we will use the org admin TLS enrollment to connect
+      # to the endpoint. See https://cloud.ibm.com/docs/blockchain?topic=blockchain-ibp-monitoring-prometheus
+      # for an example of creating a custom org enrollment for external monitors via ingress.
+      #
+      # tlsca-signcert.pem : temp/cas/$org-ca/tlsca-signcert.pem
+      # cert.pem : temp/enrollments/$org/users/org0admin/tls/signcerts/cert.pem
+      # key.pem : temp/enrollments/$org/users/org0admin/tls/keystore/key.pem
       tlsConfig:
         serverName: org1-peer1
         ca:
           secret:
             name: org1-servicemonitor-tls-secret
-            key: cacert.pem
+            key: tlsca-signcert.pem
         cert:
           secret:
             name: org1-servicemonitor-tls-secret
diff --git a/sample-network/config/prometheus/servicemonitor-org1-peer2.yaml b/sample-network/config/prometheus/servicemonitor-org1-peer2.yaml
index c454b44a..716c56f8 100644
--- a/sample-network/config/prometheus/servicemonitor-org1-peer2.yaml
+++ b/sample-network/config/prometheus/servicemonitor-org1-peer2.yaml
@@ -5,25 +5,30 @@ metadata:
   namespace: monitoring
 spec:
   namespaceSelector:
-    # todo: use ${NS} and establish RBAC for visibility from system:serviceaccount:monitoring:prometheus-k8s
+    # todo: establish RBAC for visibility from system:serviceaccount:monitoring:prometheus-k8s
     matchNames:
-      - default
+      - ${NS}
   selector:
     matchLabels:
       app: org1-peer2
   endpoints:
     - port: operations
       scheme: https
-      # todo: add kustomization target to create org1 monitoring secret from
-      # cacert.pem = temp/enrollments/org1/users/org1admin/tls/cacerts/default-org1-ca-ca-localho-st-443-tlsca.pem
-      # cert.pem = temp/enrollments/org1/users/org1admin/tls/signcerts/cert.pem
-      # key.pem = temp/enrollments/org1/users/org1admin/tls/keystore/key.pem
+
+      # The operations /metrics endpoint is secured by mTLS, and must be presented with a client certificate
+      # and key signed by the org CA.  In this example we will use the org admin TLS enrollment to connect
+      # to the endpoint. See https://cloud.ibm.com/docs/blockchain?topic=blockchain-ibp-monitoring-prometheus
+      # for an example of creating a custom org enrollment for external monitors via ingress.
+      #
+      # tlsca-signcert.pem : temp/cas/$org-ca/tlsca-signcert.pem
+      # cert.pem : temp/enrollments/$org/users/org0admin/tls/signcerts/cert.pem
+      # key.pem : temp/enrollments/$org/users/org0admin/tls/keystore/key.pem
       tlsConfig:
         serverName: org1-peer2
         ca:
           secret:
             name: org1-servicemonitor-tls-secret
-            key: cacert.pem
+            key: tlsca-signcert.pem
         cert:
           secret:
             name: org1-servicemonitor-tls-secret
@@ -31,3 +36,5 @@ spec:
         keySecret:
           name: org1-servicemonitor-tls-secret
           key: key.pem
+
+
diff --git a/sample-network/config/prometheus/servicemonitor-org2-peer1.yaml b/sample-network/config/prometheus/servicemonitor-org2-peer1.yaml
new file mode 100644
index 00000000..e1ea4889
--- /dev/null
+++ b/sample-network/config/prometheus/servicemonitor-org2-peer1.yaml
@@ -0,0 +1,40 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: org2-peer1
+  namespace: monitoring
+spec:
+  namespaceSelector:
+    # todo: establish RBAC for visibility from system:serviceaccount:monitoring:prometheus-k8s
+    matchNames:
+      - ${NS}
+  selector:
+    matchLabels:
+      app: org2-peer1
+  endpoints:
+    - port: operations
+      scheme: https
+
+      # The operations /metrics endpoint is secured by mTLS, and must be presented with a client certificate
+      # and key signed by the org CA.  In this example we will use the org admin TLS enrollment to connect
+      # to the endpoint. See https://cloud.ibm.com/docs/blockchain?topic=blockchain-ibp-monitoring-prometheus
+      # for an example of creating a custom org enrollment for external monitors via ingress.
+      #
+      # tlsca-signcert.pem : temp/cas/$org-ca/tlsca-signcert.pem
+      # cert.pem : temp/enrollments/$org/users/org0admin/tls/signcerts/cert.pem
+      # key.pem : temp/enrollments/$org/users/org0admin/tls/keystore/key.pem
+      tlsConfig:
+        serverName: org2-peer1
+        ca:
+          secret:
+            name: org2-servicemonitor-tls-secret
+            key: tlsca-signcert.pem
+        cert:
+          secret:
+            name: org2-servicemonitor-tls-secret
+            key: cert.pem
+        keySecret:
+          name: org2-servicemonitor-tls-secret
+          key: key.pem
+
+
diff --git a/sample-network/config/prometheus/servicemonitor-org2-peer2.yaml b/sample-network/config/prometheus/servicemonitor-org2-peer2.yaml
new file mode 100644
index 00000000..a1ae56a4
--- /dev/null
+++ b/sample-network/config/prometheus/servicemonitor-org2-peer2.yaml
@@ -0,0 +1,40 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: org2-peer2
+  namespace: monitoring
+spec:
+  namespaceSelector:
+    # todo: establish RBAC for visibility from system:serviceaccount:monitoring:prometheus-k8s
+    matchNames:
+      - ${NS}
+  selector:
+    matchLabels:
+      app: org2-peer2
+  endpoints:
+    - port: operations
+      scheme: https
+
+      # The operations /metrics endpoint is secured by mTLS, and must be presented with a client certificate
+      # and key signed by the org CA.  In this example we will use the org admin TLS enrollment to connect
+      # to the endpoint. See https://cloud.ibm.com/docs/blockchain?topic=blockchain-ibp-monitoring-prometheus
+      # for an example of creating a custom org enrollment for external monitors via ingress.
+      #
+      # tlsca-signcert.pem : temp/cas/$org-ca/tlsca-signcert.pem
+      # cert.pem : temp/enrollments/$org/users/org0admin/tls/signcerts/cert.pem
+      # key.pem : temp/enrollments/$org/users/org0admin/tls/keystore/key.pem
+      tlsConfig:
+        serverName: org2-peer2
+        ca:
+          secret:
+            name: org2-servicemonitor-tls-secret
+            key: tlsca-signcert.pem
+        cert:
+          secret:
+            name: org2-servicemonitor-tls-secret
+            key: cert.pem
+        keySecret:
+          name: org2-servicemonitor-tls-secret
+          key: key.pem
+
+
diff --git a/sample-network/config/rbac/kustomization.yaml b/sample-network/config/rbac/kustomization.yaml
index 9a5132c7..6d79975f 100644
--- a/sample-network/config/rbac/kustomization.yaml
+++ b/sample-network/config/rbac/kustomization.yaml
@@ -18,6 +18,8 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
+namespace: ${NS}
+
 resources:
   - hlf-psp.yaml
   - hlf-operator-serviceaccount.yaml
diff --git a/sample-network/scripts/channel.sh b/sample-network/scripts/channel.sh
index 36fe802e..0a276c80 100644
--- a/sample-network/scripts/channel.sh
+++ b/sample-network/scripts/channel.sh
@@ -38,10 +38,6 @@ function channel_command_group() {
 }
 
 function channel_up() {
-  set -x
-
-  enroll_org_admins
-
   create_channel_msp
   create_genesis_block
 
@@ -120,6 +116,7 @@ function enroll_org_admin() {
   CA_URL=https://${CA_AUTH}@${CA_HOST}:${CA_PORT}
 
   jq -r .tls.cert $CONNECTION_PROFILE | base64 -d >& $CA_DIR/tls-cert.pem
+  jq -r .tlsca.signcerts $CONNECTION_PROFILE | base64 -d >& $CA_DIR/tlsca-signcert.pem
 
   # enroll the admin user
   FABRIC_CA_CLIENT_HOME=${ORG_ADMIN_DIR} fabric-ca-client enroll --url ${CA_URL} --tls.certfiles ${CA_DIR}/tls-cert.pem
diff --git a/sample-network/scripts/test_network.sh b/sample-network/scripts/test_network.sh
index fa9f194b..f875b6b1 100644
--- a/sample-network/scripts/test_network.sh
+++ b/sample-network/scripts/test_network.sh
@@ -35,6 +35,7 @@ function network_up() {
   launch_operator
 
   launch_network_CAs
+  enroll_org_admins
 
   apply_network_peers
   apply_network_orderers
@@ -47,6 +48,10 @@ function network_up() {
   wait_for ibporderer org0-orderersnode1
   wait_for ibporderer org0-orderersnode2
   wait_for ibporderer org0-orderersnode3
+
+  if [ "${PROMETHEUS_MONITORING}" == true ]; then
+    apply_service_monitors
+  fi
 }
 
 function init_namespace() {
@@ -123,6 +128,34 @@ function apply_network_orderers() {
   pop_fn
 }
 
+# Create kube secrets to store the monitoring client TLS certificate and key for access
+# to the operations/metrics endpoint.
+function create_servicemonitor_tls_secret() {
+  local org=$1
+  local secret_name=${org}-servicemonitor-tls-secret
+
+  kubectl -n monitoring delete secret ${secret_name} --ignore-not-found
+
+  kubectl -n monitoring \
+    create secret \
+      generic ${secret_name} \
+      --from-file=temp/cas/${org}-ca/tlsca-signcert.pem \
+      --from-file=temp/enrollments/${org}/users/${org}admin/tls/signcerts/cert.pem \
+      --from-file=temp/enrollments/${org}/users/${org}admin/tls/keystore/key.pem
+}
+
+function apply_service_monitors() {
+  push_fn "Creating Prometheus service monitors"
+
+  create_servicemonitor_tls_secret org0
+  create_servicemonitor_tls_secret org1
+  create_servicemonitor_tls_secret org2
+
+  apply_kustomization config/prometheus
+
+  pop_fn
+}
+
 function stop_services() {
   push_fn "Stopping Fabric Services"
 
@@ -130,6 +163,7 @@ function stop_services() {
   undo_kustomization config/cas
   undo_kustomization config/peers
   undo_kustomization config/orderers
+  undo_kustomization config/prometheus
 
   # give the operator a chance to reconcile the deletion and then shut down the operator.
   sleep 10
diff --git a/sample-network/scripts/utils.sh b/sample-network/scripts/utils.sh
index db7d60f5..db9b3b8e 100644
--- a/sample-network/scripts/utils.sh
+++ b/sample-network/scripts/utils.sh
@@ -109,11 +109,11 @@ function absolute_path() {
 }
 
 function apply_kustomization() {
-  $KUSTOMIZE_BUILD $1 | envsubst | kubectl -n $NS apply -f -
+  $KUSTOMIZE_BUILD $1 | envsubst | kubectl apply -f -
 }
 
 function undo_kustomization() {
-  $KUSTOMIZE_BUILD $1 | envsubst | kubectl -n $NS delete --ignore-not-found=true -f -
+  $KUSTOMIZE_BUILD $1 | envsubst | kubectl delete --ignore-not-found=true -f -
 }
 
 function create_image_pull_secret() {

From 29a138bf15a5d5b3b1638d2db675168cdbffd691 Mon Sep 17 00:00:00 2001
From: Josh Kneubuhl <jkneubuh@us.ibm.com>
Date: Fri, 29 Jul 2022 07:25:34 -0400
Subject: [PATCH 5/5] only remove servicemonitors when monitoring

Signed-off-by: Josh Kneubuhl <jkneubuh@us.ibm.com>
---
 sample-network/scripts/test_network.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/sample-network/scripts/test_network.sh b/sample-network/scripts/test_network.sh
index f875b6b1..24f3abb8 100644
--- a/sample-network/scripts/test_network.sh
+++ b/sample-network/scripts/test_network.sh
@@ -163,7 +163,11 @@ function stop_services() {
   undo_kustomization config/cas
   undo_kustomization config/peers
   undo_kustomization config/orderers
-  undo_kustomization config/prometheus
+
+  if [ "${PROMETHEUS_MONITORING}" == true ]; then
+    undo_kustomization config/prometheus
+  fi
+
 
   # give the operator a chance to reconcile the deletion and then shut down the operator.
   sleep 10