manifest-generator: add network-policies generation support

A new environment variable CDI_DEPLOY_NP has been added with the
options of true and false (defaults to false) to control the deployment
of CDI's network policies.

The network policies deployed when CDI_DEPLOY_NP is set to true are the
ones generated by the createNetworkPolicies factory function as well as
static ones for denying all traffic in the namespace and allow traffic
to kube-apiserver and DNS for pods labeled with `np.kubevirt.io/allow-access-cluster-services`.

Signed-off-by: Adi Aloni <[email protected]>

Author: Acedus

Makefile

@@ -64,6 +64,7 @@ help: ## Print this message and exit
 	@echo "    WHAT                        Path to the package to test. Default is './pkg/... ./cmd/...' for unit tests and './test/...' for functional tests."
 	@echo "    RELREF                      Required by release-description. Must be a commit or tag. Should be newer than $$PREREF."
 	@echo "    PREREF                      Required by release-description. Must also be a commit or tag. Should be older than $$RELREF."
+	@echo "    CDI_DEPLOY_NP        Deploy CDI's utility network policies. Default is 'false'."
 
 all: manifests bazel-build-images ## Clean up previous build artifacts, compile all CDI packages and build containers
 
@@ -170,10 +171,10 @@ cluster-clean-test-infra:
 	CDI_CLEAN="test-infra" ./cluster-sync/clean.sh
 
 cluster-sync-cdi: cluster-clean-cdi
-	./cluster-sync/sync.sh CDI_AVAILABLE_TIMEOUT=${CDI_AVAILABLE_TIMEOUT} DOCKER_PREFIX=${DOCKER_PREFIX} DOCKER_TAG=${DOCKER_TAG} PULL_POLICY=${PULL_POLICY} CDI_NAMESPACE=${CDI_NAMESPACE}
+	./cluster-sync/sync.sh CDI_AVAILABLE_TIMEOUT=${CDI_AVAILABLE_TIMEOUT} DOCKER_PREFIX=${DOCKER_PREFIX} DOCKER_TAG=${DOCKER_TAG} PULL_POLICY=${PULL_POLICY} CDI_NAMESPACE=${CDI_NAMESPACE} CDI_DEPLOY_NP=${CDI_DEPLOY_NP}
 
 cluster-sync-test-infra: cluster-clean-test-infra
-	CDI_SYNC="test-infra" ./cluster-sync/sync.sh CDI_AVAILABLE_TIMEOUT=${CDI_AVAILABLE_TIMEOUT} DOCKER_PREFIX=${DOCKER_PREFIX} DOCKER_TAG=${DOCKER_TAG} PULL_POLICY=${PULL_POLICY} CDI_NAMESPACE=${CDI_NAMESPACE}
+	CDI_SYNC="test-infra" ./cluster-sync/sync.sh CDI_AVAILABLE_TIMEOUT=${CDI_AVAILABLE_TIMEOUT} DOCKER_PREFIX=${DOCKER_PREFIX} DOCKER_TAG=${DOCKER_TAG} PULL_POLICY=${PULL_POLICY} CDI_NAMESPACE=${CDI_NAMESPACE} CDI_DEPLOY_NP=${CDI_DEPLOY_NP}
 
 cluster-sync: cluster-sync-cdi cluster-sync-test-infra ## Build the controller/importer/cloner, and push it into a running cluster. The cluster must be up before running a cluster sync. Also generates a manifest and applies it to the running cluster after pushing the images to it.
 

cluster-sync/clean.sh

@@ -80,6 +80,7 @@ for n in ${NAMESPACES[@]}; do
     _kubectl -n ${n} delete rolebinding -l ${label}
     _kubectl -n ${n} delete roles -l ${label}
     _kubectl -n ${n} delete serviceaccounts -l ${label}
+    _kubectl -n ${n} delete networkpolicies -l ${label}
   done
 done
 

cluster-sync/networkpolicies/cdi-testing-np.yaml

@@ -0,0 +1,17 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: allow-all-cdi-test
+  labels:
+    cdi.kubevirt.io: ""
+spec:
+  podSelector:
+    matchLabels:
+      cdi.kubevirt.io/testing: ""
+  policyTypes:
+    - Ingress
+    - Egress
+  ingress: 
+  - {}
+  egress: 
+  - {}

cluster-sync/networkpolicies/cluster-services-np.yaml

@@ -0,0 +1,43 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: default-deny
+  labels:
+    cdi.kubevirt.io: ""
+spec:
+  podSelector: { }
+  policyTypes:
+    - Ingress
+    - Egress
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: cdi-allow-egress-to-cluster-services
+  labels:
+    cdi.kubevirt.io: ""
+spec:
+  podSelector:
+    matchExpressions:
+      - key: np.kubevirt.io/allow-access-cluster-services
+        operator: In
+        values:
+          - "true"
+  policyTypes:
+    - Egress
+  egress:
+    - ports:
+        - protocol: TCP
+          port: 6443
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: kube-system
+          podSelector:
+            matchLabels:
+              k8s-app: kube-dns
+      ports:
+        - protocol: TCP
+          port: dns-tcp
+        - protocol: UDP
+          port: dns

cluster-sync/sync.sh

@@ -11,9 +11,9 @@ source ./cluster-up/hack/common.sh
 source ./cluster-up/cluster/${KUBEVIRT_PROVIDER}/provider.sh
 
 if [ "${KUBEVIRT_PROVIDER}" = "external" ]; then
-   CDI_SYNC_PROVIDER="external"
+  CDI_SYNC_PROVIDER="external"
 else
-   CDI_SYNC_PROVIDER="kubevirtci"
+  CDI_SYNC_PROVIDER="kubevirtci"
 fi
 
 source ./cluster-sync/${CDI_SYNC_PROVIDER}/provider.sh
@@ -68,8 +68,8 @@ function kill_running_operator {
   out=$(_kubectl get pods -n $CDI_NAMESPACE)
   out=($out)
   length=${#out[@]}
-  for ((idx=0; idx<${#out[@]}; idx=idx+5)); do
-    if [[ ${out[idx]} == cdi-operator-* ]] && [[ ${out[idx+2]} == "Running" ]]; then
+  for ((idx = 0; idx < ${#out[@]}; idx = idx + 5)); do
+    if [[ ${out[idx]} == cdi-operator-* ]] && [[ ${out[idx + 2]} == "Running" ]]; then
       _kubectl delete pod ${out[idx]} -n $CDI_NAMESPACE --grace-period=0 --force
       return
     fi
@@ -132,25 +132,25 @@ NEW_CDI_VER_PODS="./_out/tests/new_cdi_ver_pods"
 # Note it will fail update to the same version
 function wait_cdi_pods_updated {
   echo "Waiting $CDI_PODS_UPDATE_TIMEOUT seconds for all CDI pods to update"
-  if [ -f $NEW_CDI_VER_PODS ] ; then
+  if [ -f $NEW_CDI_VER_PODS ]; then
     mv $NEW_CDI_VER_PODS $OLD_CDI_VER_PODS
   fi
   wait_time=0
   ret=0
   while [[ $ret -eq 0 ]] && [[ $wait_time -lt ${CDI_PODS_UPDATE_TIMEOUT} ]]; do
     wait_time=$((wait_time + 5))
-    _kubectl get pods -n $CDI_NAMESPACE -l '!cdi.kubevirt.io/testing' -o=jsonpath='{range .items[*]}{.metadata.name}{"\n"}{.metadata.uid}{"\n"}{.spec.containers[*].image}{"\n"}{end}' > $NEW_CDI_VER_PODS
-    if [ -f $OLD_CDI_VER_PODS ] ; then
+    _kubectl get pods -n $CDI_NAMESPACE -l '!cdi.kubevirt.io/testing' -o=jsonpath='{range .items[*]}{.metadata.name}{"\n"}{.metadata.uid}{"\n"}{.spec.containers[*].image}{"\n"}{end}' >$NEW_CDI_VER_PODS
+    if [ -f $OLD_CDI_VER_PODS ]; then
       grep -qFxf $OLD_CDI_VER_PODS $NEW_CDI_VER_PODS || ret=$?
-      if [ $ret -eq 0 ] ; then
+      if [ $ret -eq 0 ]; then
         sleep 5
       fi
     else
       ret=1
     fi
   done
   echo "Waited $wait_time seconds"
-  if [ $ret -eq 0 ] ; then
+  if [ $ret -eq 0 ]; then
     echo "Not all pods updated"
     exit 1
   fi
@@ -168,7 +168,7 @@ function dump_upgrade_info {
 # Setup some datavolumes in older version for testing upgrades
 # Done unconditionally to make it easier to write tests.
 function setup_for_upgrade_testing {
-  if _kubectl get namespace cdi-testing-old-version-artifacts ; then
+  if _kubectl get namespace cdi-testing-old-version-artifacts; then
     echo "Old version testing environment already setup"
     return
   fi
@@ -199,6 +199,9 @@ function setup_for_upgrade_testing {
 # Start functional test HTTP server.
 # We skip the functional test additions for external provider for now, as they're specific
 if [ "${CDI_SYNC}" == "test-infra" ]; then
+  if [ "${CDI_DEPLOY_NP}" == true ]; then
+    _kubectl apply -n ${CDI_NAMESPACE} -f "./cluster-sync/networkpolicies/cdi-testing-np.yaml"
+  fi
   configure_storage
   _kubectl apply -f "./_out/manifests/cdi-testing-sa.yaml"
   _kubectl apply -f "./_out/manifests/bad-webserver.yaml"
@@ -227,10 +230,11 @@ if [ "${CDI_SYNC}" == "test-infra" ]; then
     # vCenter (VDDK) test service:
     _kubectl apply -f "./_out/manifests/vcenter.yaml"
   fi
- 
+
   if _kubectl get crd securitycontextconstraints.security.openshift.io >/dev/null 2>&1; then
     _kubectl apply -f "./_out/manifests/cdi-testing-scc.yaml"
   fi
+
   exit 0
 fi
 
@@ -249,7 +253,7 @@ wait_cdi_crd_installed $CDI_INSTALL_TIMEOUT
 
 # If we are upgrading, verify our current value.
 if [[ ! -z "$UPGRADE_FROM" ]]; then
-  UPGRADE_FROM_LIST=( $UPGRADE_FROM )
+  UPGRADE_FROM_LIST=($UPGRADE_FROM)
   for VERSION in ${UPGRADE_FROM_LIST[@]}; do
     echo $VERSION
     if [ "$VERSION" != "${UPGRADE_FROM_LIST[0]}" ]; then
@@ -265,10 +269,10 @@ if [[ ! -z "$UPGRADE_FROM" ]]; then
     retry_counter=0
     kill_count=0
     while [[ $retry_counter -lt $CDI_UPGRADE_RETRY_COUNT ]] && [ "$operator_version" != "$VERSION" ]; do
-      cdi_cr_phase=`_kubectl get CDI -o=jsonpath='{.items[*].status.phase}{"\n"}'`
-      observed_version=`_kubectl get CDI -o=jsonpath='{.items[*].status.observedVersion}{"\n"}'`
-      target_version=`_kubectl get CDI -o=jsonpath='{.items[*].status.targetVersion}{"\n"}'`
-      operator_version=`_kubectl get CDI -o=jsonpath='{.items[*].status.operatorVersion}{"\n"}'`
+      cdi_cr_phase=$(_kubectl get CDI -o=jsonpath='{.items[*].status.phase}{"\n"}')
+      observed_version=$(_kubectl get CDI -o=jsonpath='{.items[*].status.observedVersion}{"\n"}')
+      target_version=$(_kubectl get CDI -o=jsonpath='{.items[*].status.targetVersion}{"\n"}')
+      operator_version=$(_kubectl get CDI -o=jsonpath='{.items[*].status.operatorVersion}{"\n"}')
       echo "Phase: $cdi_cr_phase, observedVersion: $observed_version, operatorVersion: $operator_version, targetVersion: $target_version"
       retry_counter=$((retry_counter + 1))
       if [[ $kill_count -lt 1 ]]; then
@@ -292,20 +296,20 @@ if [[ ! -z "$UPGRADE_FROM" ]]; then
   retry_counter=0
   _kubectl apply -f "./_out/manifests/release/cdi-operator.yaml"
   while [[ $retry_counter -lt $CDI_UPGRADE_RETRY_COUNT ]] && [ "$observed_version" != "latest" ]; do
-    cdi_cr_phase=`_kubectl get CDI -o=jsonpath='{.items[*].status.phase}{"\n"}'`
-    observed_version=`_kubectl get CDI -o=jsonpath='{.items[*].status.observedVersion}{"\n"}'`
-    target_version=`_kubectl get CDI -o=jsonpath='{.items[*].status.targetVersion}{"\n"}'`
-    operator_version=`_kubectl get CDI -o=jsonpath='{.items[*].status.operatorVersion}{"\n"}'`
+    cdi_cr_phase=$(_kubectl get CDI -o=jsonpath='{.items[*].status.phase}{"\n"}')
+    observed_version=$(_kubectl get CDI -o=jsonpath='{.items[*].status.observedVersion}{"\n"}')
+    target_version=$(_kubectl get CDI -o=jsonpath='{.items[*].status.targetVersion}{"\n"}')
+    operator_version=$(_kubectl get CDI -o=jsonpath='{.items[*].status.operatorVersion}{"\n"}')
     echo "Phase: $cdi_cr_phase, observedVersion: $observed_version, operatorVersion: $operator_version, targetVersion: $target_version"
     retry_counter=$((retry_counter + 1))
     _kubectl get pods -n $CDI_NAMESPACE
     sleep 5
   done
   if [ $retry_counter -eq $CDI_UPGRADE_RETRY_COUNT ]; then
-	  echo "Unable to deploy to latest version"
-	  cdi_obj=$(_kubectl get CDI -o yaml)
-	  echo $cdi_obj
-	  exit 1
+    echo "Unable to deploy to latest version"
+    cdi_obj=$(_kubectl get CDI -o yaml)
+    echo $cdi_obj
+    exit 1
   fi
   wait_cdi_available
   wait_cdi_pods_updated
@@ -323,6 +327,13 @@ if [ "${KUBEVIRT_PROVIDER}" != "external" ]; then
   configure_prometheus
 fi
 
+# Network policies
+if [ "${CDI_DEPLOY_NP}" == true ]; then
+  _kubectl apply -n ${CDI_NAMESPACE} -f "./cluster-sync/networkpolicies/cluster-services-np.yaml"
+  _kubectl apply -n ${CDI_NAMESPACE} -f "./_out/manifests/release/network-policies.yaml"
+fi
+
+
 # Grab all the CDI crds so we can check if they are structural schemas
 cdi_crds=$(_kubectl get crd -l cdi.kubevirt.io -o jsonpath={.items[*].metadata.name})
 crds=($cdi_crds)

hack/build/build-manifests.sh

@@ -35,6 +35,9 @@ mkdir -p "${MANIFEST_GENERATED_DIR}/"
 #generate operator related manifests used to deploy cdi with operator-framework
 generateResourceManifest $generator $MANIFEST_GENERATED_DIR "operator" "everything" "operator-everything.yaml.in"
 
+#generate networkpolicy manifests
+generateResourceManifest $generator $MANIFEST_GENERATED_DIR "namespaced" "networkpolicies" "network-policies.yaml.in"
+
 #process templated manifests and populate them with generated manifests
 tempDir=${MANIFEST_TEMPLATE_DIR}
 processDirTemplates ${tempDir} ${OUT_DIR}/manifests ${OUT_DIR}/manifests/templates ${generator} ${MANIFEST_GENERATED_DIR}

manifests/templates/release/network-policies.yaml.in

@@ -0,0 +1 @@
+{{index .GeneratedManifests "network-policies.yaml.in"}}