From f88b2f25e3a370108a3dee83608e6ac0e5d29917 Mon Sep 17 00:00:00 2001 From: meiza Date: Thu, 3 Sep 2020 01:57:48 -0700 Subject: [PATCH 1/2] Add automated security policy tests, update README --- go.mod | 5 +- go.sum | 21 ++++++ licenses/google.golang.org/protobuf/LICENSE | 27 +++++++ .../security/generate_policies/README.md | 50 +++++++++++++ .../generate_policies/testCases/Pipfile | 14 ++++ .../generate_policies/testCases/authZPaths.sh | 71 ++++++++++++++++++ .../testCases/authZPathsVariablePolicies.sh | 72 ++++++++++++++++++ .../testCases/authZSourceIP.sh | 73 ++++++++++++++++++ .../authZSourceIPVariablePolicies.sh | 73 ++++++++++++++++++ .../testCases/requestAuthN.sh | 74 +++++++++++++++++++ 10 files changed, 478 insertions(+), 2 deletions(-) create mode 100644 licenses/google.golang.org/protobuf/LICENSE mode change 100644 => 100755 perf/benchmark/security/generate_policies/README.md create mode 100644 perf/benchmark/security/generate_policies/testCases/Pipfile create mode 100755 perf/benchmark/security/generate_policies/testCases/authZPaths.sh create mode 100755 perf/benchmark/security/generate_policies/testCases/authZPathsVariablePolicies.sh create mode 100755 perf/benchmark/security/generate_policies/testCases/authZSourceIP.sh create mode 100755 perf/benchmark/security/generate_policies/testCases/authZSourceIPVariablePolicies.sh create mode 100755 perf/benchmark/security/generate_policies/testCases/requestAuthN.sh diff --git a/go.mod b/go.mod index 91a54c13f0..a2973ac322 100644 --- a/go.mod +++ b/go.mod @@ -26,8 +26,8 @@ require ( github.com/gliderlabs/ssh v0.2.2 // indirect github.com/gogo/protobuf v1.3.1 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b - github.com/golang/protobuf v1.3.5 - github.com/google/go-cmp v0.4.0 + github.com/golang/protobuf v1.4.1 + github.com/google/go-cmp v0.5.0 github.com/hhatto/gorst v0.0.0-20181029133204-ca9f730cac5b // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jdkato/prose v1.1.0 // indirect @@ -46,6 +46,7 @@ require ( github.com/xanzy/ssh-agent v0.2.0 // indirect golang.org/x/tools v0.0.0-20200113154838-30cae5f2fb06 gonum.org/v1/netlib v0.0.0-20191031114514-eccb95939662 // indirect + google.golang.org/protobuf v1.25.0 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/neurosnap/sentences.v1 v1.0.6 // indirect gopkg.in/russross/blackfriday.v2 v2.0.0 // indirect diff --git a/go.sum b/go.sum index 0928a50a84..f6d2301b6e 100644 --- a/go.sum +++ b/go.sum @@ -200,12 +200,22 @@ github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.5 h1:F768QJ1E9tib+q5Sc8MkdJi1RxLTbRcTf8LJV56aRls= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1 h1:ZFgWrT+bLgsYPirOnRfKLYJLvssAegOj/hgyMFdJZe0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -562,13 +572,24 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.28.1/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/licenses/google.golang.org/protobuf/LICENSE b/licenses/google.golang.org/protobuf/LICENSE new file mode 100644 index 0000000000..49ea0f9288 --- /dev/null +++ b/licenses/google.golang.org/protobuf/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2018 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/perf/benchmark/security/generate_policies/README.md b/perf/benchmark/security/generate_policies/README.md old mode 100644 new mode 100755 index b80d7d342a..f348b846f8 --- a/perf/benchmark/security/generate_policies/README.md +++ b/perf/benchmark/security/generate_policies/README.md @@ -216,6 +216,12 @@ The values which can be used to create custom RequestAuthentication are as follo For more information see [RequestAuthentication Reference](https://istio.io/latest/docs/reference/config/security/request_authentication/). +## Test Cases + +There are 4 automated scripts that generate policies, apply those policies, runs the performance tests on them, and then fetches the data of those tests from the fortio client. + +The cases are as follows: + ## Examples generate_policies.go also allows a user to create mutliple kinds of policies in one command. @@ -296,6 +302,50 @@ To apply largePolicy.yaml that was just created to istio use the following comma kubectl apply -f largePolicy.yaml ``` +## Test Cases + +There are 5 automated scripts that generate policies, apply those policies, runs the performance tests on them, and then fetches the data of those tests from the fortio client. + +Each script compares a single policy or a number of policies against another set of policies. This gives us easy access to data which we can use to compare the performance of two policies against each other. For each set of policies there are 3 different configurations of the performance tests. The first represents a small load on the system (qps=100, conn=8), the second represents a medium load (qps=500, conn=32), and finally the 3rd represents a large load (qps=1000, conn=64). + +The cases are as follows: + +1. To get the data of having a single AuthZ policy with 10 path rules as well as running the performance tests with a single AuthZ policy with 1000 path rules. + To run this test run the following commands + +```bash +cd testCases +./authZPaths.sh +``` + +2. To get the data of having a single AuthZ policy with 10 sourceIP rules as well as running the performance tests with a single AuthZ policy with 1000 sourceIP rules. + To run this test run the following commands in the testCases folder: + +```bash +./authZSourceIP.sh +``` + +3. To get the data of having 10 AuthZ policies each with 1 sourceIP rule as well as running the performance tests with 1000 AuthZ policies each with 1 sourceIP rule. + To run this test run the following commands in the testCases folder: + +```bash +./authZSourceIPVariable.sh +``` + +4. To get the data of having 10 AuthZ policies each with 1 paths rule as well as running the performance tests with 1000 AuthZ policies each with 1 path rule. + To run this test run the following commands in the testCases folder: + +```bash +./authZPathVariable.sh +``` + +5. To get the data of having 10 AuthZ policies each with 1 paths rule as well as running the performance tests with 1000 AuthZ policies each with 1 path rule. + To run this test run the following commands in the testCases folder: + +```bash +./authZPathVariable.sh +``` + ## Example 1 - By creating a config file called config.json with the following data, and then running the following command: diff --git a/perf/benchmark/security/generate_policies/testCases/Pipfile b/perf/benchmark/security/generate_policies/testCases/Pipfile new file mode 100644 index 0000000000..c55eb95d61 --- /dev/null +++ b/perf/benchmark/security/generate_policies/testCases/Pipfile @@ -0,0 +1,14 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] +requests = "*" +pytz = "*" +bokeh = "*" +pandas = "==0.24.2" +numpy = "*" +pyyaml = "*" diff --git a/perf/benchmark/security/generate_policies/testCases/authZPaths.sh b/perf/benchmark/security/generate_policies/testCases/authZPaths.sh new file mode 100755 index 0000000000..9215338760 --- /dev/null +++ b/perf/benchmark/security/generate_policies/testCases/authZPaths.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +# Copyright Istio Authors + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "Creating tests for authorizationPolicy with a variable number of path rules" +echo "Running each test in small load (conn=8, qps=100), medium load (conn=32, qps=500), and large load (conn=64, qps=1000)" + +echo ' +{ + "authZ":{ + "numPolicies":1, + "numPaths":10 + } +}' > authZPath10.json +go run ../generate_policies.go ../generate.go ../jwt.go -configFile="authZPath10.json" > authZPath10.yaml +echo "Generated a single authZ policy with 10 path rules" +kubectl apply -f authZPath10.yaml +echo "Running variable number of path rules" +echo "Running perf test with conn=8 and qps=100" +pipenv run python3 ../../../runner/runner.py --conn 8 --qps 100 --baseline --duration 240 --load_gen_type=nighthawk --telemetry_mode=none +echo "Running perf test with conn=32 and qps=500" +pipenv run python3 ../../../runner/runner.py --conn 32 --qps 500 --baseline --duration 240 --load_gen_type=nighthawk --telemetry_mode=none +echo "Running perf test with conn=64 and qps=1000" +pipenv run python3 ../../../runner/runner.py --conn 64 --qps 1000 --baseline --duration 240 --load_gen_type=nighthawk --telemetry_mode=none +echo "Removing policies" +kubectl delete -f authZPath10.yaml +rm authZPath10.json +rm authZPath10.yaml + +echo ' +{ + "authZ":{ + "numPolicies":1, + "numPaths":1000 + } +}' > authZPath1000.json +go run ../generate_policies.go ../generate.go ../jwt.go -configFile="authZPath1000.json" > authZPath1000.yaml +echo "Generated a single authZ policy with 1000 paths" +kubectl apply -f authZSourceIP1000.yaml +echo "Running perf test with conn=8 and qps=100" +pipenv run python3 ../../../runner/runner.py --conn 8 --qps 100 --baseline --duration 240 --load_gen_type=nighthawk --telemetry_mode=none +echo "Running perf test with conn=32 and qps=500" +pipenv run python3 ../../../runner/runner.py --conn 32 --qps 500 --baseline --duration 240 --load_gen_type=nighthawk --telemetry_mode=none +echo "Running perf test with conn=64 and qps=1000" +pipenv run python3 ../../../runner/runner.py --conn 64 --qps 1000 --baseline --duration 240 --load_gen_type=nighthawk --telemetry_mode=none +echo "Removing policies" +kubectl delete -f authZPath1000.yaml +rm authZPath1000.json +rm authZPath1000.yaml + +echo "Fetching data" +FORTIO_CLIENT_URL=http://$(kubectl get services -n twopods-istio fortioclient -o jsonpath="{.status.loadBalancer.ingress[0].ip}"):9076 +kubectl -n istio-prometheus port-forward svc/istio-prometheus 9090:9090 & +PROMETHEUS_URL=http://localhost:9090 +python3 ./../../../runner/fortio.py "$FORTIO_CLIENT_URL" --prometheus=$PROMETHEUS_URL --csv StartTime,ActualDuration,Labels,NumThreads,ActualQPS,p50,p90,p99,cpu_mili_avg_istio_proxy_fortioclient,cpu_mili_avg_istio_proxy_fortioserver,cpu_mili_avg_istio_proxy_istio-ingressgateway,mem_Mi_avg_istio_proxy_fortioclient,mem_Mi_avg_istio_proxy_fortioserver,mem_Mi_avg_istio_proxy_istio-ingressgateway + +echo "Cleanup started" +kubectl delete --all pods --namespace=twopods-istio +echo "Cleanup finished, data collected" diff --git a/perf/benchmark/security/generate_policies/testCases/authZPathsVariablePolicies.sh b/perf/benchmark/security/generate_policies/testCases/authZPathsVariablePolicies.sh new file mode 100755 index 0000000000..397cc76a3f --- /dev/null +++ b/perf/benchmark/security/generate_policies/testCases/authZPathsVariablePolicies.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +# Copyright Istio Authors + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "Creating tests for authorizationPolicy with a variable number of paths" +echo "Running each test in small load (conn=8, qps=100), medium load (conn=32, qps=500), and large load (conn=64, qps=1000)" + +echo ' +{ + "authZ":{ + "numPolicies":10, + "numPaths":1 + } +}' > authZ10Path1.json +go run ../generate_policies.go ../generate.go ../jwt.go -configFile="authZ10Path1.json" > authZ10Path.yaml +echo "Generated 10 authZ policy with 1 path rule each" +kubectl apply -f authZ10Path.yaml +echo "Running variable number of policies" +echo "Running perf test with conn=8 and qps=100" +pipenv run python3 ../../../runner/runner.py --conn 8 --qps 100 --baseline --duration 240 --load_gen_type=nighthawk --telemetry_mode=none +echo "Running perf test with conn=32 and qps=500" +pipenv run python3 ../../../runner/runner.py --conn 32 --qps 500 --baseline --duration 240 --load_gen_type=nighthawk --telemetry_mode=none +echo "Running perf test with conn=64 and qps=1000" +pipenv run python3 ../../../runner/runner.py --conn 64 --qps 1000 --baseline --duration 240 --load_gen_type=nighthawk --telemetry_mode=none +echo "Removing policies" +kubectl delete -f authZ10Path.yaml +rm authZ10Path1.json +rm authZ10Path.yaml + +echo ' +{ + "authZ": + { + "numPolicies":1000, + "numPaths":1 + } +}' > authZ1000Path1.json +go run ../generate_policies.go ../generate.go ../jwt.go -configFile="authZ1000Path1.json" > authZ1000Path.yaml +echo "Generated 1000 authZ policy with 1 path each" +kubectl apply -f authZ1000Path.yaml +echo "Running perf test with conn=8 and qps=100" +pipenv run python3 ../../../runner/runner.py --conn 8 --qps 100 --baseline --duration 240 --load_gen_type=nighthawk --telemetry_mode=none +echo "Running perf test with conn=32 and qps=500" +pipenv run python3 ../../../runner/runner.py --conn 32 --qps 500 --baseline --duration 240 --load_gen_type=nighthawk --telemetry_mode=none +echo "Running perf test with conn=64 and qps=1000" +pipenv run python3 ../../../runner/runner.py --conn 64 --qps 1000 --baseline --duration 240 --load_gen_type=nighthawk --telemetry_mode=none +echo "Removing policies" +kubectl delete -f authZ1000Path.yaml +rm authZ1000Path.yaml +rm authZ1000Path1.json + +echo "Fetching data" +FORTIO_CLIENT_URL=http://$(kubectl get services -n twopods-istio fortioclient -o jsonpath="{.status.loadBalancer.ingress[0].ip}"):9076 +kubectl -n istio-prometheus port-forward svc/istio-prometheus 9090:9090 & +PROMETHEUS_URL=http://localhost:9090 +python3 ./../../../runner/fortio.py "$FORTIO_CLIENT_URL" --prometheus=$PROMETHEUS_URL --csv StartTime,ActualDuration,Labels,NumThreads,ActualQPS,p50,p90,p99,cpu_mili_avg_istio_proxy_fortioclient,cpu_mili_avg_istio_proxy_fortioserver,cpu_mili_avg_istio_proxy_istio-ingressgateway,mem_Mi_avg_istio_proxy_fortioclient,mem_Mi_avg_istio_proxy_fortioserver,mem_Mi_avg_istio_proxy_istio-ingressgateway + +echo "Cleanup started" +kubectl delete --all pods --namespace=twopods-istio +echo "Cleanup finished, data collected" diff --git a/perf/benchmark/security/generate_policies/testCases/authZSourceIP.sh b/perf/benchmark/security/generate_policies/testCases/authZSourceIP.sh new file mode 100755 index 0000000000..cb05c5ed4d --- /dev/null +++ b/perf/benchmark/security/generate_policies/testCases/authZSourceIP.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +# Copyright Istio Authors + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "Creating tests for authorizationPolicy with a variable number of sourceIP rules (10 vs 1000)" +echo "Running each test in small load (conn=8, qps=100), medium load (conn=32, qps=500), and large load (conn=64, qps=1000)" + +echo ' +{ + "authZ": + { + "numPolicies":1, + "numSourceIP":10 + } +}' > authZSourceIP10.json +go run ../../generate_policies.go ../../generate.go ../../jwt.go -configFile="authZSourceIP10.json" > authZSourceIP10.yaml +echo "Generated a single authZ policy with 10 sourceIP rules" +kubectl apply -f authZSourceIP10.yaml +echo "Running variable number of sourceIP rules" +echo "Running perf test with conn=8 and qps=100" +pipenv run python3 ../../../runner/runner.py --conn 8 --qps 100 --baseline --duration 240 --load_gen_type=nighthawk --telemetry_mode=none +echo "Running perf test with conn=32 and qps=500" +pipenv run python3 ../../../runner/runner.py --conn 32 --qps 500 --baseline --duration 240 --load_gen_type=nighthawk --telemetry_mode=none +echo "Running perf test with conn=64 and qps=1000" +pipenv run python3 ../../../runner/runner.py --conn 64 --qps 1000 --baseline --duration 240 --load_gen_type=nighthawk --telemetry_mode=none +echo "Removing policies" +kubectl delete -f authZSourceIP10.yaml +rm authZSourceIP10.json +rm authZSourceIP10.yaml + +echo ' +{ + "authZ": + { + "numPolicies":1, + "numSourceIP":1000 + } +}' > authZSourceIP1000.json +go run ../../generate_policies.go ../../generate.go ../../jwt.go -configFile="authZSourceIP1000.json" > authZSourceIP1000.yaml +echo "Generated a single authZ policy with 1000 paths" +kubectl apply -f authZSourceIP1000.yaml +echo "Running perf test with conn=8 and qps=100" +pipenv run python3 ../../../runner/runner.py --conn 8 --qps 100 --baseline --duration 240 --load_gen_type=nighthawk --telemetry_mode=none +echo "Running perf test with conn=32 and qps=500" +pipenv run python3 ../../../runner/runner.py --conn 32 --qps 500 --baseline --duration 240 --load_gen_type=nighthawk --telemetry_mode=none +echo "Running perf test with conn=64 and qps=1000" +pipenv run python3 ../../../runner/runner.py --conn 64 --qps 1000 --baseline --duration 240 --load_gen_type=nighthawk --telemetry_mode=none +echo "Removing policies" +kubectl delete -f authZSourceIP1000.yaml +rm authZSourceIP1000.json +rm authZSourceIP1000.yaml + +echo "Fetching data" +FORTIO_CLIENT_URL=http://$(kubectl get services -n twopods-istio fortioclient -o jsonpath="{.status.loadBalancer.ingress[0].ip}"):9076 +kubectl -n istio-prometheus port-forward svc/istio-prometheus 9090:9090 & +PROMETHEUS_URL=http://localhost:9090 +python3 ./../../../runner/fortio.py "$FORTIO_CLIENT_URL" --prometheus=$PROMETHEUS_URL --csv StartTime,ActualDuration,Labels,NumThreads,ActualQPS,p50,p90,p99,cpu_mili_avg_istio_proxy_fortioclient,cpu_mili_avg_istio_proxy_fortioserver,cpu_mili_avg_istio_proxy_istio-ingressgateway,mem_Mi_avg_istio_proxy_fortioclient,mem_Mi_avg_istio_proxy_fortioserver,mem_Mi_avg_istio_proxy_istio-ingressgateway + +echo "Cleanup started" +kubectl delete --all pods --namespace=twopods-istio +echo "Cleanup finished, data collected" diff --git a/perf/benchmark/security/generate_policies/testCases/authZSourceIPVariablePolicies.sh b/perf/benchmark/security/generate_policies/testCases/authZSourceIPVariablePolicies.sh new file mode 100755 index 0000000000..b6ef05465d --- /dev/null +++ b/perf/benchmark/security/generate_policies/testCases/authZSourceIPVariablePolicies.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +# Copyright Istio Authors + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "Creating tests for authorizationPolicy with a variable number of policies each with 1 sourceIP rule" +echo "Running each test in small load (conn=8, qps=100), medium load (conn=32, qps=500), and large load (conn=64, qps=1000)" + +echo ' +{ + "authZ": + { + "numPolicies":10, + "numSourceIP":1 + } +}' > authZ10SourceIP.json +go run ../generate_policies.go ../generate.go ../jwt.go -configFile="authZ10SourceIP.json" > authZ10SourceIP.yaml +echo "Generated 10 authZ policy with 1 sourceIP rule each" +kubectl apply -f authZ10SourceIP.yaml +echo "Running variable number of policies" +echo "Running perf test with conn=8 and qps=100" +pipenv run python3 ../../../runner/runner.py --conn 8 --qps 100 --baseline --duration 240 --load_gen_type=nighthawk --telemetry_mode=none +echo "Running perf test with conn=32 and qps=500" +pipenv run python3 ../../../runner/runner.py --conn 32 --qps 500 --baseline --duration 240 --load_gen_type=nighthawk --telemetry_mode=none +echo "Running perf test with conn=64 and qps=1000" +pipenv run python3 ../../../runner/runner.py --conn 64 --qps 1000 --baseline --duration 240 --load_gen_type=nighthawk --telemetry_mode=none +echo "Removing policies" +kubectl delete -f authZ10SourceIP.yaml +rm authZ10SourceIP.json +rm authZ10SourceIP.yaml + +echo ' +{ + "authZ": + { + "numPolicies":1000, + "numSourceIP":1 + } +}' > authZ1000SourceIP.json +go run ../generate_policies.go ../generate.go ../jwt.go -configFile="authZ1000SourceIP.json" > authZ1000SourceIP.yaml +echo "Generated 1000 authZ policy with 1 sourceIP rule each" +kubectl apply -f authZ1000SourceIP.yaml +echo "Running perf test with conn=8 and qps=100" +pipenv run python3 ../../../runner/runner.py --conn 8 --qps 100 --baseline --duration 240 --load_gen_type=nighthawk --telemetry_mode=none +echo "Running perf test with conn=32 and qps=500" +pipenv run python3 ../../../runner/runner.py --conn 32 --qps 500 --baseline --duration 240 --load_gen_type=nighthawk --telemetry_mode=none +echo "Running perf test with conn=64 and qps=1000" +pipenv run python3 ../../../runner/runner.py --conn 64 --qps 1000 --baseline --duration 240 --load_gen_type=nighthawk --telemetry_mode=none +echo "Removing policies" +kubectl delete -f authZ1000SourceIP.yaml +rm authZ1000SourceIP.json +rm authZ1000SourceIP.yaml + +echo "Fetching data" +FORTIO_CLIENT_URL=http://$(kubectl get services -n twopods-istio fortioclient -o jsonpath="{.status.loadBalancer.ingress[0].ip}"):9076 +kubectl -n istio-prometheus port-forward svc/istio-prometheus 9090:9090 & +PROMETHEUS_URL=http://localhost:9090 +python3 ./../../../runner/fortio.py "$FORTIO_CLIENT_URL" --prometheus=$PROMETHEUS_URL --csv StartTime,ActualDuration,Labels,NumThreads,ActualQPS,p50,p90,p99,cpu_mili_avg_istio_proxy_fortioclient,cpu_mili_avg_istio_proxy_fortioserver,cpu_mili_avg_istio_proxy_istio-ingressgateway,mem_Mi_avg_istio_proxy_fortioclient,mem_Mi_avg_istio_proxy_fortioserver,mem_Mi_avg_istio_proxy_istio-ingressgateway + +echo "Cleanup started" +kubectl delete --all pods --namespace=twopods-istio +echo "Cleanup finished, data collected" diff --git a/perf/benchmark/security/generate_policies/testCases/requestAuthN.sh b/perf/benchmark/security/generate_policies/testCases/requestAuthN.sh new file mode 100755 index 0000000000..d2e78d8a33 --- /dev/null +++ b/perf/benchmark/security/generate_policies/testCases/requestAuthN.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +# Copyright Istio Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "Creating tests for requestAuthentication with a variable number of jwks (1 vs 10)" +echo "Running each test in small load (conn=8, qps=100), medium load (conn=32, qps=500), and large load (conn=64, qps=1000)" + +echo ' +{ + "requestAuthN": + { + "numPolicies":1, + "numJwks":1 + } +}' > requestAuthN1jwks.json +go run ../generate_policies.go ../generate.go ../jwt.go -configFile="requestAuthN1jwks.json" > requestAuthN1jwks.yaml +echo "generated requestAuthN policy with 1 jwks rules" +kubectl apply -f requestAuthN1jwks.yaml +echo "Running variable number of jwks rules" +echo "Running perf test with conn=8 and qps=100" +pipenv run python3 ../../../runner/runner.py --conn 8 --qps 100 --baseline --duration 240 --load_gen_type=nighthawk --telemetry_mode=none +echo "Running perf test with conn=32 and qps=500" +pipenv run python3 ../../../runner/runner.py --conn 32 --qps 500 --baseline --duration 240 --load_gen_type=nighthawk --telemetry_mode=none +echo "Running perf test with conn=64 and qps=1000" +pipenv run python3 ../../../runner/runner.py --conn 64 --qps 1000 --baseline --duration 240 --load_gen_type=nighthawk --telemetry_mode=none +echo "Removing policies" +kubectl delete -f requestAuthN1jwks.yaml +rm requestAuthN1jwks.json +rm requestAuthN1jwks.yaml + +echo ' +{ + "requestAuthN": + { + "numPolicies":1, + "numJwks":10 + } +}' > requestAuthN10jwks.json +go run ../generate_policies.go ../generate.go ../jwt.go -configFile="requestAuthN10jwks.json" > requestAuthN10jwks.yaml +echo "generated requestAuthN policy with 10 jwks rules" +kubectl apply -f requestAuthN10jwks.yaml +echo "Running perf test with conn=8 and qps=100" +pipenv run python3 ../../../runner/runner.py --conn 8 --qps 100 --baseline --duration 240 --load_gen_type=nighthawk --telemetry_mode=none +echo "Running perf test with conn=32 and qps=500" +pipenv run python3 ../../../runner/runner.py --conn 32 --qps 500 --baseline --duration 240 --load_gen_type=nighthawk --telemetry_mode=none +echo "Running perf test with conn=64 and qps=1000" +pipenv run python3 ../../../runner/runner.py --conn 64 --qps 1000 --baseline --duration 240 --load_gen_type=nighthawk --telemetry_mode=none +echo "Removing policies" +kubectl delete -f requestAuthN10jwks.yaml +rm requestAuthN10jwks.json +rm requestAuthN10jwks.yaml +rm token.txt + +echo "Fetching data" +FORTIO_CLIENT_URL=http://$(kubectl get services -n twopods-istio fortioclient -o jsonpath="{.status.loadBalancer.ingress[0].ip}"):9076 +kubectl -n istio-prometheus port-forward svc/istio-prometheus 9090:9090 & +PROMETHEUS_URL=http://localhost:9090 +python3 ./../../../runner/fortio.py "$FORTIO_CLIENT_URL" --prometheus=$PROMETHEUS_URL --csv StartTime,ActualDuration,Labels,NumThreads,ActualQPS,p50,p90,p99,cpu_mili_avg_istio_proxy_fortioclient,cpu_mili_avg_istio_proxy_fortioserver,cpu_mili_avg_istio_proxy_istio-ingressgateway,mem_Mi_avg_istio_proxy_fortioclient,mem_Mi_avg_istio_proxy_fortioserver,mem_Mi_avg_istio_proxy_istio-ingressgateway + +echo "Cleanup started" +kubectl delete --all pods --namespace=twopods-istio +echo "Cleanup finished, data collected" From 99a2f6e523281d8300b4a90ab324b5d5f9ba24e0 Mon Sep 17 00:00:00 2001 From: meiza Date: Thu, 3 Sep 2020 02:04:58 -0700 Subject: [PATCH 2/2] update position of Test Cases in README --- .../security/generate_policies/README.md | 94 +++++++++---------- 1 file changed, 44 insertions(+), 50 deletions(-) diff --git a/perf/benchmark/security/generate_policies/README.md b/perf/benchmark/security/generate_policies/README.md index f348b846f8..5b667062dc 100755 --- a/perf/benchmark/security/generate_policies/README.md +++ b/perf/benchmark/security/generate_policies/README.md @@ -216,12 +216,6 @@ The values which can be used to create custom RequestAuthentication are as follo For more information see [RequestAuthentication Reference](https://istio.io/latest/docs/reference/config/security/request_authentication/). -## Test Cases - -There are 4 automated scripts that generate policies, apply those policies, runs the performance tests on them, and then fetches the data of those tests from the fortio client. - -The cases are as follows: - ## Examples generate_policies.go also allows a user to create mutliple kinds of policies in one command. @@ -302,50 +296,6 @@ To apply largePolicy.yaml that was just created to istio use the following comma kubectl apply -f largePolicy.yaml ``` -## Test Cases - -There are 5 automated scripts that generate policies, apply those policies, runs the performance tests on them, and then fetches the data of those tests from the fortio client. - -Each script compares a single policy or a number of policies against another set of policies. This gives us easy access to data which we can use to compare the performance of two policies against each other. For each set of policies there are 3 different configurations of the performance tests. The first represents a small load on the system (qps=100, conn=8), the second represents a medium load (qps=500, conn=32), and finally the 3rd represents a large load (qps=1000, conn=64). - -The cases are as follows: - -1. To get the data of having a single AuthZ policy with 10 path rules as well as running the performance tests with a single AuthZ policy with 1000 path rules. - To run this test run the following commands - -```bash -cd testCases -./authZPaths.sh -``` - -2. To get the data of having a single AuthZ policy with 10 sourceIP rules as well as running the performance tests with a single AuthZ policy with 1000 sourceIP rules. - To run this test run the following commands in the testCases folder: - -```bash -./authZSourceIP.sh -``` - -3. To get the data of having 10 AuthZ policies each with 1 sourceIP rule as well as running the performance tests with 1000 AuthZ policies each with 1 sourceIP rule. - To run this test run the following commands in the testCases folder: - -```bash -./authZSourceIPVariable.sh -``` - -4. To get the data of having 10 AuthZ policies each with 1 paths rule as well as running the performance tests with 1000 AuthZ policies each with 1 path rule. - To run this test run the following commands in the testCases folder: - -```bash -./authZPathVariable.sh -``` - -5. To get the data of having 10 AuthZ policies each with 1 paths rule as well as running the performance tests with 1000 AuthZ policies each with 1 path rule. - To run this test run the following commands in the testCases folder: - -```bash -./authZPathVariable.sh -``` - ## Example 1 - By creating a config file called config.json with the following data, and then running the following command: @@ -452,3 +402,47 @@ To remove the policies applied navigate to the generate_policies folder and run ```bash kubectl delete -f largePolicy.yaml ``` + +## Test Cases + +There are 5 automated scripts that generate policies, apply those policies, runs the performance tests on them, and then fetches the data of those tests from the fortio client. + +Each script compares a single policy or a number of policies against another set of policies. This gives us easy access to data which we can use to compare the performance of two policies against each other. For each set of policies there are 3 different configurations of the performance tests. The first represents a small load on the system (qps=100, conn=8), the second represents a medium load (qps=500, conn=32), and finally the 3rd represents a large load (qps=1000, conn=64). + +The cases are as follows: + +1. To get the data of having a single AuthZ policy with 10 path rules as well as running the performance tests with a single AuthZ policy with 1000 path rules. + To run this test run the following commands + +```bash +cd testCases +./authZPaths.sh +``` + +2. To get the data of having a single AuthZ policy with 10 sourceIP rules as well as running the performance tests with a single AuthZ policy with 1000 sourceIP rules. + To run this test run the following commands in the testCases folder: + +```bash +./authZSourceIP.sh +``` + +3. To get the data of having 10 AuthZ policies each with 1 sourceIP rule as well as running the performance tests with 1000 AuthZ policies each with 1 sourceIP rule. + To run this test run the following commands in the testCases folder: + +```bash +./authZSourceIPVariable.sh +``` + +4. To get the data of having 10 AuthZ policies each with 1 paths rule as well as running the performance tests with 1000 AuthZ policies each with 1 path rule. + To run this test run the following commands in the testCases folder: + +```bash +./authZPathVariable.sh +``` + +5. To get the data of having 10 AuthZ policies each with 1 paths rule as well as running the performance tests with 1000 AuthZ policies each with 1 path rule. + To run this test run the following commands in the testCases folder: + +```bash +./authZPathVariable.sh +```