diff --git a/go.mod b/go.mod index 2895da57..dc79737f 100644 --- a/go.mod +++ b/go.mod @@ -23,17 +23,13 @@ require ( k8s.io/kubernetes v0.0.0-00010101000000-000000000000 k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 kusionstack.io/kube-api v0.7.4-0.20250909095208-496f60eea9b5 - kusionstack.io/kube-utils v0.2.1-0.20251031115408-8f00ba110277 + kusionstack.io/kube-utils v0.2.1-0.20251120063041-6043805ee00d + kusionstack.io/kube-xset v0.0.0-20251111060928-069410a15229 kusionstack.io/resourceconsist v0.0.1 sigs.k8s.io/controller-runtime v0.17.3 ) require ( - github.com/Azure/go-autorest/autorest v0.11.18 // indirect - github.com/Azure/go-autorest/autorest/adal v0.9.13 // indirect - github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect - github.com/Azure/go-autorest/logger v0.2.1 // indirect - github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/docker/distribution v2.8.2+incompatible // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect @@ -41,8 +37,6 @@ require ( ) require ( - cloud.google.com/go v0.65.0 // indirect - github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.4 // indirect github.com/alibabacloud-go/darabonba-openapi/v2 v2.0.4 // indirect github.com/alibabacloud-go/debug v0.0.0-20190504072949-9472017b5c68 // indirect @@ -59,7 +53,6 @@ require ( github.com/clbanning/mxj/v2 v2.5.5 // indirect github.com/cyphar/filepath-securejoin v0.2.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logr/zapr v1.2.4 // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -81,7 +74,6 @@ require ( github.com/tjfoc/gmsm v1.3.2 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect - golang.org/x/crypto v0.26.0 // indirect golang.org/x/oauth2 v0.15.0 // indirect golang.org/x/sys v0.23.0 // indirect golang.org/x/term v0.23.0 // indirect diff --git a/go.sum b/go.sum index 0479a26d..715823f5 100644 --- a/go.sum +++ b/go.sum @@ -11,50 +11,32 @@ cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gc cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0 h1:Dg9iHVQfrhq82rUNu9ZxUDrJLaxFUe/HlCVaLyRruq8= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go v55.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.18 h1:90Y4srNYrwOtAgVo3ndrQkTYn6kf1Eg/AjTFJ8Is2aM= github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= -github.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZtoxRXqUEzCfJt3+/Q= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= -github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= github.com/Azure/go-autorest/autorest/validation v0.1.0/go.mod h1:Ha3z/SqBeaalWQvokg3NZAlQTalVMtOIAs1aGK7G6u8= -github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -221,7 +203,6 @@ github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSw github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -275,14 +256,12 @@ github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -305,9 +284,7 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -319,14 +296,11 @@ github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -615,10 +589,8 @@ github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.30/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -635,7 +607,6 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= @@ -677,8 +648,6 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= -golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= -golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -739,7 +708,6 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -750,12 +718,8 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -786,7 +750,6 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -830,16 +793,10 @@ golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200509044756-6aff5f38e54f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -933,20 +890,10 @@ golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= @@ -975,19 +922,12 @@ google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/api v0.15.1-0.20200106000736-b8fc810ca6b5/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= @@ -1009,20 +949,10 @@ google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= @@ -1036,10 +966,7 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= @@ -1102,7 +1029,6 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.22.6 h1:acjE5ABt0KpsBI9QCtLqaQEPSF94jOtE/LoFxSYasSE= k8s.io/api v0.22.6/go.mod h1:q1F7IfaNrbi/83ebLy3YFQYLjPSNyunZ/IXQxMmbwCg= k8s.io/apiextensions-apiserver v0.22.6 h1:TH+9+EGtoVzzbrlfSDnObzFTnyXKqw1NBfT5XFATeJI= @@ -1153,8 +1079,10 @@ k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6J k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= kusionstack.io/kube-api v0.7.4-0.20250909095208-496f60eea9b5 h1:/jbKYMeXiYnuxyJQs72MoL6vxVoY15FX/7tCKWljscY= kusionstack.io/kube-api v0.7.4-0.20250909095208-496f60eea9b5/go.mod h1:e1jtrQH2LK5fD2nTyfIXG6nYrYbU8VXShRxTRwVPaLk= -kusionstack.io/kube-utils v0.2.1-0.20251031115408-8f00ba110277 h1:jsaRBLVOfkpAgPz1JA0Cp+fs7tqi+9Uccu0JJlCeaO0= -kusionstack.io/kube-utils v0.2.1-0.20251031115408-8f00ba110277/go.mod h1:KEHTfo1Y8SWMODnckF6daO2cSIW0FJ8fkk8PBA5O2GU= +kusionstack.io/kube-utils v0.2.1-0.20251120063041-6043805ee00d h1:iQtnK03ia/MN4K/6O75EMI91ep7jpcIG0pWyeREBqtg= +kusionstack.io/kube-utils v0.2.1-0.20251120063041-6043805ee00d/go.mod h1:KEHTfo1Y8SWMODnckF6daO2cSIW0FJ8fkk8PBA5O2GU= +kusionstack.io/kube-xset v0.0.0-20251111060928-069410a15229 h1:fTXG+HNxgM982pU/SnETY1fTRIcrrovs0Apn6h//zIo= +kusionstack.io/kube-xset v0.0.0-20251111060928-069410a15229/go.mod h1:FceKgqapMHhwiyIqCziTQRW27fsSXpPS611AApnyiYI= kusionstack.io/resourceconsist v0.0.1 h1:+k/jriq5Ld7fQUYfWSMGynz/FesHtl3Rk2fmQPjBe0g= kusionstack.io/resourceconsist v0.0.1/go.mod h1:816xS/fY6EOUbPFjXIWW/TGs8/YE46qP4ElKeIiwFdU= modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= diff --git a/main.go b/main.go index d1e729e2..46855fb3 100644 --- a/main.go +++ b/main.go @@ -22,20 +22,18 @@ import ( "os" "path/filepath" - _ "k8s.io/client-go/plugin/pkg/client/auth" - "github.com/spf13/pflag" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/klog/v2" "k8s.io/klog/v2/klogr" appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" + xsetfeatures "kusionstack.io/kube-xset/features" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "kusionstack.io/kuperator/pkg/controllers" "kusionstack.io/kuperator/pkg/controllers/operationjob" - _ "kusionstack.io/kuperator/pkg/features" "kusionstack.io/kuperator/pkg/utils/feature" "kusionstack.io/kuperator/pkg/utils/inject" "kusionstack.io/kuperator/pkg/webhook" @@ -71,6 +69,7 @@ func main() { defer klog.Flush() feature.DefaultMutableFeatureGate.AddFlag(pflag.CommandLine) + xsetfeatures.DefaultMutableFeatureGate.AddFlag(pflag.CommandLine) pflag.CommandLine.AddGoFlagSet(flag.CommandLine) pflag.Parse() diff --git a/pkg/controllers/collaset/collaset_controller.go b/pkg/controllers/collaset/collaset_controller.go index cb2d8e05..0c41a73b 100644 --- a/pkg/controllers/collaset/collaset_controller.go +++ b/pkg/controllers/collaset/collaset_controller.go @@ -1,5 +1,5 @@ /* -Copyright 2023 The KusionStack Authors. +Copyright 2025 The KusionStack Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,38 +18,20 @@ package collaset import ( "context" - "fmt" - "time" + "encoding/json" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/equality" - "k8s.io/apimachinery/pkg/api/errors" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/clock" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" - kubeutilsclient "kusionstack.io/kube-utils/client" - kubeutilsexpectations "kusionstack.io/kube-utils/controller/expectations" - "kusionstack.io/kube-utils/controller/history" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" - - "kusionstack.io/kuperator/pkg/controllers/collaset/podcontext" - "kusionstack.io/kuperator/pkg/controllers/collaset/podcontrol" - "kusionstack.io/kuperator/pkg/controllers/collaset/pvccontrol" - "kusionstack.io/kuperator/pkg/controllers/collaset/synccontrol" + kubexset "kusionstack.io/kube-xset" + xsetapi "kusionstack.io/kube-xset/api" + "kusionstack.io/kube-xset/synccontrols" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + "kusionstack.io/kuperator/pkg/controllers/collaset/utils" - collasetutils "kusionstack.io/kuperator/pkg/controllers/collaset/utils" controllerutils "kusionstack.io/kuperator/pkg/controllers/utils" - utilspoddecoration "kusionstack.io/kuperator/pkg/controllers/utils/poddecoration" - "kusionstack.io/kuperator/pkg/controllers/utils/poddecoration/strategy" - "kusionstack.io/kuperator/pkg/controllers/utils/podopslifecycle" - commonutils "kusionstack.io/kuperator/pkg/utils" - "kusionstack.io/kuperator/pkg/utils/mixin" ) const ( @@ -58,370 +40,271 @@ const ( preReclaimFinalizer = "apps.kusionstack.io/pre-reclaim" ) -// CollaSetReconciler reconciles a CollaSet object -type CollaSetReconciler struct { - *mixin.ReconcilerMixin - revisionManager history.HistoryManager - syncControl synccontrol.Interface - pvcControl pvccontrol.Interface - podContextControl podcontext.Interface - cacheExpectations kubeutilsexpectations.CacheExpectationsInterface +func Add(mgr manager.Manager) error { + xSetController := &CollaSetController{} + synccontrols.RegisterInPlaceIfPossibleUpdater(&inPlaceIfPossibleUpdater{}) + return kubexset.SetUpWithManager(mgr, xSetController) } -func Add(mgr ctrl.Manager) error { - return AddToMgr(mgr, NewReconciler(mgr)) -} +var _ xsetapi.XSetController = &CollaSetController{} -// NewReconciler returns a new reconcile.Reconciler -func NewReconciler(mgr ctrl.Manager) reconcile.Reconciler { - mixin := mixin.NewReconcilerMixin(controllerName, mgr) - - cacheExpectations := kubeutilsexpectations.NewxCacheExpectations(mixin.Client, mixin.Scheme, clock.RealClock{}) - pvcControl := pvccontrol.NewRealPvcControl(mixin.Client, mixin.Scheme, cacheExpectations) - podContextControl := podcontext.NewRealPodContextControl(mixin.Client, cacheExpectations) - syncControl := synccontrol.NewRealSyncControl(mixin.Client, mixin.Logger, podcontrol.NewRealPodControl(mixin.Client, mixin.Scheme), pvcControl, podContextControl, mixin.Recorder, cacheExpectations) - revisionManager := history.NewHistoryManager(history.NewRevisionControl(mixin.Client, mixin.Client), &revisionOwnerAdapter{podControl: podcontrol.NewRealPodControl(mixin.Client, mixin.Scheme)}) - - return &CollaSetReconciler{ - ReconcilerMixin: mixin, - revisionManager: revisionManager, - syncControl: syncControl, - pvcControl: pvcControl, - podContextControl: podContextControl, - cacheExpectations: cacheExpectations, - } +type CollaSetController struct { + XSetOperation + XOperation + LifecycleAdapterGetter + GetLabelManagerAdapterGetter + SubResourcePvcAdapter + DecorationAdapter + ResourceContextAdapterGetter } -func AddToMgr(mgr ctrl.Manager, r reconcile.Reconciler) error { - // Create a new controller - c, err := controller.New(controllerName, mgr, controller.Options{ - MaxConcurrentReconciles: 5, - Reconciler: r, - }) - if err != nil { - return err - } - - err = c.Watch(&source.Kind{Type: &appsv1alpha1.CollaSet{}}, &handler.EnqueueRequestForObject{}) - if err != nil { - return err - } +func (m *CollaSetController) ControllerName() string { + return controllerName +} - // Only for starting SharedStrategyController - err = c.Watch(strategy.SharedStrategyController, &handler.Funcs{}) - if err != nil { - return err - } +func (m *CollaSetController) FinalizerName() string { + return preReclaimFinalizer +} - ch := make(chan event.GenericEvent, 1<<10) - strategy.SharedStrategyController.RegisterGenericEventChannel(ch) - // Watch PodDecoration related events - err = c.Watch(&source.Channel{Source: ch}, &handler.EnqueueRequestForObject{}) - if err != nil { - return err - } +func (m *CollaSetController) XSetMeta() metav1.TypeMeta { + return metav1.TypeMeta{APIVersion: appsv1alpha1.SchemeGroupVersion.String(), Kind: "CollaSet"} +} - err = c.Watch(&source.Kind{Type: &corev1.Pod{}}, &handler.EnqueueRequestForOwner{ - IsController: true, - OwnerType: &appsv1alpha1.CollaSet{}, - }, &PodPredicate{}) - if err != nil { - return err - } +func (m *CollaSetController) XMeta() metav1.TypeMeta { + return metav1.TypeMeta{APIVersion: corev1.SchemeGroupVersion.String(), Kind: "Pod"} +} - return nil +func (m *CollaSetController) NewXSetObject() xsetapi.XSetObject { + return &appsv1alpha1.CollaSet{} } -// +kubebuilder:rbac:groups=apps.kusionstack.io,resources=collasets,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=apps.kusionstack.io,resources=collasets/status,verbs=get;update;patch -// +kubebuilder:rbac:groups=apps.kusionstack.io,resources=collasets/finalizers,verbs=update -// +kubebuilder:rbac:groups=apps.kusionstack.io,resources=resourcecontexts,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=apps.kusionstack.io,resources=resourcecontexts/status,verbs=get;update;patch -// +kubebuilder:rbac:groups=apps.kusionstack.io,resources=resourcecontexts/finalizers,verbs=update -// +kubebuilder:rbac:groups=core,resources=persistentvolumeclaims,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=apps,resources=controllerrevisions,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=core,resources=events,verbs=create;update;patch - -// Reconcile is part of the main kubernetes reconciliation loop which aims to -// move the current state of the cluster closer to the desired state. -func (r *CollaSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - logger := r.Logger.WithValues("collaset", req.String()) - instance := &appsv1alpha1.CollaSet{} - if err := r.Client.Get(ctx, req.NamespacedName, instance); err != nil { - if !errors.IsNotFound(err) { - logger.Error(err, "failed to find CollaSet") - return reconcile.Result{}, err - } +func (m *CollaSetController) NewXObject() client.Object { + return &corev1.Pod{} +} - logger.Info("collaSet is deleted") - r.cacheExpectations.DeleteExpectations(req.String()) - return ctrl.Result{}, nil - } +func (m *CollaSetController) NewXObjectList() client.ObjectList { + return &corev1.PodList{} +} - // if expectation not satisfied, shortcut this reconciling till informer cache is updated. - if satisfied := r.cacheExpectations.SatisfiedExpectations(req.String()); !satisfied { - logger.Info("CollaSet is not satisfied to reconcile") - return ctrl.Result{RequeueAfter: 30 * time.Second}, nil +type XSetOperation struct{} + +func (s *XSetOperation) GetXSetSpec(xset xsetapi.XSetObject) *xsetapi.XSetSpec { + xSetSpec := xsetapi.XSetSpec{} + set := xset.(*appsv1alpha1.CollaSet) + xSetSpec.Paused = set.Spec.Paused + xSetSpec.Replicas = set.Spec.Replicas + xSetSpec.Selector = set.Spec.Selector + xSetSpec.HistoryLimit = set.Spec.HistoryLimit + + // update strategy + switch set.Spec.UpdateStrategy.PodUpdatePolicy { + case appsv1alpha1.CollaSetRecreatePodUpdateStrategyType: + xSetSpec.UpdateStrategy.UpdatePolicy = xsetapi.XSetRecreateTargetUpdateStrategyType + case appsv1alpha1.CollaSetReplacePodUpdateStrategyType: + xSetSpec.UpdateStrategy.UpdatePolicy = xsetapi.XSetReplaceTargetUpdateStrategyType + case appsv1alpha1.CollaSetInPlaceIfPossiblePodUpdateStrategyType: + xSetSpec.UpdateStrategy.UpdatePolicy = xsetapi.XSetInPlaceIfPossibleTargetUpdateStrategyType + case appsv1alpha1.CollaSetInPlaceOnlyPodUpdateStrategyType: + xSetSpec.UpdateStrategy.UpdatePolicy = xsetapi.XSetInPlaceOnlyTargetUpdateStrategyType + default: + xSetSpec.UpdateStrategy.UpdatePolicy = xsetapi.XSetInPlaceIfPossibleTargetUpdateStrategyType } - - if instance.DeletionTimestamp != nil { - if err := r.ensureReclaimPvcs(ctx, instance); err != nil { - // reclaim pvcs before remove finalizers - return ctrl.Result{}, err + if set.Spec.UpdateStrategy.RollingUpdate != nil { + rollingUpdate := xsetapi.RollingUpdateStrategy{} + if set.Spec.UpdateStrategy.RollingUpdate.ByLabel != nil { + rollingUpdate.ByLabel = &xsetapi.ByLabel{} } - if err := r.ensureReclaimPodOwnerReferences(instance); err != nil { - // reclaim pods ownerReferences before remove finalizers - return ctrl.Result{}, err + if set.Spec.UpdateStrategy.RollingUpdate.ByPartition != nil { + rollingUpdate.ByPartition = &xsetapi.ByPartition{} + rollingUpdate.ByPartition.Partition = set.Spec.UpdateStrategy.RollingUpdate.ByPartition.Partition } - if err := r.ensureReclaimPodsDeletion(instance); err != nil { - // reclaim pods deletion before remove finalizers - return ctrl.Result{}, err - } - if controllerutil.ContainsFinalizer(instance, preReclaimFinalizer) { - // reclaim owner IDs in ResourceContext - if err := r.reclaimResourceContext(ctx, instance); err != nil { - return ctrl.Result{}, err - } + xSetSpec.UpdateStrategy.RollingUpdate = &rollingUpdate + } + xSetSpec.UpdateStrategy.OperationDelaySeconds = set.Spec.UpdateStrategy.OperationDelaySeconds + // scale strategy + xSetSpec.ScaleStrategy.TargetToDelete = set.Spec.ScaleStrategy.PodToDelete + xSetSpec.ScaleStrategy.TargetToExclude = set.Spec.ScaleStrategy.PodToExclude + xSetSpec.ScaleStrategy.TargetToInclude = set.Spec.ScaleStrategy.PodToInclude + xSetSpec.ScaleStrategy.Context = set.Spec.ScaleStrategy.Context + xSetSpec.ScaleStrategy.OperationDelaySeconds = set.Spec.ScaleStrategy.OperationDelaySeconds + // naming strategy + if set.Spec.NamingStrategy != nil { + namingStrategy := &xsetapi.NamingStrategy{} + switch set.Spec.NamingStrategy.PodNamingSuffixPolicy { + case appsv1alpha1.PodNamingSuffixPolicyPersistentSequence: + namingStrategy.TargetNamingSuffixPolicy = xsetapi.TargetNamingSuffixPolicyPersistentSequence + case appsv1alpha1.PodNamingSuffixPolicyRandom: + namingStrategy.TargetNamingSuffixPolicy = xsetapi.TargetNamingSuffixPolicyRandom + default: + namingStrategy.TargetNamingSuffixPolicy = xsetapi.TargetNamingSuffixPolicyRandom } - - return ctrl.Result{}, nil + xSetSpec.NamingStrategy = namingStrategy } + return &xSetSpec +} - if !controllerutil.ContainsFinalizer(instance, preReclaimFinalizer) { - return ctrl.Result{}, controllerutils.AddFinalizer(context.TODO(), r.Client, instance, preReclaimFinalizer) +func (s *XSetOperation) GetXSetPatch(object metav1.Object) ([]byte, error) { + set := object.(*appsv1alpha1.CollaSet) + dsBytes, err := json.Marshal(set) + if err != nil { + return nil, err } - key := commonutils.ObjectKeyString(instance) - currentRevision, updatedRevision, revisions, collisionCount, _, err := r.revisionManager.ConstructRevisions(ctx, instance) + var raw map[string]interface{} + err = json.Unmarshal(dsBytes, &raw) if err != nil { - return ctrl.Result{}, fmt.Errorf("fail to construct revision for CollaSet %s: %w", key, err) + return nil, err } + objCopy := make(map[string]interface{}) + specCopy := make(map[string]interface{}) - newStatus := &appsv1alpha1.CollaSetStatus{ - // record collisionCount - CollisionCount: &collisionCount, - CurrentRevision: currentRevision.Name, - UpdatedRevision: updatedRevision.Name, - Conditions: instance.Status.Conditions, - } + // Create a patch of the CollaSet that replaces spec.template + spec := raw["spec"].(map[string]interface{}) + template := spec["template"].(map[string]interface{}) + template["$patch"] = "replace" + specCopy["template"] = template - getter, err := utilspoddecoration.NewPodDecorationGetter(r.Client, instance.Namespace) - if err != nil { - return ctrl.Result{}, err - } - resources := &collasetutils.RelatedResources{ - Revisions: revisions, - CurrentRevision: currentRevision, - UpdatedRevision: updatedRevision, - NewStatus: newStatus, - PDGetter: getter, - } - requeueAfter, newStatus, err := r.DoReconcile(ctx, instance, resources) - // sort conditions - collasetutils.SortCollaSetConditions(newStatus.Conditions) - // update status anyway - if err := r.updateStatus(ctx, instance, newStatus); err != nil { - return requeueResult(requeueAfter), fmt.Errorf("fail to update status of CollaSet %s: %w", req, err) + if _, exist := spec["volumeClaimTemplates"]; exist { + specCopy["volumeClaimTemplates"] = spec["volumeClaimTemplates"] } - return requeueResult(requeueAfter), err + objCopy["spec"] = specCopy + patch, err := json.Marshal(objCopy) + return patch, err } -func (r *CollaSetReconciler) DoReconcile( - ctx context.Context, - instance *appsv1alpha1.CollaSet, - resources *collasetutils.RelatedResources, -) (*time.Duration, *appsv1alpha1.CollaSetStatus, error) { - podWrappers, requeueAfter, syncErr := r.doSync(ctx, instance, resources) - return requeueAfter, calculateStatus(instance, resources, podWrappers), syncErr +func (s *XSetOperation) GetXSetStatus(object xsetapi.XSetObject) *xsetapi.XSetStatus { + xSetStatus := xsetapi.XSetStatus{} + set := object.(*appsv1alpha1.CollaSet) + xSetStatus.Replicas = set.Status.Replicas + xSetStatus.ReadyReplicas = set.Status.ReadyReplicas + xSetStatus.UpdatedReplicas = set.Status.UpdatedReplicas + xSetStatus.OperatingReplicas = set.Status.OperatingReplicas + xSetStatus.UpdatedReadyReplicas = set.Status.UpdatedReadyReplicas + xSetStatus.AvailableReplicas = set.Status.AvailableReplicas + xSetStatus.UpdatedAvailableReplicas = set.Status.UpdatedAvailableReplicas + xSetStatus.ScheduledReplicas = set.Status.ScheduledReplicas + xSetStatus.Conditions = convertClsConditionToMetaCondition(set.Status.Conditions) + xSetStatus.ObservedGeneration = set.Status.ObservedGeneration + xSetStatus.CurrentRevision = set.Status.CurrentRevision + xSetStatus.UpdatedRevision = set.Status.UpdatedRevision + xSetStatus.CollisionCount = set.Status.CollisionCount + return &xSetStatus } -// doSync is responsible for reconcile Pods with CollaSet spec. -// 1. sync Pods to prepare information, especially IDs, for following Scale and Update -// 2. scale Pods to match the Pod number indicated in `spec.replcas`. if an error thrown out or Pods is not matched recently, update will be skipped. -// 3. update Pods, to update each Pod to the updated revision indicated by `spec.template` -func (r *CollaSetReconciler) doSync( - ctx context.Context, - instance *appsv1alpha1.CollaSet, - resources *collasetutils.RelatedResources) ( - []*collasetutils.PodWrapper, *time.Duration, error, -) { - synced, podWrappers, ownedIDs, err := r.syncControl.SyncPods(ctx, instance, resources) - if err != nil || synced { - return podWrappers, nil, err - } - - podWrappers, ownedIDs, err = r.syncControl.Replace(ctx, instance, podWrappers, ownedIDs, resources) - if err != nil { - return podWrappers, nil, err +func convertClsConditionToMetaCondition(conditions []appsv1alpha1.CollaSetCondition) []metav1.Condition { + var newCondition []metav1.Condition + for i := range conditions { + newCondition = append(newCondition, metav1.Condition{ + Type: string(conditions[i].Type), + Status: metav1.ConditionStatus(conditions[i].Status), + LastTransitionTime: conditions[i].LastTransitionTime, + Reason: conditions[i].Reason, + Message: conditions[i].Message, + }) } + return newCondition +} - _, scaleRequeueAfter, scaleErr := r.syncControl.Scale(ctx, instance, resources, podWrappers, ownedIDs) - _, updateRequeueAfter, updateErr := r.syncControl.Update(ctx, instance, resources, podWrappers, ownedIDs) +func (s *XSetOperation) SetXSetStatus(object xsetapi.XSetObject, xSetStatus *xsetapi.XSetStatus) { + set := object.(*appsv1alpha1.CollaSet) + set.Status.Replicas = xSetStatus.Replicas + set.Status.ReadyReplicas = xSetStatus.ReadyReplicas + set.Status.UpdatedReplicas = xSetStatus.UpdatedReplicas + set.Status.OperatingReplicas = xSetStatus.OperatingReplicas + set.Status.UpdatedReadyReplicas = xSetStatus.UpdatedReadyReplicas + set.Status.AvailableReplicas = xSetStatus.AvailableReplicas + set.Status.UpdatedAvailableReplicas = xSetStatus.UpdatedAvailableReplicas + set.Status.ScheduledReplicas = xSetStatus.ScheduledReplicas + set.Status.Conditions = convertMetaConditionToClsCondition(xSetStatus.Conditions) + set.Status.ObservedGeneration = xSetStatus.ObservedGeneration + set.Status.CurrentRevision = xSetStatus.CurrentRevision + set.Status.UpdatedRevision = xSetStatus.UpdatedRevision + set.Status.CollisionCount = xSetStatus.CollisionCount +} - err = controllerutils.AggregateErrors([]error{scaleErr, updateErr}) - if updateRequeueAfter != nil && (scaleRequeueAfter == nil || *updateRequeueAfter < *scaleRequeueAfter) { - return podWrappers, updateRequeueAfter, err +func convertMetaConditionToClsCondition(conditions []metav1.Condition) []appsv1alpha1.CollaSetCondition { + var newCondition []appsv1alpha1.CollaSetCondition + for i := range conditions { + newCondition = append(newCondition, appsv1alpha1.CollaSetCondition{ + Type: appsv1alpha1.CollaSetConditionType(conditions[i].Type), + Status: corev1.ConditionStatus(conditions[i].Status), + LastTransitionTime: conditions[i].LastTransitionTime, + Reason: conditions[i].Reason, + Message: conditions[i].Message, + }) } - return podWrappers, scaleRequeueAfter, err + return newCondition } -func calculateStatus( - instance *appsv1alpha1.CollaSet, - resources *collasetutils.RelatedResources, - podWrappers []*collasetutils.PodWrapper, -) *appsv1alpha1.CollaSetStatus { - newStatus := resources.NewStatus - newStatus.ObservedGeneration = instance.Generation - - var scheduledReplicas, readyReplicas, availableReplicas, replicas, updatedReplicas, operatingReplicas, - updatedReadyReplicas, updatedAvailableReplicas int32 - - activePods := synccontrol.FilterOutPlaceHolderPodWrappers(podWrappers) - for _, podWrapper := range activePods { - if podWrapper.DeletionTimestamp != nil && !collasetutils.IsPodNamingSuffixPolicyPersistentSequence(instance) { - continue - } - - replicas++ - - isUpdated := false - if isUpdated = utils.IsPodUpdatedRevision(podWrapper.Pod, resources.UpdatedRevision.Name); isUpdated { - updatedReplicas++ - } +func (s *XSetOperation) UpdateScaleStrategy(ctx context.Context, c client.Client, object xsetapi.XSetObject, scaleStrategy *xsetapi.ScaleStrategy) (err error) { + set := object.(*appsv1alpha1.CollaSet) + set.Spec.ScaleStrategy.PodToDelete = scaleStrategy.TargetToDelete + set.Spec.ScaleStrategy.PodToExclude = scaleStrategy.TargetToExclude + set.Spec.ScaleStrategy.PodToInclude = scaleStrategy.TargetToInclude + return c.Update(ctx, object) +} - if podopslifecycle.IsDuringOps(utils.UpdateOpsLifecycleAdapter, podWrapper) || podopslifecycle.IsDuringOps(utils.ScaleInOpsLifecycleAdapter, podWrapper) { - operatingReplicas++ - } +func (s *XSetOperation) GetXSetTemplatePatcher(_ metav1.Object) func(client.Object) error { + return func(obj client.Object) error { + return nil + } +} - if controllerutils.IsPodScheduled(podWrapper.Pod) { - scheduledReplicas++ - } +type XOperation struct{} - if controllerutils.IsPodReady(podWrapper.Pod) { - readyReplicas++ - if isUpdated { - updatedReadyReplicas++ - } - } +func (o *XOperation) GetXOpsPriority(_ context.Context, _ client.Client, _ client.Object) (*xsetapi.OpsPriority, error) { + return &xsetapi.OpsPriority{}, nil +} - if controllerutils.IsPodServiceAvailable(podWrapper.Pod) { - availableReplicas++ - if isUpdated { - updatedAvailableReplicas++ - } - } +func (o *XOperation) GetXObjectFromRevision(revision *appsv1.ControllerRevision) (client.Object, error) { + pod, err := utils.ApplyPatchFromRevision(&corev1.Pod{}, revision) + if err != nil { + return nil, err } + return pod, nil +} - newStatus.ScheduledReplicas = scheduledReplicas - newStatus.ReadyReplicas = readyReplicas - newStatus.AvailableReplicas = availableReplicas - newStatus.Replicas = replicas - newStatus.UpdatedReplicas = updatedReplicas - newStatus.OperatingReplicas = operatingReplicas - newStatus.UpdatedReadyReplicas = updatedReadyReplicas - newStatus.UpdatedAvailableReplicas = updatedAvailableReplicas - - if (instance.Spec.Replicas == nil && newStatus.UpdatedReadyReplicas >= 0) || - newStatus.UpdatedReadyReplicas >= *instance.Spec.Replicas { - newStatus.CurrentRevision = resources.UpdatedRevision.Name +func (o *XOperation) CheckReadyTime(object client.Object) (bool, *metav1.Time) { + pod := object.(*corev1.Pod) + if controllerutils.IsPodReady(pod) { + return true, utils.PodReadyTime(pod) } - - return newStatus + return false, &metav1.Time{} } -func (r *CollaSetReconciler) updateStatus( - ctx context.Context, - instance *appsv1alpha1.CollaSet, - newStatus *appsv1alpha1.CollaSetStatus, -) error { - if equality.Semantic.DeepEqual(instance.Status, newStatus) { - return nil - } +func (o *XOperation) CheckAvailable(object client.Object) bool { + pod := object.(*corev1.Pod) + return controllerutils.IsPodServiceAvailable(pod) +} - instance.Status = *newStatus +func (o *XOperation) CheckScheduled(object client.Object) bool { + pod := object.(*corev1.Pod) + return controllerutils.IsPodScheduled(pod) +} - err := r.Client.Status().Update(ctx, instance) - if err == nil { - return r.cacheExpectations.ExpectUpdation(kubeutilsclient.ObjectKeyString(instance), - collasetutils.CollaSetGVK, instance.Namespace, instance.Name, instance.ResourceVersion) - } - return err +func (o *XOperation) CheckInactive(object client.Object) bool { + pod := object.(*corev1.Pod) + return utils.IsPodInactive(pod) } -func (r *CollaSetReconciler) reclaimResourceContext(ctx context.Context, cls *appsv1alpha1.CollaSet) error { - // clean the owner IDs from this CollaSet - if err := r.podContextControl.UpdateToPodContext(ctx, cls, nil); err != nil { - return err - } +type LifecycleAdapterGetter struct{} - return controllerutils.RemoveFinalizer(context.TODO(), r.Client, cls, preReclaimFinalizer) +func (l *LifecycleAdapterGetter) GetScaleInOpsLifecycleAdapter() xsetapi.LifecycleAdapter { + return ScaleInOpsLifecycleAdapter } -func requeueResult(requeueTime *time.Duration) reconcile.Result { - if requeueTime != nil { - if *requeueTime == 0 { - return reconcile.Result{Requeue: true} - } - return reconcile.Result{RequeueAfter: *requeueTime} - } - return reconcile.Result{} +func (l *LifecycleAdapterGetter) GetUpdateOpsLifecycleAdapter() xsetapi.LifecycleAdapter { + return UpdateOpsLifecycleAdapter } -func (r *CollaSetReconciler) ensureReclaimPvcs(ctx context.Context, cls *appsv1alpha1.CollaSet) error { - var needReclaimPvcs []*corev1.PersistentVolumeClaim - pvcs, err := r.pvcControl.GetFilteredPvcs(ctx, cls) - if err != nil { - return err - } - // reclaim pvcs according to whenDelete retention policy - for i := range pvcs { - owned := pvcs[i].OwnerReferences != nil && len(pvcs[i].OwnerReferences) > 0 - if owned && collasetutils.PvcPolicyWhenDelete(cls) == appsv1alpha1.RetainPersistentVolumeClaimRetentionPolicyType { - needReclaimPvcs = append(needReclaimPvcs, pvcs[i]) - } - } - for i := range needReclaimPvcs { - if err = r.pvcControl.OrphanPvc(cls, needReclaimPvcs[i]); err != nil { - return err - } - } - return nil -} +type GetLabelManagerAdapterGetter struct{} -func (r *CollaSetReconciler) ensureReclaimPodOwnerReferences(cls *appsv1alpha1.CollaSet) error { - podControl := podcontrol.NewRealPodControl(r.Client, r.Scheme) - filteredPods, _, err := podControl.GetFilteredPods(cls.Spec.Selector, cls) - if err != nil { - return fmt.Errorf("fail to get filtered Pods: %w", err) - } - // reclaim podDecoration ownerReferences on filteredPods - for i := range filteredPods { - if len(filteredPods[i].OwnerReferences) == 0 { - continue - } - var newOwnerRefs []v1.OwnerReference - for j := range filteredPods[i].OwnerReferences { - if filteredPods[i].OwnerReferences[j].Kind == "PodDecoration" { - continue - } - newOwnerRefs = append(newOwnerRefs, filteredPods[i].OwnerReferences[j]) - } - if len(newOwnerRefs) != len(filteredPods[i].OwnerReferences) { - filteredPods[i].OwnerReferences = newOwnerRefs - if err := podControl.UpdatePod(filteredPods[i]); err != nil { - return err - } - } - } - return nil +func (g *GetLabelManagerAdapterGetter) GetLabelManagerAdapter() map[xsetapi.XSetLabelAnnotationEnum]string { + return defaultXSetControllerLabelManager } -func (r *CollaSetReconciler) ensureReclaimPodsDeletion(cls *appsv1alpha1.CollaSet) error { - podControl := podcontrol.NewRealPodControl(r.Client, r.Scheme) - filteredPods, _, err := podControl.GetFilteredPods(cls.Spec.Selector, cls) - if err != nil { - return fmt.Errorf("fail to get filtered Pods: %w", err) - } - return synccontrol.DeletePodsByLabel(podControl, filteredPods) +type ResourceContextAdapterGetter struct{} + +func (r *ResourceContextAdapterGetter) GetResourceContextAdapter() xsetapi.ResourceContextAdapter { + return &ResourceContextAdapter{} } diff --git a/pkg/controllers/collaset/collaset_controller_test.go b/pkg/controllers/collaset/collaset_controller_test.go index df840e35..3d3e8640 100644 --- a/pkg/controllers/collaset/collaset_controller_test.go +++ b/pkg/controllers/collaset/collaset_controller_test.go @@ -41,6 +41,7 @@ import ( "k8s.io/client-go/util/retry" "k8s.io/utils/ptr" appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" + "kusionstack.io/kube-xset/features" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" @@ -48,14 +49,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "kusionstack.io/kuperator/pkg/controllers/collaset/synccontrol" + "kusionstack.io/kuperator/pkg/controllers/collaset/legacy" collasetutils "kusionstack.io/kuperator/pkg/controllers/collaset/utils" "kusionstack.io/kuperator/pkg/controllers/poddecoration" "kusionstack.io/kuperator/pkg/controllers/poddeletion" "kusionstack.io/kuperator/pkg/controllers/utils/poddecoration/strategy" "kusionstack.io/kuperator/pkg/controllers/utils/podopslifecycle" - "kusionstack.io/kuperator/pkg/features" - "kusionstack.io/kuperator/pkg/utils/feature" "kusionstack.io/kuperator/pkg/utils/inject" ) @@ -115,7 +114,10 @@ var _ = Describe("collaset controller", func() { return len(podList.Items) == 2 }, 5*time.Second, 1*time.Second).Should(BeTrue()) Expect(c.Get(context.TODO(), types.NamespacedName{Namespace: cs.Namespace, Name: cs.Name}, cs)).Should(BeNil()) - Expect(expectedStatusReplicas(c, cs, 0, 0, 0, 2, 2, 0, 0, 0)).Should(BeNil()) + + Eventually(func() error { + return expectedStatusReplicas(c, cs, 0, 0, 0, 2, 2, 0, 0, 0) + }, time.Second*10, time.Second).Should(Succeed()) observedGeneration := cs.Status.ObservedGeneration // update CollaSet template metadata @@ -135,7 +137,7 @@ var _ = Describe("collaset controller", func() { for i := range podList.Items { pod := &podList.Items[i] Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { - labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.UpdateOpsLifecycleAdapter.GetID()) + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, legacy.UpdateOpsLifecycleAdapter.GetID()) pod.Labels[labelOperate] = fmt.Sprintf("%d", time.Now().UnixNano()) return true })).Should(BeNil()) @@ -165,7 +167,7 @@ var _ = Describe("collaset controller", func() { Expect(c.List(context.TODO(), podList, client.InNamespace(cs.Namespace))).Should(BeNil()) for i := range podList.Items { pod := &podList.Items[i] - if podopslifecycle.IsDuringOps(collasetutils.UpdateOpsLifecycleAdapter, pod) { + if podopslifecycle.IsDuringOps(legacy.UpdateOpsLifecycleAdapter, pod) { return false } } @@ -266,7 +268,7 @@ var _ = Describe("collaset controller", func() { // allow origin pod to update Expect(updatePodWithRetry(c, originPod.Namespace, originPodName, func(pod *corev1.Pod) bool { - labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.UpdateOpsLifecycleAdapter.GetID()) + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, legacy.UpdateOpsLifecycleAdapter.GetID()) pod.Labels[labelOperate] = fmt.Sprintf("%d", time.Now().UnixNano()) return true })).Should(BeNil()) @@ -408,7 +410,7 @@ var _ = Describe("collaset controller", func() { })).Should(BeNil()) Eventually(func() error { return expectedStatusReplicas(c, cs, 0, 0, 0, 2, 2, 0, 0, 0) - }, 30*time.Second, 1*time.Second).Should(BeNil()) + }, 3000*time.Second, 1*time.Second).Should(BeNil()) Eventually(func() bool { Expect(c.List(context.TODO(), podList, client.InNamespace(cs.Namespace))).Should(BeNil()) @@ -442,7 +444,7 @@ var _ = Describe("collaset controller", func() { // allow origin pod to update Expect(updatePodWithRetry(c, originPod.Namespace, originPodName, func(pod *corev1.Pod) bool { - labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.UpdateOpsLifecycleAdapter.GetID()) + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, legacy.UpdateOpsLifecycleAdapter.GetID()) pod.Labels[labelOperate] = fmt.Sprintf("%d", time.Now().UnixNano()) return true })).Should(BeNil()) @@ -627,7 +629,7 @@ var _ = Describe("collaset controller", func() { for i := range podList.Items { pod := &podList.Items[i] Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { - labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.ScaleInOpsLifecycleAdapter.GetID()) + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, legacy.ScaleInOpsLifecycleAdapter.GetID()) pod.Labels[labelOperate] = fmt.Sprintf("%d", time.Now().UnixNano()) return true })).Should(BeNil()) @@ -774,7 +776,7 @@ var _ = Describe("collaset controller", func() { for i := range podList.Items { pod := &podList.Items[i] Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { - labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.ScaleInOpsLifecycleAdapter.GetID()) + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, legacy.ScaleInOpsLifecycleAdapter.GetID()) pod.Labels[labelOperate] = fmt.Sprintf("%d", time.Now().UnixNano()) return true })).Should(BeNil()) @@ -864,7 +866,7 @@ var _ = Describe("collaset controller", func() { Expect(c.List(context.TODO(), podList, client.InNamespace(cs.Namespace))).Should(BeNil()) for i := range podList.Items { pod := podList.Items[i] - if podopslifecycle.IsDuringOps(collasetutils.ScaleInOpsLifecycleAdapter, &pod) { + if podopslifecycle.IsDuringOps(legacy.ScaleInOpsLifecycleAdapter, &pod) { podToScaleIn = &pod return true } @@ -885,8 +887,8 @@ var _ = Describe("collaset controller", func() { for i := range podList.Items { pod := podList.Items[i] Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { - if podopslifecycle.IsDuringOps(collasetutils.ScaleInOpsLifecycleAdapter, pod) { - labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.ScaleInOpsLifecycleAdapter.GetID()) + if podopslifecycle.IsDuringOps(legacy.ScaleInOpsLifecycleAdapter, pod) { + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, legacy.ScaleInOpsLifecycleAdapter.GetID()) pod.Labels[labelOperate] = fmt.Sprintf("%d", time.Now().UnixNano()) triggerAllowed = podToScaleIn.Name == pod.Name } @@ -982,8 +984,8 @@ var _ = Describe("collaset controller", func() { for i := range podList.Items { pod := podList.Items[i] Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { - if podopslifecycle.IsDuringOps(collasetutils.ScaleInOpsLifecycleAdapter, pod) { - labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.ScaleInOpsLifecycleAdapter.GetID()) + if podopslifecycle.IsDuringOps(legacy.ScaleInOpsLifecycleAdapter, pod) { + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, legacy.ScaleInOpsLifecycleAdapter.GetID()) pod.Labels[labelOperate] = fmt.Sprintf("%d", time.Now().UnixNano()) triggerAllowed = true } @@ -1075,16 +1077,16 @@ var _ = Describe("collaset controller", func() { Expect(c.List(context.TODO(), podList, client.InNamespace(cs.Namespace))).Should(BeNil()) for i := range podList.Items { pod := &podList.Items[i] - if !podopslifecycle.IsDuringOps(collasetutils.UpdateOpsLifecycleAdapter, pod) { + if !podopslifecycle.IsDuringOps(legacy.UpdateOpsLifecycleAdapter, pod) { continue } - if _, allowed := podopslifecycle.AllowOps(collasetutils.UpdateOpsLifecycleAdapter, 0, pod); allowed { + if _, allowed := podopslifecycle.AllowOps(legacy.UpdateOpsLifecycleAdapter, 0, pod); allowed { continue } // allow Pod to do update Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { - labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.UpdateOpsLifecycleAdapter.GetID()) + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, legacy.UpdateOpsLifecycleAdapter.GetID()) pod.Labels[labelOperate] = fmt.Sprintf("%d", time.Now().UnixNano()) return true })).Should(BeNil()) @@ -1103,7 +1105,7 @@ var _ = Describe("collaset controller", func() { Expect(pod.Annotations).ShouldNot(BeNil()) Expect(pod.Annotations[appsv1alpha1.LastPodStatusAnnotationKey]).ShouldNot(BeEquivalentTo("")) - podStatus := &synccontrol.PodStatus{} + podStatus := &PodStatus{} Expect(json.Unmarshal([]byte(pod.Annotations[appsv1alpha1.LastPodStatusAnnotationKey]), podStatus)).Should(BeNil()) Expect(len(podStatus.ContainerStates)).Should(BeEquivalentTo(1)) Expect(podStatus.ContainerStates["foo"].LatestImage).Should(BeEquivalentTo("nginx:v2")) @@ -1163,12 +1165,12 @@ var _ = Describe("collaset controller", func() { Eventually(func() bool { Expect(c.Get(context.TODO(), types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}, &pod)).Should(BeNil()) - if !podopslifecycle.IsDuringOps(collasetutils.UpdateOpsLifecycleAdapter, &pod) { + if !podopslifecycle.IsDuringOps(legacy.UpdateOpsLifecycleAdapter, &pod) { return false } // allow Pod to do update Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { - labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.UpdateOpsLifecycleAdapter.GetID()) + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, legacy.UpdateOpsLifecycleAdapter.GetID()) pod.Labels[labelOperate] = fmt.Sprintf("%d", time.Now().UnixNano()) return true })).Should(BeNil()) @@ -1199,7 +1201,7 @@ var _ = Describe("collaset controller", func() { tmpPod := &corev1.Pod{} Expect(c.Get(context.TODO(), types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}, tmpPod)).Should(BeNil()) return tmpPod.Labels[appsv1alpha1.CollaSetUpdateIndicateLabelKey] == "" - }, 5*time.Second, 1*time.Second).Should(BeTrue()) + }, 500*time.Second, 1*time.Second).Should(BeTrue()) if pod.Spec.Containers[0].Image == cs.Spec.Template.Spec.Containers[0].Image { updatedReplicas++ } @@ -1279,14 +1281,14 @@ var _ = Describe("collaset controller", func() { // allow Pod to do update for i := range podList.Items { pod := &podList.Items[i] - if !podopslifecycle.IsDuringOps(collasetutils.UpdateOpsLifecycleAdapter, pod) { + if !podopslifecycle.IsDuringOps(legacy.UpdateOpsLifecycleAdapter, pod) { continue } - if _, allowed := podopslifecycle.AllowOps(collasetutils.UpdateOpsLifecycleAdapter, 0, pod); allowed { + if _, allowed := podopslifecycle.AllowOps(legacy.UpdateOpsLifecycleAdapter, 0, pod); allowed { continue } Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { - labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.UpdateOpsLifecycleAdapter.GetID()) + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, legacy.UpdateOpsLifecycleAdapter.GetID()) pod.Labels[labelOperate] = fmt.Sprintf("%d", time.Now().UnixNano()) return true })).Should(BeNil()) @@ -1381,14 +1383,14 @@ var _ = Describe("collaset controller", func() { Expect(c.List(context.TODO(), podList, client.InNamespace(cs.Namespace))).Should(BeNil()) for i := range podList.Items { pod := &podList.Items[i] - if !podopslifecycle.IsDuringOps(collasetutils.UpdateOpsLifecycleAdapter, pod) { + if !podopslifecycle.IsDuringOps(legacy.UpdateOpsLifecycleAdapter, pod) { continue } - if _, allowed := podopslifecycle.AllowOps(collasetutils.UpdateOpsLifecycleAdapter, 0, pod); allowed { + if _, allowed := podopslifecycle.AllowOps(legacy.UpdateOpsLifecycleAdapter, 0, pod); allowed { continue } Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { - labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.UpdateOpsLifecycleAdapter.GetID()) + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, legacy.UpdateOpsLifecycleAdapter.GetID()) pod.Labels[labelOperate] = fmt.Sprintf("%d", time.Now().UnixNano()) return true })).Should(BeNil()) @@ -1406,14 +1408,14 @@ var _ = Describe("collaset controller", func() { Expect(c.List(context.TODO(), podList, client.InNamespace(cs.Namespace))).Should(BeNil()) for i := range podList.Items { pod := &podList.Items[i] - if !podopslifecycle.IsDuringOps(collasetutils.UpdateOpsLifecycleAdapter, pod) { + if !podopslifecycle.IsDuringOps(legacy.UpdateOpsLifecycleAdapter, pod) { continue } - if _, allowed := podopslifecycle.AllowOps(collasetutils.UpdateOpsLifecycleAdapter, 0, pod); allowed { + if _, allowed := podopslifecycle.AllowOps(legacy.UpdateOpsLifecycleAdapter, 0, pod); allowed { continue } Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { - labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.UpdateOpsLifecycleAdapter.GetID()) + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, legacy.UpdateOpsLifecycleAdapter.GetID()) pod.Labels[labelOperate] = fmt.Sprintf("%d", time.Now().UnixNano()) return true })).Should(BeNil()) @@ -1518,7 +1520,7 @@ var _ = Describe("collaset controller", func() { // allow Pod to update Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { - labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.UpdateOpsLifecycleAdapter.GetID()) + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, legacy.UpdateOpsLifecycleAdapter.GetID()) pod.Labels[labelOperate] = "true" return true })).Should(BeNil()) @@ -1638,7 +1640,7 @@ var _ = Describe("collaset controller", func() { count++ // allow Pod to update Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { - labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.UpdateOpsLifecycleAdapter.GetID()) + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, legacy.UpdateOpsLifecycleAdapter.GetID()) pod.Labels[labelOperate] = "true" return true })).Should(BeNil()) @@ -2576,7 +2578,7 @@ var _ = Describe("collaset controller", func() { // allow new pod to scaleIn Expect(updatePodWithRetry(c, newPod.Namespace, newPod.Name, func(pod *corev1.Pod) bool { - labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.ScaleInOpsLifecycleAdapter.GetID()) + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, legacy.ScaleInOpsLifecycleAdapter.GetID()) pod.Labels[labelOperate] = fmt.Sprintf("%d", time.Now().UnixNano()) return true })).Should(BeNil()) @@ -2926,7 +2928,7 @@ var _ = Describe("collaset controller", func() { return false }, time.Second*10, time.Second).Should(BeTrue()) Expect(updatePodWithRetry(c, originPod2.Namespace, originPod2.Name, func(pod *corev1.Pod) bool { - labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.UpdateOpsLifecycleAdapter.GetID()) + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, legacy.UpdateOpsLifecycleAdapter.GetID()) pod.Labels[labelOperate] = fmt.Sprintf("%d", time.Now().UnixNano()) return true })).Should(BeNil()) @@ -2942,7 +2944,7 @@ var _ = Describe("collaset controller", func() { // originPod1 is during inPlaceUpdate lifecycle, but continue to be replaced by updated pod // originPod2 is during inPlaceUpdate lifecycle, and being updated by inPlace if pod.Name == originPod1.Name || pod.Name == originPod2.Name { - Expect(podopslifecycle.IsDuringOps(collasetutils.UpdateOpsLifecycleAdapter, &pod)).Should(BeTrue()) + Expect(podopslifecycle.IsDuringOps(legacy.UpdateOpsLifecycleAdapter, &pod)).Should(BeTrue()) } } }) @@ -3072,7 +3074,7 @@ var _ = Describe("collaset controller", func() { for i := range podList.Items { pod := &podList.Items[i] Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { - labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.ScaleInOpsLifecycleAdapter.GetID()) + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, legacy.ScaleInOpsLifecycleAdapter.GetID()) pod.Labels[labelOperate] = fmt.Sprintf("%d", time.Now().UnixNano()) return true })).Should(BeNil()) @@ -3114,7 +3116,7 @@ var _ = Describe("collaset controller", func() { for i := range podList.Items { pod := &podList.Items[i] Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { - labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.ScaleInOpsLifecycleAdapter.GetID()) + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, legacy.ScaleInOpsLifecycleAdapter.GetID()) delete(pod.Labels, labelOperate) return true })).Should(BeNil()) @@ -3299,15 +3301,15 @@ var _ = Describe("collaset controller", func() { } for i := range podList.Items { pod := &podList.Items[i] - if !podopslifecycle.IsDuringOps(collasetutils.UpdateOpsLifecycleAdapter, pod) { + if !podopslifecycle.IsDuringOps(legacy.UpdateOpsLifecycleAdapter, pod) { continue } - if _, allowed := podopslifecycle.AllowOps(collasetutils.UpdateOpsLifecycleAdapter, 0, pod); allowed { + if _, allowed := podopslifecycle.AllowOps(legacy.UpdateOpsLifecycleAdapter, 0, pod); allowed { continue } Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { - labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.UpdateOpsLifecycleAdapter.GetID()) + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, legacy.UpdateOpsLifecycleAdapter.GetID()) pod.Labels[labelOperate] = fmt.Sprintf("%d", time.Now().UnixNano()) return true })).Should(BeNil()) @@ -3630,7 +3632,7 @@ var _ = Describe("collaset controller", func() { for i := range podList.Items { pod := &podList.Items[i] Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { - labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.ScaleInOpsLifecycleAdapter.GetID()) + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, legacy.ScaleInOpsLifecycleAdapter.GetID()) pod.Labels[labelOperate] = fmt.Sprintf("%d", time.Now().UnixNano()) return true })).Should(BeNil()) @@ -3663,7 +3665,7 @@ var _ = Describe("collaset controller", func() { for i := range podList.Items { pod := &podList.Items[i] Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { - labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.ScaleInOpsLifecycleAdapter.GetID()) + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, legacy.ScaleInOpsLifecycleAdapter.GetID()) delete(pod.Labels, labelOperate) return true })).Should(BeNil()) @@ -3701,10 +3703,6 @@ var _ = Describe("collaset controller", func() { })).Should(BeNil()) // delete collaset Expect(c.Delete(context.TODO(), cs)).Should(BeNil()) - Eventually(func() bool { - err := c.Get(context.TODO(), types.NamespacedName{Namespace: cs.Namespace, Name: cs.Name}, cs) - return errors.IsNotFound(err) - }, time.Second*10, time.Second).Should(BeTrue()) // delete collaset pods podList := &corev1.PodList{} Expect(c.List(context.TODO(), podList, client.InNamespace(cs.Namespace))).Should(BeNil()) @@ -3715,6 +3713,10 @@ var _ = Describe("collaset controller", func() { Expect(c.List(context.TODO(), podList, client.InNamespace(cs.Namespace))).Should(BeNil()) return len(podList.Items) }, time.Second*10, time.Second).Should(BeEquivalentTo(0)) + Eventually(func() bool { + err := c.Get(context.TODO(), types.NamespacedName{Namespace: cs.Namespace, Name: cs.Name}, cs) + return errors.IsNotFound(err) + }, time.Second*10, time.Second).Should(BeTrue()) // there should be 8 pvcs allPvcs := &corev1.PersistentVolumeClaimList{} activePvcs := make([]*corev1.PersistentVolumeClaim, 0) @@ -3750,13 +3752,26 @@ var _ = Describe("collaset controller", func() { Expect(pvcNames.Has(volume.PersistentVolumeClaim.ClaimName)).Should(BeTrue()) } } + Eventually(func() int { + Expect(c.List(context.TODO(), allPvcs, client.InNamespace(cs.Namespace))).Should(BeNil()) + activePvcs = make([]*corev1.PersistentVolumeClaim, 0) + pvcNames = sets.String{} + for i := range allPvcs.Items { + if allPvcs.Items[i].DeletionTimestamp != nil { + continue + } + activePvcs = append(activePvcs, &allPvcs.Items[i]) + pvcNames.Insert(allPvcs.Items[i].Name) + } + return len(activePvcs) + }, time.Second*10, time.Second).Should(BeEquivalentTo(8)) } }) It("podToDelete", func() { testcase := "test-pod-to-delete" Expect(createNamespace(c, testcase)).Should(BeNil()) - _ = feature.DefaultMutableFeatureGate.Set(fmt.Sprintf("%s=%s", features.ReclaimPodScaleStrategy, "true")) + _ = features.DefaultMutableFeatureGate.Set(fmt.Sprintf("%s=%s", features.ReclaimScaleStrategy, "true")) cs := &appsv1alpha1.CollaSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: testcase, @@ -3834,7 +3849,8 @@ var _ = Describe("collaset controller", func() { It("[podToDelete] scale", func() { testcase := "test-pod-to-delete-scale" Expect(createNamespace(c, testcase)).Should(BeNil()) - _ = feature.DefaultMutableFeatureGate.Set(fmt.Sprintf("%s=%s", features.ReclaimPodScaleStrategy, "true")) + + _ = features.DefaultMutableFeatureGate.Set(fmt.Sprintf("%s=%s", features.ReclaimScaleStrategy, "true")) cs := &appsv1alpha1.CollaSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: testcase, @@ -3889,7 +3905,7 @@ var _ = Describe("collaset controller", func() { // mark pod allowed to operate in ScaleInPodOpsLifecycle pod := &podList.Items[0] Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { - labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.ScaleInOpsLifecycleAdapter.GetID()) + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, legacy.ScaleInOpsLifecycleAdapter.GetID()) pod.Labels[labelOperate] = fmt.Sprintf("%d", time.Now().UnixNano()) return true })).Should(BeNil()) @@ -3953,7 +3969,7 @@ var _ = Describe("collaset controller", func() { It("Exclude and Include pod", func() { testcase := "test-pod-to-exclude-include" Expect(createNamespace(c, testcase)).Should(BeNil()) - _ = feature.DefaultMutableFeatureGate.Set(fmt.Sprintf("%s=%s", features.ReclaimPodScaleStrategy, "true")) + _ = features.DefaultMutableFeatureGate.Set(fmt.Sprintf("%s=%s", features.ReclaimScaleStrategy, "true")) cs := &appsv1alpha1.CollaSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: testcase, @@ -4229,8 +4245,8 @@ var _ = Describe("collaset controller", func() { for i := range podList.Items { pod := podList.Items[i] Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { - if podopslifecycle.IsDuringOps(collasetutils.ScaleInOpsLifecycleAdapter, pod) { - labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.ScaleInOpsLifecycleAdapter.GetID()) + if podopslifecycle.IsDuringOps(legacy.ScaleInOpsLifecycleAdapter, pod) { + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, legacy.ScaleInOpsLifecycleAdapter.GetID()) pod.Labels[labelOperate] = fmt.Sprintf("%d", time.Now().UnixNano()) triggerAllowed = true } @@ -4387,7 +4403,7 @@ var _ = Describe("collaset controller", func() { It("[Include] pod without revision", func() { testcase := "test-include-pod-without-revision" Expect(createNamespace(c, testcase)).Should(BeNil()) - _ = feature.DefaultMutableFeatureGate.Set(fmt.Sprintf("%s=%s", features.ReclaimPodScaleStrategy, "true")) + _ = features.DefaultMutableFeatureGate.Set(fmt.Sprintf("%s=%s", features.ReclaimScaleStrategy, "true")) cs := &appsv1alpha1.CollaSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: testcase, @@ -4590,7 +4606,7 @@ var _ = Describe("collaset controller", func() { for i := range podList.Items { pod := podList.Items[i] Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { - labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.UpdateOpsLifecycleAdapter.GetID()) + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, legacy.UpdateOpsLifecycleAdapter.GetID()) pod.Labels[labelOperate] = fmt.Sprintf("%d", time.Now().UnixNano()) return true })).Should(BeNil()) @@ -4666,6 +4682,10 @@ var _ = Describe("collaset controller", func() { // delete this CollaSet Expect(c.Delete(context.TODO(), cs)).Should(BeNil()) Eventually(func() bool { + Expect(c.List(context.TODO(), podList, client.InNamespace(cs.Namespace))).Should(BeNil()) + for i := range podList.Items { + Expect(c.Delete(context.TODO(), &podList.Items[i])).Should(BeNil()) + } if err := c.Get(context.TODO(), types.NamespacedName{Namespace: cs.Namespace, Name: cs.Name}, cs); err != nil && errors.IsNotFound(err) { return true } @@ -4774,7 +4794,7 @@ var _ = Describe("collaset controller", func() { return false }, time.Second*10, time.Second).Should(BeTrue()) Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { - labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.UpdateOpsLifecycleAdapter.GetID()) + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, legacy.UpdateOpsLifecycleAdapter.GetID()) pod.Labels[labelOperate] = fmt.Sprintf("%d", time.Now().UnixNano()) return true })).Should(BeNil()) @@ -4801,13 +4821,6 @@ var _ = Describe("collaset controller", func() { // delete this CollaSet Expect(c.Delete(context.TODO(), cs)).Should(BeNil()) - Eventually(func() bool { - if err := c.Get(context.TODO(), types.NamespacedName{Namespace: cs.Namespace, Name: cs.Name}, cs); err != nil && errors.IsNotFound(err) { - return true - } - - return false - }, 5*time.Second, 1*time.Second).Should(BeTrue()) Eventually(func() bool { Expect(c.List(context.TODO(), podList, client.InNamespace(cs.Namespace))).Should(BeNil()) @@ -4817,7 +4830,21 @@ var _ = Describe("collaset controller", func() { } } return true - }, 50*time.Second, 1*time.Second).Should(BeTrue()) + }, 5*time.Second, 1*time.Second).Should(BeTrue()) + + Eventually(func() bool { + podList := &corev1.PodList{} + Expect(c.List(context.TODO(), podList, client.InNamespace(cs.Namespace))).Should(BeNil()) + for i := range podList.Items { + Expect(c.Delete(context.TODO(), &podList.Items[i])).Should(BeNil()) + } + + if err := c.Get(context.TODO(), types.NamespacedName{Namespace: cs.Namespace, Name: cs.Name}, cs); err != nil && errors.IsNotFound(err) { + return true + } + + return false + }, 5*time.Second, 1*time.Second).Should(BeTrue()) rc := &resourceContextList.Items[0] Eventually(func() bool { @@ -5102,8 +5129,7 @@ var _ = BeforeSuite(func() { c = mgr.GetClient() var r reconcile.Reconciler - r, request = testReconcile(NewReconciler(mgr)) - err = AddToMgr(mgr, r) + err = Add(mgr) Expect(err).NotTo(HaveOccurred()) r, request = testReconcile(poddeletion.NewReconciler(mgr)) err = poddeletion.AddToMgr(mgr, r) diff --git a/pkg/controllers/collaset/decoration_adapter.go b/pkg/controllers/collaset/decoration_adapter.go new file mode 100644 index 00000000..b832c43d --- /dev/null +++ b/pkg/controllers/collaset/decoration_adapter.go @@ -0,0 +1,138 @@ +/* +Copyright 2025 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package collaset + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" + xsetapi "kusionstack.io/kube-xset/api" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/source" + + utilspoddecoration "kusionstack.io/kuperator/pkg/controllers/utils/poddecoration" + "kusionstack.io/kuperator/pkg/controllers/utils/poddecoration/anno" + "kusionstack.io/kuperator/pkg/controllers/utils/poddecoration/strategy" +) + +var _ xsetapi.DecorationAdapter = &DecorationAdapter{} + +type DecorationAdapter struct{} + +func (d *DecorationAdapter) WatchDecoration(c controller.Controller) error { + // Only for starting SharedStrategyController + err := c.Watch(strategy.SharedStrategyController, &handler.Funcs{}) + if err != nil { + return err + } + + ch := make(chan event.GenericEvent, 1<<10) + strategy.SharedStrategyController.RegisterGenericEventChannel(ch) + // Watch PodDecoration related events + err = c.Watch(&source.Channel{Source: ch}, &handler.EnqueueRequestForObject{}) + if err != nil { + return err + } + return nil +} + +func (d *DecorationAdapter) GetDecorationGroupVersionKind() metav1.GroupVersionKind { + return metav1.GroupVersionKind{ + Group: appsv1alpha1.SchemeGroupVersion.Group, + Version: appsv1alpha1.SchemeGroupVersion.Version, + Kind: "PodDecoration", + } +} + +func (d *DecorationAdapter) GetDecorationPatcherByRevisions(ctx context.Context, c client.Client, target client.Object, revision string) (func(client.Object) error, error) { + var pds map[string]*appsv1alpha1.PodDecoration + pod := target.(*corev1.Pod) + infos, marshallErr := anno.UnmarshallFromString(revision) + if marshallErr != nil { + return nil, marshallErr + } + var revisions []string + for _, info := range infos { + revisions = append(revisions, info.Revision) + } + getter, err := utilspoddecoration.NewPodDecorationGetter(c, pod.Namespace) + if err != nil { + return nil, err + } + pds, err = getter.GetByRevisions(ctx, revisions...) + if err != nil { + return nil, err + } + return func(object client.Object) error { + p := object.(*corev1.Pod) + return utilspoddecoration.PatchListOfDecorations(p, pds) + }, nil +} + +func (d *DecorationAdapter) GetTargetCurrentDecorationRevisions(ctx context.Context, c client.Client, target client.Object) (string, error) { + var pds map[string]*appsv1alpha1.PodDecoration + pod := target.(*corev1.Pod) + getter, err := utilspoddecoration.NewPodDecorationGetter(c, pod.Namespace) + if err != nil { + return "", err + } + pds, err = getter.GetOnPod(ctx, pod) + return anno.GetDecorationInfoString(pds), err +} + +func (d *DecorationAdapter) GetTargetUpdatedDecorationRevisions(ctx context.Context, c client.Client, target client.Object) (string, error) { + var pds map[string]*appsv1alpha1.PodDecoration + pod := target.(*corev1.Pod) + getter, err := utilspoddecoration.NewPodDecorationGetter(c, pod.Namespace) + if err != nil { + return "", err + } + pds, err = getter.GetEffective(ctx, pod) + return anno.GetDecorationInfoString(pds), err +} + +func (d *DecorationAdapter) IsTargetDecorationChanged(current, updated string) (bool, error) { + var currentInfos, updatedInfos []*anno.DecorationInfo + var err error + if currentInfos, err = anno.UnmarshallFromString(current); err != nil { + return false, err + } + if updatedInfos, err = anno.UnmarshallFromString(updated); err != nil { + return false, err + } + + if len(currentInfos) != len(updatedInfos) { + return true, nil + } else { + revisionSets := sets.NewString() + for i := range currentInfos { + revisionSets.Insert(currentInfos[i].Revision) + } + for i := range updatedInfos { + if !revisionSets.Has(updatedInfos[i].Revision) { + return true, nil + } + } + } + return false, nil +} diff --git a/pkg/controllers/collaset/utils/lifecycle_adapter.go b/pkg/controllers/collaset/legacy/lifecycle_adapter.go similarity index 99% rename from pkg/controllers/collaset/utils/lifecycle_adapter.go rename to pkg/controllers/collaset/legacy/lifecycle_adapter.go index 3c768ba5..a3779261 100644 --- a/pkg/controllers/collaset/utils/lifecycle_adapter.go +++ b/pkg/controllers/collaset/legacy/lifecycle_adapter.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package utils +package legacy import ( appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" diff --git a/pkg/controllers/collaset/utils/expectation.go b/pkg/controllers/collaset/legacy/podcontext.go similarity index 52% rename from pkg/controllers/collaset/utils/expectation.go rename to pkg/controllers/collaset/legacy/podcontext.go index 19eabb0c..3b5151cd 100644 --- a/pkg/controllers/collaset/utils/expectation.go +++ b/pkg/controllers/collaset/legacy/podcontext.go @@ -1,5 +1,5 @@ /* -Copyright 2023 The KusionStack Authors. +Copyright 2025 The KusionStack Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,16 +14,15 @@ See the License for the specific language governing permissions and limitations under the License. */ -package utils +package legacy -import ( - corev1 "k8s.io/api/core/v1" - appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" -) - -var ( - PodGVK = corev1.SchemeGroupVersion.WithKind("Pod") - PVCGVK = corev1.SchemeGroupVersion.WithKind("PersistentVolumeClaim") - CollaSetGVK = appsv1alpha1.SchemeGroupVersion.WithKind("CollaSet") - ResourceContextGVK = appsv1alpha1.SchemeGroupVersion.WithKind("ResourceContext") +const ( + OwnerContextKey = "Owner" + RevisionContextDataKey = "Revision" + PodDecorationRevisionKey = "PodDecorationRevisions" + JustCreateContextDataKey = "PodJustCreate" + RecreateUpdateContextDataKey = "PodRecreateUpdate" + ScaleInContextDataKey = "ScaleIn" + ReplaceNewPodIDContextDataKey = "ReplaceNewPodID" + ReplaceOriginPodIDContextDataKey = "ReplaceOriginPodID" ) diff --git a/pkg/controllers/collaset/pvccontrol/pvc_control.go b/pkg/controllers/collaset/legacy/pvc_control.go similarity index 92% rename from pkg/controllers/collaset/pvccontrol/pvc_control.go rename to pkg/controllers/collaset/legacy/pvc_control.go index 0e2b3d9f..2362ca52 100644 --- a/pkg/controllers/collaset/pvccontrol/pvc_control.go +++ b/pkg/controllers/collaset/legacy/pvc_control.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package pvccontrol +package legacy import ( "context" @@ -29,7 +29,7 @@ import ( kubeutilsexpectations "kusionstack.io/kube-utils/controller/expectations" "sigs.k8s.io/controller-runtime/pkg/client" - collasetutils "kusionstack.io/kuperator/pkg/controllers/collaset/utils" + "kusionstack.io/kuperator/pkg/controllers/collaset/utils" refmanagerutil "kusionstack.io/kuperator/pkg/controllers/utils/refmanager" "kusionstack.io/kuperator/pkg/utils/inject" ) @@ -125,7 +125,7 @@ func (pc *RealPvcControl) provisionUpdatedPvc(c client.Client, ctx context.Conte } // create new pvc - claim, err := collasetutils.BuildPvcWithHash(cls, &pvcTmp, id) + claim, err := utils.BuildPvcWithHash(cls, &pvcTmp, id) if err != nil { return nil, err } @@ -133,7 +133,7 @@ func (pc *RealPvcControl) provisionUpdatedPvc(c client.Client, ctx context.Conte if err := c.Create(ctx, claim); err != nil { return nil, fmt.Errorf("fail to create pvc for id %s: %w", id, err) } else { - if err = pc.cacheExpectations.ExpectCreation(kubeutilsclient.ObjectKeyString(cls), collasetutils.PVCGVK, claim.Namespace, claim.Name); err != nil { + if err = pc.cacheExpectations.ExpectCreation(kubeutilsclient.ObjectKeyString(cls), utils.PVCGVK, claim.Namespace, claim.Name); err != nil { return nil, err } } @@ -159,7 +159,7 @@ func (pc *RealPvcControl) DeletePodPvcs(ctx context.Context, cls *appsv1alpha1.C // delete pvcs labeled same id with pod if err := pc.client.Delete(ctx, pvc); err != nil { return err - } else if err := pc.cacheExpectations.ExpectDeletion(kubeutilsclient.ObjectKeyString(cls), collasetutils.PVCGVK, pvc.Namespace, pvc.Name); err != nil { + } else if err := pc.cacheExpectations.ExpectDeletion(kubeutilsclient.ObjectKeyString(cls), utils.PVCGVK, pvc.Namespace, pvc.Name); err != nil { return err } } @@ -193,7 +193,7 @@ func (pc *RealPvcControl) DeletePodUnusedPvcs(ctx context.Context, cls *appsv1al } // delete old pvc if new pvc is provisioned and WhenScaled is "Delete" - if collasetutils.PvcPolicyWhenScaled(cls) == appsv1alpha1.DeletePersistentVolumeClaimRetentionPolicyType { + if utils.PvcPolicyWhenScaled(cls) == appsv1alpha1.DeletePersistentVolumeClaimRetentionPolicyType { return pc.deleteOldPvcs(pc.client, ctx, cls, newPvcs, oldPvcs) } return nil @@ -237,7 +237,7 @@ func (pc *RealPvcControl) AdoptPvc(cls *appsv1alpha1.CollaSet, pvc *corev1.Persi func classifyPodPvcs(cls *appsv1alpha1.CollaSet, id string, existingPvcs []*corev1.PersistentVolumeClaim) (*map[string]*corev1.PersistentVolumeClaim, *map[string]*corev1.PersistentVolumeClaim, error) { newPvcs := map[string]*corev1.PersistentVolumeClaim{} oldPvcs := map[string]*corev1.PersistentVolumeClaim{} - newTmpHash, err := collasetutils.PvcTmpHashMapping(cls.Spec.VolumeClaimTemplates) + newTmpHash, err := utils.PvcTmpHashMapping(cls.Spec.VolumeClaimTemplates) if err != nil { return &newPvcs, &oldPvcs, err } @@ -261,7 +261,7 @@ func classifyPodPvcs(cls *appsv1alpha1.CollaSet, id string, existingPvcs []*core continue } hash := pvc.Labels[appsv1alpha1.PvcTemplateHashLabelKey] - pvcTmpName, err := collasetutils.ExtractPvcTmpName(cls, pvc) + pvcTmpName, err := utils.ExtractPvcTmpName(cls, pvc) if err != nil { return nil, nil, err } @@ -279,7 +279,7 @@ func classifyPodPvcs(cls *appsv1alpha1.CollaSet, id string, existingPvcs []*core func IsPodPvcTmpChanged(cls *appsv1alpha1.CollaSet, pod *corev1.Pod, existingPvcs []*corev1.PersistentVolumeClaim) (bool, error) { // get pvc template hash values - newHashMapping, err := collasetutils.PvcTmpHashMapping(cls.Spec.VolumeClaimTemplates) + newHashMapping, err := utils.PvcTmpHashMapping(cls.Spec.VolumeClaimTemplates) if err != nil { return false, err } @@ -329,7 +329,7 @@ func (pc *RealPvcControl) deleteUnclaimedPvcs(c client.Client, ctx context.Conte } if err := c.Delete(ctx, pvc); err != nil { return err - } else if err := pc.cacheExpectations.ExpectDeletion(kubeutilsclient.ObjectKeyString(cls), collasetutils.PVCGVK, pvc.Namespace, pvc.Name); err != nil { + } else if err := pc.cacheExpectations.ExpectDeletion(kubeutilsclient.ObjectKeyString(cls), utils.PVCGVK, pvc.Namespace, pvc.Name); err != nil { return err } } @@ -343,7 +343,7 @@ func (pc *RealPvcControl) deleteOldPvcs(c client.Client, ctx context.Context, cl } if err := c.Delete(ctx, pvc); err != nil { return err - } else if err := pc.cacheExpectations.ExpectDeletion(kubeutilsclient.ObjectKeyString(cls), collasetutils.PVCGVK, pvc.Namespace, pvc.Name); err != nil { + } else if err := pc.cacheExpectations.ExpectDeletion(kubeutilsclient.ObjectKeyString(cls), utils.PVCGVK, pvc.Namespace, pvc.Name); err != nil { return err } } diff --git a/pkg/controllers/collaset/lifecycle_adapter.go b/pkg/controllers/collaset/lifecycle_adapter.go new file mode 100644 index 00000000..a2db1cbb --- /dev/null +++ b/pkg/controllers/collaset/lifecycle_adapter.go @@ -0,0 +1,99 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package collaset + +import ( + appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" + xsetapi "kusionstack.io/kube-xset/api" + "sigs.k8s.io/controller-runtime/pkg/client" + + "kusionstack.io/kuperator/pkg/controllers/utils/podopslifecycle" +) + +var ( + UpdateOpsLifecycleAdapter = &CollaSetUpdateOpsLifecycleAdapter{} + ScaleInOpsLifecycleAdapter = &CollaSetScaleInOpsLifecycleAdapter{} +) + +// CollaSetUpdateOpsLifecycleAdapter tells PodOpsLifecycle the basic workload update ops info +type CollaSetUpdateOpsLifecycleAdapter struct{} + +// GetID indicates ID of one PodOpsLifecycle +func (a *CollaSetUpdateOpsLifecycleAdapter) GetID() string { + return "collaset" +} + +// GetType indicates type for an Operator +func (a *CollaSetUpdateOpsLifecycleAdapter) GetType() xsetapi.OperationType { + return xsetapi.OpsLifecycleTypeUpdate +} + +// AllowMultiType indicates whether multiple IDs which have the same Type are allowed +func (a *CollaSetUpdateOpsLifecycleAdapter) AllowMultiType() bool { + return true +} + +// WhenBegin will be executed when begin a lifecycle +func (a *CollaSetUpdateOpsLifecycleAdapter) WhenBegin(_ client.Object) (bool, error) { + return false, nil +} + +// WhenFinish will be executed when finish a lifecycle +func (a *CollaSetUpdateOpsLifecycleAdapter) WhenFinish(pod client.Object) (bool, error) { + needUpdated := false + if _, exist := pod.GetLabels()[appsv1alpha1.CollaSetUpdateIndicateLabelKey]; exist { + delete(pod.GetLabels(), appsv1alpha1.CollaSetUpdateIndicateLabelKey) + needUpdated = true + } + + if pod.GetAnnotations() != nil { + if _, exist := pod.GetAnnotations()[appsv1alpha1.LastPodStatusAnnotationKey]; exist { + delete(pod.GetAnnotations(), appsv1alpha1.LastPodStatusAnnotationKey) + needUpdated = true + } + } + + return needUpdated, nil +} + +// CollaSetScaleInOpsLifecycleAdapter tells PodOpsLifecycle the basic workload scaling in ops info +type CollaSetScaleInOpsLifecycleAdapter struct{} + +// GetID indicates ID of one PodOpsLifecycle +func (a *CollaSetScaleInOpsLifecycleAdapter) GetID() string { + return "collaset" +} + +// GetType indicates type for an Operator +func (a *CollaSetScaleInOpsLifecycleAdapter) GetType() xsetapi.OperationType { + return xsetapi.OpsLifecycleTypeScaleIn +} + +// AllowMultiType indicates whether multiple IDs which have the same Type are allowed +func (a *CollaSetScaleInOpsLifecycleAdapter) AllowMultiType() bool { + return true +} + +// WhenBegin will be executed when begin a lifecycle +func (a *CollaSetScaleInOpsLifecycleAdapter) WhenBegin(pod client.Object) (bool, error) { + return podopslifecycle.WhenBeginDelete(pod) +} + +// WhenFinish will be executed when finish a lifecycle +func (a *CollaSetScaleInOpsLifecycleAdapter) WhenFinish(_ client.Object) (bool, error) { + return false, nil +} diff --git a/pkg/controllers/collaset/pod_updater.go b/pkg/controllers/collaset/pod_updater.go new file mode 100644 index 00000000..4808ec44 --- /dev/null +++ b/pkg/controllers/collaset/pod_updater.go @@ -0,0 +1,275 @@ +/* +Copyright 2025 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package collaset + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/util/sets" + appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" + clientutil "kusionstack.io/kube-utils/client" + "kusionstack.io/kube-xset/api" + "kusionstack.io/kube-xset/synccontrols" + "sigs.k8s.io/controller-runtime/pkg/client" + + "kusionstack.io/kuperator/pkg/controllers/collaset/utils" +) + +var _ synccontrols.TargetUpdater = &inPlaceIfPossibleUpdater{} + +type inPlaceIfPossibleUpdater struct { + synccontrols.GenericTargetUpdater +} + +func (u *inPlaceIfPossibleUpdater) FulfillTargetUpdatedInfo(ctx context.Context, revision *appsv1.ControllerRevision, targetUpdateInfo *synccontrols.TargetUpdateInfo) error { + // 1. build target from current and updated revision + // TODO: use cache + currentTarget, err := synccontrols.NewTargetFrom(u.XsetController, u.XsetLabelAnnoMgr, u.OwnerObject, targetUpdateInfo.CurrentRevision, targetUpdateInfo.ID, + func(object client.Object) error { + if decorationAdapter, ok := u.XsetController.(api.DecorationAdapter); ok { + if fn, err := decorationAdapter.GetDecorationPatcherByRevisions(ctx, u.Client, targetUpdateInfo.TargetWrapper.Object, targetUpdateInfo.TargetWrapper.DecorationCurrentRevisions); err != nil { + return err + } else { + return fn(object) + } + } + return nil + }, + ) + if err != nil { + return fmt.Errorf("fail to build Target from current revision %s: %v", targetUpdateInfo.CurrentRevision.GetName(), err.Error()) + } + + // TODO: use cache + targetUpdateInfo.UpdatedTarget, err = synccontrols.NewTargetFrom(u.XsetController, u.XsetLabelAnnoMgr, u.OwnerObject, targetUpdateInfo.UpdateRevision, targetUpdateInfo.ID, + func(object client.Object) error { + if decorationAdapter, ok := u.XsetController.(api.DecorationAdapter); ok { + if fn, err := decorationAdapter.GetDecorationPatcherByRevisions(ctx, u.Client, targetUpdateInfo.TargetWrapper.Object, targetUpdateInfo.TargetWrapper.DecorationUpdatedRevisions); err != nil { + return err + } else { + return fn(object) + } + } + return nil + }, + ) + if err != nil { + return fmt.Errorf("fail to build Target from updated revision %s: %v", targetUpdateInfo.UpdateRevision.GetName(), err.Error()) + } + + if targetUpdateInfo.PvcTmpHashChanged { + targetUpdateInfo.InPlaceUpdateSupport, targetUpdateInfo.OnlyMetadataChanged = false, false + return nil + } + + // TODO check diff + var imageChangedContainers sets.String + targetUpdateInfo.InPlaceUpdateSupport, targetUpdateInfo.OnlyMetadataChanged, imageChangedContainers = u.diffPod(currentTarget, targetUpdateInfo.UpdatedTarget) + + if !targetUpdateInfo.InPlaceUpdateSupport { + return nil + } + + targetUpdateInfo.UpdatedTarget, err = utils.PatchToPod(currentTarget, targetUpdateInfo.UpdatedTarget, targetUpdateInfo.TargetWrapper.Object) + if err != nil { + return err + } + + var podStatus *PodStatus + if targetUpdateInfo.OnlyMetadataChanged { + podStatus = &PodStatus{ContainerStates: nil} + } else { + containerCurrentStatusMapping := map[string]*corev1.ContainerStatus{} + currentPod := targetUpdateInfo.TargetWrapper.Object.(*corev1.Pod) + for i := range currentPod.Status.ContainerStatuses { + status := currentPod.Status.ContainerStatuses[i] + // only store and compare imageID of changed containers + if imageChangedContainers != nil && imageChangedContainers.Has(status.Name) { + containerCurrentStatusMapping[status.Name] = &status + } + } + + podStatus = &PodStatus{ContainerStates: map[string]*ContainerStatus{}} + updatedPod := targetUpdateInfo.UpdatedTarget.(*corev1.Pod) + for _, container := range updatedPod.Spec.Containers { + podStatus.ContainerStates[container.Name] = &ContainerStatus{ + // store image of each container in updated Pod + LatestImage: container.Image, + } + + containerCurrentStatus, exist := containerCurrentStatusMapping[container.Name] + if !exist { + continue + } + + // store image ID of each container in current Pod + podStatus.ContainerStates[container.Name].LastImageID = containerCurrentStatus.ImageID + } + } + + podStatusStr, err := json.Marshal(podStatus) + if err != nil { + return err + } + anno := targetUpdateInfo.UpdatedTarget.GetAnnotations() + if anno == nil { + anno = make(map[string]string) + } + anno[appsv1alpha1.LastPodStatusAnnotationKey] = string(podStatusStr) + targetUpdateInfo.UpdatedTarget.SetAnnotations(anno) + return nil +} + +func (u *inPlaceIfPossibleUpdater) diffPod(currentObj, updatedObj client.Object) (inPlaceSetUpdateSupport, onlyMetadataChanged bool, imageChangedContainers sets.String) { + currentPod := currentObj.(*corev1.Pod) + updatedPod := updatedObj.(*corev1.Pod) + if len(currentPod.Spec.Containers) != len(updatedPod.Spec.Containers) { + return false, false, nil + } + + currentPod = currentPod.DeepCopy() + // sync metadata + currentPod.ObjectMeta = updatedPod.ObjectMeta + + // sync image + imageChanged := false + imageChangedContainers = sets.String{} + for i := range currentPod.Spec.Containers { + if currentPod.Spec.Containers[i].Image != updatedPod.Spec.Containers[i].Image { + imageChanged = true + imageChangedContainers.Insert(currentPod.Spec.Containers[i].Name) + currentPod.Spec.Containers[i].Image = updatedPod.Spec.Containers[i].Image + } + } + + if !equality.Semantic.DeepEqual(currentPod, updatedPod) { + return false, false, nil + } + + if !imageChanged { + return true, true, nil + } + + return true, false, imageChangedContainers +} + +func (u *inPlaceIfPossibleUpdater) UpgradeTarget(ctx context.Context, targetInfo *synccontrols.TargetUpdateInfo) error { + if targetInfo.OnlyMetadataChanged || targetInfo.InPlaceUpdateSupport { + // if target template changes only include metadata or support in-place update, just apply these changes to target directly + if err := u.TargetControl.UpdateTarget(ctx, targetInfo.UpdatedTarget); err != nil { + return fmt.Errorf("fail to update Target %s/%s when updating by in-place: %s", targetInfo.GetNamespace(), targetInfo.GetName(), err.Error()) + } + targetInfo.Object = targetInfo.UpdatedTarget + u.Recorder.Eventf(targetInfo.Object, + corev1.EventTypeNormal, + "UpdateTarget", + "succeed to update Target %s/%s to from revision %s to revision %s by in-place", + targetInfo.GetNamespace(), targetInfo.GetName(), + targetInfo.CurrentRevision.GetName(), + targetInfo.UpdateRevision.GetName()) + return u.CacheExpectations.ExpectUpdation(clientutil.ObjectKeyString(u.OwnerObject), u.TargetGVK, targetInfo.Object.GetNamespace(), targetInfo.Object.GetName(), targetInfo.Object.GetResourceVersion()) + } else { + // if target has changes not in-place supported, recreate it + if err := u.GenericTargetUpdater.RecreateTarget(ctx, targetInfo); err != nil { + return err + } + return u.CacheExpectations.ExpectDeletion(clientutil.ObjectKeyString(u.OwnerObject), u.TargetGVK, targetInfo.Object.GetNamespace(), targetInfo.Object.GetName()) + } +} + +func (u *inPlaceIfPossibleUpdater) GetTargetUpdateFinishStatus(_ context.Context, targetUpdateInfo *synccontrols.TargetUpdateInfo) (finished bool, msg string, err error) { + if !targetUpdateInfo.IsUpdatedRevision || targetUpdateInfo.DecorationChanged { + return false, "add on not updated", nil + } + + pod := targetUpdateInfo.TargetWrapper.Object.(*corev1.Pod) + if pod.Status.ContainerStatuses == nil { + return false, "no container status", nil + } + + if pod.Spec.Containers == nil { + return false, "no container spec", nil + } + + if len(pod.Spec.Containers) != len(pod.Status.ContainerStatuses) { + return false, "container status number does not match", nil + } + + if targetUpdateInfo.GetAnnotations() == nil { + return false, "no annotations for last container status", nil + } + + targetLastState := &PodStatus{} + if lastStateJson, exist := targetUpdateInfo.GetAnnotations()[appsv1alpha1.LastPodStatusAnnotationKey]; !exist { + return false, "no pod last state annotation", nil + } else if err := json.Unmarshal([]byte(lastStateJson), targetLastState); err != nil { + msg := fmt.Sprintf("malformat target last state annotation [%s]: %s", lastStateJson, err.Error()) + return false, msg, errors.New(msg) + } + + if targetLastState.ContainerStates == nil { + return true, "pod updated with only metadata changed", nil + } + + imageMapping := map[string]string{} + for _, containerSpec := range pod.Spec.Containers { + imageMapping[containerSpec.Name] = containerSpec.Image + } + + imageIdMapping := map[string]string{} + for _, containerStatus := range pod.Status.ContainerStatuses { + imageIdMapping[containerStatus.Name] = containerStatus.ImageID + } + + for containerName, lastContainerState := range targetLastState.ContainerStates { + latestImage := lastContainerState.LatestImage + lastImageId := lastContainerState.LastImageID + + if currentImage, exist := imageMapping[containerName]; !exist { + // If no this container image recorded, ignore this container. + continue + } else if currentImage != latestImage { + // If container image in pod spec has changed, ignore this container. + continue + } + + if currentImageId, exist := imageIdMapping[containerName]; !exist { + // If no this container image id recorded, ignore this container. + continue + } else if currentImageId == lastImageId { + // No image id changed means the pod in-place update has not finished by kubelet. + return false, fmt.Sprintf("container has %s not been updated: last image id %s, current image id %s", containerName, lastImageId, currentImageId), nil + } + } + + return true, "", nil +} + +type PodStatus struct { + ContainerStates map[string]*ContainerStatus `json:"containerStates,omitempty"` +} + +type ContainerStatus struct { + LatestImage string `json:"latestImage,omitempty"` + LastImageID string `json:"lastImageID,omitempty"` +} diff --git a/pkg/controllers/collaset/podcontext/podcontext.go b/pkg/controllers/collaset/podcontext/podcontext.go deleted file mode 100644 index 0e5f6460..00000000 --- a/pkg/controllers/collaset/podcontext/podcontext.go +++ /dev/null @@ -1,230 +0,0 @@ -/* -Copyright 2023 The KusionStack Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package podcontext - -import ( - "context" - "fmt" - "sort" - - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" - kubeutilsclient "kusionstack.io/kube-utils/client" - kubeutilsexpectations "kusionstack.io/kube-utils/controller/expectations" - "sigs.k8s.io/controller-runtime/pkg/client" - - collasetutils "kusionstack.io/kuperator/pkg/controllers/collaset/utils" -) - -const ( - OwnerContextKey = "Owner" - RevisionContextDataKey = "Revision" - PodDecorationRevisionKey = "PodDecorationRevisions" - JustCreateContextDataKey = "PodJustCreate" - RecreateUpdateContextDataKey = "PodRecreateUpdate" -) - -type Interface interface { - AllocateID(ctx context.Context, instance *appsv1alpha1.CollaSet, defaultRevision string, replicas int) (map[int]*appsv1alpha1.ContextDetail, error) - UpdateToPodContext(ctx context.Context, instance *appsv1alpha1.CollaSet, ownedIDs map[int]*appsv1alpha1.ContextDetail) error -} - -type RealPodContextControl struct { - client.Client - cacheExpectations kubeutilsexpectations.CacheExpectationsInterface -} - -func NewRealPodContextControl(c client.Client, cacheExpectations kubeutilsexpectations.CacheExpectationsInterface) Interface { - return &RealPodContextControl{ - Client: c, - cacheExpectations: cacheExpectations, - } -} - -func (r *RealPodContextControl) AllocateID(ctx context.Context, instance *appsv1alpha1.CollaSet, defaultRevision string, replicas int) (map[int]*appsv1alpha1.ContextDetail, error) { - contextName := getContextName(instance) - podContext := &appsv1alpha1.ResourceContext{} - notFound := false - if err := r.Client.Get(ctx, types.NamespacedName{Namespace: instance.Namespace, Name: contextName}, podContext); err != nil { - if !errors.IsNotFound(err) { - return nil, fmt.Errorf("fail to find ResourceContext %s/%s for owner %s: %w", instance.Namespace, contextName, instance.Name, err) - } - - notFound = true - podContext.Namespace = instance.Namespace - podContext.Name = contextName - } - - // store all the IDs crossing Multiple workload - existingIDs := map[int]*appsv1alpha1.ContextDetail{} - // only store the IDs belonging to this owner - ownedIDs := map[int]*appsv1alpha1.ContextDetail{} - for i := range podContext.Spec.Contexts { - detail := &podContext.Spec.Contexts[i] - if detail.Contains(OwnerContextKey, instance.Name) { - ownedIDs[detail.ID] = detail - existingIDs[detail.ID] = detail - } else if instance.Spec.ScaleStrategy.Context != "" { - // add other collaset podContexts only if context pool enabled - existingIDs[detail.ID] = detail - } - } - - // if owner has enough ID, return - if len(ownedIDs) >= replicas { - return ownedIDs, nil - } - - // find new IDs for owner - candidateID := 0 - for len(ownedIDs) < replicas { - // find one new ID - for { - if _, exist := existingIDs[candidateID]; exist { - candidateID++ - continue - } - - break - } - - detail := &appsv1alpha1.ContextDetail{ - ID: candidateID, - // TODO choose just create pods' revision according to scaleStrategy - Data: map[string]string{ - OwnerContextKey: instance.Name, - RevisionContextDataKey: defaultRevision, - JustCreateContextDataKey: "true", - }, - } - existingIDs[candidateID] = detail - ownedIDs[candidateID] = detail - } - - if notFound { - return ownedIDs, r.doCreatePodContext(ctx, instance, ownedIDs) - } - - return ownedIDs, r.doUpdatePodContext(ctx, instance, ownedIDs, podContext) -} - -func (r *RealPodContextControl) UpdateToPodContext(ctx context.Context, instance *appsv1alpha1.CollaSet, ownedIDs map[int]*appsv1alpha1.ContextDetail) error { - contextName := getContextName(instance) - podContext := &appsv1alpha1.ResourceContext{} - if err := r.Client.Get(context.TODO(), types.NamespacedName{Namespace: instance.Namespace, Name: contextName}, podContext); err != nil { - if !errors.IsNotFound(err) { - return fmt.Errorf("fail to find ResourceContext %s/%s: %w", instance.Namespace, contextName, err) - } - - if len(ownedIDs) == 0 { - return nil - } - - if err := r.doCreatePodContext(ctx, instance, ownedIDs); err != nil { - return fmt.Errorf("fail to create ResourceContext %s/%s after not found: %w", instance.Namespace, contextName, err) - } - } - - return r.doUpdatePodContext(ctx, instance, ownedIDs, podContext) -} - -func (r *RealPodContextControl) doCreatePodContext(ctx context.Context, instance *appsv1alpha1.CollaSet, ownerIDs map[int]*appsv1alpha1.ContextDetail) error { - contextName := getContextName(instance) - podContext := &appsv1alpha1.ResourceContext{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: instance.Namespace, - Name: contextName, - }, - Spec: appsv1alpha1.ResourceContextSpec{ - Contexts: make([]appsv1alpha1.ContextDetail, len(ownerIDs)), - }, - } - - i := 0 - for _, detail := range ownerIDs { - podContext.Spec.Contexts[i] = *detail - i++ - } - - return r.Client.Create(ctx, podContext) -} - -func (r *RealPodContextControl) doUpdatePodContext(ctx context.Context, instance client.Object, ownedIDs map[int]*appsv1alpha1.ContextDetail, podContext *appsv1alpha1.ResourceContext) error { - // store all IDs crossing all workload - existingIDs := map[int]*appsv1alpha1.ContextDetail{} - - // add other collaset podContexts only if context pool enabled - cls := instance.(*appsv1alpha1.CollaSet) - if cls.Spec.ScaleStrategy.Context != "" { - for i := range podContext.Spec.Contexts { - detail := podContext.Spec.Contexts[i] - if detail.Contains(OwnerContextKey, instance.GetName()) { - continue - } - existingIDs[detail.ID] = &detail - } - } - - for _, contextDetail := range ownedIDs { - existingIDs[contextDetail.ID] = contextDetail - } - - // delete PodContext if it is empty - if len(existingIDs) == 0 { - err := r.Client.Delete(context.TODO(), podContext) - if err != nil { - return err - } - return r.cacheExpectations.ExpectDeletion(kubeutilsclient.ObjectKeyString(instance), collasetutils.ResourceContextGVK, podContext.Namespace, podContext.Name) - } - - podContext.Spec.Contexts = make([]appsv1alpha1.ContextDetail, len(existingIDs)) - idx := 0 - for _, contextDetail := range existingIDs { - podContext.Spec.Contexts[idx] = *contextDetail - idx++ - } - - // keep context detail in order by ID - sort.Sort(ContextDetailsByOrder(podContext.Spec.Contexts)) - err := r.Client.Update(ctx, podContext) - if err != nil { - return err - } - - return r.cacheExpectations.ExpectUpdation(kubeutilsclient.ObjectKeyString(instance), collasetutils.ResourceContextGVK, podContext.Namespace, podContext.Name, podContext.ResourceVersion) -} - -func getContextName(instance *appsv1alpha1.CollaSet) string { - if instance.Spec.ScaleStrategy.Context != "" { - return instance.Spec.ScaleStrategy.Context - } - - return instance.Name -} - -type ContextDetailsByOrder []appsv1alpha1.ContextDetail - -func (s ContextDetailsByOrder) Len() int { return len(s) } -func (s ContextDetailsByOrder) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -func (s ContextDetailsByOrder) Less(i, j int) bool { - l, r := s[i], s[j] - return l.ID < r.ID -} diff --git a/pkg/controllers/collaset/podcontext/podcontext_test.go b/pkg/controllers/collaset/podcontext/podcontext_test.go deleted file mode 100644 index b50e5596..00000000 --- a/pkg/controllers/collaset/podcontext/podcontext_test.go +++ /dev/null @@ -1,115 +0,0 @@ -/* -Copyright 2023 The KusionStack Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package podcontext - -import ( - "context" - "testing" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/clock" - appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" - kubeutilsexpectations "kusionstack.io/kube-utils/controller/expectations" - "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -func init() { - corev1.AddToScheme(scheme) - appsv1alpha1.AddToScheme(scheme) -} - -var scheme = runtime.NewScheme() - -var _ = Describe("ResourceContext allocation", func() { - It("allocate ID", func() { - c := fake.NewClientBuilder().WithScheme(scheme).Build() - rpc := NewRealPodContextControl(c, kubeutilsexpectations.NewxCacheExpectations(c, scheme, clock.RealClock{})) - namespace := "test" - - instance1 := &appsv1alpha1.CollaSet{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: "foo1", - }, - Spec: appsv1alpha1.CollaSetSpec{ - ScaleStrategy: appsv1alpha1.ScaleStrategy{ - Context: "foo", // use the same Context - }, - }, - } - - instance2 := &appsv1alpha1.CollaSet{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: "foo2", - }, - Spec: appsv1alpha1.CollaSetSpec{ - ScaleStrategy: appsv1alpha1.ScaleStrategy{ - Context: "foo", // use the same Context - }, - }, - } - - ownedIDs, err := rpc.AllocateID(context.TODO(), instance1, "", 10) - Expect(err).Should(BeNil()) - Expect(len(ownedIDs)).Should(BeEquivalentTo(10)) - for i := 0; i < 10; i++ { - _, exist := ownedIDs[i] - Expect(exist).Should(BeTrue()) - } - - ownedIDs, err = rpc.AllocateID(context.TODO(), instance2, "", 4) - Expect(err).Should(BeNil()) - Expect(len(ownedIDs)).Should(BeEquivalentTo(4)) - for i := 10; i < 14; i++ { - _, exist := ownedIDs[i] - Expect(exist).Should(BeTrue()) - } - - ownedIDs, err = rpc.AllocateID(context.TODO(), instance1, "", 10) - Expect(err).NotTo(HaveOccurred()) - delete(ownedIDs, 4) - delete(ownedIDs, 6) - Expect(rpc.UpdateToPodContext(context.TODO(), instance1, ownedIDs)).Should(BeNil()) - - ownedIDs, err = rpc.AllocateID(context.TODO(), instance2, "", 7) - Expect(err).Should(BeNil()) - Expect(len(ownedIDs)).Should(BeEquivalentTo(7)) - for _, i := range []int{4, 6, 10, 11, 12, 13, 14} { - _, exist := ownedIDs[i] - Expect(exist).Should(BeTrue()) - } - - ownedIDs, err = rpc.AllocateID(context.TODO(), instance1, "", 12) - Expect(err).Should(BeNil()) - Expect(len(ownedIDs)).Should(BeEquivalentTo(12)) - for _, i := range []int{0, 1, 2, 3, 5, 7, 8, 9, 15, 16, 17, 18} { - _, exist := ownedIDs[i] - Expect(exist).Should(BeTrue()) - } - }) -}) - -func TestPodContext(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "ResourceContext Test Suite") -} diff --git a/pkg/controllers/collaset/podcontrol/pod_control.go b/pkg/controllers/collaset/podcontrol/pod_control.go deleted file mode 100644 index 1ecbfc90..00000000 --- a/pkg/controllers/collaset/podcontrol/pod_control.go +++ /dev/null @@ -1,173 +0,0 @@ -/* -Copyright 2023 The KusionStack Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package podcontrol - -import ( - "context" - "fmt" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/runtime" - appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" - "sigs.k8s.io/controller-runtime/pkg/client" - - refmanagerutil "kusionstack.io/kuperator/pkg/controllers/utils/refmanager" - "kusionstack.io/kuperator/pkg/utils/inject" -) - -type Interface interface { - GetFilteredPods(selector *metav1.LabelSelector, owner client.Object) ([]*corev1.Pod, []*corev1.Pod, error) - CreatePod(pod *corev1.Pod) (*corev1.Pod, error) - DeletePod(pod *corev1.Pod) error - UpdatePod(pod *corev1.Pod) error - PatchPod(pod *corev1.Pod, patch client.Patch) error - OrphanPod(cls *appsv1alpha1.CollaSet, pod *corev1.Pod) error - AdoptPod(cls *appsv1alpha1.CollaSet, pod *corev1.Pod) error -} - -func NewRealPodControl(client client.Client, scheme *runtime.Scheme) Interface { - return &RealPodControl{ - client: client, - scheme: scheme, - } -} - -type RealPodControl struct { - client client.Client - scheme *runtime.Scheme -} - -// GetFilteredPods returns filtered pods and all pods -func (pc *RealPodControl) GetFilteredPods(selector *metav1.LabelSelector, owner client.Object) ([]*corev1.Pod, []*corev1.Pod, error) { - // get the pods with ownerReference points to this CollaSet - podList := &corev1.PodList{} - err := pc.client.List(context.TODO(), podList, &client.ListOptions{Namespace: owner.GetNamespace(), FieldSelector: fields.OneTermEqualSelector(inject.FieldIndexOwnerRefUID, string(owner.GetUID()))}) - if err != nil { - return nil, nil, err - } - - var allPods []*corev1.Pod - for i := range podList.Items { - allPods = append(allPods, &podList.Items[i]) - } - - filteredPods := FilterOutInactivePod(podList.Items) - filteredPods, err = pc.getPodSetPods(filteredPods, selector, owner) - if err != nil { - return nil, nil, err - } - - return filteredPods, allPods, nil -} - -func (pc *RealPodControl) CreatePod(pod *corev1.Pod) (*corev1.Pod, error) { - if err := pc.client.Create(context.TODO(), pod); err != nil { - return nil, fmt.Errorf("fail to create Pod: %w", err) - } - - return pod, nil -} - -func (pc *RealPodControl) DeletePod(pod *corev1.Pod) error { - return pc.client.Delete(context.TODO(), pod) -} - -func (pc *RealPodControl) UpdatePod(pod *corev1.Pod) error { - return pc.client.Update(context.TODO(), pod) -} - -func (pc *RealPodControl) PatchPod(pod *corev1.Pod, patch client.Patch) error { - return pc.client.Patch(context.TODO(), pod, patch) -} - -func (pc *RealPodControl) OrphanPod(cls *appsv1alpha1.CollaSet, pod *corev1.Pod) error { - if cls.Spec.Selector.MatchLabels == nil { - return nil - } - cm, err := refmanagerutil.NewRefManager(pc.client, cls.Spec.Selector, cls, pc.scheme) - if err != nil { - return fmt.Errorf("fail to create ref manager: %w", err) - } - - if pod.Labels == nil { - pod.Labels = make(map[string]string) - } - if pod.Annotations == nil { - pod.Annotations = make(map[string]string) - } - return cm.Release(pod) -} - -func (pc *RealPodControl) AdoptPod(cls *appsv1alpha1.CollaSet, pod *corev1.Pod) error { - if cls.Spec.Selector.MatchLabels == nil { - return nil - } - cm, err := refmanagerutil.NewRefManager(pc.client, cls.Spec.Selector, cls, pc.scheme) - if err != nil { - return fmt.Errorf("fail to create ref manager: %w", err) - } - - _, err = cm.ClaimOwned([]client.Object{pod}) - if err != nil { - return err - } - return nil -} - -func (pc *RealPodControl) getPodSetPods(pods []*corev1.Pod, selector *metav1.LabelSelector, owner client.Object) ([]*corev1.Pod, error) { - // Use ControllerRefManager to adopt/orphan as needed. - cm, err := refmanagerutil.NewRefManager(pc.client, selector, owner, pc.scheme) - if err != nil { - return nil, fmt.Errorf("fail to create ref manager: %w", err) - } - - candidates := make([]client.Object, len(pods)) - for i, pod := range pods { - candidates[i] = pod - } - - claims, err := cm.ClaimOwned(candidates) - if err != nil { - return nil, err - } - - claimPods := make([]*corev1.Pod, len(claims)) - for i, mt := range claims { - claimPods[i] = mt.(*corev1.Pod) - } - - return claimPods, nil -} - -func FilterOutInactivePod(pods []corev1.Pod) []*corev1.Pod { - var filteredPod []*corev1.Pod - - for i := range pods { - if IsPodInactive(&pods[i]) { - continue - } - - filteredPod = append(filteredPod, &pods[i]) - } - return filteredPod -} - -func IsPodInactive(pod *corev1.Pod) bool { - return pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed -} diff --git a/pkg/controllers/collaset/predicts.go b/pkg/controllers/collaset/predicts.go deleted file mode 100644 index 4ac0a2d2..00000000 --- a/pkg/controllers/collaset/predicts.go +++ /dev/null @@ -1,45 +0,0 @@ -/* -Copyright 2023 The KusionStack Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package collaset - -import ( - "sigs.k8s.io/controller-runtime/pkg/event" - - "kusionstack.io/kuperator/pkg/utils" -) - -type PodPredicate struct{} - -// Create returns true if the Create event should be processed -func (p *PodPredicate) Create(e event.CreateEvent) bool { - return utils.ControlledByKusionStack(e.Object) -} - -// Delete returns true if the Delete event should be processed -func (p *PodPredicate) Delete(e event.DeleteEvent) bool { - return utils.ControlledByKusionStack(e.Object) -} - -// Update returns true if the Update event should be processed -func (p *PodPredicate) Update(e event.UpdateEvent) bool { - return utils.ControlledByKusionStack(e.ObjectNew) || utils.ControlledByKusionStack(e.ObjectOld) -} - -// Generic returns true if the Generic event should be processed -func (p *PodPredicate) Generic(e event.GenericEvent) bool { - return utils.ControlledByKusionStack(e.Object) -} diff --git a/pkg/controllers/collaset/pvc_adapter.go b/pkg/controllers/collaset/pvc_adapter.go new file mode 100644 index 00000000..17037b7f --- /dev/null +++ b/pkg/controllers/collaset/pvc_adapter.go @@ -0,0 +1,70 @@ +/* +Copyright 2025 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package collaset + +import ( + corev1 "k8s.io/api/core/v1" + appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" + xsetapi "kusionstack.io/kube-xset/api" + "sigs.k8s.io/controller-runtime/pkg/client" + + "kusionstack.io/kuperator/pkg/controllers/collaset/utils" +) + +type SubResourcePvcAdapter struct{} + +// RetainPvcWhenXSetDeleted returns true if pvc should be retained when XSet is deleted. +func (p *SubResourcePvcAdapter) RetainPvcWhenXSetDeleted(object xsetapi.XSetObject) bool { + cls := object.(*appsv1alpha1.CollaSet) + return utils.PvcPolicyWhenDelete(cls) == appsv1alpha1.RetainPersistentVolumeClaimRetentionPolicyType +} + +// RetainPvcWhenXSetScaled returns true if pvc should be retained when XSet replicas is scaledIn. +func (p *SubResourcePvcAdapter) RetainPvcWhenXSetScaled(object xsetapi.XSetObject) bool { + cls := object.(*appsv1alpha1.CollaSet) + return utils.PvcPolicyWhenScaled(cls) == appsv1alpha1.RetainPersistentVolumeClaimRetentionPolicyType +} + +// GetXSetPvcTemplate returns pvc template from XSet object. +func (p *SubResourcePvcAdapter) GetXSetPvcTemplate(object xsetapi.XSetObject) []corev1.PersistentVolumeClaim { + cls := object.(*appsv1alpha1.CollaSet) + return cls.Spec.VolumeClaimTemplates +} + +// GetXSpecVolumes returns spec.volumes from X object. +func (p *SubResourcePvcAdapter) GetXSpecVolumes(object client.Object) []corev1.Volume { + pod := object.(*corev1.Pod) + return pod.Spec.Volumes +} + +// GetXVolumeMounts returns containers volumeMounts from X (pod) object. +func (p *SubResourcePvcAdapter) GetXVolumeMounts(object client.Object) []corev1.VolumeMount { + pod := object.(*corev1.Pod) + var volumeMounts []corev1.VolumeMount + for _, container := range pod.Spec.Containers { + for _, v := range container.VolumeMounts { + volumeMounts = append(volumeMounts, v) + } + } + return volumeMounts +} + +// SetXSpecVolumes sets spec.volumes to X object. +func (p *SubResourcePvcAdapter) SetXSpecVolumes(object client.Object, pvcs []corev1.Volume) { + pod := object.(*corev1.Pod) + pod.Spec.Volumes = pvcs +} diff --git a/pkg/controllers/collaset/resourcecontext_adapter.go b/pkg/controllers/collaset/resourcecontext_adapter.go new file mode 100644 index 00000000..65e1a59c --- /dev/null +++ b/pkg/controllers/collaset/resourcecontext_adapter.go @@ -0,0 +1,81 @@ +/* +Copyright 2025 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package collaset + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" + xsetapi "kusionstack.io/kube-xset/api" + + "kusionstack.io/kuperator/pkg/controllers/collaset/legacy" +) + +var _ xsetapi.ResourceContextAdapter = &ResourceContextAdapter{} + +var defaultResourceContextKeys = map[xsetapi.ResourceContextKeyEnum]string{ + xsetapi.EnumOwnerContextKey: legacy.OwnerContextKey, + xsetapi.EnumRevisionContextDataKey: legacy.RevisionContextDataKey, + xsetapi.EnumTargetDecorationRevisionKey: legacy.PodDecorationRevisionKey, + xsetapi.EnumJustCreateContextDataKey: legacy.JustCreateContextDataKey, + xsetapi.EnumRecreateUpdateContextDataKey: legacy.RecreateUpdateContextDataKey, + xsetapi.EnumScaleInContextDataKey: legacy.ScaleInContextDataKey, + xsetapi.EnumReplaceNewTargetIDContextDataKey: legacy.ReplaceNewPodIDContextDataKey, + xsetapi.EnumReplaceOriginTargetIDContextDataKey: legacy.ReplaceOriginPodIDContextDataKey, +} + +// ResourceContextAdapter is the adapter to xsetapi apps.kusionstack.io.resourcecontexts +type ResourceContextAdapter struct{} + +func (*ResourceContextAdapter) ResourceContextMeta() metav1.TypeMeta { + return metav1.TypeMeta{APIVersion: appsv1alpha1.SchemeGroupVersion.String(), Kind: "ResourceContext"} +} + +func (*ResourceContextAdapter) GetResourceContextSpec(object xsetapi.ResourceContextObject) *xsetapi.ResourceContextSpec { + rc := object.(*appsv1alpha1.ResourceContext) + var contexts []xsetapi.ContextDetail + for i := range rc.Spec.Contexts { + c := rc.Spec.Contexts[i] + contexts = append(contexts, xsetapi.ContextDetail{ + ID: c.ID, + Data: c.Data, + }) + } + return &xsetapi.ResourceContextSpec{ + Contexts: contexts, + } +} + +func (*ResourceContextAdapter) SetResourceContextSpec(spec *xsetapi.ResourceContextSpec, object xsetapi.ResourceContextObject) { + rc := object.(*appsv1alpha1.ResourceContext) + var contexts []appsv1alpha1.ContextDetail + for i := range spec.Contexts { + c := spec.Contexts[i] + contexts = append(contexts, appsv1alpha1.ContextDetail{ + ID: c.ID, + Data: c.Data, + }) + } + rc.Spec.Contexts = contexts +} + +func (*ResourceContextAdapter) GetContextKeys() map[xsetapi.ResourceContextKeyEnum]string { + return defaultResourceContextKeys +} + +func (*ResourceContextAdapter) NewResourceContext() xsetapi.ResourceContextObject { + return &appsv1alpha1.ResourceContext{} +} diff --git a/pkg/controllers/collaset/revision.go b/pkg/controllers/collaset/revision.go deleted file mode 100644 index 8dc5e433..00000000 --- a/pkg/controllers/collaset/revision.go +++ /dev/null @@ -1,114 +0,0 @@ -/* -Copyright 2023 The KusionStack Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package collaset - -import ( - "encoding/json" - - appsv1 "k8s.io/api/apps/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/util/sets" - appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" - - "kusionstack.io/kuperator/pkg/controllers/collaset/podcontrol" -) - -func getCollaSetPatch(cls *appsv1alpha1.CollaSet) ([]byte, error) { - dsBytes, err := json.Marshal(cls) - if err != nil { - return nil, err - } - var raw map[string]interface{} - err = json.Unmarshal(dsBytes, &raw) - if err != nil { - return nil, err - } - objCopy := make(map[string]interface{}) - specCopy := make(map[string]interface{}) - - // Create a patch of the CollaSet that replaces spec.template - spec := raw["spec"].(map[string]interface{}) - template := spec["template"].(map[string]interface{}) - template["$patch"] = "replace" - specCopy["template"] = template - - if _, exist := spec["volumeClaimTemplates"]; exist { - specCopy["volumeClaimTemplates"] = spec["volumeClaimTemplates"] - } - - objCopy["spec"] = specCopy - patch, err := json.Marshal(objCopy) - return patch, err -} - -type revisionOwnerAdapter struct { - podControl podcontrol.Interface -} - -func (roa *revisionOwnerAdapter) GetGroupVersionKind() schema.GroupVersionKind { - return appsv1alpha1.SchemeGroupVersion.WithKind("CollaSet") -} - -func (roa *revisionOwnerAdapter) GetMatchLabels(obj metav1.Object) map[string]string { - cls, _ := obj.(*appsv1alpha1.CollaSet) - selector := cls.Spec.Selector - if selector == nil { - return nil - } - return selector.MatchLabels -} - -func (roa *revisionOwnerAdapter) GetCollisionCount(obj metav1.Object) *int32 { - cls, _ := obj.(*appsv1alpha1.CollaSet) - return cls.Status.CollisionCount -} - -func (roa *revisionOwnerAdapter) GetHistoryLimit(obj metav1.Object) int32 { - cls, _ := obj.(*appsv1alpha1.CollaSet) - return cls.Spec.HistoryLimit -} - -func (roa *revisionOwnerAdapter) GetPatch(obj metav1.Object) ([]byte, error) { - cs, _ := obj.(*appsv1alpha1.CollaSet) - return getCollaSetPatch(cs) -} - -func (roa *revisionOwnerAdapter) GetCurrentRevision(obj metav1.Object) string { - cls, _ := obj.(*appsv1alpha1.CollaSet) - return cls.Status.CurrentRevision -} - -func (roa *revisionOwnerAdapter) GetInUsedRevisions(obj metav1.Object) (sets.String, error) { - inUsed := sets.NewString() - cls, _ := obj.(*appsv1alpha1.CollaSet) - if cls.Status.UpdatedRevision != "" { - inUsed.Insert(cls.Status.UpdatedRevision) - } - if cls.Status.CurrentRevision != "" { - inUsed.Insert(cls.Status.CurrentRevision) - } - filteredPods, _, _ := roa.podControl.GetFilteredPods(cls.Spec.Selector, cls) - for _, pod := range filteredPods { - if pod.Labels != nil { - if currentRevisionName, exist := pod.Labels[appsv1.ControllerRevisionHashLabelKey]; exist { - inUsed.Insert(currentRevisionName) - } - } - } - return inUsed, nil -} diff --git a/pkg/controllers/collaset/synccontrol/replace.go b/pkg/controllers/collaset/synccontrol/replace.go deleted file mode 100644 index 4f039a02..00000000 --- a/pkg/controllers/collaset/synccontrol/replace.go +++ /dev/null @@ -1,444 +0,0 @@ -/* -Copyright 2024 The KusionStack Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package synccontrol - -import ( - "context" - "encoding/json" - "fmt" - "strconv" - "strings" - "time" - - "github.com/go-logr/logr" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/client-go/tools/record" - appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" - kubeutilsclient "kusionstack.io/kube-utils/client" - "sigs.k8s.io/controller-runtime/pkg/client" - - "kusionstack.io/kuperator/pkg/controllers/collaset/podcontext" - collasetutils "kusionstack.io/kuperator/pkg/controllers/collaset/utils" - controllerutils "kusionstack.io/kuperator/pkg/controllers/utils" - utilspoddecoration "kusionstack.io/kuperator/pkg/controllers/utils/poddecoration" - "kusionstack.io/kuperator/pkg/controllers/utils/podopslifecycle" -) - -const ( - ReplaceNewPodIDContextDataKey = "ReplaceNewPodID" - ReplaceOriginPodIDContextDataKey = "ReplaceOriginPodID" -) - -func (r *RealSyncControl) cleanReplacePodLabels( - needCleanLabelPods []*corev1.Pod, - podsNeedCleanLabels [][]string, - ownedIDs map[int]*appsv1alpha1.ContextDetail, - currentIDs map[int]struct{}, - logger logr.Logger) ( - bool, sets.Int, error, -) { - needUpdateContext := false - needDeletePodsIDs := sets.Int{} - mapOriginToNewPodContext := mapReplaceOriginToNewPodContext(ownedIDs) - mapNewToOriginPodContext := mapReplaceNewToOriginPodContext(ownedIDs) - _, err := controllerutils.SlowStartBatch(len(needCleanLabelPods), controllerutils.SlowStartInitialBatchSize, false, func(i int, err error) error { - defer func() { - if err == nil { - logger.Info("cleanReplacePodLabels clean replace labels success", "pod", needCleanLabelPods[i].Name, "labels", podsNeedCleanLabels[i]) - } - }() - - pod := needCleanLabelPods[i] - needCleanLabels := podsNeedCleanLabels[i] - var deletePatch []map[string]string - for _, labelKey := range needCleanLabels { - patchOperation := map[string]string{ - "op": "remove", - "path": fmt.Sprintf("/metadata/labels/%s", strings.ReplaceAll(labelKey, "/", "~1")), - } - deletePatch = append(deletePatch, patchOperation) - // replace finished, (1) remove ReplaceNewPodID, ReplaceOriginPodID key from IDs, (2) try to delete origin Pod's ID - if labelKey == appsv1alpha1.PodReplacePairOriginName { - needUpdateContext = true - newPodId, _ := collasetutils.GetPodInstanceID(pod) - if originPodContext, exist := mapOriginToNewPodContext[newPodId]; exist && originPodContext != nil { - originPodContext.Remove(ReplaceNewPodIDContextDataKey) - if _, exist := currentIDs[originPodContext.ID]; !exist { - needDeletePodsIDs.Insert(originPodContext.ID) - } - } - if contextDetail, exist := ownedIDs[newPodId]; exist { - contextDetail.Remove(ReplaceOriginPodIDContextDataKey) - } - } - // replace canceled, (1) remove ReplaceNewPodID, ReplaceOriginPodID key from IDs, (2) try to delete new Pod's ID - _, replaceIndicate := pod.Labels[appsv1alpha1.PodReplaceIndicationLabelKey] - if !replaceIndicate && labelKey == appsv1alpha1.PodReplacePairNewId { - needUpdateContext = true - originPodId, _ := collasetutils.GetPodInstanceID(pod) - if newPodContext, exist := mapNewToOriginPodContext[originPodId]; exist && newPodContext != nil { - newPodContext.Remove(ReplaceOriginPodIDContextDataKey) - if _, exist := currentIDs[newPodContext.ID]; !exist { - needDeletePodsIDs.Insert(newPodContext.ID) - } - } - if contextDetail, exist := ownedIDs[originPodId]; exist { - contextDetail.Remove(ReplaceNewPodIDContextDataKey) - } - } - } - // patch to bytes - patchBytes, err := json.Marshal(deletePatch) - if err != nil { - return err - } - if err = r.podControl.PatchPod(pod, client.RawPatch(types.JSONPatchType, patchBytes)); err != nil { - return fmt.Errorf("failed to remove replace pair label %s/%s: %w", pod.Namespace, pod.Name, err) - } - return nil - }) - - return needUpdateContext, needDeletePodsIDs, err -} - -func (r *RealSyncControl) replaceOriginPods( - ctx context.Context, - instance *appsv1alpha1.CollaSet, - resources *collasetutils.RelatedResources, - needReplaceOriginPods []*corev1.Pod, - ownedIDs map[int]*appsv1alpha1.ContextDetail, - availableContexts []*appsv1alpha1.ContextDetail, - logger logr.Logger, -) (int, error) { - mapNewToOriginPodContext := mapReplaceNewToOriginPodContext(ownedIDs) - successCount, err := controllerutils.SlowStartBatch(len(needReplaceOriginPods), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) error { - originPod := needReplaceOriginPods[i] - originPodId, _ := collasetutils.GetPodInstanceID(originPod) - if ownedIDs[originPodId] == nil { - r.recorder.Eventf(originPod, corev1.EventTypeWarning, "OriginPodContext", "cannot found resource context id %d of origin pod %s/%s", originPodId, originPod.Namespace, originPod.Name) - return fmt.Errorf("cannot found context for replace origin pod %s/%s", originPod.Namespace, originPod.Name) - } - - ownerRef := metav1.NewControllerRef(instance, instance.GroupVersionKind()) - updatedPDs, err := resources.PDGetter.GetEffective(ctx, originPod) - replaceRevision := getReplaceRevision(originPod, resources) - if err != nil { - return err - } - // add instance id and replace pair label - var instanceId string - var newPodContext *appsv1alpha1.ContextDetail - if contextDetail, exist := mapNewToOriginPodContext[originPodId]; exist && contextDetail != nil { - newPodContext = contextDetail - // reuse podContext ID if pair-relation exists - instanceId = fmt.Sprintf("%d", newPodContext.ID) - logger.Info("replaceOriginPods", "try to reuse new pod resourceContext id", instanceId) - } else { - if availableContexts[i] == nil { - r.recorder.Eventf(originPod, corev1.EventTypeWarning, "AvailableContext", "cannot found available context for replace new pod when replacing origin pod %s/%s", originPod.Namespace, originPod.Name) - return fmt.Errorf("cannot find available context for replace new pod when replacing origin pod %s/%s", originPod.Namespace, originPod.Name) - } - newPodContext = availableContexts[i] - // add replace pair-relation to podContexts for originPod and newPod - instanceId = fmt.Sprintf("%d", newPodContext.ID) - ownedIDs[originPodId].Put(ReplaceNewPodIDContextDataKey, instanceId) - ownedIDs[newPodContext.ID].Put(ReplaceOriginPodIDContextDataKey, strconv.Itoa(originPodId)) - ownedIDs[newPodContext.ID].Remove(podcontext.JustCreateContextDataKey) - } - // create pod using update revision if replaced by update, otherwise using current revision - newPod, err := collasetutils.NewPodFrom(instance, ownerRef, replaceRevision, instanceId, func(in *corev1.Pod) error { - return utilspoddecoration.PatchListOfDecorations(in, updatedPDs) - }) - if err != nil { - return err - } - newPod.Labels[appsv1alpha1.PodReplacePairOriginName] = originPod.GetName() - newPod.Labels[appsv1alpha1.PodCreatingLabel] = strconv.FormatInt(time.Now().UnixNano(), 10) - newPodContext.Put(podcontext.RevisionContextDataKey, replaceRevision.Name) - // create pvcs for new pod - err = r.pvcControl.CreatePodPvcs(ctx, instance, newPod, resources.ExistingPvcs) - if err != nil { - return fmt.Errorf("fail to migrate PVCs from origin pod %s to replace pod %s: %w", originPod.Name, newPod.Name, err) - } - if newCreatedPod, err := r.podControl.CreatePod(newPod); err == nil { - r.recorder.Eventf(originPod, - corev1.EventTypeNormal, - "CreatePairPod", - "succeed to create replace pair Pod %s/%s with revision %s by replace", - originPod.Namespace, - originPod.Name, - replaceRevision.Name) - if err := r.cacheExpectations.ExpectCreation(kubeutilsclient.ObjectKeyString(instance), collasetutils.PodGVK, newCreatedPod.Namespace, newCreatedPod.Name); err != nil { - return err - } - - patch := client.RawPatch(types.StrategicMergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{"%s":"%s"}}}`, appsv1alpha1.PodReplacePairNewId, instanceId))) - if err = r.podControl.PatchPod(originPod, patch); err != nil { - return fmt.Errorf("fail to update origin pod %s/%s pair label %s when updating by replaceUpdate: %w", originPod.Namespace, originPod.Name, newCreatedPod.Name, err) - } - logger.Info("replaceOriginPods", "replacing originPod", originPod.Name, "originPodId", originPodId, "newPodContextID", instanceId) - } else { - r.recorder.Eventf(originPod, - corev1.EventTypeNormal, - "ReplacePod", - "failed to create replace pair Pod %s/%s from revision %s by replace update: %s", - originPod.Namespace, - originPod.Name, - replaceRevision.Name, - err.Error()) - return err - } - return nil - }) - - return successCount, err -} - -func dealReplacePods(pods []*corev1.Pod, logger logr.Logger) ( - needReplacePods, needCleanLabelPods []*corev1.Pod, podNeedCleanLabels [][]string, needDeletePods []*corev1.Pod, -) { - podInstanceIdMap := make(map[string]*corev1.Pod) - podNameMap := make(map[string]*corev1.Pod) - for _, pod := range pods { - if instanceId, exist := pod.Labels[appsv1alpha1.PodInstanceIDLabelKey]; exist { - podInstanceIdMap[instanceId] = pod - } - podNameMap[pod.Name] = pod - } - - // deal need replace pods - for _, pod := range pods { - // no replace indication label - if _, exist := pod.Labels[appsv1alpha1.PodReplaceIndicationLabelKey]; !exist { - continue - } - - // origin pod is about to scaleIn, skip replace - if podopslifecycle.IsDuringOps(collasetutils.ScaleInOpsLifecycleAdapter, pod) { - logger.Info("dealReplacePods", "pod is during scaleInOps, skip replacing", pod.Name) - continue - } - - // pod is replace new created pod, skip replace - if originPodName, exist := pod.Labels[appsv1alpha1.PodReplacePairOriginName]; exist { - if _, exist := podNameMap[originPodName]; exist { - continue - } - } - - // pod already has a new created pod for replacement - if newPairPodId, exist := pod.Labels[appsv1alpha1.PodReplacePairNewId]; exist { - if _, exist := podInstanceIdMap[newPairPodId]; exist { - continue - } - } - - needReplacePods = append(needReplacePods, pod) - } - - for _, pod := range pods { - _, replaceByUpdate := pod.Labels[appsv1alpha1.PodReplaceByReplaceUpdateLabelKey] - var needCleanLabels []string - - // pod is replace new created pod, skip replace - if originPodName, exist := pod.Labels[appsv1alpha1.PodReplacePairOriginName]; exist { - // replace pair origin pod is not exist, clean label. - if originPod, exist := podNameMap[originPodName]; !exist { - needCleanLabels = append(needCleanLabels, appsv1alpha1.PodReplacePairOriginName) - } else if originPod.Labels[appsv1alpha1.PodReplaceIndicationLabelKey] == "" { - // replace canceled, delete replace new pod if new pod is not service available - if _, exist := pod.Labels[appsv1alpha1.PodServiceAvailableLabel]; !exist { - needDeletePods = append(needDeletePods, pod) - } - } else if !replaceByUpdate { - // not replace update, delete origin pod when new created pod is service available - if _, serviceAvailable := pod.Labels[appsv1alpha1.PodServiceAvailableLabel]; serviceAvailable { - needDeletePods = append(needDeletePods, originPod) - } - } - } - - if newPairPodId, exist := pod.Labels[appsv1alpha1.PodReplacePairNewId]; exist { - if _, exist := podInstanceIdMap[newPairPodId]; !exist { - needCleanLabels = append(needCleanLabels, appsv1alpha1.PodReplacePairNewId) - } - } - - if len(needCleanLabels) > 0 { - needCleanLabelPods = append(needCleanLabelPods, pod) - podNeedCleanLabels = append(podNeedCleanLabels, needCleanLabels) - } - } - - return -} - -func updateReplaceOriginPod( - ctx context.Context, - c client.Client, - recorder record.EventRecorder, - originPodUpdateInfo, newPodUpdateInfo *PodUpdateInfo, -) error { - originPod := originPodUpdateInfo.Pod - // 1. delete the new pod if not updated - if newPodUpdateInfo != nil { - newPod := newPodUpdateInfo.Pod - _, deletionIndicate := newPod.Labels[appsv1alpha1.PodDeletionIndicationLabelKey] - currentRevision, exist := newPod.Labels[appsv1.ControllerRevisionHashLabelKey] - if exist && currentRevision != originPodUpdateInfo.UpdateRevision.Name && !deletionIndicate { - patch := client.RawPatch(types.StrategicMergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{"%s":"%d"}}}`, appsv1alpha1.PodDeletionIndicationLabelKey, time.Now().UnixNano()))) - if patchErr := c.Patch(ctx, newPod, patch); patchErr != nil { - err := fmt.Errorf("failed to delete replace pair new pod %s/%s %w", - newPod.Namespace, newPod.Name, patchErr) - return err - } - recorder.Eventf(originPod, - corev1.EventTypeNormal, - "DeleteOldNewPod", - "succeed to delete replace new Pod %s/%s by label to-replace", - originPod.Namespace, - originPod.Name, - ) - } - } - - // 2. replace the origin pod with updated pod - _, replaceIndicate := originPod.Labels[appsv1alpha1.PodReplaceIndicationLabelKey] - replaceRevision, replaceByUpdate := originPod.Labels[appsv1alpha1.PodReplaceByReplaceUpdateLabelKey] - if !replaceIndicate || !replaceByUpdate || replaceRevision != originPodUpdateInfo.UpdateRevision.Name { - now := time.Now().UnixNano() - patch := client.RawPatch(types.StrategicMergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{"%s":"%v", "%s": "%v"}}}`, appsv1alpha1.PodReplaceIndicationLabelKey, now, appsv1alpha1.PodReplaceByReplaceUpdateLabelKey, originPodUpdateInfo.UpdateRevision.Name))) - if err := c.Patch(ctx, originPod, patch); err != nil { - return fmt.Errorf("fail to label origin pod %s/%s with replace indicate label by replaceUpdate: %w", originPod.Namespace, originPod.Name, err) - } - recorder.Eventf(originPod, - corev1.EventTypeNormal, - "UpdateOriginPod", - "succeed to update Pod %s/%s by label to-replace", - originPod.Namespace, - originPod.Name, - ) - } - - return nil -} - -// getReplaceRevision finds replaceNewPod's revision from originPod -func getReplaceRevision(originPod *corev1.Pod, resources *collasetutils.RelatedResources) *appsv1.ControllerRevision { - // replace update, first find revision from label, if revision not found, just replace with updated revision - if updateRevisionName, exist := originPod.Labels[appsv1alpha1.PodReplaceByReplaceUpdateLabelKey]; exist { - for _, rv := range resources.Revisions { - if updateRevisionName == rv.Name { - return rv - } - } - return resources.UpdatedRevision - } - - // replace by to-replace label, just replace with current revision - podCurrentRevisionName, exist := originPod.Labels[appsv1.ControllerRevisionHashLabelKey] - if !exist { - return resources.CurrentRevision - } - - for _, revision := range resources.Revisions { - if revision.Name == podCurrentRevisionName { - return revision - } - } - - return resources.CurrentRevision -} - -// classify the pair relationship for Pod replacement. -func classifyPodReplacingMapping(podWrappers []*collasetutils.PodWrapper) map[string]*collasetutils.PodWrapper { - podNameMap := make(map[string]*collasetutils.PodWrapper) - podIdMap := make(map[string]*collasetutils.PodWrapper) - for _, podWrapper := range podWrappers { - podNameMap[podWrapper.Name] = podWrapper - instanceId := podWrapper.Labels[appsv1alpha1.PodInstanceIDLabelKey] - podIdMap[instanceId] = podWrapper - } - - replacePodMapping := make(map[string]*collasetutils.PodWrapper) - for _, podWrapper := range podWrappers { - name := podWrapper.Name - if replacePairNewIdStr, exist := podWrapper.Labels[appsv1alpha1.PodReplacePairNewId]; exist { - if pairNewPod, exist := podIdMap[replacePairNewIdStr]; exist { - replacePodMapping[name] = pairNewPod - // if one of pair pods is to Exclude, both pods should not scaleIn - podWrapper.ToExclude = podWrapper.ToExclude || pairNewPod.ToExclude - continue - } - } else if replaceOriginStr, exist := podWrapper.Labels[appsv1alpha1.PodReplacePairOriginName]; exist { - if originPod, exist := podNameMap[replaceOriginStr]; exist { - if originPod.Labels[appsv1alpha1.PodReplacePairNewId] == podWrapper.Labels[appsv1alpha1.PodInstanceIDLabelKey] { - continue - } - } - } - - replacePodMapping[name] = nil - } - return replacePodMapping -} - -func mapReplaceNewToOriginPodContext(ownedIDs map[int]*appsv1alpha1.ContextDetail) map[int]*appsv1alpha1.ContextDetail { - mapNewToOriginPodContext := make(map[int]*appsv1alpha1.ContextDetail) - for id, contextDetail := range ownedIDs { - if val, exist := contextDetail.Data[ReplaceNewPodIDContextDataKey]; exist { - newPodId, _ := strconv.ParseInt(val, 10, 32) - newPodContextDetail, exist := ownedIDs[int(newPodId)] - if exist && newPodContextDetail.Data[ReplaceOriginPodIDContextDataKey] == strconv.Itoa(id) { - mapNewToOriginPodContext[id] = newPodContextDetail - } else { - mapNewToOriginPodContext[id] = nil - } - } - } - return mapNewToOriginPodContext -} - -func mapReplaceOriginToNewPodContext(ownedIDs map[int]*appsv1alpha1.ContextDetail) map[int]*appsv1alpha1.ContextDetail { - mapOriginToNewPodContext := make(map[int]*appsv1alpha1.ContextDetail) - for id, contextDetail := range ownedIDs { - if val, exist := contextDetail.Data[ReplaceOriginPodIDContextDataKey]; exist { - originPodId, _ := strconv.ParseInt(val, 10, 32) - originPodContextDetail, exist := ownedIDs[int(originPodId)] - if exist && originPodContextDetail.Data[ReplaceNewPodIDContextDataKey] == strconv.Itoa(id) { - mapOriginToNewPodContext[id] = originPodContextDetail - } else { - mapOriginToNewPodContext[id] = nil - } - } - } - return mapOriginToNewPodContext -} - -func podDuringReplace(pod *corev1.Pod) bool { - if pod.Labels == nil { - return false - } - _, replaceIndicate := pod.Labels[appsv1alpha1.PodReplaceIndicationLabelKey] - _, replaceOriginPod := pod.Labels[appsv1alpha1.PodReplacePairNewId] - _, replaceNewPod := pod.Labels[appsv1alpha1.PodReplacePairOriginName] - return replaceIndicate || replaceOriginPod || replaceNewPod -} diff --git a/pkg/controllers/collaset/synccontrol/scale.go b/pkg/controllers/collaset/synccontrol/scale.go deleted file mode 100644 index 9ced7eef..00000000 --- a/pkg/controllers/collaset/synccontrol/scale.go +++ /dev/null @@ -1,397 +0,0 @@ -/* -Copyright 2023 The KusionStack Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package synccontrol - -import ( - "context" - "fmt" - "sort" - "strconv" - "strings" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/client-go/util/retry" - "k8s.io/utils/ptr" - appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" - kubeutilsclient "kusionstack.io/kube-utils/client" - "sigs.k8s.io/controller-runtime/pkg/client" - - collasetutils "kusionstack.io/kuperator/pkg/controllers/collaset/utils" - controllerutils "kusionstack.io/kuperator/pkg/controllers/utils" - "kusionstack.io/kuperator/pkg/controllers/utils/podopslifecycle" - "kusionstack.io/kuperator/pkg/features" - "kusionstack.io/kuperator/pkg/utils/feature" -) - -// getPodsToDelete -// 1. finds number of diff pods from filteredPods to do scaleIn -// 2. finds pods allowed to scale in out of diff -func getPodsToDelete(cls *appsv1alpha1.CollaSet, filteredPods []*collasetutils.PodWrapper, replaceMapping map[string]*collasetutils.PodWrapper, diff int) []*collasetutils.PodWrapper { - targetsPods := getTargetsDeletePods(filteredPods, replaceMapping) - // select pods to delete in first round according to diff - sort.Sort(ActivePodsForDeletion(targetsPods)) - if diff > len(targetsPods) { - diff = len(targetsPods) - } - - var needDeletePods []*collasetutils.PodWrapper - // select pods to delete in second round according to replace, delete, exclude - for i, pod := range targetsPods { - // find pods to be scaled in out of diff, if allowed to ops - _, allowed := podopslifecycle.AllowOps(collasetutils.ScaleInOpsLifecycleAdapter, ptr.Deref(cls.Spec.ScaleStrategy.OperationDelaySeconds, 0), pod) - if i >= diff && !allowed { - continue - } - - // don't scaleIn exclude pod and its newPod (if exist) - if pod.ToExclude { - continue - } - - if replacePairPod, exist := replaceMapping[pod.Name]; exist && replacePairPod != nil { - // don't selective scaleIn newPod (and its originPod) until replace finished - if replacePairPod.ToDelete && !pod.ToDelete { - continue - } - // when scaleIn origin Pod, newPod should be deleted if not service available - if _, serviceAvailable := replacePairPod.Labels[appsv1alpha1.PodServiceAvailableLabel]; !serviceAvailable { - needDeletePods = append(needDeletePods, replacePairPod) - } - } - needDeletePods = append(needDeletePods, pod) - } - - return needDeletePods -} - -// getTargetsDeletePods finds pods to choose delete, only finds those (1) replace origin pods, (2) non-exclude pods -func getTargetsDeletePods(filteredPods []*collasetutils.PodWrapper, replaceMapping map[string]*collasetutils.PodWrapper) []*collasetutils.PodWrapper { - targetPods := make([]*collasetutils.PodWrapper, len(replaceMapping)) - index := 0 - for _, pod := range filteredPods { - if _, exist := replaceMapping[pod.Name]; exist { - targetPods[index] = pod - index++ - } - } - - return targetPods -} - -type ActivePodsForDeletion []*collasetutils.PodWrapper - -func (s ActivePodsForDeletion) Len() int { return len(s) } -func (s ActivePodsForDeletion) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -// Less sort deletion order by: podToDelete > podToExclude > duringScaleIn > others -func (s ActivePodsForDeletion) Less(i, j int) bool { - l, r := s[i], s[j] - - if l.ToDelete != r.ToDelete { - return l.ToDelete - } - - if l.ToExclude != r.ToExclude { - return l.ToExclude - } - - // pods which are during scaleInOps should be deleted before those not during - lDuringScaleIn := podopslifecycle.IsDuringOps(collasetutils.ScaleInOpsLifecycleAdapter, l) - rDuringScaleIn := podopslifecycle.IsDuringOps(collasetutils.ScaleInOpsLifecycleAdapter, r) - if lDuringScaleIn != rDuringScaleIn { - return lDuringScaleIn - } - - // TODO consider service available timestamps - return collasetutils.ComparePod(l.Pod, r.Pod) -} - -// dealIncludeExcludePods returns pods which are allowed to exclude and include -func (r *RealSyncControl) dealIncludeExcludePods(ctx context.Context, cls *appsv1alpha1.CollaSet, pods []*corev1.Pod) (sets.String, sets.String, error) { - ownedPods := sets.String{} - excludePodNames := sets.String{} - includePodNames := sets.String{} - for _, pod := range pods { - ownedPods.Insert(pod.Name) - if _, exist := pod.Labels[appsv1alpha1.PodExcludeIndicationLabelKey]; exist { - excludePodNames.Insert(pod.Name) - } - } - - for _, podName := range cls.Spec.ScaleStrategy.PodToExclude { - if ownedPods.Has(podName) { - excludePodNames.Insert(podName) - } - } - - for _, podName := range cls.Spec.ScaleStrategy.PodToInclude { - includePodNames.Insert(podName) - } - - intersection := excludePodNames.Intersection(includePodNames) - if len(intersection) > 0 { - r.recorder.Eventf(cls, corev1.EventTypeWarning, "DupExIncludedPod", "duplicated pods %s in both excluding and including sets", strings.Join(intersection.List(), ", ")) - includePodNames.Delete(intersection.List()...) - excludePodNames.Delete(intersection.List()...) - } - - if includePodNames.Len() > 0 { - for _, pod := range pods { - if includePodNames.Has(pod.Name) { - includePodNames.Delete(pod.Name) - } - } - } - - toExcludePods, notAllowedExcludePods, exErr := r.allowIncludeExcludePods(ctx, cls, excludePodNames.List(), collasetutils.AllowResourceExclude) - toIncludePods, notAllowedIncludePods, inErr := r.allowIncludeExcludePods(ctx, cls, includePodNames.List(), collasetutils.AllowResourceInclude) - if notAllowedExcludePods.Len() > 0 { - r.recorder.Eventf(cls, corev1.EventTypeWarning, "ExcludeNotAllowed", fmt.Sprintf("pods [%v] are not allowed to exclude, please find out the reason from pod's event", notAllowedExcludePods.List())) - } - if notAllowedIncludePods.Len() > 0 { - r.recorder.Eventf(cls, corev1.EventTypeWarning, "IncludeNotAllowed", fmt.Sprintf("pods [%v] are not allowed to include, please find out the reason from pod's event", notAllowedIncludePods.List())) - } - return toExcludePods, toIncludePods, controllerutils.AggregateErrors([]error{exErr, inErr}) -} - -// checkAllowFunc refers to AllowResourceExclude and AllowResourceInclude -type checkAllowFunc func(obj metav1.Object, ownerName, ownerKind string) (bool, string) - -// allowIncludeExcludePods try to classify podNames to allowedPods and notAllowedPods, using checkAllowFunc func -func (r *RealSyncControl) allowIncludeExcludePods(ctx context.Context, cls *appsv1alpha1.CollaSet, podNames []string, fn checkAllowFunc) (allowPods, notAllowPods sets.String, err error) { - allowPods = sets.String{} - notAllowPods = sets.String{} - for i := range podNames { - pod := &corev1.Pod{} - err = r.client.Get(ctx, types.NamespacedName{Namespace: cls.Namespace, Name: podNames[i]}, pod) - if errors.IsNotFound(err) { - notAllowPods.Insert(podNames[i]) - continue - } else if err != nil { - r.recorder.Eventf(cls, corev1.EventTypeWarning, "ExcludeIncludeFailed", fmt.Sprintf("failed to find pod %s: %s", podNames[i], err.Error())) - return - } - - // check allowance for pod - if allowed, reason := fn(pod, cls.Name, cls.Kind); !allowed { - r.recorder.Eventf(pod, corev1.EventTypeWarning, "ExcludeIncludeNotAllowed", fmt.Sprintf("pod is not allowed to exclude/include from/to collaset %s/%s: %s", cls.Namespace, cls.Name, reason)) - notAllowPods.Insert(pod.Name) - continue - } - - pvcsAllowed := true - // check allowance for pvcs - for _, volume := range pod.Spec.Volumes { - if volume.PersistentVolumeClaim == nil { - continue - } - pvc := &corev1.PersistentVolumeClaim{} - err = r.client.Get(ctx, types.NamespacedName{Namespace: pod.Namespace, Name: volume.PersistentVolumeClaim.ClaimName}, pvc) - // If pvc not found, ignore it. In case of pvc is filtered out by controller-mesh - if errors.IsNotFound(err) { - continue - } else if err != nil { - r.recorder.Eventf(pod, corev1.EventTypeWarning, "ExcludeIncludeNotAllowed", fmt.Sprintf("failed to check allowed to exclude/include from/to collaset %s/%s: %s", cls.Namespace, cls.Name, err.Error())) - pvcsAllowed = false - break - } - if allowed, reason := fn(pvc, cls.Name, cls.Kind); !allowed { - r.recorder.Eventf(pod, corev1.EventTypeWarning, "ExcludeIncludeNotAllowed", fmt.Sprintf("pvc is not allowed to exclude/include from/to collaset %s/%s: %s", cls.Namespace, cls.Name, reason)) - notAllowPods.Insert(pod.Name) - pvcsAllowed = false - break - } - } - if pvcsAllowed { - allowPods.Insert(pod.Name) - } - } - return allowPods, notAllowPods, nil -} - -// doIncludeExcludePods do real include and exclude for pods which are allowed to in/exclude -func (r *RealSyncControl) doIncludeExcludePods(ctx context.Context, cls *appsv1alpha1.CollaSet, excludePods, includePods []string, availableContexts []*appsv1alpha1.ContextDetail) error { - var excludeErrs, includeErrs []error - _, _ = controllerutils.SlowStartBatch(len(excludePods), controllerutils.SlowStartInitialBatchSize, false, func(idx int, _ error) (err error) { - defer func() { excludeErrs = append(excludeErrs, err) }() - return r.excludePod(ctx, cls, excludePods[idx]) - }) - _, _ = controllerutils.SlowStartBatch(len(includePods), controllerutils.SlowStartInitialBatchSize, false, func(idx int, _ error) (err error) { - defer func() { includeErrs = append(includeErrs, err) }() - return r.includePod(ctx, cls, includePods[idx], strconv.Itoa(availableContexts[idx].ID)) - }) - return controllerutils.AggregateErrors(append(includeErrs, excludeErrs...)) -} - -// excludePod try to exclude a pod from collaset -func (r *RealSyncControl) excludePod(ctx context.Context, cls *appsv1alpha1.CollaSet, podName string) error { - pod := &corev1.Pod{} - if err := r.client.Get(ctx, types.NamespacedName{Namespace: cls.Namespace, Name: podName}, pod); err != nil { - return err - } - var pvcs []*corev1.PersistentVolumeClaim - for _, volume := range pod.Spec.Volumes { - if volume.PersistentVolumeClaim == nil { - continue - } - pvc := &corev1.PersistentVolumeClaim{} - err := r.client.Get(ctx, types.NamespacedName{Namespace: pod.Namespace, Name: volume.PersistentVolumeClaim.ClaimName}, pvc) - // If pvc not found, ignore it. In case of pvc is filtered out by controller-mesh - if errors.IsNotFound(err) { - continue - } else if err != nil { - return err - } - pvcs = append(pvcs, pvc) - } - - pod.Labels[appsv1alpha1.PodOrphanedIndicateLabelKey] = "true" - if err := r.podControl.OrphanPod(cls, pod); err != nil { - return err - } - for i := range pvcs { - pvcs[i].Labels[appsv1alpha1.PodOrphanedIndicateLabelKey] = "true" - if err := r.pvcControl.OrphanPvc(cls, pvcs[i]); err != nil { - return err - } - } - return r.cacheExpectations.ExpectUpdation(kubeutilsclient.ObjectKeyString(cls), collasetutils.PodGVK, pod.Namespace, pod.Name, pod.ResourceVersion) -} - -// includePod try to include a pod into collaset -func (r *RealSyncControl) includePod(ctx context.Context, cls *appsv1alpha1.CollaSet, podName, instanceId string) error { - pod := &corev1.Pod{} - if err := r.client.Get(ctx, types.NamespacedName{Namespace: cls.Namespace, Name: podName}, pod); err != nil { - return err - } - - var pvcs []*corev1.PersistentVolumeClaim - for _, volume := range pod.Spec.Volumes { - if volume.PersistentVolumeClaim == nil { - continue - } - pvc := &corev1.PersistentVolumeClaim{} - err := r.client.Get(ctx, types.NamespacedName{Namespace: pod.Namespace, Name: volume.PersistentVolumeClaim.ClaimName}, pvc) - // If pvc not found, ignore it. In case of pvc is filtered out by controller-mesh - if errors.IsNotFound(err) { - continue - } else if err != nil { - return err - } - pvcs = append(pvcs, pvc) - } - - pod.Labels[appsv1alpha1.PodInstanceIDLabelKey] = instanceId - delete(pod.Labels, appsv1alpha1.PodOrphanedIndicateLabelKey) - if err := r.podControl.AdoptPod(cls, pod); err != nil { - return err - } - for i := range pvcs { - pvcs[i].Labels[appsv1alpha1.PodInstanceIDLabelKey] = instanceId - delete(pvcs[i].Labels, appsv1alpha1.PodOrphanedIndicateLabelKey) - if err := r.pvcControl.AdoptPvc(cls, pvcs[i]); err != nil { - return err - } - } - return r.cacheExpectations.ExpectUpdation(kubeutilsclient.ObjectKeyString(cls), collasetutils.PodGVK, pod.Namespace, pod.Name, pod.ResourceVersion) -} - -// adoptPvcsLeftByRetainPolicy adopts pvcs with respect of "whenDelete=true" pvc retention policy -func (r *RealSyncControl) adoptPvcsLeftByRetainPolicy(ctx context.Context, cls *appsv1alpha1.CollaSet) ([]*corev1.PersistentVolumeClaim, error) { - ownerSelector := cls.Spec.Selector.DeepCopy() - if ownerSelector.MatchLabels == nil { - ownerSelector.MatchLabels = map[string]string{} - } - ownerSelector.MatchLabels[appsv1alpha1.ControlledByKusionStackLabelKey] = "true" - ownerSelector.MatchExpressions = append(ownerSelector.MatchExpressions, metav1.LabelSelectorRequirement{ - Key: appsv1alpha1.PodOrphanedIndicateLabelKey, // should not be excluded pvcs - Operator: metav1.LabelSelectorOpDoesNotExist, - }) - ownerSelector.MatchExpressions = append(ownerSelector.MatchExpressions, metav1.LabelSelectorRequirement{ - Key: appsv1alpha1.PodInstanceIDLabelKey, // instance-id label should exist - Operator: metav1.LabelSelectorOpExists, - }) - ownerSelector.MatchExpressions = append(ownerSelector.MatchExpressions, metav1.LabelSelectorRequirement{ - Key: appsv1alpha1.PvcTemplateHashLabelKey, // pvc-hash label should exist - Operator: metav1.LabelSelectorOpExists, - }) - - selector, err := metav1.LabelSelectorAsSelector(ownerSelector) - if err != nil { - return nil, err - } - - orphanedPvcList := &corev1.PersistentVolumeClaimList{} - if err = r.client.List(ctx, orphanedPvcList, &client.ListOptions{Namespace: cls.Namespace, LabelSelector: selector}); err != nil { - return nil, err - } - - // adopt orphaned pvcs - var claims []*corev1.PersistentVolumeClaim - for i := range orphanedPvcList.Items { - pvc := orphanedPvcList.Items[i] - if pvc.OwnerReferences != nil && len(pvc.OwnerReferences) > 0 { - continue - } - if pvc.Labels == nil { - pvc.Labels = make(map[string]string) - } - if pvc.Annotations == nil { - pvc.Annotations = make(map[string]string) - } - - claims = append(claims, &pvc) - } - for i := range claims { - if err := r.pvcControl.AdoptPvc(cls, claims[i]); err != nil { - return nil, err - } - } - return claims, nil -} - -// reclaimScaleStrategy updates podToDelete, podToExclude, podToInclude in scaleStrategy -func (r *RealSyncControl) reclaimScaleStrategy(ctx context.Context, deletedPods, excludedPods, includedPods sets.String, cls *appsv1alpha1.CollaSet) error { - // ReclaimPodScaleStrategy FeatureGate defaults to true - // Add '--feature-gates=ReclaimPodScaleStrategy=false' to container args, to disable reclaim of podToDelete, podToExclude, podToInclude - if feature.DefaultFeatureGate.Enabled(features.ReclaimPodScaleStrategy) { - // reclaim PodToDelete - toDeletePods := sets.NewString(cls.Spec.ScaleStrategy.PodToDelete...) - notDeletedPods := toDeletePods.Delete(deletedPods.List()...) - cls.Spec.ScaleStrategy.PodToDelete = notDeletedPods.List() - // reclaim PodToExclude - toExcludePods := sets.NewString(cls.Spec.ScaleStrategy.PodToExclude...) - notExcludePods := toExcludePods.Delete(excludedPods.List()...) - cls.Spec.ScaleStrategy.PodToExclude = notExcludePods.List() - // reclaim PodToInclude - toIncludePodNames := sets.NewString(cls.Spec.ScaleStrategy.PodToInclude...) - notIncludePods := toIncludePodNames.Delete(includedPods.List()...) - cls.Spec.ScaleStrategy.PodToInclude = notIncludePods.List() - // update cls.spec.scaleStrategy - if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - return r.client.Update(ctx, cls) - }); err != nil { - return err - } - return r.cacheExpectations.ExpectUpdation(kubeutilsclient.ObjectKeyString(cls), collasetutils.CollaSetGVK, cls.Namespace, cls.Name, cls.ResourceVersion) - } - return nil -} diff --git a/pkg/controllers/collaset/synccontrol/sync_control.go b/pkg/controllers/collaset/synccontrol/sync_control.go deleted file mode 100644 index 0580fd18..00000000 --- a/pkg/controllers/collaset/synccontrol/sync_control.go +++ /dev/null @@ -1,888 +0,0 @@ -/* -Copyright 2023 The KusionStack Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package synccontrol - -import ( - "context" - "fmt" - "strconv" - "sync/atomic" - "time" - - "github.com/go-logr/logr" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/client-go/tools/record" - "k8s.io/client-go/util/retry" - appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" - kubeutilsclient "kusionstack.io/kube-utils/client" - kubeutilsexpectations "kusionstack.io/kube-utils/controller/expectations" - "sigs.k8s.io/controller-runtime/pkg/client" - - "kusionstack.io/kuperator/pkg/controllers/collaset/podcontext" - "kusionstack.io/kuperator/pkg/controllers/collaset/podcontrol" - "kusionstack.io/kuperator/pkg/controllers/collaset/pvccontrol" - collasetutils "kusionstack.io/kuperator/pkg/controllers/collaset/utils" - controllerutils "kusionstack.io/kuperator/pkg/controllers/utils" - utilspoddecoration "kusionstack.io/kuperator/pkg/controllers/utils/poddecoration" - "kusionstack.io/kuperator/pkg/controllers/utils/poddecoration/anno" - "kusionstack.io/kuperator/pkg/controllers/utils/podopslifecycle" - commonutils "kusionstack.io/kuperator/pkg/utils" -) - -const ( - ScaleInContextDataKey = "ScaleIn" -) - -type Interface interface { - SyncPods( - ctx context.Context, - instance *appsv1alpha1.CollaSet, - resources *collasetutils.RelatedResources, - ) (bool, []*collasetutils.PodWrapper, map[int]*appsv1alpha1.ContextDetail, error) - - Replace( - ctx context.Context, - instance *appsv1alpha1.CollaSet, - podWrappers []*collasetutils.PodWrapper, - ownedIDs map[int]*appsv1alpha1.ContextDetail, - resources *collasetutils.RelatedResources, - ) ([]*collasetutils.PodWrapper, map[int]*appsv1alpha1.ContextDetail, error) - - Scale( - ctx context.Context, - instance *appsv1alpha1.CollaSet, - resources *collasetutils.RelatedResources, - podWrappers []*collasetutils.PodWrapper, - ownedIDs map[int]*appsv1alpha1.ContextDetail, - ) (bool, *time.Duration, error) - - Update( - ctx context.Context, - instance *appsv1alpha1.CollaSet, - resources *collasetutils.RelatedResources, - filteredPods []*collasetutils.PodWrapper, - ownedIDs map[int]*appsv1alpha1.ContextDetail, - ) (bool, *time.Duration, error) -} - -func NewRealSyncControl(client client.Client, - logger logr.Logger, - podControl podcontrol.Interface, - pvcControl pvccontrol.Interface, - podContextControl podcontext.Interface, - recorder record.EventRecorder, - cacheExpectations kubeutilsexpectations.CacheExpectationsInterface, -) Interface { - return &RealSyncControl{ - client: client, - logger: logger, - podControl: podControl, - pvcControl: pvcControl, - podContextControl: podContextControl, - recorder: recorder, - cacheExpectations: cacheExpectations, - } -} - -var _ Interface = &RealSyncControl{} - -type RealSyncControl struct { - client client.Client - logger logr.Logger - podControl podcontrol.Interface - pvcControl pvccontrol.Interface - podContextControl podcontext.Interface - recorder record.EventRecorder - cacheExpectations kubeutilsexpectations.CacheExpectationsInterface -} - -// SyncPods is used to parse podWrappers and reclaim Pod instance ID -func (r *RealSyncControl) SyncPods( - ctx context.Context, - instance *appsv1alpha1.CollaSet, - resources *collasetutils.RelatedResources, -) (bool, []*collasetutils.PodWrapper, map[int]*appsv1alpha1.ContextDetail, error) { - filteredPods, allPods, err := r.podControl.GetFilteredPods(instance.Spec.Selector, instance) - if err != nil { - return false, nil, nil, fmt.Errorf("fail to get filtered Pods: %w", err) - } - - if collasetutils.IsPodNamingSuffixPolicyPersistentSequence(instance) { - // pods with same number should not exist at same time - resources.FilteredPods = allPods - } else { - resources.FilteredPods = filteredPods - } - - // list pvcs using ownerReference - if resources.ExistingPvcs, err = r.pvcControl.GetFilteredPvcs(ctx, instance); err != nil { - return false, nil, nil, fmt.Errorf("fail to get filtered PVCs: %w", err) - } - // adopt and retain orphaned pvcs according to PVC retention policy - if adoptedPvcs, err := r.adoptPvcsLeftByRetainPolicy(ctx, instance); err != nil { - return false, nil, nil, fmt.Errorf("fail to adopt orphaned left by whenDelete retention policy PVCs: %w", err) - } else { - resources.ExistingPvcs = append(resources.ExistingPvcs, adoptedPvcs...) - } - - toExcludePodNames, toIncludePodNames, err := r.dealIncludeExcludePods(ctx, instance, resources.FilteredPods) - if err != nil { - return false, nil, nil, fmt.Errorf("fail to deal with include exclude pods: %w", err) - } - - // get owned IDs - var ownedIDs map[int]*appsv1alpha1.ContextDetail - if err = retry.RetryOnConflict(retry.DefaultRetry, func() error { - ownedIDs, err = r.podContextControl.AllocateID(ctx, instance, resources.UpdatedRevision.Name, int(realValue(instance.Spec.Replicas))) - return err - }); err != nil { - return false, nil, ownedIDs, fmt.Errorf("fail to allocate %d IDs using context when sync Pods: %w", instance.Spec.Replicas, err) - } - - // stateless case - var podWrappers []*collasetutils.PodWrapper - resources.CurrentIDs = make(map[int]struct{}) - idToReclaim := sets.Int{} - toDeletePodNames := sets.NewString(instance.Spec.ScaleStrategy.PodToDelete...) - for i := range resources.FilteredPods { - pod := resources.FilteredPods[i] - id, _ := collasetutils.GetPodInstanceID(pod) - toDelete := toDeletePodNames.Has(pod.Name) - toExclude := toExcludePodNames.Has(pod.Name) - - // priority: toDelete > toReplace > toExclude - if toDelete { - toDeletePodNames.Delete(pod.Name) - } - if toExclude { - if podDuringReplace(pod) || toDelete { - // skip exclude until replace and toDelete done - toExcludePodNames.Delete(pod.Name) - } else { - // exclude pod and delete its podContext - idToReclaim.Insert(id) - } - } - - // pods with PersistentSequence naming policy should always count a replicas, to avoid scaleOut a pod with the same name - if pod.DeletionTimestamp != nil && !collasetutils.IsPodNamingSuffixPolicyPersistentSequence(instance) { - // 1. Reclaim ID from Pod which is scaling in and terminating. - if contextDetail, exist := ownedIDs[id]; exist && contextDetail.Contains(ScaleInContextDataKey, "true") { - idToReclaim.Insert(id) - } - - _, replaceIndicate := pod.Labels[appsv1alpha1.PodReplaceIndicationLabelKey] - // 2. filter out Pods which are terminating and not replace indicate - if !replaceIndicate { - continue - } - } - - // delete unused pvcs - if err := r.pvcControl.DeletePodUnusedPvcs(ctx, instance, pod, resources.ExistingPvcs); err != nil { - return false, nil, nil, fmt.Errorf("fail to delete unused pvcs %w", err) - } - - podWrappers = append(podWrappers, &collasetutils.PodWrapper{ - Pod: pod, - ID: id, - ContextDetail: ownedIDs[id], - ToDelete: toDelete, - ToExclude: toExclude, - PlaceHolder: false, - }) - - if id >= 0 { - resources.CurrentIDs[id] = struct{}{} - } - } - - // do include exclude pods, and skip doSync() if succeeded - var inExSucceed bool - if len(toExcludePodNames) > 0 || len(toIncludePodNames) > 0 { - var availableContexts []*appsv1alpha1.ContextDetail - var getErr error - availableContexts, ownedIDs, getErr = r.getAvailablePodIDs(ctx, len(toIncludePodNames), instance, resources, ownedIDs, resources.CurrentIDs) - if getErr != nil { - return false, nil, nil, getErr - } - if err = r.doIncludeExcludePods(ctx, instance, toExcludePodNames.List(), toIncludePodNames.List(), availableContexts); err != nil { - r.recorder.Eventf(instance, corev1.EventTypeWarning, "ExcludeIncludeFailed", "collaset syncPods include exclude with error: %s", err.Error()) - return false, nil, nil, err - } - inExSucceed = true - } - - // reclaim Pod ID which is (1) during ScalingIn, (2) ExcludePods - err = r.reclaimOwnedIDs(ctx, false, instance, idToReclaim, ownedIDs, resources.CurrentIDs) - if err != nil { - r.recorder.Eventf(instance, corev1.EventTypeWarning, "ReclaimOwnedIDs", "reclaim pod contexts with error: %s", err.Error()) - return false, nil, nil, err - } - - // reclaim scaleStrategy for delete, exclude, include - err = r.reclaimScaleStrategy(ctx, toDeletePodNames, toExcludePodNames, toIncludePodNames, instance) - if err != nil { - r.recorder.Eventf(instance, corev1.EventTypeWarning, "ReclaimScaleStrategy", "reclaim scaleStrategy with error: %s", err.Error()) - return false, nil, nil, err - } - - return inExSucceed, podWrappers, ownedIDs, nil -} - -// Replace is used to replace replace-indicate pods -func (r *RealSyncControl) Replace( - ctx context.Context, - instance *appsv1alpha1.CollaSet, - podWrappers []*collasetutils.PodWrapper, - ownedIDs map[int]*appsv1alpha1.ContextDetail, - resources *collasetutils.RelatedResources, -) ([]*collasetutils.PodWrapper, map[int]*appsv1alpha1.ContextDetail, error) { - var err error - var needUpdateContext bool - var idToReclaim sets.Int - logger := r.logger.WithValues("collaset", commonutils.ObjectKeyString(instance)) - - needReplaceOriginPods, needCleanLabelPods, podsNeedCleanLabels, needDeletePods := dealReplacePods(resources.FilteredPods, logger) - - // delete origin pods for replace - err = DeletePodsByLabel(r.podControl, needDeletePods) - if err != nil { - r.recorder.Eventf(instance, corev1.EventTypeWarning, "ReplacePod", "delete pods by label with error: %s", err.Error()) - return podWrappers, ownedIDs, err - } - - // clean labels for replace pods - needUpdateContext, idToReclaim, err = r.cleanReplacePodLabels(needCleanLabelPods, podsNeedCleanLabels, ownedIDs, resources.CurrentIDs, logger) - if err != nil { - r.recorder.Eventf(instance, corev1.EventTypeWarning, "ReplacePod", fmt.Sprintf("clean pods replace pair origin name label with error: %s", err.Error())) - return podWrappers, ownedIDs, err - } - - // create new pods for need replace pods - if len(needReplaceOriginPods) > 0 { - var availableContexts []*appsv1alpha1.ContextDetail - var getErr error - availableContexts, ownedIDs, getErr = r.getAvailablePodIDs(ctx, len(needReplaceOriginPods), instance, resources, ownedIDs, resources.CurrentIDs) - if getErr != nil { - return podWrappers, ownedIDs, getErr - } - successCount, err := r.replaceOriginPods(ctx, instance, resources, needReplaceOriginPods, ownedIDs, availableContexts, logger) - needUpdateContext = needUpdateContext || successCount > 0 - if err != nil { - r.recorder.Eventf(instance, corev1.EventTypeWarning, "ReplacePod", "deal replace pods with error: %s", err.Error()) - return podWrappers, ownedIDs, err - } - } - - // reclaim Pod ID which is ReplaceOriginPod - err = r.reclaimOwnedIDs(ctx, needUpdateContext, instance, idToReclaim, ownedIDs, resources.CurrentIDs) - if err != nil { - r.recorder.Eventf(instance, corev1.EventTypeWarning, "ReclaimOwnedIDs", "reclaim pod contexts with error: %s", err.Error()) - return podWrappers, ownedIDs, err - } - - // create podWrappers for non-exist pods - for id, contextDetail := range ownedIDs { - if _, inUsed := resources.CurrentIDs[id]; inUsed { - continue - } - podWrappers = append(podWrappers, &collasetutils.PodWrapper{ - ID: id, - Pod: nil, - ContextDetail: contextDetail, - PlaceHolder: true, - }) - } - - return podWrappers, ownedIDs, nil -} - -// Scale is used to reconcile replicas to spec.replicas -func (r *RealSyncControl) Scale( - ctx context.Context, - cls *appsv1alpha1.CollaSet, - resources *collasetutils.RelatedResources, - podWrappers []*collasetutils.PodWrapper, - ownedIDs map[int]*appsv1alpha1.ContextDetail, -) (bool, *time.Duration, error) { - logger := r.logger.WithValues("collaset", commonutils.ObjectKeyString(cls)) - var recordedRequeueAfter *time.Duration - activePods := FilterOutPlaceHolderPodWrappers(podWrappers) - replacePodMap := classifyPodReplacingMapping(activePods) - - diff := int(realValue(cls.Spec.Replicas)) - len(replacePodMap) - scaling := false - - if diff >= 0 { - // trigger delete pods indicated in ScaleStrategy.PodToDelete by label - for _, podWrapper := range activePods { - if podWrapper.ToDelete { - err := DeletePodsByLabel(r.podControl, []*corev1.Pod{podWrapper.Pod}) - if err != nil { - return false, recordedRequeueAfter, err - } - } - } - - // scale out pods and return if diff > 0 - if diff > 0 { - // collect instance ID in used from owned Pods - podInstanceIDSet := collasetutils.CollectPodInstanceID(activePods) - // find IDs and their contexts which have not been used by owned Pods - var availableContexts []*appsv1alpha1.ContextDetail - var getErr error - availableContexts, ownedIDs, getErr = r.getAvailablePodIDs(ctx, diff, cls, resources, ownedIDs, podInstanceIDSet) - if getErr != nil { - return false, recordedRequeueAfter, getErr - } - needUpdateContext := atomic.Bool{} - succCount, err := controllerutils.SlowStartBatch(diff, controllerutils.SlowStartInitialBatchSize, false, func(idx int, _ error) (err error) { - availableIDContext := availableContexts[idx] - defer func() { - if decideContextRevision(availableIDContext, resources.UpdatedRevision, err == nil) { - needUpdateContext.Store(true) - } - }() - // use revision recorded in Context - revision := resources.UpdatedRevision - if revisionName, exist := availableIDContext.Data[podcontext.RevisionContextDataKey]; exist && revisionName != "" { - for i := range resources.Revisions { - if resources.Revisions[i].Name == revisionName { - revision = resources.Revisions[i] - break - } - } - } - // scale out new Pods with updatedRevision - // TODO use cache - instanceId := fmt.Sprintf("%d", availableIDContext.ID) - pod, err := collasetutils.NewPodFrom( - cls, - metav1.NewControllerRef(cls, appsv1alpha1.SchemeGroupVersion.WithKind("CollaSet")), - revision, - instanceId, - func(in *corev1.Pod) (localErr error) { - if availableIDContext.Data[podcontext.JustCreateContextDataKey] == "true" { - in.Labels[appsv1alpha1.PodCreatingLabel] = strconv.FormatInt(time.Now().UnixNano(), 10) - } else { - in.Labels[appsv1alpha1.PodCompletingLabel] = strconv.FormatInt(time.Now().UnixNano(), 10) - } - revisionsInfo, ok := availableIDContext.Get(podcontext.PodDecorationRevisionKey) - var pds map[string]*appsv1alpha1.PodDecoration - if !ok { - // get default PodDecorations if no revision in context - pds, localErr = resources.PDGetter.GetEffective(ctx, in) - if localErr != nil { - return localErr - } - needUpdateContext.Store(true) - availableIDContext.Put(podcontext.PodDecorationRevisionKey, anno.GetDecorationInfoString(pds)) - } else { - // upgrade by recreate pod case - infos, marshallErr := anno.UnmarshallFromString(revisionsInfo) - if marshallErr != nil { - return marshallErr - } - var revisions []string - for _, info := range infos { - revisions = append(revisions, info.Revision) - } - pds, localErr = resources.PDGetter.GetByRevisions(ctx, revisions...) - if localErr != nil { - return localErr - } - } - logger.Info("get pod effective decorations before create it", "EffectivePodDecorations", utilspoddecoration.BuildInfo(pds)) - return utilspoddecoration.PatchListOfDecorations(in, pds) - }, - ) - if err != nil { - return fmt.Errorf("fail to new Pod from revision %s: %w", revision.Name, err) - } - err = r.pvcControl.CreatePodPvcs(ctx, cls, pod, resources.ExistingPvcs) - if err != nil { - return fmt.Errorf("fail to create PVCs for pod %s: %w", pod.Name, err) - } - newPod := pod.DeepCopy() - logger.Info("try to create Pod with revision of collaSet", "revision", revision.Name) - if pod, err = r.podControl.CreatePod(newPod); err != nil { - return err - } - // add an expectation for this pod creation, before next reconciling - return r.cacheExpectations.ExpectCreation(kubeutilsclient.ObjectKeyString(cls), collasetutils.PodGVK, pod.Namespace, pod.Name) - }) - if needUpdateContext.Load() { - logger.Info("try to update ResourceContext for CollaSet after scaling out", "Context", ownedIDs) - if updateContextErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { - return r.podContextControl.UpdateToPodContext(ctx, cls, ownedIDs) - }); updateContextErr != nil { - err = controllerutils.AggregateErrors([]error{updateContextErr, err}) - } - } - if err != nil { - collasetutils.AddOrUpdateCondition(resources.NewStatus, appsv1alpha1.CollaSetScale, err, "ScaleOutFailed", err.Error()) - return succCount > 0, recordedRequeueAfter, err - } - r.recorder.Eventf(cls, corev1.EventTypeNormal, "ScaleOut", "scale out %d Pod(s)", succCount) - collasetutils.AddOrUpdateCondition(resources.NewStatus, appsv1alpha1.CollaSetScale, nil, "ScaleOut", "") - return succCount > 0, recordedRequeueAfter, err - } - } - - if diff <= 0 { - // chose the pods to scale in - podsToScaleIn := getPodsToDelete(cls, activePods, replacePodMap, diff*-1) - // filter out Pods need to trigger PodOpsLifecycle - podCh := make(chan *collasetutils.PodWrapper, len(podsToScaleIn)) - for i := range podsToScaleIn { - if podopslifecycle.IsDuringOps(collasetutils.ScaleInOpsLifecycleAdapter, podsToScaleIn[i].Pod) { - continue - } - podCh <- podsToScaleIn[i] - } - - // trigger Pods to enter PodOpsLifecycle - succCount, err := controllerutils.SlowStartBatch(len(podCh), controllerutils.SlowStartInitialBatchSize, false, func(_ int, err error) error { - pod := <-podCh - - // trigger PodOpsLifecycle with scaleIn OperationType - logger.Info("try to begin PodOpsLifecycle for scaling in Pod in CollaSet", "pod", commonutils.ObjectKeyString(pod)) - if updated, err := podopslifecycle.Begin(r.client, collasetutils.ScaleInOpsLifecycleAdapter, pod.Pod); err != nil { - return fmt.Errorf("fail to begin PodOpsLifecycle for Scaling in Pod %s/%s: %w", pod.Namespace, pod.Name, err) - } else if updated { - r.recorder.Eventf(pod.Pod, corev1.EventTypeNormal, "BeginScaleInLifecycle", "succeed to begin PodOpsLifecycle for scaling in") - // add an expectation for this pod creation, before next reconciling - if err := r.cacheExpectations.ExpectUpdation(kubeutilsclient.ObjectKeyString(cls), collasetutils.PodGVK, pod.Namespace, pod.Name, pod.ResourceVersion); err != nil { - return err - } - } - return nil - }) - scaling = succCount > 0 - - if err != nil { - collasetutils.AddOrUpdateCondition(resources.NewStatus, appsv1alpha1.CollaSetScale, err, "ScaleInFailed", err.Error()) - return scaling, recordedRequeueAfter, err - } else if scaling { - collasetutils.AddOrUpdateCondition(resources.NewStatus, appsv1alpha1.CollaSetScale, nil, "ScaleIn", "") - } - - needUpdateContext := false - for i, podWrapper := range podsToScaleIn { - requeueAfter, allowed := podopslifecycle.AllowOps(collasetutils.ScaleInOpsLifecycleAdapter, realValue(cls.Spec.ScaleStrategy.OperationDelaySeconds), podWrapper.Pod) - if !allowed && podWrapper.DeletionTimestamp == nil { - r.recorder.Eventf(podWrapper.Pod, corev1.EventTypeNormal, "PodScaleInLifecycle", "Pod is not allowed to scale in") - continue - } - - if requeueAfter != nil { - r.recorder.Eventf(podWrapper.Pod, corev1.EventTypeNormal, "PodScaleInLifecycle", "delay Pod scale in for %d seconds", requeueAfter.Seconds()) - if recordedRequeueAfter == nil || *requeueAfter < *recordedRequeueAfter { - recordedRequeueAfter = requeueAfter - } - - continue - } - - // if Pod is allowed to operate or Pod has already been deleted, promte to delete Pod - if contextDetail, exist := ownedIDs[podWrapper.ID]; exist && !contextDetail.Contains(ScaleInContextDataKey, "true") { - needUpdateContext = true - contextDetail.Put(ScaleInContextDataKey, "true") - } - - if podWrapper.DeletionTimestamp != nil { - continue - } - - podCh <- podsToScaleIn[i] - } - - // mark these Pods to scalingIn - if needUpdateContext { - logger.Info("try to update ResourceContext for CollaSet when scaling in Pod", "Context", ownedIDs) - err = retry.RetryOnConflict(retry.DefaultRetry, func() error { - return r.podContextControl.UpdateToPodContext(ctx, cls, ownedIDs) - }) - - if err != nil { - collasetutils.AddOrUpdateCondition(resources.NewStatus, appsv1alpha1.CollaSetScale, err, "ScaleInFailed", fmt.Sprintf("failed to update Context for scaling in: %s", err)) - return scaling, recordedRequeueAfter, err - } else { - collasetutils.AddOrUpdateCondition(resources.NewStatus, appsv1alpha1.CollaSetScale, nil, "ScaleIn", "") - } - } - - // do delete Pod resource - succCount, err = controllerutils.SlowStartBatch(len(podCh), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) error { - pod := <-podCh - logger.Info("try to scale in Pod", "pod", commonutils.ObjectKeyString(pod)) - if err := r.podControl.DeletePod(pod.Pod); err != nil { - return fmt.Errorf("fail to delete Pod %s/%s when scaling in: %w", pod.Namespace, pod.Name, err) - } - - r.recorder.Eventf(cls, corev1.EventTypeNormal, "PodDeleted", "succeed to scale in Pod %s/%s", pod.Namespace, pod.Name) - if err := r.cacheExpectations.ExpectDeletion(kubeutilsclient.ObjectKeyString(cls), collasetutils.PodGVK, pod.Namespace, pod.Name); err != nil { - return err - } - - // delete PVC if pod is in update replace, or retention policy is "Deleted" - _, originExist := pod.Labels[appsv1alpha1.PodReplacePairNewId] - _, replaceExist := pod.Labels[appsv1alpha1.PodReplacePairOriginName] - if originExist || replaceExist || collasetutils.PvcPolicyWhenScaled(cls) == appsv1alpha1.DeletePersistentVolumeClaimRetentionPolicyType { - return r.pvcControl.DeletePodPvcs(ctx, cls, pod.Pod, resources.ExistingPvcs) - } - return nil - }) - scaling = scaling || succCount > 0 - - if succCount > 0 { - r.recorder.Eventf(cls, corev1.EventTypeNormal, "ScaleIn", "scale in %d Pod(s)", succCount) - } - if err != nil { - collasetutils.AddOrUpdateCondition(resources.NewStatus, appsv1alpha1.CollaSetScale, err, "ScaleInFailed", fmt.Sprintf("fail to delete Pod for scaling in: %s", err)) - return scaling, recordedRequeueAfter, err - } else if succCount > 0 { - collasetutils.AddOrUpdateCondition(resources.NewStatus, appsv1alpha1.CollaSetScale, nil, "ScaleIn", "") - } - } - - // reset ContextDetail.ScalingIn, if there are Pods had its PodOpsLifecycle reverted - needUpdatePodContext := false - for _, podWrapper := range activePods { - if contextDetail, exist := ownedIDs[podWrapper.ID]; exist && contextDetail.Contains(ScaleInContextDataKey, "true") && - !podopslifecycle.IsDuringOps(collasetutils.ScaleInOpsLifecycleAdapter, podWrapper) { - needUpdatePodContext = true - contextDetail.Remove(ScaleInContextDataKey) - } - } - - if needUpdatePodContext { - logger.Info("try to update ResourceContext for CollaSet after scaling", "Context", ownedIDs) - if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - return r.podContextControl.UpdateToPodContext(ctx, cls, ownedIDs) - }); err != nil { - return scaling, recordedRequeueAfter, fmt.Errorf("fail to reset ResourceContext: %w", err) - } - } - - if !scaling { - collasetutils.AddOrUpdateCondition(resources.NewStatus, appsv1alpha1.CollaSetScale, nil, "Scaled", "") - } - - return scaling, recordedRequeueAfter, nil -} - -// FilterOutPlaceHolderPodWrappers filter out placeholder pods -func FilterOutPlaceHolderPodWrappers(pods []*collasetutils.PodWrapper) []*collasetutils.PodWrapper { - var filteredPodWrappers []*collasetutils.PodWrapper - for _, pod := range pods { - if pod.PlaceHolder { - continue - } - filteredPodWrappers = append(filteredPodWrappers, pod) - } - return filteredPodWrappers -} - -func extractAvailableContexts(diff int, ownedIDs map[int]*appsv1alpha1.ContextDetail, podInstanceIDSet map[int]struct{}) []*appsv1alpha1.ContextDetail { - var availableContexts []*appsv1alpha1.ContextDetail - if diff <= 0 { - return availableContexts - } - - idx := 0 - for id := range ownedIDs { - if _, inUsed := podInstanceIDSet[id]; inUsed { - continue - } - - availableContexts = append(availableContexts, ownedIDs[id]) - idx++ - if idx == diff { - break - } - } - - return availableContexts -} - -// DeletePodsByLabel try to trigger pod deletion by to-delete label -func DeletePodsByLabel(podControl podcontrol.Interface, needDeletePods []*corev1.Pod) error { - _, err := controllerutils.SlowStartBatch(len(needDeletePods), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) error { - pod := needDeletePods[i] - if _, exist := pod.Labels[appsv1alpha1.PodDeletionIndicationLabelKey]; !exist { - patch := client.RawPatch(types.StrategicMergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{"%s":"%d"}}}`, appsv1alpha1.PodDeletionIndicationLabelKey, time.Now().UnixNano()))) - if err := podControl.PatchPod(pod, patch); err != nil { - return fmt.Errorf("failed to delete pod when syncPods %s/%s %w", pod.Namespace, pod.Name, err) - } - } - return nil - }) - return err -} - -// decideContextRevision decides revision for 3 pod create types: (1) just create, (2) upgrade by recreate, (3) delete and recreate -func decideContextRevision(contextDetail *appsv1alpha1.ContextDetail, updatedRevision *appsv1.ControllerRevision, createSucceeded bool) bool { - needUpdateContext := false - if !createSucceeded { - if contextDetail.Contains(podcontext.JustCreateContextDataKey, "true") { - // TODO choose just create pods' revision according to scaleStrategy - contextDetail.Put(podcontext.RevisionContextDataKey, updatedRevision.Name) - delete(contextDetail.Data, podcontext.PodDecorationRevisionKey) - needUpdateContext = true - } else if contextDetail.Contains(podcontext.RecreateUpdateContextDataKey, "true") { - contextDetail.Put(podcontext.RevisionContextDataKey, updatedRevision.Name) - delete(contextDetail.Data, podcontext.PodDecorationRevisionKey) - needUpdateContext = true - } - // if pod is delete and recreate, never change revisionKey - } else { - // TODO delete ID if create succeeded - contextDetail.Remove(podcontext.JustCreateContextDataKey) - contextDetail.Remove(podcontext.RecreateUpdateContextDataKey) - needUpdateContext = true - } - return needUpdateContext -} - -// Update is used to update pods to spec.template -func (r *RealSyncControl) Update( - ctx context.Context, - cls *appsv1alpha1.CollaSet, - resources *collasetutils.RelatedResources, - podWrappers []*collasetutils.PodWrapper, - ownedIDs map[int]*appsv1alpha1.ContextDetail, -) (bool, *time.Duration, error) { - logger := r.logger.WithValues("collaset", commonutils.ObjectKeyString(cls)) - var recordedRequeueAfter *time.Duration - // 1. scan and analysis pods update info for active pods and PlaceHolder pods - podUpdateInfos, err := r.attachPodUpdateInfo(ctx, cls, podWrappers, resources) - if err != nil { - return false, nil, fmt.Errorf("fail to attach pod update info, %w", err) - } - - // 2. decide Pod update candidates - candidates := decidePodToUpdate(cls, podUpdateInfos) - podToUpdate := filterOutPlaceHolderUpdateInfos(candidates) - podCh := make(chan *PodUpdateInfo, len(podToUpdate)) - updater := newPodUpdater(r.client, cls, r.podControl, r.podContextControl, r.recorder, r.cacheExpectations) - updating := false - - // 3. filter already updated revision, - for i, podInfo := range podToUpdate { - if podInfo.IsUpdatedRevision && !podInfo.PodDecorationChanged && !podInfo.PvcTmpHashChanged { - continue - } - - // 3.1 fulfillPodUpdateInfo to all not updatedRevision pod - if podInfo.CurrentRevision.Name != UnknownRevision { - if err = updater.FulfillPodUpdatedInfo(ctx, resources.UpdatedRevision, podInfo); err != nil { - logger.Error(err, fmt.Sprintf("fail to analyze pod %s/%s in-place update support", podInfo.Namespace, podInfo.Name)) - continue - } - } - - if podInfo.DeletionTimestamp != nil { - continue - } - - if podInfo.isDuringUpdateOps || podInfo.isDuringScaleInOps { - continue - } - - podCh <- podToUpdate[i] - } - - // 4. begin pod update lifecycle - updating, err = updater.BeginUpdatePod(ctx, resources, podCh) - if err != nil { - return updating, recordedRequeueAfter, err - } - - // 5. (1) filter out pods not allow to ops now, such as OperationDelaySeconds strategy; (2) update PlaceHolder Pods resourceContext revision - recordedRequeueAfter, err = updater.FilterAllowOpsPods(ctx, candidates, ownedIDs, resources, podCh) - if err != nil { - collasetutils.AddOrUpdateCondition(resources.NewStatus, - appsv1alpha1.CollaSetUpdate, err, "UpdateFailed", - fmt.Sprintf("fail to update Context for updating: %s", err)) - return updating, recordedRequeueAfter, err - } else { - collasetutils.AddOrUpdateCondition(resources.NewStatus, - appsv1alpha1.CollaSetUpdate, nil, "Updated", "") - } - - // 6. update Pod - succCount, err := controllerutils.SlowStartBatch(len(podCh), controllerutils.SlowStartInitialBatchSize, false, func(_ int, _ error) error { - podInfo := <-podCh - logger.Info("before pod update operation", - "pod", commonutils.ObjectKeyString(podInfo.Pod), - "revision.from", podInfo.CurrentRevision.Name, - "revision.to", resources.UpdatedRevision.Name, - "inPlaceUpdate", podInfo.InPlaceUpdateSupport, - "onlyMetadataChanged", podInfo.OnlyMetadataChanged, - ) - - // when a pod during replace, it turns to ReplaceUpdate - if podInfo.isInReplace && cls.Spec.UpdateStrategy.PodUpdatePolicy != appsv1alpha1.CollaSetReplacePodUpdateStrategyType { - return updateReplaceOriginPod(ctx, r.client, r.recorder, podInfo, podInfo.replacePairNewPodInfo) - } - - return updater.UpgradePod(ctx, podInfo) - }) - - updating = updating || succCount > 0 - if err != nil { - collasetutils.AddOrUpdateCondition(resources.NewStatus, appsv1alpha1.CollaSetUpdate, err, "UpdateFailed", err.Error()) - return updating, recordedRequeueAfter, err - } else { - collasetutils.AddOrUpdateCondition(resources.NewStatus, appsv1alpha1.CollaSetUpdate, nil, "Updated", "") - } - - podToUpdateSet := sets.String{} - for i := range podToUpdate { - podToUpdateSet.Insert(podToUpdate[i].Name) - } - // 7. try to finish all Pods' PodOpsLifecycle if its update is finished. - succCount, err = controllerutils.SlowStartBatch(len(podUpdateInfos), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) error { - podInfo := podUpdateInfos[i] - - if !(podInfo.isDuringUpdateOps || podInfo.isInReplaceUpdate) || podInfo.PlaceHolder || podInfo.DeletionTimestamp != nil { - return nil - } - - var finishByCancelUpdate bool - var updateFinished bool - var msg string - var err error - if !podToUpdateSet.Has(podInfo.Name) { - // Pod is out of scope (partition or by label) and not start update yet, finish update by cancel - finishByCancelUpdate = !podInfo.isAllowUpdateOps - logger.Info("out of update scope", "pod", commonutils.ObjectKeyString(podInfo.Pod), "finishByCancelUpdate", finishByCancelUpdate) - } else if !podInfo.isAllowUpdateOps { - // Pod is in update scope, but is not start update yet, if pod is updatedRevision, just finish update by cancel - finishByCancelUpdate = podInfo.IsUpdatedRevision - } else { - // Pod is in update scope and allowed to update, check and finish update gracefully - if updateFinished, msg, err = updater.GetPodUpdateFinishStatus(ctx, podInfo); err != nil { - return fmt.Errorf("failed to get pod %s/%s update finished: %w", podInfo.Namespace, podInfo.Name, err) - } else if !updateFinished { - r.recorder.Eventf(podInfo.Pod, - corev1.EventTypeNormal, - "WaitingUpdateReady", - "waiting for pod %s/%s to update finished: %s", - podInfo.Namespace, podInfo.Name, msg) - } - } - - if updateFinished || finishByCancelUpdate { - if err := updater.FinishUpdatePod(ctx, podInfo, finishByCancelUpdate); err != nil { - return err - } - r.recorder.Eventf(podInfo.Pod, - corev1.EventTypeNormal, - "UpdatePodFinished", - "pod %s/%s with current revision %s is finished for upgrade to revision %s [finishByCancelUpdate=%v]", - podInfo.Namespace, podInfo.Name, podInfo.CurrentRevision.Name, podInfo.UpdateRevision.Name, finishByCancelUpdate) - } - return nil - }) - - return updating || succCount > 0, recordedRequeueAfter, err -} - -// getAvailablePodIDs try to extract and re-allocate want available IDs. -func (r *RealSyncControl) getAvailablePodIDs( - ctx context.Context, - want int, - instance *appsv1alpha1.CollaSet, - resources *collasetutils.RelatedResources, - ownedIDs map[int]*appsv1alpha1.ContextDetail, - currentIDs map[int]struct{}, -) ([]*appsv1alpha1.ContextDetail, map[int]*appsv1alpha1.ContextDetail, error) { - availableContexts := extractAvailableContexts(want, ownedIDs, currentIDs) - if len(availableContexts) >= want { - return availableContexts, ownedIDs, nil - } - - diff := want - len(availableContexts) - - var newOwnedIDs map[int]*appsv1alpha1.ContextDetail - var err error - if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - newOwnedIDs, err = r.podContextControl.AllocateID(ctx, instance, resources.UpdatedRevision.Name, len(ownedIDs)+diff) - return err - }); err != nil { - return nil, ownedIDs, fmt.Errorf("fail to allocate IDs using context when include Pods: %w", err) - } - - return extractAvailableContexts(want, newOwnedIDs, currentIDs), newOwnedIDs, nil -} - -// reclaimOwnedIDs delete and reclaim unused IDs -func (r *RealSyncControl) reclaimOwnedIDs( - ctx context.Context, - needUpdateContext bool, - cls *appsv1alpha1.CollaSet, - idToReclaim sets.Int, - ownedIDs map[int]*appsv1alpha1.ContextDetail, - currentIDs map[int]struct{}, -) error { - // TODO stateful case - // 1) only reclaim non-existing Pods' ID. Do not reclaim terminating Pods' ID until these Pods and PVC have been deleted from ETCD - // 2) do not filter out these terminating Pods - for id, contextDetail := range ownedIDs { - if _, exist := currentIDs[id]; exist { - continue - } - if contextDetail.Contains(ScaleInContextDataKey, "true") { - idToReclaim.Insert(id) - } - } - - for _, id := range idToReclaim.List() { - needUpdateContext = true - delete(ownedIDs, id) - } - - // TODO clean replace-pair-keys or dirty podContext - // 1) replace pair pod are not exists - // 2) pod exists but is not replaceIndicated - - if needUpdateContext { - logger := r.logger.WithValues("collaset", commonutils.ObjectKeyString(cls)) - logger.Info("try to update ResourceContext for CollaSet when sync", "Context", ownedIDs) - if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - return r.podContextControl.UpdateToPodContext(ctx, cls, ownedIDs) - }); err != nil { - return fmt.Errorf("fail to update ResourceContext when reclaiming IDs: %w", err) - } - } - return nil -} - -func realValue(val *int32) int32 { - if val == nil { - return 0 - } - - return *val -} diff --git a/pkg/controllers/collaset/synccontrol/update.go b/pkg/controllers/collaset/synccontrol/update.go deleted file mode 100644 index a83386ca..00000000 --- a/pkg/controllers/collaset/synccontrol/update.go +++ /dev/null @@ -1,925 +0,0 @@ -/* -Copyright 2023 The KusionStack Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package synccontrol - -import ( - "context" - "encoding/json" - "fmt" - "sort" - "time" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/equality" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/client-go/tools/record" - "k8s.io/client-go/util/retry" - "k8s.io/utils/ptr" - appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" - kubeutilsclient "kusionstack.io/kube-utils/client" - kubeutilsexpectations "kusionstack.io/kube-utils/controller/expectations" - "sigs.k8s.io/controller-runtime/pkg/client" - - "kusionstack.io/kuperator/pkg/controllers/collaset/podcontext" - "kusionstack.io/kuperator/pkg/controllers/collaset/podcontrol" - "kusionstack.io/kuperator/pkg/controllers/collaset/pvccontrol" - "kusionstack.io/kuperator/pkg/controllers/collaset/utils" - collasetutils "kusionstack.io/kuperator/pkg/controllers/collaset/utils" - ojutils "kusionstack.io/kuperator/pkg/controllers/operationjob/utils" - controllerutils "kusionstack.io/kuperator/pkg/controllers/utils" - utilspoddecoration "kusionstack.io/kuperator/pkg/controllers/utils/poddecoration" - "kusionstack.io/kuperator/pkg/controllers/utils/poddecoration/anno" - "kusionstack.io/kuperator/pkg/controllers/utils/podopslifecycle" -) - -const UnknownRevision = "__unknownRevision__" - -type PodUpdateInfo struct { - *utils.PodWrapper - - UpdatedPod *corev1.Pod - - InPlaceUpdateSupport bool - OnlyMetadataChanged bool - - // indicate if this pod has up-to-date revision from its owner, like CollaSet - IsUpdatedRevision bool - // carry the pod's current revision - CurrentRevision *appsv1.ControllerRevision - // carry the desired update revision - UpdateRevision *appsv1.ControllerRevision - - // indicates effected PodDecorations changed - PodDecorationChanged bool - // indicate if the pvc template changed - PvcTmpHashChanged bool - - CurrentPodDecorations map[string]*appsv1alpha1.PodDecoration - UpdatedPodDecorations map[string]*appsv1alpha1.PodDecoration - - // indicates the Pod is during UpdateOpsLifecycle - isDuringUpdateOps bool - // indicates the Pod is during ScaleOpsLifecycle - isDuringScaleInOps bool - // indicates operate is allowed for Update - isAllowUpdateOps bool - // requeue after for operationDelaySeconds - requeueForOperationDelay *time.Duration - - // for replace update - // judge pod in replace and replace updating - isInReplace bool - isInReplaceUpdate bool - - // replace new created pod - replacePairNewPodInfo *PodUpdateInfo - - // replace origin pod - replacePairOriginPodName string -} - -func (r *RealSyncControl) attachPodUpdateInfo(ctx context.Context, cls *appsv1alpha1.CollaSet, pods []*collasetutils.PodWrapper, resource *collasetutils.RelatedResources) ([]*PodUpdateInfo, error) { - activePods := FilterOutPlaceHolderPodWrappers(pods) - podUpdateInfoList := make([]*PodUpdateInfo, len(activePods)) - - for i, pod := range activePods { - updateInfo := &PodUpdateInfo{ - PodWrapper: pod, - } - - currentPDs, err := resource.PDGetter.GetOnPod(ctx, pod.Pod) - if err != nil { - return nil, err - } - updatedPDs, err := resource.PDGetter.GetEffective(ctx, pod.Pod) - if err != nil { - return nil, err - } - - if len(currentPDs) != len(updatedPDs) { - updateInfo.PodDecorationChanged = true - } else { - revisionSets := sets.NewString() - for rev := range currentPDs { - revisionSets.Insert(rev) - } - for rev := range updatedPDs { - if !revisionSets.Has(rev) { - updateInfo.PodDecorationChanged = true - break - } - } - } - updateInfo.CurrentPodDecorations = currentPDs - updateInfo.UpdatedPodDecorations = updatedPDs - - updateInfo.UpdateRevision = resource.UpdatedRevision - // decide this pod current revision, or nil if not indicated - if pod.Labels != nil { - currentRevisionName, exist := pod.Labels[appsv1.ControllerRevisionHashLabelKey] - if exist { - if currentRevisionName == resource.UpdatedRevision.Name { - updateInfo.IsUpdatedRevision = true - updateInfo.CurrentRevision = resource.UpdatedRevision - } else { - updateInfo.IsUpdatedRevision = false - for _, rv := range resource.Revisions { - if currentRevisionName == rv.Name { - updateInfo.CurrentRevision = rv - } - } - } - } - } - - // default CurrentRevision is an empty revision - if updateInfo.CurrentRevision == nil { - updateInfo.CurrentRevision = &appsv1.ControllerRevision{ - ObjectMeta: metav1.ObjectMeta{ - Name: UnknownRevision, - }, - } - r.recorder.Eventf(pod.Pod, - corev1.EventTypeWarning, - "PodCurrentRevisionNotFound", - "pod is going to be updated by recreate because: (1) controller-revision-hash label not found, or (2) not found in history revisions") - } - - // decide whether the PodOpsLifecycle is during ops or not - updateInfo.isDuringUpdateOps = podopslifecycle.IsDuringOps(utils.UpdateOpsLifecycleAdapter, pod) - updateInfo.isDuringScaleInOps = podopslifecycle.IsDuringOps(utils.ScaleInOpsLifecycleAdapter, pod) - updateInfo.requeueForOperationDelay, updateInfo.isAllowUpdateOps = podopslifecycle.AllowOps(collasetutils.UpdateOpsLifecycleAdapter, realValue(cls.Spec.UpdateStrategy.OperationDelaySeconds), pod) - updateInfo.PvcTmpHashChanged, err = pvccontrol.IsPodPvcTmpChanged(cls, pod.Pod, resource.ExistingPvcs) - if err != nil { - return nil, fmt.Errorf("fail to check pvc template changed, %w", err) - } - podUpdateInfoList[i] = updateInfo - } - - // attach replace info - podUpdateInfoMap := make(map[string]*PodUpdateInfo) - for _, podUpdateInfo := range podUpdateInfoList { - podUpdateInfoMap[podUpdateInfo.Name] = podUpdateInfo - } - replacePodMap := classifyPodReplacingMapping(activePods) - // originPod's isAllowUpdateOps depends on these 2 cases: - // (1) pod is during replacing but not during replaceUpdate, keep it legacy value - // (2) pod is during replaceUpdate, set to "true" if newPod is service available - for originPodName, replacePairNewPod := range replacePodMap { - originPodInfo := podUpdateInfoMap[originPodName] - _, replaceIndicated := originPodInfo.Labels[appsv1alpha1.PodReplaceIndicationLabelKey] - _, replaceByReplaceUpdate := originPodInfo.Labels[appsv1alpha1.PodReplaceByReplaceUpdateLabelKey] - isReplaceUpdating := replaceIndicated && replaceByReplaceUpdate - - originPodInfo.isInReplace = replaceIndicated - originPodInfo.isInReplaceUpdate = isReplaceUpdating - if replacePairNewPod != nil { - // origin pod is allowed to ops if new pod is serviceAvailable - _, newPodSa := replacePairNewPod.Labels[appsv1alpha1.PodServiceAvailableLabel] - originPodInfo.isAllowUpdateOps = originPodInfo.isAllowUpdateOps || newPodSa - // attach replace new pod updateInfo - replacePairNewPodInfo := podUpdateInfoMap[replacePairNewPod.Name] - replacePairNewPodInfo.isInReplace = true - // in case of to-replace label is removed from origin pod, new pod is still in replaceUpdate - replacePairNewPodInfo.isInReplaceUpdate = replaceByReplaceUpdate - - replacePairNewPodInfo.replacePairOriginPodName = originPodName - originPodInfo.replacePairNewPodInfo = replacePairNewPodInfo - } - } - - // join PlaceHolder pods in updating - for _, pod := range pods { - if !pod.PlaceHolder { - continue - } - updateInfo := &PodUpdateInfo{ - PodWrapper: pod, - UpdateRevision: resource.UpdatedRevision, - } - if revision, exist := pod.ContextDetail.Data[podcontext.RevisionContextDataKey]; exist && - revision == resource.UpdatedRevision.Name { - updateInfo.IsUpdatedRevision = true - } - podUpdateInfoList = append(podUpdateInfoList, updateInfo) - } - - return podUpdateInfoList, nil -} - -func filterOutPlaceHolderUpdateInfos(pods []*PodUpdateInfo) []*PodUpdateInfo { - var filteredPodUpdateInfos []*PodUpdateInfo - for _, pod := range pods { - if pod.PlaceHolder { - continue - } - filteredPodUpdateInfos = append(filteredPodUpdateInfos, pod) - } - return filteredPodUpdateInfos -} - -func decidePodToUpdate( - cls *appsv1alpha1.CollaSet, - podInfos []*PodUpdateInfo, -) []*PodUpdateInfo { - filteredPodInfos := getTargetsUpdatePods(podInfos) - - if cls.Spec.UpdateStrategy.RollingUpdate != nil && cls.Spec.UpdateStrategy.RollingUpdate.ByLabel != nil { - activePodInfos := filterOutPlaceHolderUpdateInfos(filteredPodInfos) - return decidePodToUpdateByLabel(cls, activePodInfos) - } - return decidePodToUpdateByPartition(cls, filteredPodInfos) -} - -func decidePodToUpdateByLabel(_ *appsv1alpha1.CollaSet, podInfos []*PodUpdateInfo) (podToUpdate []*PodUpdateInfo) { - for i := range podInfos { - if _, exist := podInfos[i].Labels[appsv1alpha1.CollaSetUpdateIndicateLabelKey]; exist { - podToUpdate = append(podToUpdate, podInfos[i]) - continue - } - - if podInfos[i].PodDecorationChanged { - if podInfos[i].isInReplace { - continue - } - // separate pd and collaset update progress - podInfos[i].IsUpdatedRevision = true - podInfos[i].UpdateRevision = podInfos[i].CurrentRevision - podToUpdate = append(podToUpdate, podInfos[i]) - } - } - return podToUpdate -} - -func decidePodToUpdateByPartition( - cls *appsv1alpha1.CollaSet, - filteredPodInfos []*PodUpdateInfo, -) []*PodUpdateInfo { - replicas := ptr.Deref(cls.Spec.Replicas, 0) - currentPodCount := int32(len(filteredPodInfos)) - partition := int32(0) - if cls.Spec.UpdateStrategy.RollingUpdate != nil && cls.Spec.UpdateStrategy.RollingUpdate.ByPartition != nil { - partition = ptr.Deref(cls.Spec.UpdateStrategy.RollingUpdate.ByPartition.Partition, 0) - } - - // update all or not update any replicas - if partition == 0 { - return filteredPodInfos - } - if partition >= replicas { - return nil - } - - // partial update replicas - ordered := orderByDefault(filteredPodInfos) - sort.Sort(ordered) - podToUpdate := ordered[:replicas-partition] - for i := replicas - partition; i < int32Min(replicas, currentPodCount); i++ { - if ordered[i].PodDecorationChanged { - // separate pd and collaset update progress - filteredPodInfos[i].IsUpdatedRevision = true - ordered[i].UpdateRevision = ordered[i].CurrentRevision - podToUpdate = append(podToUpdate, ordered[i]) - } - } - return podToUpdate -} - -// when sort pods to choose update, only sort (1) replace origin pods, (2) non-exclude pods -func getTargetsUpdatePods(podInfos []*PodUpdateInfo) (filteredPodInfos []*PodUpdateInfo) { - for _, podInfo := range podInfos { - if podInfo.replacePairOriginPodName != "" { - continue - } - - if podInfo.PlaceHolder { - _, isReplaceNewPod := podInfo.ContextDetail.Data[ReplaceOriginPodIDContextDataKey] - if isReplaceNewPod { - continue - } - } - - filteredPodInfos = append(filteredPodInfos, podInfo) - } - return filteredPodInfos -} - -type orderByDefault []*PodUpdateInfo - -func (o orderByDefault) Len() int { - return len(o) -} - -func (o orderByDefault) Swap(i, j int) { o[i], o[j] = o[j], o[i] } - -func (o orderByDefault) Less(i, j int) bool { - l, r := o[i], o[j] - if l.IsUpdatedRevision != r.IsUpdatedRevision { - return l.IsUpdatedRevision - } - - if l.isDuringUpdateOps != r.isDuringUpdateOps { - return l.isDuringUpdateOps - } - - if l.isInReplaceUpdate != r.isInReplaceUpdate { - return l.isInReplaceUpdate - } - - if l.PlaceHolder != r.PlaceHolder { - return r.PlaceHolder - } - - if l.PlaceHolder && r.PlaceHolder { - return true - } - - if controllerutils.BeforeReady(l.Pod) == controllerutils.BeforeReady(r.Pod) && - l.PodDecorationChanged != r.PodDecorationChanged { - return l.PodDecorationChanged - } - - if controllerutils.IsPodServiceAvailable(l.Pod) != controllerutils.IsPodServiceAvailable(r.Pod) { - return controllerutils.IsPodServiceAvailable(r.Pod) - } - - return utils.ComparePod(l.Pod, r.Pod) -} - -type PodUpdater interface { - Setup(client.Client, *appsv1alpha1.CollaSet, podcontrol.Interface, podcontext.Interface, record.EventRecorder, kubeutilsexpectations.CacheExpectationsInterface) - FulfillPodUpdatedInfo(ctx context.Context, revision *appsv1.ControllerRevision, podUpdateInfo *PodUpdateInfo) error - BeginUpdatePod(ctx context.Context, resources *collasetutils.RelatedResources, podCh chan *PodUpdateInfo) (bool, error) - FilterAllowOpsPods(ctx context.Context, podToUpdate []*PodUpdateInfo, ownedIDs map[int]*appsv1alpha1.ContextDetail, resources *collasetutils.RelatedResources, podCh chan *PodUpdateInfo) (*time.Duration, error) - UpgradePod(ctx context.Context, podInfo *PodUpdateInfo) error - GetPodUpdateFinishStatus(ctx context.Context, podUpdateInfo *PodUpdateInfo) (bool, string, error) - FinishUpdatePod(ctx context.Context, podInfo *PodUpdateInfo, finishByCancelUpdate bool) error -} - -type GenericPodUpdater struct { - *appsv1alpha1.CollaSet - PodControl podcontrol.Interface - PodContextControl podcontext.Interface - Recorder record.EventRecorder - client.Client - CacheExpectations kubeutilsexpectations.CacheExpectationsInterface -} - -func (u *GenericPodUpdater) Setup(client client.Client, cls *appsv1alpha1.CollaSet, podControl podcontrol.Interface, podContextControl podcontext.Interface, recorder record.EventRecorder, cacheExpectations kubeutilsexpectations.CacheExpectationsInterface) { - u.Client = client - u.CollaSet = cls - u.PodControl = podControl - u.PodContextControl = podContextControl - u.Recorder = recorder - u.CacheExpectations = cacheExpectations -} - -func (u *GenericPodUpdater) BeginUpdatePod(_ context.Context, resources *collasetutils.RelatedResources, podCh chan *PodUpdateInfo) (bool, error) { - succCount, err := controllerutils.SlowStartBatch(len(podCh), controllerutils.SlowStartInitialBatchSize, false, func(int, error) error { - podInfo := <-podCh - u.Recorder.Eventf(podInfo.Pod, corev1.EventTypeNormal, "PodUpdateLifecycle", "try to begin PodOpsLifecycle for updating Pod of CollaSet") - - if updated, err := podopslifecycle.BeginWithCleaningOld(u.Client, collasetutils.UpdateOpsLifecycleAdapter, podInfo.Pod, func(obj client.Object) (bool, error) { - if !podInfo.OnlyMetadataChanged && !podInfo.InPlaceUpdateSupport { - return podopslifecycle.WhenBeginDelete(obj) - } - return false, nil - }); err != nil { - return fmt.Errorf("fail to begin PodOpsLifecycle for updating Pod %s/%s: %w", podInfo.Namespace, podInfo.Name, err) - } else if updated { - // add an expectation for this pod update, before next reconciling - if err := u.CacheExpectations.ExpectUpdation(kubeutilsclient.ObjectKeyString(u.CollaSet), collasetutils.PodGVK, podInfo.Pod.Namespace, podInfo.Pod.Name, podInfo.Pod.ResourceVersion); err != nil { - return err - } - } - return nil - }) - - updating := succCount > 0 - if err != nil { - collasetutils.AddOrUpdateCondition(resources.NewStatus, appsv1alpha1.CollaSetUpdate, err, "UpdateFailed", err.Error()) - return updating, err - } else { - collasetutils.AddOrUpdateCondition(resources.NewStatus, appsv1alpha1.CollaSetUpdate, nil, "Updated", "") - } - return updating, nil -} - -func (u *GenericPodUpdater) FilterAllowOpsPods(ctx context.Context, candidates []*PodUpdateInfo, ownedIDs map[int]*appsv1alpha1.ContextDetail, _ *collasetutils.RelatedResources, podCh chan *PodUpdateInfo) (*time.Duration, error) { - var recordedRequeueAfter *time.Duration - needUpdateContext := false - for i := range candidates { - podInfo := candidates[i] - - if !podInfo.PlaceHolder { - if !podInfo.isAllowUpdateOps { - continue - } - if podInfo.requeueForOperationDelay != nil { - u.Recorder.Eventf(podInfo, corev1.EventTypeNormal, "PodUpdateLifecycle", "delay Pod update for %f seconds", podInfo.requeueForOperationDelay.Seconds()) - if recordedRequeueAfter == nil || *podInfo.requeueForOperationDelay < *recordedRequeueAfter { - recordedRequeueAfter = podInfo.requeueForOperationDelay - } - continue - } - } - - podInfo.isAllowUpdateOps = true - - if podInfo.IsUpdatedRevision && !podInfo.PodDecorationChanged && !podInfo.PvcTmpHashChanged { - continue - } - - if _, exist := ownedIDs[podInfo.ID]; !exist { - u.Recorder.Eventf(u.CollaSet, corev1.EventTypeWarning, "PodBeforeUpdate", "pod %s/%s is not allowed to update because cannot find context id %s in resourceContext", podInfo.Namespace, podInfo.Name, podInfo.Labels[appsv1alpha1.PodInstanceIDLabelKey]) - continue - } - - if !ownedIDs[podInfo.ID].Contains(podcontext.RevisionContextDataKey, podInfo.UpdateRevision.Name) { - needUpdateContext = true - ownedIDs[podInfo.ID].Put(podcontext.RevisionContextDataKey, podInfo.UpdateRevision.Name) - } - - // mark podContext "PodRecreateUpgrade" if upgrade by recreate - isRecreateUpdatePolicy := u.CollaSet.Spec.UpdateStrategy.PodUpdatePolicy == appsv1alpha1.CollaSetRecreatePodUpdateStrategyType - if (!podInfo.OnlyMetadataChanged && !podInfo.InPlaceUpdateSupport) || isRecreateUpdatePolicy { - ownedIDs[podInfo.ID].Put(podcontext.RecreateUpdateContextDataKey, "true") - } - - if podInfo.PodDecorationChanged { - decorationStr := anno.GetDecorationInfoString(podInfo.UpdatedPodDecorations) - if val, ok := ownedIDs[podInfo.ID].Get(podcontext.PodDecorationRevisionKey); !ok || val != decorationStr { - needUpdateContext = true - ownedIDs[podInfo.ID].Put(podcontext.PodDecorationRevisionKey, decorationStr) - } - } - - if podInfo.PlaceHolder { - continue - } - - // if Pod has not been updated, update it. - podCh <- candidates[i] - } - // mark Pod to use updated revision before updating it. - if needUpdateContext { - u.Recorder.Eventf(u.CollaSet, corev1.EventTypeNormal, "UpdateToPodContext", "try to update ResourceContext for CollaSet") - err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - return u.PodContextControl.UpdateToPodContext(ctx, u.CollaSet, ownedIDs) - }) - return recordedRequeueAfter, err - } - return recordedRequeueAfter, nil -} - -func (u *GenericPodUpdater) FinishUpdatePod(_ context.Context, podInfo *PodUpdateInfo, finishByCancelUpdate bool) error { - if finishByCancelUpdate { - // cancel update lifecycle - return ojutils.CancelOpsLifecycle(u.Client, collasetutils.UpdateOpsLifecycleAdapter, podInfo.Pod) - } - - // pod is ops finished, finish the lifecycle gracefully - if updated, err := podopslifecycle.Finish(u.Client, collasetutils.UpdateOpsLifecycleAdapter, podInfo.Pod); err != nil { - return fmt.Errorf("failed to finish PodOpsLifecycle for updating Pod %s/%s: %w", podInfo.Namespace, podInfo.Name, err) - } else if updated { - // add an expectation for this pod update, before next reconciling - if err := u.CacheExpectations.ExpectUpdation(kubeutilsclient.ObjectKeyString(u.CollaSet), collasetutils.PodGVK, podInfo.Pod.Namespace, podInfo.Pod.Name, podInfo.Pod.ResourceVersion); err != nil { - return err - } - u.Recorder.Eventf(podInfo.Pod, - corev1.EventTypeNormal, - "UpdateReady", "pod %s/%s update finished", podInfo.Namespace, podInfo.Name) - } - return nil -} - -// Support users to define inPlaceOnlyPodUpdater and register through RegisterInPlaceOnlyUpdater -var inPlaceOnlyPodUpdater PodUpdater - -func RegisterInPlaceOnlyUpdater(podUpdater PodUpdater) { - inPlaceOnlyPodUpdater = podUpdater -} - -func newPodUpdater(client client.Client, cls *appsv1alpha1.CollaSet, podControl podcontrol.Interface, podContextControl podcontext.Interface, recorder record.EventRecorder, cacheExpectations kubeutilsexpectations.CacheExpectationsInterface) PodUpdater { - var podUpdater PodUpdater - switch cls.Spec.UpdateStrategy.PodUpdatePolicy { - case appsv1alpha1.CollaSetRecreatePodUpdateStrategyType: - podUpdater = &recreatePodUpdater{} - case appsv1alpha1.CollaSetInPlaceOnlyPodUpdateStrategyType: - if inPlaceOnlyPodUpdater != nil { - podUpdater = inPlaceOnlyPodUpdater - } else { - // In case of using native K8s, Pod is only allowed to update with container image, so InPlaceOnly policy is - // implemented with InPlaceIfPossible policy as default for compatibility. - podUpdater = &inPlaceIfPossibleUpdater{} - } - case appsv1alpha1.CollaSetReplacePodUpdateStrategyType: - podUpdater = &replaceUpdatePodUpdater{} - default: - podUpdater = &inPlaceIfPossibleUpdater{} - } - podUpdater.Setup(client, cls, podControl, podContextControl, recorder, cacheExpectations) - return podUpdater -} - -type PodStatus struct { - ContainerStates map[string]*ContainerStatus `json:"containerStates,omitempty"` -} - -type ContainerStatus struct { - LatestImage string `json:"latestImage,omitempty"` - LastImageID string `json:"lastImageID,omitempty"` -} - -type inPlaceIfPossibleUpdater struct { - GenericPodUpdater -} - -func (u *inPlaceIfPossibleUpdater) FulfillPodUpdatedInfo(_ context.Context, _ *appsv1.ControllerRevision, podUpdateInfo *PodUpdateInfo) error { - // 1. build pod from current and updated revision - ownerRef := metav1.NewControllerRef(u.CollaSet, appsv1alpha1.SchemeGroupVersion.WithKind("CollaSet")) - instanceId := podUpdateInfo.Labels[appsv1alpha1.PodInstanceIDLabelKey] - // TODO: use cache - currentPod, err := collasetutils.NewPodFrom(u.CollaSet, ownerRef, podUpdateInfo.CurrentRevision, instanceId, func(in *corev1.Pod) error { - return utilspoddecoration.PatchListOfDecorations(in, podUpdateInfo.CurrentPodDecorations) - }) - if err != nil { - return fmt.Errorf("fail to build Pod from current revision %s: %w", podUpdateInfo.CurrentRevision.Name, err) - } - - // TODO: use cache - podUpdateInfo.UpdatedPod, err = collasetutils.NewPodFrom(u.CollaSet, ownerRef, podUpdateInfo.UpdateRevision, instanceId, func(in *corev1.Pod) error { - return utilspoddecoration.PatchListOfDecorations(in, podUpdateInfo.UpdatedPodDecorations) - }) - if err != nil { - return fmt.Errorf("fail to build Pod from updated revision %s: %w", podUpdateInfo.UpdateRevision.Name, err) - } - - if podUpdateInfo.PvcTmpHashChanged { - podUpdateInfo.InPlaceUpdateSupport, podUpdateInfo.OnlyMetadataChanged = false, false - return nil - } - - // 2. compare current and updated pods. Only pod image and metadata are supported to update in-place - // TODO: use cache - var imageChangedContainers sets.String - podUpdateInfo.InPlaceUpdateSupport, podUpdateInfo.OnlyMetadataChanged, imageChangedContainers = u.diffPod(currentPod, podUpdateInfo.UpdatedPod) - // 3. if pod has changes more than metadata and image - if !podUpdateInfo.InPlaceUpdateSupport { - return nil - } - - podUpdateInfo.UpdatedPod, err = utils.PatchToPod(currentPod, podUpdateInfo.UpdatedPod, podUpdateInfo.Pod) - if err != nil { - return err - } - - var podStatus *PodStatus - if podUpdateInfo.OnlyMetadataChanged { - podStatus = &PodStatus{ContainerStates: nil} - } else { - containerCurrentStatusMapping := map[string]*corev1.ContainerStatus{} - for i := range podUpdateInfo.Status.ContainerStatuses { - status := podUpdateInfo.Status.ContainerStatuses[i] - // only store and compare imageID of changed containers - if imageChangedContainers != nil && imageChangedContainers.Has(status.Name) { - containerCurrentStatusMapping[status.Name] = &status - } - } - - podStatus = &PodStatus{ContainerStates: map[string]*ContainerStatus{}} - for _, container := range podUpdateInfo.UpdatedPod.Spec.Containers { - podStatus.ContainerStates[container.Name] = &ContainerStatus{ - // store image of each container in updated Pod - LatestImage: container.Image, - } - - containerCurrentStatus, exist := containerCurrentStatusMapping[container.Name] - if !exist { - continue - } - - // store image ID of each container in current Pod - podStatus.ContainerStates[container.Name].LastImageID = containerCurrentStatus.ImageID - } - } - - podStatusStr, err := json.Marshal(podStatus) - if err != nil { - return err - } - - if podUpdateInfo.UpdatedPod.Annotations == nil { - podUpdateInfo.UpdatedPod.Annotations = map[string]string{} - } - podUpdateInfo.UpdatedPod.Annotations[appsv1alpha1.LastPodStatusAnnotationKey] = string(podStatusStr) - return nil -} - -func (u *inPlaceIfPossibleUpdater) UpgradePod(_ context.Context, podInfo *PodUpdateInfo) error { - if podInfo.OnlyMetadataChanged || podInfo.InPlaceUpdateSupport { - // if pod template changes only include metadata or support in-place update, just apply these changes to pod directly - if err := u.PodControl.UpdatePod(podInfo.UpdatedPod); err != nil { - return fmt.Errorf("fail to update Pod %s/%s when updating by in-place: %w", podInfo.Namespace, podInfo.Name, err) - } else { - podInfo.Pod = podInfo.UpdatedPod - u.Recorder.Eventf(podInfo.Pod, - corev1.EventTypeNormal, - "UpdatePod", - "succeed to update Pod %s/%s to from revision %s to revision %s by in-place", - podInfo.Namespace, podInfo.Name, - podInfo.CurrentRevision.Name, - podInfo.UpdateRevision.Name) - if err := u.CacheExpectations.ExpectUpdation(kubeutilsclient.ObjectKeyString(u.CollaSet), collasetutils.PodGVK, podInfo.Namespace, podInfo.Name, podInfo.Pod.ResourceVersion); err != nil { - return err - } - } - } else { - // if pod has changes not in-place supported, recreate it - if err := RecreatePod(podInfo, u.PodControl, u.Recorder); err != nil { - return err - } - return u.CacheExpectations.ExpectDeletion(kubeutilsclient.ObjectKeyString(u.CollaSet), collasetutils.PodGVK, podInfo.Namespace, podInfo.Name) - } - return nil -} - -func RecreatePod(podInfo *PodUpdateInfo, podControl podcontrol.Interface, recorder record.EventRecorder) error { - if err := podControl.DeletePod(podInfo.Pod); err != nil { - return fmt.Errorf("fail to delete Pod %s/%s when updating by recreate: %w", podInfo.Namespace, podInfo.Name, err) - } - recorder.Eventf(podInfo.Pod, - corev1.EventTypeNormal, - "UpdatePod", - "succeed to update Pod %s/%s to from revision %s to revision %s by recreate", - podInfo.Namespace, - podInfo.Name, - podInfo.CurrentRevision.Name, - podInfo.UpdateRevision.Name) - return nil -} - -func (u *inPlaceIfPossibleUpdater) diffPod(currentPod, updatedPod *corev1.Pod) (inPlaceSetUpdateSupport, onlyMetadataChanged bool, imageChangedContainers sets.String) { - if len(currentPod.Spec.Containers) != len(updatedPod.Spec.Containers) { - return false, false, nil - } - - currentPod = currentPod.DeepCopy() - // sync metadata - currentPod.ObjectMeta = updatedPod.ObjectMeta - - // sync image - imageChanged := false - imageChangedContainers = sets.String{} - for i := range currentPod.Spec.Containers { - if currentPod.Spec.Containers[i].Image != updatedPod.Spec.Containers[i].Image { - imageChanged = true - imageChangedContainers.Insert(currentPod.Spec.Containers[i].Name) - currentPod.Spec.Containers[i].Image = updatedPod.Spec.Containers[i].Image - } - } - - if !equality.Semantic.DeepEqual(currentPod, updatedPod) { - return false, false, nil - } - - if !imageChanged { - return true, true, nil - } - - return true, false, imageChangedContainers -} - -func (u *inPlaceIfPossibleUpdater) GetPodUpdateFinishStatus(_ context.Context, podUpdateInfo *PodUpdateInfo) (finished bool, msg string, err error) { - if !podUpdateInfo.IsUpdatedRevision || podUpdateInfo.PodDecorationChanged { - return false, "pod is not updated or add on not updated", nil - } - - if podUpdateInfo.Status.ContainerStatuses == nil { - return false, "no container status", nil - } - - if podUpdateInfo.Spec.Containers == nil { - return false, "no container spec", nil - } - - if len(podUpdateInfo.Spec.Containers) != len(podUpdateInfo.Status.ContainerStatuses) { - return false, "container status number does not match", nil - } - - if podUpdateInfo.Annotations == nil { - return false, "no annotations for last container status", nil - } - - podLastState := &PodStatus{} - if lastStateJson, exist := podUpdateInfo.Annotations[appsv1alpha1.LastPodStatusAnnotationKey]; !exist { - return false, "no pod last state annotation", nil - } else if err := json.Unmarshal([]byte(lastStateJson), podLastState); err != nil { - msg := fmt.Sprintf("malformat pod last state annotation [%s]: %s", lastStateJson, err) - return false, msg, fmt.Errorf("malformat pod last state annotation [%s]: %w", lastStateJson, err) - } - - if podLastState.ContainerStates == nil { - return true, "pod is updated with only metadata changed", nil - } - - imageMapping := map[string]string{} - for _, containerSpec := range podUpdateInfo.Spec.Containers { - imageMapping[containerSpec.Name] = containerSpec.Image - } - - imageIdMapping := map[string]string{} - for _, containerStatus := range podUpdateInfo.Status.ContainerStatuses { - imageIdMapping[containerStatus.Name] = containerStatus.ImageID - } - - for containerName, lastContainerState := range podLastState.ContainerStates { - latestImage := lastContainerState.LatestImage - lastImageId := lastContainerState.LastImageID - - if currentImage, exist := imageMapping[containerName]; !exist { - // If no this container image recorded, ignore this container. - continue - } else if currentImage != latestImage { - // If container image in pod spec has changed, ignore this container. - continue - } - - if currentImageId, exist := imageIdMapping[containerName]; !exist { - // If no this container image id recorded, ignore this container. - continue - } else if currentImageId == lastImageId { - // No image id changed means the pod in-place update has not finished by kubelet. - return false, fmt.Sprintf("container has %s not been updated: last image id %s, current image id %s", containerName, lastImageId, currentImageId), nil - } - } - - return true, "", nil -} - -type recreatePodUpdater struct { - GenericPodUpdater -} - -func (u *recreatePodUpdater) FulfillPodUpdatedInfo(_ context.Context, _ *appsv1.ControllerRevision, _ *PodUpdateInfo) error { - return nil -} - -func (u *recreatePodUpdater) UpgradePod(_ context.Context, podInfo *PodUpdateInfo) error { - if err := RecreatePod(podInfo, u.PodControl, u.Recorder); err != nil { - return err - } - return u.CacheExpectations.ExpectDeletion(kubeutilsclient.ObjectKeyString(u.CollaSet), collasetutils.PodGVK, podInfo.Namespace, podInfo.Name) -} - -func (u *recreatePodUpdater) GetPodUpdateFinishStatus(_ context.Context, podInfo *PodUpdateInfo) (finished bool, msg string, err error) { - // Recreate policy always treat Pod as update not finished - return podInfo.IsUpdatedRevision && !podInfo.PodDecorationChanged, "", nil -} - -type replaceUpdatePodUpdater struct { - collaSet *appsv1alpha1.CollaSet - podControl podcontrol.Interface - recorder record.EventRecorder - client.Client - cacheExpectations kubeutilsexpectations.CacheExpectationsInterface -} - -func (u *replaceUpdatePodUpdater) Setup(client client.Client, cls *appsv1alpha1.CollaSet, podControl podcontrol.Interface, _ podcontext.Interface, recorder record.EventRecorder, cacheExpectations kubeutilsexpectations.CacheExpectationsInterface) { - u.Client = client - u.collaSet = cls - u.podControl = podControl - u.recorder = recorder - u.cacheExpectations = cacheExpectations -} - -func (u *replaceUpdatePodUpdater) BeginUpdatePod(ctx context.Context, resources *collasetutils.RelatedResources, podCh chan *PodUpdateInfo) (bool, error) { - succCount, err := controllerutils.SlowStartBatch(len(podCh), controllerutils.SlowStartInitialBatchSize, false, func(int, error) error { - podInfo := <-podCh - if podInfo.replacePairNewPodInfo != nil { - replacePairNewPod := podInfo.replacePairNewPodInfo.Pod - newPodRevision, exist := replacePairNewPod.Labels[appsv1.ControllerRevisionHashLabelKey] - if exist && newPodRevision == podInfo.UpdateRevision.Name { - return nil - } - if _, exist := replacePairNewPod.Labels[appsv1alpha1.PodDeletionIndicationLabelKey]; exist { - return nil - } - - u.recorder.Eventf(podInfo.Pod, - corev1.EventTypeNormal, - "ReplaceUpdatePod", - "label to-delete on new pair pod %s/%s because it is not updated revision, current revision: %s, updated revision: %s", - replacePairNewPod.Namespace, - replacePairNewPod.Name, - newPodRevision, - resources.UpdatedRevision.Name) - patch := client.RawPatch(types.StrategicMergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{"%s":"%d"}}}`, appsv1alpha1.PodDeletionIndicationLabelKey, time.Now().UnixNano()))) - if patchErr := u.Patch(ctx, podInfo.replacePairNewPodInfo.Pod, patch); patchErr != nil { - err := fmt.Errorf("failed to delete replace pair new pod %s/%s %w", - podInfo.replacePairNewPodInfo.Namespace, podInfo.replacePairNewPodInfo.Name, patchErr) - return err - } - } - return nil - }) - - return succCount > 0, err -} - -func (u *replaceUpdatePodUpdater) FilterAllowOpsPods(_ context.Context, candidates []*PodUpdateInfo, _ map[int]*appsv1alpha1.ContextDetail, _ *collasetutils.RelatedResources, podCh chan *PodUpdateInfo) (requeueAfter *time.Duration, err error) { - activePodToUpdate := filterOutPlaceHolderUpdateInfos(candidates) - for i, podInfo := range activePodToUpdate { - if podInfo.IsUpdatedRevision && !podInfo.PodDecorationChanged && !podInfo.PvcTmpHashChanged { - continue - } - - podCh <- activePodToUpdate[i] - } - return nil, err -} - -func (u *replaceUpdatePodUpdater) FulfillPodUpdatedInfo(_ context.Context, _ *appsv1.ControllerRevision, _ *PodUpdateInfo) (err error) { - return -} - -func (u *replaceUpdatePodUpdater) UpgradePod(ctx context.Context, podInfo *PodUpdateInfo) error { - return updateReplaceOriginPod(ctx, u.Client, u.recorder, podInfo, podInfo.replacePairNewPodInfo) -} - -func (u *replaceUpdatePodUpdater) GetPodUpdateFinishStatus(_ context.Context, podUpdateInfo *PodUpdateInfo) (finished bool, msg string, err error) { - replaceNewPodInfo := podUpdateInfo.replacePairNewPodInfo - if replaceNewPodInfo == nil { - return - } - - return isPodUpdatedServiceAvailable(replaceNewPodInfo) -} - -func (u *replaceUpdatePodUpdater) FinishUpdatePod(_ context.Context, podInfo *PodUpdateInfo, finishByCancelUpdate bool) error { - if finishByCancelUpdate { - // cancel replace update by removing to-replace and replace-by-update label from origin pod - if podInfo.isInReplace { - patch := client.RawPatch(types.StrategicMergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{"%s":null, "%s":null}}}`, appsv1alpha1.PodReplaceIndicationLabelKey, appsv1alpha1.PodReplaceByReplaceUpdateLabelKey))) - if err := u.podControl.PatchPod(podInfo.Pod, patch); err != nil { - return fmt.Errorf("failed to delete replace pair origin pod %s/%s %w", podInfo.Namespace, podInfo.Name, err) - } - } - return nil - } - - replacePairNewPodInfo := podInfo.replacePairNewPodInfo - if replacePairNewPodInfo != nil { - if _, exist := podInfo.Labels[appsv1alpha1.PodDeletionIndicationLabelKey]; !exist { - patch := client.RawPatch(types.StrategicMergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{"%s":"%d"}}}`, appsv1alpha1.PodDeletionIndicationLabelKey, time.Now().UnixNano()))) - if err := u.podControl.PatchPod(podInfo.Pod, patch); err != nil { - return fmt.Errorf("failed to delete replace pair origin pod %s/%s %w", podInfo.Namespace, podInfo.replacePairNewPodInfo.Name, err) - } - } - } - return nil -} - -func isPodUpdatedServiceAvailable(podInfo *PodUpdateInfo) (finished bool, msg string, err error) { - if podInfo.PodDecorationChanged { - return false, "add on not updated", nil - } - - if podInfo.Labels == nil { - return false, "no labels on pod", nil - } - if podInfo.isInReplace && podInfo.replacePairNewPodInfo != nil { - return false, "replace origin pod", nil - } - - if _, serviceAvailable := podInfo.Labels[appsv1alpha1.PodServiceAvailableLabel]; serviceAvailable { - return true, "", nil - } - - return false, "pod not service available", nil -} - -func int32Min(l, r int32) int32 { - if l < r { - return l - } - - return r -} diff --git a/pkg/controllers/collaset/utils/collaset.go b/pkg/controllers/collaset/utils/collaset.go index f4bbf50f..ea9fee10 100644 --- a/pkg/controllers/collaset/utils/collaset.go +++ b/pkg/controllers/collaset/utils/collaset.go @@ -16,9 +16,19 @@ limitations under the License. package utils -import appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" +import ( + corev1 "k8s.io/api/core/v1" + appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" +) -func IsPodNamingSuffixPolicyPersistentSequence(cls *appsv1alpha1.CollaSet) bool { +var ( + PodGVK = corev1.SchemeGroupVersion.WithKind("Pod") + PVCGVK = corev1.SchemeGroupVersion.WithKind("PersistentVolumeClaim") + CollaSetGVK = appsv1alpha1.SchemeGroupVersion.WithKind("CollaSet") + ResourceContextGVK = appsv1alpha1.SchemeGroupVersion.WithKind("ResourceContext") +) + +func PodNamingSuffixPersistentSequence(cls *appsv1alpha1.CollaSet) bool { if cls.Spec.NamingStrategy == nil { return false } diff --git a/pkg/controllers/collaset/utils/collaset_test.go b/pkg/controllers/collaset/utils/collaset_test.go index f74473a9..e0c1bc6e 100644 --- a/pkg/controllers/collaset/utils/collaset_test.go +++ b/pkg/controllers/collaset/utils/collaset_test.go @@ -69,8 +69,8 @@ func TestPodNamingSuffixPersistentSequence(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := IsPodNamingSuffixPolicyPersistentSequence(tt.args.cls); got != tt.want { - t.Errorf("IsPodNamingSuffixPolicyPersistentSequence() = %v, want %v", got, tt.want) + if got := PodNamingSuffixPersistentSequence(tt.args.cls); got != tt.want { + t.Errorf("PodNamingSuffixPersistentSequence() = %v, want %v", got, tt.want) } }) } diff --git a/pkg/controllers/collaset/utils/condition.go b/pkg/controllers/collaset/utils/condition.go deleted file mode 100644 index e8583272..00000000 --- a/pkg/controllers/collaset/utils/condition.go +++ /dev/null @@ -1,110 +0,0 @@ -/* -Copyright 2023 The KusionStack Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package utils - -import ( - "sort" - "strings" - "time" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" -) - -const ConditionUpdatePeriodBackOff = 30 * time.Second - -func AddOrUpdateCondition(status *appsv1alpha1.CollaSetStatus, conditionType appsv1alpha1.CollaSetConditionType, err error, reason, message string) { - condStatus := corev1.ConditionTrue - if err != nil { - condStatus = corev1.ConditionFalse - } - - existCond := GetCondition(status, conditionType) - if existCond != nil && existCond.Reason == reason && existCond.Status == condStatus { - now := metav1.Now() - if now.Sub(existCond.LastTransitionTime.Time) < ConditionUpdatePeriodBackOff { - return - } - } - - cond := NewCondition(conditionType, condStatus, reason, message) - SetCondition(status, cond) -} - -func NewCondition(condType appsv1alpha1.CollaSetConditionType, status corev1.ConditionStatus, reason, msg string) *appsv1alpha1.CollaSetCondition { - return &appsv1alpha1.CollaSetCondition{ - Type: condType, - Status: status, - LastTransitionTime: metav1.Now(), - Reason: reason, - Message: msg, - } -} - -// GetCondition returns a inplace set condition with the provided type if it exists. -func GetCondition(status *appsv1alpha1.CollaSetStatus, condType appsv1alpha1.CollaSetConditionType) *appsv1alpha1.CollaSetCondition { - for _, c := range status.Conditions { - if c.Type == condType { - return &c - } - } - return nil -} - -// SetCondition adds/replaces the given condition in the replicaset status. If the condition that we -// are about to add already exists and has the same status and reason then we are not going to update. -func SetCondition(status *appsv1alpha1.CollaSetStatus, condition *appsv1alpha1.CollaSetCondition) { - currentCond := GetCondition(status, condition.Type) - if currentCond != nil && currentCond.Status == condition.Status && currentCond.Reason == condition.Reason && currentCond.LastTransitionTime == condition.LastTransitionTime { - return - } - newConditions := filterOutCondition(status.Conditions, condition.Type) - status.Conditions = append(newConditions, *condition) -} - -// RemoveCondition removes the condition with the provided type from the replicaset status. -func RemoveCondition(status *appsv1alpha1.CollaSetStatus, condType appsv1alpha1.CollaSetConditionType) { - status.Conditions = filterOutCondition(status.Conditions, condType) -} - -// filterOutCondition returns a new slice of replicaset conditions without conditions with the provided type. -func filterOutCondition(conditions []appsv1alpha1.CollaSetCondition, condType appsv1alpha1.CollaSetConditionType) []appsv1alpha1.CollaSetCondition { - var newConditions []appsv1alpha1.CollaSetCondition - for _, c := range conditions { - if c.Type == condType { - continue - } - newConditions = append(newConditions, c) - } - return newConditions -} - -func SortCollaSetConditions(conditions []appsv1alpha1.CollaSetCondition) []appsv1alpha1.CollaSetCondition { - sort.Sort(ByCollaSetType(conditions)) - return conditions -} - -type CollaSetConditionType string - -type ByCollaSetType []appsv1alpha1.CollaSetCondition - -func (a ByCollaSetType) Len() int { return len(a) } -func (a ByCollaSetType) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a ByCollaSetType) Less(i, j int) bool { - return strings.Compare(string(a[i].Type), string(a[j].Type)) < 0 -} diff --git a/pkg/controllers/collaset/utils/inexclude.go b/pkg/controllers/collaset/utils/inexclude.go deleted file mode 100644 index e0f20564..00000000 --- a/pkg/controllers/collaset/utils/inexclude.go +++ /dev/null @@ -1,64 +0,0 @@ -/* -Copyright 2024 The KusionStack Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package utils - -import ( - "fmt" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" -) - -// AllowResourceExclude checks if pod or pvc is allowed to exclude -func AllowResourceExclude(obj metav1.Object, ownerName, ownerKind string) (bool, string) { - labels := obj.GetLabels() - // not controlled by ks manager - if labels == nil { - return false, "object's label is empty" - } else if val, exist := labels[appsv1alpha1.ControlledByKusionStackLabelKey]; !exist || val != "true" { - return false, "object is not controlled by kusionstack system" - } - - // not controlled by current collaset - if controller := metav1.GetControllerOf(obj); controller == nil || controller.Name != ownerName || controller.Kind != ownerKind { - return false, "object is not owned by any one, not allowed to exclude" - } - return true, "" -} - -// AllowResourceInclude checks if pod or pvc is allowed to include -func AllowResourceInclude(obj metav1.Object, ownerName, ownerKind string) (bool, string) { - labels := obj.GetLabels() - ownerRefs := obj.GetOwnerReferences() - - // not controlled by ks manager - if labels == nil { - return false, "object's label is empty" - } else if val, exist := labels[appsv1alpha1.ControlledByKusionStackLabelKey]; !exist || val != "true" { - return false, "object is not controlled by kusionstack system" - } - - if ownerRefs != nil { - if controller := metav1.GetControllerOf(obj); controller != nil { - // controlled by others - if controller.Name != ownerName || controller.Kind != ownerKind { - return false, fmt.Sprintf("object's ownerReference controller is not %s/%s", ownerKind, ownerName) - } - } - } - return true, "" -} diff --git a/pkg/controllers/collaset/utils/inexclude_test.go b/pkg/controllers/collaset/utils/inexclude_test.go deleted file mode 100644 index e033def1..00000000 --- a/pkg/controllers/collaset/utils/inexclude_test.go +++ /dev/null @@ -1,302 +0,0 @@ -/* -Copyright 2024 The KusionStack Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package utils - -import ( - "testing" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/pointer" - appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" -) - -var ( - ownerName = "test" - ownerKind = "CollaSet" -) - -func TestAllowResourceExclude(t *testing.T) { - tests := []struct { - name string - obj *corev1.Pod - allow bool - reason string - }{ - { - name: "label is nil", - obj: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: nil, - }, - }, - allow: false, - reason: "object's label is empty", - }, - { - name: "KusionStack control label not satisfied", - obj: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - appsv1alpha1.ControlledByKusionStackLabelKey: "false", - }, - }, - }, - allow: false, - reason: "object is not controlled by kusionstack system", - }, - { - name: "controller is nil", - obj: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - appsv1alpha1.ControlledByKusionStackLabelKey: "true", - }, - OwnerReferences: make([]metav1.OwnerReference, 0), - }, - }, - allow: false, - reason: "object is not owned by any one, not allowed to exclude", - }, - { - name: "controller name not equals to ownerName", - obj: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - appsv1alpha1.ControlledByKusionStackLabelKey: "true", - }, - OwnerReferences: []metav1.OwnerReference{ - { - Name: "test1", - Kind: ownerKind, - Controller: pointer.Bool(true), - }, - }, - }, - }, - allow: false, - reason: "object is not owned by any one, not allowed to exclude", - }, - { - name: "controller kind not equals to ownerName", - obj: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - appsv1alpha1.ControlledByKusionStackLabelKey: "true", - }, - OwnerReferences: []metav1.OwnerReference{ - { - Name: ownerName, - Kind: "kind2", - Controller: pointer.Bool(true), - }, - }, - }, - }, - allow: false, - reason: "object is not owned by any one, not allowed to exclude", - }, - { - name: "allowed case", - obj: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - appsv1alpha1.ControlledByKusionStackLabelKey: "true", - }, - OwnerReferences: []metav1.OwnerReference{ - { - Name: ownerName, - Kind: ownerKind, - Controller: pointer.Bool(true), - }, - }, - }, - }, - allow: true, - reason: "", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, got1 := AllowResourceExclude(tt.obj, ownerName, ownerKind) - if got != tt.allow { - t.Errorf("AllowResourceExclude() got = %v, want %v", got, tt.allow) - } - if got1 != tt.reason { - t.Errorf("AllowResourceExclude() got1 = %v, want %v", got1, tt.reason) - } - }) - } -} - -func TestAllowResourceInclude(t *testing.T) { - tests := []struct { - name string - obj *corev1.Pod - allow bool - reason string - }{ - { - name: "label is nil", - obj: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: nil, - }, - }, - allow: false, - reason: "object's label is empty", - }, - { - name: "KusionStack control label not satisfied", - obj: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - appsv1alpha1.ControlledByKusionStackLabelKey: "false", - }, - }, - }, - allow: false, - reason: "object is not controlled by kusionstack system", - }, - { - name: "controller is nil", - obj: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - appsv1alpha1.ControlledByKusionStackLabelKey: "true", - }, - OwnerReferences: make([]metav1.OwnerReference, 0), - }, - }, - allow: true, - reason: "", - }, - { - name: "controller name not equals to ownerName", - obj: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - appsv1alpha1.ControlledByKusionStackLabelKey: "true", - }, - OwnerReferences: []metav1.OwnerReference{ - { - Name: "test1", - Kind: ownerKind, - Controller: pointer.Bool(true), - }, - }, - }, - }, - allow: false, - reason: "object's ownerReference controller is not CollaSet/test", - }, - { - name: "controller kind not equals to ownerName", - obj: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - appsv1alpha1.ControlledByKusionStackLabelKey: "true", - }, - OwnerReferences: []metav1.OwnerReference{ - { - Name: "test", - Kind: "kind2", - Controller: pointer.Bool(true), - }, - }, - }, - }, - allow: false, - reason: "object's ownerReference controller is not CollaSet/test", - }, - { - name: "controller kind not equals to ownerName", - obj: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - appsv1alpha1.ControlledByKusionStackLabelKey: "true", - }, - OwnerReferences: []metav1.OwnerReference{ - { - Name: ownerName, - Kind: ownerKind, - Controller: pointer.Bool(true), - }, - }, - }, - }, - allow: true, - reason: "", - }, - { - name: "allowed case1", - obj: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - appsv1alpha1.ControlledByKusionStackLabelKey: "true", - appsv1alpha1.PodOrphanedIndicateLabelKey: "true", - }, - OwnerReferences: []metav1.OwnerReference{ - { - Name: ownerName, - Kind: ownerKind, - Controller: pointer.Bool(true), - }, - }, - }, - }, - allow: true, - reason: "", - }, - { - name: "allowed case2", - obj: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - appsv1alpha1.ControlledByKusionStackLabelKey: "true", - }, - }, - }, - allow: true, - reason: "", - }, - { - name: "allowed case3", - obj: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - appsv1alpha1.ControlledByKusionStackLabelKey: "true", - appsv1alpha1.PodOrphanedIndicateLabelKey: "true", - }, - }, - }, - allow: true, - reason: "", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, got1 := AllowResourceInclude(tt.obj, ownerName, ownerKind) - if got != tt.allow { - t.Errorf("AllowResourceExclude() got = %v, want %v", got, tt.allow) - } - if got1 != tt.reason { - t.Errorf("AllowResourceExclude() got1 = %v, want %v", got1, tt.reason) - } - }) - } -} diff --git a/pkg/controllers/collaset/utils/pod.go b/pkg/controllers/collaset/utils/pod.go index b9c8a240..d799a0af 100644 --- a/pkg/controllers/collaset/utils/pod.go +++ b/pkg/controllers/collaset/utils/pod.go @@ -1,5 +1,4 @@ /* -Copyright 2014 The Kubernetes Authors. Copyright 2023 The KusionStack Authors. Licensed under the Apache License, Version 2.0 (the "License"); @@ -30,6 +29,7 @@ import ( "k8s.io/apimachinery/pkg/util/strategicpatch" "k8s.io/client-go/kubernetes/scheme" appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/client" controllerutils "kusionstack.io/kuperator/pkg/controllers/utils" "kusionstack.io/kuperator/pkg/utils" @@ -86,7 +86,7 @@ func NewPodFrom(owner metav1.Object, ownerRef *metav1.OwnerReference, revision * pod.OwnerReferences = append(pod.OwnerReferences, *ownerRef) cls, _ := owner.(*appsv1alpha1.CollaSet) - if IsPodNamingSuffixPolicyPersistentSequence(cls) { + if PodNamingSuffixPersistentSequence(cls) { pod.Name = pod.GenerateName + instanceId } @@ -133,7 +133,10 @@ func ApplyPatchFromRevision(pod *corev1.Pod, revision *appsv1.ControllerRevision } // PatchToPod Use three-way merge to get a updated pod. -func PatchToPod(currentRevisionPod, updateRevisionPod, currentPod *corev1.Pod) (*corev1.Pod, error) { +func PatchToPod(currentRevisionObj, updateRevisionObj, currentObj client.Object) (client.Object, error) { + currentRevisionPod := currentRevisionObj.(*corev1.Pod) + updateRevisionPod := updateRevisionObj.(*corev1.Pod) + currentPod := currentObj.(*corev1.Pod) currentRevisionPodBytes, err := json.Marshal(currentRevisionPod) if err != nil { return nil, err @@ -203,8 +206,8 @@ func ComparePod(l, r *corev1.Pod) bool { // see https://github.com/kubernetes/kubernetes/issues/22065 // 4. Been ready for empty time < less time < more time // If both pods are ready, the latest ready one is smaller - if controllerutils.IsPodReady(l) && controllerutils.IsPodReady(r) && !podReadyTime(l).Equal(podReadyTime(r)) { - return afterOrZero(podReadyTime(l), podReadyTime(r)) + if controllerutils.IsPodReady(l) && controllerutils.IsPodReady(r) && !PodReadyTime(l).Equal(PodReadyTime(r)) { + return afterOrZero(PodReadyTime(l), PodReadyTime(r)) } // 5. Pods with containers with higher restart counts < lower restart counts if maxContainerRestarts(l) != maxContainerRestarts(r) { @@ -227,7 +230,7 @@ func maxContainerRestarts(pod *corev1.Pod) int { return int(maxRestarts) } -func podReadyTime(pod *corev1.Pod) *metav1.Time { +func PodReadyTime(pod *corev1.Pod) *metav1.Time { if controllerutils.IsPodReady(pod) { for _, c := range pod.Status.Conditions { // we only care about pod ready conditions @@ -248,10 +251,6 @@ func afterOrZero(t1, t2 *metav1.Time) bool { return t1.After(t2.Time) } -func IsPodUpdatedRevision(pod *corev1.Pod, revision string) bool { - if pod.Labels == nil { - return false - } - - return pod.Labels[appsv1.ControllerRevisionHashLabelKey] == revision +func IsPodInactive(pod *corev1.Pod) bool { + return pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed } diff --git a/pkg/controllers/collaset/utils/pod_test.go b/pkg/controllers/collaset/utils/pod_test.go index 832568bc..bca93d35 100644 --- a/pkg/controllers/collaset/utils/pod_test.go +++ b/pkg/controllers/collaset/utils/pod_test.go @@ -113,7 +113,7 @@ var _ = Describe("Pod utils", func() { } pod, err := PatchToPod(currentRevisionPod, updateRevisionPod, currentPod) Expect(err).Should(BeNil()) - Expect(pod.Labels["foo"]).Should(Equal("bar-1")) + Expect(pod.GetLabels()["foo"]).Should(Equal("bar-1")) }) It("test ComparePod", func() { pods := []*corev1.Pod{ diff --git a/pkg/controllers/collaset/utils/pvc.go b/pkg/controllers/collaset/utils/pvc.go index 4ab1d90c..04d42558 100644 --- a/pkg/controllers/collaset/utils/pvc.go +++ b/pkg/controllers/collaset/utils/pvc.go @@ -1,11 +1,11 @@ /* -Copyright 2024 The KusionStack Authors. +Copyright 2025 The KusionStack Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/pkg/controllers/collaset/utils/pvc_test.go b/pkg/controllers/collaset/utils/pvc_test.go index 3d8d2cec..6bcdeb86 100644 --- a/pkg/controllers/collaset/utils/pvc_test.go +++ b/pkg/controllers/collaset/utils/pvc_test.go @@ -1,5 +1,5 @@ /* -Copyright 2024 The KusionStack Authors. +Copyright 2025 The KusionStack Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/controllers/collaset/utils/resource.go b/pkg/controllers/collaset/utils/resource.go deleted file mode 100644 index d8a98f1f..00000000 --- a/pkg/controllers/collaset/utils/resource.go +++ /dev/null @@ -1,39 +0,0 @@ -/* -Copyright 2023 The KusionStack Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package utils - -import ( - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" - - utilspoddecoration "kusionstack.io/kuperator/pkg/controllers/utils/poddecoration" -) - -type RelatedResources struct { - Revisions []*appsv1.ControllerRevision - CurrentRevision *appsv1.ControllerRevision - UpdatedRevision *appsv1.ControllerRevision - ExistingPvcs []*corev1.PersistentVolumeClaim - - PDGetter utilspoddecoration.Getter - - FilteredPods []*corev1.Pod - CurrentIDs map[int]struct{} - - NewStatus *appsv1alpha1.CollaSetStatus -} diff --git a/pkg/controllers/collaset/utils/utils_test.go b/pkg/controllers/collaset/utils/utils_test.go deleted file mode 100644 index 4b1045c8..00000000 --- a/pkg/controllers/collaset/utils/utils_test.go +++ /dev/null @@ -1,49 +0,0 @@ -/* -Copyright 2023 The KusionStack Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package utils - -import ( - "fmt" - "testing" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - corev1 "k8s.io/api/core/v1" - appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" -) - -func TestUtils(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "CollaSets utils tests") -} - -var _ = Describe("Condition tests", func() { - It("test AddOrUpdateCondition", func() { - status := &appsv1alpha1.CollaSetStatus{ - Conditions: []appsv1alpha1.CollaSetCondition{ - { - Type: appsv1alpha1.CollaSetScale, - Status: corev1.ConditionTrue, - }, - }, - } - AddOrUpdateCondition(status, appsv1alpha1.CollaSetScale, fmt.Errorf("test err"), "", "") - AddOrUpdateCondition(status, appsv1alpha1.CollaSetUpdate, nil, "", "") - Expect(len(status.Conditions)).Should(Equal(2)) - }) -}) diff --git a/pkg/controllers/collaset/well_known_manager.go b/pkg/controllers/collaset/well_known_manager.go new file mode 100644 index 00000000..54e9d3bc --- /dev/null +++ b/pkg/controllers/collaset/well_known_manager.go @@ -0,0 +1,47 @@ +/* +Copyright 2025 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package collaset + +import ( + appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" + "kusionstack.io/kube-xset/api" +) + +var defaultXSetControllerLabelManager = map[api.XSetLabelAnnotationEnum]string{ + api.OperatingLabelPrefix: appsv1alpha1.PodOperatingLabelPrefix, + api.OperationTypeLabelPrefix: appsv1alpha1.PodOperationTypeLabelPrefix, + api.OperateLabelPrefix: appsv1alpha1.PodOperateLabelPrefix, + api.UndoOperationTypeLabelPrefix: appsv1alpha1.PodUndoOperationTypeLabelPrefix, + api.ServiceAvailableLabel: appsv1alpha1.PodServiceAvailableLabel, + api.PreparingDeleteLabel: appsv1alpha1.PodPreparingDeleteLabel, + + api.ControlledByXSetLabel: appsv1alpha1.ControlledByKusionStackLabelKey, + api.XInstanceIdLabelKey: appsv1alpha1.PodInstanceIDLabelKey, + api.XSetUpdateIndicationLabelKey: appsv1alpha1.CollaSetUpdateIndicateLabelKey, + api.XDeletionIndicationLabelKey: appsv1alpha1.PodDeletionIndicationLabelKey, + api.XReplaceIndicationLabelKey: appsv1alpha1.PodReplaceIndicationLabelKey, + api.XReplacePairNewId: appsv1alpha1.PodReplacePairNewId, + api.XReplacePairOriginName: appsv1alpha1.PodReplacePairOriginName, + api.XReplaceByReplaceUpdateLabelKey: appsv1alpha1.PodReplaceByReplaceUpdateLabelKey, + api.XOrphanedIndicationLabelKey: appsv1alpha1.PodOrphanedIndicateLabelKey, + api.XCreatingLabel: appsv1alpha1.PodCreatingLabel, + api.XCompletingLabel: appsv1alpha1.PodCompletingLabel, + api.XExcludeIndicationLabelKey: appsv1alpha1.PodExcludeIndicationLabelKey, + + api.SubResourcePvcTemplateLabelKey: appsv1alpha1.PvcTemplateLabelKey, + api.SubResourcePvcTemplateHashLabelKey: appsv1alpha1.PvcTemplateHashLabelKey, +} diff --git a/pkg/controllers/collaset/well_known_manager_test.go b/pkg/controllers/collaset/well_known_manager_test.go new file mode 100644 index 00000000..0b9822b5 --- /dev/null +++ b/pkg/controllers/collaset/well_known_manager_test.go @@ -0,0 +1,192 @@ +/* +Copyright 2025 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package collaset + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" + "kusionstack.io/kube-xset/api" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func Test_xSetControllerLabelManager_Get(t *testing.T) { + type fields struct { + labelManager map[api.XSetLabelAnnotationEnum]string + } + type args struct { + pod *corev1.Pod + key api.XSetLabelAnnotationEnum + } + tests := []struct { + name string + fields fields + args args + want string + want1 bool + }{ + { + name: "get label ok", + fields: fields{ + labelManager: defaultXSetControllerLabelManager, + }, + args: args{ + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + appsv1alpha1.PodInstanceIDLabelKey: "0", + }, + }, + }, + key: api.XInstanceIdLabelKey, + }, + want: "0", + want1: true, + }, + { + name: "label is nil", + fields: fields{ + labelManager: defaultXSetControllerLabelManager, + }, + args: args{ + pod: &corev1.Pod{}, + key: api.XInstanceIdLabelKey, + }, + want: "", + want1: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := api.NewXSetLabelAnnotationManager(defaultXSetControllerLabelManager) + got, got1 := m.Get(tt.args.pod, tt.args.key) + if got != tt.want { + t.Errorf("Get() got = %v, want %v", got, tt.want) + } + if got1 != tt.want1 { + t.Errorf("Get() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} + +func Test_xSetControllerLabelManager_Set(t *testing.T) { + type fields struct { + labelManager map[api.XSetLabelAnnotationEnum]string + } + type args struct { + obj client.Object + key api.XSetLabelAnnotationEnum + val string + } + tests := []struct { + name string + fields fields + args args + }{ + { + name: "object label is nil", + fields: fields{ + labelManager: defaultXSetControllerLabelManager, + }, + args: args{ + obj: &corev1.Pod{}, + key: api.XInstanceIdLabelKey, + val: "0", + }, + }, + { + name: "set label ok", + fields: fields{ + labelManager: defaultXSetControllerLabelManager, + }, + args: args{ + obj: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + appsv1alpha1.PodInstanceIDLabelKey: "0", + }, + }, + }, + key: api.XInstanceIdLabelKey, + val: "0", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := api.NewXSetLabelAnnotationManager(defaultXSetControllerLabelManager) + m.Set(tt.args.obj, tt.args.key, tt.args.val) + if val, ok := m.Get(tt.args.obj, tt.args.key); !ok || val != tt.args.val { + t.Errorf("Set() = %v, want %v", val, tt.args.val) + } + }) + } +} + +func Test_xSetControllerLabelManager_Delete(t *testing.T) { + type fields struct { + labelManager map[api.XSetLabelAnnotationEnum]string + } + type args struct { + pod *corev1.Pod + key api.XSetLabelAnnotationEnum + } + tests := []struct { + name string + fields fields + args args + }{ + { + name: "delete label ok", + fields: fields{ + labelManager: defaultXSetControllerLabelManager, + }, + args: args{ + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + appsv1alpha1.PodInstanceIDLabelKey: "0", + }, + }, + }, + key: api.XInstanceIdLabelKey, + }, + }, + { + name: "delete label is nil", + fields: fields{ + labelManager: defaultXSetControllerLabelManager, + }, + args: args{ + pod: &corev1.Pod{}, + key: api.XInstanceIdLabelKey, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := api.NewXSetLabelAnnotationManager(defaultXSetControllerLabelManager) + m.Delete(tt.args.pod, tt.args.key) + if val, ok := m.Get(tt.args.pod, tt.args.key); ok || val != "" { + t.Errorf("Delete() = %v, want %v", val, "") + } + }) + } +} diff --git a/pkg/controllers/operationjob/operationjob_controller_test.go b/pkg/controllers/operationjob/operationjob_controller_test.go index a2f6af54..88f75166 100644 --- a/pkg/controllers/operationjob/operationjob_controller_test.go +++ b/pkg/controllers/operationjob/operationjob_controller_test.go @@ -806,8 +806,7 @@ var _ = BeforeSuite(func() { err = AddToMgr(mgr, r) Expect(err).NotTo(HaveOccurred()) // collaset controller - r, request = testReconcile(collaset.NewReconciler(mgr)) - err = collaset.AddToMgr(mgr, r) + err = collaset.Add(mgr) Expect(err).NotTo(HaveOccurred()) // poddeletion controller r, request = testReconcile(poddeletion.NewReconciler(mgr)) diff --git a/pkg/controllers/poddecoration/poddecoration_controller.go b/pkg/controllers/poddecoration/poddecoration_controller.go index b1222288..1cbd7961 100644 --- a/pkg/controllers/poddecoration/poddecoration_controller.go +++ b/pkg/controllers/poddecoration/poddecoration_controller.go @@ -41,7 +41,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - "kusionstack.io/kuperator/pkg/controllers/collaset/podcontrol" + collasetutils "kusionstack.io/kuperator/pkg/controllers/collaset/utils" controllerutils "kusionstack.io/kuperator/pkg/controllers/utils" "kusionstack.io/kuperator/pkg/controllers/utils/expectations" utilspoddecoration "kusionstack.io/kuperator/pkg/controllers/utils/poddecoration/anno" @@ -170,7 +170,7 @@ func (r *ReconcilePodDecoration) Reconcile(ctx context.Context, request reconcil return reconcile.Result{}, err } for i := range podList.Items { - if podcontrol.IsPodInactive(&podList.Items[i]) || podList.Items[i].DeletionTimestamp != nil { + if collasetutils.IsPodInactive(&podList.Items[i]) || podList.Items[i].DeletionTimestamp != nil { continue } selectedPods = append(selectedPods, &podList.Items[i]) diff --git a/pkg/controllers/poddecoration/poddecoration_controller_test.go b/pkg/controllers/poddecoration/poddecoration_controller_test.go index a33c675a..b5f5e6a0 100644 --- a/pkg/controllers/poddecoration/poddecoration_controller_test.go +++ b/pkg/controllers/poddecoration/poddecoration_controller_test.go @@ -43,7 +43,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "kusionstack.io/kuperator/pkg/controllers/collaset" - collasetutils "kusionstack.io/kuperator/pkg/controllers/collaset/utils" + collasetutils "kusionstack.io/kuperator/pkg/controllers/collaset/legacy" utilspoddecoration "kusionstack.io/kuperator/pkg/controllers/utils/poddecoration/anno" "kusionstack.io/kuperator/pkg/controllers/utils/poddecoration/strategy" "kusionstack.io/kuperator/pkg/controllers/utils/podopslifecycle" diff --git a/pkg/controllers/poddeletion/poddeletion_controller.go b/pkg/controllers/poddeletion/poddeletion_controller.go index ce6fc247..3a98cc41 100644 --- a/pkg/controllers/poddeletion/poddeletion_controller.go +++ b/pkg/controllers/poddeletion/poddeletion_controller.go @@ -34,7 +34,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - "kusionstack.io/kuperator/pkg/controllers/collaset/pvccontrol" + pvccontrol "kusionstack.io/kuperator/pkg/controllers/collaset/legacy" "kusionstack.io/kuperator/pkg/controllers/utils/podopslifecycle" "kusionstack.io/kuperator/pkg/utils/mixin" ) diff --git a/pkg/controllers/resourcecontext/resourcecontext_controller_test.go b/pkg/controllers/resourcecontext/resourcecontext_controller_test.go index b79f1962..147a6b6b 100644 --- a/pkg/controllers/resourcecontext/resourcecontext_controller_test.go +++ b/pkg/controllers/resourcecontext/resourcecontext_controller_test.go @@ -44,7 +44,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "kusionstack.io/kuperator/pkg/controllers/collaset" - collasetutils "kusionstack.io/kuperator/pkg/controllers/collaset/utils" + collasetutils "kusionstack.io/kuperator/pkg/controllers/collaset/legacy" "kusionstack.io/kuperator/pkg/controllers/poddeletion" "kusionstack.io/kuperator/pkg/controllers/utils/poddecoration/strategy" "kusionstack.io/kuperator/pkg/utils/inject" @@ -350,8 +350,7 @@ var _ = BeforeSuite(func() { err = AddToMgr(mgr, r) Expect(err).NotTo(HaveOccurred()) - r, request = testReconcile(collaset.NewReconciler(mgr)) - err = collaset.AddToMgr(mgr, r) + err = collaset.Add(mgr) Expect(err).NotTo(HaveOccurred()) r, request = testReconcile(poddeletion.NewReconciler(mgr)) diff --git a/pkg/features/features.go b/pkg/features/features.go index a20173f2..0d277132 100644 --- a/pkg/features/features.go +++ b/pkg/features/features.go @@ -28,14 +28,11 @@ const ( AlibabaCloudSlb featuregate.Feature = "AlibabaCloudSlb" // GraceDeleteWebhook enables the gracedelete webhook GraceDeleteWebhook featuregate.Feature = "GraceDeleteWebhook" - // ReclaimPodScaleStrategy enables reclaim of collaset.spec.scaleStrategy.podToDelete - ReclaimPodScaleStrategy featuregate.Feature = "ReclaimPodScaleStrategy" ) var defaultFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ - AlibabaCloudSlb: {Default: false, PreRelease: featuregate.Alpha}, - GraceDeleteWebhook: {Default: false, PreRelease: featuregate.Alpha}, - ReclaimPodScaleStrategy: {Default: false, PreRelease: featuregate.Alpha}, + AlibabaCloudSlb: {Default: false, PreRelease: featuregate.Alpha}, + GraceDeleteWebhook: {Default: false, PreRelease: featuregate.Alpha}, } func init() { diff --git a/pkg/utils/inject/inject.go b/pkg/utils/inject/inject.go index f56b1b38..6f96b533 100644 --- a/pkg/utils/inject/inject.go +++ b/pkg/utils/inject/inject.go @@ -19,9 +19,6 @@ package inject import ( "context" - appv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/rest" appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" @@ -48,42 +45,6 @@ func NewCacheWithFieldIndex(config *rest.Config, opts cache.Options) (cache.Cach // }, //} - runtime.Must(c.IndexField( - context.TODO(), - &corev1.Pod{}, - FieldIndexOwnerRefUID, - func(pod client.Object) []string { - ownerRef := metav1.GetControllerOf(pod) - if ownerRef == nil { - return nil - } - return []string{string(ownerRef.UID)} - })) - - runtime.Must(c.IndexField( - context.TODO(), - &corev1.PersistentVolumeClaim{}, - FieldIndexOwnerRefUID, - func(pvc client.Object) []string { - ownerRef := metav1.GetControllerOf(pvc) - if ownerRef == nil { - return nil - } - return []string{string(ownerRef.UID)} - })) - - runtime.Must(c.IndexField( - context.TODO(), - &appv1.ControllerRevision{}, - FieldIndexOwnerRefUID, - func(revision client.Object) []string { - ownerRef := metav1.GetControllerOf(revision) - if ownerRef == nil { - return nil - } - return []string{string(ownerRef.UID)} - })) - runtime.Must(c.IndexField( context.TODO(), &appsv1alpha1.PodTransitionRule{}, diff --git a/pkg/webhook/server/generic/resourcecontext/resourcecontext_validating_handler.go b/pkg/webhook/server/generic/resourcecontext/resourcecontext_validating_handler.go index 02008bf9..5aa4efda 100644 --- a/pkg/webhook/server/generic/resourcecontext/resourcecontext_validating_handler.go +++ b/pkg/webhook/server/generic/resourcecontext/resourcecontext_validating_handler.go @@ -27,7 +27,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/runtime/inject" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - "kusionstack.io/kuperator/pkg/controllers/collaset/podcontext" + podcontext "kusionstack.io/kuperator/pkg/controllers/collaset/legacy" "kusionstack.io/kuperator/pkg/utils/mixin" ) diff --git a/test/e2e/apps/collaset.go b/test/e2e/apps/collaset.go index 829fe472..21e4775f 100644 --- a/test/e2e/apps/collaset.go +++ b/test/e2e/apps/collaset.go @@ -39,8 +39,7 @@ import ( appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" "sigs.k8s.io/controller-runtime/pkg/client" - "kusionstack.io/kuperator/pkg/controllers/collaset/podcontext" - collasetutils "kusionstack.io/kuperator/pkg/controllers/collaset/utils" + "kusionstack.io/kuperator/pkg/controllers/collaset/legacy" "kusionstack.io/kuperator/pkg/controllers/utils/podopslifecycle" "kusionstack.io/kuperator/pkg/utils" "kusionstack.io/kuperator/test/e2e/framework" @@ -1242,7 +1241,7 @@ var _ = SIGDescribe("CollaSet", func() { var duration time.Duration Eventually(func() bool { Expect(client.Get(context.Background(), types.NamespacedName{Namespace: podToUpdate.Namespace, Name: podToUpdate.Name}, podToUpdate)).Should(BeNil()) - labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.UpdateOpsLifecycleAdapter.GetID()) + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, legacy.UpdateOpsLifecycleAdapter.GetID()) if startedTimestampStr, started := podToUpdate.Labels[labelOperate]; started { startedTimestamp, _ := strconv.ParseInt(startedTimestampStr, 10, 64) startTime = time.Unix(0, startedTimestamp) @@ -1456,7 +1455,7 @@ var _ = SIGDescribe("CollaSet", func() { Expect(err).NotTo(HaveOccurred()) duringOpsCount := 0 for i := range pods { - if podopslifecycle.IsDuringOps(collasetutils.UpdateOpsLifecycleAdapter, pods[i]) { + if podopslifecycle.IsDuringOps(legacy.UpdateOpsLifecycleAdapter, pods[i]) { duringOpsCount++ } } @@ -1524,7 +1523,7 @@ var _ = SIGDescribe("CollaSet", func() { Expect(err).NotTo(HaveOccurred()) _, exist := pods[0].Labels[fmt.Sprintf("%s/%s", appsv1alpha1.PodOperatedLabelPrefix, "collaset")] return exist - }, 10*time.Second, 3*time.Second).Should(Equal(true)) + }, 30*time.Second, 3*time.Second).Should(Equal(true)) By("Record lifecycle label values") pods, err = tester.ListPodsForCollaSet(cls) @@ -1598,7 +1597,7 @@ var _ = SIGDescribe("CollaSet", func() { Expect(err).NotTo(HaveOccurred()) Expect(len(currResourceContexts[0].Spec.Contexts)).Should(BeEquivalentTo(3)) for i := range currResourceContexts[0].Spec.Contexts { - currResourceContexts[0].Spec.Contexts[i].Data[podcontext.OwnerContextKey] = "mock-collaset" + currResourceContexts[0].Spec.Contexts[i].Data[legacy.OwnerContextKey] = "mock-collaset" } By("Update image to nginxNew but pods are not updated") @@ -1631,7 +1630,7 @@ var _ = SIGDescribe("CollaSet", func() { Expect(err).NotTo(HaveOccurred()) Expect(len(currResourceContexts[0].Spec.Contexts)).Should(BeEquivalentTo(3)) for i := range currResourceContexts[0].Spec.Contexts { - Expect(currResourceContexts[0].Spec.Contexts[i].Data[podcontext.OwnerContextKey]).Should(BeEquivalentTo(cls.Name)) + Expect(currResourceContexts[0].Spec.Contexts[i].Data[legacy.OwnerContextKey]).Should(BeEquivalentTo(cls.Name)) } }) })