Merge remote-tracking branch 'upstream/main' into feature_rbd_encryption

Conflicts:
- LibvirtStorageAdaptor.java: #13763 added VIR_STORAGE_VOL_DELETE_WITH_SNAPSHOTS
  next to LUKS2_HEADER_RESERVE_BYTES; kept both constants.
- PendingReleaseNotes: kept the UEFI NVRAM entry from main and moved the
  encrypted RBD entry under 4.23.0.0 > 24.0.0, since 4.23.0.0 has been
  released without it and main is now 24.0.0-SNAPSHOT.

Author: calvix

.asf.yaml

@@ -58,6 +58,7 @@ github:
     - GaOrtiga
     - bhouse-nexthop
     - Dogface2k
+    - prashanthr2
 
   rulesets:
     - name: "Default Branch Protection"
@@ -73,7 +74,7 @@ github:
 
   copilot_code_review:
     enabled: true
-    review_drafts: true
+    review_drafts: false
     review_on_push: true
 
 notifications:

.codespellrc

@@ -17,4 +17,4 @@
 
 [codespell]
 ignore-words = .github/linters/codespell.txt
-skip = systemvm/agent/noVNC/*,ui/package.json,ui/package-lock.json,ui/public/js/less.min.js,ui/public/locales/*.json,server/src/test/java/org/apache/cloudstack/network/ssl/CertServiceTest.java,test/integration/smoke/test_ssl_offloading.py
+skip = systemvm/agent/noVNC/*,ui/package.json,ui/package-lock.json,ui/public/js/less.min.js,ui/public/locales/*.json,server/src/test/java/org/apache/cloudstack/network/ssl/CertServiceTest.java,test/integration/smoke/test_ssl_offloading.py,tools/apidoc-versions-diff/*

.github/dependabot.yml

@@ -41,3 +41,14 @@ updates:
       interval: "daily"
     cooldown:
       default-days: 7
+  - package-ecosystem: "pre-commit"
+    directory: "/"
+    open-pull-requests-limit: 2
+    schedule:
+      interval: "weekly"
+    groups:
+      pre-commit-hooks:
+        patterns:
+          - "*"
+    cooldown:
+      default-days: 7

.github/linters/codespell.txt

@@ -184,6 +184,8 @@ environmnet
 equivalant
 erro
 erronous
+errorprone
+everthing
 everytime
 excute
 execept

.github/workflows/docker-cloudstack-simulator.yml

@@ -22,8 +22,7 @@ on:
     branches:
       - main
     tags:
-      - 4.*
-      - 5.*
+      - '[0-9]+.[0-9]+.*'
 
 permissions:
   contents: read

.github/workflows/rat.yml

@@ -41,7 +41,7 @@ jobs:
         uses: ./.github/actions/install-nonoss
       - name: RAT licence checks
         run: |
-          mvn -P developer,systemvm -Dsimulator -Dnoredist -pl . org.apache.rat:apache-rat-plugin:0.12:check
+          mvn -P developer,systemvm -Dsimulator -Dnoredist -pl . org.apache.rat:apache-rat-plugin:0.18:check
       - name: Rat Report
         if: always()
         run: |

.pre-commit-config.yaml

@@ -26,26 +26,37 @@ repos:
       - id: identity
       - id: check-hooks-apply
   - repo: https://github.com/thlorenz/doctoc.git
-    rev: v2.3.0
+    rev: 59dcae910a6ad4bbe8711275b854e3cd5c30f08b  # frozen: v2.5.0
     hooks:
       - id: doctoc
         name: Add TOC for Markdown files
         files: ^CONTRIBUTING\.md$|^INSTALL\.md$|^README\.md$
+  - repo: local
+    hooks:
+      - id: check-zip-file-is-not-committed
+        name: check no zip files are committed
+        description: Zip files are not allowed in the repository
+        language: fail
+        entry: |
+          Zip files are not allowed in the repository as they are hard to
+          track and have security implications. Please remove the zip file from the repository.
+        files: (?i)\.zip$
+        exclude: ^core/src/test/resources/vhds/test\.vhd\.zip$
   - repo: https://github.com/oxipng/oxipng
-    rev: v10.1.0
+    rev: 340cd9878d8d8289f09fa101b48a8f5f0b7783f4 # frozen: v10.2.0
     hooks:
       - id: oxipng
         name: run oxipng
         description: optimize PNG images with lossless compression
         args: ['-o', '4', '--strip', 'safe', '--alpha']
   - repo: https://github.com/gitleaks/gitleaks
-    rev: v8.30.0
+    rev: 83d9cd684c87d95d656c1458ef04895a7f1cbd8e  # frozen: v8.30.1
     hooks:
       - id: gitleaks
         name: run gitleaks
         description: detect hardcoded secrets
   - repo: https://github.com/Lucas-C/pre-commit-hooks
-    rev: v1.5.6
+    rev: ad1b27d73581aa16cca06fc4a0761fc563ffe8e8 # frozen: v1.5.6
     hooks:
       - id: chmod
         name: set file permissions
@@ -62,6 +73,16 @@ repos:
           - --license-filepath
           - .github/workflows/license-templates/LICENSE.txt
           - --fuzzy-match-generates-todo
+      - id: insert-license
+        name: add license for all conf files
+        description: automatically adds a license header to all conf files that don't have a license header
+        files: \.conf$
+        args:
+          - --comment-style
+          - '|#|'
+          - --license-filepath
+          - .github/workflows/license-templates/LICENSE.txt
+          - --fuzzy-match-generates-todo
       - id: insert-license
         name: add license for all Markdown files
         files: \.md$
@@ -122,7 +143,7 @@ repos:
           - --fuzzy-match-generates-todo
         exclude: ^\.github/workflows/.*\.lock\.yml$
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v6.0.0
+    rev: 3e8a8703264a2f4a69428a0aa4dcb512790b2c8c # frozen: v6.0.0
     hooks:
       #- id: check-added-large-files
       - id: check-case-conflict
@@ -166,18 +187,18 @@ repos:
         args: [--markdown-linebreak-ext=md]
         exclude: ^services/console-proxy/rdpconsole/src/test/doc/freerdp-debug-log\.txt$
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.4.2
+    rev: 57b21406f092110c18776e39b0bda50d37c945c8 # frozen: v2.4.3
     hooks:
       - id: codespell
         name: run codespell
         description: Check spelling with codespell
   - repo: https://github.com/pycqa/flake8
-    rev: 7.3.0
+    rev: c48217e1fc006c2dddd14df54e83b67da15de5cd  # frozen: 7.3.0
     hooks:
-    - id: flake8
-      args: [--config, .github/linters/.flake8]
+      - id: flake8
+        args: [--config, .github/linters/.flake8]
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.48.0
+    rev: 5b5dddc4fb0f83c3ea1fc5616fa63e115dce83e0 # frozen: v0.49.1
     hooks:
       - id: markdownlint
         name: run markdownlint
@@ -186,7 +207,7 @@ repos:
         types: [markdown]
         files: \.md$
   - repo: https://github.com/adrienverge/yamllint
-    rev: v1.38.0
+    rev: cba56bcde1fdd01c1deb3f945e69764c291a6530 # frozen: v1.38.0
     hooks:
       - id: yamllint
         name: run yamllint

PendingReleaseNotes

@@ -40,8 +40,21 @@ example.ver.1 > example.ver.2:
     Storage to grow to enormous sizes as Linux Distributions keep growing in
     size while a stripped down Linux should fit on a 2.88MB floppy.
 
+4.22.0.0 > 4.22.0.1:
+  * Disk-only instance snapshots for KVM UEFI VMs now include a sidecar copy of
+    the active NVRAM state so revert operations restore both disk and firmware
+    boot state consistently.
 
-4.22.0.0 > 4.23.0.0:
+  * UEFI disk-only instance snapshots taken before this change do not contain an
+    NVRAM sidecar and cannot be safely reverted. Take a new snapshot after
+    upgrading before relying on revert for UEFI VMs.
+
+  * Taking a disk-only instance snapshot for KVM UEFI VMs now briefly suspends
+    the guest while the NVRAM sidecar is copied, so that the captured firmware
+    state is consistent with the disk snapshot. Non-UEFI VMs are unaffected and
+    continue to snapshot live.
+
+4.23.0.0 > 24.0.0:
   * KVM/Ceph: RBD volumes can now be encrypted at rest using native librbd
     LUKS2 (<encryption format='luks2' engine='librbd'>), for both data disks
     and root disks. Encryption is transparent to the guest and reuses the

agent/conf/log4j-cloud.xml.in

@@ -38,8 +38,13 @@ under the License.
       <!-- ============================== -->
 
       <Console name="CONSOLE" target="SYSTEM_OUT">
-          <ThresholdFilter level="OFF" onMatch="ACCEPT" onMismatch="DENY"/>
-          <PatternLayout pattern="%-5p [%c{3}] (%t:%x) (logid:%X{logcontextid}) %m%ex%n"/>
+         <ThresholdFilter level="ERROR" onMatch="DENY" onMismatch="ACCEPT"/>
+         <PatternLayout pattern="%d{DEFAULT} %-5p [%c{3}] (%t:%x) (logid:%X{logcontextid}) %m%ex%n"/>
+      </Console>
+
+      <Console name="CONSOLE_ERR" target="SYSTEM_ERR">
+         <ThresholdFilter level="ERROR" onMatch="ACCEPT" onMismatch="DENY"/>
+         <PatternLayout pattern="%d{DEFAULT} %-5p [%c{3}] (%t:%x) (logid:%X{logcontextid}) %m%ex%n"/>
       </Console>
    </Appenders>
 
@@ -50,17 +55,11 @@ under the License.
       <!-- ================ -->
 
       <Logger name="com.cloud" level="INFO"/>
-
       <Logger name="org.apache" level="INFO"/>
-
       <Logger name="org" level="INFO"/>
-
       <Logger name="net" level="INFO"/>
-
       <Logger name="com.amazonaws" level="INFO"/>
-
       <Logger name="httpclient.wire" level="INFO"/>
-
       <Logger name="org.apache.http.wire" level="INFO"/>
 
       <!-- ======================= -->
@@ -69,6 +68,7 @@ under the License.
 
       <Root level="INFO">
          <AppenderRef ref="CONSOLE"/>
+         <AppenderRef ref="CONSOLE_ERR"/>
          <AppenderRef ref="FILE"/>
       </Root>
 

agent/pom.xml

@@ -24,7 +24,7 @@
     <parent>
         <groupId>org.apache.cloudstack</groupId>
         <artifactId>cloudstack</artifactId>
-        <version>4.23.0.0-SNAPSHOT</version>
+        <version>24.0.0-SNAPSHOT</version>
     </parent>
     <dependencies>
         <dependency>