Add a data/valid/generated-parallel-wrapped-by-declarative to the mix

data/valid/generated-parallel-wrapped-by-declarative/Jenkinsfile

@@ -0,0 +1,343 @@
+#!/usr/bin/env groovy
+
+// (C) 2018 - 2021 by Jim Klimov <[email protected]>
+// A slightly simplified copy of pipeline for regular rescan of OrgFolder
+// jobs and MBPs generated by those, and SCM URL tracking in refrepos, from
+// https://github.com/jimklimov/git-refrepo-scripts/blob/main/Jenkinsfile-rescan-MBPs
+//
+// This Jenkinsfile makes-believe it is a Declarative pipeline for
+// the sake of commonality of our pipeline scripts (build args,
+// various options and stuff), but in fact this is a scripted
+// pipeline most of the way.
+// Its job is to find MultiBranchPipeline items spawned inside an
+// organization folder and issue a rescan request (build operation),
+// so we can detect new or closed PRs and branches quickly (due to
+// JENKINS-49526 MBPs are otherwise polled once a day, hardcoded).
+
+// Note: A lot of methods below must be approved by Jenkins admin
+// To do that, "replay" the job and edit the script in Web-GUI to
+// remove try lines and catch blocks, and replay it time and again
+// until all method signatures have been submitted for approval and
+// approved in $JENKINS_URL/scriptApproval/ by an admin, one by one.
+
+import jenkins.model.*
+import hudson.model.*
+import hudson.util.PersistedList
+import jenkins.branch.*
+
+// This shared array variable will be populated with the list of parallel stages
+def scan_stages = [:]
+
+// Jenkins script security since late 2020 does not like generated code
+// clauses made by a routine with such decoration. "Passive" returned
+// values like git_urls list here are okay, but stages are "unsandboxed".
+// @NonCPS
+def parallelSubtasks(String verbose, String regexSubset, String regexExclude) {
+    def subbuilds = [:]
+    def jobs = Jenkins.instance.getAllItems()
+    String git_urls = ""
+    def candidateurls = [:]
+
+    jobs.each { j ->
+        if (j instanceof com.cloudbees.hudson.plugins.folder.Folder) {
+            if (verbose.equals("true")) {
+                echo 'SKIP-REINDEX+URL: Ignoring JOB which is a com.cloudbees.hudson.plugins.folder.Folder : ' + j.fullName
+            }
+            return
+        }
+        if (j instanceof jenkins.branch.OrganizationFolder) {
+            if (verbose.equals("true")) {
+                echo 'SKIP-REINDEX+URL: Ignoring JOB which is a jenkins.branch.OrganizationFolder : ' + j.fullName
+            }
+            return
+        }
+
+        if ( regexSubset.equals("") ) {
+/*
+            // Note: this was not a correct diagnosis. The snowflake job
+            // is rebuit every time we changed the CI repo with this
+            // Jenkinsfile (and compile-web script) as we iterated :)
+            if ( j.fullName.contains("SPECIAL/snowflake") ) {
+                echo 'SKIP-REINDEX+URL: Ignoring MBP JOB which is rebuilt every time we touch it (and/or the secondary SCM repo it fetches is changed): ' + j.fullName
+                return;
+            }
+*/
+        } else {
+            if ( ! ( j.fullName =~ regexSubset ) ) {
+                echo "SKIP-REINDEX+URL: Ignoring MBP JOB whose name '${j.fullName}' did not match specified regex of interesting jobs to rescan '${regexSubset}'"
+                return;
+            }
+        }
+
+        if ( ! regexExclude.equals("") ) {
+            if ( j.fullName =~ regexExclude ) {
+                echo "SKIP-REINDEX+URL: Ignoring MBP JOB whose name '${j.fullName}' matched specified regex of jobs to exclude from rescan '${regexExclude}'"
+                return;
+            }
+        }
+
+        // Determine Git URL for any build job, not only MBPs
+        def gotUrl = false
+        def skipJob = false
+        try {
+            if (j.disabled) {
+                echo "SKIP-URL: job '${j.fullName}' is disabled, ignoring the URLs it refers to"
+                skipJob = true
+            }
+        } catch (Exception eDisabled) {
+            echo "WARNING: failed to query whether the job '${j.fullName}' is disabled, so considering its Git URLs just in case: " + eDisabled
+        }
+
+        if (!skipJob) {
+          try {
+            if (verbose.equals("true")) {
+                echo "Analyzing Git URL for job '${j.fullName}' ..."
+            }
+
+            def scmsrc = []
+            if (j instanceof org.jenkinsci.plugins.workflow.job.WorkflowJob) {
+                scmsrc = j.getSCMs()
+            } else {
+                try {
+                    scmsrc = j.SCMSources
+                } catch (Exception eSCMsrc) {
+                    if (verbose.equals("true")) {
+                        echo "No supported SCM source variable found: " + eSCMsrc;
+// TODO: Support legacy jobs, they should have SCM's too (the CI repo at least):
+// 'SCMless-freestyle' : No supported SCM source variable found: groovy.lang.MissingPropertyException: No such property: SCMSources for class: hudson.model.FreeStyleProject
+// 'SCMless-MultiJob-toplevel' : No supported SCM source variable found: groovy.lang.MissingPropertyException: No such property: SCMSources for class: com.tikal.jenkins.plugins.multijob.MultiJobProject
+                    }
+                }
+            } // See if we have any scmsrc's
+
+            if (scmsrc.size() >0)
+            for (scm in scmsrc) {
+                String url = ""
+                if (scm instanceof hudson.plugins.git.GitSCM) {
+                    if (verbose.equals("true")) {
+                        echo "GitSCM...";
+                    }
+                    try {
+                        // Inspired by https://github.com/jenkinsci/git-plugin/blob/master/src/main/java/hudson/plugins/git/GitSCM.java#L256
+                        def CfgList = scm.getUserRemoteConfigs()
+                        for (int i = 0; i < CfgList.size(); ++i) {
+                            try {
+                                url = CfgList.get(i).getUrl();
+                                if ( !candidateurls["${url}"] ) candidateurls["${url}"] = "";
+                                candidateurls["${url}"] += "${j.fullName} ";
+                                gotUrl = true
+                            } catch (Exception e0g0) { echo "SKIP-URL: Failed scm.getUserRemoteConfigs().get().getUrl() #" + i + " : " + e0g0; }
+                        }
+                        continue; // This "scm" instance is fully processed
+                    } catch (Exception e0g) { echo "Failed scm.getUserRemoteConfigs().get().getUrl(): " + e0g; throw e0g; }
+                } else
+                if (scm instanceof org.jenkinsci.plugins.github_branch_source.GitHubSCMSource) {
+                    if (verbose.equals("true")) {
+                        echo "GitHubSCM...";
+                    }
+                    try {
+                        url = scm.remote
+                        gotUrl = true
+                    } catch (Exception e0gh) { echo "Failed scm.remote: " + e0hg; throw e0gh; }
+                } else
+                if (scm instanceof com.cloudbees.jenkins.plugins.bitbucket.BitbucketSCMSource) {
+                    if (verbose.equals("true")) {
+                        echo "BitbucketSCM...";
+                    }
+                    try {
+                        def repoo = scm.getRepoOwner()
+                        def repon = scm.getRepository()
+                        def repou = scm.getServerUrl()
+                        if (verbose.equals("true")) {
+                            echo "U='${repou}' O='${repoo}' N='${repon}'"
+                        }
+// Alas, there seems to be no good easy method to just get the URL (SSH or HTTP)
+//                        url = scm.getRemote( repoo, repon )
+                        url = "${repou}/scm/${repoo}/${repon}.git"
+                        gotUrl = true
+                    } catch (Exception e0b) { echo "Failed scm.getRepository(): " + e0b; throw e0b; }
+                } else {
+                    echo "WARNING : SCMSources type not supported in MBP Job '${j.fullName}'"
+                }
+                if ( !candidateurls["${url}"] ) candidateurls["${url}"] = "";
+                candidateurls["${url}"] += "${j.fullName} ";
+            } // Find candidateurl's in discovered scmsrc's
+
+          } catch (Exception e1) {
+            echo "SKIP-URL: Failed to research SCM URL for '${j.fullName}' : " + e1
+          } // try to investigate job attributes
+        } // if !skipJob
+
+        if (!gotUrl) {
+            try {
+                if (j.buildable)
+                    echo "SKIP-URL: Failed to find any SCM URL for buildable job '${j.fullName}'"
+            } catch (Exception e) { }
+        }
+
+
+        if (! (j instanceof org.jenkinsci.plugins.workflow.multibranch.WorkflowMultiBranchProject) ) {
+            if (verbose.equals("true")) {
+                echo 'SKIP-REINDEX: Ignoring JOB which is not an MBP: ' + j.fullName
+            }
+            return;
+        }
+        if ( ! j.fullName.contains("/") ) {
+            echo 'SKIP-REINDEX: Ignoring MBP JOB which is not under a (organization) folder, so has and hopefully honours a schedule of its own: ' + j.fullName
+            return;
+        }
+        // TODO: Determine that MBP's schedule and only fire below if it is "once a day" (hardcoded)?
+
+        subbuilds["${j.fullName}"] = {
+            stage("Scan ${j.fullName}") {
+                // Per non-declarative pipeline syntax, no steps{} here
+                    echo "Rescanning '${j.fullName}' ..."
+                    // Jenkins refuses to "wait" for this type of job...
+                    // TODO: Sleep-poll the results (and logs?) of the
+                    // spawned child jobs.
+                    try {
+                        def bbb = build job: "${j.fullName}", quietPeriod: 0, wait: false, propagate: true
+                    } catch (Exception e) {
+                        // e.g. a disabled repo, like one where a Jenkinsfile
+                        // script existed earlier but was removed
+                        echo "Failed to trigger scan for ${j.fullName}, skipped"
+                    }
+            }
+        } // end of def subbuilds[j.fullName]
+
+    } // jobs.each() clause
+
+    if (candidateurls.size() >0)
+    candidateurls.each { url, jobnames ->
+        if ( url.startsWith("git@") ) {
+            if ( url.startsWith("[email protected]:") )
+                url = url.replaceAll("[email protected]:", "[email protected]/") ;
+            url = "ssh://" + url;
+        }
+        if ( url.contains("://") ) {
+            if (verbose.equals("true"))
+                echo "GIT-URL: MBP Job(s) '${jobnames}' have Git URL '" + url + "'";
+            if ( url.contains("gitlab.modernrepos.com") || url.contains("bb.modernrepos.com") ) {
+                echo "SKIP-URL: Skipping obsoleted-server Git URL '" + url + "'";
+            } else {
+                git_urls += url + " ";
+            }
+        } else {
+            echo "SKIP-URL: MBP Job(s) '${jobnames}' have invalid URL: '" + url + "'";
+        }
+    } // Populate git_urls string with unique URLs for discovered jobs (if any)
+
+    return [subbuilds, git_urls]
+}
+
+
+// A declarative pipeline shiny wrapper (we need it
+// for common ways of autosetup and params, mostly)
+pipeline {
+    options {
+/*
+        description("This job runs regularly to find and rescan multibranch pipeline jobs (e.g. generated via organization folders) to add monitoring of new PRs (and disable monitoring of merged PRs) in a timely fashion.")
+*/
+        disableConcurrentBuilds()
+        disableResume()
+        durabilityHint('PERFORMANCE_OPTIMIZED')
+        buildDiscarder(logRotator(numToKeepStr: '10'))
+        skipDefaultCheckout true
+    }
+    triggers {
+        cron('H/30 * * * *')
+    }
+    agent {label "master"}
+    parameters {
+        booleanParam (
+            defaultValue: false,
+            description: 'Print found and skipped items?',
+            name: 'JOBLIST_VERBOSE'
+        )
+        booleanParam (
+            defaultValue: true,
+            description: 'Update git reference repo?',
+            name: 'UPDATE_REFREPO'
+        )
+        string (
+            defaultValue: '',
+            description: 'Groovy regex for constraining a list of rescans to schedule (if not empty, then only matching MBP job names are scanned)',
+            name: 'JOBLIST_SUBSET')
+        string (
+            defaultValue: '.*(/PP/|/CB/|git\\.modernrepos\\.com|cvs\\.legacyrepos).*',
+            description: 'Groovy regex for constraining a list of rescans to schedule (if not empty, then matching MBP job names are excluded)\nDefault: PP and CB projects whom we help build but do not provide a cache for',
+            name: 'JOBLIST_EXCLUDE')
+    }
+    stages {
+        stage('Single-exec milestone') {
+            steps {
+                milestone 1
+            } // steps clause
+        }
+    }
+}
+
+// Non-declarative pipeline payload.
+// This code below does not work inside a pipeline{} stage{}
+// clause nor in the post{} clause - not even with the added
+// try/catches for exceptions all around.
+// Curiously, when this pipeline job is built by hand
+// or triggered by timer, it fails with lots of CPS and
+// marshalling exception messages in the end. Replaying
+// the same groovy script with no changes just works.
+try {
+    String git_urls = ""
+    echo "DISCOVERING JOBS..."
+    (scan_stages, git_urls) = parallelSubtasks( "${params.JOBLIST_VERBOSE}", "${params.JOBLIST_SUBSET}", "${params.JOBLIST_EXCLUDE}" )
+
+    // Note: the "sh" step should not be referenced inside the @NonCPS method
+    if ( git_urls.equals("") ) {
+        echo "SKIP to register Git URLs with git reference repo : list seems empty"
+    } else {
+      if ( params.UPDATE_REFREPO.equals("false") ) {
+        echo "SKIP to register Git URLs with git reference repo : requested so. List is: " + git_urls
+      } else {
+        scan_stages["RefRepo"] = {
+            stage("RefRepoConfig") {
+                try {
+                    // The update script by itself is quite fast, but its
+                    // startup can be blocked by other copies running or
+                    // NFS server unavailability. Timing out allows the
+                    // rescans to still happen, and try/catch block makes it
+                    // non-fatal as it is not a big issue to register later.
+                    node {
+                        // The shell part must run on a node (NOTE: and also as
+                        // "gitowner" with write-access to the jenkins-gitcache
+                        // directory if locally or over NFS)
+                        stage("Update Git Reference repo pointers") {
+                            timeout (time: 5, unit: 'MINUTES', activity: true) {
+                                sh """ cd /home/gitowner/jenkins-gitcache && ./register-git-cache.sh add ${git_urls} """
+                            }
+                        }
+                    }
+                } catch (Exception e) {
+                    echo "SKIP : Failed to register Git URLs with git reference repo, can try with a later re-run: " + e
+                }
+            }
+        }
+      } // Add a RefRepo subbuild if not skipped by build arg
+    } // Add a RefRepo subbuild if got not-empty git_urls
+} catch (java.io.NotSerializableException e) {
+    echo "Ignoring NotSerializableException during parallelSubtasks(), a groovy/JenkinsDSL thing"
+} catch (Exception e) {
+    echo "Failed to find jobs or set up parallel scans: " + e
+    currentBuild.result = 'FAILED'
+    manager.buildAborted()
+}
+
+try {
+    echo "SCHEDULING RESCANS..."
+    parallel scan_stages
+    echo "SCHEDULING OF RESCANS COMPLETED"
+} catch (java.io.NotSerializableException e) {
+    echo "Ignoring NotSerializableException during parallelized work, a groovy/JenkinsDSL thing"
+} catch (Exception e) {
+    echo "Failed to trigger parallel scans: " + e
+    currentBuild.result = 'FAILED'
+    manager.buildAborted()
+}