diff --git a/data/valid/generated-parallel-wrapped-by-declarative/Jenkinsfile b/data/valid/generated-parallel-wrapped-by-declarative/Jenkinsfile new file mode 100644 index 0000000..1c22a12 --- /dev/null +++ b/data/valid/generated-parallel-wrapped-by-declarative/Jenkinsfile @@ -0,0 +1,343 @@ +#!/usr/bin/env groovy + +// (C) 2018 - 2021 by Jim Klimov +// A slightly simplified copy of pipeline for regular rescan of OrgFolder +// jobs and MBPs generated by those, and SCM URL tracking in refrepos, from +// https://github.com/jimklimov/git-refrepo-scripts/blob/main/Jenkinsfile-rescan-MBPs +// +// This Jenkinsfile makes-believe it is a Declarative pipeline for +// the sake of commonality of our pipeline scripts (build args, +// various options and stuff), but in fact this is a scripted +// pipeline most of the way. +// Its job is to find MultiBranchPipeline items spawned inside an +// organization folder and issue a rescan request (build operation), +// so we can detect new or closed PRs and branches quickly (due to +// JENKINS-49526 MBPs are otherwise polled once a day, hardcoded). + +// Note: A lot of methods below must be approved by Jenkins admin +// To do that, "replay" the job and edit the script in Web-GUI to +// remove try lines and catch blocks, and replay it time and again +// until all method signatures have been submitted for approval and +// approved in $JENKINS_URL/scriptApproval/ by an admin, one by one. + +import jenkins.model.* +import hudson.model.* +import hudson.util.PersistedList +import jenkins.branch.* + +// This shared array variable will be populated with the list of parallel stages +def scan_stages = [:] + +// Jenkins script security since late 2020 does not like generated code +// clauses made by a routine with such decoration. "Passive" returned +// values like git_urls list here are okay, but stages are "unsandboxed". +// @NonCPS +def parallelSubtasks(String verbose, String regexSubset, String regexExclude) { + def subbuilds = [:] + def jobs = Jenkins.instance.getAllItems() + String git_urls = "" + def candidateurls = [:] + + jobs.each { j -> + if (j instanceof com.cloudbees.hudson.plugins.folder.Folder) { + if (verbose.equals("true")) { + echo 'SKIP-REINDEX+URL: Ignoring JOB which is a com.cloudbees.hudson.plugins.folder.Folder : ' + j.fullName + } + return + } + if (j instanceof jenkins.branch.OrganizationFolder) { + if (verbose.equals("true")) { + echo 'SKIP-REINDEX+URL: Ignoring JOB which is a jenkins.branch.OrganizationFolder : ' + j.fullName + } + return + } + + if ( regexSubset.equals("") ) { +/* + // Note: this was not a correct diagnosis. The snowflake job + // is rebuit every time we changed the CI repo with this + // Jenkinsfile (and compile-web script) as we iterated :) + if ( j.fullName.contains("SPECIAL/snowflake") ) { + echo 'SKIP-REINDEX+URL: Ignoring MBP JOB which is rebuilt every time we touch it (and/or the secondary SCM repo it fetches is changed): ' + j.fullName + return; + } +*/ + } else { + if ( ! ( j.fullName =~ regexSubset ) ) { + echo "SKIP-REINDEX+URL: Ignoring MBP JOB whose name '${j.fullName}' did not match specified regex of interesting jobs to rescan '${regexSubset}'" + return; + } + } + + if ( ! regexExclude.equals("") ) { + if ( j.fullName =~ regexExclude ) { + echo "SKIP-REINDEX+URL: Ignoring MBP JOB whose name '${j.fullName}' matched specified regex of jobs to exclude from rescan '${regexExclude}'" + return; + } + } + + // Determine Git URL for any build job, not only MBPs + def gotUrl = false + def skipJob = false + try { + if (j.disabled) { + echo "SKIP-URL: job '${j.fullName}' is disabled, ignoring the URLs it refers to" + skipJob = true + } + } catch (Exception eDisabled) { + echo "WARNING: failed to query whether the job '${j.fullName}' is disabled, so considering its Git URLs just in case: " + eDisabled + } + + if (!skipJob) { + try { + if (verbose.equals("true")) { + echo "Analyzing Git URL for job '${j.fullName}' ..." + } + + def scmsrc = [] + if (j instanceof org.jenkinsci.plugins.workflow.job.WorkflowJob) { + scmsrc = j.getSCMs() + } else { + try { + scmsrc = j.SCMSources + } catch (Exception eSCMsrc) { + if (verbose.equals("true")) { + echo "No supported SCM source variable found: " + eSCMsrc; +// TODO: Support legacy jobs, they should have SCM's too (the CI repo at least): +// 'SCMless-freestyle' : No supported SCM source variable found: groovy.lang.MissingPropertyException: No such property: SCMSources for class: hudson.model.FreeStyleProject +// 'SCMless-MultiJob-toplevel' : No supported SCM source variable found: groovy.lang.MissingPropertyException: No such property: SCMSources for class: com.tikal.jenkins.plugins.multijob.MultiJobProject + } + } + } // See if we have any scmsrc's + + if (scmsrc.size() >0) + for (scm in scmsrc) { + String url = "" + if (scm instanceof hudson.plugins.git.GitSCM) { + if (verbose.equals("true")) { + echo "GitSCM..."; + } + try { + // Inspired by https://github.com/jenkinsci/git-plugin/blob/master/src/main/java/hudson/plugins/git/GitSCM.java#L256 + def CfgList = scm.getUserRemoteConfigs() + for (int i = 0; i < CfgList.size(); ++i) { + try { + url = CfgList.get(i).getUrl(); + if ( !candidateurls["${url}"] ) candidateurls["${url}"] = ""; + candidateurls["${url}"] += "${j.fullName} "; + gotUrl = true + } catch (Exception e0g0) { echo "SKIP-URL: Failed scm.getUserRemoteConfigs().get().getUrl() #" + i + " : " + e0g0; } + } + continue; // This "scm" instance is fully processed + } catch (Exception e0g) { echo "Failed scm.getUserRemoteConfigs().get().getUrl(): " + e0g; throw e0g; } + } else + if (scm instanceof org.jenkinsci.plugins.github_branch_source.GitHubSCMSource) { + if (verbose.equals("true")) { + echo "GitHubSCM..."; + } + try { + url = scm.remote + gotUrl = true + } catch (Exception e0gh) { echo "Failed scm.remote: " + e0hg; throw e0gh; } + } else + if (scm instanceof com.cloudbees.jenkins.plugins.bitbucket.BitbucketSCMSource) { + if (verbose.equals("true")) { + echo "BitbucketSCM..."; + } + try { + def repoo = scm.getRepoOwner() + def repon = scm.getRepository() + def repou = scm.getServerUrl() + if (verbose.equals("true")) { + echo "U='${repou}' O='${repoo}' N='${repon}'" + } +// Alas, there seems to be no good easy method to just get the URL (SSH or HTTP) +// url = scm.getRemote( repoo, repon ) + url = "${repou}/scm/${repoo}/${repon}.git" + gotUrl = true + } catch (Exception e0b) { echo "Failed scm.getRepository(): " + e0b; throw e0b; } + } else { + echo "WARNING : SCMSources type not supported in MBP Job '${j.fullName}'" + } + if ( !candidateurls["${url}"] ) candidateurls["${url}"] = ""; + candidateurls["${url}"] += "${j.fullName} "; + } // Find candidateurl's in discovered scmsrc's + + } catch (Exception e1) { + echo "SKIP-URL: Failed to research SCM URL for '${j.fullName}' : " + e1 + } // try to investigate job attributes + } // if !skipJob + + if (!gotUrl) { + try { + if (j.buildable) + echo "SKIP-URL: Failed to find any SCM URL for buildable job '${j.fullName}'" + } catch (Exception e) { } + } + + + if (! (j instanceof org.jenkinsci.plugins.workflow.multibranch.WorkflowMultiBranchProject) ) { + if (verbose.equals("true")) { + echo 'SKIP-REINDEX: Ignoring JOB which is not an MBP: ' + j.fullName + } + return; + } + if ( ! j.fullName.contains("/") ) { + echo 'SKIP-REINDEX: Ignoring MBP JOB which is not under a (organization) folder, so has and hopefully honours a schedule of its own: ' + j.fullName + return; + } + // TODO: Determine that MBP's schedule and only fire below if it is "once a day" (hardcoded)? + + subbuilds["${j.fullName}"] = { + stage("Scan ${j.fullName}") { + // Per non-declarative pipeline syntax, no steps{} here + echo "Rescanning '${j.fullName}' ..." + // Jenkins refuses to "wait" for this type of job... + // TODO: Sleep-poll the results (and logs?) of the + // spawned child jobs. + try { + def bbb = build job: "${j.fullName}", quietPeriod: 0, wait: false, propagate: true + } catch (Exception e) { + // e.g. a disabled repo, like one where a Jenkinsfile + // script existed earlier but was removed + echo "Failed to trigger scan for ${j.fullName}, skipped" + } + } + } // end of def subbuilds[j.fullName] + + } // jobs.each() clause + + if (candidateurls.size() >0) + candidateurls.each { url, jobnames -> + if ( url.startsWith("git@") ) { + if ( url.startsWith("git@github.com:") ) + url = url.replaceAll("git@github.com:", "git@github.com/") ; + url = "ssh://" + url; + } + if ( url.contains("://") ) { + if (verbose.equals("true")) + echo "GIT-URL: MBP Job(s) '${jobnames}' have Git URL '" + url + "'"; + if ( url.contains("gitlab.modernrepos.com") || url.contains("bb.modernrepos.com") ) { + echo "SKIP-URL: Skipping obsoleted-server Git URL '" + url + "'"; + } else { + git_urls += url + " "; + } + } else { + echo "SKIP-URL: MBP Job(s) '${jobnames}' have invalid URL: '" + url + "'"; + } + } // Populate git_urls string with unique URLs for discovered jobs (if any) + + return [subbuilds, git_urls] +} + + +// A declarative pipeline shiny wrapper (we need it +// for common ways of autosetup and params, mostly) +pipeline { + options { +/* + description("This job runs regularly to find and rescan multibranch pipeline jobs (e.g. generated via organization folders) to add monitoring of new PRs (and disable monitoring of merged PRs) in a timely fashion.") +*/ + disableConcurrentBuilds() + disableResume() + durabilityHint('PERFORMANCE_OPTIMIZED') + buildDiscarder(logRotator(numToKeepStr: '10')) + skipDefaultCheckout true + } + triggers { + cron('H/30 * * * *') + } + agent {label "master"} + parameters { + booleanParam ( + defaultValue: false, + description: 'Print found and skipped items?', + name: 'JOBLIST_VERBOSE' + ) + booleanParam ( + defaultValue: true, + description: 'Update git reference repo?', + name: 'UPDATE_REFREPO' + ) + string ( + defaultValue: '', + description: 'Groovy regex for constraining a list of rescans to schedule (if not empty, then only matching MBP job names are scanned)', + name: 'JOBLIST_SUBSET') + string ( + defaultValue: '.*(/PP/|/CB/|git\\.modernrepos\\.com|cvs\\.legacyrepos).*', + description: 'Groovy regex for constraining a list of rescans to schedule (if not empty, then matching MBP job names are excluded)\nDefault: PP and CB projects whom we help build but do not provide a cache for', + name: 'JOBLIST_EXCLUDE') + } + stages { + stage('Single-exec milestone') { + steps { + milestone 1 + } // steps clause + } + } +} + +// Non-declarative pipeline payload. +// This code below does not work inside a pipeline{} stage{} +// clause nor in the post{} clause - not even with the added +// try/catches for exceptions all around. +// Curiously, when this pipeline job is built by hand +// or triggered by timer, it fails with lots of CPS and +// marshalling exception messages in the end. Replaying +// the same groovy script with no changes just works. +try { + String git_urls = "" + echo "DISCOVERING JOBS..." + (scan_stages, git_urls) = parallelSubtasks( "${params.JOBLIST_VERBOSE}", "${params.JOBLIST_SUBSET}", "${params.JOBLIST_EXCLUDE}" ) + + // Note: the "sh" step should not be referenced inside the @NonCPS method + if ( git_urls.equals("") ) { + echo "SKIP to register Git URLs with git reference repo : list seems empty" + } else { + if ( params.UPDATE_REFREPO.equals("false") ) { + echo "SKIP to register Git URLs with git reference repo : requested so. List is: " + git_urls + } else { + scan_stages["RefRepo"] = { + stage("RefRepoConfig") { + try { + // The update script by itself is quite fast, but its + // startup can be blocked by other copies running or + // NFS server unavailability. Timing out allows the + // rescans to still happen, and try/catch block makes it + // non-fatal as it is not a big issue to register later. + node { + // The shell part must run on a node (NOTE: and also as + // "gitowner" with write-access to the jenkins-gitcache + // directory if locally or over NFS) + stage("Update Git Reference repo pointers") { + timeout (time: 5, unit: 'MINUTES', activity: true) { + sh """ cd /home/gitowner/jenkins-gitcache && ./register-git-cache.sh add ${git_urls} """ + } + } + } + } catch (Exception e) { + echo "SKIP : Failed to register Git URLs with git reference repo, can try with a later re-run: " + e + } + } + } + } // Add a RefRepo subbuild if not skipped by build arg + } // Add a RefRepo subbuild if got not-empty git_urls +} catch (java.io.NotSerializableException e) { + echo "Ignoring NotSerializableException during parallelSubtasks(), a groovy/JenkinsDSL thing" +} catch (Exception e) { + echo "Failed to find jobs or set up parallel scans: " + e + currentBuild.result = 'FAILED' + manager.buildAborted() +} + +try { + echo "SCHEDULING RESCANS..." + parallel scan_stages + echo "SCHEDULING OF RESCANS COMPLETED" +} catch (java.io.NotSerializableException e) { + echo "Ignoring NotSerializableException during parallelized work, a groovy/JenkinsDSL thing" +} catch (Exception e) { + echo "Failed to trigger parallel scans: " + e + currentBuild.result = 'FAILED' + manager.buildAborted() +}