feat(build): add utils.deploy.env.add build plugin API

This changeset adds a new utility to build plugins:
`utils.deploy.env.add()`. This utility function allows build plugins to
register environment variables at build time that will be injected into
a specific deploy.

Userland build plugins cannot see or manipulate the deploy environment
variables registered by core or other userland plugins.

Deploy-specific environment variables are collected by the `build`
supervisor process and processed by the deploy step, which sends them to
Buildbot for processing. They are also returned by the programmatic `dev`
interface; we'll use these returned variables in the CLI, where we'll
e.g. set them on functions.

While I was in here I also converted a few files to TypeScript/generally
did my best to improve the types along the way.

Author: ndhoule

packages/build/src/core/build.ts

@@ -660,6 +660,7 @@ const runBuild = async function ({
     configMutations,
     metrics,
     returnValues,
+    deployEnvVars,
   } = await runSteps({
     steps,
     buildbotServerSocket,
@@ -708,5 +709,6 @@ const runBuild = async function ({
     configMutations,
     metrics,
     returnValues,
+    deployEnvVars,
   }
 }

packages/build/src/core/dev.js

@@ -13,6 +13,7 @@ export const startDev = async (devCommand, flags = {}) => {
       netlifyConfig: netlifyConfigA,
       configMutations,
       returnValues,
+      deployEnvVars,
     } = await execBuild({
       ...normalizedFlags,
       errorMonitor,
@@ -33,6 +34,7 @@ export const startDev = async (devCommand, flags = {}) => {
       logs,
       configMutations,
       generatedFunctions: getGeneratedFunctions(returnValues),
+      deployEnvVars,
     }
   } catch (error) {
     const { severity, message, stack } = await handleBuildError(error, errorParams)

packages/build/src/plugins/child/lazy.js packages/build/src/plugins/child/lazy.tspackages/build/src/plugins/child/lazy.js renamed to packages/build/src/plugins/child/lazy.ts

@@ -2,10 +2,12 @@ import memoizeOne from 'memoize-one'
 
 // Add a `object[propName]` whose value is the return value of `getFunc()`, but
 // is only retrieved when accessed.
-export const addLazyProp = function (object, propName, getFunc) {
-  const mGetFunc = memoizeOne(getFunc, returnTrue)
-  // Mutation is required due to the usage of `Object.defineProperty()`
+export const addLazyProp = function (object: object, propName: string, getFunc: () => unknown): void {
+  // @ts-expect-error(ndhoule): dis be angry
+  // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+  const mGetFunc: typeof getFunc = memoizeOne(getFunc, returnTrue)
 
+  // Mutation is required due to the usage of `Object.defineProperty()`
   Object.defineProperty(object, propName, {
     get: mGetFunc,
     enumerable: true,

packages/build/src/plugins/child/run.js

@@ -20,8 +20,9 @@ export const run = async function (
     const method = methods[event]
     const runState = {}
     const generatedFunctions = []
+    const deployEnvVars = []
     const systemLog = getSystemLog()
-    const utils = getUtils({ event, constants, generatedFunctions, runState })
+    const utils = getUtils({ event, constants, deployEnvVars, generatedFunctions, runState })
     const netlifyConfigCopy = cloneNetlifyConfig(netlifyConfig)
     const runOptions = {
       utils,
@@ -45,6 +46,6 @@ export const run = async function (
 
     const configMutations = getConfigMutations(netlifyConfig, netlifyConfigCopy, event)
     const returnValue = generatedFunctions.length ? { generatedFunctions } : undefined
-    return { ...runState, newEnvChanges, configMutations, returnValue }
+    return { ...runState, deployEnvVars, newEnvChanges, configMutations, returnValue }
   })
 }

…ckages/build/src/plugins/child/status.js …ckages/build/src/plugins/child/status.tspackages/build/src/plugins/child/status.js renamed to packages/build/src/plugins/child/status.ts packages/build/src/plugins/child/status.js renamed to packages/build/src/plugins/child/status.ts

@@ -3,29 +3,32 @@ import isPlainObj from 'is-plain-obj'
 import { addErrorInfo } from '../../error/info.js'
 
 // Report status information to the UI
-export const show = function (runState, showArgs) {
+export const show = function (runState: Record<string, unknown>, showArgs: Record<string, unknown>) {
   validateShowArgs(showArgs)
   const { title, summary, text, extraData } = removeEmptyStrings(showArgs)
   runState.status = { state: 'success', title, summary, text, extraData }
 }
 
 // Validate arguments of `utils.status.show()`
-const validateShowArgs = function (showArgs) {
+const validateShowArgs = function (showArgs: Record<string, unknown>) {
   try {
     validateShowArgsObject(showArgs)
     const { title, summary, text, extraData, ...otherArgs } = showArgs
     validateShowArgsKeys(otherArgs)
     Object.entries({ title, summary, text }).forEach(validateStringArg)
     validateShowArgsSummary(summary)
     validateShowArgsExtraData(extraData)
-  } catch (error) {
+  } catch (error: unknown) {
+    if (!(error instanceof Error)) {
+      throw error
+    }
     error.message = `utils.status.show() ${error.message}`
     addErrorInfo(error, { type: 'pluginValidation' })
     throw error
   }
 }
 
-const validateShowArgsObject = function (showArgs) {
+function validateShowArgsObject(showArgs: unknown): asserts showArgs is Record<string, unknown> {
   if (showArgs === undefined) {
     throw new Error('requires an argument')
   }
@@ -35,36 +38,36 @@ const validateShowArgsObject = function (showArgs) {
   }
 }
 
-const validateShowArgsKeys = function (otherArgs) {
+const validateShowArgsKeys = function (otherArgs: Record<string, unknown>) {
   const otherKeys = Object.keys(otherArgs).map((arg) => `"${arg}"`)
   if (otherKeys.length !== 0) {
     throw new Error(`must only contain "title", "summary" or "text" properties, not ${otherKeys.join(', ')}`)
   }
 }
 
-const validateStringArg = function ([key, value]) {
+const validateStringArg = function ([key, value]: [string, unknown]) {
   if (value !== undefined && typeof value !== 'string') {
     throw new Error(`"${key}" property must be a string`)
   }
 }
 
-const validateShowArgsSummary = function (summary) {
-  if (summary === undefined || summary.trim() === '') {
+const validateShowArgsSummary = function (summary: unknown) {
+  if (typeof summary !== 'string' || summary.trim() === '') {
     throw new Error('requires specifying a "summary" property')
   }
 }
 
-const validateShowArgsExtraData = function (extraData) {
-  if (extraData !== undefined && Array.isArray(extraData) === false) {
+const validateShowArgsExtraData = function (extraData: unknown) {
+  if (extraData !== undefined && !Array.isArray(extraData)) {
     throw new TypeError('provided extra data must be an array')
   }
 }
 
-const removeEmptyStrings = function (showArgs) {
-  return Object.fromEntries(Object.entries(showArgs).map(removeEmptyString))
+const removeEmptyStrings = function (showArgs: Record<string, unknown>): Record<string, unknown> {
+  return Object.fromEntries(Object.entries(showArgs).map(removeEmptyString)) as Record<string, unknown>
 }
 
-const removeEmptyString = function ([key, value]) {
+const removeEmptyString = function ([key, value]: [string, unknown]) {
   if (typeof value === 'string' && value.trim() === '') {
     return [key]
   }

packages/build/src/plugins/child/utils.js

@@ -0,0 +1,118 @@
+import { bindOpts as cacheBindOpts } from '@netlify/cache-utils'
+import { add as functionsAdd, list as functionsList, listAll as functionsListAll } from '@netlify/functions-utils'
+import { getGitUtils } from '@netlify/git-utils'
+import { run as baseRun, runCommand } from '@netlify/run-utils'
+
+import { failBuild, failPlugin, cancelBuild, failPluginWithWarning } from '../error.js'
+import { isSoftFailEvent } from '../events.js'
+import { isReservedEnvironmentVariableKey } from '../../utils/environment.js'
+import type { NetlifyPluginConstants } from '../../core/constants.js'
+import type { ReturnValue } from '../../steps/return_values.ts'
+import type { NetlifyPluginUtils } from '../../types/options/netlify_plugin_utils.js'
+
+import { addLazyProp } from './lazy.js'
+import { show } from './status.js'
+
+type BuildEvent = 'onPreBuild' | 'onBuild' | 'onPostBuild' | 'onError' | 'onSuccess' | 'onEnd'
+
+type RunState = Record<string, unknown>
+
+type DeployEnvVarsData = { key: string; value: string; isSecret: boolean }[]
+
+// Retrieve the `utils` argument.
+export const getUtils = function ({
+  event,
+  constants: { FUNCTIONS_SRC, INTERNAL_FUNCTIONS_SRC, CACHE_DIR },
+  generatedFunctions = [],
+  runState,
+  deployEnvVars = [],
+}: {
+  event: BuildEvent
+  constants: NetlifyPluginConstants
+  generatedFunctions?: ReturnValue['generatedFunctions']
+  runState: RunState
+  deployEnvVars: DeployEnvVarsData
+}): NetlifyPluginUtils {
+  const run = Object.assign(baseRun, { command: runCommand }) as unknown as NetlifyPluginUtils['run']
+  const build = getBuildUtils(event)
+  const cache = getCacheUtils(CACHE_DIR)
+  const deploy = getDeployUtils({ deployEnvVars })
+  const functions = getFunctionsUtils(FUNCTIONS_SRC, INTERNAL_FUNCTIONS_SRC, generatedFunctions)
+  const status = getStatusUtils(runState)
+  const utils = { build, cache, deploy, functions, run, status }
+  addLazyProp(utils, 'git', () => getGitUtils())
+  return utils as typeof utils & { git: NetlifyPluginUtils['git'] }
+}
+
+const getDeployUtils = ({ deployEnvVars }: { deployEnvVars: DeployEnvVarsData }): NetlifyPluginUtils['deploy'] => {
+  const env: NetlifyPluginUtils['deploy']['env'] = {
+    add(key, value, { isSecret = false, scopes = [] } = {}) {
+      // Do some basic client-side validation on the injected variables. The build happens long
+      // before we attempt to create these environment variables via the Netlify API. We want to
+      // give the user as much immediate feedback as possible, so we perform this validation in the
+      // build plugin as to direct the user on which build plugin is causing a problem.
+      if (isReservedEnvironmentVariableKey(key)) {
+        throw new Error(
+          `utils.deploy.env.add() failed: "${key}" is a reserved environment variable in Netlify deployments. Use a different environment variable key.`,
+        )
+      }
+
+      let normalizedScopes = new Set(...scopes)
+      if (normalizedScopes.size == 0) {
+        normalizedScopes = new Set(
+          // If the user did not specify scopes, we assume they mean all valid scopes. Secrets are
+          // not permitted in the post-processing scope.
+          isSecret ? ['builds', 'functions', 'runtime'] : ['builds', 'functions', 'post_processing', 'runtime'],
+        )
+      }
+      if (isSecret && normalizedScopes.has('post_processing')) {
+        throw new Error(`utils.deploy.env.add() failed: The "post_processing" scope cannot be used with isSecret=true.`)
+      }
+
+      const existingDeployEnvVarIdx = deployEnvVars.findIndex((env) => env.key === key)
+      if (existingDeployEnvVarIdx !== -1) {
+        deployEnvVars[existingDeployEnvVarIdx] = { key, value, isSecret }
+      } else {
+        deployEnvVars.push({ key, value, isSecret })
+      }
+
+      return env
+    },
+  }
+  return Object.freeze({ env: Object.freeze(env) })
+}
+
+const getBuildUtils = function (event: 'onPreBuild' | 'onBuild' | 'onPostBuild' | 'onError' | 'onSuccess' | 'onEnd') {
+  if (isSoftFailEvent(event)) {
+    return {
+      failPlugin,
+      failBuild: failPluginWithWarning.bind(null, 'failBuild', event),
+      cancelBuild: failPluginWithWarning.bind(null, 'cancelBuild', event),
+    }
+  }
+
+  return { failBuild, failPlugin, cancelBuild }
+}
+
+const getCacheUtils = function (CACHE_DIR: string) {
+  return cacheBindOpts({ cacheDir: CACHE_DIR })
+}
+
+const getFunctionsUtils = function (
+  FUNCTIONS_SRC: string | undefined,
+  INTERNAL_FUNCTIONS_SRC: string | undefined,
+  generatedFunctions: { path: string }[],
+) {
+  const functionsDirectories = [INTERNAL_FUNCTIONS_SRC, FUNCTIONS_SRC].filter(Boolean)
+  const add = (src: string | undefined) => functionsAdd(src, INTERNAL_FUNCTIONS_SRC, { fail: failBuild })
+  const list = functionsList.bind(null, functionsDirectories, { fail: failBuild })
+  const listAll = functionsListAll.bind(null, functionsDirectories, { fail: failBuild })
+  const generate = (functionPath: string) => generatedFunctions.push({ path: functionPath })
+
+  /** @type import('../../types/options/netlify_plugin_functions_util.js').NetlifyPluginFunctionsUtil */
+  return { add, list, listAll, generate }
+}
+
+const getStatusUtils = function (runState) {
+  return { show: show.bind(undefined, runState) }
+}

packages/build/src/plugins/child/utils.ts

@@ -4,37 +4,47 @@ import { logFailPluginWarning } from '../log/messages/plugins.js'
 // Stop build.
 // As opposed to throwing an error directly or to uncaught exceptions, this is
 // displayed as a user error, not an implementation error.
-export const failBuild = function (message, opts) {
+export const failBuild = function (message: string, opts?: { error: Error; errorMetadata: string[] }) {
   throw normalizeError('failBuild', failBuild, message, opts)
 }
 
 // Stop plugin. Same as `failBuild` but only stops plugin not whole build
-export const failPlugin = function (message, opts) {
+export const failPlugin = function (message: string, opts?: { error: Error; errorMetadata: string[] }) {
   throw normalizeError('failPlugin', failPlugin, message, opts)
 }
 
 // Cancel build. Same as `failBuild` except it marks the build as "canceled".
-export const cancelBuild = function (message, opts) {
+export const cancelBuild = function (message: string, opts?: { error: Error; errorMetadata: string[] }) {
   throw normalizeError('cancelBuild', cancelBuild, message, opts)
 }
 
 // `onSuccess`, `onEnd` and `onError` cannot cancel the build since they are run
 // or might be run after deployment. When calling `failBuild()` or
 // `cancelBuild()`, `failPlugin()` is run instead and a warning is printed.
-export const failPluginWithWarning = function (methodName, event, message, opts) {
+export const failPluginWithWarning = function (
+  methodName: string,
+  event: string,
+  message: string,
+  opts: { error: Error; errorMetadata: string[] },
+) {
   logFailPluginWarning(methodName, event)
   failPlugin(message, opts)
 }
 
 // An `error` option can be passed to keep the original error message and
 // stack trace. An additional `message` string is always required.
-const normalizeError = function (type, func, message, { error, errorMetadata } = {}) {
+const normalizeError = function (
+  type: string,
+  func: (...args: unknown[]) => unknown,
+  message: string,
+  { error, errorMetadata }: { error?: Error; errorMetadata?: string[] } = {},
+) {
   const errorA = getError(error, message, func)
   addErrorInfo(errorA, { type, errorMetadata })
   return errorA
 }
 
-const getError = function (error, message, func) {
+const getError = function (error: unknown, message: string, func: (...args: unknown[]) => unknown) {
   if (error instanceof Error) {
     // This might fail if `name` is a getter or is non-writable.
     try {