diff --git a/autodeploy/docker/Dockerfile b/autodeploy/docker/Dockerfile new file mode 100644 index 0000000000..7608f65aba --- /dev/null +++ b/autodeploy/docker/Dockerfile @@ -0,0 +1,13 @@ +FROM alpine:3.12 + +RUN apk update && \ + apk add --no-cache jq bash curl + +RUN mkdir -p /usr/autodeploy/volume +WORKDIR /usr/autodeploy + +COPY autodeploy.sh . +RUN chmod +x autodeploy.sh +WORKDIR /usr/autodeploy/volume + +ENTRYPOINT ["bash", "../autodeploy.sh"] diff --git a/autodeploy/docker/autodeploy.sh b/autodeploy/docker/autodeploy.sh new file mode 100755 index 0000000000..cf7d6105f5 --- /dev/null +++ b/autodeploy/docker/autodeploy.sh @@ -0,0 +1,253 @@ +#!/usr/bin/env sh + +API_TOKEN='fab1d1a2ace51f11ccb29246a3a1d99143908ceb' +SILO_TOKEN='eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoiYWRtaW4iLCJ1c2VybmFtZSI6ImFkbWluIn0.OAK7h0FKjwpoarCJdiuQ9q2zKW7D4KiyseyQLwfO5A8' +CLUSTER="oarfish" + +#+------------------------------------------------------ +#| NAME: validateJSON() +#| PARAMETERS: url_snip, json_file +#+------------------------------------------------------ +validateJSON() { + RESPONSE="$(curl -skLX POST \ + -H "Cookie: csrftoken=${CSRF_TOKEN}" \ + -H "Authorization: TOKEN ${API_TOKEN}" \ + -H "Content-Type: application/json" -T $2 \ + https://scale.${CLUSTER}.aisohio.net/api/v7/$1/validation)" + if [[ $(echo "${RESPONSE}" | jq '.is_valid') = "true" ]]; then + if [[ $(echo "${RESPONSE}" | jq '.diff | length') = 0 ]]; then + echo "Autodeploy: $1: '$2' is valid" + else + echo "Autodeploy: $1: '$2' is valid with a diff" + #echo "${RESPONSE}" | jq '.diff' + fi + else + echo "Error: $1: '$2' is not valid" + echo "${RESPONSE}" | jq '.' + exit 1 + fi +} + +#+------------------------------------------------------ +#| NAME: postJSON() +#| PARAMETERS: url_snip, json_file +#+------------------------------------------------------ +postJSON() { + RESPONSE=$(curl -skLX GET \ + -H "Cookie: csrftoken=${CSRF_TOKEN}" \ + -H "Authorization: TOKEN ${API_TOKEN}" \ + https://scale.${CLUSTER}.aisohio.net/api/v7/$1) + + case $1 in + workspaces|recipe-types|strikes) + NAME=$(jq '.name' $2);; + job-types) + NAME=$(jq '.manifest.job.name' $2) + VERSION=$(jq '.manifest.job.jobVersion' $2);; + esac + + if [[ $(echo "${RESPONSE}" | jq "has(\"results\")") = "true" ]]; then + if [[ $(echo "${RESPONSE}" | jq ".results[] | select(.name == "${NAME}") | has(\"id\")") = "true" ]]; then + echo "Autodeploy: $1: '$2' is a duplicate" + case $1 in + workspaces|strikes) + ID=$(echo "${RESPONSE}" | jq ".results[] | select(.name == "${NAME}") | .id") + URL=$(echo "https://scale.${CLUSTER}.aisohio.net/api/v7/$1/${ID}" | sed 's/"//g');; + job-types) + URL=$(echo "https://scale.${CLUSTER}.aisohio.net/api/v7/$1/${NAME}/${VERSION}" | sed 's/"//g');; + recipe-types) + URL=$(echo "https://scale.${CLUSTER}.aisohio.net/api/v7/$1/${NAME}" | sed 's/"//g');; + esac + + RESPONSE=$(curl -skLX PATCH \ + -H "Cookie: csrftoken=${CSRF_TOKEN}" \ + -H "Authorization: TOKEN ${API_TOKEN}" \ + -H "Content-Type: application/json" -T $2 \ + ${URL}) + if [[ $(echo "${RESPONSE}" | jq '.is_valid') != "true" ]]; then + if [ "${RESPONSE}" != "" ]; then + echo "Error: $1: '$2' was not able to PATCH" + echo "${RESPONSE}" + exit 1 + fi + fi + echo "Autodeploy: $1: '$2' PATCH was successful" + RESPONSE=$(curl -skLX GET \ + -H "Cookie: csrftoken=${CSRF_TOKEN}" \ + -H "Authorization: TOKEN ${API_TOKEN}" \ + ${URL}) + else + echo "Autodeploy: $1: '$2' is original" + RESPONSE=$(curl -skLX POST \ + -H "Cookie: csrftoken=${CSRF_TOKEN}" \ + -H "Authorization: TOKEN ${API_TOKEN}" \ + -H "Content-Type: application/json" -T $2 \ + https://scale.${CLUSTER}.aisohio.net/api/v7/$1) + if [[ ! $(echo "${RESPONSE}" | jq 'has("id","name")') =~ "false" ]]; then + echo "Autodeploy: $1: '$2' POST was successful" + else + echo "Error: $1: '$2' was not able to POST" + echo "${RESPONSE}" | jq '.' + exit 1 + fi + fi + else + echo "Error: $1: Response expected 'results[]'" + echo "${RESPONSE}" + exit 1 + fi +} + +#+------------------------------------------------------ +#| NAME: checkDuplicates() +#| PARAMETERS: variable_checked, parameter +#+------------------------------------------------------ +checkDuplicates() { + if [[ -n $1 ]]; then + echo "Error: '$2' is a duplicate" + echo "Note: './autodeploy.sh --help' or './autodeploy.sh -h' for more information" + exit 1 + fi +} + +#+------------------------------------------------------ +#| NAME: checkFile() +#| PARAMETERS: file_name +#+------------------------------------------------------ +checkFile() { + if [[ ! -f $1 ]]; then + echo "Error: '$1' doesn't exists" + echo "Usage: ./autodeploy.sh $2 " + exit 1 + fi +} + +# Replace equal sign format +set -- $(echo "$@" | sed 's/[=]/ /g') + +# Argument Parsing +if [ $# == 0 ]; then + echo "Usage: ./autodeploy.sh [options...]" + echo "Note: './autodeploy.sh --help' or './autodeploy.sh -h' for more information" + exit 0 +fi + +while [[ "$#" -gt 0 ]]; do + OPTION=$1 + case $1 in + -a|--api-token) + checkDuplicates "${WORKSPACE}" "$1" + shift + API_TOKEN=$1 + echo "Autodeploy: api-token: '${API_TOKEN}'";; + -h|--help) + echo "Usage: ./autodeploy.sh [options...]" + echo " -a --api-token replace the default api token" + echo " -w, --workspace verifies and creates a workspace based on the json file provied" + echo " -j, --job-type verifies and creates a job type based on the json file provied" + echo " -r, --recipe-type verifies and creates a recipe type based on the json file provied" + echo " -s, --strike verifies and creates a strike based on the json file provied" + echo " --silo scans registries to update images" + echo "Note: short fromat not supported (ex. './autodeploy.sh -wjrs')" + exit 0;; + -w|--workspace) + checkDuplicates "${WORKSPACE}" "$1" + shift + checkFile "$1" "${OPTION}" + WORKSPACE="$1";; + -j|--job-type) + checkDuplicates "${JOB_TYPE}" "$1" + shift + checkFile "$1" "${OPTION}" + JOB_TYPE="$1";; + -r|--recipe-type) + checkDuplicates "${RECIPE_TYPE}" "$1" + shift + checkFile "$1" "${OPTION}" + RECIPE_TYPE="$1";; + -s|--strike) + checkDuplicates "${STRIKE}" "$1" + shift + checkFile "$1" "${OPTION}" + STRIKE="$1";; + --silo) + checkDuplicates "${SILO}" "$1" + SILO="true";; + *) + echo "Unkown parameter passed: $1" + exit 1;; + esac + shift +done + +# SILO +if [[ -n ${SILO} ]]; then + echo "Autodeploy: silo: scanning registries..." + curl -sk \ + -H "Authorization: Token ${SILO_TOKEN}" \ + -H "Content-Type: application/json" \ + https://scale-silo.${CLUSTER}.aisohio.net/registries/scan +fi + +# Get CSRF Token +CSRF_TOKEN=$(curl -skLX GET \ +https://scale.${CLUSTER}.aisohio.net/api/admin --cookie-jar - \ +| grep 'csrftoken' | sed 's/^.*csrftoken[[:space:]]]*//g' ) +echo "Autodeploy: csrf-token: '${CSRF_TOKEN}'" + +# Workspace +if [[ -n ${WORKSPACE} ]]; then + validateJSON "workspaces" ${WORKSPACE} + postJSON "workspaces" ${WORKSPACE} + + # Add Workspace Name to Job-type + if [[ -n ${JOB_TYPE} ]]; then + NAME_WORKSPACE="$(echo ${RESPONSE} | jq '.name')" + JOB_TYPE_EDIT="$(jq ".configuration.output_workspaces.default=${NAME_WORKSPACE}" ${JOB_TYPE})" && \ + echo "${JOB_TYPE_EDIT}" > "${JOB_TYPE}" + echo "Autodeploy: job-types: '${JOB_TYPE}' workspace updated to ${NAME_WORKSPACE}" + fi + + # Add Workspace Name to Strike + if [[ -n ${STRIKE} ]]; then + NAME_WORKSPACE="$(echo ${RESPONSE} | jq '.name')" + STRIKE_EDIT="$(jq ".configuration.workspace=${NAME_WORKSPACE}" ${STRIKE})" && \ + echo "${STRIKE_EDIT}" > "${STRIKE}" + echo "Autodeploy: strikes: '${STRIKE}' workspace updated to ${NAME_WORKSPACE}" + fi +fi + +# Job-Type +if [[ -n ${JOB_TYPE} ]]; then + validateJSON 'job-types' "${JOB_TYPE}" + postJSON 'job-types' "${JOB_TYPE}" + + # Add Job-Type to Recipe-type + if [[ -n ${RECIPE_TYPE} ]]; then + JOB_TYPE_JSON="$(echo ${RESPONSE} | jq '.')" + RECIPE_TYPE_EDIT="$(jq ".job_types=${JOB_TYPE_JSON}" ${RECIPE_TYPE})" && \ + echo "${RECIPE_TYPE_EDIT}" > "${RECIPE_TYPE}" + echo "Autodeploy: recipe-types: '${RECIPE_TYPE}' job-type updated to $(echo ${JOB_TYPE_JSON} | jq '.name')" + fi + +fi + +# Recipe-Type +if [[ -n ${RECIPE_TYPE} ]]; then + validateJSON "recipe-types" ${RECIPE_TYPE} + postJSON 'recipe-types' "${RECIPE_TYPE}" + + # Add Recipe Name to strike + if [[ -n ${STRIKE} ]]; then + NAME_RECIPE="$(echo ${RESPONSE} | jq '.name')" + STRIKE_EDIT="$(jq ".configuration.recipe.name=${NAME_RECIPE}" ${STRIKE})" && \ + echo "${STRIKE_EDIT}" > "${STRIKE}" + echo "Autodeploy: strikes: '${STRIKE}' recipe updated to ${NAME_RECIPE}" + fi +fi + +# Strike +if [[ -n ${STRIKE} ]]; then + validateJSON "strikes" "${STRIKE}" + postJSON 'strikes' "${STRIKE}" +fi diff --git a/autodeploy/run-autodeploy.sh b/autodeploy/run-autodeploy.sh new file mode 100755 index 0000000000..c027c36737 --- /dev/null +++ b/autodeploy/run-autodeploy.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env sh + +ARGS=$@ +docker run --rm -v "${PWD}:/usr/autodeploy/volume" mtalda/autodeploy ${ARGS} + \ No newline at end of file diff --git a/autodeploy/templates/job-type-template.json b/autodeploy/templates/job-type-template.json new file mode 100644 index 0000000000..5cc5184426 --- /dev/null +++ b/autodeploy/templates/job-type-template.json @@ -0,0 +1,32 @@ +{ + "id": null, + "name": "", + "version": "1.0.0", + "title": "", + "description": "<DESCIPTION>", + "is_active": true, + "is_system": false, + "is_paused": false, + "is_published": false, + "icon_code": "<ICON_CODE", + "unmet_resources": null, + "max_scheduled": null, + "max_tries": 3, + "revision_num": 1, + "docker_image": "docker.io/<URL>", + "created": null, + "deprecated": null, + "paused": null, + "last_modified": null, + "manifest": "<MANIFEST>", + "configuration": { + "mounts": {}, + "priority": 201, + "output_workspaces": { + "default": "<WORKSPACE>", + "outputs": {} + }, + "settings": {} + }, + "recipe_types": [] +} diff --git a/autodeploy/templates/recipe-type-template.json b/autodeploy/templates/recipe-type-template.json new file mode 100644 index 0000000000..6e701bdb05 --- /dev/null +++ b/autodeploy/templates/recipe-type-template.json @@ -0,0 +1,45 @@ +{ + "id": null, + "name": "<NAME>", + "title": "<TITLE>", + "description": "<DESCRIPTION>", + "revision_num": 1, + "is_active": true, + "is_system": false, + "created": null, + "deprecated": null, + "last_modified": null, + "definition": { + "input": { + "files": [ + { + "media_types": [], + "required": true, + "multiple": false, + "name": "File" + } + ], + "json": [] + }, + "nodes": { + "job": { + "input": { + "INPUT_FILE": { + "input": "File", + "type": "recipe" + } + }, + "node_type": { + "job_type_version": "1.0.0", + "node_type": "job", + "job_type_name": "<JOB_NAME>", + "job_type_revision": 1 + }, + "dependencies": [] + } + } + }, + "job_types": [], + "sub_recipe_types": [], + "super_recipe_types": [] +} diff --git a/autodeploy/templates/strike-template.json b/autodeploy/templates/strike-template.json new file mode 100644 index 0000000000..7a2f94cb9e --- /dev/null +++ b/autodeploy/templates/strike-template.json @@ -0,0 +1,41 @@ +{ + "id": null, + "name": "<NAME>", + "title": "<TITLE>", + "description": "<DESCRIPTION>", + "job": { + "id": 233235, + "job_type": { + "id": 7, + "name": "scale-strike", + "version": "1.0.0", + "title": "Scale Strike", + "description": "Monitors a directory for incoming source files to ingest", + "is_active": true, + "is_system": true, + "is_paused": false, + "is_published": false, + "icon_code": "f0e7", + "unmet_resources": null + }, + "status": "CANCELED" + }, + "created": null, + "last_modified": null, + "configuration": { + "recipe": { + "name": null + }, + "monitor": { + "transfer_suffix": "_tmp", + "type": "dir-watcher" + }, + "workspace": "<WORKSPACE>", + "files_to_ingest": [ + { + "data_types": [], + "filename_regex": ".*" + } + ] + } +} diff --git a/autodeploy/templates/workspace-template.json b/autodeploy/templates/workspace-template.json new file mode 100644 index 0000000000..89d2be7fbf --- /dev/null +++ b/autodeploy/templates/workspace-template.json @@ -0,0 +1,17 @@ +{ + "id": null, + "name": "<NAME>", + "title": "<TITLE>", + "description": "<DESCRIPTION>", + "base_url": null, + "is_active": true, + "created": null, + "deprecated": null, + "last_modified": null, + "configuration": { + "broker": { + "host_path": "/dfs/<PATH>", + "type": "host" + } + } +}