From bef864b9667ac5fd526dc60e2d66e9dd4cfa3796 Mon Sep 17 00:00:00 2001 From: Matt Talda Date: Tue, 4 Aug 2020 10:32:56 -0400 Subject: [PATCH 1/7] initial commit --- autodeploy/autodeploy.sh | 37 +++++++++++++++++++++++++++++++++++++ autodeploy/cookies.txt | 4 ++++ autodeploy/workspace.json | 18 ++++++++++++++++++ 3 files changed, 59 insertions(+) create mode 100755 autodeploy/autodeploy.sh create mode 100644 autodeploy/cookies.txt create mode 100644 autodeploy/workspace.json diff --git a/autodeploy/autodeploy.sh b/autodeploy/autodeploy.sh new file mode 100755 index 0000000000..ff42d44585 --- /dev/null +++ b/autodeploy/autodeploy.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env sh + +# Arguments +WORKSPACE=$1 +API_TOKEN="fab1d1a2ace51f11ccb29246a3a1d99143908ceb" +CLUSTER="oarfish" + +# Get CSRF Token +CSRF_TOKEN=$(curl -skLX GET \ +https://scale.${CLUSTER}.aisohio.net/api/admin --cookie-jar - \ +| grep 'csrftoken' | sed 's/^.*csrftoken[[:space:]]]*//g') +echo "CSRF TOKEN: ${CSRF_TOKEN}" + +# Validate Workspace +IS_VALID=$(curl -skLX POST \ +-H "Cookie: csrftoken=${CSRF_TOKEN}" \ +-H "Authorization: TOKEN ${API_TOKEN}" \ +-H "Content-Type: application/json" -T ${WORKSPACE} \ +https://scale.oarfish.aisohio.net/api/v7/workspaces/validation) + +# Check for Validation +if [ "${IS_VALID}" = '{"is_valid":true,"errors":[],"warnings":[]}' ]; then + echo "LOG> Workspace valid!" +elif [ "${IS_VALID}" = '{"detail":"Missing required parameter: \"configuration\""}' ]; then + echo "LOG> Workspace invalid!" + exit 1 +else + echo "LOG> Unkown error during workspace validation!" + exit 2 +fi + +# Create Workspace +curl -skLX POST \ +-H "Cookie: csrftoken=${CSRF_TOKEN}" \ +-H "Authorization: TOKEN ${API_TOKEN}" \ +-H "Content-Type: application/json" -T ${WORKSPACE} \ +https://scale.oarfish.aisohio.net/api/v7/workspaces \ No newline at end of file diff --git a/autodeploy/cookies.txt b/autodeploy/cookies.txt new file mode 100644 index 0000000000..150e9f9c93 --- /dev/null +++ b/autodeploy/cookies.txt @@ -0,0 +1,4 @@ +# Netscape HTTP Cookie File +# https://curl.haxx.se/docs/http-cookies.html +# This file was generated by libcurl! Edit at your own risk. + diff --git a/autodeploy/workspace.json b/autodeploy/workspace.json new file mode 100644 index 0000000000..d4eb8ed94f --- /dev/null +++ b/autodeploy/workspace.json @@ -0,0 +1,18 @@ +{ + + "id": null, + "name": "auto-deploy-workspace", + "title": "Auto Deploy Workspace", + "description": "A Test Workspace created by a script", + "base_url": null, + "is_active": true, + "created": null, + "deprecated": null, + "last_modified": null, + "configuration": { + "broker": { + "host_path": "/dfs/testing/auto-deploy", + "type": "host" + } + } +} \ No newline at end of file From 9473dfa4f4b2658df2727f78c68902ba35976e63 Mon Sep 17 00:00:00 2001 From: Matt Talda Date: Wed, 5 Aug 2020 11:52:09 -0400 Subject: [PATCH 2/7] validation checks added --- autodeploy/autodeploy.sh | 76 ++++++++++++++------- autodeploy/cookies.txt | 4 -- autodeploy/job-type.json | 53 ++++++++++++++ autodeploy/recipe-type.json | 133 ++++++++++++++++++++++++++++++++++++ autodeploy/strike.json | 41 +++++++++++ autodeploy/test.sh | 55 +++++++++++++++ 6 files changed, 334 insertions(+), 28 deletions(-) delete mode 100644 autodeploy/cookies.txt create mode 100644 autodeploy/job-type.json create mode 100644 autodeploy/recipe-type.json create mode 100644 autodeploy/strike.json create mode 100755 autodeploy/test.sh diff --git a/autodeploy/autodeploy.sh b/autodeploy/autodeploy.sh index ff42d44585..5aacb2af0c 100755 --- a/autodeploy/autodeploy.sh +++ b/autodeploy/autodeploy.sh @@ -1,7 +1,36 @@ #!/usr/bin/env sh +validateJSON() { + # Parameters + local url_snip=$1 + local json_file=$2 + + # Validate Workspace + IS_VALID=$(curl -skLX POST \ + -H "Cookie: csrftoken=${CSRF_TOKEN}" \ + -H "Authorization: TOKEN ${API_TOKEN}" \ + -H "Content-Type: application/json" -T $json_file \ + https://scale.oarfish.aisohio.net/api/v7/$url_snip/validation) + + # Check for Validation + if [ "${IS_VALID}" = '{"is_valid":true,"errors":[],"warnings":[]}' ]; then + echo "LOG> $url_snip valid!" + return 0 + elif [ "${IS_VALID}" = '{"diff":{},"is_valid":true,"errors":[],"warnings":[]}' ]; then + echo "LOG> $url_snip valid and diff is empty!" + return 0 + else + echo ${IS_VALID} + echo "LOG> $url_snip invalid!" + exit 1 + fi +} + # Arguments WORKSPACE=$1 +JOBTYPE=$2 +RECIPETYPE=$3 +STRIKE=$4 API_TOKEN="fab1d1a2ace51f11ccb29246a3a1d99143908ceb" CLUSTER="oarfish" @@ -9,29 +38,28 @@ CLUSTER="oarfish" CSRF_TOKEN=$(curl -skLX GET \ https://scale.${CLUSTER}.aisohio.net/api/admin --cookie-jar - \ | grep 'csrftoken' | sed 's/^.*csrftoken[[:space:]]]*//g') -echo "CSRF TOKEN: ${CSRF_TOKEN}" - -# Validate Workspace -IS_VALID=$(curl -skLX POST \ --H "Cookie: csrftoken=${CSRF_TOKEN}" \ --H "Authorization: TOKEN ${API_TOKEN}" \ --H "Content-Type: application/json" -T ${WORKSPACE} \ -https://scale.oarfish.aisohio.net/api/v7/workspaces/validation) - -# Check for Validation -if [ "${IS_VALID}" = '{"is_valid":true,"errors":[],"warnings":[]}' ]; then - echo "LOG> Workspace valid!" -elif [ "${IS_VALID}" = '{"detail":"Missing required parameter: \"configuration\""}' ]; then - echo "LOG> Workspace invalid!" - exit 1 -else - echo "LOG> Unkown error during workspace validation!" - exit 2 -fi +echo "LOG> CSRF TOKEN: ${CSRF_TOKEN}" + +# Validate +validateJSON "workspaces" ${WORKSPACE} +validateJSON "job-types" ${JOBTYPE} +validateJSON "recipe-types" ${RECIPETYPE} +validateJSON "strikes" ${STRIKE} + + + + + +# curl -skLX GET \ +# -H "Cookie: csrftoken=${CSRF_TOKEN}" \ +# -H "Authorization: TOKEN ${API_TOKEN}" \ +# https://scale.oarfish.aisohio.net/api/v7/jobs + + # Create Workspace -curl -skLX POST \ --H "Cookie: csrftoken=${CSRF_TOKEN}" \ --H "Authorization: TOKEN ${API_TOKEN}" \ --H "Content-Type: application/json" -T ${WORKSPACE} \ -https://scale.oarfish.aisohio.net/api/v7/workspaces \ No newline at end of file +# curl -skLX POST \ +# -H "Cookie: csrftoken=${CSRF_TOKEN}" \ +# -H "Authorization: TOKEN ${API_TOKEN}" \ +# -H "Content-Type: application/json" -T ${WORKSPACE} \ +# https://scale.oarfish.aisohio.net/api/v7/workspaces \ No newline at end of file diff --git a/autodeploy/cookies.txt b/autodeploy/cookies.txt deleted file mode 100644 index 150e9f9c93..0000000000 --- a/autodeploy/cookies.txt +++ /dev/null @@ -1,4 +0,0 @@ -# Netscape HTTP Cookie File -# https://curl.haxx.se/docs/http-cookies.html -# This file was generated by libcurl! Edit at your own risk. - diff --git a/autodeploy/job-type.json b/autodeploy/job-type.json new file mode 100644 index 0000000000..f2765638f2 --- /dev/null +++ b/autodeploy/job-type.json @@ -0,0 +1,53 @@ +{ + "icon_code": "f236", + "docker_image": "docker.io/fizz11/simplesleeper-1.0.0-seed:1.0.0", + "manifest": { + "seedVersion": "1.0.0", + "job": { + "name": "simplesleeper", + "jobVersion": "1.0.0", + "packageVersion": "1.0.0", + "title": "Simple Sleeper", + "description": "Logs its filename and sleeps for 5 seconds", + "maintainer": { + "name": "AIS", + "organization": "Applied Information Sciences", + "email": "n/a" + }, + "timeout": 3600, + "interface": { + "command": "", + "inputs": { + "files": [{ + "name": "INPUT_FILE", + "multiple": false, + "partial": false, + "required": true + }] + }, + "outputs": {} + }, + "resources": { + "scalar": [{ + "name": "cpus", + "value": 0.1, + "inputMultiplier": 0 + }, { + "name": "mem", + "value": 32, + "inputMultiplier": 0 + }] + } + } + }, + "configuration": { + "output_workspaces": { + "default": "auto-deploy-workspace-b241736dad5a48ff8c8" + }, + "priority": 201 + }, + "is_published": false, + "is_active": true, + "is_paused": false, + "max_scheduled": null +} \ No newline at end of file diff --git a/autodeploy/recipe-type.json b/autodeploy/recipe-type.json new file mode 100644 index 0000000000..f13a9f5cac --- /dev/null +++ b/autodeploy/recipe-type.json @@ -0,0 +1,133 @@ +{ + "id": null, + "name": "simple-sleeper-autodeploy", + "title": "Simple Sleeper Autodeploy", + "description": null, + "revision_num": 1, + "is_active": true, + "is_system": false, + "created": null, + "deprecated": null, + "last_modified": null, + "definition": { + "input": { + "files": [ + { + "media_types": [], + "required": true, + "multiple": false, + "name": "File" + } + ], + "json": [] + }, + "nodes": { + "simplesleeper": { + "input": { + "INPUT_FILE": { + "input": "File", + "type": "recipe" + } + }, + "node_type": { + "job_type_version": "1.0.0", + "node_type": "job", + "job_type_name": "simplesleeper", + "job_type_revision": 1 + }, + "dependencies": [] + } + } + }, + "job_types": [ + { + "id": 23, + "name": "simplesleeper", + "version": "1.0.0", + "title": "Simple Sleeper", + "description": "Logs its filename and sleeps for 5 seconds", + "is_active": true, + "is_system": false, + "is_paused": false, + "is_published": false, + "icon_code": "f236", + "unmet_resources": null, + "max_scheduled": null, + "max_tries": 3, + "revision_num": 1, + "docker_image": "docker.io/fizz11/simplesleeper-1.0.0-seed:1.0.0", + "created": "2020-07-27T19:41:02.119907Z", + "deprecated": null, + "paused": null, + "last_modified": "2020-07-27T19:41:02.119978Z", + "manifest": { + "job": { + "maintainer": { + "organization": "Applied Information Sciences", + "name": "AIS", + "email": "n/a" + }, + "jobVersion": "1.0.0", + "title": "Simple Sleeper", + "description": "Logs its filename and sleeps for 5 seconds", + "timeout": 3600, + "interface": { + "inputs": { + "files": [ + { + "multiple": false, + "required": true, + "partial": false, + "name": "INPUT_FILE", + "mediaTypes": [] + } + ] + } + }, + "packageVersion": "1.0.0", + "resources": { + "scalar": [ + { + "inputMultiplier": 0, + "name": "cpus", + "value": 0.1 + }, + { + "inputMultiplier": 0, + "name": "mem", + "value": 32 + } + ] + }, + "name": "simplesleeper" + }, + "seedVersion": "1.0.0" + }, + "configuration": { + "mounts": {}, + "priority": 201, + "output_workspaces": { + "default": "default-output", + "outputs": {} + }, + "settings": {} + }, + "recipe_types": [ + { + "id": 11, + "name": "simple-sleeper", + "title": "Simple Sleeper", + "description": null, + "revision_num": 1, + "is_active": true, + "is_system": false, + "created": "2020-07-27T19:41:35.621061Z", + "deprecated": null, + "last_modified": "2020-07-27T19:41:35.621094Z" + } + ] + } + ], + "sub_recipe_types": [], + "super_recipe_types": [] +} \ No newline at end of file diff --git a/autodeploy/strike.json b/autodeploy/strike.json new file mode 100644 index 0000000000..aac8f84f84 --- /dev/null +++ b/autodeploy/strike.json @@ -0,0 +1,41 @@ +{ + "id": 5, + "name": "simple-sleeper-strike", + "title": "Simple Sleeper Strike", + "description": null, + "job": { + "id": 233235, + "job_type": { + "id": 7, + "name": "scale-strike", + "version": "1.0.0", + "title": "Scale Strike", + "description": "Monitors a directory for incoming source files to ingest", + "is_active": true, + "is_system": true, + "is_paused": false, + "is_published": false, + "icon_code": "f0e7", + "unmet_resources": null + }, + "status": "CANCELED" + }, + "created": "2020-07-27T19:42:05.448704Z", + "last_modified": "2020-07-27T19:42:05.595187Z", + "configuration": { + "recipe": { + "name": "simple-sleeper" + }, + "monitor": { + "transfer_suffix": "_tmp", + "type": "dir-watcher" + }, + "workspace": "sleeper-input-workspace", + "files_to_ingest": [ + { + "data_types": [], + "filename_regex": ".*" + } + ] + } +} \ No newline at end of file diff --git a/autodeploy/test.sh b/autodeploy/test.sh new file mode 100755 index 0000000000..2771532c99 --- /dev/null +++ b/autodeploy/test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env sh + +args=$@ +ARGS_ARRAY=$(echo ${args} | sed 's/[=]/ /g') + +for i in ${ARGS_ARRAY}; do + echo ${ARGS_ARRAY} + case $i in + -h|--help) + echo "Workspace parameter passed: $i" + exit 0;; + -w|--workspace) + echo "Workspace parameter passed: $i" + exit 0;; + *) + echo "Unkown parameter passed: $i" + exit 1;; + esac + shift +done + + +# while [[ "$#" -gt 0 ]]; do +# case $1 in +# -d|--deploy) deploy="$2"; shift ;; +# -u|--uglify) uglify=1 ;; +# *) echo "Unknown parameter passed: $1"; exit 1 ;; +# esac +# shift +# done + +# for i in "$@" +# do +# case $i in +# -e=*|--extension=*) +# EXTENSION="${i#*=}" +# shift # past argument=value +# ;; +# -s=*|--searchpath=*) +# SEARCHPATH="${i#*=}" +# shift # past argument=value +# ;; +# -l=*|--lib=*) +# LIBPATH="${i#*=}" +# shift # past argument=value +# ;; +# --default) +# DEFAULT=YES +# shift # past argument with no value +# ;; +# *) +# # unknown option +# ;; +# esac +# done \ No newline at end of file From 4dc05d6bca14df67ba2052715aa38700dbed367e Mon Sep 17 00:00:00 2001 From: Matt Talda Date: Wed, 5 Aug 2020 17:21:21 -0400 Subject: [PATCH 3/7] added option args and handling --- autodeploy/autodeploy.sh | 10 +++- autodeploy/test.sh | 107 ++++++++++++++++++++++++++++++++++---- autodeploy/workspace.json | 1 - 3 files changed, 105 insertions(+), 13 deletions(-) diff --git a/autodeploy/autodeploy.sh b/autodeploy/autodeploy.sh index 5aacb2af0c..cc38f5c9ee 100755 --- a/autodeploy/autodeploy.sh +++ b/autodeploy/autodeploy.sh @@ -46,8 +46,14 @@ validateJSON "job-types" ${JOBTYPE} validateJSON "recipe-types" ${RECIPETYPE} validateJSON "strikes" ${STRIKE} - - +# Find Workspace (not 100%) +# WORKSPACE_NAME=$(grep '"name":' ${WORKSPACE} | awk -F: '{print $2}' | sed 's/[[:space:]",]//g') +# WORKSPACE_LIST=$(curl -skLX GET \ +# -H "Cookie: csrftoken=${CSRF_TOKEN}" \ +# -H "Authorization: TOKEN ${API_TOKEN}" \ +# https://scale.oarfish.aisohio.net/api/v7/workspaces | +# grep "${WORKSPACE_NAME}") +# echo ${WORKSPACE_LIST} # curl -skLX GET \ diff --git a/autodeploy/test.sh b/autodeploy/test.sh index 2771532c99..14711d2141 100755 --- a/autodeploy/test.sh +++ b/autodeploy/test.sh @@ -1,23 +1,110 @@ #!/usr/bin/env sh -args=$@ -ARGS_ARRAY=$(echo ${args} | sed 's/[=]/ /g') +API_TOKEN="fab1d1a2ace51f11ccb29246a3a1d99143908ceb" +CLUSTER="oarfish" -for i in ${ARGS_ARRAY}; do - echo ${ARGS_ARRAY} - case $i in +#+------------------------------------------------------ +#| NAME: validateJSON() +#| PARAMETERS: url_snip, json_file +#+------------------------------------------------------ +validateJSON() { + # Validate Workspace + IS_VALID=$(curl -skLX POST \ + -H "Cookie: csrftoken=${CSRF_TOKEN}" \ + -H "Authorization: TOKEN ${API_TOKEN}" \ + -H "Content-Type: application/json" -T $2 \ + https://scale.oarfish.aisohio.net/api/v7/$1/validation) + + # Check for Validation + if [ "${IS_VALID}" = '{"is_valid":true,"errors":[],"warnings":[]}' ]; then + echo "LOG> $1 valid!" + return 0 + elif [ "${IS_VALID}" = '{"diff":{},"is_valid":true,"errors":[],"warnings":[]}' ]; then + echo "LOG> $1 valid and diff is empty!" + return 0 + else + echo ${IS_VALID} + echo "LOG> $1 invalid!" + exit 1 + fi +} + +#+------------------------------------------------------ +#| NAME: checkDuplicates() +#| PARAMETERS: variable_checked, parameter +#+------------------------------------------------------ +checkDuplicates() { + if [[ -n $1 ]]; then + echo "Duplicate parameter \"$2\"" + exit 1 + fi +} + +#+------------------------------------------------------ +#| NAME: checkFile() +#| PARAMETERS: file_name +#+------------------------------------------------------ +checkFile() { + if [[ ! -f $1 ]]; then + echo "\"$1\" doesn't exists" + exit 2 + fi +} + +# Replace equal sign format +set -- $(echo "$@" | sed 's/[=]/ /g') + +# Argument Parsing +while [[ "$#" -gt 0 ]]; do + case $1 in -h|--help) - echo "Workspace parameter passed: $i" + echo "Usage: ./autodeploy.sh [options...]" + echo " -w, --workspace verifies and creates a workspace based on the json file provied" + echo " -j, --job-type verifies and creates a job type based on the json file provied" + echo " -r, --recipe-type verifies and creates a recipe type based on the json file provied" + echo " -s, --strike verifies and creates a strike based on the json file provied" + echo 'Note: short fromat not supported (ex. % ./autodeploy.sh -wjrs)' exit 0;; -w|--workspace) - echo "Workspace parameter passed: $i" - exit 0;; + checkDuplicates "${WORKSPACE}" "-w" + shift + checkFile "$1" + WORKSPACE="$1";; + -j|--job-type) + checkDuplicates "${JOB_TYPE}" "-j" + shift + checkFile "$1" + JOB_TYPE="$1";; + -r|--recipe-type) + checkDuplicates "${RECIPE_TYPE}" "-r" + shift + checkFile "$1" + RECIPE_TYPE="$1";; + -s|--strike) + checkDuplicates "${STRIKE}" "-s" + shift + checkFile "$1" + STRIKE="$1";; *) - echo "Unkown parameter passed: $i" - exit 1;; + echo "Unkown parameter passed: $1";; esac shift done + +# Get CSRF Token +CSRF_TOKEN=$(curl -skLX GET \ +https://scale.${CLUSTER}.aisohio.net/api/admin --cookie-jar - \ +| grep 'csrftoken' | sed 's/^.*csrftoken[[:space:]]]*//g') +echo "LOG> CSRF TOKEN: ${CSRF_TOKEN}" + +# Workspace +if [[ -n ${WORKSPACE} ]]; then + validateJSON "workspaces" ${WORKSPACE} + + + +fi + # while [[ "$#" -gt 0 ]]; do diff --git a/autodeploy/workspace.json b/autodeploy/workspace.json index d4eb8ed94f..a1660946d8 100644 --- a/autodeploy/workspace.json +++ b/autodeploy/workspace.json @@ -1,5 +1,4 @@ { - "id": null, "name": "auto-deploy-workspace", "title": "Auto Deploy Workspace", From 874e6cb7e23db145542235ae4824c30bddbc03e9 Mon Sep 17 00:00:00 2001 From: Matt Talda Date: Fri, 21 Aug 2020 14:50:24 -0400 Subject: [PATCH 4/7] added templates, dockerfile, and updated autodeploy.sh --- autodeploy/Dockerfile | 12 + autodeploy/autodeploy.sh | 294 +++++++++++++--- autodeploy/input.txt | 1 + autodeploy/job-type.json | 107 +++--- autodeploy/recipe-type.json | 250 +++++++------- autodeploy/strike.json | 78 ++--- autodeploy/templates/job-type-template.json | 76 +++++ .../templates/recipe-type-template.json | 45 +++ autodeploy/templates/strike-template.json | 41 +++ autodeploy/templates/workspace-template.json | 17 + autodeploy/test.json | 76 +++++ autodeploy/test.sh | 321 ++++++++++++++---- autodeploy/workspace.json | 8 +- 13 files changed, 989 insertions(+), 337 deletions(-) create mode 100644 autodeploy/Dockerfile create mode 100644 autodeploy/input.txt create mode 100644 autodeploy/templates/job-type-template.json create mode 100644 autodeploy/templates/recipe-type-template.json create mode 100644 autodeploy/templates/strike-template.json create mode 100644 autodeploy/templates/workspace-template.json create mode 100644 autodeploy/test.json diff --git a/autodeploy/Dockerfile b/autodeploy/Dockerfile new file mode 100644 index 0000000000..1eb319b5de --- /dev/null +++ b/autodeploy/Dockerfile @@ -0,0 +1,12 @@ +FROM alpine:3.12 + +RUN apk update && \ + apk add --no-cache jq bash curl + +RUN mkdir -p /usr/autodeploy +WORKDIR /usr/autodeploy + +COPY autodeploy.sh /usr/autodeploy +RUN chmod +x autodeploy.sh + +ENTRYPOINT ["bash", "autodeploy.sh"] \ No newline at end of file diff --git a/autodeploy/autodeploy.sh b/autodeploy/autodeploy.sh index cc38f5c9ee..e99c31eb44 100755 --- a/autodeploy/autodeploy.sh +++ b/autodeploy/autodeploy.sh @@ -1,71 +1,253 @@ #!/usr/bin/env sh +API_TOKEN='fab1d1a2ace51f11ccb29246a3a1d99143908ceb' +SILO_TOKEN='eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoiYWRtaW4iLCJ1c2VybmFtZSI6ImFkbWluIn0.OAK7h0FKjwpoarCJdiuQ9q2zKW7D4KiyseyQLwfO5A8' +CLUSTER="oarfish" + +#+------------------------------------------------------ +#| NAME: validateJSON() +#| PARAMETERS: url_snip, json_file +#+------------------------------------------------------ validateJSON() { - # Parameters - local url_snip=$1 - local json_file=$2 + RESPONSE="$(curl -skLX POST \ + -H "Cookie: csrftoken=${CSRF_TOKEN}" \ + -H "Authorization: TOKEN ${API_TOKEN}" \ + -H "Content-Type: application/json" -T $2 \ + https://scale.${CLUSTER}.aisohio.net/api/v7/$1/validation)" + if [[ $(echo "${RESPONSE}" | jq '.is_valid') = "true" ]]; then + if [[ $(echo "${RESPONSE}" | jq '.diff | length') = 0 ]]; then + echo "Autodeploy: $1: '$2' is valid" + else + echo "Autodeploy: $1: '$2' is valid with a diff" + #echo "${RESPONSE}" | jq '.diff' + fi + else + echo "Error: $1: '$2' is not valid" + echo "${RESPONSE}" | jq '.' + exit 1 + fi +} - # Validate Workspace - IS_VALID=$(curl -skLX POST \ +#+------------------------------------------------------ +#| NAME: postJSON() +#| PARAMETERS: url_snip, json_file +#+------------------------------------------------------ +postJSON() { + RESPONSE=$(curl -skLX GET \ -H "Cookie: csrftoken=${CSRF_TOKEN}" \ -H "Authorization: TOKEN ${API_TOKEN}" \ - -H "Content-Type: application/json" -T $json_file \ - https://scale.oarfish.aisohio.net/api/v7/$url_snip/validation) - - # Check for Validation - if [ "${IS_VALID}" = '{"is_valid":true,"errors":[],"warnings":[]}' ]; then - echo "LOG> $url_snip valid!" - return 0 - elif [ "${IS_VALID}" = '{"diff":{},"is_valid":true,"errors":[],"warnings":[]}' ]; then - echo "LOG> $url_snip valid and diff is empty!" - return 0 + https://scale.${CLUSTER}.aisohio.net/api/v7/$1) + + case $1 in + workspaces|recipe-types|strikes) + NAME=$(jq '.name' $2);; + job-types) + NAME=$(jq '.manifest.job.name' $2) + VERSION=$(jq '.manifest.job.jobVersion' $2);; + esac + + if [[ $(echo "${RESPONSE}" | jq "has(\"results\")") = "true" ]]; then + if [[ $(echo "${RESPONSE}" | jq ".results[] | select(.name == "${NAME}") | has(\"id\")") = "true" ]]; then + echo "Autodeploy: $1: '$2' is a duplicate" + case $1 in + workspaces|strikes) + ID=$(echo "${RESPONSE}" | jq ".results[] | select(.name == "${NAME}") | .id") + URL=$(echo "https://scale.${CLUSTER}.aisohio.net/api/v7/$1/${ID}" | sed 's/"//g');; + job-types) + URL=$(echo "https://scale.${CLUSTER}.aisohio.net/api/v7/$1/${NAME}/${VERSION}" | sed 's/"//g');; + recipe-types) + URL=$(echo "https://scale.${CLUSTER}.aisohio.net/api/v7/$1/${NAME}" | sed 's/"//g');; + esac + + RESPONSE=$(curl -skLX PATCH \ + -H "Cookie: csrftoken=${CSRF_TOKEN}" \ + -H "Authorization: TOKEN ${API_TOKEN}" \ + -H "Content-Type: application/json" -T $2 \ + ${URL}) + if [[ $(echo "${RESPONSE}" | jq '.is_valid') != "true" ]]; then + if [ "${RESPONSE}" != "" ]; then + echo "Error: $1: '$2' was not able to PATCH" + echo "${RESPONSE}" + exit 1 + fi + fi + echo "Autodeploy: $1: '$2' PATCH was successful" + RESPONSE=$(curl -skLX GET \ + -H "Cookie: csrftoken=${CSRF_TOKEN}" \ + -H "Authorization: TOKEN ${API_TOKEN}" \ + ${URL}) + else + echo "Autodeploy: $1: '$2' is original" + RESPONSE=$(curl -skLX POST \ + -H "Cookie: csrftoken=${CSRF_TOKEN}" \ + -H "Authorization: TOKEN ${API_TOKEN}" \ + -H "Content-Type: application/json" -T $2 \ + https://scale.${CLUSTER}.aisohio.net/api/v7/$1) + if [[ ! $(echo "${RESPONSE}" | jq 'has("id","name")') =~ "false" ]]; then + echo "Autodeploy: $1: '$2' POST was successful" + else + echo "Error: $1: '$2' was not able to POST" + echo "${RESPONSE}" | jq '.' + exit 1 + fi + fi else - echo ${IS_VALID} - echo "LOG> $url_snip invalid!" + echo "Error: $1: Response expected 'results[]'" + echo "${RESPONSE}" exit 1 fi } -# Arguments -WORKSPACE=$1 -JOBTYPE=$2 -RECIPETYPE=$3 -STRIKE=$4 -API_TOKEN="fab1d1a2ace51f11ccb29246a3a1d99143908ceb" -CLUSTER="oarfish" +#+------------------------------------------------------ +#| NAME: checkDuplicates() +#| PARAMETERS: variable_checked, parameter +#+------------------------------------------------------ +checkDuplicates() { + if [[ -n $1 ]]; then + echo "Error: '$2' is a duplicate" + echo "Note: './autodeploy.sh --help' or './autodeploy.sh -h' for more information" + exit 1 + fi +} + +#+------------------------------------------------------ +#| NAME: checkFile() +#| PARAMETERS: file_name +#+------------------------------------------------------ +checkFile() { + if [[ ! -f $1 ]]; then + echo "Error: '$1' doesn't exists" + echo "Usage: ./autodeploy.sh $2 " + exit 1 + fi +} + +# Replace equal sign format +set -- $(echo "$@" | sed 's/[=]/ /g') + +# Argument Parsing +if [ $# == 0 ]; then + echo "Usage: ./autodeploy.sh [options...]" + echo "Note: './autodeploy.sh --help' or './autodeploy.sh -h' for more information" + exit 0 +fi + +while [[ "$#" -gt 0 ]]; do + OPTION=$1 + case $1 in + -a|--api-token) + checkDuplicates "${WORKSPACE}" "$1" + shift + API_TOKEN=$1 + echo "Autodeploy: api-token: '${API_TOKEN}'";; + -h|--help) + echo "Usage: ./autodeploy.sh [options...]" + echo " -a --api-token replace the default api token" + echo " -w, --workspace verifies and creates a workspace based on the json file provied" + echo " -j, --job-type verifies and creates a job type based on the json file provied" + echo " -r, --recipe-type verifies and creates a recipe type based on the json file provied" + echo " -s, --strike verifies and creates a strike based on the json file provied" + echo " --silo scans registries to update images" + echo "Note: short fromat not supported (ex. './autodeploy.sh -wjrs')" + exit 0;; + -w|--workspace) + checkDuplicates "${WORKSPACE}" "$1" + shift + checkFile "$1" "${OPTION}" + WORKSPACE="$1";; + -j|--job-type) + checkDuplicates "${JOB_TYPE}" "$1" + shift + checkFile "$1" "${OPTION}" + JOB_TYPE="$1";; + -r|--recipe-type) + checkDuplicates "${RECIPE_TYPE}" "$1" + shift + checkFile "$1" "${OPTION}" + RECIPE_TYPE="$1";; + -s|--strike) + checkDuplicates "${STRIKE}" "$1" + shift + checkFile "$1" "${OPTION}" + STRIKE="$1";; + --silo) + checkDuplicates "${SILO}" "$1" + SILO="true";; + *) + echo "Unkown parameter passed: $1" + exit 1;; + esac + shift +done + +# SILO +if [[ -n ${SILO} ]]; then + echo "Autodeploy: silo: scanning registries..." + curl -sk \ + -H "Authorization: Token ${SILO_TOKEN}" \ + -H "Content-Type: application/json" \ + https://scale-silo.${CLUSTER}.aisohio.net/registries/scan +fi # Get CSRF Token CSRF_TOKEN=$(curl -skLX GET \ https://scale.${CLUSTER}.aisohio.net/api/admin --cookie-jar - \ -| grep 'csrftoken' | sed 's/^.*csrftoken[[:space:]]]*//g') -echo "LOG> CSRF TOKEN: ${CSRF_TOKEN}" - -# Validate -validateJSON "workspaces" ${WORKSPACE} -validateJSON "job-types" ${JOBTYPE} -validateJSON "recipe-types" ${RECIPETYPE} -validateJSON "strikes" ${STRIKE} - -# Find Workspace (not 100%) -# WORKSPACE_NAME=$(grep '"name":' ${WORKSPACE} | awk -F: '{print $2}' | sed 's/[[:space:]",]//g') -# WORKSPACE_LIST=$(curl -skLX GET \ -# -H "Cookie: csrftoken=${CSRF_TOKEN}" \ -# -H "Authorization: TOKEN ${API_TOKEN}" \ -# https://scale.oarfish.aisohio.net/api/v7/workspaces | -# grep "${WORKSPACE_NAME}") -# echo ${WORKSPACE_LIST} - - -# curl -skLX GET \ -# -H "Cookie: csrftoken=${CSRF_TOKEN}" \ -# -H "Authorization: TOKEN ${API_TOKEN}" \ -# https://scale.oarfish.aisohio.net/api/v7/jobs - - - -# Create Workspace -# curl -skLX POST \ -# -H "Cookie: csrftoken=${CSRF_TOKEN}" \ -# -H "Authorization: TOKEN ${API_TOKEN}" \ -# -H "Content-Type: application/json" -T ${WORKSPACE} \ -# https://scale.oarfish.aisohio.net/api/v7/workspaces \ No newline at end of file +| grep 'csrftoken' | sed 's/^.*csrftoken[[:space:]]]*//g' ) +echo "Autodeploy: csrf-token: '${CSRF_TOKEN}'" + +# Workspace +if [[ -n ${WORKSPACE} ]]; then + validateJSON "workspaces" ${WORKSPACE} + postJSON "workspaces" ${WORKSPACE} + + # Add Workspace Name to Job-type + if [[ -n ${JOB_TYPE} ]]; then + NAME_WORKSPACE="$(echo ${RESPONSE} | jq '.name')" + JOB_TYPE_EDIT="$(jq ".configuration.output_workspaces.default=${NAME_WORKSPACE}" ${JOB_TYPE})" && \ + echo "${JOB_TYPE_EDIT}" > "${JOB_TYPE}" + echo "Autodeploy: job-types: '${JOB_TYPE}' workspace updated to ${NAME_WORKSPACE}" + fi + + # Add Workspace Name to Strike + if [[ -n ${STRIKE} ]]; then + NAME_WORKSPACE="$(echo ${RESPONSE} | jq '.name')" + STRIKE_EDIT="$(jq ".configuration.workspace=${NAME_WORKSPACE}" ${STRIKE})" && \ + echo "${STRIKE_EDIT}" > "${STRIKE}" + echo "Autodeploy: strikes: '${STRIKE}' workspace updated to ${NAME_WORKSPACE}" + fi +fi + +# Job-Type +if [[ -n ${JOB_TYPE} ]]; then + validateJSON 'job-types' "${JOB_TYPE}" + postJSON 'job-types' "${JOB_TYPE}" + + # Add Job-Type to Recipe-type + if [[ -n ${RECIPE_TYPE} ]]; then + JOB_TYPE_JSON="$(echo ${RESPONSE} | jq '.')" + RECIPE_TYPE_EDIT="$(jq ".job_types=${JOB_TYPE_JSON}" ${RECIPE_TYPE})" && \ + echo "${RECIPE_TYPE_EDIT}" > "${RECIPE_TYPE}" + echo "Autodeploy: recipe-types: '${RECIPE_TYPE}' job-type updated to $(echo ${JOB_TYPE_JSON} | jq '.name')" + fi + +fi + +# Recipe-Type +if [[ -n ${RECIPE_TYPE} ]]; then + validateJSON "recipe-types" ${RECIPE_TYPE} + postJSON 'recipe-types' "${RECIPE_TYPE}" + + # Add Recipe Name to strike + if [[ -n ${STRIKE} ]]; then + NAME_RECIPE="$(echo ${RESPONSE} | jq '.name')" + STRIKE_EDIT="$(jq ".configuration.recipe.name=${NAME_RECIPE}" ${STRIKE})" && \ + echo "${STRIKE_EDIT}" > "${STRIKE}" + echo "Autodeploy: strikes: '${STRIKE}' recipe updated to ${NAME_RECIPE}" + fi +fi + +# Strike +if [[ -n ${STRIKE} ]]; then + validateJSON "strikes" "${STRIKE}" + postJSON 'strikes' "${STRIKE}" +fi \ No newline at end of file diff --git a/autodeploy/input.txt b/autodeploy/input.txt new file mode 100644 index 0000000000..bc7774a7b1 --- /dev/null +++ b/autodeploy/input.txt @@ -0,0 +1 @@ +hello world! \ No newline at end of file diff --git a/autodeploy/job-type.json b/autodeploy/job-type.json index f2765638f2..f1e99ec39b 100644 --- a/autodeploy/job-type.json +++ b/autodeploy/job-type.json @@ -1,53 +1,62 @@ { - "icon_code": "f236", - "docker_image": "docker.io/fizz11/simplesleeper-1.0.0-seed:1.0.0", - "manifest": { - "seedVersion": "1.0.0", - "job": { - "name": "simplesleeper", - "jobVersion": "1.0.0", - "packageVersion": "1.0.0", - "title": "Simple Sleeper", - "description": "Logs its filename and sleeps for 5 seconds", - "maintainer": { - "name": "AIS", - "organization": "Applied Information Sciences", - "email": "n/a" - }, - "timeout": 3600, - "interface": { - "command": "", - "inputs": { - "files": [{ - "name": "INPUT_FILE", - "multiple": false, - "partial": false, - "required": true - }] - }, - "outputs": {} - }, - "resources": { - "scalar": [{ - "name": "cpus", - "value": 0.1, - "inputMultiplier": 0 - }, { - "name": "mem", - "value": 32, - "inputMultiplier": 0 - }] + "is_active": true, + "is_paused": false, + "is_published": false, + "icon_code": "f236", + "max_scheduled": null, + "docker_image": "docker.io/fizz11/simplesleeper-1.0.0-seed:1.0.0", + "manifest": { + "job": { + "maintainer": { + "organization": "Applied Information Sciences", + "name": "AIS", + "email": "n/a" + }, + "jobVersion": "1.0.0", + "title": "Test Deploy Sleeper Job", + "description": "", + "timeout": 3600, + "interface": { + "inputs": { + "files": [ + { + "multiple": false, + "required": true, + "partial": false, + "name": "INPUT_FILE", + "mediaTypes": [] } - } - }, - "configuration": { - "output_workspaces": { - "default": "auto-deploy-workspace-b241736dad5a48ff8c8" + ] }, - "priority": 201 + "command": "", + "outputs": {} + }, + "packageVersion": "1.0.0", + "resources": { + "scalar": [ + { + "inputMultiplier": 0, + "name": "cpus", + "value": 0.1 + }, + { + "inputMultiplier": 0, + "name": "mem", + "value": 32 + } + ] + }, + "name": "test-deploy-sleeper-job" + }, + "seedVersion": "1.0.0" + }, + "configuration": { + "mounts": {}, + "priority": 201, + "output_workspaces": { + "default": "test-deploy-workspace", + "outputs": {} }, - "is_published": false, - "is_active": true, - "is_paused": false, - "max_scheduled": null -} \ No newline at end of file + "settings": {} + } +} diff --git a/autodeploy/recipe-type.json b/autodeploy/recipe-type.json index f13a9f5cac..c46dda6992 100644 --- a/autodeploy/recipe-type.json +++ b/autodeploy/recipe-type.json @@ -1,133 +1,133 @@ { - "id": null, - "name": "simple-sleeper-autodeploy", - "title": "Simple Sleeper Autodeploy", - "description": null, - "revision_num": 1, + "id": null, + "name": "test-deploy-recipe", + "title": "Test Deploy Recipe", + "description": "", + "revision_num": 1, + "is_active": true, + "is_system": false, + "created": null, + "deprecated": null, + "last_modified": null, + "definition": { + "input": { + "files": [ + { + "media_types": [], + "required": true, + "multiple": false, + "name": "File" + } + ], + "json": [] + }, + "nodes": { + "job-1": { + "input": { + "INPUT_FILE": { + "input": "File", + "type": "recipe" + } + }, + "node_type": { + "job_type_version": "1.0.0", + "node_type": "job", + "job_type_name": "test-deploy-sleeper-job", + "job_type_revision": 1 + }, + "dependencies": [] + } + } + }, + "job_types": { + "id": 13, + "name": "test-deploy-sleeper-job", + "version": "1.0.0", + "title": "Test Deploy Sleeper Job", + "description": "", "is_active": true, "is_system": false, - "created": null, + "is_paused": false, + "is_published": false, + "icon_code": "f236", + "unmet_resources": null, + "max_scheduled": null, + "max_tries": 3, + "revision_num": 2, + "docker_image": "docker.io/fizz11/simplesleeper-1.0.0-seed:1.0.0", + "created": "2020-08-21T18:37:51.676636Z", "deprecated": null, - "last_modified": null, - "definition": { - "input": { + "paused": null, + "last_modified": "2020-08-21T18:47:49.718635Z", + "manifest": { + "job": { + "maintainer": { + "organization": "Applied Information Sciences", + "name": "AIS", + "email": "n/a" + }, + "jobVersion": "1.0.0", + "title": "Test Deploy Sleeper Job", + "description": "", + "timeout": 3600, + "interface": { + "inputs": { "files": [ - { - "media_types": [], - "required": true, - "multiple": false, - "name": "File" - } - ], - "json": [] + { + "multiple": false, + "required": true, + "partial": false, + "name": "INPUT_FILE", + "mediaTypes": [] + } + ] + }, + "command": "", + "outputs": {} }, - "nodes": { - "simplesleeper": { - "input": { - "INPUT_FILE": { - "input": "File", - "type": "recipe" - } - }, - "node_type": { - "job_type_version": "1.0.0", - "node_type": "job", - "job_type_name": "simplesleeper", - "job_type_revision": 1 - }, - "dependencies": [] + "packageVersion": "1.0.0", + "resources": { + "scalar": [ + { + "inputMultiplier": 0, + "name": "cpus", + "value": 0.1 + }, + { + "inputMultiplier": 0, + "name": "mem", + "value": 32 } - } + ] + }, + "name": "test-deploy-sleeper-job" + }, + "seedVersion": "1.0.0" }, - "job_types": [ - { - "id": 23, - "name": "simplesleeper", - "version": "1.0.0", - "title": "Simple Sleeper", - "description": "Logs its filename and sleeps for 5 seconds", - "is_active": true, - "is_system": false, - "is_paused": false, - "is_published": false, - "icon_code": "f236", - "unmet_resources": null, - "max_scheduled": null, - "max_tries": 3, - "revision_num": 1, - "docker_image": "docker.io/fizz11/simplesleeper-1.0.0-seed:1.0.0", - "created": "2020-07-27T19:41:02.119907Z", - "deprecated": null, - "paused": null, - "last_modified": "2020-07-27T19:41:02.119978Z", - "manifest": { - "job": { - "maintainer": { - "organization": "Applied Information Sciences", - "name": "AIS", - "email": "n/a" - }, - "jobVersion": "1.0.0", - "title": "Simple Sleeper", - "description": "Logs its filename and sleeps for 5 seconds", - "timeout": 3600, - "interface": { - "inputs": { - "files": [ - { - "multiple": false, - "required": true, - "partial": false, - "name": "INPUT_FILE", - "mediaTypes": [] - } - ] - } - }, - "packageVersion": "1.0.0", - "resources": { - "scalar": [ - { - "inputMultiplier": 0, - "name": "cpus", - "value": 0.1 - }, - { - "inputMultiplier": 0, - "name": "mem", - "value": 32 - } - ] - }, - "name": "simplesleeper" - }, - "seedVersion": "1.0.0" - }, - "configuration": { - "mounts": {}, - "priority": 201, - "output_workspaces": { - "default": "default-output", - "outputs": {} - }, - "settings": {} - }, - "recipe_types": [ - { - "id": 11, - "name": "simple-sleeper", - "title": "Simple Sleeper", - "description": null, - "revision_num": 1, - "is_active": true, - "is_system": false, - "created": "2020-07-27T19:41:35.621061Z", - "deprecated": null, - "last_modified": "2020-07-27T19:41:35.621094Z" - } - ] - } - ], - "sub_recipe_types": [], - "super_recipe_types": [] -} \ No newline at end of file + "configuration": { + "mounts": {}, + "priority": 201, + "output_workspaces": { + "default": "test-deploy-workspace", + "outputs": {} + }, + "settings": {} + }, + "recipe_types": [ + { + "id": 6, + "name": "test-deploy-recipe", + "title": "Test Deploy Recipe", + "description": "", + "revision_num": 1, + "is_active": true, + "is_system": false, + "created": "2020-08-21T18:37:53.344495Z", + "deprecated": null, + "last_modified": "2020-08-21T18:37:53.344547Z" + } + ] + }, + "sub_recipe_types": [], + "super_recipe_types": [] +} diff --git a/autodeploy/strike.json b/autodeploy/strike.json index aac8f84f84..3e43c6a900 100644 --- a/autodeploy/strike.json +++ b/autodeploy/strike.json @@ -1,41 +1,41 @@ { - "id": 5, - "name": "simple-sleeper-strike", - "title": "Simple Sleeper Strike", - "description": null, - "job": { - "id": 233235, - "job_type": { - "id": 7, - "name": "scale-strike", - "version": "1.0.0", - "title": "Scale Strike", - "description": "Monitors a directory for incoming source files to ingest", - "is_active": true, - "is_system": true, - "is_paused": false, - "is_published": false, - "icon_code": "f0e7", - "unmet_resources": null - }, - "status": "CANCELED" + "id": null, + "name": "test-deploy-strike", + "title": "Test Deploy Strike", + "description": "", + "job": { + "id": 233235, + "job_type": { + "id": 7, + "name": "scale-strike", + "version": "1.0.0", + "title": "Scale Strike", + "description": "Monitors a directory for incoming source files to ingest", + "is_active": true, + "is_system": true, + "is_paused": false, + "is_published": false, + "icon_code": "f0e7", + "unmet_resources": null }, - "created": "2020-07-27T19:42:05.448704Z", - "last_modified": "2020-07-27T19:42:05.595187Z", - "configuration": { - "recipe": { - "name": "simple-sleeper" - }, - "monitor": { - "transfer_suffix": "_tmp", - "type": "dir-watcher" - }, - "workspace": "sleeper-input-workspace", - "files_to_ingest": [ - { - "data_types": [], - "filename_regex": ".*" - } - ] - } -} \ No newline at end of file + "status": "CANCELED" + }, + "created": null, + "last_modified": null, + "configuration": { + "recipe": { + "name": "test-deploy-recipe" + }, + "monitor": { + "transfer_suffix": "_tmp", + "type": "dir-watcher" + }, + "workspace": "test-deploy-workspace", + "files_to_ingest": [ + { + "data_types": [], + "filename_regex": ".*" + } + ] + } +} diff --git a/autodeploy/templates/job-type-template.json b/autodeploy/templates/job-type-template.json new file mode 100644 index 0000000000..eab7529d9b --- /dev/null +++ b/autodeploy/templates/job-type-template.json @@ -0,0 +1,76 @@ +{ + "id": null, + "name": "template-job", + "version": "1.0.0", + "title": "Template Job", + "description": "Template for Job Types", + "is_active": true, + "is_system": false, + "is_paused": false, + "is_published": false, + "icon_code": "f236", + "unmet_resources": null, + "max_scheduled": null, + "max_tries": 3, + "revision_num": 1, + "docker_image": "docker.io/fizz11/simplesleeper-1.0.0-seed:1.0.0", + "created": null, + "deprecated": null, + "paused": null, + "last_modified": null, + "manifest": { + "job": { + "maintainer": { + "organization": "Applied Information Sciences", + "name": "AIS", + "email": "n/a" + }, + "jobVersion": "1.0.0", + "title": "Simple Sleeper Autodeploy", + "description": "Logs its filename and sleeps for 5 seconds", + "timeout": 3600, + "interface": { + "inputs": { + "files": [ + { + "multiple": false, + "required": true, + "partial": false, + "name": "INPUT_FILE", + "mediaTypes": [] + } + ] + }, + "command": "", + "outputs": {} + }, + "packageVersion": "1.0.0", + "resources": { + "scalar": [ + { + "inputMultiplier": 0, + "name": "cpus", + "value": 0.1 + }, + { + "inputMultiplier": 0, + "name": "mem", + "value": 32 + } + ] + }, + "name": "simplesleeper-autodeploy" + }, + "seedVersion": "1.0.0" + }, + "configuration": { + "mounts": {}, + "priority": 201, + "output_workspaces": { + "default": null, + "outputs": {} + }, + "settings": {} + }, + "recipe_types": [] +} diff --git a/autodeploy/templates/recipe-type-template.json b/autodeploy/templates/recipe-type-template.json new file mode 100644 index 0000000000..3472537cbc --- /dev/null +++ b/autodeploy/templates/recipe-type-template.json @@ -0,0 +1,45 @@ +{ + "id": null, + "name": "template-recipe", + "title": "Template Recipe", + "description": "Template for Recipe Types", + "revision_num": 1, + "is_active": true, + "is_system": false, + "created": null, + "deprecated": null, + "last_modified": null, + "definition": { + "input": { + "files": [ + { + "media_types": [], + "required": true, + "multiple": false, + "name": "File" + } + ], + "json": [] + }, + "nodes": { + "template-job": { + "input": { + "INPUT_FILE": { + "input": "File", + "type": "recipe" + } + }, + "node_type": { + "job_type_version": "1.0.0", + "node_type": "job", + "job_type_name": "template-job", + "job_type_revision": 1 + }, + "dependencies": [] + } + } + }, + "job_types": [], + "sub_recipe_types": [], + "super_recipe_types": [] +} diff --git a/autodeploy/templates/strike-template.json b/autodeploy/templates/strike-template.json new file mode 100644 index 0000000000..7764d063a5 --- /dev/null +++ b/autodeploy/templates/strike-template.json @@ -0,0 +1,41 @@ +{ + "id": null, + "name": "template-strike", + "title": "Template Strike", + "description": "Template for Job Types", + "job": { + "id": 233235, + "job_type": { + "id": 7, + "name": "scale-strike", + "version": "1.0.0", + "title": "Scale Strike", + "description": "Monitors a directory for incoming source files to ingest", + "is_active": true, + "is_system": true, + "is_paused": false, + "is_published": false, + "icon_code": "f0e7", + "unmet_resources": null + }, + "status": "CANCELED" + }, + "created": null, + "last_modified": null, + "configuration": { + "recipe": { + "name": null + }, + "monitor": { + "transfer_suffix": "_tmp", + "type": "dir-watcher" + }, + "workspace": null, + "files_to_ingest": [ + { + "data_types": [], + "filename_regex": ".*" + } + ] + } +} \ No newline at end of file diff --git a/autodeploy/templates/workspace-template.json b/autodeploy/templates/workspace-template.json new file mode 100644 index 0000000000..14aa4db6bd --- /dev/null +++ b/autodeploy/templates/workspace-template.json @@ -0,0 +1,17 @@ +{ + "id": null, + "name": "auto-deploy-workspace-b241736dad5a48ff8c8", + "title": "Auto Deploy Workspace (active)", + "description": "A Test Workspace created by a script", + "base_url": null, + "is_active": true, + "created": null, + "deprecated": null, + "last_modified": null, + "configuration": { + "broker": { + "host_path": "/dfs/testing/auto-deploy", + "type": "host" + } + } +} \ No newline at end of file diff --git a/autodeploy/test.json b/autodeploy/test.json new file mode 100644 index 0000000000..6e4f8d4aef --- /dev/null +++ b/autodeploy/test.json @@ -0,0 +1,76 @@ +{ + "id": 24, + "name": "simplesleeper-autodeploy", + "version": "1.0.0", + "title": "Simple Sleeper Autodeploy", + "description": "Logs its filename and sleeps for 5 seconds", + "is_active": true, + "is_system": false, + "is_paused": false, + "is_published": false, + "icon_code": "f236", + "unmet_resources": null, + "max_scheduled": null, + "max_tries": 3, + "revision_num": 1, + "docker_image": "docker.io/fizz11/simplesleeper-1.0.0-seed:1.0.0", + "created": "2020-08-06T18:30:16.448764Z", + "deprecated": null, + "paused": null, + "last_modified": "2020-08-06T18:30:16.448807Z", + "manifest": { + "job": { + "maintainer": { + "organization": "Applied Information Sciences", + "name": "AIS", + "email": "n/a" + }, + "jobVersion": "1.0.0", + "title": "Simple Sleeper Autodeploy", + "description": "Logs its filename and sleeps for 5 seconds", + "timeout": 3600, + "interface": { + "inputs": { + "files": [ + { + "multiple": false, + "required": true, + "partial": false, + "name": "INPUT_FILE", + "mediaTypes": [] + } + ] + }, + "command": "", + "outputs": {} + }, + "packageVersion": "1.0.0", + "resources": { + "scalar": [ + { + "inputMultiplier": 0, + "name": "cpus", + "value": 0.1 + }, + { + "inputMultiplier": 0, + "name": "mem", + "value": 32 + } + ] + }, + "name": "simplesleeper-autodeploy" + }, + "seedVersion": "1.0.0" + }, + "configuration": { + "mounts": {}, + "priority": 201, + "output_workspaces": { + "default": "auto-deploy-workspace-b241736dad5a48ff8c8", + "outputs": {} + }, + "settings": {} + }, + "recipe_types": [] + } \ No newline at end of file diff --git a/autodeploy/test.sh b/autodeploy/test.sh index 14711d2141..e3a4051d5c 100755 --- a/autodeploy/test.sh +++ b/autodeploy/test.sh @@ -1,6 +1,7 @@ #!/usr/bin/env sh -API_TOKEN="fab1d1a2ace51f11ccb29246a3a1d99143908ceb" +API_TOKEN='fab1d1a2ace51f11ccb29246a3a1d99143908ceb' +SILO_TOKEN='eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoiYWRtaW4iLCJ1c2VybmFtZSI6ImFkbWluIn0.OAK7h0FKjwpoarCJdiuQ9q2zKW7D4KiyseyQLwfO5A8' CLUSTER="oarfish" #+------------------------------------------------------ @@ -8,34 +9,156 @@ CLUSTER="oarfish" #| PARAMETERS: url_snip, json_file #+------------------------------------------------------ validateJSON() { - # Validate Workspace - IS_VALID=$(curl -skLX POST \ + RESPONSE="$(curl -skLX POST \ -H "Cookie: csrftoken=${CSRF_TOKEN}" \ -H "Authorization: TOKEN ${API_TOKEN}" \ -H "Content-Type: application/json" -T $2 \ - https://scale.oarfish.aisohio.net/api/v7/$1/validation) - - # Check for Validation - if [ "${IS_VALID}" = '{"is_valid":true,"errors":[],"warnings":[]}' ]; then - echo "LOG> $1 valid!" - return 0 - elif [ "${IS_VALID}" = '{"diff":{},"is_valid":true,"errors":[],"warnings":[]}' ]; then - echo "LOG> $1 valid and diff is empty!" - return 0 + https://scale.${CLUSTER}.aisohio.net/api/v7/$1/validation)" + if [[ $(echo "${RESPONSE}" | jq '.is_valid') = "true" ]]; then + if [[ $(echo "${RESPONSE}" | jq '.diff | length') = 0 ]]; then + echo "LOG> $1: $2 is valid" + else + echo "Log> $1: Diff detected in $2\n${DIFF}" + echo "${RESPONSE}" | jq '.diff' + fi + # elif [[ $(echo "${RESPONSE}" | jq 'has("detail")') = "true" ]]; then + # ERROR=$(echo "${RESPONSE}" | jq '.detail') + # echo "LOG> $1: $2 is not valid\n${ERROR}" + # exit 1 else - echo ${IS_VALID} - echo "LOG> $1 invalid!" + ERROR=$(echo "${RESPONSE}" | jq '.') + echo "LOG> $1: $2 is not valid\n${ERROR}" exit 1 fi } +#+------------------------------------------------------ +#| NAME: postJSON() +#| PARAMETERS: url_snip, json_file +#+------------------------------------------------------ +postJSON() { + RESPONSE=$(curl -skLX GET \ + -H "Cookie: csrftoken=${CSRF_TOKEN}" \ + -H "Authorization: TOKEN ${API_TOKEN}" \ + https://scale.${CLUSTER}.aisohio.net/api/v7/$1) + + case $1 in + workspaces|recipe-types|strikes) + NAME=$(jq '.name' $2);; + job-types) + NAME=$(jq '.manifest.job.name' $2) + VERSION=$(jq '.manifest.job.jobVersion' $2);; + esac + + if [[ $(echo "${RESPONSE}" | jq ".results[] | select(.name == "${NAME}") | has(\"id\")") = "true" ]]; then + echo "LOG> $1: $2 is a duplicate" + case $1 in + workspaces|strikes) + ID=$(echo "${RESPONSE}" | jq ".results[] | select(.name == "${NAME}") | .id") + URL=$(echo "https://scale.${CLUSTER}.aisohio.net/api/v7/$1/${ID}" | sed 's/"//g');; + job-types) + URL=$(echo "https://scale.${CLUSTER}.aisohio.net/api/v7/$1/${NAME}/${VERSION}" | sed 's/"//g');; + recipe-types) + URL=$(echo "https://scale.${CLUSTER}.aisohio.net/api/v7/$1/${NAME}" | sed 's/"//g');; + esac + + RESPONSE=$(curl -skLX PATCH \ + -H "Cookie: csrftoken=${CSRF_TOKEN}" \ + -H "Authorization: TOKEN ${API_TOKEN}" \ + -H "Content-Type: application/json" -T $2 \ + ${URL}) + if [[ $(echo "${RESPONSE}" | jq '.is_valid') != "true" ]]; then + if [ "${RESPONSE}" != "" ]; then + echo "LOG> $1: Response Invalid, could not PATCH $2" + echo "${RESPONSE}" + exit 1 + fi + fi + echo "LOG> $1: Successful PATCH of $2" + RESPONSE=$(curl -skLX GET \ + -H "Cookie: csrftoken=${CSRF_TOKEN}" \ + -H "Authorization: TOKEN ${API_TOKEN}" \ + ${URL}) + echo "${RESPONSE}" | jq '.' + exit 0 + else + echo "LOG> $1: $2 is original" + RESPONSE=$(curl -skLX POST \ + -H "Cookie: csrftoken=${CSRF_TOKEN}" \ + -H "Authorization: TOKEN ${API_TOKEN}" \ + -H "Content-Type: application/json" -T $2 \ + https://scale.${CLUSTER}.aisohio.net/api/v7/$1) + if [[ ! $(echo "${RESPONSE}" | jq 'has("id","name")') =~ "false" ]]; then + echo "LOG> $1: $2 posted!" + echo "${RESPONSE}" | jq '.' + else + echo "LOG> $1: Response invalid, could not POST $2" + echo "${RESPONSE}" | jq '.' + exit 1 + fi + fi +} + +# oldPostJSON() { +# RESPONSE=$(curl -skLX POST \ +# -H "Cookie: csrftoken=${CSRF_TOKEN}" \ +# -H "Authorization: TOKEN ${API_TOKEN}" \ +# -H "Content-Type: application/json" -T $2 \ +# https://scale.${CLUSTER}.aisohio.net/api/v7/$1) + +# # Check for ID and Name Fields +# if [ "${RESPONSE}" = '{"is_valid":true,"errors":[],"warnings":[]}' ]; then +# echo "LOG> $1 already exists!" +# NAME=$(jq '.manifest.job.name' $2) +# RESPONSE=$(curl -skLX GET \ +# -H "Cookie: csrftoken=${CSRF_TOKEN}" \ +# -H "Authorization: TOKEN ${API_TOKEN}" \ +# -H "Content-Type: application/json" -T $2 \ +# https://scale.${CLUSTER}.aisohio.net/api/v7/$1/${NAME}) + +# # Get JSON for existing Workspace/Job/Recipe/Strike +# case $1 in +# workspaces|strikes) +# ID=$(jq '.id' $2 | sed 's/"//g') +# RESPONSE=$(curl -skLX GET \ +# -H "Cookie: csrftoken=${CSRF_TOKEN}" \ +# -H "Authorization: TOKEN ${API_TOKEN}" \ +# https://scale.${CLUSTER}.aisohio.net/api/v7/$1/${ID});; +# job-types) +# NAME=$(jq '.manifest.job.name' $2 | sed 's/"//g') +# RESPONSE=$(curl -skLX GET \ +# -H "Cookie: csrftoken=${CSRF_TOKEN}" \ +# -H "Authorization: TOKEN ${API_TOKEN}" \ +# https://scale.${CLUSTER}.aisohio.net/api/v7/$1/${NAME});; +# recipe-types) +# NAME=$(jq '.name' $2 | sed 's/"//g') +# RESPONSE=$(curl -skLX GET \ +# -H "Cookie: csrftoken=${CSRF_TOKEN}" \ +# -H "Authorization: TOKEN ${API_TOKEN}" \ +# https://scale.${CLUSTER}.aisohio.net/api/v7/$1/${NAME});; +# *) +# echo "LOG> Incorrect URL!";; +# esac + +# elif [[ ! $(echo "${RESPONSE}" | jq 'has("id","name")') =~ "false" ]]; then +# echo "LOG> $1 posted!" +# echo "${RESPONSE}" | jq '.' +# exit 1 +# else +# echo "LOG> Response invalid, unsuccessful post" +# echo "${RESPONSE}" | jq '.' +# exit 1 +# fi +# } + #+------------------------------------------------------ #| NAME: checkDuplicates() #| PARAMETERS: variable_checked, parameter #+------------------------------------------------------ checkDuplicates() { if [[ -n $1 ]]; then - echo "Duplicate parameter \"$2\"" + echo "Error: '$2' is a duplicate" + echo "Note: './autodeploy.sh --help' or './autodeploy.sh -h' for more information" exit 1 fi } @@ -46,8 +169,9 @@ checkDuplicates() { #+------------------------------------------------------ checkFile() { if [[ ! -f $1 ]]; then - echo "\"$1\" doesn't exists" - exit 2 + echo "Error: '$1' doesn't exists" + echo "Usage: ./autodeploy.sh $2 " + exit 1 fi } @@ -55,88 +179,157 @@ checkFile() { set -- $(echo "$@" | sed 's/[=]/ /g') # Argument Parsing +if [ $# == 0 ]; then + echo "Usage: ./autodeploy.sh [options...]" + echo "Note: './autodeploy.sh --help' or './autodeploy.sh -h' for more information" + exit 0 +fi + while [[ "$#" -gt 0 ]]; do + OPTION=$1 case $1 in -h|--help) echo "Usage: ./autodeploy.sh [options...]" - echo " -w, --workspace verifies and creates a workspace based on the json file provied" + echo " -w, --workspace verifies and creates a workspace based on the json file provied" echo " -j, --job-type verifies and creates a job type based on the json file provied" echo " -r, --recipe-type verifies and creates a recipe type based on the json file provied" echo " -s, --strike verifies and creates a strike based on the json file provied" - echo 'Note: short fromat not supported (ex. % ./autodeploy.sh -wjrs)' + echo " --silo scans registries to update images" + echo "Note: short fromat not supported (ex. './autodeploy.sh -wjrs')" exit 0;; -w|--workspace) - checkDuplicates "${WORKSPACE}" "-w" + checkDuplicates "${WORKSPACE}" "$1" shift - checkFile "$1" + checkFile "$1" "${OPTION}" WORKSPACE="$1";; -j|--job-type) - checkDuplicates "${JOB_TYPE}" "-j" + checkDuplicates "${JOB_TYPE}" "$1" shift - checkFile "$1" + checkFile "$1" "${OPTION}" JOB_TYPE="$1";; -r|--recipe-type) - checkDuplicates "${RECIPE_TYPE}" "-r" + checkDuplicates "${RECIPE_TYPE}" "$1" shift - checkFile "$1" + checkFile "$1" "${OPTION}" RECIPE_TYPE="$1";; -s|--strike) - checkDuplicates "${STRIKE}" "-s" + checkDuplicates "${STRIKE}" "$1" shift - checkFile "$1" + checkFile "$1" "${OPTION}" STRIKE="$1";; + --silo) + # checkDuplicates "${STRIKE}" "-s" + # echo "Login into ${CLUSTER} SILO:\n-------------------------------" + # read -p 'Username: ' USERNAME + # read -sp 'Password: ' PASSWORD + # echo "" + SILO="true";; *) - echo "Unkown parameter passed: $1";; + echo "Unkown parameter passed: $1" + exit 1;; esac shift done +# SILO +if [[ -n ${SILO} ]]; then + SILO_TOKEN=$(curl -sk \ + -H "Content-Type: application/json" \ + -d "{\"username\":\"${USERNAME}\", \"password\": \"${PASSWORD}\"}" \ + https://scale-silo.${CLUSTER}.aisohio.net/login | jq '.token' | sed 's/"//g') + if [[ ${SILO_TOKEN} = "null" ]]; then + echo "LOG> Incorrect Username or Password" + exit 0 + fi + +curl -skLX GET \ +https://scale.oarfish.aisohio.net/api/admin --cookie-jar - + + +curl -skDX GET "https://scale.oarfish.aisohio.net/api/login/?next=https://scale.oarfish.aisohio.net/" + +curl -skX POST \ +-H "Cookie: csrftoken=Ee6qDB4rQ1StvmelEsGVwLPlYqFJPDhfpiP1DHp94FliJkGn2zUn75NaR9nAOH8l" \ +-H "Content-Type: application/json" \ +-d "{\"username\":\"admin\", \"password\": \"admin\", \"csrfmiddlewaretoken\": \"20Z4logmOEj1qwqbUvGmVlVt5hN54bC8N4IFluB42iMQEuSdiCUOwFTiY0vW3fte\"}" \ +"https://scale.oarfish.aisohio.net/api/login/?next=https://scale.oarfish.aisohio.net/" + +csrfmiddlewaretoken=uJsTXB2vDMCak5H5pfrc3LyCmuBZGpjmMpU6RuAGCZ45DQqDZcyaugKlhkKDZeAL&username=admin&password=sdfsd + + + +csrfmiddlewaretoken: uJsTXB2vDMCak5H5pfrc3LyCmuBZGpjmMpU6RuAGCZ45DQqDZcyaugKlhkKDZeAL + +oiViw3q5f1oKl9OpRphWU9MKeylwNyoS0cVrev2voy65uqUav242XN0Iw + + + +RESPONSE=$(curl -skLX PATCH \ + -H "Cookie: csrftoken=${CSRF_TOKEN}" \ + -H "Authorization: TOKEN ${API_TOKEN}" \ + -H "Content-Type: application/json" -T $2 \ + #echo "LOG> SILO TOKEN: ${SILO_TOKEN}" + + # curl -sk \ + # -H "Authorization: Token ${SILO_TOKEN}" \ + # -H "Content-Type: application/json" \ + # -d "{\"name\":\"${USERNAME}\", \"url\":\"https://hub.docker.com\", \"org\":\"${USERNAME}\", \"username\":\"${USERNAME}\", \"password\": \"${PASSWORD}\"}" \ + # https://scale-silo.${CLUSTER}.aisohio.net/registries/add + + curl -sk \ + -H "Authorization: Token ${SILO_TOKEN}" \ + -H "Content-Type: application/json" \ + https://scale-silo.${CLUSTER}.aisohio.net/registries/scan + +fi + # Get CSRF Token CSRF_TOKEN=$(curl -skLX GET \ https://scale.${CLUSTER}.aisohio.net/api/admin --cookie-jar - \ -| grep 'csrftoken' | sed 's/^.*csrftoken[[:space:]]]*//g') +| grep 'csrftoken' | sed 's/^.*csrftoken[[:space:]]]*//g' ) echo "LOG> CSRF TOKEN: ${CSRF_TOKEN}" # Workspace if [[ -n ${WORKSPACE} ]]; then validateJSON "workspaces" ${WORKSPACE} + postJSON "workspaces" ${WORKSPACE} + # Add Workspace Name to job-type + if [[ -n ${JOB_TYPE} ]]; then + NAME_WORKSPACE="$(echo ${RESPONSE} | jq '.name' ${RESPONSE})" + JOB_TYPE_EDIT="$(jq ".configuration.output_workspaces.default=${NAME_WORKSPACE}" ${JOB_TYPE})" && \ + echo "${JOB_TYPE_EDIT}" > "${JOB_TYPE}" + fi +fi +# Job-Type +if [[ -n ${JOB_TYPE} ]]; then + validateJSON 'job-types' "${JOB_TYPE}" + postJSON 'job-types' "${JOB_TYPE}" + # Add Job-Type to Recipe-type + if [[ -n ${RECIPE_TYPE} ]]; then + JOB_TYPE_JSON="$(echo ${RESPONSE} | jq '.results')" + RECIPE_TYPE_EDIT="$(jq ".job_types=${JOB_TYPE_JSON}" ${RECIPE_TYPE})" && \ + echo "${RECIPE_TYPE_EDIT}" > "${RECIPE_TYPE}" + fi +fi + +# Recipe-Type +if [[ -n ${RECIPE_TYPE} ]]; then + validateJSON "recipe-types" ${RECIPE_TYPE} + postJSON 'recipe-types' "${RECIPE_TYPE}" + + # Add Recipe Name to strike + if [[ -n ${STRIKE} ]]; then + NAME_RECIPE="$(echo ${RESPONSE} | jq '.name')" + STRIKE_EDIT="$(jq ".configuration.recipe.name=${NAME_RECIPE}" ${STRIKE})" && \ + echo "${STRIKE_EDIT}" > "${STRIKE}" + fi fi - - -# while [[ "$#" -gt 0 ]]; do -# case $1 in -# -d|--deploy) deploy="$2"; shift ;; -# -u|--uglify) uglify=1 ;; -# *) echo "Unknown parameter passed: $1"; exit 1 ;; -# esac -# shift -# done - -# for i in "$@" -# do -# case $i in -# -e=*|--extension=*) -# EXTENSION="${i#*=}" -# shift # past argument=value -# ;; -# -s=*|--searchpath=*) -# SEARCHPATH="${i#*=}" -# shift # past argument=value -# ;; -# -l=*|--lib=*) -# LIBPATH="${i#*=}" -# shift # past argument=value -# ;; -# --default) -# DEFAULT=YES -# shift # past argument with no value -# ;; -# *) -# # unknown option -# ;; -# esac -# done \ No newline at end of file +# Strike +if [[ -n ${STRIKE} ]]; then + validateJSON "strikes" "${STRIKE}" + postJSON 'strikes' "${STRIKE}" +fi \ No newline at end of file diff --git a/autodeploy/workspace.json b/autodeploy/workspace.json index a1660946d8..178b93142d 100644 --- a/autodeploy/workspace.json +++ b/autodeploy/workspace.json @@ -1,8 +1,8 @@ { "id": null, - "name": "auto-deploy-workspace", - "title": "Auto Deploy Workspace", - "description": "A Test Workspace created by a script", + "name": "test-deploy-workspace", + "title": "Test Deploy Workspace", + "description": "", "base_url": null, "is_active": true, "created": null, @@ -10,7 +10,7 @@ "last_modified": null, "configuration": { "broker": { - "host_path": "/dfs/testing/auto-deploy", + "host_path": "/dfs/testing/test-deploy", "type": "host" } } From c24b8738614f161e26dcc94e7f6f189f9449a87b Mon Sep 17 00:00:00 2001 From: Matt Talda Date: Mon, 24 Aug 2020 10:27:18 -0400 Subject: [PATCH 5/7] dockerized, removed test files --- autodeploy/{ => docker}/Dockerfile | 7 +- autodeploy/{ => docker}/autodeploy.sh | 0 autodeploy/input.txt | 1 - autodeploy/job-type.json | 62 ---- autodeploy/recipe-type.json | 133 ------- autodeploy/run-autodeploy.sh | 4 + autodeploy/strike.json | 41 --- autodeploy/templates/job-type-template.json | 58 +-- .../templates/recipe-type-template.json | 10 +- autodeploy/templates/strike-template.json | 8 +- autodeploy/templates/workspace-template.json | 8 +- autodeploy/test.json | 76 ---- autodeploy/test.sh | 335 ------------------ autodeploy/workspace.json | 17 - 14 files changed, 28 insertions(+), 732 deletions(-) rename autodeploy/{ => docker}/Dockerfile (50%) rename autodeploy/{ => docker}/autodeploy.sh (100%) delete mode 100644 autodeploy/input.txt delete mode 100644 autodeploy/job-type.json delete mode 100644 autodeploy/recipe-type.json create mode 100755 autodeploy/run-autodeploy.sh delete mode 100644 autodeploy/strike.json delete mode 100644 autodeploy/test.json delete mode 100755 autodeploy/test.sh delete mode 100644 autodeploy/workspace.json diff --git a/autodeploy/Dockerfile b/autodeploy/docker/Dockerfile similarity index 50% rename from autodeploy/Dockerfile rename to autodeploy/docker/Dockerfile index 1eb319b5de..7608f65aba 100644 --- a/autodeploy/Dockerfile +++ b/autodeploy/docker/Dockerfile @@ -3,10 +3,11 @@ FROM alpine:3.12 RUN apk update && \ apk add --no-cache jq bash curl -RUN mkdir -p /usr/autodeploy +RUN mkdir -p /usr/autodeploy/volume WORKDIR /usr/autodeploy -COPY autodeploy.sh /usr/autodeploy +COPY autodeploy.sh . RUN chmod +x autodeploy.sh +WORKDIR /usr/autodeploy/volume -ENTRYPOINT ["bash", "autodeploy.sh"] \ No newline at end of file +ENTRYPOINT ["bash", "../autodeploy.sh"] diff --git a/autodeploy/autodeploy.sh b/autodeploy/docker/autodeploy.sh similarity index 100% rename from autodeploy/autodeploy.sh rename to autodeploy/docker/autodeploy.sh diff --git a/autodeploy/input.txt b/autodeploy/input.txt deleted file mode 100644 index bc7774a7b1..0000000000 --- a/autodeploy/input.txt +++ /dev/null @@ -1 +0,0 @@ -hello world! \ No newline at end of file diff --git a/autodeploy/job-type.json b/autodeploy/job-type.json deleted file mode 100644 index f1e99ec39b..0000000000 --- a/autodeploy/job-type.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "is_active": true, - "is_paused": false, - "is_published": false, - "icon_code": "f236", - "max_scheduled": null, - "docker_image": "docker.io/fizz11/simplesleeper-1.0.0-seed:1.0.0", - "manifest": { - "job": { - "maintainer": { - "organization": "Applied Information Sciences", - "name": "AIS", - "email": "n/a" - }, - "jobVersion": "1.0.0", - "title": "Test Deploy Sleeper Job", - "description": "", - "timeout": 3600, - "interface": { - "inputs": { - "files": [ - { - "multiple": false, - "required": true, - "partial": false, - "name": "INPUT_FILE", - "mediaTypes": [] - } - ] - }, - "command": "", - "outputs": {} - }, - "packageVersion": "1.0.0", - "resources": { - "scalar": [ - { - "inputMultiplier": 0, - "name": "cpus", - "value": 0.1 - }, - { - "inputMultiplier": 0, - "name": "mem", - "value": 32 - } - ] - }, - "name": "test-deploy-sleeper-job" - }, - "seedVersion": "1.0.0" - }, - "configuration": { - "mounts": {}, - "priority": 201, - "output_workspaces": { - "default": "test-deploy-workspace", - "outputs": {} - }, - "settings": {} - } -} diff --git a/autodeploy/recipe-type.json b/autodeploy/recipe-type.json deleted file mode 100644 index c46dda6992..0000000000 --- a/autodeploy/recipe-type.json +++ /dev/null @@ -1,133 +0,0 @@ -{ - "id": null, - "name": "test-deploy-recipe", - "title": "Test Deploy Recipe", - "description": "", - "revision_num": 1, - "is_active": true, - "is_system": false, - "created": null, - "deprecated": null, - "last_modified": null, - "definition": { - "input": { - "files": [ - { - "media_types": [], - "required": true, - "multiple": false, - "name": "File" - } - ], - "json": [] - }, - "nodes": { - "job-1": { - "input": { - "INPUT_FILE": { - "input": "File", - "type": "recipe" - } - }, - "node_type": { - "job_type_version": "1.0.0", - "node_type": "job", - "job_type_name": "test-deploy-sleeper-job", - "job_type_revision": 1 - }, - "dependencies": [] - } - } - }, - "job_types": { - "id": 13, - "name": "test-deploy-sleeper-job", - "version": "1.0.0", - "title": "Test Deploy Sleeper Job", - "description": "", - "is_active": true, - "is_system": false, - "is_paused": false, - "is_published": false, - "icon_code": "f236", - "unmet_resources": null, - "max_scheduled": null, - "max_tries": 3, - "revision_num": 2, - "docker_image": "docker.io/fizz11/simplesleeper-1.0.0-seed:1.0.0", - "created": "2020-08-21T18:37:51.676636Z", - "deprecated": null, - "paused": null, - "last_modified": "2020-08-21T18:47:49.718635Z", - "manifest": { - "job": { - "maintainer": { - "organization": "Applied Information Sciences", - "name": "AIS", - "email": "n/a" - }, - "jobVersion": "1.0.0", - "title": "Test Deploy Sleeper Job", - "description": "", - "timeout": 3600, - "interface": { - "inputs": { - "files": [ - { - "multiple": false, - "required": true, - "partial": false, - "name": "INPUT_FILE", - "mediaTypes": [] - } - ] - }, - "command": "", - "outputs": {} - }, - "packageVersion": "1.0.0", - "resources": { - "scalar": [ - { - "inputMultiplier": 0, - "name": "cpus", - "value": 0.1 - }, - { - "inputMultiplier": 0, - "name": "mem", - "value": 32 - } - ] - }, - "name": "test-deploy-sleeper-job" - }, - "seedVersion": "1.0.0" - }, - "configuration": { - "mounts": {}, - "priority": 201, - "output_workspaces": { - "default": "test-deploy-workspace", - "outputs": {} - }, - "settings": {} - }, - "recipe_types": [ - { - "id": 6, - "name": "test-deploy-recipe", - "title": "Test Deploy Recipe", - "description": "", - "revision_num": 1, - "is_active": true, - "is_system": false, - "created": "2020-08-21T18:37:53.344495Z", - "deprecated": null, - "last_modified": "2020-08-21T18:37:53.344547Z" - } - ] - }, - "sub_recipe_types": [], - "super_recipe_types": [] -} diff --git a/autodeploy/run-autodeploy.sh b/autodeploy/run-autodeploy.sh new file mode 100755 index 0000000000..3abd35498a --- /dev/null +++ b/autodeploy/run-autodeploy.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env sh + +ARGS=$@ +docker run --rm -v "${PWD}:/usr/autodeploy/volume" mtalda/autodeploy ${ARGS} \ No newline at end of file diff --git a/autodeploy/strike.json b/autodeploy/strike.json deleted file mode 100644 index 3e43c6a900..0000000000 --- a/autodeploy/strike.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "id": null, - "name": "test-deploy-strike", - "title": "Test Deploy Strike", - "description": "", - "job": { - "id": 233235, - "job_type": { - "id": 7, - "name": "scale-strike", - "version": "1.0.0", - "title": "Scale Strike", - "description": "Monitors a directory for incoming source files to ingest", - "is_active": true, - "is_system": true, - "is_paused": false, - "is_published": false, - "icon_code": "f0e7", - "unmet_resources": null - }, - "status": "CANCELED" - }, - "created": null, - "last_modified": null, - "configuration": { - "recipe": { - "name": "test-deploy-recipe" - }, - "monitor": { - "transfer_suffix": "_tmp", - "type": "dir-watcher" - }, - "workspace": "test-deploy-workspace", - "files_to_ingest": [ - { - "data_types": [], - "filename_regex": ".*" - } - ] - } -} diff --git a/autodeploy/templates/job-type-template.json b/autodeploy/templates/job-type-template.json index eab7529d9b..5cc5184426 100644 --- a/autodeploy/templates/job-type-template.json +++ b/autodeploy/templates/job-type-template.json @@ -1,73 +1,29 @@ { "id": null, - "name": "template-job", + "name": "", "version": "1.0.0", - "title": "Template Job", - "description": "Template for Job Types", + "title": "", + "description": "<DESCIPTION>", "is_active": true, "is_system": false, "is_paused": false, "is_published": false, - "icon_code": "f236", + "icon_code": "<ICON_CODE", "unmet_resources": null, "max_scheduled": null, "max_tries": 3, "revision_num": 1, - "docker_image": "docker.io/fizz11/simplesleeper-1.0.0-seed:1.0.0", + "docker_image": "docker.io/<URL>", "created": null, "deprecated": null, "paused": null, "last_modified": null, - "manifest": { - "job": { - "maintainer": { - "organization": "Applied Information Sciences", - "name": "AIS", - "email": "n/a" - }, - "jobVersion": "1.0.0", - "title": "Simple Sleeper Autodeploy", - "description": "Logs its filename and sleeps for 5 seconds", - "timeout": 3600, - "interface": { - "inputs": { - "files": [ - { - "multiple": false, - "required": true, - "partial": false, - "name": "INPUT_FILE", - "mediaTypes": [] - } - ] - }, - "command": "", - "outputs": {} - }, - "packageVersion": "1.0.0", - "resources": { - "scalar": [ - { - "inputMultiplier": 0, - "name": "cpus", - "value": 0.1 - }, - { - "inputMultiplier": 0, - "name": "mem", - "value": 32 - } - ] - }, - "name": "simplesleeper-autodeploy" - }, - "seedVersion": "1.0.0" - }, + "manifest": "<MANIFEST>", "configuration": { "mounts": {}, "priority": 201, "output_workspaces": { - "default": null, + "default": "<WORKSPACE>", "outputs": {} }, "settings": {} diff --git a/autodeploy/templates/recipe-type-template.json b/autodeploy/templates/recipe-type-template.json index 3472537cbc..6e701bdb05 100644 --- a/autodeploy/templates/recipe-type-template.json +++ b/autodeploy/templates/recipe-type-template.json @@ -1,8 +1,8 @@ { "id": null, - "name": "template-recipe", - "title": "Template Recipe", - "description": "Template for Recipe Types", + "name": "<NAME>", + "title": "<TITLE>", + "description": "<DESCRIPTION>", "revision_num": 1, "is_active": true, "is_system": false, @@ -22,7 +22,7 @@ "json": [] }, "nodes": { - "template-job": { + "job": { "input": { "INPUT_FILE": { "input": "File", @@ -32,7 +32,7 @@ "node_type": { "job_type_version": "1.0.0", "node_type": "job", - "job_type_name": "template-job", + "job_type_name": "<JOB_NAME>", "job_type_revision": 1 }, "dependencies": [] diff --git a/autodeploy/templates/strike-template.json b/autodeploy/templates/strike-template.json index 7764d063a5..cedafaa3f7 100644 --- a/autodeploy/templates/strike-template.json +++ b/autodeploy/templates/strike-template.json @@ -1,8 +1,8 @@ { "id": null, - "name": "template-strike", - "title": "Template Strike", - "description": "Template for Job Types", + "name": "<NAME>", + "title": "<TITLE>", + "description": "<DESCRIPTION>", "job": { "id": 233235, "job_type": { @@ -30,7 +30,7 @@ "transfer_suffix": "_tmp", "type": "dir-watcher" }, - "workspace": null, + "workspace": "<WORKSPACE>", "files_to_ingest": [ { "data_types": [], diff --git a/autodeploy/templates/workspace-template.json b/autodeploy/templates/workspace-template.json index 14aa4db6bd..e41d818952 100644 --- a/autodeploy/templates/workspace-template.json +++ b/autodeploy/templates/workspace-template.json @@ -1,8 +1,8 @@ { "id": null, - "name": "auto-deploy-workspace-b241736dad5a48ff8c8", - "title": "Auto Deploy Workspace (active)", - "description": "A Test Workspace created by a script", + "name": "<NAME>", + "title": "<TITLE>", + "description": "<DESCRIPTION>", "base_url": null, "is_active": true, "created": null, @@ -10,7 +10,7 @@ "last_modified": null, "configuration": { "broker": { - "host_path": "/dfs/testing/auto-deploy", + "host_path": "/dfs/<PATH>", "type": "host" } } diff --git a/autodeploy/test.json b/autodeploy/test.json deleted file mode 100644 index 6e4f8d4aef..0000000000 --- a/autodeploy/test.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "id": 24, - "name": "simplesleeper-autodeploy", - "version": "1.0.0", - "title": "Simple Sleeper Autodeploy", - "description": "Logs its filename and sleeps for 5 seconds", - "is_active": true, - "is_system": false, - "is_paused": false, - "is_published": false, - "icon_code": "f236", - "unmet_resources": null, - "max_scheduled": null, - "max_tries": 3, - "revision_num": 1, - "docker_image": "docker.io/fizz11/simplesleeper-1.0.0-seed:1.0.0", - "created": "2020-08-06T18:30:16.448764Z", - "deprecated": null, - "paused": null, - "last_modified": "2020-08-06T18:30:16.448807Z", - "manifest": { - "job": { - "maintainer": { - "organization": "Applied Information Sciences", - "name": "AIS", - "email": "n/a" - }, - "jobVersion": "1.0.0", - "title": "Simple Sleeper Autodeploy", - "description": "Logs its filename and sleeps for 5 seconds", - "timeout": 3600, - "interface": { - "inputs": { - "files": [ - { - "multiple": false, - "required": true, - "partial": false, - "name": "INPUT_FILE", - "mediaTypes": [] - } - ] - }, - "command": "", - "outputs": {} - }, - "packageVersion": "1.0.0", - "resources": { - "scalar": [ - { - "inputMultiplier": 0, - "name": "cpus", - "value": 0.1 - }, - { - "inputMultiplier": 0, - "name": "mem", - "value": 32 - } - ] - }, - "name": "simplesleeper-autodeploy" - }, - "seedVersion": "1.0.0" - }, - "configuration": { - "mounts": {}, - "priority": 201, - "output_workspaces": { - "default": "auto-deploy-workspace-b241736dad5a48ff8c8", - "outputs": {} - }, - "settings": {} - }, - "recipe_types": [] - } \ No newline at end of file diff --git a/autodeploy/test.sh b/autodeploy/test.sh deleted file mode 100755 index e3a4051d5c..0000000000 --- a/autodeploy/test.sh +++ /dev/null @@ -1,335 +0,0 @@ -#!/usr/bin/env sh - -API_TOKEN='fab1d1a2ace51f11ccb29246a3a1d99143908ceb' -SILO_TOKEN='eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoiYWRtaW4iLCJ1c2VybmFtZSI6ImFkbWluIn0.OAK7h0FKjwpoarCJdiuQ9q2zKW7D4KiyseyQLwfO5A8' -CLUSTER="oarfish" - -#+------------------------------------------------------ -#| NAME: validateJSON() -#| PARAMETERS: url_snip, json_file -#+------------------------------------------------------ -validateJSON() { - RESPONSE="$(curl -skLX POST \ - -H "Cookie: csrftoken=${CSRF_TOKEN}" \ - -H "Authorization: TOKEN ${API_TOKEN}" \ - -H "Content-Type: application/json" -T $2 \ - https://scale.${CLUSTER}.aisohio.net/api/v7/$1/validation)" - if [[ $(echo "${RESPONSE}" | jq '.is_valid') = "true" ]]; then - if [[ $(echo "${RESPONSE}" | jq '.diff | length') = 0 ]]; then - echo "LOG> $1: $2 is valid" - else - echo "Log> $1: Diff detected in $2\n${DIFF}" - echo "${RESPONSE}" | jq '.diff' - fi - # elif [[ $(echo "${RESPONSE}" | jq 'has("detail")') = "true" ]]; then - # ERROR=$(echo "${RESPONSE}" | jq '.detail') - # echo "LOG> $1: $2 is not valid\n${ERROR}" - # exit 1 - else - ERROR=$(echo "${RESPONSE}" | jq '.') - echo "LOG> $1: $2 is not valid\n${ERROR}" - exit 1 - fi -} - -#+------------------------------------------------------ -#| NAME: postJSON() -#| PARAMETERS: url_snip, json_file -#+------------------------------------------------------ -postJSON() { - RESPONSE=$(curl -skLX GET \ - -H "Cookie: csrftoken=${CSRF_TOKEN}" \ - -H "Authorization: TOKEN ${API_TOKEN}" \ - https://scale.${CLUSTER}.aisohio.net/api/v7/$1) - - case $1 in - workspaces|recipe-types|strikes) - NAME=$(jq '.name' $2);; - job-types) - NAME=$(jq '.manifest.job.name' $2) - VERSION=$(jq '.manifest.job.jobVersion' $2);; - esac - - if [[ $(echo "${RESPONSE}" | jq ".results[] | select(.name == "${NAME}") | has(\"id\")") = "true" ]]; then - echo "LOG> $1: $2 is a duplicate" - case $1 in - workspaces|strikes) - ID=$(echo "${RESPONSE}" | jq ".results[] | select(.name == "${NAME}") | .id") - URL=$(echo "https://scale.${CLUSTER}.aisohio.net/api/v7/$1/${ID}" | sed 's/"//g');; - job-types) - URL=$(echo "https://scale.${CLUSTER}.aisohio.net/api/v7/$1/${NAME}/${VERSION}" | sed 's/"//g');; - recipe-types) - URL=$(echo "https://scale.${CLUSTER}.aisohio.net/api/v7/$1/${NAME}" | sed 's/"//g');; - esac - - RESPONSE=$(curl -skLX PATCH \ - -H "Cookie: csrftoken=${CSRF_TOKEN}" \ - -H "Authorization: TOKEN ${API_TOKEN}" \ - -H "Content-Type: application/json" -T $2 \ - ${URL}) - if [[ $(echo "${RESPONSE}" | jq '.is_valid') != "true" ]]; then - if [ "${RESPONSE}" != "" ]; then - echo "LOG> $1: Response Invalid, could not PATCH $2" - echo "${RESPONSE}" - exit 1 - fi - fi - echo "LOG> $1: Successful PATCH of $2" - RESPONSE=$(curl -skLX GET \ - -H "Cookie: csrftoken=${CSRF_TOKEN}" \ - -H "Authorization: TOKEN ${API_TOKEN}" \ - ${URL}) - echo "${RESPONSE}" | jq '.' - exit 0 - else - echo "LOG> $1: $2 is original" - RESPONSE=$(curl -skLX POST \ - -H "Cookie: csrftoken=${CSRF_TOKEN}" \ - -H "Authorization: TOKEN ${API_TOKEN}" \ - -H "Content-Type: application/json" -T $2 \ - https://scale.${CLUSTER}.aisohio.net/api/v7/$1) - if [[ ! $(echo "${RESPONSE}" | jq 'has("id","name")') =~ "false" ]]; then - echo "LOG> $1: $2 posted!" - echo "${RESPONSE}" | jq '.' - else - echo "LOG> $1: Response invalid, could not POST $2" - echo "${RESPONSE}" | jq '.' - exit 1 - fi - fi -} - -# oldPostJSON() { -# RESPONSE=$(curl -skLX POST \ -# -H "Cookie: csrftoken=${CSRF_TOKEN}" \ -# -H "Authorization: TOKEN ${API_TOKEN}" \ -# -H "Content-Type: application/json" -T $2 \ -# https://scale.${CLUSTER}.aisohio.net/api/v7/$1) - -# # Check for ID and Name Fields -# if [ "${RESPONSE}" = '{"is_valid":true,"errors":[],"warnings":[]}' ]; then -# echo "LOG> $1 already exists!" -# NAME=$(jq '.manifest.job.name' $2) -# RESPONSE=$(curl -skLX GET \ -# -H "Cookie: csrftoken=${CSRF_TOKEN}" \ -# -H "Authorization: TOKEN ${API_TOKEN}" \ -# -H "Content-Type: application/json" -T $2 \ -# https://scale.${CLUSTER}.aisohio.net/api/v7/$1/${NAME}) - -# # Get JSON for existing Workspace/Job/Recipe/Strike -# case $1 in -# workspaces|strikes) -# ID=$(jq '.id' $2 | sed 's/"//g') -# RESPONSE=$(curl -skLX GET \ -# -H "Cookie: csrftoken=${CSRF_TOKEN}" \ -# -H "Authorization: TOKEN ${API_TOKEN}" \ -# https://scale.${CLUSTER}.aisohio.net/api/v7/$1/${ID});; -# job-types) -# NAME=$(jq '.manifest.job.name' $2 | sed 's/"//g') -# RESPONSE=$(curl -skLX GET \ -# -H "Cookie: csrftoken=${CSRF_TOKEN}" \ -# -H "Authorization: TOKEN ${API_TOKEN}" \ -# https://scale.${CLUSTER}.aisohio.net/api/v7/$1/${NAME});; -# recipe-types) -# NAME=$(jq '.name' $2 | sed 's/"//g') -# RESPONSE=$(curl -skLX GET \ -# -H "Cookie: csrftoken=${CSRF_TOKEN}" \ -# -H "Authorization: TOKEN ${API_TOKEN}" \ -# https://scale.${CLUSTER}.aisohio.net/api/v7/$1/${NAME});; -# *) -# echo "LOG> Incorrect URL!";; -# esac - -# elif [[ ! $(echo "${RESPONSE}" | jq 'has("id","name")') =~ "false" ]]; then -# echo "LOG> $1 posted!" -# echo "${RESPONSE}" | jq '.' -# exit 1 -# else -# echo "LOG> Response invalid, unsuccessful post" -# echo "${RESPONSE}" | jq '.' -# exit 1 -# fi -# } - -#+------------------------------------------------------ -#| NAME: checkDuplicates() -#| PARAMETERS: variable_checked, parameter -#+------------------------------------------------------ -checkDuplicates() { - if [[ -n $1 ]]; then - echo "Error: '$2' is a duplicate" - echo "Note: './autodeploy.sh --help' or './autodeploy.sh -h' for more information" - exit 1 - fi -} - -#+------------------------------------------------------ -#| NAME: checkFile() -#| PARAMETERS: file_name -#+------------------------------------------------------ -checkFile() { - if [[ ! -f $1 ]]; then - echo "Error: '$1' doesn't exists" - echo "Usage: ./autodeploy.sh $2 <file>" - exit 1 - fi -} - -# Replace equal sign format -set -- $(echo "$@" | sed 's/[=]/ /g') - -# Argument Parsing -if [ $# == 0 ]; then - echo "Usage: ./autodeploy.sh [options...]" - echo "Note: './autodeploy.sh --help' or './autodeploy.sh -h' for more information" - exit 0 -fi - -while [[ "$#" -gt 0 ]]; do - OPTION=$1 - case $1 in - -h|--help) - echo "Usage: ./autodeploy.sh [options...]" - echo " -w, --workspace <file> verifies and creates a workspace based on the json file provied" - echo " -j, --job-type <file> verifies and creates a job type based on the json file provied" - echo " -r, --recipe-type <file> verifies and creates a recipe type based on the json file provied" - echo " -s, --strike <file> verifies and creates a strike based on the json file provied" - echo " --silo scans registries to update images" - echo "Note: short fromat not supported (ex. './autodeploy.sh -wjrs')" - exit 0;; - -w|--workspace) - checkDuplicates "${WORKSPACE}" "$1" - shift - checkFile "$1" "${OPTION}" - WORKSPACE="$1";; - -j|--job-type) - checkDuplicates "${JOB_TYPE}" "$1" - shift - checkFile "$1" "${OPTION}" - JOB_TYPE="$1";; - -r|--recipe-type) - checkDuplicates "${RECIPE_TYPE}" "$1" - shift - checkFile "$1" "${OPTION}" - RECIPE_TYPE="$1";; - -s|--strike) - checkDuplicates "${STRIKE}" "$1" - shift - checkFile "$1" "${OPTION}" - STRIKE="$1";; - --silo) - # checkDuplicates "${STRIKE}" "-s" - # echo "Login into ${CLUSTER} SILO:\n-------------------------------" - # read -p 'Username: ' USERNAME - # read -sp 'Password: ' PASSWORD - # echo "" - SILO="true";; - *) - echo "Unkown parameter passed: $1" - exit 1;; - esac - shift -done - -# SILO -if [[ -n ${SILO} ]]; then - SILO_TOKEN=$(curl -sk \ - -H "Content-Type: application/json" \ - -d "{\"username\":\"${USERNAME}\", \"password\": \"${PASSWORD}\"}" \ - https://scale-silo.${CLUSTER}.aisohio.net/login | jq '.token' | sed 's/"//g') - if [[ ${SILO_TOKEN} = "null" ]]; then - echo "LOG> Incorrect Username or Password" - exit 0 - fi - -curl -skLX GET \ -https://scale.oarfish.aisohio.net/api/admin --cookie-jar - - - -curl -skDX GET "https://scale.oarfish.aisohio.net/api/login/?next=https://scale.oarfish.aisohio.net/" - -curl -skX POST \ --H "Cookie: csrftoken=Ee6qDB4rQ1StvmelEsGVwLPlYqFJPDhfpiP1DHp94FliJkGn2zUn75NaR9nAOH8l" \ --H "Content-Type: application/json" \ --d "{\"username\":\"admin\", \"password\": \"admin\", \"csrfmiddlewaretoken\": \"20Z4logmOEj1qwqbUvGmVlVt5hN54bC8N4IFluB42iMQEuSdiCUOwFTiY0vW3fte\"}" \ -"https://scale.oarfish.aisohio.net/api/login/?next=https://scale.oarfish.aisohio.net/" - -csrfmiddlewaretoken=uJsTXB2vDMCak5H5pfrc3LyCmuBZGpjmMpU6RuAGCZ45DQqDZcyaugKlhkKDZeAL&username=admin&password=sdfsd - - - -csrfmiddlewaretoken: uJsTXB2vDMCak5H5pfrc3LyCmuBZGpjmMpU6RuAGCZ45DQqDZcyaugKlhkKDZeAL - -oiViw3q5f1oKl9OpRphWU9MKeylwNyoS0cVrev2voy65uqUav242XN0Iw - - - -RESPONSE=$(curl -skLX PATCH \ - -H "Cookie: csrftoken=${CSRF_TOKEN}" \ - -H "Authorization: TOKEN ${API_TOKEN}" \ - -H "Content-Type: application/json" -T $2 \ - #echo "LOG> SILO TOKEN: ${SILO_TOKEN}" - - # curl -sk \ - # -H "Authorization: Token ${SILO_TOKEN}" \ - # -H "Content-Type: application/json" \ - # -d "{\"name\":\"${USERNAME}\", \"url\":\"https://hub.docker.com\", \"org\":\"${USERNAME}\", \"username\":\"${USERNAME}\", \"password\": \"${PASSWORD}\"}" \ - # https://scale-silo.${CLUSTER}.aisohio.net/registries/add - - curl -sk \ - -H "Authorization: Token ${SILO_TOKEN}" \ - -H "Content-Type: application/json" \ - https://scale-silo.${CLUSTER}.aisohio.net/registries/scan - -fi - -# Get CSRF Token -CSRF_TOKEN=$(curl -skLX GET \ -https://scale.${CLUSTER}.aisohio.net/api/admin --cookie-jar - \ -| grep 'csrftoken' | sed 's/^.*csrftoken[[:space:]]]*//g' ) -echo "LOG> CSRF TOKEN: ${CSRF_TOKEN}" - -# Workspace -if [[ -n ${WORKSPACE} ]]; then - validateJSON "workspaces" ${WORKSPACE} - postJSON "workspaces" ${WORKSPACE} - - # Add Workspace Name to job-type - if [[ -n ${JOB_TYPE} ]]; then - NAME_WORKSPACE="$(echo ${RESPONSE} | jq '.name' ${RESPONSE})" - JOB_TYPE_EDIT="$(jq ".configuration.output_workspaces.default=${NAME_WORKSPACE}" ${JOB_TYPE})" && \ - echo "${JOB_TYPE_EDIT}" > "${JOB_TYPE}" - fi -fi - -# Job-Type -if [[ -n ${JOB_TYPE} ]]; then - validateJSON 'job-types' "${JOB_TYPE}" - postJSON 'job-types' "${JOB_TYPE}" - - # Add Job-Type to Recipe-type - if [[ -n ${RECIPE_TYPE} ]]; then - JOB_TYPE_JSON="$(echo ${RESPONSE} | jq '.results')" - RECIPE_TYPE_EDIT="$(jq ".job_types=${JOB_TYPE_JSON}" ${RECIPE_TYPE})" && \ - echo "${RECIPE_TYPE_EDIT}" > "${RECIPE_TYPE}" - fi -fi - -# Recipe-Type -if [[ -n ${RECIPE_TYPE} ]]; then - validateJSON "recipe-types" ${RECIPE_TYPE} - postJSON 'recipe-types' "${RECIPE_TYPE}" - - # Add Recipe Name to strike - if [[ -n ${STRIKE} ]]; then - NAME_RECIPE="$(echo ${RESPONSE} | jq '.name')" - STRIKE_EDIT="$(jq ".configuration.recipe.name=${NAME_RECIPE}" ${STRIKE})" && \ - echo "${STRIKE_EDIT}" > "${STRIKE}" - fi -fi - -# Strike -if [[ -n ${STRIKE} ]]; then - validateJSON "strikes" "${STRIKE}" - postJSON 'strikes' "${STRIKE}" -fi \ No newline at end of file diff --git a/autodeploy/workspace.json b/autodeploy/workspace.json deleted file mode 100644 index 178b93142d..0000000000 --- a/autodeploy/workspace.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "id": null, - "name": "test-deploy-workspace", - "title": "Test Deploy Workspace", - "description": "", - "base_url": null, - "is_active": true, - "created": null, - "deprecated": null, - "last_modified": null, - "configuration": { - "broker": { - "host_path": "/dfs/testing/test-deploy", - "type": "host" - } - } -} \ No newline at end of file From 0f08c7f0be078bac4baee28fb04fc2d2ec286f73 Mon Sep 17 00:00:00 2001 From: Matt Talda <matthew.talda@appliedis.com> Date: Mon, 24 Aug 2020 10:33:39 -0400 Subject: [PATCH 6/7] added new lines --- autodeploy/docker/autodeploy.sh | 2 +- autodeploy/templates/strike-template.json | 2 +- autodeploy/templates/workspace-template.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/autodeploy/docker/autodeploy.sh b/autodeploy/docker/autodeploy.sh index e99c31eb44..cf7d6105f5 100755 --- a/autodeploy/docker/autodeploy.sh +++ b/autodeploy/docker/autodeploy.sh @@ -250,4 +250,4 @@ fi if [[ -n ${STRIKE} ]]; then validateJSON "strikes" "${STRIKE}" postJSON 'strikes' "${STRIKE}" -fi \ No newline at end of file +fi diff --git a/autodeploy/templates/strike-template.json b/autodeploy/templates/strike-template.json index cedafaa3f7..7a2f94cb9e 100644 --- a/autodeploy/templates/strike-template.json +++ b/autodeploy/templates/strike-template.json @@ -38,4 +38,4 @@ } ] } -} \ No newline at end of file +} diff --git a/autodeploy/templates/workspace-template.json b/autodeploy/templates/workspace-template.json index e41d818952..89d2be7fbf 100644 --- a/autodeploy/templates/workspace-template.json +++ b/autodeploy/templates/workspace-template.json @@ -14,4 +14,4 @@ "type": "host" } } -} \ No newline at end of file +} From 1ec671a425ff0acaa8e61be9d88b49d01bfab4ee Mon Sep 17 00:00:00 2001 From: Matt Talda <matthew.talda@appliedis.com> Date: Mon, 24 Aug 2020 10:52:10 -0400 Subject: [PATCH 7/7] last nl --- autodeploy/run-autodeploy.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/autodeploy/run-autodeploy.sh b/autodeploy/run-autodeploy.sh index 3abd35498a..c027c36737 100755 --- a/autodeploy/run-autodeploy.sh +++ b/autodeploy/run-autodeploy.sh @@ -1,4 +1,5 @@ #!/usr/bin/env sh ARGS=$@ -docker run --rm -v "${PWD}:/usr/autodeploy/volume" mtalda/autodeploy ${ARGS} \ No newline at end of file +docker run --rm -v "${PWD}:/usr/autodeploy/volume" mtalda/autodeploy ${ARGS} + \ No newline at end of file