Suggestion for a spark build script

spark-2.0.1-install

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+###############################################
+# Installing spark
+#
+# by James Hetherington, 2016
+#
+
+APPNAME=${APPNAME:-spark}
+VERSION=${VERSION:-2.0.1}
+# We do not need Hadoop. But the built jars "binaries" contain a Hadoop version.
+HADOOPVERSION=${HADOOPVERSION:-2.7}
+INSTALL_ZONE=/home/ucgajhe/software/
+# INSTALL will be to /shared/ucl/apps/
+INSTALL_PREFIX=${INSTALL_PREFIX:-$INSTALL_ZONE/$APPNAME/$VERSION}
+# TEMPDIR will be on /dev/shm
+TEMP_ZONE=/home/ucgajhe/software/shm
+
+SHA256=${SHA256:-3d017807650f41377118a736e2f2298cd0146a593e7243a28c2ed72a88b9a043}
+SRC_ARCHIVE=${SRC_ARCHIVE:-http://mirror.catn.com/pub/apache/$APPNAME/$APPNAME-$VERSION/$APPNAME-$VERSION-bin-hadoop${HADOOPVERSION}.tgz}
+
+set -e
+
+export PATH=$INSTALL_PREFIX/bin:$PATH
+
+echo Install prefix : $INSTALL_PREFIX
+
+mkdir -p $TEMP_ZONE/$APPNAME
+temp_dir=`mktemp -d -p $TEMP_ZONE/$APPNAME`
+
+cd $temp_dir
+
+wget -O ${APPNAME}-${VERSION}.tgz $SRC_ARCHIVE
+
+CHECKSUM=`sha256sum ${APPNAME}-${VERSION}.tgz |  awk '{print $1}'`
+
+if [ "$SHA256" == "$CHECKSUM" ]
+then
+  mkdir -p $INSTALL_PREFIX
+  cd $INSTALL_PREFIX
+  tar -xzvf $temp_dir/${APPNAME}-${VERSION}.tgz
+  mv spark-${VERSION}-bin-hadoop${HADOOPVERSION}/* .
+  rmdir spark-${VERSION}-bin-hadoop${HADOOPVERSION}
+  wget -O bin/sparkrun https://raw.githubusercontent.com/jamespjh/rcps-buildscripts/master/spark-files/sparkrun
+  chmod a+x bin/sparkrun
+else
+  echo "Hash mismatch."
+  echo "Expected: $SHA256"
+  echo "Got: $CHECKSUM"
+fi

spark-files/sparkrun

@@ -0,0 +1,36 @@
+# A gerun-inspired wrapper for running spark jobs on UCL Clusters
+# Mostly based on a script by Jonathan Dursi
+# from this post: http://www.dursi.ca/spark-in-hpc-clusters/
+
+# Make a per-job output directory
+OUTPUT_DIR="$(pwd)/run.${JOB_ID}"
+mkdir -p "$OUTPUT_DIR"
+
+# Get our list of allocated nodes and how many
+nodes=($( sort -u <"$TMPDIR/machines" | sed -e 's/$/.data.legion.ucl.ac.uk/' ))
+nnodes=${#nodes[@]}  
+last=$(( nnodes - 1 ))
+
+export SPARK_LOCAL_DIRS="$TMPDIR"
+
+ssh "${nodes[0]}" "module load java; cd ${SPARK_HOME}; ./sbin/start-master.sh"  
+sparkmaster="spark://${nodes[0]}:7077"
+
+# Start the spark workers on all nodes
+for i in $( seq 0 $last )  
+do  
+    ssh "${nodes[$i]}" "cd ${SPARK_HOME}; module load java; export SPARK_LOCAL_DIRS=\"$TMPDIR\"; nohup spark-class org.apache.spark.deploy.worker.Worker ${sparkmaster} &> ${OUTPUT_DIR}/nohup-${nodes[$i]}.out" &
+done
+
+# Submit the script to the Spark cluster
+
+spark-submit --master "${sparkmaster}" "$@"
+
+# Stop the Spark master and kill all the Spark processes to clean up
+for i in $( seq 0 $last )  
+do  
+    ssh "${nodes[$i]}" "module load java; cd ${SPARK_HOME}; ./sbin/stop-slaves.sh" &
+done  
+
+ssh "${nodes[0]}" "module load java; cd ${SPARK_HOME}; ./sbin/stop-master.sh" &
+wait