diff --git a/CondorMonitoring/JobCounter.sh b/CondorMonitoring/JobCounter.sh index 7f9997a2a..25f94118c 100755 --- a/CondorMonitoring/JobCounter.sh +++ b/CondorMonitoring/JobCounter.sh @@ -8,10 +8,12 @@ location="/afs/cern.ch/user/c/cmst1/CondorMonitoring" outputdir="/afs/cern.ch/user/c/cmst1/www/CondorMonitoring/" -#Set environment +# Set environment +# shellcheck disable=SC1091 source /data/admin/wmagent/env.sh +# shellcheck disable=SC1091 source /data/srv/wmagent/current/apps/wmagent/etc/profile.d/init.sh -cd $location +cd $location||exit #Email if things are running slowly if [ -f scriptRunning.run ]; @@ -33,6 +35,6 @@ fi python JobCounter.py &> JobCounter.log exitstatus="$?" echo "JobCounter.py exit status: $exitstatus" -cp *.json $outputdir -cp *.txt $outputdir +cp ./*.json $outputdir +cp ./*.txt $outputdir rm scriptRunning.run \ No newline at end of file diff --git a/PhEDEXFix/phedex-fix.sh b/PhEDEXFix/phedex-fix.sh index e5f6304ae..ae54da7a0 100755 --- a/PhEDEXFix/phedex-fix.sh +++ b/PhEDEXFix/phedex-fix.sh @@ -1,10 +1,12 @@ #!/bin/bash ./config/wmagent/manage execute-agent wmcoreD --shutdown --components=PhEDExInjector ./config/wmagent/manage mysql-prompt wmagentlfns.txt +# shellcheck disable=SC2002 cat lfns.txt | grep /store > lfnsgreped.txt #tail -n 28 install/wmagent/PhEDExInjector/ComponentLog | head -n1 > phedex.log #for j in $(for i in $(cat phedex.log) ; do echo $i | awk -F'"' {'print $2'} ; done | grep root) ; do echo $j; done > lfns.txt ./fileInPhedex.py lfnsgreped.txt > lfnsinPhedex.txt +# shellcheck disable=SC2013 for i in $(cat lfnsinPhedex.txt); do echo "UPDATE dbsbuffer_file set in_phedex = 1 where lfn='$i';";done > meuscript.sql ./config/wmagent/manage mysql-prompt wmagent < meuscript.sql ./config/wmagent/manage execute-agent wmcoreD --shutdown --components=PhEDExInjector diff --git a/StoreResults/StoreResultsAddCMSSWReleases.sh b/StoreResults/StoreResultsAddCMSSWReleases.sh index c9a2ed54b..152acaacd 100755 --- a/StoreResults/StoreResultsAddCMSSWReleases.sh +++ b/StoreResults/StoreResultsAddCMSSWReleases.sh @@ -1,3 +1,4 @@ #!/bin/bash +# shellcheck disable=SC1091 source /data/srv/wmagent/current/apps/wmagent/etc/profile.d/init.sh python2.6 /home/cmsdataops/storeResults/cronjobs/StoreResultsAddCMSSWReleases.py diff --git a/Unified/exec_expose.sh b/Unified/exec_expose.sh index e3c01522e..e72e115d0 100755 --- a/Unified/exec_expose.sh +++ b/Unified/exec_expose.sh @@ -5,7 +5,8 @@ loc=$4 dest=$5 short=$6 -echo "Parameters passed to execution:" $wf $p $er $loc $dest $short +echo "Parameters passed to execution: $wf $p $er $loc $dest $short" #sudo -u cmst1 /bin/bash --init-file ~cmst1/.bashrc $loc/WmAgentScripts/Unified/expose.sh $wf $p $er $dest $short -source $loc/WmAgentScripts/Unified/expose.sh $wf $p $er $dest $short \ No newline at end of file +# shellcheck disable=SC2034,SC1090,SC1091 +source "$loc"/WmAgentScripts/Unified/expose.sh "$wf $p $er $dest $short" \ No newline at end of file diff --git a/Unified/expose.sh b/Unified/expose.sh index 2be1801fa..8f32c5eb2 100755 --- a/Unified/expose.sh +++ b/Unified/expose.sh @@ -1,7 +1,7 @@ #!/bin/bash -host=`echo $HOSTNAME | cut -d "." -f 1` -echo "looking for condor log on" $host +host=$(echo "$HOSTNAME" | cut -d "." -f 1) +echo "looking for condor log on $host" taskName=$1 wmbsID=$2 code=$3 @@ -9,56 +9,57 @@ cluster=${wmbsID:0:3} loc=$4 short=$5 -#end_dir=/afs/cern.ch/user/c/cmst1/www/JobLogs/$taskName/$cluster/$host\_$wmbsID -#end_dir=/afs/cern.ch/user/c/cmst2/www/unified/condorlogs/$taskName/$code/$cluster/$host\_$wmbsID +# end_dir=/afs/cern.ch/user/c/cmst1/www/JobLogs/$taskName/$cluster/$host\_$wmbsID +# end_dir=/afs/cern.ch/user/c/cmst2/www/unified/condorlogs/$taskName/$code/$cluster/$host\_$wmbsID +# shellcheck disable=SC1001 end_dir=$loc/condorlogs/$taskName/$code/$short/$cluster/$host\_$wmbsID echo '==================================== Condor log retrieval =============================================' -echo 'LISTING: ' /data/srv/wmagent/current/install/wmagent/JobCreator/JobCache/$taskName/*/job_$wmbsID/ -dir=`ls -d /data/srv/wmagent/current/install/wmagent/JobCreator/JobCache/$taskName/*/job_$wmbsID/` -if [ -z $dir ]; then +echo 'LISTING: ' /data/srv/wmagent/current/install/wmagent/JobCreator/JobCache/"$taskName"/*/job_"$wmbsID"/ +dir=$(ls -d /data/srv/wmagent/current/install/wmagent/JobCreator/JobCache/"$taskName"/*/job_"$wmbsID"/) +if [ -z "$dir" ]; then echo "no on-going directory" else - echo " Found some logs in" $dir - echo "Will put it in " $end_dir - mkdir -p $end_dir - cp -r $dir/* $end_dir/. + echo " Found some logs in $dir" + echo "Will put it in $end_dir" + mkdir -p "$end_dir" + cp -r "$dir"/* "$end_dir"/. fi echo '==================================== Condor log retrieval =============================================' -echo 'LISTING: ' /data/srv/wmagent/current/install/wmagent/JobCreator/JobCache/$taskName/*/*/job_$wmbsID/ -dir=`ls -d /data/srv/wmagent/current/install/wmagent/JobCreator/JobCache/$taskName/*/*/job_$wmbsID/` -if [ -z $dir ]; then +echo 'LISTING: ' /data/srv/wmagent/current/install/wmagent/JobCreator/JobCache/"$taskName"/*/*/job_"$wmbsID"/ +dir=$(ls -d /data/srv/wmagent/current/install/wmagent/JobCreator/JobCache/"$taskName"/*/*/job_"$wmbsID"/) +if [ -z "$dir" ]; then echo "no on-going directory" else - echo " Found some logs in" $dir - echo "Will put it in " $end_dir - mkdir -p $end_dir - cp -r $dir/* $end_dir/. + echo " Found some logs in $dir" + echo "Will put it in $end_dir" + mkdir -p "$end_dir" + cp -r "$dir"/* "$end_dir"/. fi echo '==================================== Condor log retrieval =============================================' -echo 'LISTING: ' /data/srv/wmagent/current/install/wmagent/JobArchiver/logDir/${taskName:0:1}/$taskName/JobCluster_$cluster/Job_$wmbsID.tar.bz2 -file=`ls /data/srv/wmagent/current/install/wmagent/JobArchiver/logDir/${taskName:0:1}/$taskName/JobCluster_$cluster/Job_$wmbsID.tar.bz2` -if [ -z $file ]; then +echo 'LISTING: ' /data/srv/wmagent/current/install/wmagent/JobArchiver/logDir/"${taskName:0:1}"/"$taskName"/JobCluster_"$cluster"/Job_"$wmbsID".tar.bz2 +file=$(ls /data/srv/wmagent/current/install/wmagent/JobArchiver/logDir/"${taskName:0:1}"/"$taskName"/JobCluster_"$cluster"/Job_"$wmbsID".tar.bz2) +if [ -z "$file" ]; then echo "no archived file" else - echo " Found some logs in" $file - echo "Will put it in " $end_dir - mkdir -p $end_dir - cd $end_dir - tar xvf $file + echo " Found some logs in" "$file" + echo "Will put it in $end_dir" + mkdir -p "$end_dir" + cd "$end_dir"||exit + tar xvf "$file" fi echo '==================================== Condor log retrieval =============================================' cluster=${wmbsID:0:4} -echo 'LISTING: ' /data/srv/wmagent/current/install/wmagent/JobArchiver/logDir/${taskName:0:1}/$taskName/JobCluster_$cluster/Job_$wmbsID.tar.bz2 -file=`ls /data/srv/wmagent/current/install/wmagent/JobArchiver/logDir/${taskName:0:1}/$taskName/JobCluster_$cluster/Job_$wmbsID.tar.bz2` -if [ -z $file ]; then +echo 'LISTING: ' /data/srv/wmagent/current/install/wmagent/JobArchiver/logDir/"${taskName:0:1}"/"$taskName"/JobCluster_"$cluster"/Job_"$wmbsID".tar.bz2 +file=$(ls /data/srv/wmagent/current/install/wmagent/JobArchiver/logDir/"${taskName:0:1}"/"$taskName"/JobCluster_"$cluster"/Job_"$wmbsID".tar.bz2) +if [ -z "$file" ]; then echo "no archived file" else - echo " Found some logs in" $file - echo "Will put it in " $end_dir - mkdir -p $end_dir - cd $end_dir - tar xvf $file + echo " Found some logs in" "$file" + echo "Will put it in " "$end_dir" + mkdir -p "$end_dir" + cd "$end_dir"||exit + tar xvf "$file" fi \ No newline at end of file diff --git a/WFComplete_submitter.sh b/WFComplete_submitter.sh index 685e930e3..f083f89c3 100755 --- a/WFComplete_submitter.sh +++ b/WFComplete_submitter.sh @@ -1,4 +1,4 @@ #!/bin/sh echo "Executing $1 as cmst1..." -sudo -u cmst1 /bin/bashs --init-file $1 +sudo -u cmst1 /bin/bashs --init-file "$1" diff --git a/actcycle.sh b/actcycle.sh index c91dc47c7..a9df2a77c 100755 --- a/actcycle.sh +++ b/actcycle.sh @@ -1,14 +1,18 @@ BASE_DIR=/data/unified/WmAgentScripts/ +# shellcheck disable=SC2034 HTML_DIR=/var/www/html/unified/ -lock_name=`echo $BASH_SOURCE | cut -f 1 -d "."`.lock -source $BASE_DIR/cycle_common.sh $lock_name +# shellcheck disable=SC1090,SC2128 +lock_name=$(echo "$BASH_SOURCE" | cut -f 1 -d ".").lock +# shellcheck disable=SC1090 +source $BASE_DIR/cycle_common.sh "$lock_name" ## get sso cookie and new grid proxy +# shellcheck disable=SC1090 source $BASE_DIR/credentials.sh ## submit ACDCs and clones from actions submitted via new recovery tools $BASE_DIR/cWrap.sh Unified/actor.py -rm -f $lock_name +rm -f "$lock_name" diff --git a/assigncycle.sh b/assigncycle.sh index ed84de84b..d1f768599 100755 --- a/assigncycle.sh +++ b/assigncycle.sh @@ -1,8 +1,11 @@ BASE_DIR=/data/unified/WmAgentScripts/ +# shellcheck disable=SC2034 HTML_DIR=/var/www/html/unified/ -lock_name=`echo $BASH_SOURCE | cut -f 1 -d "."`.lock -source $BASE_DIR/cycle_common.sh $lock_name +# shellcheck disable=SC2128 +lock_name=$(echo "$BASH_SOURCE" | cut -f 1 -d ".").lock +# shellcheck disable=SC1090 +source $BASE_DIR/cycle_common.sh "$lock_name" ## get the workflow in/out of the system $BASE_DIR/cWrap.sh Unified/injector.py @@ -26,5 +29,5 @@ $BASE_DIR/cWrap.sh Unified/assignor.py _PR_ref ## assign the workflow to sites $BASE_DIR/cWrap.sh Unified/assignor.py -rm -f $lock_name +rm -f "$lock_name" diff --git a/cWrap.sh b/cWrap.sh index f315d126b..81df89842 100755 --- a/cWrap.sh +++ b/cWrap.sh @@ -7,74 +7,79 @@ if [ ! -d $FINAL_HTML_DIR ] ; then echo "Cannot read the log destination",$FINAL_HTML_DIR exit fi -cd $BASE_DIR +cd $BASE_DIR||exit -modulename=`echo $1 | sed 's/\.py//' | sed 's/Unified\///'` -mkdir -p $HTML_DIR/logs/$modulename/ -env EOS_MGM_URL=root://eoscms.cern.ch eos mkdir -p $FINAL_HTML_DIR/logs/$modulename/ +modulename=$(echo "$1" | sed 's/\.py//' | sed 's/Unified\///') +mkdir -p $HTML_DIR/logs/"$modulename"/ +env EOS_MGM_URL=root://eoscms.cern.ch eos mkdir -p $FINAL_HTML_DIR/logs/"$modulename"/ last_log=$HTML_DIR/logs/$modulename/last.log -s_dated_log=$modulename/`date +%F_%T`.log +s_dated_log=$modulename/$(date +%F_%T).log dated_log=$HTML_DIR/logs/$s_dated_log log=$dated_log -echo `date` > $log -echo $$ >> $log +# shellcheck disable=SC2005 +echo "$(date)" > "$log" +echo $$ >> "$log" if [ -r unified_drain ] ; then - echo "System is locally draining" >> $log - cp $log $last_log - env EOS_MGM_URL=root://eoscms.cern.ch eos cp $log $FINAL_HTML_DIR/logs/$modulename/. - env EOS_MGM_URL=root://eoscms.cern.ch eos cp $log $FINAL_HTML_DIR/logs/$modulename/last.log + echo "System is locally draining" >> "$log" + cp "$log" "$last_log" + env EOS_MGM_URL=root://eoscms.cern.ch eos cp "$log" $FINAL_HTML_DIR/logs/"$modulename"/. + env EOS_MGM_URL=root://eoscms.cern.ch eos cp "$log" $FINAL_HTML_DIR/logs/"$modulename"/last.log exit fi if [ -r /eos/cms/store/unified/unified_drain ] ; then - echo "System is globally draining" >> $log - cp $log $last_log - env EOS_MGM_URL=root://eoscms.cern.ch eos cp $log $FINAL_HTML_DIR/logs/$modulename/. - env EOS_MGM_URL=root://eoscms.cern.ch eos cp $log $FINAL_HTML_DIR/logs/$modulename/last.log + echo "System is globally draining" >> "$log" + cp "$log" "$last_log" + env EOS_MGM_URL=root://eoscms.cern.ch eos cp "$log" $FINAL_HTML_DIR/logs/"$modulename"/. + env EOS_MGM_URL=root://eoscms.cern.ch eos cp "$log" $FINAL_HTML_DIR/logs/"$modulename"/last.log exit fi +# shellcheck disable=SC2129 +echo "$USER" >> "$log" +echo "$HOSTNAME" >> "$log" +echo module "$modulename">> "$log" - -echo $USER >> $log -echo $HOSTNAME >> $log -echo module $modulename>> $log - +# shellcheck disable=SC1091 source ./set.sh -echo >> $log +echo >> "$log" -start=`date +%s` -python ssi.py $modulename $start +start=$(date +%s) +python ssi.py "$modulename" "$start" -python $* &>> $log +python "$@" &>> "$log" +# shellcheck disable=SC2181 if [ $? == 0 ]; then - echo "finished" >> $log + echo "finished" >> "$log" else - echo -e "\nAbnormal termination with exit code $?" >> $log - top -n1 -o %MEM -c >> $log + echo -e "\nAbnormal termination with exit code $?" >> "$log" + top -n1 -o %MEM -c >> "$log" emaillog=$log.txt failed_pid=$! - echo "Abnormal termination, check $log" > $emaillog - echo "https://cms-unified.web.cern.ch/cms-unified/logs/$s_dated_log" >> $emaillog - echo $failed_pid >> $emaillog - echo $USER >> $emaillog - echo $HOSTNAME >> $emaillog - echo -e "module $modulename \n" >> $emaillog - tail $log >> $emaillog - cat $emaillog | mail -s "[Ops] module "$modulename" failed" cmsunified@cern.ch + echo "Abnormal termination, check $log" > "$emaillog" + # shellcheck disable=SC2129 + echo "https://cms-unified.web.cern.ch/cms-unified/logs/$s_dated_log" >> "$emaillog" + echo $failed_pid >> "$emaillog" + echo "$USER" >> "$emaillog" + echo "$HOSTNAME" >> "$emaillog" + echo -e "module $modulename \n" >> "$emaillog" + tail "$log" >> "$emaillog" + # shellcheck disable=SC2002 + cat "$emaillog" | mail -s "[Ops] module $modulename failed" cmsunified@cern.ch fi -stop=`date +%s` -python ssi.py $modulename $start $stop -echo `date` >> $log +stop=$(date +%s) +python ssi.py "$modulename $start $stop" +# shellcheck disable=SC2005 +echo "$(date)" >> "$log" #cp $log $dated_log -cp $log $last_log -env EOS_MGM_URL=root://eoscms.cern.ch eos cp $log $FINAL_HTML_DIR/logs/$modulename/. -env EOS_MGM_URL=root://eoscms.cern.ch eos cp $log $FINAL_HTML_DIR/logs/$modulename/last.log +cp "$log" "$last_log" +env EOS_MGM_URL=root://eoscms.cern.ch eos cp "$log" $FINAL_HTML_DIR/logs/"$modulename"/. +env EOS_MGM_URL=root://eoscms.cern.ch eos cp "$log" $FINAL_HTML_DIR/logs/"$modulename"/last.log -#rm $log +# rm $log diff --git a/checkmem.sh b/checkmem.sh index 2673d2d5d..2ea23062b 100644 --- a/checkmem.sh +++ b/checkmem.sh @@ -1,8 +1,8 @@ #log the memory of all for each in 0268 0269 0272 0273 0274 0275; do - lf='/data/unified/www/logs/checkmem/'`date +%F_%T`.$each.memlog - date > $lf - ssh vocms$each "ps -o pid,user,rss,%mem,args --sort -pmem -e | head -10" >> $lf + lf='/data/unified/www/logs/checkmem/'$(date +%F_%T).$each.memlog + date > "$lf" + ssh vocms$each "ps -o pid,user,rss,%mem,args --sort -pmem -e | head -10" >> "$lf" done ## keep it clean find /data/unified/www/logs/checkmem/ -type f -name '*.memlog' -mtime +1 -exec rm {} \; diff --git a/cleanBackfills.sh b/cleanBackfills.sh index 8dd7e63ec..44a5407e5 100755 --- a/cleanBackfills.sh +++ b/cleanBackfills.sh @@ -1,7 +1,9 @@ -cd /data/unified/WmAgentScripts +# shellcheck disable=SC1091 +cd /data/unified/WmAgentScripts||exit -oweek=`date +%W` +oweek=$(date +%W) week=${oweek#0} +# shellcheck disable=SC2219,SC2034 let oddity=week%2 export X509_USER_PROXY=$HOME/private/personal/voms_proxy.cert diff --git a/clearcycle.sh b/clearcycle.sh index 525a7d86d..e728704f9 100755 --- a/clearcycle.sh +++ b/clearcycle.sh @@ -1,8 +1,11 @@ BASE_DIR=/data/unified/WmAgentScripts/ +# shellcheck disable=SC2034 HTML_DIR=/var/www/html/unified/ -lock_name=`echo $BASH_SOURCE | cut -f 1 -d "."`.lock -source $BASE_DIR/cycle_common.sh $lock_name +# shellcheck disable=SC2128 +lock_name=$(echo "$BASH_SOURCE" | cut -f 1 -d ".").lock +# shellcheck disable=SC1090 +source $BASE_DIR/cycle_common.sh "$lock_name" $BASE_DIR/cWrap.sh Unified/completor.py @@ -11,5 +14,5 @@ $BASE_DIR/cWrap.sh Unified/closor.py ## early announce what can be announced already $BASE_DIR/cWrap.sh Unified/closor.py --announce -rm -f $lock_name +rm -f "$lock_name" diff --git a/condor_rm48h.sh b/condor_rm48h.sh index 74bb9d9dd..9281a4fec 100755 --- a/condor_rm48h.sh +++ b/condor_rm48h.sh @@ -1,42 +1,45 @@ #!/bin/sh +# shellcheck disable=SC1091 source ~cmst1/.bashrc agentenv -tmp=`mktemp` -condor_overview | sed -n '/48 hours by workflow/,/-----/p' | grep " : " >> $tmp +tmp=$(mktemp) +condor_overview | sed -n '/48 hours by workflow/,/-----/p' | grep " : " >> "$tmp" -while read line ; do +while read -r line ; do - WF=`echo "$line" | awk -F' : ' '{print $1}'` - Jobs=`echo "$line" | awk -F' : ' '{print $2}'` - echo "Checking : " $WF + WF=$(echo "$line" | awk -F' : ' '{print $1}') + Jobs=$(echo "$line" | awk -F' : ' '{print $2}') + echo "Checking : $WF" nRem=0 for iJob in $Jobs ; do - nStarts=`condor_q -l $iJob | grep NumJobStarts | awk -F' = ' '{print $2}'` + nStarts=$(condor_q -l "$iJob" | grep NumJobStarts | awk -F' = ' '{print $2}') if [[ $nStarts -gt 1 ]]; then - condor_rm $iJob - nRem=`expr $nRem + 1` + condor_rm "$iJob" + # shellcheck disable=SC2003 + nRem=$(expr $nRem + 1) fi done - echo " ---> "$nRem" Jobs Removed" + echo " ---> $nRem Jobs Removed" done < "$tmp" -rm $tmp +rm "$tmp" -tmp=`mktemp` -condor_overview | sed -n '/Jobs with RemoveReason!=UNDEFINED/,/-----/p' | grep " : " >> $tmp +tmp=$(mktemp) +condor_overview | sed -n '/Jobs with RemoveReason!=UNDEFINED/,/-----/p' | grep " : " >> "$tmp" -while read line ; do +while read -r line ; do - WF=`echo "$line" | awk -F' : ' '{print $1}'` - Jobs=`echo "$line" | awk -F' : ' '{print $2}'` - echo "Checking : " $WF + WF=$(echo "$line" | awk -F' : ' '{print $1}') + Jobs=$(echo "$line" | awk -F' : ' '{print $2}') + echo "Checking : $WF" nRem=0 for iJob in $Jobs ; do - condor_rm -forcex $iJob - nRem=`expr $nRem + 1` + condor_rm -forcex "$iJob" + # shellcheck disable=SC2003 + nRem=$(expr $nRem + 1) done done < "$tmp" -rm $tmp +rm "$tmp" diff --git a/copyLFNfromSite.sh b/copyLFNfromSite.sh index 65d8c9c36..7e04d6386 100755 --- a/copyLFNfromSite.sh +++ b/copyLFNfromSite.sh @@ -4,7 +4,7 @@ if [[ "$1" == T1* ]]; then else CURL_URL="https://cmsweb.cern.ch/phedex/datasvc/xml/prod/lfn2pfn?node=$1&protocol=srmv2&lfn=$2" fi -echo $CURL_URL -file=$(curl -s --insecure -X GET $CURL_URL | awk -F"'" '{print $32}') -echo $file -lcg-cp -v -n 1 $file file:////`pwd`/$3 +echo "$CURL_URL" +file=$(curl -s --insecure -X GET "$CURL_URL" | awk -F"'" '{print $32}') +echo "$file" +lcg-cp -v -n 1 "$file" file:////"$(pwd)"/"$3" diff --git a/credentials.sh b/credentials.sh index a1ffe5290..afdce3c21 100755 --- a/credentials.sh +++ b/credentials.sh @@ -2,26 +2,27 @@ #talk to mcm export JIRA_SSO_COOKIE=/tmp/$USER-$HOSTNAME-vsdf43vfs if [ "$1" == "create" ] ; then - echo "creating jira cookie" $JIRA_SSO_COOKIE - cern-get-sso-cookie -u https://its.cern.ch/jira/loginCern.jsp -o $JIRA_SSO_COOKIE --krb + echo "creating jira cookie $JIRA_SSO_COOKIE" + cern-get-sso-cookie -u https://its.cern.ch/jira/loginCern.jsp -o "$JIRA_SSO_COOKIE" --krb else - echo "using jira cookie" $JIRA_SSO_COOKIE + echo "using jira cookie $JIRA_SSO_COOKIE" fi export MCM_SSO_COOKIE=/tmp/$USER-$HOSTNAME-vfsvdka573t if [ "$1" == "create" ] ; then - echo "creating mcm cookie" $MCM_SSO_COOKIE - cern-get-sso-cookie -u https://cms-pdmv.cern.ch/mcm/ -o $MCM_SSO_COOKIE --krb + echo "creating mcm cookie $MCM_SSO_COOKIE" + cern-get-sso-cookie -u https://cms-pdmv.cern.ch/mcm/ -o "$MCM_SSO_COOKIE" --krb else - echo "using mcm cookie" $MCM_SSO_COOKIE + echo "using mcm cookie $MCM_SSO_COOKIE" fi #talk to cmsweb export X509_USER_PROXY=/tmp/$USER-4hty64k793hj if [ "$1" == "create" ] ; then - echo "creating grid proxy" $X509_USER_PROXY - cat $HOME/private/$USER.txt | voms-proxy-init -voms cms --valid 140:00 --rfc -pwstdin + echo "creating grid proxy00 $X509_USER_PROXY" + # shellcheck disable=SC2002 + cat "$HOME"/private/"$USER".txt | voms-proxy-init -voms cms --valid 140:00 --rfc -pwstdin else - echo "using grid proxy" $X509_USER_PROXY + echo "using grid proxy $X509_USER_PROXY" fi diff --git a/cycle_common.sh b/cycle_common.sh index 5a583a3c2..98f1ecc5f 100644 --- a/cycle_common.sh +++ b/cycle_common.sh @@ -1,25 +1,31 @@ BASE_DIR=/data/unified/WmAgentScripts/ +# shellcheck disable=SC2034 HTML_DIR=/var/www/html/unified/ lock_name=$1 -if [ -r $lock_name ] ; then +if [ -r "$lock_name" ] ; then echo "lock file $lock_name is present" echo current id is $$ - lock_id=`tail -1 $lock_name` - echo looking for $lock_id - lock_running=`ps -e -f | grep " $lock_id " | grep -c -v grep` + lock_id=$(tail -1 "$lock_name") + echo looking for "$lock_id" + # shellcheck disable=SC2009 + lock_running=$(ps -e -f | grep " $lock_id " | grep -c -v grep) + # shellcheck disable=SC2009 ps -e -f | grep " $lock_id " | grep -v grep - echo $lock_running + echo "$lock_running" if [ "$lock_running" == "0" ] ; then echo "The cycle is locked but $lock_id is not running. Lifting the lock" + # shellcheck disable=SC2009 ps -e -f | grep Unified - cat $lock_name | mail -s "[Ops] Emergency On Cycle Lock. Unified isn't running." cmsunified@cern.ch - rm -f $lock_name + # shellcheck disable=SC2002 + cat "$lock_name" | mail -s "[Ops] Emergency On Cycle Lock. Unified isn't running." cmsunified@cern.ch + rm -f "$lock_name" else echo "cycle is locked" - echo $lock_id,"is running" + echo "$lock_id, is running" + # shellcheck disable=SC2009 ps -e -f | grep Unified exit fi @@ -32,12 +38,14 @@ if [ ! -r $BASE_DIR/credentials.sh ] ; then exit fi -echo $lock_name > $lock_name -echo `date` >> $lock_name -echo $HOSTNAME >> $lock_name -echo $$ >> $lock_name +echo "$lock_name" > "$lock_name" +# shellcheck disable=SC2005,SC2129 +echo "$(date)" >> "$lock_name" +echo "$HOSTNAME" >> "$lock_name" +echo $$ >> "$lock_name" ## set sso cookie and grid proxy locations: not creating them +# shellcheck disable=SC1090 source $BASE_DIR/credentials.sh diff --git a/errorcycle.sh b/errorcycle.sh index 23d2d8443..06b2cce73 100755 --- a/errorcycle.sh +++ b/errorcycle.sh @@ -1,10 +1,13 @@ BASE_DIR=/data/unified/WmAgentScripts/ +# shellcheck disable=SC2034 HTML_DIR=/var/www/html/unified/ -lock_name=`echo $BASH_SOURCE | cut -f 1 -d "."`.lock -source $BASE_DIR/cycle_common.sh $lock_name +# shellcheck disable=SC2128 +lock_name=$(echo "$BASH_SOURCE" | cut -f 1 -d ".").lock +# shellcheck disable=SC1090 +source $BASE_DIR/cycle_common.sh "$lock_name" $BASE_DIR/cWrap.sh Unified/showError.py --manual --expose 2 --threads 2 --log_threads 2 -rm -f $lock_name +rm -f "$lock_name" diff --git a/gqcycle.sh b/gqcycle.sh index 9dbe968ef..5df4aeadd 100755 --- a/gqcycle.sh +++ b/gqcycle.sh @@ -1,10 +1,13 @@ BASE_DIR=/data/unified/WmAgentScripts/ +# shellcheck disable=SC2034 HTML_DIR=/var/www/html/unified/ -lock_name=`echo $BASH_SOURCE | cut -f 1 -d "."`.lock -source $BASE_DIR/cycle_common.sh $lock_name +# shellcheck disable=SC2128 +lock_name=$(echo "$BASH_SOURCE" | cut -f 1 -d ".").lock +# shellcheck disable=SC1090 +source $BASE_DIR/cycle_common.sh "$lock_name" $BASE_DIR/cWrap.sh Unified/GQ.py -rm -f $lock_name +rm -f "$lock_name" diff --git a/inputscycle.sh b/inputscycle.sh index 7392c751d..3182026b8 100755 --- a/inputscycle.sh +++ b/inputscycle.sh @@ -1,11 +1,13 @@ -cd /data/unified/WmAgentScripts +cd /data/unified/WmAgentScripts||exit -oweek=`date +%W` +oweek=$(date +%W) week=${oweek#0} +# shellcheck disable=SC2219,SC2034 let oddity=week%2 export X509_USER_PROXY=$HOME/private/personal/voms_proxy.cert +# shellcheck disable=SC1091 source /data/srv/wmagent/current/apps/wmagent/etc/profile.d/init.sh python Unified/inputs_summary.py 200 diff --git a/lastlog.sh b/lastlog.sh index 1b0eb82a8..61998b390 100644 --- a/lastlog.sh +++ b/lastlog.sh @@ -1,7 +1,8 @@ #!/bin/bash module=$1 -filename=`ls -tr ../www/logs/$module/*.log | tail -1` -#echo $module -echo $filename -tail $filename +# shellcheck disable=SC2012 +filename=$(ls -tr ../www/logs/"$module"/*.log | tail -1) +# echo $module +echo "$filename" +tail "$filename" diff --git a/logscycle.sh b/logscycle.sh index 8fb799156..479cf0069 100755 --- a/logscycle.sh +++ b/logscycle.sh @@ -1,10 +1,13 @@ BASE_DIR=/data/unified/WmAgentScripts/ +# shellcheck disable=SC2034 HTML_DIR=/var/www/html/unified/ -lock_name=`echo $BASH_SOURCE | cut -f 1 -d "."`.lock -source $BASE_DIR/cycle_common.sh $lock_name +# shellcheck disable=SC2128 +lock_name=$(echo "$BASH_SOURCE" | cut -f 1 -d ".").lock +# shellcheck disable=SC1090 +source $BASE_DIR/cycle_common.sh "$lock_name" $BASE_DIR/cWrap.sh Unified/efficiencor.py 1 -rm -f $lock_name +rm -f "$lock_name" diff --git a/pokeeos.sh b/pokeeos.sh index 770bdaad6..224b689ae 100644 --- a/pokeeos.sh +++ b/pokeeos.sh @@ -1,15 +1,16 @@ #!/bin/bash out=/data/unified-cache/listingeos.$$.$HOSTNAME.log -date > $out -ls /eos/cms/store/unified/ >> $out -date >> $out -echo >> $out -ls /eos/cms/store/unified/www/ >>$out -date >> $out -echo >> $out -ls /eos/cms/store/logs/prod/2018/* -l >>$out -date >> $out +date > "$out" +# shellcheck disable=SC2129 +ls /eos/cms/store/unified/ >> "$out" +date >> "$out" +echo >> "$out" +ls /eos/cms/store/unified/www/ >>"$out" +date >> "$out" +echo >> "$out" +ls /eos/cms/store/logs/prod/2018/* -l >>"$out" +date >> "$out" -cp $out /eos/cms/store/unified/www/eospoke/. +cp "$out" /eos/cms/store/unified/www/eospoke/. ls /eos/cms/store/unified/www/eospoke/*.log diff --git a/postcycle-review-manual.sh b/postcycle-review-manual.sh index 9859a2b44..761ae8e98 100755 --- a/postcycle-review-manual.sh +++ b/postcycle-review-manual.sh @@ -1,14 +1,17 @@ BASE_DIR=/data/unified/WmAgentScripts/ +# shellcheck disable=SC2034 HTML_DIR=/var/www/html/unified/ ## export RUCIO_HOME export RUCIO_HOME=~/.local/ -lock_name=`echo $BASH_SOURCE | cut -f 1 -d "."`.lock -source $BASE_DIR/cycle_common.sh $lock_name +# shellcheck disable=SC2128 +lock_name=$(echo "$BASH_SOURCE" | cut -f 1 -d ".").lock +# shellcheck disable=SC1090 +source $BASE_DIR/cycle_common.sh "$lock_name" ## look at everything that has to be taken care of : add clear to expedite $BASE_DIR/cWrap.sh Unified/checkor.py --review --manual --threads 5 -rm -f $lock_name +rm -f "$lock_name" diff --git a/postcycle-review-recovering.sh b/postcycle-review-recovering.sh index 742cba725..a87bd54b4 100755 --- a/postcycle-review-recovering.sh +++ b/postcycle-review-recovering.sh @@ -1,14 +1,17 @@ BASE_DIR=/data/unified/WmAgentScripts/ +# shellcheck disable=SC2034 HTML_DIR=/var/www/html/unified/ ## export RUCIO_HOME export RUCIO_HOME=~/.local/ -lock_name=`echo $BASH_SOURCE | cut -f 1 -d "."`.lock -source $BASE_DIR/cycle_common.sh $lock_name +# shellcheck disable=SC2128 +lock_name=$(echo "$BASH_SOURCE" | cut -f 1 -d ".").lock +# shellcheck disable=SC1090 +source $BASE_DIR/cycle_common.sh "$lock_name" ## look at everything that had been taken care of already : add clear to expedite $BASE_DIR/cWrap.sh Unified/checkor.py --review --recovering --threads 5 -rm -f $lock_name +rm -f "$lock_name" diff --git a/postcycle-strict.sh b/postcycle-strict.sh index af88ddd02..cdb456d10 100755 --- a/postcycle-strict.sh +++ b/postcycle-strict.sh @@ -1,14 +1,17 @@ BASE_DIR=/data/unified/WmAgentScripts/ +# shellcheck disable=SC2034 HTML_DIR=/var/www/html/unified/ ## export RUCIO_HOME export RUCIO_HOME=~/.local/ -lock_name=`echo $BASH_SOURCE | cut -f 1 -d "."`.lock -source $BASE_DIR/cycle_common.sh $lock_name +# shellcheck disable=SC2128 +lock_name=$(echo "$BASH_SOURCE" | cut -f 1 -d ".").lock +# shellcheck disable=SC1090 +source $BASE_DIR/cycle_common.sh "$lock_name" ## those that just completed $BASE_DIR/cWrap.sh Unified/checkor.py --strict --clear --threads 5 -rm -f $lock_name +rm -f "$lock_name" diff --git a/postcycle-update.sh b/postcycle-update.sh index 80ae781ef..4835c147f 100755 --- a/postcycle-update.sh +++ b/postcycle-update.sh @@ -1,14 +1,17 @@ BASE_DIR=/data/unified/WmAgentScripts/ +# shellcheck disable=SC2034 HTML_DIR=/var/www/html/unified/ ## export RUCIO_HOME export RUCIO_HOME=~/.local/ -lock_name=`echo $BASH_SOURCE | cut -f 1 -d "."`.lock -source $BASE_DIR/cycle_common.sh $lock_name +# shellcheck disable=SC2128 +lock_name=$(echo "$BASH_SOURCE" | cut -f 1 -d ".").lock +# shellcheck disable=SC1090 +source $BASE_DIR/cycle_common.sh "$lock_name" ## look at what is still running : add clear to expedite $BASE_DIR/cWrap.sh Unified/checkor.py --update --threads 5 -rm -f $lock_name +rm -f "$lock_name" diff --git a/remainingcycle.sh b/remainingcycle.sh index 526061cdf..6382b7437 100755 --- a/remainingcycle.sh +++ b/remainingcycle.sh @@ -1,10 +1,14 @@ BASE_DIR=/data/unified/WmAgentScripts/ +# shellcheck disable=SC2034 HTML_DIR=/var/www/html/unified/ -lock_name=`echo $BASH_SOURCE | cut -f 1 -d "."`.lock -source $BASE_DIR/cycle_common.sh $lock_name +# shellcheck disable=SC2128 +lock_name=$(echo "$BASH_SOURCE" | cut -f 1 -d ".").lock +# shellcheck disable=SC1090 +source $BASE_DIR/cycle_common.sh "$lock_name" +# shellcheck disable=SC1090 $BASE_DIR/cWrap.sh Unified/remainor.py -n 5 -rm -f $lock_name +rm -f "$lock_name" diff --git a/rmOldJobs.sh b/rmOldJobs.sh index 5c8a28106..9706ae56a 100755 --- a/rmOldJobs.sh +++ b/rmOldJobs.sh @@ -1,4 +1,5 @@ #!/bin/bash +# shellcheck disable=SC1091 source /data/admin/wmagent/env.sh condor_rm -constraint 'JobStatus == 3 && ( CurrentTime - EnteredCurrentStatus ) > 3600' -forcex condor_rm -constraint 'JobStatus == 5 && ( CurrentTime - EnteredCurrentStatus ) > 86400' diff --git a/runerrorcycle.sh b/runerrorcycle.sh index 0fbc9681c..5aae8dc7c 100755 --- a/runerrorcycle.sh +++ b/runerrorcycle.sh @@ -1,10 +1,13 @@ BASE_DIR=/data/unified/WmAgentScripts/ +# shellcheck disable=SC2034 HTML_DIR=/var/www/html/unified/ -lock_name=`echo $BASH_SOURCE | cut -f 1 -d "."`.lock -source $BASE_DIR/cycle_common.sh $lock_name +# shellcheck disable=SC2128 +lock_name=$(echo "$BASH_SOURCE" | cut -f 1 -d ".").lock +# shellcheck disable=SC1090 +source $BASE_DIR/cycle_common.sh "$lock_name" $BASE_DIR/cWrap.sh Unified/showError.py --ongoing --expose 2 --threads 2 --log_threads 2 -rm -f $lock_name +rm -f "$lock_name" diff --git a/set.sh b/set.sh index b4a2c1bb0..63aa5afe2 100755 --- a/set.sh +++ b/set.sh @@ -1,2 +1,3 @@ export PYTHONPATH=$PYTHONPATH:/usr/lib64/python2.7/site-packages:/usr/lib/python2.7/site-packages +# shellcheck disable=SC1091 source /data/srv/wmagent/current/apps/wmagent/etc/profile.d/init.sh diff --git a/setenvscripts.sh b/setenvscripts.sh index d0530539a..95b9a7367 100755 --- a/setenvscripts.sh +++ b/setenvscripts.sh @@ -1,11 +1,13 @@ echo 'INFO: Setting identity' USER=$(whoami) -kinit $USER@CERN.CH +kinit "$USER"@CERN.CH echo 'INFO: Setting Grid env and proxy' +# shellcheck disable=SC1091 source /afs/cern.ch/project/gd/LCG-share/3.2.11-1/etc/profile.d/grid-env.sh voms-proxy-init -voms cms { echo 'INFO: Setting agent environment' +# shellcheck disable=SC1091 source /data/srv/wmagent/current/apps/wmagent/etc/profile.d/init.sh } || { echo 'ERROR: This machine does not have WMAgent installed. Not sourcing Agent environment' diff --git a/shortcycle.sh b/shortcycle.sh index 576e2ab50..143aff1bc 100755 --- a/shortcycle.sh +++ b/shortcycle.sh @@ -1,8 +1,11 @@ BASE_DIR=/data/unified/WmAgentScripts/ +# shellcheck disable=SC2034 HTML_DIR=/var/www/html/unified/ -lock_name=`echo $BASH_SOURCE | cut -f 1 -d "."`.lock -source $BASE_DIR/cycle_common.sh $lock_name +# shellcheck disable=SC2128 +lock_name=$(echo "$BASH_SOURCE" | cut -f 1 -d ".").lock +# shellcheck disable=SC1090 +source $BASE_DIR/cycle_common.sh "$lock_name" ##mapping the sites $BASE_DIR/cWrap.sh Unified/mappor.py @@ -14,5 +17,5 @@ $BASE_DIR/cWrap.sh Unified/equalizor.py $BASE_DIR/cWrap.sh Unified/addHoc.py -rm -f $lock_name +rm -f "$lock_name" diff --git a/subscribecycle.sh b/subscribecycle.sh index 7d796b960..806c442c5 100755 --- a/subscribecycle.sh +++ b/subscribecycle.sh @@ -1,17 +1,21 @@ BASE_DIR=/data/unified/WmAgentScripts/ +# shellcheck disable=SC2034 HTML_DIR=/var/www/html/unified/ -lock_name=`echo $BASH_SOURCE | cut -f 1 -d "."`.lock -source $BASE_DIR/cycle_common.sh $lock_name +# shellcheck disable=SC2128 +lock_name=$(echo "$BASH_SOURCE" | cut -f 1 -d ".").lock +# shellcheck disable=SC1090 +source $BASE_DIR/cycle_common.sh "$lock_name" -## finsih subscribing output blocks +## finish subscribing output blocks $BASE_DIR/cWrap.sh Unified/subscribor.py away $BASE_DIR/cWrap.sh Unified/subscribor.py assistance* +# shellcheck disable=SC1010 $BASE_DIR/cWrap.sh Unified/subscribor.py done $BASE_DIR/cWrap.sh Unified/subscribor.py close ## subscribe everything that is being produced or waiting around $BASE_DIR/cWrap.sh Unified/subscribor.py wmagent -rm -f $lock_name +rm -f "$lock_name" diff --git a/sync.sh b/sync.sh index baa36489f..06ef3e702 100755 --- a/sync.sh +++ b/sync.sh @@ -1,6 +1,6 @@ #!/bin/bash -cd /data/unified/WmAgentScripts +cd /data/unified/WmAgentScripts||exit git stash git fetch origin git rebase origin/master diff --git a/toperrorcycle.sh b/toperrorcycle.sh index 926bddfda..6b2ff5d8a 100755 --- a/toperrorcycle.sh +++ b/toperrorcycle.sh @@ -1,10 +1,13 @@ BASE_DIR=/data/unified/WmAgentScripts/ +# shellcheck disable=SC2034 HTML_DIR=/var/www/html/unified/ -lock_name=`echo $BASH_SOURCE | cut -f 1 -d "."`.lock -source $BASE_DIR/cycle_common.sh $lock_name +# shellcheck disable=SC2128 +lock_name=$(echo "$BASH_SOURCE" | cut -f 1 -d ".").lock +# shellcheck disable=SC1090 +source $BASE_DIR/cycle_common.sh "$lock_name" $BASE_DIR/cWrap.sh Unified/showError.py --top --expose 2 --threads 2 --log_threads 2 -rm -f $lock_name +rm -f "$lock_name" diff --git a/viewcycle.sh b/viewcycle.sh index 9663c6d2d..f39f319df 100755 --- a/viewcycle.sh +++ b/viewcycle.sh @@ -1,10 +1,13 @@ BASE_DIR=/data/unified/WmAgentScripts/ +# shellcheck disable=SC2034 HTML_DIR=/var/www/html/unified/ -lock_name=`echo $BASH_SOURCE | cut -f 1 -d "."`.lock -source $BASE_DIR/cycle_common.sh $lock_name +# shellcheck disable=SC2128 +lock_name=$(echo "$BASH_SOURCE" | cut -f 1 -d ".").lock +# shellcheck disable=SC1090 +source $BASE_DIR/cycle_common.sh "$lock_name" $BASE_DIR/cWrap.sh Unified/htmlor.py -rm -f $lock_name +rm -f "$lock_name" diff --git a/whatjob.sh b/whatjob.sh index ce331ca6e..ecf7b2b00 100755 --- a/whatjob.sh +++ b/whatjob.sh @@ -3,22 +3,22 @@ date #echo "################ QUOTA #######################" #eos quota ls -u cmsvoc /eos/cms/store/unified nodes="0268 0269 0272 0273 0274 0275" -if [ ! -z $1 ] ; then +if [ -n "$1" ] ; then nodes=$1 fi for each in $nodes; do echo echo "#############################################" - echo "Set on node" $each + echo "Set on node $each" echo "################ CRON #######################" - acrontab -l | grep $each + acrontab -l | grep "$each" echo "############## UPTIME #######################" - ssh vocms$each "uptime" + ssh vocms"$each uptime" echo "############## RUNNING ######################" - ssh vocms$each "ps -e -f | grep Uni| grep -v grep" + ssh vocms"$each ps -e -f | grep Uni| grep -v grep" - ssh vocms$each "cd /data/unified/WmAgentScripts ; source /data/srv/wmagent/current/apps/wmagent/etc/profile.d/init.sh ; python /data/unified/WmAgentScripts/Unified/deadlock.py" + ssh vocms"$each cd /data/unified/WmAgentScripts ; source /data/srv/wmagent/current/apps/wmagent/etc/profile.d/init.sh ; python /data/unified/WmAgentScripts/Unified/deadlock.py" #ssh vocms$each "sudo chmod g+rw -R /data/unified/" #ssh vocms$each "sudo chmod g+rw -R /data/unified-cache/"