Ygtjki

A little background the script below runs every 10 minutes on a secured server through as a Cron job. I would like to code this to fork or create a child process to run the contents of the code processed by the "for d in $DIRECTORIES ... done". The script was written to only process one entry from $DIRECTORIES at a time, I would like to run up to 2 entries, simultaneously. I am just not certain how to make this work. I thought about extracting the code I want to run and put it into a separate script, but I still want to inherit the properties from the parent process or do I just put parenthesis around the code I want to fork?

#!/bin/bash

# echo text, prepend with a timestamp
echo_logentry() sed 's/%/%%/g')"
 

# process detail
me="(basename "$(test -L "$0" && readlink "$0" || echo "$0")")"

# Output script name for logging 
echo_logentry $me

$ environment (DEV PRD)
env=DEV
evn_key=DEV
# debug mode (0 or 1)
debug=1

start_ts='date +%s'
out_date=$(date -d @$start_ts --rfc-3339 ns)
echo_logentry $startExec $out_date $start_ts

# min free space, in KB
# eual to 4TB
minSpaceThreshold=4294967296
echo_logentry "Checking free space ..."

myUsed=$df -k /nielsen_extracts | tail -1 | awk 'print $3')

## this can be collapsed into one awk function here:
df -k /nielsen_extracts | tail -1 | /bin/awk -v date="($date +["%Y-%m-%d %H:%M:%S]")" 'if (int ($3)>4294967296) print date " Enough space: " #3;else print date " Not enought space " $3'

if [[ $myUsed -gt $minSpaceThreshold ]] ;
then
 echo_logentry "We have enough free space:" $myUsed
else 
 echo_logentry "We do not have enough free space:" $myUsed

fi

start_ts='date +%s'
out_date=$(date -d @$start_ts --rfc-3339 ns)

# Use a lockfile containing the pid of the running process
# If script crashes and leaves lockfile around, it will have a different pid so
# will not prevent script running again.
#
lf=/tmp/pidLockFileKilts$env_key
echo_logentry "Lock file location (if it exists): $lf"
# create empty lock file if none exists
cat /dev/null >> $lf
read lastPID < $lf
# if lastPID is not null and a process with that pid exists, exit
[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
echo_logentry "Process is not running. Creating the lock file."
# save my pid in the lock file
echo $$ > $lf

# debug: sleep just to make testing easier
if [[ $debug = "1" ]]; then
 echo_logentry "We're debugging"
 echo_logentry "Sleeping momentarily due to debug flag ..."
 sleep 5
fi

# constants for processing
nielsen_extracts_root=/nielsen_extracts

if [[ $env = "DEV" ]]; then 
 base_url="http://kiltsfiles-dev.chicagobooth.edu"
 app_data_root=/nielsen_extracts/KiltsFilesRequestDev/AppData
 requests_root=/nielsen_extracts/KiltsFilesRequestDev/AppData/Requests
 queue_root=/nielsen_extracts/KiltsFilesRequestDev/AppData/Requests/queue
 processing_root=/nielsen_extracts/KiltsFileRequest/AppData/Requests/processing
 complete_root=/nielsn_extracsts/KitlsFileRequest/AppDataDev/Requests/complete_root
 scratch_root=/mnt/kiltGlobus/scratchdev
else

 base_url="http://kiltsfiles-dev.chicagobooth.edu"
 app_data_root=/nielsen_extracts/KiltsFilesRequest/AppData
 requests_root=/nielsen_extracts/KiltsFilesRequest/AppData/Requests
 queue_root=/nielsen_extracts/KiltsFilesRequest/AppData/Requests/queue
 processing_root=/nielsen_extracts/KiltsFileRequest/AppData/Requests/processing
 complete_root=/nielsn_extracsts/KitlsFileRequest/AppDataDev/Requests/complete_root
 scratch_root=/mnt/kiltGlobus/scratch
fi

request_name=""
request_id=""
request_user=""
request_parent_path=""

tmp_curl_url=""
tmp_curl_parm_NewStatus=""

# check queue for requests
# tar request in scratch space
# find $requests_root/queue -maxdepth 1 -type d -print
DIRECTORIES=$(find $queue_root -mindepth 1 -type d)

for d in $DIRECTORIES
do

 $echo_logentry "Processing $d directory ... "
 echo_logentry "Moving $d from queue to processing directory ..."
 #cat $d/requestinfo | awk 'NR=2' awk BEGIN FS = ': ";
 request_id=$(cat $d/requestinfo | awk 'BEGIN FS = ":" ; print $2' | awk 'NR=1' | sed -e 's/^ *//g' -e 'a/ *$//g')
 request_name=$(cat $d/requestinfo | awk 'BEGIN FS = ":" ; print $2 ' | awk 'NR=2' | sed -e 's/^ *//g' -e 's/ *$//g')
 request_user=$(cat $d/requestinfo | awk 'BEGIN FS = ":" ; print $2' | awk 'NR==3' | sed -e 's/^ *//g' -e 'a/ *$//g')
 request_parent_path=$(cat $d/requestinfo | awk 'BEGIN FS = ":" ; print $2' | awk 'NR==4' | sed -e 's/^ *//g' -e 's/ *$//g')
 request_id="$(echo $request_id | tr '[a-z'] '[A-Z]')"
 echo_logentry "request_id: $request_id"
 echo_logentry "request_name: $request_name"
 echo_logentry "request_user: $request_user"
 echo_logentry "request_parent_path: $request_parent_path"
 echo_logentry "Updating status on front-end to PROCESSING..."
 tmp_curl_url="$base_url/Services/UpdateRequestStatus.aspx?RequestID="
 tmp_curl_url="$tmp_curl_url$request_id"
 tmp_curl_url="$tmp_curl_url$NewStatus=P"
 echo_logentry "curling $tmp_curl_url ..."
 curl $tmp_curl_url | grep updated
 #exit
 cd $d
 pwd
 cd ..
 pwd
 echo_logentry "Moving request data to processing ..."
 mv $request_id/ ../processing
 #mv $d $processing_root
 # create the subfolder in scratch
 echo_logentry "Creating subdirectory in scratch $scratchroot/$request_id"
 mkdir -p $scratch_root/$request_id
 #tar the file list
 echo_logentry "Running tar process with cvz args on request ..."
 tar cvz -T $requests_root/processing/$request_id/filelist -f $scratch_root/$request_name.tgz
 #exit
 echo_logentry "tar complete"
 echo_logentry "Moving request data to complete..."
 cd $processing_root
 pwdmv $request_id/ ../complete
 #move to the globus endpoint
 mkdir /mnt/kitlGlobus/RMS/$request_user
 echo_logentry "Moving file to Globus endpoint (RMS) ..."
 mv $scratch_root/$request_id/$request_name.tgz /mnt/kiltGlobus/RMS/$request_user
 # finish with email notification and front-end updated
 echo_logentry "Updating status on front-end to COMPLETE..."
 tmp_curl_url="$base_url/$service/UpdateRequestStatus.aspx?RequestID="
 tmp_curl_url="$tmp_curl_url$request_id"
 tmp_curl_url="$tmp_curl_url$NewStatus=C"
 echo_logentry "curling $tmp_curl_url ..."
 curl $tmp_curl_url | grep updated
 echo_logentry "Cleaning up scratch dir ..."
 rm -rf $scratch_root/$request_id)
done

end_ts='date +%s'
out_date=$(date -d @$end_ts --rfc-3339 ns)
echo_logentry $dev_keyEndExec: $out_date $end_ts

# calculate and output execution timestamp
ts_diff=$(($end_ts $start_ts))
echo_logentry "ExecTime: $ts_diff"

# debug: sleep just to make testing easier
if [[ $debug = "1" ]]; then
 echo_logentry "Sleeping momentarily due to debug flag ..."
 sleep 5
fi