Compiling datastage jobs in parallel

function ds_compile_batch {
  if [[ "$DEBUG" = "Y" ]]; then
    set -x
  fi
  logVerbose "Compiling routines in ${ds_project}@${ds_host}"
  $CMD_DSCC /f /r '*' ${ds_project} >> $COMPILE_LOG 2>&1
  typeset -i COUNTER=0
  SLEEP=5
  cat ${DSJOB_LIST} | while read DSJOB ; do
    logVerbose "Compiling $DSJOB"
    if [[ "$DRY_RUN_FLAG" != "Y" ]]; then
      let COUNTER=$COUNTER+1
      ID="${temp_dir}/${COUNTER}_$(date '+%Y%m%d%H%M%S')"
      $CMD_DSCC /f /j ${DSJOB} ${ds_project} >>${ID}.log 2>&1 &
      PID=$!
      echo $PID > ${ID}.pid
      echo "Compiling $DSJOB (P${COUNTER})" >> ${ID}.log
      # sleep to avoid write errors to c:/Program\ Files/ITCB/ds_logs/dstage_wrapper_trace_n.log
      #sleep 2

      # wait if max processes running
      logDebug "Number of compile processes running $COUNTER (MAX=$MAX_PROCESS)"
      if [[ $COUNTER -ge $MAX_PROCESS ]]; then
      # wait for processes to finish
        WAIT="Y"
        while [[ "$WAIT" = "Y" ]]; do
          sleep $SLEEP
          for PID_FILE in ${temp_dir}/*.pid ; do
            PID=$(cat ${PID_FILE})
            ps -p ${PID} >/dev/null 2>&1
            if [[ $? -ne 0 ]]; then
              THIS_LOG="${temp_dir}/$(basename $PID_FILE .pid).log"
              cat "$THIS_LOG" >> $COMPILE_LOG
              echo "===========================================================" >> $COMPILE_LOG
              WAIT="N"
              rm "$PID_FILE"
              rm "$THIS_LOG"
            fi
          done
          COUNTER=$(ls ${temp_dir}/*.pid 2>/dev/null | wc -l)
          logDebug "Number of compile processes running $COUNTER (MAX=$MAX_PROCESS)"
        done
      fi
    fi
  done
}