#!/bin/bash

set -e

# note - this requires Montage 3.2beta6 or higher

#######################################################################
#
#  Settings
#

DEGREES=2.0

LOCAL_MONTAGE_LOCATION="/home/rcf-proj/gmj/pegasus/SOFTWARE/montage/default"
LOCAL_GLOBUS_TCP_PORT_RANGE="40000,50000"

CLUSTER_NAME="HPCC"
#CLUSTER_HOSTNAME="obelix.isi.edu"
CLUSTER_HOSTNAME="hpc-login2.usc.edu"
CLUSTER_GATEKEEPER_TYPE="gt5"
CLUSTER_GATEKEEPER_PORT="2119"
CLUSTER_SCHEDULER="condor"
CLUSTER_WORK_DIR="/staging/gmj/vahi/glite-montage-pmc/SCRATCH/HPCC"
CLUSTER_PEGASUS_HOME="//home/rcf-proj/gmj/pegasus/SOFTWARE/pegasus/default/"
#CLUSTER_GLOBUS_LOCATION="/ccg/software/globus/default"
CLUSTER_MONTAGE_LOCATION="/home/rcf-proj/gmj/pegasus/SOFTWARE/montage/default"
CLUSTER_PMC_WRAPPER="/home/rcf-proj/gmj/pegasus/SOFTWARE/pegasus/pegasus-mpi-cluster-wrapper"
# this will bundle N jobs togehter
JOB_CLUSTERS_SIZE="25"

#######################################################################

TOP_DIR=`pwd`
LOCAL_HOSTNAME=`hostname -f`

export PATH=$LOCAL_MONTAGE_LOCATION/bin:$PATH

# unique directory for this run
RUN_ID=`/bin/date +'%F_%H%M%S'`
RUN_DIR=`pwd`/work/$RUN_ID

echo "Work directory: $RUN_DIR"

mkdir -p $RUN_DIR
cd $RUN_DIR

cp $TOP_DIR/pegasusrc .

# create the transformation catalogue (tc)
echo
echo "Creating the transformation catalog..."
for BINARY in `(cd $LOCAL_MONTAGE_LOCATION/bin && ls)`; do
    cat >>tc <<EOF
tr $BINARY:3.3 {
    site $CLUSTER_NAME {
        pfn "$CLUSTER_MONTAGE_LOCATION/bin/$BINARY"
        arch "x86_64"
        os "linux"
        type "INSTALLED"
        profile pegasus "label"  "montage"
        profile pegasus "clusters.size" "$JOB_CLUSTERS_SIZE" 
        profile globus "maxwalltime" "1000" 
    }
}
EOF

cat >>tc<<EOF
tr pegasus::mpiexec {
    site $CLUSTER_NAME {
        pfn "$CLUSTER_PMC_WRAPPER"
        arch "x86_64"
        os "linux"
        type "INSTALLED"
        profile globus "jobtype" "mpi"
        profile globus "maxwalltime" "2880"
        # specfiy the ppn parameter.
        profile globus "xcount" "16:IB"
        # specify the nodes parameter
        profile globus "hostcount" "1"
        #specify the pmem parameter
        profile globus "maxmemory" "1gb"
        #specify the mem parameter
        profile globus "totalmemory" "16gb"
    }
}
EOF
done
#        <directory type="shared-scratch" path="$TOP_DIR/work">
#            <file-server operation="all" url="file://$TOP_DIR/work"/>
#        </directory>

# site catalog
echo
echo "Creating the site catalog..."
cat >sites.xml <<EOF
<?xml version="1.0" encoding="UTF-8"?>
<sitecatalog xmlns="http://pegasus.isi.edu/schema/sitecatalog" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://pegasus.isi.edu/schema/sitecatalog http://pegasus.isi.edu/schema/sc-4.0.xsd" version="4.0">

    <site  handle="local" arch="x86_64" os="LINUX">
        <directory type="local-storage" path="$TOP_DIR/outputs">
            <file-server operation="all" url="file://$TOP_DIR/outputs"/>
        </directory>
        <profile namespace="env" key="GLOBUS_LOCATION" >$GLOBUS_LOCATION</profile>
        <profile namespace="env" key="GLOBUS_TCP_PORT_RANGE" >$LOCAL_GLOBUS_TCP_PORT_RANGE</profile>
    </site>

    <site  handle="$CLUSTER_NAME" arch="x86_64" os="LINUX">
        <directory type="shared-scratch" path="$CLUSTER_WORK_DIR">
            <file-server operation="all" url="gsiftp://$CLUSTER_HOSTNAME/$CLUSTER_WORK_DIR"/>
        </directory>
        <profile namespace="env" key="PEGASUS_HOME" >$CLUSTER_PEGASUS_HOME</profile>
        <profile namespace="env" key="MONTAGE_HOME" >$CLUSTER_MONTAGE_LOCATION</profile>
        <profile namespace="pegasus" key="style" >glite</profile>
        <profile namespace="pegasus" key="change.dir">true</profile>

        <profile namespace="condor" key="grid_resource">pbs</profile>
        <!-- <profile namespace="condor" key="batch_queue">main</profile>-->
        <profile namespace="globus" key="maxwalltime">300</profile>

    </site>

</sitecatalog>
EOF

echo
echo "Running mDAG (finding input images, generating DAX, ...)..."
#mDAG 2mass j M17 $DEGREES $DEGREES 0.0002777778 . "gsiftp://$LOCAL_HOSTNAME$RUN_DIR" "gsiftp://$LOCAL_HOSTNAME$RUN_DIR/inputs"
mDAG 2mass j M17 $DEGREES $DEGREES 0.0002777778 . "file://$RUN_DIR" "file://$RUN_DIR/inputs"

echo
echo "Adding input images to the replica catalog..."
echo "  " `cat cache.list | wc -l` "images found"
cat cache.list | grep -v ".fits " | sed "s/ipac_cluster/$CLUSTER_NAME/" > rc
cat url.list | sed 's/ http:.*ref=/ http:\/\/obelix.isi.edu\/irsa-cache/' >>rc
#cat url.list | sed 's/ http:.*ref=/ http:\/\/obelix.isi.edu\/irsa-cache/' | sed "s/ipac_cluster/$CLUSTER_NAME/" >>rc

echo
echo "Planning and submitting the workflow..."
pegasus-plan \
    --conf pegasusrc \
    --sites $CLUSTER_NAME \
    --output-site local \
    --cluster label \
    --dir work \
    --dax dag.xml \
    -v \
    --nocleanup \
    --submit \
    2>&1 | tee pegasus-plan.out


