#!/bin/bash
#PBS -N pdelphi95
#PBS -q compbio
#PBS -l place=scatter
###---------------------for tests on palmetto----------------------###
###1st group of nodes(Count-430):                                  ###
###node range:node1112-1541; model:Sun X6250;                      ###
###Processor:Intel Xeon L5420 @ 2.5GHz x 2; L2 cache:6MB           ###
###Cores:8; Memory: 32GB                                           ###
###                                                                ###
###2nd group of nodes(Count-70):                                   ### 
###node range:node1553-1622; model:HP DL 165 G7;                   ###
###Processor:AMD Opteron 6172 @ 2.1GHz x 2; L2 cache:12MB          ###
###Cores:24; Memory: 48GB                                          ###
###----------------------------------------------------------------###
###                                                                ###
###----------------------------------------------------------------###
###                                                                ###
###               for tests on 1st group of nodes                  ###
###                                                                ###
###----------------------------------------------------------------###
###p3,90gb   : #PBS -l select=3:ncpus=1:mpiprocs=1:chip_type=l5420:mem=30gb:myrinet=true
###p6,105gb  : #PBS -l select=1:ncpus=1:mpiprocs=1:chip_type=l5420:mem=30gb:myrinet=true+5:ncpus=1:mpiprocs=1:chip_type=l5420:mem=15gb:myrinet=true
###p11,60gb  : #PBS -l select=1:ncpus=1:mpiprocs=1:chip_type=l5420:mem=30gb:myrinet=true+10:ncpus=1:mpiprocs=1:chip_type=l5420:mem=3gb:myrinet=true
###p16,75gb  : #PBS -l select=1:ncpus=1:mpiprocs=1:chip_type=l5420:mem=30gb:myrinet=true+15:ncpus=1:mpiprocs=1:chip_type=l5420:mem=3gb:myrinet=true
###p21,90gb  : #PBS -l select=1:ncpus=1:mpiprocs=1:chip_type=l5420:mem=30gb:myrinet=true+20:ncpus=1:mpiprocs=1:chip_type=l5420:mem=3gb:myrinet=true
###p31,120gb : #PBS -l select=1:ncpus=1:mpiprocs=1:chip_type=l5420:mem=30gb:myrinet=true+30:ncpus=1:mpiprocs=1:chip_type=l5420:mem=3gb:myrinet=true
###p41,150gb : #PBS -l select=1:ncpus=1:mpiprocs=1:chip_type=l5420:mem=30gb:myrinet=true+40:ncpus=1:mpiprocs=1:chip_type=l5420:mem=3gb:myrinet=true
###p51,180gb : #PBS -l select=1:ncpus=1:mpiprocs=1:chip_type=l5420:mem=30gb:myrinet=true+50:ncpus=1:mpiprocs=1:chip_type=l5420:mem=3gb:myrinet=true
###p81,270gb : #PBS -l select=1:ncpus=1:mpiprocs=1:chip_type=l5420:mem=30gb:myrinet=true+80:ncpus=1:mpiprocs=1:chip_type=l5420:mem=3gb:myrinet=true
###p101,330gb: #PBS -l select=1:ncpus=1:mpiprocs=1:chip_type=l5420:mem=30gb:myrinet=true+100:ncpus=1:mpiprocs=1:chip_type=l5420:mem=3gb:myrinet=true
###----------------------------------------------------------------###
###                                                                ###
###             for tests on mix of 1st and 2nd groups             ###
###       MR on nodes in 2nd gp and WRs on nodes in 1st gp         ###
###                                                                ###
###----------------------------------------------------------------###
###p3,105gb  : #PBS -l select=1:ncpus=1:mpiprocs=1:mem=45gb:myrinet=true+2:ncpus=1:mpiprocs=1:chip_type=l5420:mem=30gb:myrinet=true
###p6,120gb  : #PBS -l select=1:ncpus=1:mpiprocs=1:mem=45gb:myrinet=true+5:ncpus=1:mpiprocs=1:chip_type=l5420:mem=15gb:myrinet=true
###p11,75gb  : #PBS -l select=1:ncpus=1:mpiprocs=1:mem=45gb:myrinet=true+10:ncpus=1:mpiprocs=1:chip_type=l5420:mem=3gb:myrinet=true
###p16,90gb  : #PBS -l select=1:ncpus=1:mpiprocs=1:mem=45gb:myrinet=true+15:ncpus=1:mpiprocs=1:chip_type=l5420:mem=3gb:myrinet=true
###p21,105gb : #PBS -l select=1:ncpus=1:mpiprocs=1:mem=45gb:myrinet=true+20:ncpus=1:mpiprocs=1:chip_type=l5420:mem=3gb:myrinet=true
###p31,135gb : #PBS -l select=1:ncpus=1:mpiprocs=1:mem=45gb:myrinet=true+30:ncpus=1:mpiprocs=1:chip_type=l5420:mem=3gb:myrinet=true
###p41,165gb : #PBS -l select=1:ncpus=1:mpiprocs=1:mem=45gb:myrinet=true+40:ncpus=1:mpiprocs=1:chip_type=l5420:mem=3gb:myrinet=true
###p51,195gb : #PBS -l select=1:ncpus=1:mpiprocs=1:mem=45gb:myrinet=true+50:ncpus=1:mpiprocs=1:chip_type=l5420:mem=3gb:myrinet=true
###p81,285gb : #PBS -l select=1:ncpus=1:mpiprocs=1:mem=45gb:myrinet=true+80:ncpus=1:mpiprocs=1:chip_type=l5420:mem=3gb:myrinet=true
###p101,345gb: #PBS -l select=1:ncpus=1:mpiprocs=1:mem=45gb:myrinet=true+100:ncpus=1:mpiprocs=1:chip_type=l5420:mem=3gb:myrinet=true
###p121,405gb: #PBS -l select=1:ncpus=1:mpiprocs=1:mem=45gb:myrinet=true+120:ncpus=1:mpiprocs=1:chip_type=l5420:mem=3gb:myrinet=true
###p151,495gb: #PBS -l select=1:ncpus=1:mpiprocs=1:mem=45gb:myrinet=true+150:ncpus=1:mpiprocs=1:chip_type=l5420:mem=3gb:myrinet=true
###----------------------------------------------------------------###
###                                                                ###
###               for tests on 2nd group of nodes                  ###
###                                                                ###
###----------------------------------------------------------------###
###p3,105gb  : #PBS -l select=1:ncpus=1:mpiprocs=1:mem=45gb:myrinet=true+2:ncpus=1:mpiprocs=1:chip_type=6176:mem=30gb:myrinet=true
###p6,120gb  : #PBS -l select=1:ncpus=1:mpiprocs=1:mem=45gb:myrinet=true+5:ncpus=1:mpiprocs=1:chip_type=6176:mem=15gb:myrinet=true
###p11,75gb  : #PBS -l select=1:ncpus=1:mpiprocs=1:mem=45gb:myrinet=true+10:ncpus=1:mpiprocs=1:chip_type=6176:mem=3gb:myrinet=true
###p16,90gb  : #PBS -l select=1:ncpus=1:mpiprocs=1:mem=45gb:myrinet=true+15:ncpus=1:mpiprocs=1:chip_type=6176:mem=3gb:myrinet=true
###p21,105gb : #PBS -l select=1:ncpus=1:mpiprocs=1:mem=45gb:myrinet=true+20:ncpus=1:mpiprocs=1:chip_type=6176:mem=3gb:myrinet=true
###p31,135gb : #PBS -l select=1:ncpus=1:mpiprocs=1:mem=45gb:myrinet=true+30:ncpus=1:mpiprocs=1:chip_type=6176:mem=3gb:myrinet=true
###p41,165gb : #PBS -l select=1:ncpus=1:mpiprocs=1:mem=45gb:myrinet=true+40:ncpus=1:mpiprocs=1:chip_type=6176:mem=3gb:myrinet=true
###p51,195gb : #PBS -l select=1:ncpus=1:mpiprocs=1:mem=45gb:myrinet=true+50:ncpus=1:mpiprocs=1:chip_type=6176:mem=3gb:myrinet=true
###p81,285gb : #PBS -l select=1:ncpus=1:mpiprocs=1:mem=45gb:myrinet=true+40:ncpus=2:mpiprocs=2:chip_type=6176:mem=6gb:myrinet=true
###p101,345gb: #PBS -l select=1:ncpus=1:mpiprocs=1:mem=45gb:myrinet=true+50:ncpus=2:mpiprocs=2:chip_type=6176:mem=6gb:myrinet=true
###----------------------------------------------------------------###
###                                                                ###
###                        for general runs                        ###
###                                                                ###
###----------------------------------------------------------------###
###---------------------single node on palmetto:
###p2 : #PBS -l select=1:ncpus=2:mpiprocs=2:mem=4gb:myrinet=true
###p3 : #PBS -l select=1:ncpus=3:mpiprocs=3:mem=4gb:myrinet=true
###p6 : #PBS -l select=1:ncpus=6:mpiprocs=6:mem=20gb:myrinet=true
###p11: #PBS -l select=1:ncpus=11:mpiprocs=11:mem=40gb:myrinet=true
###p16: #PBS -l select=1:ncpus=16:mpiprocs=16:mem=40gb:myrinet=true
###p21: #PBS -l select=1:ncpus=21:mpiprocs=21:mem=40gb:myrinet=true
###---------------------multiple nodes on palmetto:
###p11: #PBS -l select=1:ncpus=6:mpiprocs=6:mem=20gb+1:ncpus=5:mpiprocs=5:mem=20gb:myrinet=true
###p21: #PBS -l select=3:ncpus=7:mpiprocs=7:mem=30gb:myrinet=true
###p31: #PBS -l select=1:ncpus=11:mpiprocs=11:mem=30gb+2:ncpus=10:mpiprocs=10:mem=20gb:myrinet=true
###p41: #PBS -l select=1:ncpus=11:mpiprocs=11:mem=30gb+3:ncpus=10:mpiprocs=10:mem=20gb:myrinet=true
###p51: #PBS -l select=3:ncpus=17:mpiprocs=17:mem=40gb:myrinet=true
###p81: #PBS -l select=9:ncpus=9:mpiprocs=9:mem=40gb:myrinet=true
#PBS -l walltime=168:00:00
#PBS -j oe
#PBS -k e
#PBS -m n

source /etc/profile.d/modules.sh 
module purge 
###module add gcc/4.4 openmpi/1.4.3
module add gcc/4.4 mpich2/1.4
NCORES=`wc -l $PBS_NODEFILE |gawk '//{print $1}'`

echo Master node = `hostname` 
echo Working directory = `echo $PBS_O_WORKDIR` 
echo Node file = $PBS_NODEFILE 
echo This job runs on the following nodes: 
echo `cat $PBS_NODEFILE`

cd $PBS_O_WORKDIR

date

mpiexec -np $NCORES ./pdelphi95 ./param.txt >./out95.run 2>./out95.err

date

###-----------------------for openmpi
###mpiexec -mca mtl mx -mca btl mx,self,sm,tcp -np $NCORES ./pdelphi95 ./param.txt > ./out95.txt


###use command "pbsnodes nodeXXXX" to find out what the features are of 
###any node
###following keywords can be used in "#PBS -l" line to specify hardware details:
###   chip_manufacturer=intel,amd
###   chip_model=opteron,xeon
###   chip_type=e5345,e5410,o2356,l5420,6176,x7542,x7560
###   node_manufacturer=ibm,dell,sun,hp
