PBS Sample Job Scripts

From UF HPC Wiki

(Redirected from Job Submission Scripts)

PBS Sample Job Scripts

This is a summary of all the sample PBS job scripts available, click on the link for the application to see more information.

This is a long page, use the table of contents!

You should read (if you have not already) the Getting Started and Job Submission Queues pages before using/editing these scripts.

Please refer to the Sample Submission Script section of the wiki page of the application for more information on the script.

General

Run it on the Test Queue for a quick test of the application

#!/bin/bash
#
#PBS -r n
#PBS -N testjob
#PBS -o testjob.out
#PBS -e testjob.err
#PBS -q testq
#PBS -l nodes=1:ppn=1:infiniband
#PBS -l walltime=00:07:00
#

TorqueHowto :

#! /bin/bash
#PBS -r n
#PBS -N testjob
#PBS -o testjob.out
#PBS -e testjob.err
#PBS -m abe
#PBS -M YOUR_MAIL_ADDRESS_HERE
#PBS -l walltime=01:00:00
#PBS -l pmem=100mb
#PBS -l nodes=1:ppn=1:gige

date
hostname
sleep 20
date

By Application

Amber

Amber Scripts

AUTO3DEM

AUTO3DEM :

#!/bin/bash                                                                 
#
#PBS -r n

##Job settings
#PBS -N a3d
#PBS -o a3d.out
#PBS -e a3d.err
#PBS -j oe
#PBS -m abe
#PBS -M YOUR_MAIL_ADDRESS_HERE

##Job configuration
#PBS -V

##Job Resources
#PBS -l nodes=8:ppn=1:infiniband
#PBS -l pmem=100mb
#PBS -l walltime=01:00:00


CPUS=`cat $PBS_NODEFILE | wc -l`

if [ `/usr/local/sbin/IbEnabled` -gt 0 ] ; then
    echo "Running on IB-enabled node set"
    MPIRUN="mpirun --mca btl openib"
else
    echo "Running on GigE-enabled node set"
    MPIRUN="mpirun --mca btl ^udapl,openib --mca btl_tcp_if_include eth0"
fi

setenv MPIRUN "$MPIRUN"

PATH=/apps/auto3dem/bin:$path

EXE=/apps/auto3dem/bin/auto3dem
INPUT=/scratch/ufhpc/taylor/auto3dem/AAV9/

cd $PBS_O_WORKDIR
auto3dem -ncpu=$CPUS -input=####_master

Blender

Blender :

#!/bin/bash
#
#PBS -r n

##Job settings
#PBS -N tux
#PBS -o tux.out
#PBS -e tux.err
#PBS -j oe
#PBS -m abe
#PBS -M YOUR_MAIL_ADDRESS_HERE

##Job configuration

##Job reesources
#PBS -l nodes=1:ppn=2
#PBS -l pmem=100mb
#PBS -l walltime=01:00:00


EXE=/apps/blender/2.45/blender

cd /home/jka/
$EXE -b Tux.blend -f 1 -x 1 -t 2

CMSSW

CMSSW :

#! /bin/bash
#
#PBS -r n

##Job settings
#PBS -N cmsRun
#PBS -o job.out
#PBS -e job.err
#PBS -m abe
#PBS -M YOUR_MAIL_ADDRESS_HERE

##Job Configuration

##Job Resources
#PBS -l walltime=01:00:00
#PBS -l nodes=1:ppn=1
#PBS -l pmem=1000mb

# initialize environment for worker
export SCRAM_ARCH=slc4_ia32_gcc345
export OSG_APP=/scratch/ufhpc/osg/app
export VO_CMS_SW_DIR=${OSG_APP}/cmssoft/cms
export CMS_PATH=${VO_CMS_SW_DIR}
. ${CMS_PATH}/cmsset_default.sh;

echo "Job running on `hostname` at `date`"

# enter working area
cd /scratch/ufhpc/mschmitt/CMSSW_1_6_8/src
eval `scramv1 runtime -sh`
cmsRun FastSimulation/EventProducer/test/FamosProducer.cfg

Cnssolve

Cnssolve :

#!/bin/bash
#
#PBS -r n

##Job settings
#PBS -N rmc
#PBS -o rmc.out
#PBS -e rmc.err
#PBS -j oe
#PBS -m abe
#PBS -M YOUR_MAIL_ADDRESS_HERE

##Job configuration
#PBS -V

##Job resources
#PBS -l nodes=1:ppn=1
#PBS -l pmem=100mb
#PBS -l walltime=01:00:00

PATH=/apps/cnssolve:$path

source  /apps/cnssolve/cns_solve_1.2/cns_solve_env
INPUT=$UFHPC_SCRATCH/$USER/cns

cd $UFHPC_SCRATCH/$USER/cns


cns_solve <generate.inp>generate.log

Dlpoly

Dlpoly :

#!/bin/bash
#
#PBS -r n

##Job settings
#PBS -N test1
#PBS -o test.out
#PBS -e test.err
#PBS -m abe
#PBS -M YOUR_MAIL_ADDRESS_HERE

##Job configuration

##Job resources
#PBS -l nodes=4:ppn=1
#PBS -l walltime=01:00:00
#PBS -l pmem=100mb

EXE=/apps/dlpoly/3.07/execute/DLPOLY.Y

CPUS=`cat $PBS_NODEFILE | wc -l`

if [ `/usr/local/sbin/IbEnabled` -gt 0 ] ; then
    echo "Running on IB-enabled node set"
    MPIRUN="mpirun --mca btl openib"
else
    echo "Running on GigE-enabled node set"
    MPIRUN="mpirun --mca btl ^udapl,openib --mca btl_tcp_if_include eth0"
fi

cd /scratch/ufhpc/prescott/dlpoly/TEST1
pwd
which mpirun

echo Running dlpoly...
$MPIRUN -np $CPUS $EXE
echo "Done."

Dock

Dock :

#!/bin/bash
#
#PBS -r n

## Job settings
#PBS -N noflex1of3
#PBS -o noflex1of3.out
#PBS -e noflex1of3.err
#PBS -j oe
#PBS -m abe
#PBS -M YOUR_MAIL_ADDRESS_HERE

## Job configuration

##Job resources
#PBS -l nodes=4:ppn=1
#PBS -l walltime=2:00:00
#PBS -l pmem=100mb

CPUS=`cat $PBS_NODEFILE | wc -l`

EXE="/apps/dock/61/dock6/bin/dock6.mpi"

if [ `/usr/local/sbin/IbEnabled` -gt 0 ] ; then
    echo "Running on IB-enabled node set"
    MPIRUN="mpirun --mca btl openib"
else
    echo "Running on GigE-enabled node set"
    MPIRUN="mpirun --mca btl ^udapl,openib --mca btl_tcp_if_include eth0"
fi

# Execution directory
cd <YOUR_SCRATCH_DIR>/doc/my_dock_jobdir

which mpirun
echo Running $EXE
$MPIRUN -np $CPUS $EXE -i dock_no_flex.in -o nci_1of3_100_noflex_auto.out

Freesurfer

Freesurfer :

#!/bin/bash
#
#PBS -r n

##Job settings
#PBS -N freesurfer
#PBS -o dnet.out
#PBS -e dnet.err
#PBS -j oe
#PBS -m abe
#PBS -M YOUR_MAIL_ADDRESS_HERE

##Job configuration

##Job resources
#PBS -l nodes=1:ppn=1
#PBS -l walltime=01:00:00
#PBS -l pmem=100mb

FREESURFER_HOME=/apps/freesurfer
source $FREESURFER_HOME/SetUpFreeSurfer.sh

EXE=/apps/freesurfer/bin/recon-all

$EXE -s bert -all

Gaussian

Gaussian :

#!/bin/csh -f
#
#PBS -N Moderate
#PBS -o Moderate.out
#PBS -e Moderate.err
#PBS -j oe
#PBS -m abe
#PBS -r n
#PBS -M YOUR_MAIL_ADDRESS_HERE
#PBS -l walltime=400:00:00
#PBS -l nodes=1:ppn=1
#PBS -l pmem=900m
## Change your per-process memory request as needed

groups
setenv g03root /apps/gaussian/e01/amd64

source $g03root/g03/bsd/g03.login

setenv job gaus_inp
setenv GAUSS_SCRDIR ${UFHPC_SCRATCH}/${USER}/gaus1
setenv INPUT $job.com

# Your job will run here
# make sure your input files live here

cd $GAUSS_SCRDIR

g03  $INPUT

Script Explanation

Notes

Processors

In the above example, the line

#PBS -l nodes=1:ppn=4

tells Torque to use four processors on a single node to perform this job. As Gaussian is only SMP aware, it is not feasible to try to run the same job on multiple nodes, as it will not work. It is possible to use eight processors on a single node, but due to the scarcity of eight-way nodes, this is not likely to be a good solution unless absolutely required.

Also, if you are going to use more than one processor, your input file needs to tell Gaussian to use these extra processors. For instance, in the above example we tell torque that we want to use four processors. As such, we also need to tell Gaussian to use four processors in the input file in the Link 0 section of the input file:

%nproc=4

Error Handling

If you happen to get something like the following:

gaussian
cp: cannot create regular file `./h2o.com': Permission denied
Unable to open input file "h2o.com".
/var/spool/PBS/mom_priv/jobs/163242.pbs.local.SC: line 30:  5889 Segmentation fault      g03 $INPUT
cp: cannot stat `h2o.log': No such file or directory

this is a possible indication that you have invalid permissions to a directory or file. In the case where this particular error was generated, the directory that was being referenced in $GAUSS_SRCDIR was not owned by the user, and therefore the files could not be created.

MPB

MPB :

#!/bin/bash
#
#PBS -r n

##Job settings
#PBS -N test
#PBS -o test.out
#PBS -e test.err
#PBS -j oe
#PBS -m abe
#PBS -M YOUR_MAIL_ADDRESS_HERE

##Job configuration

##Job resources
#PBS -l nodes=4:ppn=1
#PBS -l walltime=01:00:00
#PBS -l pmem=100mb

CPUS=`cat $PBS_NODEFILE | wc -l`

if [ `/usr/local/sbin/IbEnabled` -gt 0 ] ; then
    echo "Running on IB-enabled node set"
    MPIRUN="mpirun --mca btl openib"
else
    echo "Running on GigE-enabled node set"
    MPIRUN="mpirun --mca btl ^udapl,openib --mca btl_tcp_if_include eth0"
fi


cd /scratch/mri/jka
$MPIRUN -np $CPUS /apps/mpb/bin/mpb-mpi -v mpb.ctl > mpb.out

R

R :

#!/bin/bash
#
#PBS -r n

##Job settings
#PBS -N TestOfR2
#PBS -o TestOfR2.out
#PBS -e TestOfR2.err
#PBS -j oe
#PBS -m abe
#PBS -M YOUR_MAIL_ADDRESS_HERE

##Job configuration

##Job resources
#PBS -l nodes=1:ppn=1
#PBS -l walltime=01:00:00
#PBS -l pmem=100mb

EXE=/usr/bin/R

SCRDIR=/scratch/ufhpc/$USER/$PBS_JOBNAME

if [ ! -d $SCRDIR ]; then
  mkdir -p $SCRDIR
fi

cd $SCRDIR
echo running R on `hostname -s` in `pwd`
$EXE  --vanilla  < test.r > r2.out
echo Done

PWSCF

Using MVAPICH

Note that this submission script is for an MVAPICH-based PWSCF executable. The default MPI environment for most users is OpenMPI. If you want to run this executable, you must change your mpi-environment to mvapich_intel10-0.9.9 using mpi-selector. You can do so by

Run the command mpi-selector --set mvapich_intel10-0.9.9
Log out
Log back in

Script

#PBS -N MyJob
#PBS -o pwscf.out
#PBS -e pwscf.err
#PBS -j oe
#PBS -m ae
#PBS -M <your email address goes here>
#PBS -r n
#PBS -q submit
#PBS -l nodes=32:ppn=1:infiniband
#PBS -l pmem=800mb
#PBS -l walltime=18:00:00
#
NAME=relax
INFILE=$NAME.in
OUTFILE=$NAME.out
EXE=/apps/espresso/4.0.4/mvapich/bin/pw.x
#
# Make sure CPUS matches the number of cpus you have requested from PBS/Torque.
#
CPUS=`cat $PBS_NODEFILE | wc -l`

if [ `/usr/local/sbin/IbEnabled` -gt 0 ] ; then
     echo "Running on IB-enabled node set"
     MPIRUN="mpiexec"
else 
     echo " Running on GigE-only node set"
     echo "MVAPICH is not supported on GigE-only hosts"
fi 

cd $PBS_O_WORKDIR
echo RUnning on nodes:
cat $PBS_NODEFILE
echo Running pwscf with $CPUS processors...
rm -f pwscf.log

Siesta

Siesta :

#!/bin/bash
#
#PBS -N SiestaJob
#PBS -o SiestaJob.out
#PBS -e SiestaJob.err
#PBS -j oe
#PBS -m abe
#PBS -r n
#PBS -q submit
#PBS -l nodes=1:ppn=4:infiniband
#PBS -l mem=1800mb
#PBS -l walltime=76:00:00
#
EXE=/apps/siesta/2.0/mvapich/0.9.9/siesta
CPUS=`cat $PBS_NODEFILE | wc -l`

if [ `/usr/local/sbin/IbEnabled` -gt 0 ]; then
     echo "Running on IB-enabled node set"
     MPIRUN=mpiexec
else
     echo "MVAPICH is not supported on GigE-only nodes"
     exit
fi

cd $PBS_O_WORKDIR
echo Running $EXE:t on $CPUS processors...
$MPIRUN $EXE < dia111hcf3mdhot.fdf 
echo "Done."

Stata

Stata :

#!/bin/bash
#
#PBS -r n

#Job settings
#PBS -N stata
#PBS -o stata.out
#PBS -e stata.err
#PBS -m abe
#PBS -M YOUR_MAIL_ADDRESS_HERE

##Job configuration

##Job resources
#PBS -l nodes=1:ppn=1
#PBS -l walltime=01:00:00
#PBS -l pmem=100mb

PATH=/apps/stata/stata10:$PATH

stata -b do <stata input file>

VASP

VASP :

#!/bin/bash
#
#PBS -r n
#PBS -N Moderate
#PBS -o Moderate.out
#PBS -e Moderate.err
#PBS -j oe
#PBS -m abe
#PBS -M YOUR_MAIL_ADDRESS_HERE
#PBS -l walltime=01:00:00
#PBS -l nodes=2:ppn=4:infiniband
#PBS -l pmem=400mb

set EXE = /apps/vasp/4.6.34/intel/mvapich/bin/nebvasp-cd

CPUS=`cat $PBS_NODEFILE | wc -l`


set IbEnabled = `/usr/local/sbin/IbEnabled`
if ( $IbEnabled ) then
     echo "Running on IB-enabled node set"
     set MPIRUN = "mpiexec"
else
     echo "MVAPICH is not supported over GigE"
     exit
endif

cd $PBS_O_WORKDIR
$MPIRUN $EXE > vasp.log 2>&1

MATLAB

MATLAB :

#!/bin/sh
#PBS -N MyMatlabJob
#PBS -o MyMatlabJob.out
#PBS -e MyMatlabJob.err
#PBS -j oe
#PBS -m abe
#PBS -M YOUR_EMAIL_ADDRESS_HERE
#PBS -l nodes=1:ppn=1
#PBS -l walltime=00:10:00                                                                                                                     

export LD_LIBRARY_PATH=/apps/matlab/2008a/bin/glnxa64:/apps/matlab/2008a/sys/os/glnxa64
export PATH=${PATH}:/apps/matlab/2008a/bin
export MCR_CACHE_ROOT=$TMPDIR
cd /scratch/ufhpc/prescott/matlab
./mymatlabexe

Other

Generic Perl Script

Job Submission Scripts : this script is actually generating a PBS script from user input

#!/usr/bin/perl
if ($#ARGV != 1) 
  {
   die "\nExecutable or parameter files missing! \n Usage: submitjobs.pl <executable file> <parameter file>\n";
   }
print "Enter maximum wall time (HH:MM:SS): ";
$walltime = <STDIN>;
chop $walltime;
print "\n\n";

print "Enter maximum memory usage per node (ex. 600mb): ";
$memory = <STDIN>;
chop $memory;
print "\n\n";

foreach $i (0 .. $#ARGV) {
	print $ARGV[$i] . "\n";
}

$executable_file=$ARGV[0];
$parameter_file=$ARGV[1];

#### THE FOLLOWING IS THE BASE NAME GIVEN TO YOUR PBS JOB   ####
#### THE INDEX OF THE PARAMETER VALUE WILL BE APPENDED TO THIS NAME ####
#### TO DISTINGUISH JOBS                               ####
my $PBS_job_identifier=$ARGV[0];
print $PBS_job_identifier;

#### COMPILE C CODE TO FORM OUTPUT exe ####
#### system("pathcc -lm -o exe $directory/$source_file -O3")==0
#### or die "failed compilation";

open(PARAMETERS, $parameter_file) || die "Could not open parameters file\n";

$directory=`pwd`;
chop $directory;
print $pwd . "\n";

### Construct the pbs scripts and submit the jobs

$i=1;
while($param=<PARAMETERS>)
{
	print $param;
	$PBS_jobname="$PBS_job_identifier-".$i;
	$i++;
	print "$PBS_jobname\n";
	open(PBS_SCRIPT,">pbs_script");
	print PBS_SCRIPT "#!/bin/bash\n";
	print PBS_SCRIPT "##Generated with the PBS script generator...\n";
	print PBS_SCRIPT "#PBS -r n\n\n";
	print PBS_SCRIPT "##Job settings\n"
	print PBS_SCRIPT "#PBS -N $PBS_jobname\n";
	print PBS_SCRIPT "#PBS -j oe\n";
	#print PBS_SCRIPT "#PBS -m abe\n";
# If you enable the mailing option, please be
# sure to replace the username (and possibly the domain name) below
	#print PBS_SCRIPT "#PBS -M YOUR_MAIL_ADDRESS_HERE\n";
	print PBS_SCRIPT "\n";
	print PBS_SCRIPT "##Job configuration\n\n";
	print PBS_SCRIPT "##Job resources\n";
	print PBS_SCRIPT "#PBS -l walltime=$walltime\n";
	print PBS_SCRIPT "#PBS -l pmem=$memory\n";
	print PBS_SCRIPT "#PBS -l nodes=1:ppn=1:gige\n";
	print PBS_SCRIPT $directory . "/" . $executable_file . " ";
	print PBS_SCRIPT $param . "\n";

  system("qsub pbs_script")==0
             or die "failed submitting PBS job \n"; 
  
  #### GIVE THE JOB A SECOND TO GET SUBMITTED ####
  sleep(1);
  print "\n";

}

Stata Specific

Another script that creates a script for each "do" file in a stata environment, then submits them:

#!/bin/csh -f
#
#foreach Node ( `cat /etc/dsh/group/phase3 | grep -v r10a` )
foreach F ( *.do )
  echo $F
  set JobScript = `echo $F | sed -e 's/\.do//g'`
  echo JobScript = $JobScript
  set JobScript = ${JobScript}.sh
  cat << EOF > $JobScript
#!/bin/csh -f
#
#PBS -N $JobScript
#PBS -o $JobScript.log
#PBS -e $JobScript.err
#PBS -j oe
#PBS -m abe
#PBS -r n
#PBS -M erten@ufl.edu
#PBS -q submit
#PBS -l walltime=48:10:00
#PBS -l nodes=1:ppn=1
#PBS -l pmem=600mb
#
cd \$PBS_O_WORKDIR

set EXE  = /apps/stata/stata10/stata 

\$EXE -b do $F
EOF
  qsub ./$JobScript
  endif
end

Retrieved from "http://wiki.hpc.ufl.edu/index.php/PBS_Sample_Job_Scripts"

PBS Sample Job Scripts

From UF HPC Wiki

Contents