The CCSM 2 0 Scripts Lawrence Buja CCSM

The CCSM 2. 0 Scripts Lawrence Buja CCSM Software Engineering Group ccsm@ucar. edu June 29 2002 www. ccsm. ucar. edu

Three levels of c-shell scripts • run script coordinates building/running the complete system ( eg. $SCRIPTS/test. a 1. run ) • setup scripts configure each CCSM component model (eg. $SCRIPTS/cpl. setup. csh ) (eg. $SCRIPTS/ocn. setup. csh ) • tool scripts generic operations, file transfer and positioning (eg. $TOOLS/ccsm_getinput ) www. ccsm. ucar. edu .

$SCRIPTS/test. a 1. run a. Set batch system options b. Define common environment variables c. Set multi-processing & resolution specifications d. Run the setup script for each component e. Set hardware specific variables f. Run the CCSM Integration. g. Archive/harvest/ h. resubmit when run completes www. ccsm. ucar. edu

A. Set batch system options • CVS Documentation • Batch Queue settings – – – queue name wall clock limits memory limits default shell printed output options www. ccsm. ucar. edu

B. Define environment setenv setenv CASE GRID RUNTYPE SETBLD BASEDATE setenv REFCASE setenv REFDATE setenv setenv test. a 1 T 42_gx 1 v 3 startup auto 0001 -01 -01 # # # test. a 1 0001 -01 -06 case name T 42_gx 1 v 3 or T 31_gx 3 startup, continue, branch, hybrid auto, true, false initial start date yyyy-mm-dd # Runtype = branch data case # Runtype = branch start date CASESTR "fully coupled $GRID test" MSSDIR mss: /$MSSNAME/csm/$CASE MSSDIR null: /dev/nul MSSRPD 0 MSSPWD $LOGNAME # # # www. ccsm. ucar. edu short descriptive text string MSS directory path name MSS file retention period MSS file write password

B. Define environment setenv CSMROOT /fs/cgd/home 0/$LOGNAME/ccsm 2 setenv SCRIPTS $CSMROOT/scripts/$CASE setenv TOOLS $CSMROOT/scripts/tools setenv LOGDIR $CSMROOT/scripts/$CASE/logs setenv CSMCODE $CSMROOT/models setenv CSMUTL $CSMROOT/models/utils setenv CSMSHR $CSMROOT/models/csm_share setenv CSMBLD $CSMROOT/models/bld setenv CSMDATA /fs/cgd/csm/inputdata setenv LID "`date +%y%m%d-%H%M%S`" unsetenv MODEL_SRCDIR www. ccsm. ucar. edu # # # root directory of source run scripts are here some tools are here save stdout here base dir for src code Util directory shared code dir makefiles are here base dir for input data time-stamp/file-ID string use atm/lnd coupled code

B. Define environment setenv setenv EXEROOT OBJROOT LIBROOT INCROOT ARCROOT setenv setenv LFSINP LMSOUT MACINP RFSINP RMSINP MACOUT RFSOUT /ptmp/$LOGNAME/$CASE $EXEROOT/lib $LIBROOT/include /XXX/$LOGNAME/archive/$CASE # # # run model here build code here Locof supplemental libraries supplemental includes/modfiles archive root directory $CSMDATA /CCSM/inputdata /$MSSNAME/csm/$CASE dataproc. ucar. edu /fs/cgd/csm/inputdata /CCSM/inputdataproc. ucar. edu /ptmp/$LOGNAME/archive/$CASE # # # # LOCAL INPUTDATA FSROOT LOCAL INPUTDATA MSROOT LOCAL OUTPUT MSROOT REMOTE INPUTDATA MACHINE REMOTE INPUTDATA FSROOT REMOTE INPUTDATA MSROOT REMOTE OUTPUT MACHINE REMOTE OUTPUT FSROOT www. ccsm. ucar. edu

B. Define environment #--- logic to set BLDTYPE based on SETBLD above setenv BLDTYPE $SETBLD if ($SETBLD =~ auto*) then setenv BLDTYPE false if ($RUNTYPE == 'startup' || $RUNTYPE == 'hybrid') setenv BLDTYPE true endif if ($BLDTYPE != 'true' && $BLDTYPE != 'false') then echo "error in BLDTYPE: $BLDTYPE" exit 1 endif www. ccsm. ucar. edu

B. Determine os/machine/site echo ------------------------------------echo b 1. Determine os/machine/site echo ------------------------------------setenv OS ARCH MACH SITE unknown setenv OS `uname -s` if ($status == 0) then if ( $OS == 'AIX') setenv ARCH IBM if ( $OS == 'OSF 1') setenv ARCH CPQ if ( $OS == 'IRIX 64') setenv ARCH SGI endif www. ccsm. ucar. edu # operating system # architecture

B. Determine os/machine/site setenv MACHKEY `hostname` if ($status == 0) then if ($MACHKEY =~ bb* ) if ($MACHKEY =~ bf* ) if ($MACHKEY =~ s* ) if ($MACHKEY =~ prosp* ) if ($MACHKEY =~ ute* ) if ($MACHKEY =~ n* ) if ($MACHKEY =~ eag* ) if ($MACHKEY =~ fal* ) setenv SITE ncar if ($MACHKEY =~ n* ) if ($MACHKEY =~ s* ) if ($MACHKEY =~ eag* ) if ($MACHKEY =~ fal* ) endif setenv setenv MACH MACH babyblue blackforest seaborg prospect ute nirvana eagle falcon # machine # site, default is ncar setenv SITE lanl nersc ornl www. ccsm. ucar. edu

B. Create ccsm_joe echo ------------------------------------echo b 2. Create ccsm_joe echo ------------------------------------setenv CSMJOE $SCRIPTS/ccsm_joe rm -f $CSMJOE $TOOLS/ccsm_checkenvs > $CSMJOE www. ccsm. ucar. edu

C. Select processors and resolutions set MODELS set SETUPS = ( atm lnd ice ocn if ($GRID == T 42_gx 1 v 3 ) then set NTASKS = ( 8 3 16 40 set NTHRDS = ( 4 4 1 1 else if ($GRID == T 31_gx 3 ) then set NTASKS = ( 4 4 2 4 set NTHRDS = ( 4 1 1 1 else if ($GRID == T 62_gx 1 v 3 ) then set NTASKS = ( 1 1 16 40 set NTHRDS = ( 1 1 else echo "unsupported configuration: $GRID" exit 1 endif cpl ) ) # generic model names. # setup script name 1 4 ) ) # use NTASK = 1 for data model # use NTHRD = 1 for data model 1 2 ) ) # use NTASK = 1 for data model # use NTHRD = 1 for data model 1 4 ) ) # use NTASK = 1 for data model # use NTHRD = 1 for data model setenv ATM_GRID `echo $GRID | sed s/_. *//`; setenv LND_GRID setenv OCN_GRID `echo $GRID | sed s/. *_//`; setenv ICE_GRID www. ccsm. ucar. edu $ATM_GRID $OCN_GRID

D. Prepare models echo ------------------------------------echo d. Prepare $GRID component models for execution echo - create execution directories for atm, cpl, lnd, ice, ocn echo - invoke component model setup scripts found in $SCRIPTS echo ------------------------------------#--- create working directories foreach DIR ( $EXEROOT $LIBROOT $INCROOT $OBJROOT $LOGDIR) if !(-d $DIR) mkdir -p $DIR end #--- run machine dependent commands (i. e. modules on SGI). if (-f $TOOLS/modules. $OS. $MACH) source $TOOLS/modules. $OS. $MACH || exit 1 #--- create env variables for use in components foreach n (1 2 3 4 5) set model = $MODELS[$n] setenv ${model}_dir $EXEROOT/$model; setenv ${model}_setup $SETUPS[$n] setenv ${model}_in $model. stdin ; setenv ${model}_out $model. log. $LID end #--- get restart files #$TOOLS/ccsm_getrestart www. ccsm. ucar. edu

D. Build Earth System Modeling Framework echo ------------------------------------echo d 1. Build Earth System Modeling Framework http: //www. esmf. scd. ucar. edu echo ------------------------------------setenv EXEDIR $EXEROOT/esmf ; if !(-d $EXEDIR) mkdir -p $EXEDIR cd $EXEDIR echo `date` $EXEDIR/esmf. log. $LID | tee esmf. log. $LID $SCRIPTS/esmf. setup. csh >>& esmf. log. $LID || exit 1 #--- create model directories for each platform #--- ($EXEROOT/all for SGI, poe. cmdfile for AIX, prun. cmdfile for OSF 1) if (-e $EXEROOT/all) rm -rf $EXEROOT/all mkdir -p $EXEROOT/all echo "#! /bin/csh -f" >! $EXEROOT/prun. cmdfile www. ccsm. ucar. edu

D. Run Component setup scripts #--- initialize processor count @ PROC = 0 foreach n (1 2 3 4 5) setenv MODEL $MODELS[$n] setenv NTHRD $NTHRDS[$n] setenv OBJDIR $OBJROOT/$MODEL/obj setenv EXEDIR $EXEROOT/$MODEL setenv THREAD FALSE ; ; ; setenv SETUP $SETUPS[$n] setenv NTASK $NTASKS[$n] if !(-d $OBJDIR) mkdir -p $OBJDIR if !(-d $EXEDIR) mkdir -p $EXEDIR if ($NTHRD > 1) setenv THREAD TRUE cd $EXEDIR echo `date` $EXEDIR/$MODEL. log. $LID | tee $MODEL. log. $LID $SCRIPTS/$SETUP. setup. csh >>& $MODEL. log. $LID @ M = 0 while ( $M < $NTASK ) echo "env OMP_NUM_THREADS=$NTHRD $MODEL/$MODEL" >>! $EXEROOT/poe. cmdfile echo "if ("$"RMS_RANK == $PROC). /$MODEL" >>! $EXEROOT/prun. cmdfile; @ M++ ; @ PROC++ end ln -s $EXEROOT/$MODEL $EXEROOT/all/. end www. ccsm. ucar. edu

D. Run Component setup scripts #--- create model directories and processor counts for each platform #--- ($EXEROOT/all for SGI, poe. cmdfile for AIX, prun. cmdfile for OSF 1) if ($n == 1) then rm -rf $EXEROOT/poe. cmdfile $EXEROOT/all; mkdir -p $EXEROOT/all echo "#! /bin/csh -f" >! $EXEROOT/prun. cmdfile @ PROC = 0 if ($BLDTYPE == 'true') then cd $EXEROOT tar -cf $EXEROOT/$CASE. exe. $LID. tar $MODEL/$MODEL endif else if ($BLDTYPE == 'true') then cd $EXEROOT tar -rf $EXEROOT/$CASE. exe. $LID. tar $MODEL/$MODEL endif www. ccsm. ucar. edu

E. Set Hardware Specific variables > e. Setup hardware specific env variables if ( $OS == 'AIX') then limit datasize unlimited ; setenv XLSMPOPTS "stack=86000000" setenv MP_EUILIB us ; setenv MP_RMPOOL 1 setenv MP_NODES $PROC ; setenv MP_PROCS $PROC setenv MP_PGMMODEL mpmd ; setenv MP_CMDFILE poe. cmdfile setenv MP_STDOUTMODE ordered ; setenv MP_SHARED_MEMORY yes setenv MP_EAGER_LIMIT 65536 ; setenv MP_INFOLEVEL 6 else if ( $OS == 'IRIX 64') then setenv TRAP_FPE "UNDERFL=FLUSH_ZERO; OVERFL=ABORT, TRACE; DIVZERO=ABORT, TRACE" setenv OMP_DYNAMIC FALSE ; setenv MPC_GANG OFF; setenv _DSM_WAIT SPIN setenv _DSM_VERBOSE ; setenv _DSM_PLACEMENT ROUND_ROBIN endif env | egrep '(MP_|LOADL|XLS|FPE|DSM|OMP|MPC)' # document above env vars www. ccsm. ucar. edu

F. Run the CCSM integration echo if ( "`date` -- CSM EXECUTION BEGINS HERE" $OS == 'AIX') timex poe $OS == 'OSF 1') prun -n $PROC csh -c prun. cmdfile $OS == 'IRIX 64') mpirun -v -d $EXEROOT/all -np $NTASKS[1] "env OMP_NUM_THREADS=$NTHRDS[1] $MODELS[1]" -np $NTASKS[2] "env OMP_NUM_THREADS=$NTHRDS[2] $MODELS[2]" -np $NTASKS[3] "env OMP_NUM_THREADS=$NTHRDS[3] $MODELS[3]" -np $NTASKS[4] "env OMP_NUM_THREADS=$NTHRDS[4] $MODELS[4]" -np $NTASKS[5] "env OMP_NUM_THREADS=$NTHRDS[5] $MODELS[5]" wait echo "`date` -- CSM EXECUTION HAS FINISHED" www. ccsm. ucar. edu : : &

G. Archive and harvest cd $EXEROOT if (! -d $LOGDIR) mkdir -p $LOGDIR gzip */*. $LID if ($LOGDIR != "" ) cp -p */*. $LID. * $LOGDIR #$SCRIPTS/ccsm_archive if ($OS == 'AIX') then set num = `llq | grep -i $LOGNAME | grep -i share | wc -l` cd $SCRIPTS # if ($num < 1) llsubmit $CASE. har endif #if ($OS != 'AIX') qsub $SCRIPTS/$CASE. har www. ccsm. ucar. edu

H. Resubmit set echo if ( -e $SCRIPTS/RESUBMIT ) then @ N = `cat $SCRIPTS/RESUBMIT` if ( $N > 0 ) then echo "Note: resubmitting run script $CASE. run" @ N-echo $N >! $SCRIPTS/RESUBMIT cd $SCRIPTS if ($OS == 'AIX') llsubmit $CASE. run if ($OS != 'AIX') qsub $CASE. run endif >end of script echo ===================================== echo i. end of nqs shell script echo ===================================== www. ccsm. ucar. edu

Component Setup Script: cpl. setup. csh • The CCSM is designed to allow new component models to easily replace andexisting component in the system. To encapsulate the different buildprocedures required by different component models, each CCSM componenthas a setup script designed to: • Define the run environment of the individual component • Position any input datasets needed by the component • Build the component executable www. ccsm. ucar. edu

Document the setup script #! /bin/csh -f #======================================== # CVS $Id: sample. setup. csh. tex, v 1. 4 2002/06/18 21: 25: 52 southern Exp $ # CVS Source: $ # CVS $Name: $ #======================================== # cpl. setup. csh: Preparing a CSM coupler, cpl 5, for execution # # (a) set environment variables, preposition input data files # (b) create the namelist input file # (b) build this component executable # # For help, see: http: //www. ccsm. ucar. edu/models/cpl #======================================== cat $0; $TOOLS/ccsm_checkenvs || exit -1 www. ccsm. ucar. edu # cat this file, check envs

a. Set local component script variables echo -------------------------------------echo a. set environment variables, preposition input data files echo -------------------------------------if if ($GRID ($GRID =~ =~ T 21* T 31* T 42* T 62* T 85* *gx 3 *gx 1 v 3 ) ) ) ) set set ATM ATM ATM OCN = = = = ( T 21 ( T 31 ( T 42 ( T 62 ( T 85 ( gx 3 ( gx 1 v 3 64 32 ) # name, x dimension, y dimensio 96 48 ) 128 64 ) 192 94 ) 256 128 ) 100 116 ) 320 384 ) www. ccsm. ucar. edu

a. Define & position the input datasets set MAP_A 2 OF_FILE = set MAP_A 2 OS_FILE = set MAP_O 2 AF_FILE = set MAP_R 2 O_FILE = $TOOLS/ccsm_getinput map_T 42_to_gx 1 v 3_aave_da_010709. nc map_T 42_to_gx 1 v 3_bilin_da_010710. nc map_gx 1 v 3_to_T 42_aave_da_010709. nc map_r 05_to_gx 1 v 3_roff_smooth_010718. nc cpl/cpl 5/$MAP_A 2 OF_FILE cpl/cpl 5/$MAP_A 2 OS_FILE cpl/cpl 5/$MAP_O 2 AF_FILE cpl/cpl 5/$MAP_R 2 O_FILE www. ccsm. ucar. edu || || exit 1 1

b. Parse $RUNTYPE and $DATE set RUN_TYPE = $RUNTYPE if ($RUNTYPE == startup) set RUN_TYPE = initial if ($RUNTYPE == hybrid) set RUN_TYPE = initial set BASEDATE_NUM = `echo $BASEDATE | sed -e 's/-//g'` if ($RUNTYPE == branch) then set REST_BFILE = $REFCASE. cpl 5. r. ${REFDATE-00000 echo set REST_BFILE = $REST_BFILE $TOOLS/ccsm_getfile $REFCASE/$MODEL/rest/${REST_BFILE set BASEDATE_NUM = `echo $REFDATE | sed -e 's/-//g'` else set REST_BFILE = 'null' endif www. ccsm. ucar. edu || exit 99

b. Write the input namelist cat >! $MODEL. stdin << EOF &inparm case_name = '$CASE ' case_desc = '$CASESTR ' rest_type = '$RUN_TYPE ' rest_date = $BASEDATE_NUM rest_bfile = '${REST_BFILE ' rest_pfile = '$SCRIPTS/rpointer. $MODEL' map_a 2 of_fn = '$MAP_A 2 OF_FILE' map_a 2 os_fn = '$MAP_A 2 OS_FILE' map_o 2 af_fn = '$MAP_O 2 AF_FILE' map_r 2 o_fn = '$MAP_R 2 O_FILE' www. ccsm. ucar. edu

b. Write the input namelist rest_freq rest_n diag_freq diag_n stop_option stop_n hist_freq hist_n info_bcheck orb_year flx_epbal flx_albav mss_dir mss_rtpd mss_pass mss_rmlf = = = = 'monthly' 3 'ndays' 1 'ndays' 5 'monthly' 1 2 1990 'off' 0 '$MSSDIR/$MODEL/ ' $MSSRPD '$MSSPWD' 0 www. ccsm. ucar. edu

b. Write the input namelist nx_a = $ATM[2] , ny_a = $ATM[3], nx_l = $ATM[2] , ny_l = $ATM[3] nx_o = $OCN[2] , ny_o = $OCN[3], nx_i = $OCN[2] , ny_i = $OCN[3] / EOF echo o contents of $MODEL. stdin: ; cat $MODEL. stdin ; echo ' ' www. ccsm. ucar. edu

c. Build the component executable echo -------------------------------------echo c. Build an executable in $OBJDIR echo -------------------------------------cd $OBJDIR # Filepath: List of source code directories (in order of importance). #----------------------------------cat >! Filepath << EOF $SCRIPTS/src. $MODEL $CSMCODE/cpl 5 $CSMSHR EOF # run make #----------------------------------if ($BLDTYPE == 'true') then cc -o makdep $CSMBLD/makdep. c || exit 2 gmake -j 6 VPFILE=Filepath MODEL=cpl 5 EXEC=$EXEDIR/$MODEL -f $CSMBLD/Makefile MACFILE=$CSMBLD/Macros. $OS || exit 2 else echo "BLDTYPE = $BLDTYPE" endif www. ccsm. ucar. edu

d. Document and cleanup # document the source code used, cleanup $EXEDIR/obj files #----------------------------------grep 'CVS' *. [hf]* #gmake -f $CSMBLD/Makefile MACFILE=$CSMBLD/Macros. $OS mostlyclean echo ' ' echo ====================================== echo End of setup shell script `date` echo ====================================== www. ccsm. ucar. edu