C $Header: /u/gcmpack/MITgcm/eesupp/inc/EESUPPORT.h,v 1.10 2009/04/21 16:00:53 jmc Exp $ C $Name: $ CBOP C !ROUTINE: EESUPPORT.h C !INTERFACE: C include "EESUPPORT.h" C C !DESCRIPTION: C *==========================================================* C | EESUPPORT.h | C *==========================================================* C | Support data structures for the MITgcm UV ``execution | C | environment'' code. This data should be private to the | C | execution environment routines. Data which needs to be | C | accessed directly by a numerical model goes in | C | EEPARAMS.h. | C *==========================================================* CEOP C ERROR_HEADER - String which prefixes error messages CHARACTER*(*) ERROR_HEADER PARAMETER ( ERROR_HEADER = ' *** ERROR ***' ) C PROCESS_HEADER - String which prefixes processor number CHARACTER*(*) PROCESS_HEADER PARAMETER ( PROCESS_HEADER = 'PID.TID' ) C MAX_NUM_COMM_MODES - Maximum number of communication modes C COMM_NONE - No edge communication C COMM_MSG - Use messages to communicate edges C COMM_PUT - Use put to communicate edges C COMM_GET - Use get to communicate edges C Note - commName holds an identifying name for each communication C mode. The COMM_ parameters are used to index commName C so the COMM_ parameters need to be in the range C 1 : MAX_NUM_COMM_MODES. INTEGER MAX_NUM_COMM_MODES PARAMETER ( MAX_NUM_COMM_MODES = 4 ) INTEGER COMM_NONE PARAMETER ( COMM_NONE = 1 ) INTEGER COMM_MSG PARAMETER ( COMM_MSG = 2 ) INTEGER COMM_PUT PARAMETER ( COMM_PUT = 3 ) INTEGER COMM_GET PARAMETER ( COMM_GET = 4 ) COMMON /EESUPP_COMMNAME/ commName CHARACTER*10 commName(MAX_NUM_COMM_MODES) C Tile identifiers C Tiles have a number that is unique over the global domain. C A tile that is not there has its number set to NULL_TILE INTEGER NULL_TILE PARAMETER ( NULL_TILE = -1 ) C-- COMMON /EESUPP_C/ Execution environment support character variables C myProcessStr - String identifying my process number COMMON /EESUPP_C/ myProcessStr CHARACTER*128 myProcessStr C-- COMMON /EESUPP_L/ Execution environment support logical variables C initMPError - Flag indicating error during multi-processing C initialisation. C finMPError - Flag indicating error during multi-processing C termination. C ThError - Thread detected an error. C usingMPI - Flag controlling use of MPI routines. This flag C allows either MPI or threads to be used in a C shared memory environment which can be a useful C debugging/performance analysis tool. C usingSyncMessages - Flag that causes blocking communication to be used C if possible. When false non-blocking EXCH routines C will be used if possible. C notUsingXPeriodicity - Flag indicating no X/Y boundary wrap around C notUsingYPeriodicity This affects the communication routines but C is generally ignored in the numerical model C code. C threadIsRunning, threadIsComplete - Flags used to check for correct behaviour C of multi-threaded code. C threadIsRunning is used to check that the C threads we need are running. This catches the C situation where a program eedata file has nTthreads C greater than the setenv PARALLEL or NCPUS variable. C threadIsComplete is used to flag that a thread has C reached the end of the model. This is used as a check to C trap problems that might occur if one thread "escapes" C the main.F master loop. This should not happen C if the multi-threading compilation tools works right. C But (see for example KAP) this is not always the case! COMMON /EESUPP_L/ thError, threadIsRunning, threadIsComplete, & allMyEdgesAreSharedMemory, usingMPI, usingSyncMessages, & notUsingXPeriodicity, notUsingYPeriodicity LOGICAL thError(MAX_NO_THREADS) LOGICAL threadIsRunning(MAX_NO_THREADS) LOGICAL threadIsComplete(MAX_NO_THREADS) LOGICAL allMyEdgesAreSharedMemory(MAX_NO_THREADS) LOGICAL usingMPI LOGICAL usingSyncMessages LOGICAL notUsingXPeriodicity LOGICAL notUsingYPeriodicity C-- COMMON /EESUPP_I/ Parallel support integer globals C pidW - Process ID of neighbor to West C pidE - ditto East C pidN - ditto North C pidS - ditto South C Note: pid[XY] is not necessairily the UNIX C process id - it is just an identifying C number. C myPid - My own process id C nProcs - Number of processes C westCommunicationMode - Mode of communication for each tile face C eastCommunicationMode C northCommunicationMode C southCommunicationMode C bi0 - Low cartesian tile index for this process C bj0 Note - In a tile distribution with holes bi0 and bj0 C are not useful. Neighboring tile indices must C be derived some other way. C tileNo - Tile identification number for my tile and C tileNo[WENS] my N,S,E,W neighbor tiles. C tilePid[WENS] - Process identification number for C my N,S,E,W neighbor tiles. C nTx, nTy - No. threads in X and Y. This assumes a simple C cartesian gridding of the threads which is not C required elsewhere but that makes it easier. COMMON /EESUPP_I/ & myPid, nProcs, pidW, pidE, pidN, pidS, & tileCommModeW, tileCommModeE, & tileCommModeN, tileCommModeS, & tileNo, tileNoW, tileNoE, tileNoS, tileNoN, & tilePidW, tilePidE, tilePidS, tilePidN, & tileBiW, tileBiE, tileBiS, tileBiN, & tileBjW, tileBjE, tileBjS, tileBjN, & tileTagSendW, tileTagSendE, tileTagSendS, tileTagSendN, & tileTagRecvW, tileTagRecvE, tileTagRecvS, tileTagRecvN INTEGER myPid INTEGER nProcs INTEGER pidW INTEGER pidE INTEGER pidN INTEGER pidS INTEGER tileCommModeW ( nSx, nSy ) INTEGER tileCommModeE ( nSx, nSy ) INTEGER tileCommModeN ( nSx, nSy ) INTEGER tileCommModeS ( nSx, nSy ) INTEGER tileNo( nSx, nSy ) INTEGER tileNoW( nSx, nSy ) INTEGER tileNoE( nSx, nSy ) INTEGER tileNoN( nSx, nSy ) INTEGER tileNoS( nSx, nSy ) INTEGER tilePidW( nSx, nSy ) INTEGER tilePidE( nSx, nSy ) INTEGER tilePidN( nSx, nSy ) INTEGER tilePidS( nSx, nSy ) INTEGER tileBiW( nSx, nSy ) INTEGER tileBiE( nSx, nSy ) INTEGER tileBiN( nSx, nSy ) INTEGER tileBiS( nSx, nSy ) INTEGER tileBjW( nSx, nSy ) INTEGER tileBjE( nSx, nSy ) INTEGER tileBjN( nSx, nSy ) INTEGER tileBjS( nSx, nSy ) INTEGER tileTagSendW( nSx, nSy ) INTEGER tileTagSendE( nSx, nSy ) INTEGER tileTagSendN( nSx, nSy ) INTEGER tileTagSendS( nSx, nSy ) INTEGER tileTagRecvW( nSx, nSy ) INTEGER tileTagRecvE( nSx, nSy ) INTEGER tileTagRecvN( nSx, nSy ) INTEGER tileTagRecvS( nSx, nSy ) #ifdef ALLOW_USE_MPI C-- Include MPI standard Fortran header file C#include "mpif.h" include 'mpif.h' #define _mpiTRUE_ 1 #define _mpiFALSE_ 0 C-- COMMON /EESUPP_MPI_I/ MPI parallel support integer globals C mpiPidW - MPI process id for west neighbor. C mpiPidE - MPI process id for east neighbor. C mpiPidN - MPI process id for north neighbor. C mpiPidS - MPI process id for south neighbor. C mpiPidNW - MPI process id for northwest neighbor. C mpiPidNE - MPI process id for northeast neighbor. C mpiPidSW - MPI process id for southwest neighbor. C mpiPidSE - MPI process id for southeast neighbor. C mpiPidIO - MPI process to use for IO. C mpiNprocs - No. of MPI processes. C mpiMyId - MPI process id of me. C mpiComm - MPI communicator to use. C mpiPx - My MPI proc. grid X coord C mpiPy - My MPI proc. grid Y coord C mpiXGlobalLo - My bottom-left (south-west) x-coordinate in C global domain. C mpiYGlobalLo - My bottom-left (south-west) y-coordinate in C global domain. C mpiTypeXFaceBlock_xy_r4 - Primitives for communicating edge C mpiTypeXFaceBlock_xy_r8 of a block. C mpiTypeYFaceBlock_xy_r4 XFace is used in east-west transfer C mpiTypeYFaceBlock_xy_r8 YFace is used in nrth-south transfer C mpiTypeXFaceBlock_xyz_r4 xy is used in two-dimensional arrays C mpiTypeXFaceBlock_xyz_r8 xyz is used with three-dimensional arrays C mpiTypeYFaceBlock_xyz_r4 r4 is used for real*4 data C mpiTypeYFaceBlock_xyz_r8 r8 is used for real*8 data C mpiTypeXFaceThread_xy_r4 - Composites of the above primitives C mpiTypeXFaceThread_xy_r8 for communicating edges of all blocks C mpiTypeYFaceThread_xy_r4 owned by a thread. C mpiTypeYFaceThread_xy_r8 C mpiTypeXFaceThread_xyz_r4 C mpiTypeXFaceThread_xyz_r8 C mpiTypeYFaceThread_xyz_r4 C mpiTypeYFaceBlock_xyz_r8 C mpiTagE - Tags are needed to mark requests when MPI is running C mpiTagW between multithreaded processes or when the same process. C mpiTagS is a neighbor in more than one direction. The tags ensure that C mpiTagN a thread will get the message it is looking for. C mpiTagSW The scheme adopted is to tag messages according to C mpiTagSE the direction they are travelling. Thus a message C mpiTagNW travelling east is tagged mpiTagE. However, in a C mpiTagNE multi-threaded environemnt several messages could C be travelling east from the same process at the C same time. The tag is therefore modified to C be mpiTag[EWS...]*nThreads+myThid. This requires that C each thread also know the thread ids of its "neighbor" C threads. COMMON /EESUPP_MPI_I/ & mpiPidW, mpiPidE, mpiPidS, mpiPidN, & mpiPidSE, mpiPidSW, mpiPidNE, mpiPidNW, & mpiPidIo, mpiMyId, mpiNProcs, mpiComm, & mpiPx, mpiPy, mpiXGlobalLo, mpiYGlobalLo, & mpiTypeXFaceBlock_xy_r4, mpiTypeXFaceBlock_xy_r8, & mpiTypeYFaceBlock_xy_r4, mpiTypeYFaceBlock_xy_r8, & mpiTypeXFaceBlock_xyz_r4, mpiTypeXFaceBlock_xyz_r8, & mpiTypeYFaceBlock_xyz_r4, mpiTypeYFaceBlock_xyz_r8, & mpiTypeXFaceThread_xy_r4, mpiTypeXFaceThread_xy_r8, & mpiTypeYFaceThread_xy_r4, mpiTypeYFaceThread_xy_r8, & mpiTypeXFaceThread_xyz_r4, mpiTypeXFaceThread_xyz_r8, & mpiTypeYFaceThread_xyz_r4, mpiTypeYFaceThread_xyz_r8, & mpiTagE, mpiTagW, mpiTagN, mpiTagS, & mpiTagSE, mpiTagSW, mpiTagNW, mpiTagNE INTEGER mpiPidW INTEGER mpiPidE INTEGER mpiPidS INTEGER mpiPidN INTEGER mpiPidSW INTEGER mpiPidSE INTEGER mpiPidNW INTEGER mpiPidNE INTEGER mpiPidIO INTEGER mpiMyId INTEGER mpiNProcs INTEGER mpiComm INTEGER mpiPx INTEGER mpiPy INTEGER mpiXGlobalLo INTEGER mpiYGlobalLo INTEGER mpiTypeXFaceBlock_xy_r4 INTEGER mpiTypeXFaceBlock_xy_r8 INTEGER mpiTypeYFaceBlock_xy_r4 INTEGER mpiTypeYFaceBlock_xy_r8 INTEGER mpiTypeXFaceBlock_xyz_r4 INTEGER mpiTypeXFaceBlock_xyz_r8 INTEGER mpiTypeYFaceBlock_xyz_r4 INTEGER mpiTypeYFaceBlock_xyz_r8 INTEGER mpiTypeXFaceThread_xy_r4(MAX_NO_THREADS) INTEGER mpiTypeXFaceThread_xy_r8(MAX_NO_THREADS) INTEGER mpiTypeYFaceThread_xy_r4(MAX_NO_THREADS) INTEGER mpiTypeYFaceThread_xy_r8(MAX_NO_THREADS) INTEGER mpiTypeXFaceThread_xyz_r4(MAX_NO_THREADS) INTEGER mpiTypeXFaceThread_xyz_r8(MAX_NO_THREADS) INTEGER mpiTypeYFaceThread_xyz_r4(MAX_NO_THREADS) INTEGER mpiTypeYFaceThread_xyz_r8(MAX_NO_THREADS) INTEGER mpiTagNW INTEGER mpiTagNE INTEGER mpiTagSW INTEGER mpiTagSE INTEGER mpiTagW INTEGER mpiTagE INTEGER mpiTagN INTEGER mpiTagS C-- COMMON /MPI_FULLMAP_I/ holds integer arrays of the full list of MPI process C mpi_myXGlobalLo :: List of all processors bottom-left X-index in global domain C mpi_myYGlobalLo :: List of all processors bottom-left Y-index in global domain C Note: needed for mpi gather/scatter routines & singleCpuIO. COMMON /MPI_FULLMAP_I/ & mpi_myXGlobalLo, mpi_myYGlobalLo INTEGER mpi_myXGlobalLo(nPx*nPy) INTEGER mpi_myYGlobalLo(nPx*nPy) C MPI communicator describing this model realization COMMON /MPI_COMMS/ & MPI_COMM_MODEL INTEGER MPI_COMM_MODEL #endif /* ALLOW_USE_MPI */