SPModel: libSPModel/src/CommHandler.c Source File
VPAC - Computational Software Development
Main | SPModel | StGermain FrameWork |
Main Page | Alphabetical List | Class List | Directories | File List | Class Members | File Members

CommHandler.c

Go to the documentation of this file.
00001 /*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
00002 **
00003 ** Copyright (C), 2004, Victorian Partnership for Advanced Computing (VPAC) Ltd, 110 Victoria Street, Melbourne, 3053, Australia.
00004 **
00005 ** Authors:
00006 **  Ogar R. Widjaja, Computational Scientist, VPAC.
00007 **  Raquibul Hassan, Software Engineer, VPAC. (raq@vpac.org)
00008 **  Keith Hsuan, Computational Scientist, VPAC (keith@vpac.org)
00009 **  William F. Appelbe, Director, VPAC. (bill@vpac.org)
00010 **  Stevan M. Quenette, Senior Software Engineer, VPAC. (steve@vpac.org)
00011 **  Patrick D. Sunter, Software Engineer, VPAC. (patrick@vpac.org)
00012 **
00013 ** This file may be distributed under the terms of the VPAC Public License
00014 ** as defined by VPAC of Australia and appearing in the file
00015 ** LICENSE.VPL included in the packaging of this file.
00016 **
00017 ** This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
00018 ** WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
00019 **
00020 */
00030 #include <mpi.h>
00031 #include <StGermain/StGermain.h>
00032 
00033 #include <stdio.h>
00034 #include <stdlib.h>
00035 #include <string.h>
00036 #include <assert.h>
00037 #include <math.h>
00038 #include <limits.h>
00039 #include "types.h"
00040 #include "SurfaceMesh.h"
00041 #include "SurfaceMeshDecomp.h"
00042 #include "SurfaceRegularMesh.h"
00043 #include "_Interpolator.h"
00044 #include "LinearInterpolator.h"
00045 #include "SplineInterpolator.h"
00046 #include "Context.h"
00047 
00048 void BlockingSend_LinkedList( LinkedList *list, size_t dataSize, int destination, int tag, MPI_Comm comm )
00049 {
00050     LinkedListIterator *iter = NULL;
00051     void *data;
00052     
00053     assert( list );
00054     
00055     MPI_Send( &(list->nodeCount), 1, MPI_INT, destination, tag, comm );
00056 
00057     if( list->nodeCount > 0 ){
00058         iter = LinkedListIterator_New( list );
00059 
00060         for( data=LinkedListIterator_First( iter ); data!=NULL; data=LinkedListIterator_Next(iter) ){
00061             
00062             MPI_Send( data, dataSize, MPI_BYTE, destination, tag, comm );
00063         }
00064 
00065         Stg_Class_Delete( iter );
00066     }
00067 }
00068 
00069 void BlockingReceive_LinkedList( LinkedList *list, size_t dataSize, int source, int tag, MPI_Comm comm, MPI_Status *status )
00070 {
00071     int listNodeCount = 0;
00072     void *data = 0;
00073     int i = 0;
00074     
00075     assert( list );
00076     assert( status );
00077     
00078     MPI_Recv( &listNodeCount, 1, MPI_INT, source, tag, comm, status );
00079 
00080     if( listNodeCount > 0 ){
00081         
00082         data = (void*)malloc( dataSize );
00083         
00084         for( i=0; i<listNodeCount; i++ ){
00085             MPI_Recv( data, dataSize, MPI_BYTE, source, tag, comm, status );
00086             LinkedList_InsertNode( list, data, dataSize );
00087         }
00088 
00089         free( data );
00090     }
00091 }
00092 
00093 void NonBlockingSend_LinkedList( LinkedList *list, size_t dataSize, int destination, int tag, MPI_Comm comm,
00094         MPI_Request *requestTable, MPI_Status *statusTable )
00095 {
00096     LinkedListIterator *iter = NULL;
00097     void *data;
00098     int i = 0;
00099     
00100     assert( list );
00101     
00102     MPI_Isend( &(list->nodeCount), 1, MPI_INT, destination, tag, comm, &(requestTable[0]) );
00103     MPI_Wait( &(requestTable[0]), &(statusTable[0]) );
00104 
00105     if( list->nodeCount > 0 ){
00106         iter = LinkedListIterator_New( list );
00107 
00108         i = 0;
00109         for( data=LinkedListIterator_First( iter ); data!=NULL; data=LinkedListIterator_Next(iter), i++ ){
00110             
00111             MPI_Isend( data, dataSize, MPI_BYTE, destination, tag, comm, &(requestTable[i]) );
00112         }
00113 
00114         Stg_Class_Delete( iter );
00115 
00116         MPI_Waitall( i, requestTable, statusTable );
00117     }
00118 }
00119 
00120 void NonBlockingReceive_LinkedList( LinkedList *list, size_t dataSize, int source, int tag, MPI_Comm comm,
00121         MPI_Request *requestTable, MPI_Status *statusTable )
00122 {
00123     int listNodeCount = 0;
00124     void *data = 0;
00125     int i = 0;
00126     
00127     assert( list );
00128     
00129     MPI_Irecv( &listNodeCount, 1, MPI_INT, source, tag, comm, &(requestTable[0]) );
00130     MPI_Wait( &(requestTable[0]), &(statusTable[0]) );
00131 
00132     if( listNodeCount > 0 ){
00133         
00134         data = (void*)malloc( dataSize * listNodeCount * 20 );
00135         memset( data, 0, dataSize * listNodeCount * 20 );
00136         
00137         for( i=0; i<listNodeCount; i++ ){
00138             MPI_Irecv( (data + i*dataSize ), dataSize, MPI_BYTE, source, tag, comm, &(requestTable[i]) );
00139         }
00140 
00141         MPI_Waitall( i, requestTable, statusTable );
00142     
00143         for( i=0; i<listNodeCount; i++ ){
00144             LinkedList_InsertNode( list, (data + i*dataSize), dataSize );
00145         }
00146         
00147         free( data );
00148     }
00149 }
00150 
00151 void SPModelSimulationContextExtension_Gather( void *_context )
00152 {
00153     typedef struct Package_t{
00154         int id;
00155         float water, erosion, sedimentHistory, uplift;
00156     }Package; 
00157     
00158     int i, j;
00159     Package package;
00160     MPI_Status status;
00161 
00162     SPModel_Context *context;
00163     SurfaceMesh *localMesh, *globalMesh;
00164     _SurfaceMeshDecomp *meshDecomp;
00165     SPModelSimulationContextExtension *simulationExt = NULL;
00166     ExtensionInfo_Index SPModel_Simulation_ContextExtHandle = -1;
00167 
00168     assert( _context );
00169     
00170     context = (SPModel_Context*)_context;
00171     
00172     localMesh = context->localMesh;
00173     globalMesh = context->globalMesh;
00174     meshDecomp = context->meshDecomp;
00175 
00176     SPModel_Simulation_ContextExtHandle = ExtensionManager_GetHandle( context->extensionMgr, "Simulation" );
00177     
00178     simulationExt = (SPModelSimulationContextExtension*)ExtensionManager_Get
00179                 ( context->extensionMgr, context, SPModel_Simulation_ContextExtHandle );
00180     assert( simulationExt );
00181     
00182     if( context->rank == MASTER_PROC ){
00183         for( i=0; i<context->nproc; i++ ){
00184             for( j=0; j<meshDecomp->processorLoad[i]; j++ ){
00185                 if( i == MASTER_PROC ){
00186                     assert( globalMesh->id[localMesh->id[j]-1] == localMesh->id[j] );
00187                     
00188                     simulationExt->globalWater[localMesh->id[j]-1] = simulationExt->water[j];
00189                     simulationExt->globalErosion[localMesh->id[j]-1] = simulationExt->erosion[j];
00190                     simulationExt->globalSedimentHistory[localMesh->id[j]-1] = simulationExt->sedimentHistory[j];
00191                     simulationExt->globalUplift[localMesh->id[j]-1] = simulationExt->uplift[j];
00192                 }
00193                 else{
00194                     MPI_Recv( &package, sizeof(Package), MPI_BYTE, i,
00195                         PACKAGE_TAG, MPI_COMM_WORLD, &status );
00196                     
00197                     assert( globalMesh->id[package.id-1] == package.id );
00198 
00199                     simulationExt->globalWater[package.id-1] = package.water;
00200                     simulationExt->globalErosion[package.id-1] = package.erosion;
00201                     simulationExt->globalSedimentHistory[package.id-1] = package.sedimentHistory;
00202                     simulationExt->globalUplift[package.id-1] = package.uplift;
00203                 }
00204             }
00205         }
00206     }
00207     else{
00208         for( i=0; i<localMesh->myLoad; i++ ){
00209             package.id = localMesh->id[i];
00210             package.water = simulationExt->water[i];
00211             package.erosion = simulationExt->erosion[i];
00212             package.sedimentHistory = simulationExt->sedimentHistory[i];
00213             package.uplift = simulationExt->uplift[i];
00214             //package.hisoPrev = simulationExt->hisoPrev[localMesh->id[i]-1];
00215 
00216             MPI_Send( &package, sizeof(Package), MPI_BYTE, MASTER_PROC,
00217                     PACKAGE_TAG, MPI_COMM_WORLD );
00218         }
00219     }
00220 }
00221 
00222 void SPModelSimulationContextExtension_Scatter( void *_context )
00223 {
00224     typedef struct Package_t{
00225         int id;
00226         float water, erosion, sedimentHistory, uplift, hisoPrev;
00227     }Package; 
00228     
00229     int i, j;
00230     Package package;
00231     MPI_Status status;
00232 
00233     SPModel_Context *context;
00234     SurfaceMesh *localMesh, *globalMesh;
00235     _SurfaceMeshDecomp *meshDecomp;
00236     SPModelSimulationContextExtension *simulationExt = NULL;
00237     ExtensionInfo_Index SPModel_Simulation_ContextExtHandle = -1;
00238 
00239     assert( _context );
00240     
00241     context = (SPModel_Context*)_context;
00242     
00243     localMesh = context->localMesh;
00244     globalMesh = context->globalMesh;
00245     meshDecomp = context->meshDecomp;
00246 
00247     SPModel_Simulation_ContextExtHandle = ExtensionManager_GetHandle( context->extensionMgr, "Simulation" );
00248     
00249     simulationExt = (SPModelSimulationContextExtension*)ExtensionManager_Get
00250                 ( context->extensionMgr, context, SPModel_Simulation_ContextExtHandle );
00251     assert( simulationExt );
00252     
00253     if( context->rank == MASTER_PROC ){
00254         
00255         for( j=0; j<localMesh->myLoad; j++ ){
00256             assert( globalMesh->id[localMesh->id[j]-1] == localMesh->id[j] );
00257                 
00258             simulationExt->water[j] = simulationExt->globalWater[localMesh->id[j]-1];
00259             simulationExt->erosion[j] = simulationExt->globalErosion[localMesh->id[j]-1];
00260             simulationExt->sedimentHistory[j] = simulationExt->globalSedimentHistory[localMesh->id[j]-1];
00261             simulationExt->uplift[j] = simulationExt->globalUplift[localMesh->id[j]-1];
00262         }
00263         
00264         for( j=0; j<globalMesh->numNodes; j++ ){
00265             if( localMesh->mapGlobalToLocal[globalMesh->id[j]-1] == localMesh->myLoad ){
00266                 package.id = globalMesh->id[j];
00267                 package.water = simulationExt->globalWater[j];
00268                 package.erosion = simulationExt->globalErosion[j];
00269                 package.sedimentHistory = simulationExt->globalSedimentHistory[j];
00270                 package.uplift = simulationExt->globalUplift[j];
00271 
00272                 MPI_Send( &package, sizeof(Package), MPI_BYTE, globalMesh->processor[j],
00273                     PACKAGE_TAG, MPI_COMM_WORLD );
00274             }
00275         }
00276     }
00277     else{
00278         for( i=0; i<localMesh->myLoad; i++ ){
00279             MPI_Recv( &package, sizeof(Package), MPI_BYTE, MASTER_PROC,
00280                     PACKAGE_TAG, MPI_COMM_WORLD, &status );
00281             
00282             assert( localMesh->id[i] == package.id );
00283             
00284             simulationExt->water[i] = package.water;
00285             simulationExt->erosion[i] = package.erosion;
00286             simulationExt->sedimentHistory[i] = package.sedimentHistory;
00287             simulationExt->uplift[i] = package.uplift;
00288         }
00289     }
00290 }
00291