SPModel: libSPModel/src/SurfaceMeshDecomp.c Source File
VPAC - Computational Software Development
Main | SPModel | StGermain FrameWork |
Main Page | Alphabetical List | Class List | Directories | File List | Class Members | File Members

SurfaceMeshDecomp.c

Go to the documentation of this file.
00001 /*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
00002 **
00003 ** Copyright (C), 2004, Victorian Partnership for Advanced Computing (VPAC) Ltd, 110 Victoria Street, Melbourne, 3053, Australia.
00004 **
00005 ** Authors:
00006 **  Ogar R. Widjaja, Computational Scientist, VPAC.
00007 **  Raquibul Hassan, Software Engineer, VPAC. (raq@vpac.org)
00008 **  Keith Hsuan, Computational Scientist, VPAC (keith@vpac.org)
00009 **  William F. Appelbe, Director, VPAC. (bill@vpac.org)
00010 **  Stevan M. Quenette, Senior Software Engineer, VPAC. (steve@vpac.org)
00011 **  Patrick D. Sunter, Software Engineer, VPAC. (patrick@vpac.org)
00012 **
00013 ** This file may be distributed under the terms of the VPAC Public License
00014 ** as defined by VPAC of Australia and appearing in the file
00015 ** LICENSE.VPL included in the packaging of this file.
00016 **
00017 ** This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
00018 ** WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
00019 **
00020 */
00030 #include <mpi.h>
00031 #include <StGermain/StGermain.h>
00032 #include <Cascade/cascade.h>
00033 
00034 #include <stdio.h>
00035 #include <stdlib.h>
00036 #include <string.h>
00037 #include <assert.h>
00038 #include <math.h>
00039 #include <limits.h>
00040 #include "types.h"
00041 #include "SurfaceMeshDecomp.h"
00042 #include "SurfaceMesh.h"
00043 #include "Misc.h"
00044 #include "CommHandler.h"
00045 
00046 /* Textual name of this class */
00047 const Type SurfaceMeshDecomp_Type = "_SurfaceMeshDecomp";
00048     
00049 _SurfaceMeshDecomp                  *_SurfaceMeshDecomp_New( SizeT                          _sizeOfSelf,
00050                                     Type                            type,
00051                                     Stg_Class_DeleteFunction*               _delete,
00052                                     Stg_Class_PrintFunction*                _print,
00053                                     Stg_Class_CopyFunction*             _copy, 
00054                                     Stg_Component_DefaultConstructorFunction*   _defaultConstructor,
00055                                     Stg_Component_ConstructFunction*            _construct,
00056                                     Stg_Component_BuildFunction*        _build,
00057                                     Stg_Component_InitialiseFunction*       _initialise,
00058                                     Stg_Component_ExecuteFunction*      _execute,
00059                                     Stg_Component_DestroyFunction*      _destroy,
00060                                     Name                            name,
00061                                     Bool                            initFlag,
00062                                     Dictionary                      *dictionary,
00063                                     SurfaceMesh                     *_mesh,
00064                                     SurfaceMeshDecomp_SyncMeshFunction *_syncMesh,
00065                                     SurfaceMeshDecomp_ComputeHaloNodesFunction *_computeHaloNodes,
00066                                     SurfaceMeshDecomp_AllocateNodesFunction *_allocateNodes )
00067 {
00068     _SurfaceMeshDecomp* self;
00069     
00070     /* Allocate memory */
00071     assert( _sizeOfSelf >= sizeof(_SurfaceMeshDecomp) );
00072     self = (_SurfaceMeshDecomp*)_Stg_Component_New( _sizeOfSelf, type, _delete, _print, _copy, _defaultConstructor, _construct, _build, 
00073             _initialise, _execute, _destroy, name, NON_GLOBAL );
00074 
00075     assert( dictionary );
00076     assert( _mesh );
00077     assert( _syncMesh );
00078     assert( _computeHaloNodes );
00079     assert( _allocateNodes );
00080     
00081     self->dictionary = dictionary;
00082 
00083     self->mesh = _mesh;
00084     self->syncMesh = _syncMesh;
00085     self->computeHaloNodes = _computeHaloNodes;
00086     self->allocateNodes = _allocateNodes;
00087     
00088     if( initFlag ){
00089         _SurfaceMeshDecomp_Init( self );
00090     }
00091 
00092     return self;
00093 }
00094 
00095 void _SurfaceMeshDecomp_Init( _SurfaceMeshDecomp *self )
00096 {
00097     int i = 0, j = 0;
00098     SurfaceMesh *mesh = NULL;
00099 
00100     assert( self );
00101 
00102     mesh = self->mesh;
00103 
00104     assert( mesh );
00105 
00106     if( mesh->rank == MASTER_PROC ){
00107     
00108         self->processorHaloNodes = (BTree***) malloc( sizeof( BTree** ) * mesh->numProcs );
00109         self->processorForeignHaloNodes = (BTree***) malloc( sizeof( BTree** ) * mesh->numProcs );
00110         for( i=0; i<mesh->numProcs; i++ ){
00111             self->processorHaloNodes[i] = (BTree**) malloc( sizeof( BTree* ) * mesh->numProcs );
00112             self->processorForeignHaloNodes[i] = (BTree**) malloc( sizeof( BTree* ) * mesh->numProcs );
00113         
00114             for( j=0; j<mesh->numProcs; j++ ){
00115                 self->processorHaloNodes[i][j] = BTree_New( providersCompareFunction,
00116                                                         NULL,
00117                                                         providersDataDeleteFunction,
00118                                                         (BTree_dataPrintFunction*)providersDataPrintFunction,
00119                                                         BTREE_NO_DUPLICATES );
00120                 self->processorForeignHaloNodes[i][j] = BTree_New( providersCompareFunction,
00121                                                         NULL,
00122                                                         providersDataDeleteFunction,
00123                                                         (BTree_dataPrintFunction*)providersDataPrintFunction,
00124                                                         BTREE_NO_DUPLICATES );
00125             }
00126         }
00127     
00128         self->processorLoad = (int*) malloc( mesh->numProcs * sizeof(int) );
00129     }
00130     else{
00131     }
00132 }
00133 
00134 void _SurfaceMeshDecomp_Print( void *surfaceMeshDecomp, Stream *stream )
00135 {
00136     _SurfaceMeshDecomp *self = (_SurfaceMeshDecomp*) surfaceMeshDecomp;
00137 
00138     assert( self );
00139     assert( stream );
00140 
00141     /* print parent */
00142     _Stg_Component_Print( (void*) self, stream );
00143 
00144     /* general info */
00145     Journal_Printf( stream, "_SurfaceMeshDecomp (ptr): (%p)\n", self );
00146 
00147     /* Virtual Info */
00148 
00149     /* SurfaceMeshDecomp Info */
00150 }
00151 
00152 void _SurfaceMeshDecomp_Delete( void *surfaceMeshDecomp )
00153 {
00154     int i, j;
00155     SurfaceMesh *mesh = NULL;
00156     _SurfaceMeshDecomp *self = (_SurfaceMeshDecomp*)surfaceMeshDecomp;
00157     
00158     assert( self );
00159 
00160     mesh = self->mesh;
00161 
00162     assert( mesh );
00163 
00164     if( mesh->rank == MASTER_PROC ){
00165     
00166         if( self->processorHaloNodes ){
00167             for( i=0; i<mesh->numProcs; i++ ){
00168                 for( j=0; j<mesh->numProcs; j++ ){
00169                     Stg_Class_Delete( self->processorHaloNodes[i][j] );
00170                     Stg_Class_Delete( self->processorForeignHaloNodes[i][j] );
00171                 }
00172                 free( self->processorHaloNodes[i] );
00173                 free( self->processorForeignHaloNodes[i] );
00174             }
00175             free( self->processorHaloNodes );
00176             free( self->processorForeignHaloNodes );
00177         }
00178 
00179         free( self->processorLoad );
00180     }
00181     else{
00182     }
00183 
00184     for( i=0; i<mesh->numProcs; i++ ){
00185         free( self->incomingHaloPackage[i] );
00186         free( self->outgoingHaloPackage[i] );
00187     }
00188     free( self->incomingHaloPackage );
00189     free( self->outgoingHaloPackage );
00190     
00192     _Stg_Component_Delete( self );
00193 }
00194 
00195 void _SurfaceMeshDecomp_Construct( void *surfaceMeshDecomp, Stg_ComponentFactory *cf )
00196 {
00197     
00198 }
00199 
00200 void _SurfaceMeshDecomp_Build( void *surfaceMeshDecomp, void *data )
00201 {
00202     SurfaceMesh *mesh = NULL;
00203     _SurfaceMeshDecomp *self = (_SurfaceMeshDecomp*) surfaceMeshDecomp;
00204 
00205     assert( self );
00206 
00207     mesh = self->mesh;
00208 
00209     assert( mesh );
00210     
00211     if( mesh->rank == MASTER_PROC ){
00212         assert( self->allocateNodes );
00213         self->allocateNodes( self );
00214 
00215         assert( self->computeHaloNodes );
00216         self->computeHaloNodes( self );
00217     }
00218     else{
00219     }
00220 }
00221 
00222 void _SurfaceMeshDecomp_Initialise( void *surfaceMeshDecomp, void *data )
00223 {
00224     
00225 }
00226 
00227 void _SurfaceMeshDecomp_Execute( void *surfaceMeshDecomp, void *data )
00228 {
00229     typedef struct Package_t{
00230         float x, y, h, hi, h0, slope, length, surface;
00231         int id, receiver, highestNeighbour, size, nLake, nCatch;
00232     }Package; 
00233 
00234     typedef struct HaloPackage_t{
00235         float x, y, h;
00236         int id, receiver;
00237         float surface;
00238     }HaloPackage;
00239 
00240     _SurfaceMeshDecomp *meshDecomp = NULL;
00241     SurfaceMesh *globalMesh = NULL, *localMesh = NULL;
00242     MPI_Status status;
00243     int i = 0, j = 0, k = 0, globalNum = 0;
00244     BTreeIterator *iter = NULL;
00245     Package package={0};
00246     HaloPackage haloPackage={0};
00247     void *result = NULL;
00248 
00249     assert( surfaceMeshDecomp );
00250     assert( data );
00251 
00252     meshDecomp = (_SurfaceMeshDecomp*)surfaceMeshDecomp;
00253     globalMesh = meshDecomp->mesh;
00254     localMesh = (SurfaceMesh*)data;
00255     
00256     if( localMesh->rank == MASTER_PROC ){
00257 
00258         localMesh->myLoad = meshDecomp->processorLoad[globalMesh->rank];
00259         
00260         Stg_Component_Initialise( localMesh, (void*)&(localMesh->myLoad), True );
00261         
00262         memcpy( localMesh->boundaryConditions, globalMesh->boundaryConditions, sizeof(float)*globalMesh->numNodes );
00263         memcpy( localMesh->numNeigh, globalMesh->numNeigh, sizeof(int)*globalMesh->numNodes );
00264         
00265         for( i=1; i<globalMesh->numProcs; i++ ){
00266             MPI_Send( &(meshDecomp->processorLoad[i]), 1, MPI_INT, i,
00267                     MY_LOAD_TAG, MPI_COMM_WORLD );
00268 
00269         }
00270         
00271         for( i=0,j=0; i<globalMesh->numNodes; i++ ){
00272             if( globalMesh->processor[i] != MASTER_PROC ){
00273                 package.id = globalMesh->id[i];
00274                 package.x = globalMesh->x[i];
00275                 package.y = globalMesh->y[i];
00276                 package.h = globalMesh->h[i];
00277                 package.hi = globalMesh->hi[i];
00278                 package.h0 = globalMesh->h0[i];
00279                 package.slope = globalMesh->slope[i];
00280                 package.length = globalMesh->length[i];
00281                 package.surface = globalMesh->surface[i];
00282                 package.receiver = globalMesh->receiver[i];
00283                 package.highestNeighbour = globalMesh->highestNeighbour[i];
00284                 package.size = globalMesh->size[i];
00285                 package.nLake = globalMesh->nLake[i];
00286                 package.nCatch = globalMesh->nCatch[i];
00287                 
00288                 MPI_Send( &package, sizeof(Package), MPI_BYTE, globalMesh->processor[i],
00289                         PACKAGE_TAG, MPI_COMM_WORLD );
00290             }
00291             else{
00292                 localMesh->id[j] = globalMesh->id[i];
00293                 localMesh->sortedId[j] = globalMesh->id[i];
00294                 localMesh->x[j] = globalMesh->x[i];
00295                 localMesh->y[j] = globalMesh->y[i];
00296                 localMesh->h[j] = globalMesh->h[i];
00297                 localMesh->hi[j] = globalMesh->hi[i];
00298                 localMesh->h0[j] = globalMesh->h0[i];
00299                 localMesh->slope[j] = globalMesh->slope[i];
00300                 localMesh->length[j] = globalMesh->length[i];
00301                 localMesh->surface[j] = globalMesh->surface[i];
00302                 localMesh->receiver[j] = globalMesh->receiver[i];
00303                 localMesh->highestNeighbour[j] = globalMesh->highestNeighbour[i];
00304                 localMesh->size[j] = globalMesh->size[i];
00305                 localMesh->nLake[j] = globalMesh->nLake[i];
00306                 localMesh->nCatch[j] = globalMesh->nCatch[i];
00307                 j++;
00308             }
00309         }
00310     
00311         MPI_Bcast ( localMesh->numNeigh, localMesh->numNodes, MPI_INT, MASTER_PROC, MPI_COMM_WORLD );
00312 
00313         for( i=0,j=0; i<globalMesh->numNodes; i++ ){
00314             if( globalMesh->processor[i] != MASTER_PROC ){
00315                 MPI_Send( globalMesh->nodeNeighbours[i], globalMesh->numNeigh[i], MPI_INT, globalMesh->processor[i],
00316                         NEIGHBOUR_TAG, MPI_COMM_WORLD );
00317             }
00318             else{
00319                 memset( localMesh->nodeNeighbours[j], 0, sizeof( int ) * localMesh->maxNeighboursPerNode );
00320                 memcpy( localMesh->nodeNeighbours[j], globalMesh->nodeNeighbours[i], globalMesh->numNeigh[i] * sizeof(int) );
00321                 j++;
00322             }
00323         }
00324         
00325         for( i=0, j=0; i<globalMesh->numNodes; i++ ){
00326             if( globalMesh->processor[i] != MASTER_PROC ){
00327                 MPI_Send( globalMesh->sides[i], globalMesh->numNeigh[i], MPI_FLOAT, globalMesh->processor[i],
00328                         SIDE_TAG, MPI_COMM_WORLD );
00329             }
00330             else{
00331                 memset( localMesh->sides[j], 0, sizeof( float ) * localMesh->maxNeighboursPerNode );
00332                 memcpy( localMesh->sides[j], globalMesh->sides[i], globalMesh->numNeigh[i] * sizeof(float) );
00333                 j++;
00334             }
00335         }
00336         
00337         for( i=0, j=0; i<globalMesh->numNodes; i++ ){
00338             if( globalMesh->processor[i] != MASTER_PROC ){
00339                 BlockingSend_LinkedList( globalMesh->nodeProviders[i], sizeof( int ), globalMesh->processor[i],
00340                         PROVIDERS_TAG, MPI_COMM_WORLD );
00341             }
00342             else{
00343                 LinkedListIterator *iter = LinkedListIterator_New( globalMesh->nodeProviders[i] );
00344                 void *provider = NULL;
00345                 
00346                 for( provider=LinkedListIterator_First(iter); provider != NULL; provider=LinkedListIterator_Next(iter) ){
00347                     LinkedList_InsertNode( localMesh->nodeProviders[j], provider, sizeof(int) );
00348                 }
00349                 j++;
00350             }
00351         }
00352         
00353         for( i=0; i<globalMesh->numProcs; i++ ){
00354             for( j=0; j<globalMesh->numProcs; j++ ){
00355                 if( i == MASTER_PROC ){
00356                     localMesh->haloNodes[j].numHaloNodes = meshDecomp->processorHaloNodes[i][j]->nodeCount;
00357                     localMesh->foreignHaloNodes[j].numForeignHaloNodes = meshDecomp->processorForeignHaloNodes[i][j]->nodeCount;
00358                 }
00359                 else{
00360                     MPI_Send( &(meshDecomp->processorHaloNodes[i][j]->nodeCount), 1, MPI_INT, i,
00361                             HALONODE_COUNT_TAG, MPI_COMM_WORLD );
00362                     MPI_Send( &(meshDecomp->processorForeignHaloNodes[i][j]->nodeCount), 1, MPI_INT, i,
00363                             FOREIGN_HALONODE_COUNT_TAG, MPI_COMM_WORLD );
00364                 }
00365             }
00366         }
00367         
00368         SurfaceMesh_AllocateMemoryForHaloNodes( localMesh );
00369         
00370         for( i=0; i<globalMesh->numProcs; i++ ){
00371             for( j=0; j<globalMesh->numProcs; j++ ){
00372                 iter = BTreeIterator_New( meshDecomp->processorHaloNodes[i][j] );
00373                 for( k=0, result=BTreeIterator_First(iter); result != NULL; k++, result=BTreeIterator_Next(iter) ){
00374                     assert( result );
00375                     if( i == MASTER_PROC ){
00376                         localMesh->haloNodes[j].haloId[k] = (*(int*)result);
00377                         localMesh->haloNodes[j].x[k] = globalMesh->x[*(int*)result - 1];
00378                         localMesh->haloNodes[j].y[k] = globalMesh->y[*(int*)result - 1];
00379                         localMesh->haloNodes[j].h[k] = globalMesh->h[*(int*)result - 1];
00380                         localMesh->haloNodes[j].receiver[k] = globalMesh->receiver[*(int*)result - 1];
00381                         localMesh->haloNodes[j].surface[k] = globalMesh->surface[*(int*)result - 1];
00382                     }
00383                     else{
00384                         haloPackage.id = (*(int*)result);
00385                         haloPackage.x = globalMesh->x[(*(int*)result)-1];
00386                         haloPackage.y = globalMesh->y[(*(int*)result)-1];
00387                         haloPackage.h = globalMesh->h[(*(int*)result)-1];
00388                         haloPackage.receiver = globalMesh->receiver[(*(int*)result)-1];
00389                         haloPackage.surface = globalMesh->surface[(*(int*)result)-1];
00390                 
00391                         MPI_Send( &haloPackage, sizeof(HaloPackage), MPI_BYTE, i,
00392                                 HALO_PACKAGE_TAG, MPI_COMM_WORLD );
00393                     }
00394                 }
00395                 Stg_Class_Delete( iter );
00396             }
00397         }
00398         
00399         for( i=0; i<globalMesh->numProcs; i++ ){
00400             for( j=0; j<globalMesh->numProcs; j++ ){
00401                 iter = BTreeIterator_New( meshDecomp->processorForeignHaloNodes[i][j] );
00402                 for( k=0, result=BTreeIterator_First(iter); result != NULL; k++, result=BTreeIterator_Next(iter) ){
00403                     assert( result );
00404                     if( i == MASTER_PROC ){
00405                         localMesh->foreignHaloNodes[j].foreignHaloId[k] = (*(int*)result);
00406                     }
00407                     else{
00408                         MPI_Send( (int*)result, 1, MPI_INT, i,
00409                                 FOREIGN_HALONODE_ID_TAG, MPI_COMM_WORLD );
00410                     }
00411                 }
00412                 Stg_Class_Delete( iter );
00413             }
00414         }
00415     }
00416     else{
00417         MPI_Recv( &(localMesh->myLoad), 1, MPI_INT, MASTER_PROC,
00418                 MY_LOAD_TAG, MPI_COMM_WORLD, &status );
00419         
00420         Stg_Component_Initialise( localMesh, (void*)&(localMesh->myLoad), True );
00421         
00422         for( i=0; i<localMesh->myLoad; i++ ){
00423             MPI_Recv( &package, sizeof(Package), MPI_BYTE, MASTER_PROC,
00424                     PACKAGE_TAG, MPI_COMM_WORLD, &status );
00425             
00426             localMesh->id[i] = package.id;
00427             localMesh->sortedId[i] = package.id;
00428             localMesh->x[i] = package.x;
00429             localMesh->y[i] = package.y;
00430             localMesh->h[i] = package.h;
00431             localMesh->hi[i] = package.hi;
00432             localMesh->h0[i] = package.h0;
00433             localMesh->slope[i] = package.slope;
00434             localMesh->length[i] = package.length;
00435             localMesh->surface[i] = package.surface;
00436             localMesh->receiver[i] = package.receiver;
00437             localMesh->highestNeighbour[i] = package.highestNeighbour;
00438             localMesh->size[i] = package.size;
00439             localMesh->nLake[i] = package.nLake;
00440             localMesh->nCatch[i] = package.nCatch;
00441         }
00442 
00443         MPI_Bcast ( localMesh->numNeigh, localMesh->numNodes, MPI_INT, MASTER_PROC, MPI_COMM_WORLD );
00444         
00445         for( i=0; i<localMesh->myLoad; i++ ){
00446             memset( localMesh->nodeNeighbours[i], 0, sizeof( int ) * localMesh->maxNeighboursPerNode );
00447             
00448             MPI_Recv( localMesh->nodeNeighbours[i], localMesh->numNeigh[localMesh->id[i]-1], MPI_INT, MASTER_PROC,
00449                     NEIGHBOUR_TAG, MPI_COMM_WORLD, &status );
00450         }
00451         
00452         for( i=0; i<localMesh->myLoad; i++ ){
00453             memset( localMesh->sides[i], 0, sizeof( float ) * localMesh->maxNeighboursPerNode );
00454             
00455             MPI_Recv( localMesh->sides[i], localMesh->numNeigh[localMesh->id[i]-1], MPI_FLOAT, MASTER_PROC,
00456                     SIDE_TAG, MPI_COMM_WORLD, &status );
00457         }
00458         
00459         for( i=0; i<localMesh->myLoad; i++ ){
00460             BlockingReceive_LinkedList( localMesh->nodeProviders[i], sizeof( int ), MASTER_PROC,
00461                                     PROVIDERS_TAG, MPI_COMM_WORLD, &status );
00462         }
00463 
00464         for( i=0; i<localMesh->numProcs; i++ ){
00465             MPI_Recv( &(localMesh->haloNodes[i].numHaloNodes), 1, MPI_INT, MASTER_PROC,
00466                     HALONODE_COUNT_TAG, MPI_COMM_WORLD, &status );
00467             MPI_Recv( &(localMesh->foreignHaloNodes[i].numForeignHaloNodes), 1, MPI_INT, MASTER_PROC,
00468                     FOREIGN_HALONODE_COUNT_TAG, MPI_COMM_WORLD, &status );
00469         }
00470         
00471         SurfaceMesh_AllocateMemoryForHaloNodes( localMesh );
00472 
00473         for( i=0; i<localMesh->numProcs; i++ ){
00474             for( j=0; j<localMesh->haloNodes[i].numHaloNodes; j++ ){
00475                 MPI_Recv( &haloPackage, sizeof(HaloPackage), MPI_BYTE, MASTER_PROC,
00476                         HALO_PACKAGE_TAG, MPI_COMM_WORLD, &status );
00477                 localMesh->haloNodes[i].haloId[j] = haloPackage.id;
00478                 localMesh->haloNodes[i].x[j] = haloPackage.x;
00479                 localMesh->haloNodes[i].y[j] = haloPackage.y;
00480                 localMesh->haloNodes[i].h[j] = haloPackage.h;
00481                 localMesh->haloNodes[i].receiver[j] = haloPackage.receiver;
00482                 localMesh->haloNodes[i].surface[j] = haloPackage.surface;
00483             }
00484         }
00485         
00486         for( i=0; i<localMesh->numProcs; i++ ){
00487             for( j=0; j<localMesh->foreignHaloNodes[i].numForeignHaloNodes; j++ ){
00488                 MPI_Recv( &(localMesh->foreignHaloNodes[i].foreignHaloId[j]), 1, MPI_INT, MASTER_PROC,
00489                         FOREIGN_HALONODE_ID_TAG, MPI_COMM_WORLD, &status );
00490             }
00491         }
00492     }
00493 
00494     for (i=0; i<localMesh->numNodes; i++){
00495         localMesh->mapGlobalToLocal[i] = localMesh->myLoad;
00496     }
00497     
00498     for (j=0; j<localMesh->myLoad; j++) {
00499         globalNum = localMesh->id[j];
00500         localMesh->mapGlobalToLocal[globalNum-1] = j;
00501     }
00502     
00503     MPI_Bcast ( localMesh->boundaryConditions, localMesh->numNodes, MPI_FLOAT, MASTER_PROC, MPI_COMM_WORLD );
00504 
00505     meshDecomp->incomingHaloPackage = malloc( sizeof( HaloPackage* ) * localMesh->numProcs );
00506     meshDecomp->outgoingHaloPackage = malloc( sizeof( HaloPackage* ) * localMesh->numProcs );
00507     for( i=0; i<localMesh->numProcs; i++ ){
00508         meshDecomp->incomingHaloPackage[i] = malloc( sizeof(HaloPackage) * localMesh->haloNodes[i].numHaloNodes );
00509         
00510         meshDecomp->outgoingHaloPackage[i] = malloc( sizeof(HaloPackage) * localMesh->foreignHaloNodes[i].numForeignHaloNodes );
00511     }
00512 }
00513 
00514 void _SurfaceMeshDecomp_Destroy( void *surfaceMeshDecomp, void *data )
00515 {
00516     
00517 }
00518 
00521 void _SurfaceMeshDecomp_SyncMesh( _SurfaceMeshDecomp *surfaceMeshDecomp, SurfaceMesh *mesh )
00522 {
00523     int i = 0, j = 0;
00524     int foreignHaloId = 0;
00525     int localIndex = 0;
00526     MPI_Status status;
00527 
00528     assert( surfaceMeshDecomp );
00529     assert( mesh );
00530 
00531     for( i=0; i<mesh->numProcs; i++ ){
00532         for( j=0; j<mesh->foreignHaloNodes[i].numForeignHaloNodes; j++ ){
00533             foreignHaloId = mesh->foreignHaloNodes[i].foreignHaloId[j];
00534 
00535             localIndex = mesh->mapGlobalToLocal[foreignHaloId-1];
00536             assert( localIndex != mesh->myLoad );
00537                 
00538             surfaceMeshDecomp->outgoingHaloPackage[i][j].id = foreignHaloId;
00539             surfaceMeshDecomp->outgoingHaloPackage[i][j].x = mesh->x[localIndex];
00540             surfaceMeshDecomp->outgoingHaloPackage[i][j].y = mesh->y[localIndex];
00541             surfaceMeshDecomp->outgoingHaloPackage[i][j].h = mesh->h[localIndex];
00542             surfaceMeshDecomp->outgoingHaloPackage[i][j].receiver = mesh->receiver[localIndex];
00543         }
00544     }
00545 
00546     for( i=0; i<mesh->numProcs; i++ ){
00547         if( mesh->rank == i ){
00548             continue;
00549         }
00550         MPI_Send( surfaceMeshDecomp->outgoingHaloPackage[i], 
00551                 sizeof(HaloPackage)*mesh->foreignHaloNodes[i].numForeignHaloNodes, MPI_BYTE, i,
00552                 HALO_PACKAGE_TAG, MPI_COMM_WORLD );
00553     }
00554     
00555     for( i=0; i<mesh->numProcs; i++ ){
00556         if( mesh->rank == i ){
00557             continue;
00558         }
00559         MPI_Recv( surfaceMeshDecomp->incomingHaloPackage[i],
00560                 sizeof(HaloPackage)*mesh->haloNodes[i].numHaloNodes, MPI_BYTE, i,
00561                 HALO_PACKAGE_TAG, MPI_COMM_WORLD, &status );
00562     }
00563     
00564     for( i=0; i<mesh->numProcs; i++ ){
00565         for( j=0; j<mesh->haloNodes[i].numHaloNodes; j++ ){
00566             
00567             assert( surfaceMeshDecomp->incomingHaloPackage[i][j].id == mesh->haloNodes[i].haloId[j] );
00568 
00569             mesh->haloNodes[i].x[j] = surfaceMeshDecomp->incomingHaloPackage[i][j].x;
00570             mesh->haloNodes[i].y[j] = surfaceMeshDecomp->incomingHaloPackage[i][j].y;
00571             mesh->haloNodes[i].h[j] = surfaceMeshDecomp->incomingHaloPackage[i][j].h;
00572             mesh->haloNodes[i].receiver[j] = surfaceMeshDecomp->incomingHaloPackage[i][j].receiver;
00573         }
00574     }
00575     /*printf( "done syncing mesh..\n" );*/
00576 }
00577 
00578 void _SurfaceMeshDecomp_AllocateNodes( _SurfaceMeshDecomp *meshDecomp )
00579 {
00580     
00581 }
00582 
00583 void _SurfaceMeshDecomp_ComputeHaloNodes( _SurfaceMeshDecomp *meshDecomp )
00584 {
00585     SurfaceMesh *mesh = NULL;
00586     int i = 0, j = 0;
00587     void *data = NULL;
00588     int nodeId, neighbour = 0;
00589     int myProcessor, neighboursProcessor;
00590     
00591     assert( meshDecomp ); 
00592 
00593     mesh = meshDecomp->mesh;
00594 
00595     for( i=0; i<mesh->numNodes; i++ ){
00596         for( j=0; j<mesh->numNeigh[i]; j++ ){
00597             neighbour = mesh->nodeNeighbours[i][j];
00598             
00599             myProcessor = mesh->processor[i];
00600             neighboursProcessor = mesh->processor[neighbour-1];
00601             
00602             if( myProcessor != neighboursProcessor ){
00603                 BTree_InsertNode( meshDecomp->processorHaloNodes[myProcessor][neighboursProcessor], (void*)&(mesh->id[neighbour-1]), sizeof(int) );
00604                 BTree_InsertNode( meshDecomp->processorForeignHaloNodes[neighboursProcessor][myProcessor], (void*)&(mesh->id[neighbour-1]), sizeof(int) );
00605             }
00606         }
00607     }
00608     
00609     for( i=0; i<mesh->numProcs; i++ ){
00610         printf( "For processor[%d]:\n", i );
00611         for( j=0; j<mesh->numProcs; j++ ){
00612             printf( "\tNumHaloNodes on processor[%d] = %d\n", j, meshDecomp->processorHaloNodes[i][j]->nodeCount );
00613             printf( "\tNumForeignHaloNodes on processor[%d] = %d\n", j, meshDecomp->processorForeignHaloNodes[i][j]->nodeCount );
00614         }
00615     }
00616 
00617     {
00618         /*test*/
00619         int k, found; 
00620         
00621         for( i=0; i<mesh->numNodes; i++ ){
00622             for( j=0; j<mesh->numNeigh[i]; j++ ){
00623                 nodeId = mesh->nodeNeighbours[i][j];
00624                 if( mesh->processor[i] != mesh->processor[nodeId-1] ){
00625                     found = 0;
00626                     for( k=0; k<mesh->numProcs; k++ ){
00627                         BTreeIterator *testIter = BTreeIterator_New( meshDecomp->processorHaloNodes[mesh->processor[i]][k] );
00628                         
00629                         for( data=BTreeIterator_First(testIter); data!=NULL; data=BTreeIterator_Next(testIter) ){
00630                             assert( data );
00631                             if( nodeId == *((int*)data) ){
00632                                 found = 1;
00633                                 break;
00634                             }
00635                         }
00636                         
00637                         Stg_Class_Delete( testIter );
00638                     }
00639                     assert(found);
00640                 }
00641                 else{
00642                 }
00643             }
00644         }
00645         printf( "HaloNodes test Passed..\n" );
00646     }
00647 }
00648