Commit ff24a13d authored by Shen Yu's avatar Shen Yu
Browse files

remove some debug code

parent e6809943
// This file is used to collect all local communicator ids from all processes
// and compare their member processes. If their member processes are same, they
// will be considered as the same communicator and assigned a global communicator
// id, and stored in the local -- global communicator id's map.
// After all local communicators are collected, the map is save to a text file
#ifndef __GLOBALCOMM_H__
#define __GLOBALCOMM_H__
#include <stdlib.h>
#include <math.h>
typedef struct COMM
{
int n; // number of members processes associated with each local communicator
int size; // size of members array
int* members; // member processes' global ranks of the communicator
} COMM;
#ifdef _DEBUG
void printCOMMmembers(COMM* c)
{
for(int i = 0; i < c->n; ++i)
printf(" %d", c->members[i]);
printf("\n");
}
#endif
void freeCOMMs(int n, COMM c[])
{
for(int i = 0; i < n; ++i)
{
c[i].n = 0;
c[i].size = 0;
free(c[i].members);
}
}
int compareCOMM(COMM* comm1, COMM* comm2)
{
int isSame = 1;
if(comm1->n != comm2->n)
{
isSame = 0;
}
else
{
for(int i = 0; i < comm1->n; ++i)
{
if(comm1->members[i] != comm2->members[i])
{
isSame = 0;
break;
}
}
}
return isSame;
}
void copyCOMMs(int nc, COMM src[], COMM dst[])
{
for(int ic = 0; ic < nc; ++ic)
{
dst[ic].n = src[ic].n;
dst[ic].size = src[ic].size;
dst[ic].members = (int*) malloc(dst[ic].size * sizeof(COMM));
for(int i = 0; i < src[ic].size; ++i)
dst[ic].members[i] = src[ic].members[i];
}
}
typedef struct ALLCOMM
{
int n; // number all global communicators
int size; // current comms array size
COMM *comms;
} ALLCOMM;
int initALLCOMM(int NPROC, ALLCOMM* ac)
{
ac->n = 0;
ac->size = 4*ceil(sqrt(NPROC));
ac->comms = (COMM*) malloc(ac->size * sizeof(COMM));
return 0;
}
void freeALLCOMM(ALLCOMM* ac)
{
freeCOMMs(ac->n, ac->comms);
free(ac->comms);
ac->n = 0;
ac->size = 0;
}
int inALLCOMM(COMM *c, ALLCOMM *ac)
{
int in = -1;
for(int i = 0; i < ac->n; ++i)
{
if(compareCOMM(c, &ac->comms[i]) == 1)
{
in = i;
break;
}
}
return in;
}
int addALLCOMM(COMM *c, ALLCOMM *ac)
{
int r = inALLCOMM(c, ac);
if(r < 0)
{
if(ac->n == ac->size)
{
COMM *oldcomms = ac->comms;
ac->size *= 2;
ac->comms = (COMM*) malloc(ac->size * sizeof(COMM));
copyCOMMs(ac->n, oldcomms, ac->comms);
freeCOMMs(ac->n, ac->comms);
}
copyCOMMs(1, c, &(ac->comms[ac->n]));
r = ac->n;
++ ac->n;
}
return r;
}
int buildSendBuf(COMMSET *CS, MPI_Group group_world, int** p_sendbuf)
{
int N = CS->n; // number of local communicators
#ifdef _DEBUG
int myid;
MPI_Comm_rank(MPI_COMM_WORLD, &myid);
printf("myid = %d : start buildSendBuf, CS->n = %d\n", myid, CS->n);
#endif
int total_n_member = 0; // total number of processes in all local communicators
const N = CS->n;
for(int i = 0; i < N; ++i)
{
total_n_member += CS->commsize[i];
}
#ifdef _DEBUG
printf("myid = %d : total_n_member = %d\n", myid, total_n_member);
#endif
// sendbuf is an array containing 4 parts:
// - length 1: total number of local communicator N
// - length N: local communicators IDs
// - length N: number of processes associated with each local communicator
// - length total_n_member: all processes global ranks of all local communicators
int sendcount = 1 + N + N + total_n_member;
int* sendbuf = (int*) malloc(sendcount * sizeof(int));
sendbuf[0] = N;
if(N > 0)
{
int* localCommID = &sendbuf[1];
int* size = &sendbuf[1+N];
int* ranks = &sendbuf[1+N+N];
int idx = 0;
for(int i = 0; i < N; ++i)
{
localCommID[i] = CS->commid[i];
size[i] = CS->commsize[i];
for(int j = 0; j < size[i]; ++j)
ranks[idx + j] = CS->commranks[i][j];
idx += size[i];
}
}
*p_sendbuf = sendbuf;
return sendcount;
}
int collectAllCommPara(int sendcount, int sendbuf[], int** p_recvcounts, int** p_recvbuf)
{
int nprocs;
int myid;
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &myid);
// gather size of member list first
int* recvcounts;
if(myid == 0)
{
recvcounts = (int*) malloc(nprocs * sizeof(int));
}
MPI_Gather(&sendcount, 1, MPI_INT, &recvcounts[0], 1, MPI_INT, 0, MPI_COMM_WORLD);
// gather all member list
int* displs;
int* recvbuf;
int recvsize=0;
if(myid == 0)
{
displs = (int*) malloc(nprocs * sizeof(int));
displs[0]=0;
for(int i=0; i<nprocs; ++i)
{
recvsize += recvcounts[i];
if(i>0) displs[i] = displs[i-1] + recvcounts[i];
}
recvbuf = (int*) malloc(recvsize * sizeof(int));
}
MPI_Gatherv(&sendbuf[0], sendcount, MPI_INT,
&recvbuf[0], &recvcounts[0], &displs[0],
MPI_INT, 0, MPI_COMM_WORLD);
*p_recvcounts = recvcounts;
*p_recvbuf = recvbuf;
if(myid == 0) free(displs);
return recvsize;
}
int buildCommList(int buff[], ALLCOMM* p_ac, int* p_commlist_size, int** p_commlist)
{
int N = buff[0]; // number of communicators
#ifdef _DEBUG
printf("start buildCommList, number of communicators is %d\n", N);
#endif
if(N > 0)
{
int* commlist = (int*) realloc(*p_commlist, 2 * N * sizeof(int));
int* localCommID = &buff[1];
int* size = &buff[1 + N];
int* ranks = &buff[1 + N + N];
//int idx = ac->n + 1;
int r_idx = 0;
COMM c_tmp;
for(int i = 0; i < N; ++i)
{
// make comm
c_tmp.n = size[i];
c_tmp.size = size[i];
c_tmp.members = &ranks[r_idx];
r_idx += c_tmp.size;
// find local comm id and global comm id in ac
commlist[2*i] = localCommID[i];
commlist[2*i + 1] = addALLCOMM(&c_tmp, p_ac);
#ifdef _DEBUG
printf("local comm (id = %d) has global id %d\n", localCommID[i], commlist[2*i + 1]);
printf("ALLCOMM.size %d, ALLCOMM.n %d\n", p_ac->size, p_ac->n);
#endif
}
*p_commlist_size = 2 * N;
*p_commlist = commlist;
}
return N;
}
int saveCommMap(COMMSET *CS, char FILENAME[])
{
// retrieve every comm from commset and find out
// the member processes global ids and put them
// in the sendbuf
int nprocs;
int myid;
MPI_Group MPI_GROUP_WORLD;
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &myid);
MPI_Comm_group(MPI_COMM_WORLD, &MPI_GROUP_WORLD);
#ifdef _DEBUG
printf("myid %d : begin save commmap\n", myid);
#endif
int* sendbuf;
int sendcount = buildSendBuf(CS, MPI_GROUP_WORLD, &sendbuf);
#ifdef _DEBUG
printf("myid %d : sendcount = %d, sendbuf:", myid, sendcount);
for(int i = 0; i < sendcount; ++i)
{
printf(" %d", sendbuf[i]);
}
printf("\n");
#endif
int* recvcounts;
int* recvbuf;
int recvsize = collectAllCommPara(sendcount, sendbuf, &recvcounts, &recvbuf);
if(myid == 0)
{
#ifdef _DEBUG
printf("myid %d : recvsize = %d, recvbuf:", myid, recvsize);
for(int i = 0; i < recvsize; ++i)
{
printf(" %d", recvbuf[i]);
}
printf("\n");
#endif
FILE* COMMLISTFILE = fopen(FILENAME, "w");
int idx = 0;
ALLCOMM ac;
initALLCOMM(&ac);
int commlist_size = 0;
int* commlist = NULL;
for(int i = 0; i < nprocs; ++i)
{
#ifdef _DEBUG
printf("start buildCommList from recvbuf[%d]\n", idx);
printf("initial ac.n = %d, ac.size = %d\n", ac.n, ac.size);
printf("initial commlist_size = %d\n", commlist_size);
#endif
int ncomm = buildCommList(&recvbuf[idx], &ac, &commlist_size, &commlist);
idx += recvcounts[i];
#ifdef _DEBUG
printf("proc %d has %d comms:\n", i, ncomm);
for(int ic = 0; ic < ncomm*2; ++ic)
{
printf("%d/%d: %d\n", ic, ncomm*2, commlist[ic]);
}
#endif
fprintf(COMMLISTFILE, "%d %d", i, ncomm);
for(int ic = 0; ic < ncomm; ++ic)
{
fprintf(COMMLISTFILE, " %d %d", commlist[2*ic], commlist[2*ic + 1]);
}
fprintf(COMMLISTFILE, "\n");
#ifdef _DEBUG
printf("proc %d has written to file\n", i);
#endif
}
fclose(COMMLISTFILE);
}
return 0;
}
#endif
#ifndef __COMMSET_INC__
#define __COMMSET_INC__
#include <stdlib.h>
#include "commset.h"
void initCOMMSET(int size, COMMSET *CS)
{
CS->n = 0;
CS->maxsize = 16;
CS->commid = (int*)malloc(sizeof(int) * CS->maxsize);
CS->commsize = (int*)malloc(sizeof(int) * CS->maxsize);
CS->commranks = (int**)malloc(sizeof(int*) * CS->maxsize);
}
int inCOMMSET(int commid, int size, int ranks[], COMMSET *CS)
{
int same_id = -1; // init value -1 means not in current commset
for(int i = 0; i < CS->n; ++i)
{
if(CS->commid[i] == commid && CS->commsize[i] == size)
{
same_id = i; // same id and size, may be same comm, check members
if(same_id == 0 || same_id == 2) // when comm's id = 0 or 2, it's not neccessary
// to check its members, because `id = 0` means
// its MPI_COMM_WORLD and `id = 2` means its
// MPI_COMM_NULL
{
break;
}
// check comm's members to make sure they are exactly same comm.
for(int rank_id = 0; rank_id < size; ++rank_id)
{
if(CS->commranks[i][rank_id] != ranks[rank_id])
{
same_id = -1; // find different member, not same comm
break; // check next one
}
}
if(same_id > 0) // find the same comm's id
{
break; // jump over other comms
}
}
}
return same_id;
}
int addComm(int commid, int size, int ranks[], COMMSET *CS)
{
int idx = inCOMMSET(commid, size, ranks, CS);
if( idx > 0)
return idx;
if(CS->n >= CS->maxsize)
{
CS->maxsize *= 2;
CS->commid = (int*)realloc(CS->commsize, sizeof(int) * CS->maxsize);
CS->commsize = (int*)realloc(CS->commsize, sizeof(int) * CS->maxsize);
CS->commranks = (int**)realloc(CS->commsize, sizeof(int*) * CS->maxsize);
if(CS->commid == NULL || CS->commsize == NULL || CS->commranks == NULL)
{
return -1;
}
}
idx = CS->n;
CS->commid[idx] = commid;
CS->commsize[idx] = size;
CS->commranks[idx] = (int*)malloc(sizeof(int) * size);
for(int i = 0; i < size; ++i)
{
CS->commranks[idx][i] = ranks[i];
}
++(CS->n);
return idx;
}
void getCOMMPROP(MPI_Group GROUP_WORLD, int localranks[], MPI_Comm c,
int* p_commid, int* p_size, int ranks[])
{
*p_commid = MPI_Comm_c2f(c);
MPI_Comm_size(c, p_size);
switch(*p_commid)
{
case 0: // MPI_COMM_WORLD is 0 in openmpi
{
ranks[0] = -1; // MPI_COMM_WORLD has all ranks
break;
}
case 1: // MPI_COMM_SELF is 1 in openmpi
{
MPI_Comm_rank(MPI_COMM_WORLD, &ranks[0]);
break;
}
case 2: // MPI_COMM_NULL is 2 in openmpi
{
ranks[0] = -1;
break;
}
default:
{
MPI_Group g;
MPI_Comm_group(c, &g);
MPI_Group_translate_ranks(g, *p_size, localranks, GROUP_WORLD, ranks);
}
}
}
void freeCOMMSET(COMMSET *CS)
{
if(CS->commid) free(CS->commid);
if(CS->commsize) free(CS->commsize);
if(CS->commranks)
{
for(int i = 0; i < CS->maxsize; ++i)
{
free(CS->commranks[i]);
}
free(CS->commranks);
}
CS->n = 0;
CS->maxsize = 0;
}
#endif
#ifndef __COMMSET_H__
#define __COMMSET_H__
typedef struct COMMSET
{
int n; // number of comms in the set
int maxsize; // set maxsize
int* commid; // the array containing the comms
int* commsize;
int** commranks;
} COMMSET;
int initCOMMSET(int size, COMMSET *CS);
int inCOMMSET(int commid, int size, int ranks[], COMMSET *CS);
int addComm(int commid, int size, int ranks[], COMMSET *CS);
void freeCOMMSET(COMMSET *CS);
#endif
\ No newline at end of file
......@@ -30,10 +30,6 @@
#include "GlobalComm.h"
#include "GlobalComm.hpp"
#ifdef _DEBUG
#include <unistd.h>
#endif
// end
#if OMPI_BUILD_MPI_PROFILING
......@@ -70,49 +66,14 @@ int MPI_Finalize(void)
const int FUNC_ID=255;
LOGMSG0(FUNC_ID, call_time, MPI_COMM_WORLD);
fclose(MSGLOGFILE);
#ifdef _DEBUG
printf("MYID %d: MPI_Finalize is loged\n", LOGMSGMYID);
sleep(1);
MPI_Barrier(MPI_COMM_WORLD);
#endif
LOGTHIS = 0;
extern char COMMFILE[]; // COMMFILE is initialized in init.c
saveCommMap(&localComm, COMMFILE);
#ifdef _DEBUG
printf("MYID %d: saveCommMap done\n", LOGMSGMYID);
sleep(1);
MPI_Barrier(MPI_COMM_WORLD);
#endif
freeCOMMSET(&localComm);
#ifdef _DEBUG
printf("MYID %d: freeCOMMSET done\n", LOGMSGMYID);
sleep(1);
MPI_Barrier(MPI_COMM_WORLD);
#endif
free_INT_POOL(&SIZE_LIST);
#ifdef _DEBUG
printf("MYID %d: SIZE_LIST is freeed\n", LOGMSGMYID);
sleep(1);
MPI_Barrier(MPI_COMM_WORLD);
#endif
free_INT_POOL(&REQ_LIST);
#ifdef _DEBUG
printf("MYID %d: REQ_LIST is freeed\n", LOGMSGMYID);
sleep(1);
MPI_Barrier(MPI_COMM_WORLD);
#endif
free_INT_POOL(&WORLD_RANKS);
#ifdef _DEBUG
printf("MYID %d: WORLD_RANKS is freeed\n", LOGMSGMYID);
sleep(1);
MPI_Barrier(MPI_COMM_WORLD);
#endif
free_INT_POOL(&GLOBAL_RANKS);
#ifdef _DEBUG
printf("MYID %d: GLOBAL_RANKS is freeed\n", LOGMSGMYID);
sleep(1);
MPI_Barrier(MPI_COMM_WORLD);
#endif
// end
return ompi_mpi_finalize();
......
struct timeval START_TIME;
FILE* MSGLOGFILE;
int LOGMSGMYID;
int LOGTHIS;
INT_POOL SIZE_LIST;
INT_POOL REQ_LIST;
INT_POOL LOCAL_RANKS;
COMMSET localComm;
char COMMFILE[4096];
int init_INT_POOL(int size, INT_POOL* p)
{
p->pool = (int*)malloc(size * sizeof(int));
if(p->pool)
{
p->size = size;
return 0;
}
else
{
return 1;
}
}
int resize_INT_POOL(int new_size, INT_POOL* p)
{
if(new_size > p->size)
{
p->pool = (int*)realloc(p->pool, new_size * sizeof(int));
if(p->pool)
{
p->size = new_size;
return 0;
}
else
{
return 1;
}
}
else
{
return 0;
}
}
void free_INT_POOL(INT_POOL* p)
{
free(p->pool);
p->size = 0;
}
#ifndef __MEMPOOL_H__
#define __MEMPOOL_H__
typedef struct INT_POOL
{
int size;
int* pool;
} INT_POOL;
int init_INT_POOL(int size, INT_POOL* p);
int resize_INT_POOL(int new_size, INT_POOL* p);
void free_INT_POOL(INT_POOL* p);
#endif
\ No newline at end of file
// for log messages
#ifndef __MSGLOG_H__
#define __MSGLOG_H__
#include <stdio.h>
#include <sys/time.h>
#include "commset.h"
#include "mempool.h"
extern COMMSET localComm;
extern FILE* MSGLOGFILE;
extern struct timeval START_TIME;
extern int LOGMSGMYID;
extern int LOGTHIS;
extern INT_POOL SIZE_LIST;
extern INT_POOL REQ_LIST;
//inline void LOGMSG(const char FUNCNAME[], const struct timeval call_time, const MPI_Comm COMM,
// const int DSTPROC, const int SENDSIZE,
// const int SRCPROC, const int RECVSIZE);
//inline int LOGMSG_SIZE(const MPI_Datatype t, const int c);
//inline int LOGMSG_SIZE_V(const int NPROC, const MPI_Datatype t, const int c[]);
//inline int LOGMSG_SIZE_W(const int NPROC, const MPI_Datatype t[], const int c[]);