Commit 045a1ae4 authored by Shen Yu's avatar Shen Yu
Browse files

first program compiled passed for mpi with log

	modified:   openmpi/ompi/mpi/c/bcast.c
	modified:   openmpi/ompi/mpi/c/finalize.c
	modified:   openmpi/ompi/mpi/c/gather.c
	modified:   openmpi/ompi/mpi/c/gatherv.c
	modified:   openmpi/ompi/mpi/c/init.c
	modified:   openmpi/ompi/mpi/c/irecv.c
	modified:   openmpi/ompi/mpi/c/isend.c
	modified:   replayer/Makefile
	modified:   replayer/include/commset.h
	modified:   replayer/include/msglog.h
	deleted:    replayer/src/GlobalComm.c
	deleted:    replayer/src/commset.c
parent 1358ba8f
// This file is used to collect all local communicator ids from all processes
// and compare their member processes. If their member processes are same, they
// will be considered as the same communicator and assigned a global communicator
// id, and stored in the local -- global communicator id's map.
// After all local communicators are collected, the map is save to a text file
#ifndef __GLOBALCOMM_H__
#define __GLOBALCOMM_H__
#include <stdlib.h>
#include <math.h>
typedef struct COMM
{
int n; // number of members processes associated with each local communicator
int size; // size of members array
int* members; // member processes' global ranks of the communicator
} COMM;
inline void freeCOMMs(int n, COMM c[])
{
for(int i = 0; i < n; ++i)
{
c[i].n = 0;
c[i].size = 0;
free(c[i].members);
}
}
inline int compareCOMM(COMM* comm1, COMM* comm2)
{
int isSame = 1;
if(comm1->n != comm2->n)
{
isSame = 0;
}
else
{
for(int i = 0; i < comm1->n; ++i)
{
if(comm1->members[i] != comm2->members[i])
{
isSame = 0;
break;
}
}
}
return isSame;
}
inline void copyCOMMs(int n, COMM src[], COMM dst[])
{
for(int ic = 0; ic < n; ++ic)
{
dst[ic].n = src[ic].n;
dst[ic].size = src[ic].size;
dst[ic].members = malloc(dst[ic].size * sizeof(COMM));
for(int i = 0; i < src[ic].size; ++i)
dst[ic].members[i] = src[ic].members[i];
}
}
typedef struct ALLCOMM
{
int n; // number all global communicators
int size; // current comms array size
COMM *comms;
} ALLCOMM;
inline int initALLCOMM(ALLCOMM* ac)
{
int nprocs;
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
ac->n = 0;
ac->size = 4*ceil(sqrt(nprocs));
ac->comms = (COMM*) malloc(ac->size * sizeof(COMM));
return 0;
}
inline void freeALLCOMM(ALLCOMM* ac)
{
freeCOMMs(ac->n, ac->comms);
free(ac->comms);
ac->n = 0;
ac->size = 0;
}
inline int inALLCOMM(COMM *c, ALLCOMM *ac)
{
int in = 0;
for(int i = 0; i < ac->n; ++i)
{
if(compareCOMM(c, &ac->comms[i]) == 1)
{
in = i;
break;
}
}
return in;
}
inline int addALLCOMM(COMM *c, ALLCOMM *ac)
{
int r = inALLCOMM(c, ac);
if(r == 0)
{
if(ac->n == ac->size)
{
COMM *oldcomms = ac->comms;
ac->size *= 2;
ac->comms = (COMM*) malloc(ac->size * sizeof(COMM));
copyCOMMs(ac->n, oldcomms, ac->comms);
freeCOMMs(ac->n, ac->comms);
}
copyCOMMs(1, c, &(ac->comms[ac->n]));
++ ac->n;
r = ac->n;
}
return r;
}
inline int buildSendBuf(COMMSET *CS, MPI_Group group_world, int** p_sendbuf)
{
int N = CS->n; // number of local communicators
int total_n_member = 0; // total number of processes in all local communicators
for(int i = 0; i < N; ++i)
{
int i_size;
MPI_Comm_size(CS->comms[i], &i_size);
total_n_member += i_size;
}
// sendbuf is an array containing 4 parts:
// - length 1: total number of local communicator N
// - length N: local communicators IDs
// - length N: number of processes associated with each local communicator
// - length total_n_member: all processes global ranks of all local communicators
int sendcount = 1 + N + N + total_n_member;
int* sendbuf = malloc(sendcount * sizeof(int));
*p_sendbuf = sendbuf;
sendbuf[0] = N;
if(N > 0)
{
int* localCommID = &sendbuf[1];
int* size = &sendbuf[1+N];
int* ranks = &sendbuf[1+N+N];
for(int i = 0; i < N; ++i)
{
int i_pid = MPI_Comm_c2f(CS->comms[i]);
localCommID[i] = i_pid;
MPI_Comm_size(CS->comms[i], &size[i]);
}
// collect local comm members
int nprocs;
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
int *localRank = malloc(nprocs * sizeof(int));
for(int i = 0; i < nprocs; ++i)
localRank[i] = i;
int idx = 0; // index of first communicator's processes list position
for(int i = 0; i < N; ++i)
{
MPI_Group igroup;
MPI_Comm_group(CS->comms[i], &igroup);
MPI_Group_translate_ranks(igroup, size[i], &localRank[0], group_world, &ranks[idx]);
idx += size[i]; // move index to the next communicator's position
}
free(localRank);
}
return sendcount;
}
inline int collectAllCommPara(int sendcount, int sendbuf[], int** p_recvcounts, int** p_recvbuf)
{
int nprocs;
int myid;
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &myid);
// gather size of member list first
int* recvcounts;
if(myid == 0)
{
recvcounts = malloc(nprocs * sizeof(int));
}
MPI_Gather(&sendcount, 1, MPI_INT, &recvcounts[0], 1, MPI_INT, 0, MPI_COMM_WORLD);
// gather all member list
int* displs;
int* recvbuf;
int recvsize=0;
if(myid == 0)
{
displs = malloc(nprocs * sizeof(int));
displs[0]=0;
for(int i=0; i<nprocs; ++i)
{
recvsize += recvcounts[i];
if(i>0) displs[i] = displs[i-1] + recvcounts[i];
}
recvbuf = malloc(recvsize * sizeof(int));
}
MPI_Gatherv(&sendbuf[0], sendcount, MPI_INT,
&recvbuf[0], &recvcounts[0], &displs[0],
MPI_INT, 0, MPI_COMM_WORLD);
*p_recvcounts = recvcounts;
*p_recvbuf = recvbuf;
if(myid == 0) free(displs);
return recvsize;
}
inline int buildCommList(int buff[], ALLCOMM* ac, int* p_commlist_size, int** p_commlist)
{
const int N = buff[0]; // number of communicators
if(N > 0)
{
int* commlist = *p_commlist;
if(*p_commlist_size < 2 * N * sizeof(int))
{
int* old_commlist = commlist;
commlist = malloc(2 * N * sizeof(int));
for(int i = 0; i < *p_commlist_size; ++i)
commlist[i] = old_commlist[i];
p_commlist = &commlist;
free(old_commlist);
}
int* localCommID = &buff[1];
int* size = &buff[1 + N];
int* ranks = &buff[1 + N + N];
int idx = ac->n + 1;
for(int i = 0; i < N; ++i)
{
// make comm
COMM c;
c.n = N;
c.size = size[i];
c.members = ranks;
// find local comm id and global comm id in ac
commlist[2*i] = localCommID[i];
commlist[2*i + 1] = addALLCOMM(&c, ac);
}
}
return N;
}
inline int saveCommMap(COMMSET *CS, char FILENAME[])
{
// retrieve every comm from commset and find out
// the member processes global ids and put them
// in the sendbuf
int nprocs;
int myid;
MPI_Group MPI_GROUP_WORLD;
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &myid);
MPI_Comm_group(MPI_COMM_WORLD, &MPI_GROUP_WORLD);
int** p_sendbuf;
int sendcount = buildSendBuf(&localComm, MPI_GROUP_WORLD, p_sendbuf);
int* recvcounts;
int* recvbuf;
int recvsize = collectAllCommPara(sendcount, *p_sendbuf, &recvcounts, &recvbuf);
if(myid == 0)
{
FILE* COMMLISTFILE = fopen(FILENAME, "w");
int idx = 0;
ALLCOMM ac;
initALLCOMM(&ac);
for(int i = 0; i < nprocs; ++i)
{
fprintf(COMMLISTFILE, "%d", i);
int commlist_size;
int* commlist;
int n = buildCommList(&recvbuf[idx], &ac, &commlist_size, &commlist);
idx += recvcounts[i];
for(int i = 0; i < n; ++i)
{
fprintf(COMMLISTFILE, " %d %d", commlist[2*i], commlist[2*i + 1]);
}
fprintf(COMMLISTFILE, "\n");
}
fclose(COMMLISTFILE);
}
return 0;
}
#endif
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment