Commits (59)
......@@ -7,7 +7,7 @@ stages:
variables:
OS: "ubuntu"
OS_VERSION: "18.04"
OS_VERSION: "20.04"
REGISTRY: "ci.cluster.math.kit.edu"
IMAGE_NAME_MPP: "mpp-${CI_COMMIT_SHORT_SHA}-${OS}${OS_VERSION}-${CI_PIPELINE_ID}"
RUN_UNIT_TESTS: "all"
......
......@@ -11,7 +11,6 @@ set(basic_src
parallel/Buffer.cpp
parallel/ExchangeBuffer.cpp
parallel/Parallel.cpp
parallel/LIB_PS_Parallel.cpp
parallel/Communicator.cpp
)
......
......@@ -14,6 +14,13 @@ Communicator::Communicator(MPI_Comm comm, int commSplit, int color) :
MPI_Comm_rank(comm, &p);
}
Communicator::Communicator(const Communicator &comm, int color)
: P(0), p(0), commSplit(-1), color(color) {
MPI_Comm_split(comm.mpiComm, color, comm.p, &mpiComm);
MPI_Comm_size(mpiComm, &P);
MPI_Comm_rank(mpiComm, &p);
}
Communicator::~Communicator() {
if (mpiComm != MPI_COMM_WORLD)
MPI_Comm_free(&mpiComm);
......@@ -50,25 +57,15 @@ Communicator *Communicator::Split() const {
return new Communicator(newComm, commSplit + 1, newColor);
}
void Communicator::Broadcast(void *data, size_t size, int q) {
char *Data = (char *) data;
while (size > MaxBroadcastSize) {
MPI_Bcast(Data, MaxBroadcastSize, MPI_BYTE, q, mpiComm);
MPI_Barrier(mpiComm);
Data += MaxBroadcastSize;
size -= MaxBroadcastSize;
}
if (size > 0)
MPI_Bcast(Data, size, MPI_BYTE, q, mpiComm);
MPI_Barrier(mpiComm);
Communicator *Communicator::Copy() const {
MPI_Comm newComm;
MPI_Comm_dup(mpiComm, &newComm);
return new Communicator(newComm, commSplit, color);
}
#ifdef BUILD_IA
void Communicator::Sum(IAInterval *a, size_t n) {
void Communicator::Sum(IAInterval *a, size_t n) const {
double *in = new double[2 * n];
double *inout = new double[2 * n];
for (int i = 0; i < n; ++i) {
......@@ -82,7 +79,7 @@ void Communicator::Sum(IAInterval *a, size_t n) {
delete[] inout;
}
void Communicator::Sum(IACInterval *a, size_t n) {
void Communicator::Sum(IACInterval *a, size_t n) const {
double *in = new double[4 * n];
double *inout = new double[4 * n];
for (int i = 0; i < n; ++i) {
......@@ -99,7 +96,7 @@ void Communicator::Sum(IACInterval *a, size_t n) {
delete[] inout;
}
void Communicator::Hull(IAInterval *a, size_t n) {
void Communicator::Hull(IAInterval *a, size_t n) const {
double *in = new double[2 * n];
double *inout = new double[2 * n];
for (int i = 0; i < n; ++i) {
......@@ -113,7 +110,7 @@ void Communicator::Hull(IAInterval *a, size_t n) {
delete[] inout;
}
void Communicator::Intersect(IAInterval *a, size_t n) {
void Communicator::Intersect(IAInterval *a, size_t n) const {
double *in = new double[2 * n];
double *inout = new double[2 * n];
for (int i = 0; i < n; ++i) {
......
......@@ -19,8 +19,6 @@
constexpr size_t MaxBroadcastSize = 2147483647;
// 2^31 - 1
class Communicator {
protected:
// communicator size
......@@ -38,9 +36,10 @@ protected:
MPI_Comm mpiComm;
public:
Communicator(MPI_Comm comm, int commSplit, int color);
Communicator(const Communicator &comm, int color);
~Communicator();
bool Master() const;
......@@ -57,7 +56,92 @@ public:
Communicator *Split() const;
void Broadcast(void *data, size_t size, int q = 0);
Communicator *Copy() const;
/// Broadcast for known mpi data types
/// Note that size is only the size of the array (NOT multiplied with sizeof(T))!
template<typename T>
typename std::enable_if<has_mpi_datatype<T>::value, void>::type
Broadcast(T *data, size_t size, int q = 0) {
while (size > MaxBroadcastSize) {
MPI_Bcast(data, MaxBroadcastSize, mpi_datatype<T>::value(), q, mpiComm);
MPI_Barrier(mpiComm);
data += MaxBroadcastSize;
size -= MaxBroadcastSize;
}
if (size > 0) {
MPI_Bcast(data, size, mpi_datatype<T>::value(), q, mpiComm);
}
MPI_Barrier(mpiComm);
}
/// Broadcast for unknown mpi data types via char array
/// Note that size is only the size of the array (NOT multiplied with sizeof(T))!
template<typename T>
typename std::enable_if<!has_mpi_datatype<T>::value, void>::type
Broadcast(T *data, size_t size, int q = 0) {
char *Data = (char *) data;
size *= sizeof(T);
while (size > MaxBroadcastSize) {
MPI_Bcast(Data, MaxBroadcastSize, MPI_BYTE, q, mpiComm);
MPI_Barrier(mpiComm);
Data += MaxBroadcastSize;
size -= MaxBroadcastSize;
}
if (size > 0) {
MPI_Bcast(Data, size, MPI_BYTE, q, mpiComm);
}
MPI_Barrier(mpiComm);
}
template<typename T>
void Broadcast(T &t) {
Broadcast(&t, 1);
}
template<typename T>
void Broadcast(const T &t) {
Assert(Master());
T data = t;
Broadcast(&data, 1);
}
template<typename T>
typename std::enable_if<has_mpi_datatype<T>::value, void>::type
Send(T *data, size_t size, int destination) const {
const int tag = 27; // TODO: Why 27?
MPI_Request request;
MPI_Isend(data, size, mpi_datatype<T>::value(), destination, tag, mpiComm, &request);
MPI_Status status;
MPI_Wait(&request, &status);
}
template<typename T>
typename std::enable_if<has_mpi_datatype<T>::value, void>::type
Receive(T *data, size_t size, int source) const {
const int tag = 27; // TODO: Why 27?
MPI_Request request;
MPI_Irecv(data, size, mpi_datatype<T>::value(), source, tag, mpiComm, &request);
MPI_Status status;
MPI_Wait(&request, &status);
}
template<typename T>
typename std::enable_if<has_mpi_datatype<T>::value, void>::type
Allgather(T *sendData, size_t sizeSend, T *receiveData) const {
MPI_Allgather(sendData, sizeSend, mpi_datatype<T>::value(),
receiveData, sizeSend, mpi_datatype<T>::value(), mpiComm);
}
template<typename T>
typename std::enable_if<has_mpi_datatype<T>::value, void>::type
Allgatherv(T *sendData, size_t sizeSend, T *receiveData, int *sizeReceive, int *displs) const {
MPI_Allgatherv(sendData, sizeSend, mpi_datatype<T>::value(),
receiveData, sizeReceive, displs, mpi_datatype<T>::value(), mpiComm);
}
template<typename T>
typename std::enable_if<has_mpi_datatype<T>::value, void>::type
......@@ -84,13 +168,13 @@ public:
}
#ifdef BUILD_IA
void Sum(IAInterval *a, size_t n);
void Sum(IAInterval *a, size_t n) const;
void Sum(IACInterval *a, size_t n);
void Sum(IACInterval *a, size_t n) const;
void Hull(IAInterval *a, size_t n);
void Hull(IAInterval *a, size_t n) const;
void Intersect(IAInterval *a, size_t n);
void Intersect(IAInterval *a, size_t n) const;
#endif
template<typename T>
......@@ -116,17 +200,6 @@ public:
bool Or(bool b);
void Barrier();
template<typename T>
void Broadcast(T &t) {
Broadcast(&t, sizeof(t));
}
template<typename T>
void Broadcast(const T &t) {
T data = t;
Broadcast(&data, sizeof(t));
}
};
struct WorldCommunicator : public Communicator {
......
// $Header: https://svn.math.kit.edu/svn/M++/trunk/src/LIB_PS/LIB_PS_Parallel.C 228 2014-10-10 16:15:48Z maurer $
#include "LIB_PS_Parallel.hpp"
#include "mpi.h"
using Scalar = double;
using namespace std;
struct Comm {
MPI_Comm MPIC;
};
LIB_PS_ParComm::~LIB_PS_ParComm() {
if (comm->MPIC != MPI_COMM_WORLD) MPI_Comm_free(&comm->MPIC);
delete comm;
comm = 0;
}
LIB_PS_ParComm::LIB_PS_ParComm() : comm(0) {
comm = new Comm;
comm->MPIC = MPI_COMM_WORLD;
MPI_Group group;
MPI_Comm_group(comm->MPIC, &group);
MPI_Group_rank(group, &p);
MPI_Group_size(group, &N);
}
LIB_PS_ParComm::LIB_PS_ParComm(const LIB_PS_ParComm &C, int col) : comm(0) {
comm = new Comm;
MPI_Comm_split(C.comm->MPIC, col, C.p, &comm->MPIC);
MPI_Comm_size(comm->MPIC, &N);
MPI_Comm_rank(comm->MPIC, &p);
}
LIB_PS_ParComm::LIB_PS_ParComm(int col) : comm(0) {
comm = new Comm;
MPI_Comm comglob = MPI_COMM_WORLD;
int q;
MPI_Comm_rank(MPI_COMM_WORLD, &q);
MPI_Comm_split(comglob, col, q, &comm->MPIC);
MPI_Comm_size(comm->MPIC, &N);
MPI_Comm_rank(comm->MPIC, &p);
}
void LIB_PS_ParComm::Broadcast(void *data, size_t size, int master) {
char *Data = (char *) data;
size_t Size = size;
MPI_Bcast(Data, Size, MPI_BYTE, master, comm->MPIC);
MPI_Barrier(comm->MPIC);
}
void LIB_PS_ParComm::Allgather(void *senddata, size_t size_send, void *receivedata) {
char *Senddata = (char *) senddata;
char *Receivedata = (char *) receivedata;
MPI_Allgather(Senddata, size_send, MPI_CHAR, Receivedata, size_send, MPI_CHAR, comm->MPIC);
}
void LIB_PS_ParComm::Allgatherv(void *senddata, int size_send, void *receivedata, int *size_receive, int *displs,
LIB_PS_DATATYPE type) {
char *Senddata = (char *) senddata;
char *Receivedata = (char *) receivedata;
switch (type) {
case LIB_PS_INT:
MPI_Allgatherv(Senddata, size_send, MPI_INT,
Receivedata, size_receive, displs,
MPI_INT, comm->MPIC);
break;
case LIB_PS_SCALAR:
#ifdef LIBCOMPLEX
MPI_Allgatherv(Senddata, size_send, MPI_COMPLEX,
Receivedata, size_receive, displs,
MPI_COMPLEX,comm->MPIC);
#else
MPI_Allgatherv(Senddata, size_send, MPI_DOUBLE,
Receivedata, size_receive, displs,
MPI_DOUBLE, comm->MPIC);
#endif
break;
}
}
void LIB_PS_ParComm::Broadcast(void *data, size_t size, LIB_PS_DATATYPE type, int master) {
switch (type) {
case LIB_PS_INT:
Broadcast(data, size * sizeof(int), master);
break;
case LIB_PS_SCALAR:
Broadcast(data, size * sizeof(Scalar), master);
break;
}
}
void LIB_PS_ParComm::AllSum(double *a, size_t n) {
double *b = new double[n];
MPI_Allreduce(a, b, n, MPI_DOUBLE, MPI_SUM, comm->MPIC);
memcpy(a, b, sizeof(double) * n);
delete[] b;
}
#ifdef LIBCOMPLEX
void LIB_PS_ParComm::AllSum(complex<double>* z, size_t n) {
double* a = new double[2*n];
double* b = new double[2*n];
for (int i=0; i<n; ++i) {
a[i] = real(z[i]);
a[i+n] = imag(z[i]);
}
// MPI_Allreduce(a,b,n,MPI_DOUBLE,MPI_SUM,comm->MPIC); // ???
MPI_Allreduce(a,b,2*n,MPI_DOUBLE,MPI_SUM,comm->MPIC);
for (int i=0; i<n; ++i)
z[i] = complex<double>(b[i],b[i+n]);
delete[] a;
delete[] b;
}
#endif
void LIB_PS_ParComm::Send(void *data, size_t size, int dest) {
const int tag = 27;
char *Data = (char *) data;
MPI_Request *r = new MPI_Request[1];
MPI_Isend(Data, size, MPI_BYTE, dest, tag, comm->MPIC, r);
MPI_Status st;
MPI_Wait(r, &st);
delete[] r;
}
void LIB_PS_ParComm::Receive(void *data, size_t size, int source) {
const int tag = 27;
char *Data = (char *) data;
MPI_Request *r = new MPI_Request[1];
MPI_Irecv(Data, size, MPI_BYTE, source, tag, comm->MPIC, r);
MPI_Status st;
MPI_Wait(r, &st);
delete[] r;
}
void LIB_PS_ParComm::Send(void *data, size_t size, LIB_PS_DATATYPE type, int dest) {
switch (type) {
case LIB_PS_INT:
Send(data, size * sizeof(int), dest);
break;
case LIB_PS_SCALAR:
Send(data, size * sizeof(Scalar), dest);
break;
}
}
void LIB_PS_ParComm::Receive(void *data, size_t size, LIB_PS_DATATYPE type, int source) {
switch (type) {
case LIB_PS_INT:
Receive(data, size * sizeof(int), source);
break;
case LIB_PS_SCALAR:
Receive(data, size * sizeof(Scalar), source);
break;
}
}
// $Header: https://svn.math.kit.edu/svn/M++/trunk/src/LIB_PS/LIB_PS_Parallel.h 194 2014-07-15 19:50:58Z wieners $
#ifndef _LIB_PS_PARALLEL_H_
#define _LIB_PS_PARALLEL_H_
#include <algorithm>
#include <vector>
#include <iostream>
#include <cstring>
enum LIB_PS_DATATYPE {
LIB_PS_INT, LIB_PS_SCALAR
};
struct Comm;
class LIB_PS_ParComm {
int N;
int p;
public:
Comm *comm;
LIB_PS_ParComm();
LIB_PS_ParComm(int col);
LIB_PS_ParComm(const LIB_PS_ParComm &C, int col);
void Broadcast(void *, size_t, int);
void Broadcast(void *, size_t, LIB_PS_DATATYPE, int);
void Send(void *, size_t, int);
void Receive(void *, size_t, int);
void Send(void *, size_t, LIB_PS_DATATYPE, int);
void Receive(void *, size_t, LIB_PS_DATATYPE, int);
void Allgather(void *senddata, size_t size_send, void *receivedata);
void Allgatherv(void *senddata, int size_send, void *receivedata, int *size_receive, int *displs,
LIB_PS_DATATYPE = LIB_PS_SCALAR);
void AllSum(double *, size_t);
#ifdef LIBCOMPLEX
void AllSum(complex<double>*, size_t);
#endif
int size() const { return N; }
int proc() const { return p; }
int *proc_() {
return &p;
}
~LIB_PS_ParComm();
};
#endif
......@@ -15,158 +15,161 @@
ParallelProgrammingModel *ParallelProgrammingModel::ppm = nullptr;
ParallelProgrammingModel::ParallelProgrammingModel(int *argc, char ***argv) {
MPI_Init(argc, argv);
MPI_Init(argc, argv);
#ifdef BUILD_IA
MPI_Op_create((MPI_User_function *)mpi_sum_iainterval, 1, &MPI_SUM_IAINTERVAL);
MPI_Op_create((MPI_User_function *)mpi_sum_iacinterval, 1, &MPI_SUM_IACINTERVAL);
MPI_Op_create((MPI_User_function *)mpi_hull_iainterval, 1, &MPI_HULL_IAINTERVAL);
MPI_Op_create((MPI_User_function *)mpi_intersect_iainterval, 1, &MPI_INTERSECT_IAINTERVAL);
MPI_Op_create((MPI_User_function *) mpi_sum_iainterval, 1, &MPI_SUM_IAINTERVAL);
MPI_Op_create((MPI_User_function *) mpi_sum_iacinterval, 1, &MPI_SUM_IACINTERVAL);
MPI_Op_create((MPI_User_function *) mpi_hull_iainterval, 1, &MPI_HULL_IAINTERVAL);
MPI_Op_create((MPI_User_function *) mpi_intersect_iainterval, 1, &MPI_INTERSECT_IAINTERVAL);
#endif
comms.push_back(new WorldCommunicator());
maxCommSplit = ceil(log2(Size(0)));
comms.push_back(new WorldCommunicator());
maxCommSplit = ceil(log2(Size(0)));
}
ParallelProgrammingModel::~ParallelProgrammingModel() {
#ifdef BUILD_IA
MPI_Op_free(&MPI_SUM_IAINTERVAL);
MPI_Op_free(&MPI_SUM_IACINTERVAL);
MPI_Op_free(&MPI_HULL_IAINTERVAL);
MPI_Op_free(&MPI_INTERSECT_IAINTERVAL);
MPI_Op_free(&MPI_SUM_IAINTERVAL);
MPI_Op_free(&MPI_SUM_IACINTERVAL);
MPI_Op_free(&MPI_HULL_IAINTERVAL);
MPI_Op_free(&MPI_INTERSECT_IAINTERVAL);
#endif
ClearCommunicators(true);
MPI_Finalize();
ClearCommunicators();
delete comms[0];
comms.clear();
MPI_Finalize();
}
void ParallelProgrammingModel::Initialize(int argc, char **argv) {
if (IsInitialized()) Exit("PPM already initialized!")
ppm = new ParallelProgrammingModel(&argc, &argv);
if (IsInitialized()) Exit("PPM already initialized!")
ppm = new ParallelProgrammingModel(&argc, &argv);
}
bool ParallelProgrammingModel::IsInitialized() {
return ppm != nullptr;
return ppm != nullptr;
}
ParallelProgrammingModel *ParallelProgrammingModel::Instance() {
if (!ppm) Exit("PPM not initialized!");
return ppm;
if (!ppm) Exit("PPM not initialized!");
return ppm;
}
void ParallelProgrammingModel::Close() {
if (ppm) delete ppm;
ppm = nullptr;
if (ppm) delete ppm;
ppm = nullptr;
}
void ParallelProgrammingModel::Abort() {
MPI_Abort(MPI_COMM_NULL, 1);
MPI_Abort(MPI_COMM_NULL, 1);
}
Communicator *ParallelProgrammingModel::GetComm(int commSplit) const {
if (commSplit == -1)
return comms.back();
if (commSplit > MaxCommSplit()) {
return comms.back();
Exit("Communicator split " + std::to_string(commSplit) + " does not exist")
}
return comms.at(commSplit);
if (commSplit == -1)
return comms.back();
if (commSplit > MaxCommSplit()) {
//TODO: Specify behaviour
return comms.back();
Exit("Communicator split " + std::to_string(commSplit) + " does not exist")
}
return comms.at(commSplit);
}
void ParallelProgrammingModel::Broadcast(void *data, size_t size, int commSplit, int q) {
GetComm(commSplit)->Broadcast(data, size, q);
Communicator *ParallelProgrammingModel::Copy(int commSplit) const {
if (commSplit == -1)
return comms.back()->Copy();
if (commSplit > MaxCommSplit()) {
Exit("Communicator split" + std::to_string(commSplit) + " does not exist")
}
return comms.at(commSplit)->Copy();
}
int ParallelProgrammingModel::Proc(int commSplit) const {
return GetComm(commSplit)->Proc();
return GetComm(commSplit)->Proc();
}
int ParallelProgrammingModel::Size(int commSplit) const {
return GetComm(commSplit)->Size();
return GetComm(commSplit)->Size();
}
bool ParallelProgrammingModel::Master(int commSplit) const {
return GetComm(commSplit)->Master();
return GetComm(commSplit)->Master();
}
int ParallelProgrammingModel::Color(int commSplit) const {
return GetComm(commSplit)->Color();
return GetComm(commSplit)->Color();
}
void ParallelProgrammingModel::PrintInfo(int commSplit) const {
return GetComm(commSplit)->PrintInfo();
return GetComm(commSplit)->PrintInfo();
}
void ParallelProgrammingModel::Communicate(ExchangeBuffer &exBuffer, int CommSplit) {
GetComm(CommSplit)->Communicate(exBuffer);
GetComm(CommSplit)->Communicate(exBuffer);
}
bool ParallelProgrammingModel::And(bool b, int CommSplit) {
return GetComm(CommSplit)->And(b);
bool ParallelProgrammingModel::And(bool b, int CommSplit) const {
return GetComm(CommSplit)->And(b);
}
bool ParallelProgrammingModel::Or(bool b, int CommSplit) {
return GetComm(CommSplit)->Or(b);
bool ParallelProgrammingModel::Or(bool b, int CommSplit) const {
return GetComm(CommSplit)->Or(b);
}
bool ParallelProgrammingModel::SplitCommunicators() {
PPM->Barrier(0); // Todo which one?
PPM->Barrier(MaxCommSplit());
bool fullSplitReached = true;
auto comm = comms.back();
for (int p = 0; p < PPM->Size(0); p++) {
if (p == PPM->Proc(0) && comm->Size() != 1) {
fullSplitReached = false;
break;
}
PPM->Barrier(0); // Todo which one?
PPM->Barrier(MaxCommSplit());
bool fullSplitReached = true;
auto comm = comms.back();
for (int p = 0; p < PPM->Size(0); p++) {
if (p == PPM->Proc(0) && comm->Size() != 1) {
fullSplitReached = false;
break;
}
if (!fullSplitReached)
comms.push_back(comm->Split());
return fullSplitReached;
}
if (!fullSplitReached)
comms.push_back(comm->Split());
return fullSplitReached;
}
bool ParallelProgrammingModel::SplitCommunicators(int CommSplit) {
if (CommSplit == -1) {
FullSplit();
return true;
}
bool fullSplitReached = false;
for (int i = 0; i < CommSplit; i++)
fullSplitReached = SplitCommunicators();
return fullSplitReached;
if (CommSplit == -1) {
FullSplit();
return true;
}
bool fullSplitReached = false;
for (int i = 0; i < CommSplit; i++)
fullSplitReached = SplitCommunicators();
return fullSplitReached;
}
void ParallelProgrammingModel::FullSplit() {
bool fullSplitReached = false;
while (!fullSplitReached)
fullSplitReached = SplitCommunicators();
bool fullSplitReached = false;
while (!fullSplitReached)
fullSplitReached = SplitCommunicators();
}
int ParallelProgrammingModel::CommsSize() const {
return comms.size();
return comms.size();
}
int ParallelProgrammingModel::MaxCommSplit() const {
if (CommsSize() != 0)
return CommsSize() - 1;
else
return 0;
}
void ParallelProgrammingModel::ClearCommunicators(bool clearWorld) {
if (CommsSize() > 1) {
if (clearWorld) {
for (auto &comm: comms)
delete comm;
comms.clear();
} else {
comms.erase(comms.begin() + 1, comms.end());
}
}
if (CommsSize() != 0)
return CommsSize() - 1;
else
return 0;
}
void ParallelProgrammingModel::ClearCommunicators() {
while (comms.size() > 1) {
delete comms[comms.size() - 1];
comms.pop_back();
}
}
void ParallelProgrammingModel::Barrier(int CommSplit) {
GetComm(CommSplit)->Barrier();
GetComm(CommSplit)->Barrier();
}
#ifdef BUILD_IA
......@@ -181,13 +184,13 @@ void ParallelProgrammingModel::Intersect(IAInterval *a, size_t n, int commSplit)
IAInterval ParallelProgrammingModel::Hull(IAInterval a) {
Hull(&a, 1);
return a;
Hull(&a, 1);
return a;
}
IAInterval ParallelProgrammingModel::Intersect(IAInterval a) {
Intersect(&a, 1);
return a;
Intersect(&a, 1);
return a;
}
#endif
......@@ -199,92 +202,92 @@ IAInterval ParallelProgrammingModel::Intersect(IAInterval a) {
void process_mem_usage(double &vm_usage, double &resident_set) {
vm_usage = 0.0;
resident_set = 0.0;
// the two fields we want
unsigned long vsize;
long rss;
{
std::string ignore;
std::ifstream ifs("/proc/self/stat", std::ios_base::in);
ifs >> ignore >> ignore >> ignore >> ignore >> ignore >> ignore >> ignore
>> ignore >> ignore
>> ignore
>> ignore >> ignore >> ignore >> ignore >> ignore >> ignore >> ignore
>> ignore >> ignore
>> ignore
>> ignore >> ignore >> vsize >> rss;
}
long page_size_b =
sysconf(_SC_PAGE_SIZE); // in case x86-64 is configured to use 2MB pages
vm_usage = vsize / (1024 * 1024.0);
resident_set = rss * page_size_b / (1024 * 1024);
vm_usage = 0.0;
resident_set = 0.0;
// the two fields we want
unsigned long vsize;
long rss;
{
std::string ignore;
std::ifstream ifs("/proc/self/stat", std::ios_base::in);
ifs >> ignore >> ignore >> ignore >> ignore >> ignore >> ignore >> ignore
>> ignore >> ignore
>> ignore
>> ignore >> ignore >> ignore >> ignore >> ignore >> ignore >> ignore
>> ignore >> ignore
>> ignore
>> ignore >> ignore >> vsize >> rss;
}
long page_size_b =
sysconf(_SC_PAGE_SIZE); // in case x86-64 is configured to use 2MB pages
vm_usage = vsize / (1024 * 1024.0);
resident_set = rss * page_size_b / (1024 * 1024);
}
double ParallelProgrammingModel::MemoryUsage(bool show_process_usage) {
// returns the physical memory im MB for all processes
// (excluding shared libraries and swap)
double vm_usage, resident_set;
Synchronize();
process_mem_usage(vm_usage, resident_set);
if (show_process_usage) {
ExchangeBuffer E;
if (!PPM->master()) {
E.Send(0) << PPM->proc();
E.Send(0) << resident_set;
E.Communicate();
} else {
E.Communicate();
mout << " proc 0 mem used: " << resident_set << " MB" << endl;
for (short q = 0; q < PPM->size(); ++q) {
while (E.Receive(q).size() < E.ReceiveSize(q)) {
int p = -1;
double p_resident_set = -1;
E.Receive(q) >> p;
E.Receive(q) >> p_resident_set;
mout << " proc " << p << " mem used: " << p_resident_set << " MB"
<< endl;
}
}
// returns the physical memory im MB for all processes
// (excluding shared libraries and swap)
double vm_usage, resident_set;
Synchronize();
process_mem_usage(vm_usage, resident_set);
if (show_process_usage) {
ExchangeBuffer E;
if (!PPM->master()) {
E.Send(0) << PPM->proc();
E.Send(0) << resident_set;
E.Communicate();
} else {
E.Communicate();
mout << " proc 0 mem used: " << resident_set << " MB" << endl;
for (short q = 0; q < PPM->size(); ++q) {
while (E.Receive(q).size() < E.ReceiveSize(q)) {
int p = -1;
double p_resident_set = -1;
E.Receive(q) >> p;
E.Receive(q) >> p_resident_set;
mout << " proc " << p << " mem used: " << p_resident_set << " MB"
<< endl;
}
}
}
}
resident_set = Sum(resident_set);
return resident_set;
resident_set = Sum(resident_set);
return resident_set;
}
void ParallelProgrammingModel::Synchronize(const char *c) {
std::cout.flush();
Sum(1.0);
if (c == 0) return;
std::cout << GetComm(-1)->Proc() << "|";
Sum(1.0); // Wait
std::cout.flush();
if (master()) std::cout << c << endl;
std::cout.flush();
Sum(1.0);
std::cout.flush();
std::cout.flush();
Sum(1.0);
if (c == 0) return;
std::cout << GetComm(-1)->Proc() << "|";
Sum(1.0); // Wait
std::cout.flush();
if (master()) std::cout << c << endl;
std::cout.flush();
Sum(1.0);
std::cout.flush();
}
double ParallelProgrammingModel::TotalMemoryUsage(bool perPro) {
// returns the physical memory im MB for all processes
// (excluding shared libraries and swap)
double vm_usage, resident_set;
Synchronize();
process_mem_usage(vm_usage, resident_set);
// returns the physical memory im MB for all processes
// (excluding shared libraries and swap)
double vm_usage, resident_set;
Synchronize();
process_mem_usage(vm_usage, resident_set);
if (perPro)
pout << "Memory on proc " << proc() << " is: " << resident_set << " MB" << endl;
if (perPro)
pout << "Memory on proc " << proc() << " is: " << resident_set << " MB" << endl;
resident_set = Sum(resident_set);
return resident_set;
resident_set = Sum(resident_set);
return resident_set;
}
/*
......@@ -294,64 +297,64 @@ double ParallelProgrammingModel::TotalMemoryUsage(bool perPro) {
*/
bool ParallelProgrammingModel::Boolean(bool b) {
return GetComm(0)->Or(b);
return GetComm(0)->Or(b);
}
ParallelProgrammingModel *ParallelProgrammingModel::instance() {
if (!ppm) Exit("PPM not initialized!");
return ppm;
if (!ppm) Exit("PPM not initialized!");
return ppm;
}
bool ParallelProgrammingModel::isInitialized() {
return ppm != nullptr;
return ppm != nullptr;
}
void ParallelProgrammingModel::BroadcastDouble(double d) {
Assert(master());
Broadcast(d);
Assert(master());
Broadcast(d);
}
double ParallelProgrammingModel::BroadcastDouble() {
Assert(!master());
double a;
Broadcast(a);
return a;
Assert(!master());
double a;
Broadcast(a);
return a;
}
void ParallelProgrammingModel::BroadcastInt(int i) {
Assert(master());
Broadcast(i);
Assert(master());
Broadcast(i);
}
int ParallelProgrammingModel::BroadcastInt() {
Assert(!master());
int i;
Broadcast(i);
return i;
Assert(!master());
int i;
Broadcast(i);
return i;
}
bool ParallelProgrammingModel::master() const {
return GetComm(0)->Master();
return GetComm(0)->Master();
}
int ParallelProgrammingModel::proc() const {
return GetComm(0)->Proc();
return GetComm(0)->Proc();
}
int ParallelProgrammingModel::size() const {
return GetComm(0)->Size();
return GetComm(0)->Size();
}
void ParallelProgrammingModel::close() {
Close();
Close();
}
void ParallelProgrammingModel::abort() {
MPI_Abort(MPI_COMM_NULL, 1);
MPI_Abort(MPI_COMM_NULL, 1);
}
void ParallelProgrammingModel::initialize(int *argc, char **argv) {
Initialize(*argc, argv);
Initialize(*argc, argv);
}
......@@ -27,10 +27,6 @@ public:
static void Abort();
Communicator *GetComm(int commSplit = 0) const;
void Broadcast(void *data, size_t size, int commSplit = 0, int q = 0);
int Proc(int commSplit = 0) const;
int Size(int commSplit = 0) const;
......@@ -41,61 +37,62 @@ public:
void PrintInfo(int commSplit = 0) const;
template<typename T>
T Min(T a, int commSplitt = 0) const {
return GetComm(commSplitt)->Min(a);
}
Communicator *GetComm(int commSplit = 0) const;
Communicator *Copy(int commSplit = 0) const;
//TODO: Remove size issue such that the following holds (NOT true yet!!!):
/// Note that size is only the size of the array (NOT multiplied with sizeof(T))!
template<typename T>
T Max(T a, int commSplitt = 0) const {
return GetComm(commSplitt)->Max(a);
void Broadcast(T *data, size_t size, int commSplit = 0, int q = 0) {
size /= sizeof(T);
GetComm(commSplit)->Broadcast(data, size, q);
}
// Tested in Exchange Buffer
void Communicate(ExchangeBuffer &exBuffer, int commSplit = 0);
bool And(bool b, int commSplit = 0);
bool Or(bool b, int commSplit = 0);
bool SplitCommunicators();
bool SplitCommunicators(int commSplit);
void FullSplit();
int CommsSize() const;
int MaxCommSplit() const;
void ClearCommunicators(bool clearWorld);
void Barrier(int commSplit = 0);
template<typename T>
void Broadcast(T &t) {
GetComm(0)->Broadcast(&t, sizeof(t));
GetComm(0)->Broadcast(t);
}
template<typename T>
void BCastOnCommSplit(T &t, int commSplit = 0) {
GetComm(commSplit)->Broadcast(&t, sizeof(t));
GetComm(commSplit)->Broadcast(t);
}
template<typename T>
void Broadcast(const T &t) {
Assert(Master(0));
T data = t;
GetComm(0)->Broadcast(&data, sizeof(t));
GetComm(0)->Broadcast(t);
}
template<typename T>
void BCastOnCommSplit(const T &t, int commSplit = 0) {
Assert(Master(commSplit));
T data = t;
GetComm(commSplit)->Broadcast(&data, sizeof(t));
GetComm(commSplit)->Broadcast(t);
}
template<typename T>
void Send(T *data, size_t size, int destination, int commSplit = 0) const {
GetComm(commSplit)->Send(data, size, destination);
};
template<typename T>
void Receive(T *data, size_t size, int source, int commSplit = 0) const {
GetComm(commSplit)->Receive(data, size, source);
};
// Tested in Exchange Buffer
void Communicate(ExchangeBuffer &exBuffer, int commSplit = 0);
template<typename T>
void Allgather(T *sendData, size_t sizeSend, T *receiveData, int commSplit = 0) const {
GetComm(commSplit)->Allgather(sendData, sizeSend, receiveData);
};
template<typename T>
void Allgatherv(T *sendData, size_t sizeSend, T *receiveData, int *sizeReceive,
int *displs, int commSplit = 0) const {
GetComm(commSplit)->Allgatherv(sendData, sizeSend, receiveData, sizeReceive, displs);
};
template<typename T>
void Sum(T *a, size_t n, int commSplit = 0) const {
GetComm(commSplit)->Sum(a, n);
......@@ -124,7 +121,7 @@ public:
}
template<typename T>
void SumOnCommSplit(std::vector<T> &v, int commSplit = 0) {
void SumOnCommSplit(std::vector<T> &v, int commSplit = 0) const {
size_t n = std::size(v);
T *a = new T[n];
for (int i = 0; i < n; ++i) a[i] = v[i];
......@@ -133,6 +130,20 @@ public:
delete[] a;
}
template<typename T>
T Min(T a, int commSplitt = 0) const {
return GetComm(commSplitt)->Min(a);
}
template<typename T>
T Max(T a, int commSplitt = 0) const {
return GetComm(commSplitt)->Max(a);
}
bool And(bool b, int commSplit = 0) const;
bool Or(bool b, int commSplit = 0) const;
#ifdef BUILD_IA
void Hull(IAInterval *a, size_t n, int commSplit = 0);
......@@ -145,6 +156,21 @@ public:
#endif
bool SplitCommunicators();
bool SplitCommunicators(int commSplit);
void FullSplit();
int CommsSize() const;
int MaxCommSplit() const;
/// Deletes a communicators except world
void ClearCommunicators();
void Barrier(int commSplit = 0);
double MemoryUsage(bool show_process_usage = false);
void Synchronize(const char *c = 0);
......
......@@ -87,26 +87,6 @@ void verifiedEVreal(const IASymRMatrix &A, IAUpperEigenvalueBounds &Lambda) {
l.getUpperBounds(Lambda);
}
void verifiedEVreal(const IARMatrix &A, IAEigenvalueEnclosures &Lambda) {
Eigenvalues l;
Eigenvectors x;
EVreal(mid(A), l, x);
IARMatrix *B = nullptr;
EVrealEnclosures(A, B, l, x, Lambda);
}
void verifiedEVreal(const IARMatrix &A, IALowerEigenvalueBounds &lambda) {
IAEigenvalueEnclosures l;
verifiedEVreal(A, l);
l.getLowerBounds(lambda);
}
void verifiedEVreal(const IARMatrix &A, IAUpperEigenvalueBounds &Lambda) {
IAEigenvalueEnclosures l;
verifiedEVreal(A, l);
l.getUpperBounds(Lambda);
}
void verifiedEVreal(const IASymRMatrix &A, const IASymRMatrix &B,
IAEigenvalueEnclosures &Lambda) {
Eigenvalues l;
......@@ -129,28 +109,6 @@ void verifiedEVreal(const IASymRMatrix &A, const IASymRMatrix &B,
l.getUpperBounds(Lambda);
}
void verifiedEVreal(const IARMatrix &A, const IARMatrix &B,
IAEigenvalueEnclosures &Lambda) {
Eigenvalues l;
Eigenvectors x;
EVreal(mid(A), mid(B), l, x);
EVrealEnclosures(A, &B, l, x, Lambda);
}
void verifiedEVreal(const IARMatrix &A, const IARMatrix &B,
IALowerEigenvalueBounds &lambda) {
IAEigenvalueEnclosures l;
verifiedEVreal(A, B, l);
l.getLowerBounds(lambda);
}
void verifiedEVreal(const IARMatrix &A, const IARMatrix &B,
IAUpperEigenvalueBounds &Lambda) {
IAEigenvalueEnclosures l;
verifiedEVreal(A, B, l);
l.getUpperBounds(Lambda);
}
template<typename IAMATRIX, typename EVALUES, typename EVECTORS>
void EVcomplexEnclosures(const IAMATRIX &A,
const IAMATRIX *B,
......@@ -235,26 +193,6 @@ void verifiedEVcomplex(const IAHermCMatrix &A, IAUpperEigenvalueBounds &Lambda)
l.getUpperBounds(Lambda);
}
void verifiedEVcomplex(const IACMatrix &A, IAEigenvalueEnclosures &Lambda) {
Eigenvalues l;
CEigenvectors x;
EVcomplex(mid(A), l, x);
IACMatrix *B = nullptr;
EVcomplexEnclosures(A, B, l, x, Lambda);
}
void verifiedEVcomplex(const IACMatrix &A, IALowerEigenvalueBounds &lambda) {
IAEigenvalueEnclosures l;
verifiedEVcomplex(A, l);
l.getLowerBounds(lambda);
}
void verifiedEVcomplex(const IACMatrix &A, IAUpperEigenvalueBounds &Lambda) {
IAEigenvalueEnclosures l;
verifiedEVcomplex(A, l);
l.getUpperBounds(Lambda);
}
void verifiedEVcomplex(const IAHermCMatrix &A, const IAHermCMatrix &B,
IAEigenvalueEnclosures &Lambda) {
Eigenvalues l;
......@@ -276,25 +214,3 @@ void verifiedEVcomplex(const IAHermCMatrix &A, const IAHermCMatrix &B,
verifiedEVcomplex(A, B, l);
l.getUpperBounds(Lambda);
}
void verifiedEVcomplex(const IACMatrix &A, const IACMatrix &B,
IAEigenvalueEnclosures &Lambda) {
Eigenvalues l;
CEigenvectors x;
EVcomplex(mid(A), mid(B), l, x);
EVcomplexEnclosures(A, &B, l, x, Lambda);
}
void verifiedEVcomplex(const IACMatrix &A, const IACMatrix &B,
IALowerEigenvalueBounds &lambda) {
IAEigenvalueEnclosures l;
verifiedEVcomplex(A, B, l);
l.getLowerBounds(lambda);
}
void verifiedEVcomplex(const IACMatrix &A, const IACMatrix &B,
IAUpperEigenvalueBounds &Lambda) {
IAEigenvalueEnclosures l;
verifiedEVcomplex(A, B, l);
l.getUpperBounds(Lambda);
}
......@@ -24,24 +24,6 @@ void verifiedEVreal(const IASymRMatrix &A, IALowerEigenvalueBounds &lambda);
*/
void verifiedEVreal(const IASymRMatrix &A, IAUpperEigenvalueBounds &Lambda);
/**
* Encloses the eigenvalues of the matrix eigenvalue problem A * x = lambda * x,
* where A is a real symmetric interval matrix.
*/
void verifiedEVreal(const IARMatrix &A, IAEigenvalueEnclosures &Lambda);
/**
* Computes lower eigenvalue bounds of the matrix eigenvalue problem A * x = lambda * x,
* where A is a real symmetric interval matrix.
*/
void verifiedEVreal(const IARMatrix &A, IALowerEigenvalueBounds &lambda);
/**
* Computes upper eigenvalue bounds of the matrix eigenvalue problem A * x = lambda * x,
* where A is a real symmetric interval matrix.
*/
void verifiedEVreal(const IARMatrix &A, IAUpperEigenvalueBounds &Lambda);
/**
* Encloses the eigenvalues of the matrix eigenvalue problem A * x = lambda * B * x,
* where A is a real symmetric interval matrix and B is a real positive definite interval matrix.
......@@ -63,27 +45,6 @@ void verifiedEVreal(const IASymRMatrix &A, const IASymRMatrix &B,
void verifiedEVreal(const IASymRMatrix &A, const IASymRMatrix &B,
IAUpperEigenvalueBounds &Lambda);
/**
* Encloses the eigenvalues of the matrix eigenvalue problem A * x = lambda * B * x,
* where A is a real symmetric interval matrix and B is a real positive definite interval matrix.
*/
void verifiedEVreal(const IARMatrix &A, const IARMatrix &B,
IAEigenvalueEnclosures &Lambda);
/**
* Computes lower eigenvalue bounds of the matrix eigenvalue problem A * x = lambda * B * x,
* where A is a real symmetric interval matrix and B is a real positive definite interval matrix.
*/
void verifiedEVreal(const IARMatrix &A, const IARMatrix &B,
IALowerEigenvalueBounds &lambda);
/**
* Computes upper eigenvalue bounds of the matrix eigenvalue problem A * x = lambda * B * x,
* where A is a real symmetric interval matrix and B is a real positive definite interval matrix.
*/
void verifiedEVreal(const IARMatrix &A, const