Unverified Commit cd3967eb authored by greole's avatar greole Committed by GitHub
Browse files

Clean ldu ldu base (#13)

* create separate IOExecutorHandler class

* add IOExecutorHandler files

* fix compilation issue

* create separate IOSortingIdxHandler class

* Implement separate IOGKOMatrixHandler class

* move host matrix handling to separate class

* fix matrix sorting/updating host matrix
parent 5ae35b80
......@@ -92,6 +92,11 @@ add_compile_definitions(
target_sources(OGL
PRIVATE
common/common.C
IOHandler/IOPtr/IOPtr.C
IOHandler/IOSortingIdxHandler/IOSortingIdxHandler.C
IOHandler/IOExecutorHandler/IOExecutorHandler.C
IOHandler/IOGKOMatrixHandler/IOGKOMatrixHandler.C
lduMatrix/GKOlduBase/GKOlduBase.C
lduMatrix/GKOCG/GKOCG.C
lduMatrix/GKOIR/GKOIR.C
......@@ -99,6 +104,11 @@ lduMatrix/GKOBiCGStab/GKOBiCGStab.C
LduMatrix/GKOLduBase/GKOLduBase.C
LduMatrix/GKOACG/GKOACG.C
PUBLIC
common/common.H
IOHandler/IOPtr/IOPtr.H
IOHandler/IOSortingIdxHandler/IOSortingIdxHandler.H
IOHandler/IOExecutorHandler/IOExecutorHandler.H
IOHandler/IOGKOMatrixHandler/IOGKOMatrixHandler.H
lduMatrix/GKOlduBase/GKOlduBase.H
lduMatrix/GKOCG/GKOCG.H
lduMatrix/GKOIR/GKOIR.H
......@@ -110,13 +120,17 @@ LduMatrix/GKOACG/GKOACG.H
target_include_directories(OGL
PUBLIC
$ENV{FOAM_SRC}/OpenFOAM/lnInclude
$ENV{FOAM_SRC}/meshTools/lnInclude
$ENV{FOAM_SRC}/finiteVolume/lnInclude
$ENV{FOAM_SRC}/meshTools/lnInclude
$ENV{FOAM_SRC}/OpenFOAM/lnInclude
$ENV{FOAM_SRC}/OSspecific/POSIX/lnInclude
common/
lduMatrix/GKOlduBase
LduMatrix/GKOLduBase
common/
IOHandler/IOPtr
IOHandler/IOExecutorHandler/
IOHandler/IOSortingIdxHandler/
IOHandler/IOGKOMatrixHandler/
lduMatrix/GKOlduBase
LduMatrix/GKOLduBase
)
......
#include <ginkgo/ginkgo.hpp>
#include <type_traits>
#include "IOExecutorHandler.H"
namespace Foam {
IOExecutorHandler::IOExecutorHandler(const objectRegistry &db,
const dictionary &controlDict)
: device_executor_name_(
controlDict.lookupOrDefault("executor", word("reference"))),
app_executor_name_(
controlDict.lookupOrDefault("app_executor", word("reference")))
{
// create executors
bool app_exec_stored = db.foundObject<regIOobject>(app_executor_name_);
if (app_exec_stored) {
ref_exec_ptr_ =
&db.lookupObjectRef<GKOReferenceExecPtr>(app_executor_name_);
if (device_executor_name_ == app_executor_name_) {
return;
}
}
bool device_exec_stored =
db.foundObject<regIOobject>(device_executor_name_);
if (device_exec_stored) {
if (device_executor_name_ == "omp") {
omp_exec_ptr_ =
&db.lookupObjectRef<GKOOmpExecPtr>(device_executor_name_);
return;
}
if (device_executor_name_ == "cuda") {
cuda_exec_ptr_ =
&db.lookupObjectRef<GKOCudaExecPtr>(device_executor_name_);
return;
}
if (device_executor_name_ == "hip") {
hip_exec_ptr_ =
&db.lookupObjectRef<GKOHipExecPtr>(device_executor_name_);
return;
}
}
const fileName app_exec_store = app_executor_name_;
ref_exec_ptr_ =
new GKOReferenceExecPtr(IOobject(app_exec_store, db),
gko::give(gko::ReferenceExecutor::create()));
const fileName device_exec_store = device_executor_name_;
const fileName omp_exec_store = "omp";
omp_exec_ptr_ = new GKOOmpExecPtr(IOobject(omp_exec_store, db),
gko::OmpExecutor::create());
if (device_executor_name_ == "cuda") {
cuda_exec_ptr_ =
new GKOCudaExecPtr(IOobject(device_exec_store, db),
gko::give(gko::CudaExecutor::create(
0, omp_exec_ptr_->get_ptr(), true)));
}
if (device_executor_name_ == "hip") {
hip_exec_ptr_ =
new GKOHipExecPtr(IOobject(device_exec_store, db),
gko::give(gko::HipExecutor::create(
0, omp_exec_ptr_->get_ptr(), true)));
}
if (device_executor_name_ == "omp") {
omp_exec_ptr_ = new GKOOmpExecPtr(IOobject(device_exec_store, db),
omp_exec_ptr_->get_ptr());
}
}
defineTemplateTypeNameWithName(GKOExecPtr, "ExecIOPtr");
defineTemplateTypeNameWithName(GKOCudaExecPtr, "CudaExecIOPtr");
defineTemplateTypeNameWithName(GKOOmpExecPtr, "OmpExecIOPtr");
defineTemplateTypeNameWithName(GKOHipExecPtr, "HipExecIOPtr");
defineTemplateTypeNameWithName(GKOReferenceExecPtr, "ReferenceExecIOPtr");
} // namespace Foam
/*---------------------------------------------------------------------------*\
License
This file is part of OGL.
OGL is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OGL. If not, see <http://www.gnu.org/licenses/>.
Class
Foam::IOExecutorHandler
Author: Gregor Olenik <go@hpsim.de>
SourceFiles
IOExecutorHandler.C
\*---------------------------------------------------------------------------*/
#ifndef OGL_IOExecutorHandler_INCLUDED_H
#define OGL_IOExecutorHandler_INCLUDED_H
#include "../IOPtr/IOPtr.H"
#include "fvCFD.H"
#include "regIOobject.H"
#include <ginkgo/ginkgo.hpp>
namespace Foam {
using vec = gko::matrix::Dense<scalar>;
using mtx = gko::matrix::Coo<scalar>;
class IOExecutorHandler {
private:
const word device_executor_name_;
const word app_executor_name_;
// executor where Ginkgo will perform the computation
GKOExecPtr *device_exec_ptr_;
// executor of the application
GKOExecPtr *app_exec_ptr_;
GKOReferenceExecPtr *ref_exec_ptr_;
GKOCudaExecPtr *cuda_exec_ptr_;
GKOOmpExecPtr *omp_exec_ptr_;
GKOHipExecPtr *hip_exec_ptr_;
public:
IOExecutorHandler(const objectRegistry &db, const dictionary &controlDict);
word get_device_executor_name() const { return device_executor_name_; }
std::shared_ptr<gko::Executor> get_device_executor() const
{
const word device_executor_name{get_device_executor_name()};
if (device_executor_name == "omp") {
return omp_exec();
}
if (device_executor_name == "cuda") {
return cuda_exec();
}
if (device_executor_name == "hip") {
return hip_exec();
}
return ref_exec();
};
std::shared_ptr<gko::Executor> app_exec() const
{
return app_exec_ptr_->get_ptr();
};
std::shared_ptr<gko::OmpExecutor> omp_exec() const
{
return omp_exec_ptr_->get_ptr();
};
std::shared_ptr<gko::CudaExecutor> cuda_exec() const
{
return cuda_exec_ptr_->get_ptr();
};
std::shared_ptr<gko::HipExecutor> hip_exec() const
{
return hip_exec_ptr_->get_ptr();
};
std::shared_ptr<gko::ReferenceExecutor> ref_exec() const
{
return ref_exec_ptr_->get_ptr();
};
};
} // namespace Foam
#endif
/*---------------------------------------------------------------------------*\
License
This file is part of OGL.
OGL is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OGL. If not, see <http://www.gnu.org/licenses/>.
Class
Foam::GKOCG
Author: Gregor Olenik <go@hpsim.de>
SourceFiles
GKOCG.C
\*---------------------------------------------------------------------------*/
#include <ginkgo/ginkgo.hpp>
#include "IOGKOMatrixHandler.H"
namespace Foam {
void IOGKOMatrixHandler::init_device_matrix(
const objectRegistry &db, std::vector<scalar> &values_host,
std::vector<label> &col_idxs_host, std::vector<label> &row_idxs_host,
const label nElems, const label nCells, const bool store) const
{
std::shared_ptr<gko::Executor> device_exec = get_device_executor();
if (sys_matrix_stored_) {
gkomatrix_ptr_ = &db.lookupObjectRef<GKOCOOIOPtr>(sys_matrix_name_);
return;
}
bool sparsity_pattern_stored =
db.foundObject<regIOobject>(sparsity_pattern_name_cols_);
std::shared_ptr<idx_array> col_idx;
std::shared_ptr<idx_array> row_idx;
if (sparsity_pattern_stored) {
io_col_idxs_ptr_ =
&db.lookupObjectRef<GKOIDXIOPtr>(sparsity_pattern_name_cols_);
io_row_idxs_ptr_ =
&db.lookupObjectRef<GKOIDXIOPtr>(sparsity_pattern_name_rows_);
col_idx = io_col_idxs_ptr_->get_ptr();
row_idx = io_row_idxs_ptr_->get_ptr();
} else {
// if not stored yet create sparsity pattern from correspondent
// views
auto col_idx_view = idx_array::view(gko::ReferenceExecutor::create(),
nElems, &col_idxs_host[0]);
auto row_idx_view = idx_array::view(gko::ReferenceExecutor::create(),
nElems, &row_idxs_host[0]);
// copy sparsity pattern to device and leave it there
col_idx = std::make_shared<idx_array>(col_idx_view);
col_idx->set_executor(device_exec);
row_idx = std::make_shared<idx_array>(row_idx_view);
row_idx->set_executor(device_exec);
}
// if system matrix is not stored create it and set shared pointer
auto gkomatrix =
gko::share(mtx::create(device_exec, gko::dim<2>(nCells, nCells),
val_array::view(gko::ReferenceExecutor::create(),
nElems, &values_host[0]),
*col_idx.get(), *row_idx.get()));
// if updating system matrix is not needed store ptr in obj registry
if (store) {
const fileName path = sys_matrix_name_;
gkomatrix_ptr_ = new GKOCOOIOPtr(IOobject(path, db), gkomatrix);
} else {
gkomatrix_ptr_ = NULL;
gkomatrix_ = gkomatrix;
}
// in any case store sparsity pattern
const fileName path_col = sparsity_pattern_name_cols_;
const fileName path_row = sparsity_pattern_name_rows_;
io_col_idxs_ptr_ = new GKOIDXIOPtr(IOobject(path_col, db), col_idx);
io_row_idxs_ptr_ = new GKOIDXIOPtr(IOobject(path_row, db), row_idx);
};
defineTemplateTypeNameWithName(GKOIDXIOPtr, "IDXIOPtr");
defineTemplateTypeNameWithName(GKOCOOIOPtr, "COOIOPtr");
} // namespace Foam
/*---------------------------------------------------------------------------*\
License
This file is part of OGL.
OGL is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OGL. If not, see <http://www.gnu.org/licenses/>.
Class
Foam::GKOCG
Author: Gregor Olenik <go@hpsim.de>
SourceFiles
GKOCG.C
\*---------------------------------------------------------------------------*/
#ifndef OGL_IOGKOMatrixHandler_INCLUDED_H
#define OGL_IOGKOMatrixHandler_INCLUDED_H
#include "fvCFD.H"
#include "regIOobject.H"
#include "../IOExecutorHandler/IOExecutorHandler.H"
#include "../IOPtr/IOPtr.H"
namespace Foam {
using mtx = gko::matrix::Coo<scalar>;
using val_array = gko::Array<scalar>;
using idx_array = gko::Array<label>;
class IOGKOMatrixHandler : public IOExecutorHandler {
private:
const word sys_matrix_name_;
const bool sys_matrix_stored_;
const word sparsity_pattern_name_cols_ = "gko_sparsity_pattern_cols";
const word sparsity_pattern_name_rows_ = "gko_sparsity_pattern_rows";
const word update_sysMatrix_;
const bool export_;
mutable std::shared_ptr<mtx> gkomatrix_ = NULL;
mutable GKOCOOIOPtr *gkomatrix_ptr_ = NULL;
mutable GKOIDXIOPtr *io_col_idxs_ptr_ = NULL;
mutable GKOIDXIOPtr *io_row_idxs_ptr_ = NULL;
public:
IOGKOMatrixHandler(const objectRegistry &db, const dictionary &controlDict,
const word sys_matrix_name)
: IOExecutorHandler(db, controlDict),
sys_matrix_name_(sys_matrix_name + "_gko"),
sys_matrix_stored_(db.foundObject<regIOobject>(sys_matrix_name_)),
update_sysMatrix_(
controlDict.lookupOrDefault("updateSysMatrix", word("yes"))),
export_(controlDict.lookupOrDefault<Switch>("export", false)){};
bool get_sys_matrix_stored() const { return sys_matrix_stored_; };
void init_device_matrix(const objectRegistry &db,
std::vector<scalar> &values_host,
std::vector<label> &col_idxs_host,
std::vector<label> &row_idxs_host,
const label nElems, const label nCells,
const bool store) const;
std::shared_ptr<mtx> get_gkomatrix() const
{
if (gkomatrix_ptr_ == NULL) {
return gkomatrix_;
}
return gkomatrix_ptr_->get_ptr();
};
bool get_update_sys_matrix() const { return (update_sysMatrix_ == "yes"); }
bool get_export() const { return export_; }
};
} // namespace Foam
#endif
#include "IOPtr.H"
namespace Foam {
} // namespace Foam
/*---------------------------------------------------------------------------*\
License
This file is part of OGL.
OGL is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OGL. If not, see <http://www.gnu.org/licenses/>.
Class
Foam::IOExecutorHandler
Author: Gregor Olenik <go@hpsim.de>
SourceFiles
IOExecutorHandler.C
\*---------------------------------------------------------------------------*/
#ifndef OGL_IOPtr_INCLUDED_H
#define OGL_IOPtr_INCLUDED_H
#include <ginkgo/ginkgo.hpp>
#include "fvCFD.H"
namespace Foam {
using mtx = gko::matrix::Coo<scalar>;
using val_array = gko::Array<scalar>;
using idx_array = gko::Array<label>;
template <class T>
class IOPtr : public regIOobject {
private:
std::shared_ptr<T> ptr_;
public:
// - Runtime type information
TypeName("IOPtr");
//- Construct from IOobject and a PtrList
IOPtr(const IOobject &io, std::shared_ptr<T> in_ptr)
: regIOobject(io), ptr_(in_ptr){};
std::shared_ptr<T> get_ptr() { return ptr_; };
bool writeData(Ostream &) const { return false; };
};
typedef IOPtr<gko::Executor> GKOExecPtr;
typedef IOPtr<gko::CudaExecutor> GKOCudaExecPtr;
typedef IOPtr<gko::ReferenceExecutor> GKOReferenceExecPtr;
typedef IOPtr<gko::OmpExecutor> GKOOmpExecPtr;
typedef IOPtr<gko::HipExecutor> GKOHipExecPtr;
typedef IOPtr<idx_array> GKOIDXIOPtr;
typedef IOPtr<gko::matrix::Coo<scalar>> GKOCOOIOPtr;
} // namespace Foam
#endif
#include "IOSortingIdxHandler.H"
namespace Foam {
IOSortingIdxHandler::IOSortingIdxHandler(const objectRegistry &db,
const label nElems, const bool sort)
: nElems_(nElems), is_sorted_(false), sort_(sort)
{
// check object registry if sorting idxs exists
word sorting_idxs_name = "sorting_idxs_";
if (db.foundObject<IOField<label>>(sorting_idxs_name)) {
IOField<label> &labelListRef =
db.lookupObjectRef<IOField<label>>(sorting_idxs_name);
sorting_idxs_ = &labelListRef;
set_is_sorted(true);
} else {
const fileName path = sorting_idxs_name;
sorting_idxs_ = new IOField<label>(IOobject(path, db));
init_sorting_idxs();
set_is_sorted(false);
}
};
void IOSortingIdxHandler::compute_sorting_idxs(
const std::vector<label> &row_idxs)
{
// sort indexes based on comparing values in v
// using std::stable_sort instead of std::sort
// to avoid unnecessary index re-orderings
// when v contains elements of equal values
std::cout << " compute sorting idxs " << nElems_ << std::endl;
std::stable_sort(sorting_idxs_->data(), &sorting_idxs_->data()[nElems_],
[this, &row_idxs](size_t i1, size_t i2) {
return row_idxs[i1] < row_idxs[i2];
});
};
void IOSortingIdxHandler::init_sorting_idxs()
{
// initialize original index locations
sorting_idxs_->resize(nElems_);
// set sorting idxs from 0 .. n
for (int i = 0; i < nElems_; i++) {
sorting_idxs_->operator[](i) = i;
}
};
} // namespace Foam
/*---------------------------------------------------------------------------*\
License
This file is part of OGL.
OGL is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OGL. If not, see <http://www.gnu.org/licenses/>.
Class
Foam::IOSortingIdxHandler
Author: Gregor Olenik <go@hpsim.de>
SourceFiles
IOSortingIdxHandler.H
\*---------------------------------------------------------------------------*/
#ifndef OGL_IOSortingIdxHandler_INCLUDED_H
#define OGL_IOSortingIdxHandler_INCLUDED_H
#include <vector>
#include "fvCFD.H"
#include "regIOobject.H"
namespace Foam {
class IOSortingIdxHandler {
private:
const label nElems_;
// if sorting_idxs_ was found in object registry it does not
// need to be resorted
mutable bool is_sorted_;