Unverified Commit 10f51111 authored by greole's avatar greole Committed by GitHub
Browse files

Ir (#1)

* update script, needs refactoring

* update

* fix benchmark

* run at least for 10 ts

* rename GKOBase to GKOlduBase

* create ldu and Ldu matrices

* add ldu GKOCG and GKOBiCGStab

* add Ldu matrix

* add asymetric solver

* clean up

* start implementing of stopping criteria

* add common file

* update common

* inject c++14

* update

* use modified boxTurb16 case

* update benchmark

* keep CF constant

* start reuse sorting idx

* add IR solver, make sorting optional

* add line buffering, further cli args

* Add basic implementation of storing sorting idxs

* add IR implementation

* make sorting default

* store gkomatrix

* add common.C

* make executor an IOPtr

* prepare for forHLR test

* fix block clean for different block start syntax

* transfer to desktop

* hardcoded stored omp executor

* load different executor

* fix min/maxIter bug

* debug reference executor issue

* add clang format to keep formating constant

* fix cuda executor

* clean up

* clean up
parent 6b5d7682
---
Language: Cpp
# BasedOnStyle: Google
AccessModifierOffset: -4
AlignAfterOpenBracket: Align
AlignConsecutiveAssignments: false
AlignConsecutiveDeclarations: false
AlignEscapedNewlines: Left
AlignOperands: true
AlignTrailingComments: true
AllowAllParametersOfDeclarationOnNextLine: true
AllowShortBlocksOnASingleLine: false
AllowShortCaseLabelsOnASingleLine: false
AllowShortFunctionsOnASingleLine: All
AllowShortIfStatementsOnASingleLine: true
AllowShortLoopsOnASingleLine: true
AlwaysBreakAfterDefinitionReturnType: None
AlwaysBreakAfterReturnType: None
AlwaysBreakBeforeMultilineStrings: true
AlwaysBreakTemplateDeclarations: true
BinPackArguments: true
BinPackParameters: true
BraceWrapping:
AfterClass: false
AfterControlStatement: false
AfterEnum: false
AfterFunction: true
AfterNamespace: false
AfterObjCDeclaration: false
AfterStruct: false
AfterUnion: false
BeforeCatch: false
BeforeElse: false
IndentBraces: false
SplitEmptyFunction: false
SplitEmptyRecord: false
SplitEmptyNamespace: false
BreakBeforeBinaryOperators: None
BreakBeforeBraces: Custom
BreakBeforeInheritanceComma: false
BreakBeforeTernaryOperators: true
BreakConstructorInitializersBeforeComma: false
BreakConstructorInitializers: BeforeColon
BreakAfterJavaFieldAnnotations: false
BreakStringLiterals: true
ColumnLimit: 80
CommentPragmas: '^ IWYU pragma:'
CompactNamespaces: false
ConstructorInitializerAllOnOneLineOrOnePerLine: true
ConstructorInitializerIndentWidth: 4
ContinuationIndentWidth: 4
Cpp11BracedListStyle: true
DerivePointerAlignment: false
DisableFormat: false
ExperimentalAutoDetectBinPacking: false
FixNamespaceComments: true
ForEachMacros:
- foreach
- Q_FOREACH
- BOOST_FOREACH
IncludeCategories:
- Regex: '^<.*\.h>'
Priority: 1
- Regex: '^<.*'
Priority: 2
- Regex: '.*'
Priority: 3
IncludeIsMainRegex: '([-_](test|unittest))?$'
IndentCaseLabels: false
IndentWidth: 4
IndentWrappedFunctionNames: false
JavaScriptQuotes: Leave
JavaScriptWrapImports: true
KeepEmptyLinesAtTheStartOfBlocks: false
MacroBlockBegin: ''
MacroBlockEnd: ''
MaxEmptyLinesToKeep: 2
NamespaceIndentation: None
ObjCBlockIndentWidth: 2
ObjCSpaceAfterProperty: false
ObjCSpaceBeforeProtocolList: false
PenaltyBreakAssignment: 2
PenaltyBreakBeforeFirstCallParameter: 1
PenaltyBreakComment: 300
PenaltyBreakFirstLessLess: 120
PenaltyBreakString: 1000
PenaltyExcessCharacter: 1000000
PenaltyReturnTypeOnItsOwnLine: 200
PointerAlignment: Right
ReflowComments: true
SortIncludes: true
SortUsingDeclarations: true
SpaceAfterCStyleCast: false
SpaceAfterTemplateKeyword: true
SpaceBeforeAssignmentOperators: true
SpaceBeforeParens: ControlStatements
SpaceInEmptyParentheses: false
SpacesBeforeTrailingComments: 2
SpacesInAngles: false
SpacesInContainerLiterals: true
SpacesInCStyleCastParentheses: false
SpacesInParentheses: false
SpacesInSquareBrackets: false
Standard: Cpp11
TabWidth: 8
UseTab: Never
...
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2011-2018 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
\*---------------------------------------------------------------------------*/
#include <ginkgo/ginkgo.hpp>
#include <map>
#include <type_traits>
#include "GKOACG.H"
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
namespace Foam {
defineTypeNameAndDebug(GKOACG, 0);
LduMatrix<vector, scalar,
scalar>::solver::addsymMatrixConstructorToTable<GKOACG>
addGKOACGSymMatrixConstructorToTable_;
} // namespace Foam
// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
// ************************************************************************* //
/*---------------------------------------------------------------------------*\
License
This file is part of OGL.
OGL is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
Class
Foam::GKOACG
Author: Gregor Olenik <go@hpsim.de>
SourceFiles
GKOACG.C
\*---------------------------------------------------------------------------*/
#ifndef GKOACG_H
#define GKOACG_H
#include "GKOLduBase.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
namespace Foam {
class GKOACGFactory {
private:
// executor where Ginkgo will perform the computation
const word preconditioner_;
public:
GKOACGFactory(const dictionary &controlDict_)
: preconditioner_(
controlDict_.lookupOrDefault("preconditioner", word("none"))){};
std::unique_ptr<gko::solver::Cg<double>::Factory,
std::default_delete<gko::solver::Cg<double>::Factory>>
create_solver(
std::shared_ptr<gko::Executor> exec,
std::vector<std::shared_ptr<const gko::stop::CriterionFactory>>
criterion_vec) const
{
if (preconditioner_ == "none")
return create_default(exec, criterion_vec);
if (preconditioner_ == "BJ") return create_BJ(exec, criterion_vec);
};
std::unique_ptr<gko::solver::Cg<double>::Factory,
std::default_delete<gko::solver::Cg<double>::Factory>>
create_default(
std::shared_ptr<gko::Executor> exec,
std::vector<std::shared_ptr<const gko::stop::CriterionFactory>>
criterion_vec) const
{
return gko::solver::Cg<scalar>::build()
.with_criteria(criterion_vec)
.on(exec);
};
std::unique_ptr<gko::solver::Cg<double>::Factory,
std::default_delete<gko::solver::Cg<double>::Factory>>
create_BJ(std::shared_ptr<gko::Executor> exec,
std::vector<std::shared_ptr<const gko::stop::CriterionFactory>>
criterion_vec) const
{
using bj = gko::preconditioner::Jacobi<>;
return gko::solver::Cg<scalar>::build()
.with_criteria(criterion_vec)
.with_preconditioner(bj::build().with_max_block_size(8u).on(exec))
.on(exec);
};
};
/*---------------------------------------------------------------------------*\
Class GKOACG Declaration
\*---------------------------------------------------------------------------*/
class GKOACG : public GKOLduBaseSolver<vector, scalar, GKOACGFactory> {
// Private Member Functions
public:
TypeName("GKOACG");
//- Disallow default bitwise copy construct
GKOACG(const GKOACG &);
//- Disallow default bitwise assignment
void operator=(const GKOACG &);
// Constructors
//- Construct from matrix components and solver controls
GKOACG(const word &fieldName,
const LduMatrix<vector, scalar, scalar> &matrix,
const dictionary &solverControls)
: GKOLduBaseSolver<vector, scalar, GKOACGFactory>(fieldName, matrix,
solverControls){};
//- Destructor
virtual ~GKOACG(){};
// Member Functions
//- Solve the matrix with this solver
virtual SolverPerformance<vector> solve(Field<vector> &psi) const
{
return solve_impl(psi);
}
};
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
} // End namespace Foam
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
#endif
// ************************************************************************* //
......@@ -26,10 +26,8 @@ License
#include <ginkgo/ginkgo.hpp>
#include <map>
#include <type_traits>
#include "GKOBase.H"
#include "GKOLduBase.H"
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
// ************************************************************************* //
/*---------------------------------------------------------------------------*\
License
This file is part of OGL.
OGL is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
Class
Foam::GKOCG
Author: Gregor Olenik <go@hpsim.de>
SourceFiles
GKOCG.C
\*---------------------------------------------------------------------------*/
#ifndef GKO_LDUBase_H
#define GKO_LDUBase_H
#include "../common/common.H"
#include "LduMatrix.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
namespace Foam {
/*---------------------------------------------------------------------------*\
Class GKOCG Declaration
\*---------------------------------------------------------------------------*/
template <class Type, class LDUType, class SolverFactory>
class GKOLduBaseSolver : public LduMatrix<Type, LDUType, LDUType>::solver,
public SolverFactory,
public lduLduCommonBase {
public:
// Some shortcuts
using IndexType = label;
using vec = gko::matrix::Dense<scalar>;
using mtx = gko::matrix::Coo<scalar>;
using val_array = gko::Array<scalar>;
using idx_array = gko::Array<label>;
using cg = gko::solver::Cg<scalar>;
private:
// for now a copy of the OF matrix is stored
// to keep values contiguous in memory
// executor where Ginkgo will perform the computation
public:
std::vector<scalar> sorting_idxs_;
mutable std::vector<scalar> values_;
mutable std::vector<label> col_idxs_;
mutable std::vector<label> row_idxs_;
public:
//- Construct from matrix components and solver controls
GKOLduBaseSolver(const word &fieldName,
const LduMatrix<Type, LDUType, LDUType> &matrix,
const dictionary &solverControls)
: LduMatrix<Type, LDUType, LDUType>::solver(fieldName, matrix,
solverControls),
SolverFactory(this->controlDict_),
lduLduCommonBase(matrix.mesh().thisDb(), matrix.diag().size(),
matrix.lduAddr().upperAddr().size(), solverControls)
{
// create executors
if (get_update_sys_matrix()) {
SIMPLE_TIME(update_gko_mtx, update_GKOMatrix();)
}
// if (!get_is_sorted()) {
// SIMPLE_TIME(compute_sort, compute_sorting_idxs();)
// }
// if (get_sort()) {
// SIMPLE_TIME(sort_gko_mtx, sort_GKOMatrix();)
// }
}
const std::vector<scalar> &vals() const { return values_; };
const std::vector<label> &col_idx() const { return col_idxs_; };
const std::vector<label> &row_idx() const { return row_idxs_; };
// NOTE move to base lduLduBase
void update_GKOMatrix() const
{
// reset vectors
values_.resize(0);
col_idxs_.resize(0);
row_idxs_.resize(0);
values_.reserve(nElems());
col_idxs_.reserve(nElems());
row_idxs_.reserve(nElems());
// fill vectors unsorted
for (IndexType i = 0; i < nNeighbours(); ++i) {
values_.push_back(this->matrix_.lower()[i]);
row_idxs_.push_back(this->matrix_.lduAddr().lowerAddr()[i]);
col_idxs_.push_back(this->matrix_.lduAddr().upperAddr()[i]);
}
for (IndexType i = 0; i < nCells(); ++i) {
values_.push_back(this->matrix_.diag()[i]);
col_idxs_.push_back(i);
row_idxs_.push_back(i);
}
for (IndexType i = 0; i < nNeighbours(); ++i) {
values_.push_back(this->matrix_.upper()[i]);
row_idxs_.push_back(this->matrix_.lduAddr().upperAddr()[i]);
col_idxs_.push_back(this->matrix_.lduAddr().lowerAddr()[i]);
}
};
scalar compute_residual(mtx *const A, vec *const x, vec *const b) const
{
auto one = gko::initialize<vec>({1.0}, exec());
auto neg_one = gko::initialize<vec>({-1.0}, exec());
auto res = gko::initialize<vec>({0.0}, exec());
A->apply(lend(one), lend(x), lend(neg_one), lend(b));
b->compute_norm2(lend(res));
return res->at(0);
};
virtual SolverPerformance<Type> solve_impl(Field<Type> &psi) const
{
// --- Setup class containing solver performance data
// Implement
word preconditionerName(this->controlDict_.lookup("preconditioner"));
SolverPerformance<Type> solverPerf(preconditionerName,
this->fieldName_);
auto source_view_x = val_array::view(
app_exec(), 3 * nCells(),
const_cast<scalar *>(&this->matrix_.source()[0].x()));
auto source_view_y = val_array::view(
app_exec(), 3 * nCells(),
const_cast<scalar *>(&this->matrix_.source()[0].y()));
auto source_view_z = val_array::view(
app_exec(), 3 * nCells(),
const_cast<scalar *>(&this->matrix_.source()[0].z()));
auto b_x =
vec::create(app_exec(), gko::dim<2>(nCells(), 1), source_view_x, 3);
auto b_y =
vec::create(app_exec(), gko::dim<2>(nCells(), 1), source_view_y, 3);
auto b_z =
vec::create(app_exec(), gko::dim<2>(nCells(), 1), source_view_z, 3);
auto psi_view_x = val_array::view(app_exec(), 3 * nCells(),
const_cast<scalar *>(&psi[0].x()));
auto psi_view_y = val_array::view(app_exec(), 3 * nCells(),
const_cast<scalar *>(&psi[0].y()));
auto psi_view_z = val_array::view(app_exec(), 3 * nCells(),
const_cast<scalar *>(&psi[0].z()));
auto x =
vec::create(app_exec(), gko::dim<2>(nCells(), 1), psi_view_x, 3);
auto y =
vec::create(app_exec(), gko::dim<2>(nCells(), 1), psi_view_y, 3);
auto z =
vec::create(app_exec(), gko::dim<2>(nCells(), 1), psi_view_z, 3);
std::vector<std::shared_ptr<const gko::stop::CriterionFactory>>
criterion_vec{};
auto tolerance_stop = gko::stop::AbsoluteResidualNorm<scalar>::build()
.with_tolerance(tolerance())
.on(exec());
// NOTE currently minIter
// TODO move to base class
if (minIter() == 0) {
auto max_iter_stop =
gko::stop::Iteration::build().with_max_iters(maxIter()).on(
exec());
criterion_vec.push_back(gko::share(tolerance_stop));
criterion_vec.push_back(gko::share(max_iter_stop));
} else {
auto max_iter_stop =
gko::stop::Iteration::build().with_max_iters(minIter()).on(
exec());
criterion_vec.push_back(gko::share(max_iter_stop));
}
// Generate solver
auto solver_gen = this->create_solver(exec(), criterion_vec);
auto gkomatrix = gko::give(
mtx::create(exec(), gko::dim<2>(nCells()),
val_array::view(app_exec(), nElems(), &values_[0]),
idx_array::view(app_exec(), nElems(), &col_idxs_[0]),
idx_array::view(app_exec(), nElems(), &row_idxs_[0])));
auto solver = solver_gen->generate(gko::give(gkomatrix));
SIMPLE_TIME(solve_x, solver->apply(gko::lend(b_x), gko::lend(x));)
SIMPLE_TIME(solve_y, solver->apply(gko::lend(b_y), gko::lend(y));)
SIMPLE_TIME(solve_z, solver->apply(gko::lend(b_z), gko::lend(z));)
auto x_view = val_array::view(app_exec(), nCells(), x->get_values());
auto x_clone =
vec::create(app_exec(), gko::dim<2>(nCells(), 1), x_view, 1);
auto y_view = val_array::view(app_exec(), nCells(), y->get_values());
auto y_clone =
vec::create(app_exec(), gko::dim<2>(nCells(), 1), y_view, 1);
auto z_view = val_array::view(app_exec(), nCells(), z->get_values());
auto z_clone =
vec::create(app_exec(), gko::dim<2>(nCells(), 1), z_view, 1);
for (size_t i = 0; i < nCells(); i++) {
psi[i].x() = x_clone->at(i);
psi[i].y() = y_clone->at(i);
psi[i].z() = z_clone->at(i);
}
return solverPerf;
};
};
} // End namespace Foam
#endif
GKOBase/GKOBase.C
GKOCG/GKOCG.C
GKOBiCGStab/GKOBiCGStab.C
common/common.C
lduMatrix/GKOlduBase/GKOlduBase.C
lduMatrix/GKOCG/GKOCG.C
lduMatrix/GKOIR/GKOIR.C
lduMatrix/GKOBiCGStab/GKOBiCGStab.C
LduMatrix/GKOLduBase/GKOLduBase.C
LduMatrix/GKOACG/GKOACG.C
LIB = $(FOAM_USER_LIBBIN)/libOGL
EXE_INC = \
-I/home/kit/scc/nq7776/code/ginkgo/build/include \
-L/home/kit/scc/nq7776/code/ginkgo/build/lib
-I$(LIB_SRC)/finiteVolume/lnInclude \
-I$(LIB_SRC)/meshTools/lnInclude \
-std=c++14
LIB_LIBS = \
-lginkgo
-lginkgo \
-lfiniteVolume \
-lmeshTools
/*--------------------------------*- C++ -*----------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Version: 6
\\/ M anipulation |
\*---------------------------------------------------------------------------*/
FoamFile
{
version 2.0;
format ascii;
class volVectorField;
location "0";
object U.orig;
}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
dimensions [0 1 -1 0 0 0 0];
internalField uniform (0 0 0);
boundaryField
{
patch0_half0
{
type fixedValue;
value uniform (0 0 0);
}
patch0_half1
{
type fixedValue;
value uniform (0 0 0);
}
patch1_half0
{
type fixedValue;