Unverified Commit 781cfd14 authored by greole's avatar greole Committed by GitHub
Browse files

Update master branch for first prototyp release (#2)

* Ir (#1)

* update script, needs refactoring

* update

* fix benchmark

* run at least for 10 ts

* rename GKOBase to GKOlduBase

* create ldu and Ldu matrices

* add ldu GKOCG and GKOBiCGStab

* add Ldu matrix

* add asymetric solver

* clean up

* start implementing of stopping criteria

* add common file

* update common

* inject c++14

* update

* use modified boxTurb16 case

* update benchmark

* keep CF constant

* start reuse sorting idx

* add IR solver, make sorting optional

* add line buffering, further cli args

* Add basic implementation of storing sorting idxs

* add IR implementation

* make sorting default

* store gkomatrix

* add common.C

* make executor an IOPtr

* prepare for forHLR test

* fix block clean for different block start syntax

* transfer to desktop

* hardcoded stored omp executor

* load different executor

* fix min/maxIter bug

* debug reference executor issue

* add clang format to keep formating constant

* fix cuda executor

* clean up

* clean up

* remove unneeded bash benchmark script

* format + prepare preconditioner

* run omp benchmarks for different number of threads

* make pressure solver settings compatible with OF8

* changed size of cases

* fix benchmarking folders

* set dns case boundaries to walls

* Fix several compiler warnings, reordering issues,

* create additional executor to force eager copy

* log succes state of run, keep retrying on failure at least for now

* move OpenFoam stopping criterion to separate file

* Use Foam switches to read dictionary entries

* Make export system a free function, clean logger and imports

* Move IterationLogger to common.H

* Fix execution if sysMatrix is not stored

if sysMatrix is not stored and thus the gkomatrix_ptr==NULL the system
matrix is recreated

* fix loop ctr

* Fix IR solver

- initialize IR with CG
- remove BJ for now

* Add Solver classes to distinguish between OF and GKO solvers, set time out for runs, fix log files

* fix formating

* Fix typos in smoothSolver class

* Fix int to unsingned long conversion

* Add initial CMakeLists file

* Add basic cmake files

* Fix return type warning

* move setting device_exec to separate function

* add third_party folder

* Update README.md

* Update README.md

* Use FOAM_ enviromental variables

* install libginkgo.so files along with libOGL.so if no external ginkgo is used

* Create a initial cmake.yml file

* clone and source OpenFOAM

* Fix non ascii symbol

* display OpenFOAM folder content

* show content of OpenFOAM-8 folder

* Update cmake.yml

* set OpenFOAM env variables manually

* add build badge

* Update Readme file

* remove Test folder since it is no longer needed

* re-enable exporting the system matrix, rhs and initial guess

* Read maxBlockSize from dictionary

* Rename controlDict_ member variable to dictionary_

* Make BiCGStabFactory consistent with CGFactory

* Clean unneeded comment

* Fix sorting bounds of sorting_idxs

* Update cmake.yml

* add basic travis file

* remove github actions file

* update .travis.yml

* fix OpenFOAM path

* update travis file

* compile full OpenFOAM for now

* install openmpi-dev

* install flexc++

* add OpenFOAM-dev to build matrix

* add fl dev

* install openfoam from the repository

* for now just test if OGL builds

* add build badge

* fix sourcing depending on of version

* switch to ubuntu 20.04

* make OGL_USE_EXTERNAL_GINKGO an option

* use --no-check-certificate since importing the key fails regularily

* pass --no-check-certificate to wget

* fix type of tmp_col_idxs and tmp_row_idxs

* sort the system matrix to avoid  cuda convergence issue

* add preconditioner to path

* remove benchmark script to separate repository

* fix find_package

* Fix find_package execution after external ginkgo was included #5

Co-authored-by: Terry Cojean's avatartcojean <terry.cojean@kit.edu>

* Add OpenFOAM version shields
Co-authored-by: Terry Cojean's avatartcojean <terry.cojean@kit.edu>
parent 6b5d7682
---
Language: Cpp
# BasedOnStyle: Google
AccessModifierOffset: -4
AlignAfterOpenBracket: Align
AlignConsecutiveAssignments: false
AlignConsecutiveDeclarations: false
AlignEscapedNewlines: Left
AlignOperands: true
AlignTrailingComments: true
AllowAllParametersOfDeclarationOnNextLine: true
AllowShortBlocksOnASingleLine: false
AllowShortCaseLabelsOnASingleLine: false
AllowShortFunctionsOnASingleLine: All
AllowShortIfStatementsOnASingleLine: true
AllowShortLoopsOnASingleLine: true
AlwaysBreakAfterDefinitionReturnType: None
AlwaysBreakAfterReturnType: None
AlwaysBreakBeforeMultilineStrings: true
AlwaysBreakTemplateDeclarations: true
BinPackArguments: true
BinPackParameters: true
BraceWrapping:
AfterClass: false
AfterControlStatement: false
AfterEnum: false
AfterFunction: true
AfterNamespace: false
AfterObjCDeclaration: false
AfterStruct: false
AfterUnion: false
BeforeCatch: false
BeforeElse: false
IndentBraces: false
SplitEmptyFunction: false
SplitEmptyRecord: false
SplitEmptyNamespace: false
BreakBeforeBinaryOperators: None
BreakBeforeBraces: Custom
BreakBeforeInheritanceComma: false
BreakBeforeTernaryOperators: true
BreakConstructorInitializersBeforeComma: false
BreakConstructorInitializers: BeforeColon
BreakAfterJavaFieldAnnotations: false
BreakStringLiterals: true
ColumnLimit: 80
CommentPragmas: '^ IWYU pragma:'
CompactNamespaces: false
ConstructorInitializerAllOnOneLineOrOnePerLine: true
ConstructorInitializerIndentWidth: 4
ContinuationIndentWidth: 4
Cpp11BracedListStyle: true
DerivePointerAlignment: false
DisableFormat: false
ExperimentalAutoDetectBinPacking: false
FixNamespaceComments: true
ForEachMacros:
- foreach
- Q_FOREACH
- BOOST_FOREACH
IncludeCategories:
- Regex: '^<.*\.h>'
Priority: 1
- Regex: '^<.*'
Priority: 2
- Regex: '.*'
Priority: 3
IncludeIsMainRegex: '([-_](test|unittest))?$'
IndentCaseLabels: false
IndentWidth: 4
IndentWrappedFunctionNames: false
JavaScriptQuotes: Leave
JavaScriptWrapImports: true
KeepEmptyLinesAtTheStartOfBlocks: false
MacroBlockBegin: ''
MacroBlockEnd: ''
MaxEmptyLinesToKeep: 2
NamespaceIndentation: None
ObjCBlockIndentWidth: 2
ObjCSpaceAfterProperty: false
ObjCSpaceBeforeProtocolList: false
PenaltyBreakAssignment: 2
PenaltyBreakBeforeFirstCallParameter: 1
PenaltyBreakComment: 300
PenaltyBreakFirstLessLess: 120
PenaltyBreakString: 1000
PenaltyExcessCharacter: 1000000
PenaltyReturnTypeOnItsOwnLine: 200
PointerAlignment: Right
ReflowComments: true
SortIncludes: true
SortUsingDeclarations: true
SpaceAfterCStyleCast: false
SpaceAfterTemplateKeyword: true
SpaceBeforeAssignmentOperators: true
SpaceBeforeParens: ControlStatements
SpaceInEmptyParentheses: false
SpacesBeforeTrailingComments: 2
SpacesInAngles: false
SpacesInContainerLiterals: true
SpacesInCStyleCastParentheses: false
SpacesInParentheses: false
SpacesInSquareBrackets: false
Standard: Cpp11
TabWidth: 8
UseTab: Never
...
language: cpp
sudo: true
compiler:
- gcc
dist: focal
os:
- linux
env:
- VERSION=8
- VERSION=7
cache:
directories:
- $HOME/OpenFOAM
addons:
apt:
update: true
packages:
- flexc++
- libfl-dev
- lcov
- libopenmpi-dev
before_install:
- sudo sh -c "wget --no-check-certificate -O - https://dl.openfoam.org/gpg.key | apt-key add -"
- sudo add-apt-repository http://dl.openfoam.org/ubuntu
- sudo apt update
install:
- sudo apt-get -y install openfoam$VERSION
- source /opt/openfoam$VERSION/etc/bashrc
before_script:
- cmake --version
- mkdir build
- cd build
- cmake ..
script:
- make -j4
cmake_minimum_required(VERSION 3.9)
project(OGL LANGUAGES C CXX DESCRIPTION "A wrapper for Ginkgo solver to provide GPGPU capabilities to OpenFOAM")
if(NOT DEFINED ENV{FOAM_SRC})
message(FATAL_ERROR "You must source OpenFOAM before building OGL")
endif()
# Configuration options
include(cmake/build_type_helpers.cmake)
list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/Modules/")
include(CheckIncludeFileCXX)
option(OGL_USE_EXTERNAL_GINKGO "Use external ginkgo" FALSE)
include(CheckIncludeFileCXX)
check_include_file_cxx(cxxabi.h GKO_HAVE_CXXABI_H)
include(cmake/package_helpers.cmake)
include(cmake/build_helpers.cmake)
include(cmake/install_helpers.cmake)
ginkgo_find_package(Ginkgo "Ginkgo::ginkgo" FALSE 1.4.0)
add_subdirectory(third_party)
# If ginkgo wasn't found so far look for the third_party installation
if (NOT Ginkgo_FOUND)
find_package(Ginkgo 1.4.0 REQUIRED PATHS ${CMAKE_CURRENT_BINARY_DIR}/third_party/ginkgo/build/install/lib/cmake)
endif()
# To interface with ginkgo at least C++ 14 is needed
set(CMAKE_CXX_STANDARD 14)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -O0 -ggdb")
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} \
-Wall -Wpedantic -Wextra -march=native -fopenmp -fPIC"
)
if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} \
-fno-omit-frame-pointer \
-fsanitize=address"
)
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} \
-fno-omit-frame-pointer \
-Rpass=loop-vectorize \
-ffast-math \
"
#-Rpass-analysis=loop-vectorize \
#-Rpass-missed=loop-vectorize \
)
elseif ("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU")
add_definitions(-DWITH_GNU)
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} \
--coverage\
-fprofile-arcs \
-fsanitize=address \
-ggdb3 \
-ftest-coverage"
)
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} \
-ffast-math \
-march=native \
-ftree-vectorize \
-rdynamic \
"
# -fopt-info-vec-missed"
)
endif()
option(BUILD_SHARED_LIBS "Build shared library" ON)
add_library(OGL "" )
add_compile_definitions(
WM_LABEL_SIZE=32
WM_ARCH_OPTION=64
NoRepository
linux64
WM_DP
)
target_sources(OGL
PRIVATE
lduMatrix/GKOlduBase/GKOlduBase.C
lduMatrix/GKOCG/GKOCG.C
lduMatrix/GKOIR/GKOIR.C
lduMatrix/GKOBiCGStab/GKOBiCGStab.C
LduMatrix/GKOLduBase/GKOLduBase.C
LduMatrix/GKOACG/GKOACG.C
PUBLIC
lduMatrix/GKOlduBase/GKOlduBase.H
lduMatrix/GKOCG/GKOCG.H
lduMatrix/GKOIR/GKOIR.H
lduMatrix/GKOBiCGStab/GKOBiCGStab.H
LduMatrix/GKOLduBase/GKOLduBase.H
LduMatrix/GKOACG/GKOACG.H
)
target_include_directories(OGL
PUBLIC
$ENV{FOAM_SRC}/OpenFOAM/lnInclude
$ENV{FOAM_SRC}/meshTools/lnInclude
$ENV{FOAM_SRC}/finiteVolume/lnInclude
$ENV{FOAM_SRC}/OSspecific/POSIX/lnInclude
common/
lduMatrix/GKOlduBase
LduMatrix/GKOLduBase
)
target_link_libraries(OGL
PUBLIC
Ginkgo::ginkgo
)
install(TARGETS OGL
DESTINATION $ENV{FOAM_USER_LIBBIN}
)
if(NOT ${OGL_USE_EXTERNAL_GINKGO})
install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/third_party/ginkgo/build/install/lib/
DESTINATION $ENV{FOAM_USER_LIBBIN}
)
endif()
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2011-2018 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
\*---------------------------------------------------------------------------*/
#include <ginkgo/ginkgo.hpp>
#include <map>
#include <type_traits>
#include "GKOACG.H"
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
namespace Foam {
defineTypeNameAndDebug(GKOACG, 0);
LduMatrix<vector, scalar,
scalar>::solver::addsymMatrixConstructorToTable<GKOACG>
addGKOACGSymMatrixConstructorToTable_;
} // namespace Foam
// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
// ************************************************************************* //
/*---------------------------------------------------------------------------*\
License
This file is part of OGL.
OGL is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
Class
Foam::GKOACG
Author: Gregor Olenik <go@hpsim.de>
SourceFiles
GKOACG.C
\*---------------------------------------------------------------------------*/
#ifndef GKOACG_H
#define GKOACG_H
#include "GKOLduBase.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
namespace Foam {
class GKOACGFactory {
private:
// executor where Ginkgo will perform the computation
const word preconditioner_;
public:
GKOACGFactory(const dictionary &controlDict_)
: preconditioner_(
controlDict_.lookupOrDefault("preconditioner", word("none"))){};
std::unique_ptr<gko::solver::Cg<double>::Factory,
std::default_delete<gko::solver::Cg<double>::Factory>>
create_solver(
std::shared_ptr<gko::Executor> exec,
std::vector<std::shared_ptr<const gko::stop::CriterionFactory>>
criterion_vec) const
{
if (preconditioner_ == "none")
return create_default(exec, criterion_vec);
if (preconditioner_ == "BJ") return create_BJ(exec, criterion_vec);
};
std::unique_ptr<gko::solver::Cg<double>::Factory,
std::default_delete<gko::solver::Cg<double>::Factory>>
create_default(
std::shared_ptr<gko::Executor> exec,
std::vector<std::shared_ptr<const gko::stop::CriterionFactory>>
criterion_vec) const
{
return gko::solver::Cg<scalar>::build()
.with_criteria(criterion_vec)
.on(exec);
};
std::unique_ptr<gko::solver::Cg<double>::Factory,
std::default_delete<gko::solver::Cg<double>::Factory>>
create_BJ(std::shared_ptr<gko::Executor> exec,
std::vector<std::shared_ptr<const gko::stop::CriterionFactory>>
criterion_vec) const
{
using bj = gko::preconditioner::Jacobi<>;
return gko::solver::Cg<scalar>::build()
.with_criteria(criterion_vec)
.with_preconditioner(bj::build().with_max_block_size(8u).on(exec))
.on(exec);
};
};
/*---------------------------------------------------------------------------*\
Class GKOACG Declaration
\*---------------------------------------------------------------------------*/
class GKOACG : public GKOLduBaseSolver<vector, scalar, GKOACGFactory> {
// Private Member Functions
public:
TypeName("GKOACG");
//- Disallow default bitwise copy construct
GKOACG(const GKOACG &);
//- Disallow default bitwise assignment
void operator=(const GKOACG &);
// Constructors
//- Construct from matrix components and solver controls
GKOACG(const word &fieldName,
const LduMatrix<vector, scalar, scalar> &matrix,
const dictionary &solverControls)
: GKOLduBaseSolver<vector, scalar, GKOACGFactory>(fieldName, matrix,
solverControls){};
//- Destructor
virtual ~GKOACG(){};
// Member Functions
//- Solve the matrix with this solver
virtual SolverPerformance<vector> solve(Field<vector> &psi) const
{
return solve_impl(psi);
}
};
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
} // End namespace Foam
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
#endif
// ************************************************************************* //
......@@ -26,10 +26,8 @@ License
#include <ginkgo/ginkgo.hpp>
#include <map>
#include <type_traits>
#include "GKOBase.H"
#include "GKOLduBase.H"
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
// ************************************************************************* //
/*---------------------------------------------------------------------------*\
License
This file is part of OGL.
OGL is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
Class
Foam::GKOCG
Author: Gregor Olenik <go@hpsim.de>
SourceFiles
GKOCG.C
\*---------------------------------------------------------------------------*/
#ifndef GKO_LDUBase_H
#define GKO_LDUBase_H
#include "../common/common.H"
#include "LduMatrix.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
namespace Foam {
/*---------------------------------------------------------------------------*\
Class GKOCG Declaration
\*---------------------------------------------------------------------------*/
template <class Type, class LDUType, class SolverFactory>
class GKOLduBaseSolver : public LduMatrix<Type, LDUType, LDUType>::solver,
public SolverFactory,
public lduLduCommonBase {
public:
// Some shortcuts
using IndexType = label;
using vec = gko::matrix::Dense<scalar>;
using mtx = gko::matrix::Coo<scalar>;
using val_array = gko::Array<scalar>;
using idx_array = gko::Array<label>;
using cg = gko::solver::Cg<scalar>;
private:
// for now a copy of the OF matrix is stored
// to keep values contiguous in memory
// executor where Ginkgo will perform the computation
public:
std::vector<scalar> sorting_idxs_;
mutable std::vector<scalar> values_;
mutable std::vector<label> col_idxs_;
mutable std::vector<label> row_idxs_;
public:
//- Construct from matrix components and solver controls
GKOLduBaseSolver(const word &fieldName,
const LduMatrix<Type, LDUType, LDUType> &matrix,
const dictionary &solverControls)
: LduMatrix<Type, LDUType, LDUType>::solver(fieldName, matrix,
solverControls),
SolverFactory(this->controlDict_),
lduLduCommonBase(matrix.mesh().thisDb(), matrix.diag().size(),
matrix.lduAddr().upperAddr().size(), solverControls)
{
// create executors
if (get_update_sys_matrix()) {
SIMPLE_TIME(update_gko_mtx, update_GKOMatrix();)
}
// if (!get_is_sorted()) {
// SIMPLE_TIME(compute_sort, compute_sorting_idxs();)
// }
// if (get_sort()) {
// SIMPLE_TIME(sort_gko_mtx, sort_GKOMatrix();)
// }
}
const std::vector<scalar> &vals() const { return values_; };
const std::vector<label> &col_idx() const { return col_idxs_; };
const std::vector<label> &row_idx() const { return row_idxs_; };
// NOTE move to base lduLduBase
void update_GKOMatrix() const
{
// reset vectors
values_.resize(0);
col_idxs_.resize(0);
row_idxs_.resize(0);
values_.reserve(nElems());
col_idxs_.reserve(nElems());
row_idxs_.reserve(nElems());
// fill vectors unsorted
for (IndexType i = 0; i < nNeighbours(); ++i) {
values_.push_back(this->matrix_.lower()[i]);
row_idxs_.push_back(this->matrix_.lduAddr().lowerAddr()[i]);
col_idxs_.push_back(this->matrix_.lduAddr().upperAddr()[i]);
}
for (IndexType i = 0; i < nCells(); ++i) {
values_.push_back(this->matrix_.diag()[i]);
col_idxs_.push_back(i);
row_idxs_.push_back(i);
}
for (IndexType i = 0; i < nNeighbours(); ++i) {
values_.push_back(this->matrix_.upper()[i]);
row_idxs_.push_back(this->matrix_.lduAddr().upperAddr()[i]);
col_idxs_.push_back(this->matrix_.lduAddr().lowerAddr()[i]);
}
};
scalar compute_residual(mtx *const A, vec *const x, vec *const b) const
{
auto one = gko::initialize<vec>({1.0}, exec());
auto neg_one = gko::initialize<vec>({-1.0}, exec());
auto res = gko::initialize<vec>({0.0}, exec());
A->apply(lend(one), lend(x), lend(neg_one), lend(b));
b->compute_norm2(lend(res));