Commit fe527718 authored by adelmann's avatar adelmann 🎗
Browse files

First stab to integrate opt-pilot

parent 430fd691

Too many changes to show.

To preserve performance only 1000 of 1000+ files are displayed.
......@@ -229,7 +229,11 @@ ELSE()
ENDIF()
SET (IPPL_USED_FROM_OPAL "TRUE")
SET (OPTP_USED_FROM_OPAL "TRUE")
ADD_SUBDIRECTORY (ippl)
ADD_SUBDIRECTORY (opt-pilot)
ADD_SUBDIRECTORY (src)
ADD_SUBDIRECTORY (tools)
ADD_SUBDIRECTORY (doc/user_guide)
......@@ -257,4 +261,4 @@ install (
FILES ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config_install.cmake
DESTINATION "${CMAKE_INSTALL_PREFIX}/lib/cmake/${PROJECT_NAME}"
RENAME ${PROJECT_NAME}Config.cmake
)
\ No newline at end of file
)
IF (NOT OPTP_USED_FROM_OPAL)
cmake_minimum_required (VERSION 2.8.2)
set (CMAKE_DISABLE_IN_SOURCE_BUILD ON)
project (opt-pilot)
set (PACKAGE \"OPT-PILOT\")
set (PACKAGE_VERSION \"0.1.4\")
ENDIF (NOT OPTP_USED_FROM_OPAL)
#-----------------------------------------------------------------------------
# config.h
IF(EXISTS "${CMAKE_SOURCE_DIR}/.git")
set (GIT_REVISION 0)
EXECUTE_PROCESS(
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
COMMAND git describe
OUTPUT_VARIABLE GIT_OUT OUTPUT_STRIP_TRAILING_WHITESPACE
)
set (GIT_REVISION \"${GIT_OUT}\")
ELSE(EXISTS "${CMAKE_SOURCE_DIR}/.git")
set (GIT_REVISION 0)
ENDIF(EXISTS "${CMAKE_SOURCE_DIR}/.git")
set (BUILD_DATE 0)
EXECUTE_PROCESS(
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
COMMAND date
OUTPUT_VARIABLE DATE_OUT OUTPUT_STRIP_TRAILING_WHITESPACE
)
set (BUILD_DATE \"${DATE_OUT}\")
configure_file(config.h.in ${CMAKE_SOURCE_DIR}/config.h)
#-----------------------------------------------------------------------------
# Flags
if (NOT CMAKE_CONFIGURATION_TYPES AND NOT CMAKE_BUILD_TYPE)
SET (CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING
"Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel."
FORCE
)
endif (NOT CMAKE_CONFIGURATION_TYPES AND NOT CMAKE_BUILD_TYPE)
message (STATUS "Build type is: " ${CMAKE_BUILD_TYPE})
# Select flags.
set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g -Wno-reorder -Wno-enum-compare")
set (CMAKE_CXX_FLAGS_RELEASE "-O2 -Wno-reorder -Wno-enum-compare")
set (CMAKE_CXX_FLAGS_DEBUG "-O0 -g -Wno-reorder -Wno-enum-compare")
message (STATUS "Building on ${CMAKE_SYSTEM_NAME} ${CMAKE_SYSTEM_PROCESSOR} on $ENV{HOSTNAME}")
#-----------------------------------------------------------------------------
# dependent packages
find_package (MPI REQUIRED)
include_directories(SYSTEM ${MPI_INCLUDE_PATH})
#-----------------------------------------------------------------------------
set (CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/CMakeModules")
IF (NOT OPTP_USED_FROM_OPAL)
find_package (OPAL REQUIRED)
find_package (IPPL REQUIRED)
find_package (H5hut REQUIRED)
find_package (HDF5 REQUIRED)
find_package (GSL REQUIRED)
find_package (Trilinos REQUIRED HINTS $ENV{TRILINOS_PREFIX} $ENV{TRILINOS_DIR} $ENV{TRILINOS})
ENDIF (NOT OPTP_USED_FROM_OPAL)
set(BOOST_ROOT $ENV{BOOST_HOME})
set(Boost_USE_STATIC_LIBS ON)
set(BOOST_NO_SYSTEM_PATHS TRUE)
set(BOOST_INCLUDEDIR $ENV{BOOST_HOME})
set(BOOST_LIBRARYDIR $ENV{BOOST_HOME}}/lib)
find_package (Boost 1.55.0 REQUIRED chrono filesystem regex serialization system iostreams)
if (Boost_INCLUDE_DIRS)
message (STATUS "Boost include dirs: ${Boost_INCLUDE_DIRS}")
include_directories (${Boost_INCLUDE_DIRS})
endif ()
if (Boost_FILESYSTEM_FOUND)
add_definitions(-DBOOST_FILESYSTEM)
endif (Boost_FILESYSTEM_FOUND)
### Uldis Locans 2014.09.05 ###
OPTION (ENABLE_DKS "Enable DKS" OFF)
### Load DKS package ###
IF (ENABLE_DKS)
FIND_PACKAGE(DKS 1.1.1 REQUIRED HINTS $ENV{DKS_PREFIX} $ENV{DKS_DIR} $ENV{DKS})
MESSAGE (STATUS "DKS Version ${DKS_VERSION}")
MESSAGE (STATUS "Found DKS library: ${DKS_LIBRARY}")
MESSAGE (STATUS "Found DKS include dir: ${DKS_INCLUDE_DIR}")
SET (DKS_PACKAGE_VERSION "\"${DKS_VERSION}\"")
SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DDKS_MPI")
### OpenCL compiler flags ###
SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -lOpenCL -pthread -DDKS_OPENCL")
### CUDA compiler flags ###
SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -lcudart -lcufft -lcublas -lnvToolsExt -DDKS_CUDA")
### if any accelerator enabled set flag to use DKS ###
SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DIPPL_DKS -DIPPL_DKS_CUDA -DOPAL_DKS")
ENDIF (ENABLE_DKS)
### END DKS part ###
#-----------------------------------------------------------------------------
# options
OPTION (ARCH_BG "Set architecture to BlueGene" OFF)
#-----------------------------------------------------------------------------
# Common sources and libs
# add_sources macro implemented by Christof Kraus
# for all common source files (Expressions and Utils)
set (_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR})
macro (add_sources)
file (RELATIVE_PATH _relPath "${_SRC_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}")
foreach (_src ${ARGN})
if (_relPath)
list (APPEND OPTP_SRCS "${CMAKE_SOURCE_DIR}/${_relPath}/${_src}")
else()
list (APPEND OPTP_SRCS "${CMAKE_SOURCE_DIR}/${_src}")
endif()
endforeach()
if (_relPath)
# propagate SRCS to parent directory
set (OPTP_SRCS ${OPTP_SRCS} PARENT_SCOPE)
endif()
endmacro()
set (OPTP_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}" PARENT_SCOPE)
SET (OPTP_USED_FROM_OPAL "TRUE")
add_subdirectory (Expression)
add_subdirectory (Util)
add_subdirectory (extlib)
set (PILOT_LIBS
pthread
m
boost_chrono
boost_regex
boost_system
boost_serialization
# nlopt
Problems
)
IF (NOT OPTP_USED_FROM_OPAL)
#-----------------------------------------------------------------------------
# Executables
add_subdirectory (Drivers)
#-----------------------------------------------------------------------------
# Testing
enable_testing()
add_subdirectory(Tests)
ENDIF (NOT OPTP_USED_FROM_OPAL)
#-----------------------------------------------------------------------------
# Doxygen
find_package(Doxygen)
if (DOXYGEN_FOUND)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile
${CMAKE_CURRENT_BINARY_DIR}/Doxyfile @ONLY)
add_custom_target(doc
${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
COMMENT "Generating API documentation with Doxygen" VERBATIM
)
endif(DOXYGEN_FOUND)
#
# Find CLASSIC includes and library
# CLASSIC_INCLUDE_DIR - where to find opal.h
# CLASSIC_LIBRARY - qualified libraries to link against.
# CLASSIC_FOUND - do not attempt to use if "no" or undefined.
find_path (CLASSIC_INCLUDE_DIR opal.h
HINTS $ENV{OPAL_DIR}/include $ENV{OPAL_HOME}/include ENV OPAL_INCLUDE_DIR
PATHS ENV C_INCLUDE_PATH CPLUS_INCLUDE_PATH
)
find_library (CLASSIC_LIBRARY CLASSIC
HINTS $ENV{OPAL_DIR}/lib $ENV{OPAL_HOME}/lib ENV OPAL_LIBRARY_DIR
PATHS ENV LIBRARY_PATH
)
find_package_handle_standard_args (CLASSIC REQUIRED_VARS CLASSIC_INCLUDE_DIR)
find_package_handle_standard_args (CLASSIC REQUIRED_VARS CLASSIC_LIBRARY)
\ No newline at end of file
#
# Find GSL includes and library
#
# GSL
# It can be found at:
# http://amas.web.psi.ch/tools/GSL/index.html
#
# GSL_INCLUDE_DIR - where to find ippl.h
# GSL_LIBRARY - qualified libraries to link against.
# GSL_FOUND - do not attempt to use if "no" or undefined.
find_path (GSL_INCLUDE_DIR gsl/gsl_fft.h
PATHS
$ENV{GSL_DIR}/include
$ENV{GSL_HOME}/include
$ENV{GSL_PREFIX}/include
ENV GSL_INLCUDE_DIR
NO_DEFAULT_PATH
)
find_library (GSL_LIBRARY gsl
PATHS
$ENV{GSL_DIR}/lib
$ENV{GSL_HOME}/lib
$ENV{GSL_PREFIX}/lib
ENV GSL_LIBRARY_DIR LIBRARY_PATH
NO_DEFAULT_PATH
)
find_library (GSL_LIBRARY_CBLAS gslcblas
PATHS
$ENV{GSL_DIR}/lib
$ENV{GSL_HOME}/lib
$ENV{GSL_PREFIX}/lib
ENV GSL_LIBRARY_DIR LIBRARY_PATH
NO_DEFAULT_PATH
)
find_package_handle_standard_args (GSL REQUIRED_VARS GSL_INCLUDE_DIR)
find_package_handle_standard_args (GSL REQUIRED_VARS GSL_LIBRARY)
find_package_handle_standard_args (GSL REQUIRED_VARS GSL_LIBRARY_CBLAS)
set (GSL_LIBRARY ${GSL_LIBRARY} ${GSL_LIBRARY_CBLAS})
\ No newline at end of file
#
# Find H5hut includes and library
#
# H5hut
# It can be found at:
# http://amas.web.psi.ch/tools/H5hut/index.html
#
# H5HUT_INCLUDE_DIR - where to find H5hut.h
# H5HUT_LIBRARY - qualified libraries to link against.
# H5HUT_FOUND - do not attempt to use if "no" or undefined.
find_path (H5HUT_INCLUDE_DIR H5hut.h
PATHS
$ENV{H5HUT_DIR}/include
$ENV{H5HUT_HOME}/include
$ENV{H5HUT_PREFIX}/include
ENV H5HUT_INCLUDE_DIR C_INCLUDE_PATH
NO_DEFAULT_PATH
)
find_library (H5HUT_LIBRARY H5hut
PATHS
$ENV{H5HUT_DIR}/lib
$ENV{H5HUT_HOME}/lib
$ENV{H5HUT_PREFIX}/lib
ENV H5HUT_LIBRARY_DIR LIBRARY_PATH
NO_DEFAULT_PATH
)
find_package_handle_standard_args (H5hut REQUIRED_VARS H5HUT_INCLUDE_DIR)
find_package_handle_standard_args (H5hut REQUIRED_VARS H5HUT_LIBRARY)
#
# Find HDF5 includes and library
#
# HDF5
# It can be found at:
# http://amas.web.psi.ch/tools/HDF5/index.html
#
# HDF5_INCLUDE_DIR - where to find hdf5.h
# HDF5_LIBRARY - qualified libraries to link against.
# HDF5_FOUND - do not attempt to use if "no" or undefined.
find_path (HDF5_INCLUDE_DIR hdf5.h
PATHS
$ENV{HDF5_DIR}/include
$ENV{HDF5_HOME}/include
$ENV{HDF5_PREFIX}/include
ENV HDF5_INCLUDE_DIR C_INCLUDE_PATH
NO_DEFAULT_PATH
)
find_library (HDF5_LIBRARY hdf5
PATHS
$ENV{HDF5_DIR}/lib
$ENV{HDF5_HOME}/lib
$ENV{HDF5_PREFIX}/lib
ENV HDF5_LIBRARY_DIR LIBRARY_PATH
NO_DEFAULT_PATH
)
find_package_handle_standard_args (hdf5 REQUIRED_VARS HDF5_INCLUDE_DIR)
find_package_handle_standard_args (hdf5 REQUIRED_VARS HDF5_LIBRARY)
#
# Find IPPL includes and library
#
# IPPL_INCLUDE_DIR - where to find ippl.h
# IPPL_LIBRARY - qualified libraries to link against.
# IPPL_FOUND - do not attempt to use if "no" or undefined.
FIND_PATH(IPPL_INCLUDE_DIR Ippl.h
PATHS
$ENV{OPAL_DIR}/include
$ENV{OPAL_HOME}/include
ENV OPAL_INCLUDE_DIR C_INCLUDE_PATH CPLUS_INCLUDE_PATH
NO_DEFAULT_PATH
)
FIND_LIBRARY(IPPL_LIBRARY ippl
PATHS
$ENV{OPAL_DIR}/lib
$ENV{OPAL_HOME}/lib
ENV OPAL_LIBRARY_DIR LIBRARY_PATH
NO_DEFAULT_PATH
)
find_package_handle_standard_args (IPPL REQUIRED_VARS IPPL_INCLUDE_DIR)
find_package_handle_standard_args (IPPL REQUIRED_VARS IPPL_LIBRARY)
set (IPPL_CMAKE_CXX_FLAGS "-DIPPL_LINUX -DIPPL_STRINGSTREAM -DIPPL_MPI -DMPICH_SKIP_MPICXX -DIPPL_GCC -DIPPL_DONT_POOL -DIPPL_USE_XDIV_RNG -DIPPL_NO_STRINGSTREAM -DPETE_BITWISE_COPY -DIPPL_HAS_TEMPLATED_COMPLEX -DIPPL_USE_STANDARD_HEADERS -DIPPL_USE_PARTIAL_SPECIALIZATION -DIPPL_STDSTL -DIPPL_LONGLONG -DWITH_BRICK -DnoCOMP_GNUOLD -Drestrict=__restrict__ -DNOCTAssert ")
#
# Find OPAL includes and library
#
# OPAL (Object Oriented Parallel Accelerator Library) is a tool for
# charged-particle optic calculations in large accelerator structures and
# beam lines including 3D space charge. OPAL is built from first principles
# as a parallel application, OPAL admits simulations of any scale: on the
# laptop and up to the largest High Performance Computing (HPC) clusters
# available today. Simulations, in particular HPC simulations, form the
# third pillar of science, complementing theory and experiment.
# OPAL includes various beam line element descriptions and methods for single
# particle optics, namely maps up to arbitrary order, symplectic integration
# schemes and lastly time integration. OPAL is based on IPPL (Independent
# Parallel Particle Layer) which adds parallel capabilities. Main functions
# inherited from IPPL are: structured rectangular grids, fields and parallel
# FFT and particles with the respective interpolation operators. Other
# features are, expression templates and massive parallelism (up to 8000
# processors) which makes is possible to tackle the largest problems in the
# field.
# It can be found at:
# http://amas.web.psi.ch/docs/index.html
#
# OPAL_INCLUDE_DIR - where to find opal.h
# OPAL_LIBRARY - qualified libraries to link against.
# OPAL_FOUND - do not attempt to use if "no" or undefined.
find_path (OPAL_INCLUDE_DIR opal.h
PATHS
$ENV{OPAL_DIR}/include
$ENV{OPAL_HOME}/include
$ENV{OPAL_PREFIX}/include
ENV OPAL_INCLUDE_DIR C_INCLUDE_PATH CPLUS_INCLUDE_PATH
NO_DEFAULT_PATH
)
find_library (OPAL_LIBRARY OPAL
PATHS
$ENV{OPAL_DIR}/lib
$ENV{OPAL_HOME}/lib
$ENV{OPAL_PREFIX}/lib
ENV OPAL_LIBRARY_DIR LIBRARY_PATH
NO_DEFAULT_PATH
)
find_package_handle_standard_args (OPAL REQUIRED_VARS OPAL_INCLUDE_DIR)
find_package_handle_standard_args (OPAL REQUIRED_VARS OPAL_LIBRARY)
if (OPAL_FOUND)
set (OPAL_INCLUDE_DIR
${OPAL_INCLUDE_DIR}
${OPAL_INCLUDE_DIR}/Classic
)
endif ()
#ifndef __COMM_SPLITTER__
#define __COMM_SPLITTER__
#include "mpi.h"
#include "Comm/types.h"
#include "Util/Types.h"
#include "Util/CmdArguments.h"
#include "Util/OptPilotException.h"
//TODO: what is the performance difference between using MPI_COMM_WORLD and
// p2p communication vs. communicator groups??
/**
* \brief Role assignment according to strategy (that might use hardware
* network information).
*
* The CommSplitter splits the passed (usually MPI_COMM_WORLD) communicator
* into several comm groups using the colors provided by the splitting
* strategy.
* After construction each processor has an assigned role (optimizer, worker
* or pilot) and a set of communicators to send and receive tasks.
* The expected colors have the following meaning:
*
* - color[0] is shared by all my co-workers
* - color[1] is shared by the optimizer leader and the pilot
* - color[2] is shared by the worker leader and the pilot
* - color[3] is shared by all processors with the same role (used for
* broadcasts)
*
*/
template< class Strategy_t >
class CommSplitter : public Strategy_t {
public:
CommSplitter(CmdArguments_t args, MPI_Comm comm = MPI_COMM_WORLD)
: Strategy_t(args, comm)
, world_comm_(comm)
{
MPI_Comm_group(world_comm_, &world_group_);
MPI_Comm_rank(world_comm_, &global_rank_);
// the splitter strategy computes the colorings
Strategy_t::split();
my_opt_comm_ = MPI_COMM_NULL;
my_worker_comm_ = MPI_COMM_NULL;
my_coworker_comm_ = MPI_COMM_NULL;
MPI_Comm_split(world_comm_, Strategy_t::colorings_[0],
global_rank_, &my_coworker_comm_ );
MPI_Comm_split(world_comm_, Strategy_t::colorings_[1],
global_rank_, &my_opt_comm_ );
MPI_Comm_split(world_comm_, Strategy_t::colorings_[2],
global_rank_, &my_worker_comm_ );
MPI_Comm_split(world_comm_, Strategy_t::colorings_[3],
global_rank_, &my_comm_world_ );
// just a precaution to make sure everybody is ready
MPI_Barrier(world_comm_);
}
virtual ~CommSplitter()
{
MPI_Group_free(&world_group_);
if(world_comm_ != MPI_COMM_WORLD)
MPI_Comm_free(&world_comm_);
}
bool isOptimizer() const { return Strategy_t::role_ == OPTIMIZER; }
bool isWorker() const { return Strategy_t::role_ == WORKER; }
bool isPilot() const { return Strategy_t::role_ == POLLER; }
MPI_Comm mpiComm() const { return world_comm_; }
int globalRank() const { return global_rank_; }
int PilotRank() const { return Strategy_t::poller_; }
int getLeader() const { return Strategy_t::leader_; }
/// construct comm bundle and return
Comm::Bundle_t getBundle() const {
Comm::Bundle_t bundle;
bundle.island_id = Strategy_t::group_id_;
bundle.leader_pid = Strategy_t::leader_;
bundle.master_pid = Strategy_t::poller_;
//FIXME: is it always 0?
bundle.master_local_pid = 0; //poller_local_pid_;
bundle.worker = my_worker_comm_;
bundle.opt = my_opt_comm_;
bundle.coworkers = my_coworker_comm_;
bundle.world = my_comm_world_;
return bundle;
}
private:
MPI_Comm world_comm_;
MPI_Group world_group_;
/// global MPI PID
int global_rank_;
// define various communicators for communication between components
MPI_Comm my_worker_comm_;
MPI_Comm my_opt_comm_;
MPI_Comm my_coworker_comm_;
MPI_Comm my_comm_world_;
/// local (wrt. the communicator group) rank of the master/pilot process
int poller_local_pid_;
};
#endif
#ifndef __NDIM_MESH__
#define __NDIM_MESH__
#include <set>
//FIXME:
//#include "Mesh.hpp"
template < class TopoDiscoveryStrategy_t >
class NDimMesh : public TopoDiscoveryStrategy_t {
public:
std::set<size_t> execute(size_t numMasters, size_t dimensions, size_t id,
int island_id) {
return Mesh::Simplex::getNeighborIDs(island_id);
}
};
#endif
#ifndef __NO_MASTER_GRAPH__
#define __NO_MASTER_GRAPH__
#include <set>
/// A simple empty master graph (no neighbors, every island works isolated).
template < class TopoDiscoveryStrategy_t >
class NoMasterGraph : public TopoDiscoveryStrategy_t {
public:
std::set<size_t> execute(size_t numMasters, size_t dimensions, size_t id,
int island_id) {
std::set<size_t> empty;
return empty;
}
};
#endif
#ifndef __SOCIAL_NETWORK_GRAPH__
#define __SOCIAL_NETWORK_GRAPH__
#include <set>
#include <cstdint>
#include <boost/random/mersenne_twister.hpp>
#include <boost/random/discrete_distribution.hpp>
// Here, we use a simple mapping from processor ID's to 2D grid
// (determining neigborhood) assuming the total number of masters:
//
// n_m = n_g * n_g
//
// This is fine, because if we have topology info the masters are numbered
// in ascending order (done in comm splitter) and we can map them from 1D to
// 2D by index calculations.
/**
* \brief Modeling social graph (Cartesian neighbors plus additional random
* link) as underlaying master network.
*
* @see MasterNode
*
* Due to its nice rumor spreading properties this master network is mimicking
* a social network graph (augmented grid) and solution states are distributed
* in a rumor fashion.
*/
template < class TopoDiscoveryStrategy_t >
class SocialNetworkGraph : public TopoDiscoveryStrategy_t {
public:
std::set<size_t> execute(size_t numMasters, size_t dimensions, size_t id,
int island_id) {
numMasters_ = numMasters;
dim_ = dimensions;
myID_ = id;
chooseRandomNeighbor();