.clang-format

+---
+Language:        Cpp
+BasedOnStyle:  Google
+AccessModifierOffset: -4
+AlignAfterOpenBracket: AlwaysBreak
+AlignConsecutiveAssignments: true
+AllowShortFunctionsOnASingleLine: None
+AllowShortIfStatementsOnASingleLine: Never
+AllowShortLambdasOnASingleLine: Empty
+AllowShortLoopsOnASingleLine: false
+BreakBeforeBinaryOperators: NonAssignment
+ColumnLimit:     100
+DerivePointerAlignment: false
+IndentWidth:     4
+IncludeBlocks: Preserve
+IndentGotoLabels: false
+NamespaceIndentation: All 
+...

.gitignore

@@ -2,7 +2,9 @@
 *~
 *.o
 *.a
+*.so
 *.aux
+__pycache__
 optimizer/Tests/*.exe
 build
 CMakeCache.txt

.gitlab/issue_templates/Release.md

+**source code and binary**
+  * [ ] create branch YEAR.N
+  * [ ] create issue "Release version YEAR.N" and merge request
+  * [ ] update version string in Doxyfile
+  * [ ] update the version string in CMakeLists.txt and commit
+  * [ ] wait for approval of MR and merge
+  * [ ] tag version YEAR.N.0
+  * [ ] upload source tar-ball to `/afs/psi.ch/project/amas/webhosting/Downloads/OPAL/src`
+  * [ ] compile new binary for Linux
+  * [ ] upload Linux binary package to `/afs/psi.ch/project/amas/webhosting/Downloads/OPAL/package/`
+  * [ ] compile new binary for macOS
+  * [ ] upload macOS binary package to `/afs/psi.ch/project/amas/webhosting/Downloads/OPAL/package/`
+
+**manual/documentation**
+  * [ ] setup a new branch for the new version of the manual
+  * [ ] fix version, branches and links in `Manual.attributes`
+  * [ ] clone repository into `/afs/psi.ch/project/amas/webhosting/opal/Documentation/x.y` and checkout new branch (`git clone https://gitlab.psi.ch/OPAL/documentation/manual.git`)
+  * [ ] add links to the binaries in the wiki
+  * [ ] update https://gitlab.psi.ch/OPAL/src/wikis/For-Developers/Compile-OPAL
+  * [ ] compile the change log/release notes and publish it in the wiki: https://gitlab.psi.ch/OPAL/src/wikis/ReleaseNotes
+  * [ ] review the file `src/addToDoxygenMainPage.h`
+  * [ ] build Doxygen documentation
+  * [ ] update https://gitlab.psi.ch/OPAL/src/wikis/home
+  * [ ] update https://gitlab.psi.ch/OPAL/src/wikis/regression-tests
+
+**tracker**
+  * [ ] new milestone for `OPAL x.(y+1)`
+  * [ ] update labels and milestones in issues
+
+**regression-tests**
+
+  * [ ] create new branch x.y
+  * [ ] setup the regression-tests to run the new version on opalrunner.psi.ch
+
+**varia**
+  * [ ] PSI module
+  * [ ] write e-mail to mailing list

CMakeLists.txt

-cmake_minimum_required (VERSION 3.1)
-project (OPAL VERSION 2.3.0)
+cmake_minimum_required (VERSION 3.12)
+cmake_policy(VERSION 3.12)
+
+project (OPAL VERSION 2024.2)
 set (PROJECT_BUGREPORT opal@lists.psi.ch)
 
 string (REGEX REPLACE "\\.([0-9])\\." ".0\\1." PACKAGE_VERSION ${PROJECT_VERSION})
@@ -26,6 +28,13 @@ set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -g ")
 set (CMAKE_CXX_FLAGS_RELEASE "-O3")
 set (CMAKE_CXX_FLAGS_DEBUG  "-O0 -g")
 
+string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE)
+if (uppercase_CMAKE_BUILD_TYPE MATCHES "^(DEBUG|RELWITHDEBINFO)$")
+    option (BUILD_FOR_PRODUCTION "Disable all assertion checks; Only use for production" OFF)
+else()
+    option (BUILD_FOR_PRODUCTION "Disable all assertion checks; Only use for production" ON)
+endif()
+
 # Resolve all library dependencies
 set (CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/CMakeModules")
 
@@ -37,7 +46,7 @@ message (STATUS "The underlying C++ compiler is: ${CMAKE_CXX_COMPILER}")
 
 option (ENABLE_OpenMP "Use hybrid parallelism MPI-OpenMP" OFF)
 
-set (CMAKE_CXX_STANDARD 11)
+set (CMAKE_CXX_STANDARD 17)
 set (CMAKE_CXX_STANDARD_REQUIRED ON)
 set (CMAKE_CXX_EXTENSIONS OFF)
 
@@ -52,15 +61,27 @@ add_compile_options (-Werror)
 add_compile_options (-funroll-loops)
 add_compile_options (-fstrict-aliasing)
 
-# An error in Trilinos/12.18.1 causes
-# -Waggressive-loop-optimizations with -O3 optimization.
-# See issue 587 and corresponding merge request 415.
-add_compile_options (-fno-aggressive-loop-optimizations)
+option (WILL_BUILD_STATIC_LIBRARY "Building static library" ON)
+option (BUILD_OPAL_PYTHON "Building Python interface for OPAL" OFF)
+option (WILL_BUILD_SHARED_LIBRARY "Building python hooks - requires dynamic libs" OFF)
+if (BUILD_OPAL_PYTHON)
+    if (NOT WILL_BUILD_SHARED_LIBRARY)
+        message (FATAL_ERROR "For building python hooks you have to enable WILL_BUILD_SHARED_LIBRARY")
+    endif()
+    find_package (Python3 COMPONENTS Development REQUIRED)
+endif ()
 
-option (USE_STATIC_LIBRARIES "Link with static libraries if available" ON)
-if (NOT ${USE_STATIC_LIBRARIES})
+if (${WILL_BUILD_SHARED_LIBRARY})
     add_compile_options (-fPIE -fPIC)
     add_link_options (-pie)
+    # force OPAL to link to shared libraries
+    if (UNIX)
+        set(CMAKE_FIND_LIBRARY_SUFFIXES ".so")
+    elseif (APPLE)
+        set(CMAKE_FIND_LIBRARY_SUFFIXES ".dylib")
+    else()
+        MESSAGE(FATAL_ERROR "Operating system not APPLE or UNIX - quitting")
+    endif()
 endif ()
 
 # compiler dependent flags
@@ -81,16 +102,28 @@ elseif (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
     # is using this visibility setting.
     add_compile_options (-fvisibility=hidden)
     add_compile_options (-fvisibility-inlines-hidden)
-    # Allow overloaded virtual functions (instances to be fixed in OPAL)
-    #add_compile_options (-Wno-overloaded-virtual)
+    add_compile_options (-Wno-deprecated-declarations)
+    add_compile_options (-Wno-deprecated-builtins)
+    add_compile_options (-Wno-deprecated-copy)
     if (ENABLE_OpenMP)
-        add_compile_options (-fopenmp)
-        add_link_options (-fopenmp)
+        if (CMAKE_HOST_APPLE)
+            message(FATAL_ERROR "Apple Clang does not support OpenMP!")
+        else ()
+            add_compile_options (-fopenmp)
+            add_link_options (-fopenmp)
+        endif ()
     endif ()
 
 elseif (${CMAKE_CXX_COMPILER_ID} MATCHES "GNU")
-    if (${CMAKE_CXX_COMPILER_VERSION} VERSION_LESS "4.7.3")
-        message (FATAL_ERROR "To build OPAL you need GCC version 4.7.3 or greater")
+    # An error in Trilinos/12.18.1 causes
+    # -Waggressive-loop-optimizations with -O3 optimization.
+    # See issue 587 and corresponding merge request 415.
+    # Clang does not support this flag!
+    add_compile_options (-fno-aggressive-loop-optimizations)
+    add_compile_options (-Wno-error=cpp)
+
+    if (${CMAKE_CXX_COMPILER_VERSION} VERSION_LESS "9.5.0")
+        message (FATAL_ERROR "To build OPAL you need GCC version 9.5.0 or greater")
     endif ()
     # Warning flags
 
@@ -99,11 +132,16 @@ elseif (${CMAKE_CXX_COMPILER_ID} MATCHES "GNU")
         add_link_options (-fopenmp)
     endif ()
 
+    # Enables extra error checking in the form of precondition assertion, such
+    # as bounds checking and null pointer checks when dereferencing smart pointers
+    if (NOT ${BUILD_FOR_PRODUCTION})
+        add_compile_options(-Wp,-D_GLIBCXX_ASSERTIONS)
+    endif()
+    enable_language (Fortran OPTIONAL)
 else ()
     message (STATUS "Unknown C++ compiler. Please use the GNU or Intel compiler, if you are having problems.")
 endif ()
 
-enable_language (Fortran OPTIONAL)
 message (STATUS "Fortran compilers: ${CMAKE_Fortran_COMPILER_NAMES}")
 if (CMAKE_Fortran_COMPILER)
     message (STATUS "The Fortran compiler identification is: ${CMAKE_Fortran_COMPILER_ID}")
@@ -112,9 +150,6 @@ if (CMAKE_Fortran_COMPILER)
     if (${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU")
         set (Fortran_LIBS "-lgfortran")
     endif ()
-    # CMake issue. Missing library libmpi_mpifh in MPI_Fortran_LINRARIES.
-    # See issue 587 and corresponding merge request 415.
-    set (Fortran_LIBS "${Fortran_LIBS} -lmpi_mpifh")
 else ()
     message (STATUS "No Fortran compiler found (optional)")
     if (ENABLE_AMR)
@@ -124,25 +159,33 @@ endif ()
 
 find_package (H5Hut REQUIRED)
 
-if (USE_STATIC_LIBRARIES)
+if (NOT ${WILL_BUILD_SHARED_LIBRARY})
     set (HDF5_USE_STATIC_LIBRARIES ON)
 endif ()
+set (HDF5_PREFER_PARALLEL ON)
 find_package (HDF5 REQUIRED)
-message (STATUS "Found HDF5 libraries: ${HDF5_C_LIBRARIES}")
+if(UNIX AND NOT APPLE)
+    set (HDF5_LIBRARIES ${HDF5_LIBRARIES} ${CMAKE_DL_LIBS} rt z)
+endif ()
+message (STATUS "Found HDF5 libraries: ${HDF5_LIBRARIES}")
 
 find_package (GSL REQUIRED)
 message (STATUS "Found gsl libraries: ${GSL_LIBRARIES}")
 
 set (BOOSTROOT $ENV{BOOST_DIR})
-if (USE_STATIC_LIBRARIES)
+if (NOT ${WILL_BUILD_SHARED_LIBRARY})
     set (Boost_USE_STATIC_LIBS ON)
 endif ()
 set (Boost_USE_MULTITHREADED OFF)
 set (Boost_USE_STATIC_RUNTIME OFF)
+set (Boost_LIBS filesystem iostreams regex serialization system timer)
 find_package (Boost 1.66.0
-    REQUIRED COMPONENTS chrono filesystem iostreams regex serialization system timer)
-message (STATUS "Found Boost include dir: ${Boost_INCLUDE_DIR}")
-message (STATUS "Found Boost library dir: ${Boost_LIBRARY_DIR}")
+    REQUIRED COMPONENTS ${Boost_LIBS})
+if(UNIX AND NOT APPLE)
+    set (Boost_LIBRARIES ${Boost_LIBRARIES} rt)
+endif ()
+message (STATUS "Found Boost include dirs: ${Boost_INCLUDE_DIRS}")
+message (STATUS "Found Boost library dirs: ${Boost_LIBRARY_DIRS}")
 message (STATUS "Found Boost libraries: ${Boost_LIBRARIES}")
 include_directories (SYSTEM ${Boost_INCLUDE_DIRS})
 
@@ -208,7 +251,7 @@ option (ENABLE_OPAL_FEL "Enable OPAL FEL" OFF)
 if (ENABLE_OPAL_FEL)
     message (STATUS "Enable OPAL FEL: " ${ENABLE_OPAL_FEL})
     find_package (MITHRA MODULE REQUIRED)
-    add_definitions (-DOPAL_FEL)
+    add_definitions (-DENABLE_OPAL_FEL)
 endif()
 
 option (DBG_SCALARFIELD "Enable dump of scalar field rho_m" OFF)
@@ -230,6 +273,17 @@ if (NO_FIELD_ASSIGN_OPTIMIZATION)
 endif ()
 
 option (ENABLE_DOXYDOC "compile Doxygen documentation" OFF)
+if (ENABLE_DOXYDOC)
+    find_package(Doxygen REQUIRED)
+    configure_file (${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in
+        ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile @ONLY
+	)
+    add_custom_target (doxydoc
+        ${DOXYGEN_EXECUTABLE} ${CMAKE_BINARY_DIR}/Doxyfile
+        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+        COMMENT "Generating API documentation with Doxygen" VERBATIM
+        )
+endif ()
 
 add_custom_target(always_rebuild ALL
     DEPENDS
@@ -257,9 +311,6 @@ endif ()
 set (OPAL_CXX_FLAGS ${OPAL_CXX_FLAGS}
     CACHE INTERNAL "" FORCE
     )
-set (OPAL_LIBS ${OPAL_LIBS}
-    CACHE INTERNAL "" FORCE
-    )
 
 # configure file
 configure_file (

CMakeModules/FindMITHRA.cmake

@@ -3,23 +3,47 @@
 # https://github.com/aryafallahi/mithra
 #
 # MITHRA_INCLUDE_DIR
+# MITHRA_LIBRARY_DIR
 # MITHRA_FOUND
 
-find_path (MITHRA_INCLUDE_DIR mithra/classes.hh
-    HINTS $ENV{MITHRA_INCLUDE_PATH} $ENV{MITHRA_INCLUDE_DIR} $ENV{MITHRA_PREFIX}/include $ENV{MITHRA_DIR}/include $ENV{MITHRA}/include
-    PATHS ENV C_INCLUDE_PATH CPLUS_INCLUDE_PATH
+find_path (MITHRA_INCLUDE_DIR mithra/classes.h
+    HINTS $ENV{MITHRA_INCLUDE_DIR} $ENV{MITHRA_INCLUDE_PATH} $ENV{MITHRA_PREFIX}/include $ENV{MITHRA}/include
 )
 
-if (MITHRA_INCLUDE_DIR)
+find_path (MITHRA_LIBRARY_DIR libmithra.a
+    HINTS $ENV{MITHRA_LIBRARY_DIR} $ENV{MITHRA_LIBRARY_PATH} $ENV{MITHRA_PREFIX}/lib $ENV{MITHRA}/lib
+)
+
+if (MITHRA_INCLUDE_DIR AND MITHRA_LIBRARY_DIR)
     set (MITHRA_FOUND "YES")
 endif ()
 
 if (MITHRA_FOUND)
     if (NOT MITHRA_FIND_QUIETLY)
         message (STATUS "Found MITHRA include dir: ${MITHRA_INCLUDE_DIR}")
+        message (STATUS "Found MITHRA library dir: ${MITHRA_LIBRARY_DIR}")
     endif ()
 else (MITHRA_FOUND)
     if (MITHRA_FIND_REQUIRED)
+        if (NOT MITHRA_INCLUDE_DIR)
+            message (WARNING 
+                "MITHRA include directory was not found! "
+                "Make sure that MITHRA is compiled and that "
+                "the directory mithra/include/mithra has been automatically created. "
+                "Also make sure that at least one of the following "
+                "environment variables is set: "
+                "MITHRA_INCLUDE_DIR, MITHRA_INCLUDE_PATH, MITHRA_PREFIX, or MITHRA.")
+        endif ()
+        if (NOT MITHRA_LIBRARY_DIR)
+            message (WARNING 
+                "MITHRA library was not found! "
+                "Make sure that MITHRA is compiled and that "
+                "the directory mithra/lib has been automatically created. "
+                "Also make sure that at least one of the following "
+                "environment variables is set: "
+                "MITHRA_LIBRARY_DIR, MITHRA_LIBRARY_PATH, MITHRA_PREFIX, or MITHRA.")
+        endif ()
+        message (STATUS "MITHRA can be downloaded and compiled from https://github.com/aryafallahi/mithra.git")
         message (FATAL_ERROR "Could not find MITHRA!")
     endif (MITHRA_FIND_REQUIRED)
 endif (MITHRA_FOUND)

Doxyfile → Doxyfile.in

@@ -38,7 +38,7 @@ PROJECT_NAME           = "OPAL (Object Oriented Parallel Accelerator Library)"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = "2.5"
+PROJECT_NUMBER         = "2024.2"
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -764,9 +764,9 @@ WARN_LOGFILE           =
 # spaces.
 # Note: If this tag is empty the current directory is searched.
 
-INPUT                  = ./src \
-                         ./ippl/src \
-                         ./optimizer
+INPUT                  = @CMAKE_SOURCE_DIR@/src \
+                         @CMAKE_SOURCE_DIR@/ippl/src \
+                         @CMAKE_SOURCE_DIR@/optimizer
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
@@ -786,15 +786,8 @@ INPUT_ENCODING         = UTF-8
 # *.md, *.mm, *.dox, *.py, *.f90, *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf,
 # *.qsf, *.as and *.js.
 
-FILE_PATTERNS          = *.c \
-                         *.cpp \
-                         *.cc \
-                         *.h \
-                         *.hh \
-                         *.H \
-                         *.f90 \
-                         *.hpp
-
+FILE_PATTERNS          = *.c* \
+                         *.h*
 
 # The RECURSIVE tag can be used to specify whether or not subdirectories should
 # be searched for input files as well.

ippl/CMakeLists.txt

@@ -6,12 +6,12 @@ set (IPPL_VERSION_NAME "V${IPPL_VERSION_MAJOR}.${IPPL_VERSION_MINOR}")
 
 set (IPPL_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 
-if (${CMAKE_BUILD_TYPE} STREQUAL "Release")
+if (${BUILD_FOR_PRODUCTION})
     add_definitions (-DNOPAssert)
 endif ()
 
 set (IPPL_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/src" PARENT_SCOPE)
-set (IPPL_LIBRARY_DIR "${CMAKE_CURRENT_SOURCE_DIR}/src" PARENT_SCOPE)
+set (IPPL_LIBRARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/src" PARENT_SCOPE)
 set (IPPL_LIBRARY "ippl" PARENT_SCOPE)
 
 add_subdirectory (src)
@@ -28,5 +28,4 @@ install (
     FILES ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config_install.cmake
     DESTINATION "${CMAKE_INSTALL_PREFIX}/lib/cmake/${PROJECT_NAME}"
     RENAME ${PROJECT_NAME}Config.cmake
-    )
-
+    )
\ No newline at end of file

ippl/src/AppTypes/AntiSymTenzor.h

@@ -379,6 +379,10 @@ public:
   // Operators
   
   Element_t operator()(unsigned int i, unsigned int j) const {
+    // PAssert and PAssert_EQ are macros. They are defined empty, if we
+    // compile for production. i and j are unused in this case. The
+    // following statement suppress the 'unused' warning.
+    (void)i; (void)j;
     PAssert_EQ(i, j);
     return T(0.0);
   }
@@ -390,6 +394,7 @@ public:
   }
 
   AssignProxy operator()(unsigned int i, unsigned int j) {
+    (void)i; (void)j;
     PAssert_EQ(i, j);
     return AssignProxy(AntiSymTenzor<T,1>::Zero, 0);
   }
@@ -401,6 +406,7 @@ public:
   }
 
   Element_t operator[](unsigned int i) const { 
+    (void)i;
     PAssert (i == 0);
     return AntiSymTenzor<T,1>::Zero;
   }
@@ -408,6 +414,7 @@ public:
   // These are the same as operator[] but with () instead:
 
   Element_t operator()(unsigned int i) const { 
+    (void)i;
     PAssert (i == 0);
     return AntiSymTenzor<T,1>::Zero;
   }
@@ -430,7 +437,7 @@ public:
   }
 
   Message& getMessage(Message& m) {
-    T zero;
+    T zero{};
     m.get(zero);
     return m;
   }

ippl/src/AppTypes/Tenzor.h

@@ -474,7 +474,6 @@ inline std::ostream& operator<<(std::ostream& out, const Tenzor<T,D>& rhs)
 // include header files for SymTenzor and AntiSymTenzor in order
 // to define constructors for Tenzor using these types
 #include "AppTypes/SymTenzor.h"
-#include "AppTypes/AntiSymTenzor.h"
 
 template <class T, unsigned D>
 Tenzor<T,D>::Tenzor(const SymTenzor<T,D>& rhs) {

ippl/src/DataSource/DataSource.cpp

@@ -2,8 +2,8 @@
 /***************************************************************************
  *
  * The IPPL Framework
- * 
- * This program was prepared by PSI. 
+ *
+ * This program was prepared by PSI.
  * All rights in the program are reserved by PSI.
  * Neither PSI nor the author(s)
  * makes any warranty, express or implied, or assumes any liability or
@@ -17,7 +17,7 @@
 /***************************************************************************
  *
  * The IPPL Framework
- * 
+ *
  *
  * Visit http://people.web.psi.ch/adelmann/ for more details
  *
@@ -161,7 +161,7 @@ bool DataSource::disconnect(DataConnect *dataconn) {
   }
 
   // if we've removed all, we can erase all
-  if (dataconn == 0 && ConnectionList.size() > 0) {
+  if (dataconn == 0 && !ConnectionList.empty()) {
     //dbgmsg<<"Erasing all " << ConnectionList.size() << " DSO's ..." << endl;
     ConnectionList.erase(ConnectionList.begin(), ConnectionList.end());
   }
@@ -217,5 +217,5 @@ void DataSource::interact(const char *str, DataConnect *dataconn) {
 /***************************************************************************
  * $RCSfile: DataSource.cpp,v $   $Author: adelmann $
  * $Revision: 1.1.1.1 $   $Date: 2003/01/23 07:40:25 $
- * IPPL_VERSION_ID: $Id: DataSource.cpp,v 1.1.1.1 2003/01/23 07:40:25 adelmann Exp $ 
- ***************************************************************************/
+ * IPPL_VERSION_ID: $Id: DataSource.cpp,v 1.1.1.1 2003/01/23 07:40:25 adelmann Exp $
+ ***************************************************************************/
\ No newline at end of file

ippl/src/DataSource/PtclBaseDataSource.cpp

@@ -47,7 +47,7 @@ IpplParticleBaseDataSource::IpplParticleBaseDataSource(const char *nm,
 IpplParticleBaseDataSource::~IpplParticleBaseDataSource() {
 
   // disconnect all our currently connected attributes
-  while (AttribList.size() > 0)
+  while (!AttribList.empty())
     disconnect_attrib(AttribList.front());
 
   // remove ourselves from the list of available IpplParticleBase containers

ippl/src/Field/Assign.hpp

@@ -159,31 +159,31 @@ public:
     return Expr::Child.offset(i,j,k)(Expr::Value.Arg);
   }
 
-  PETE_Return_t& operator*() 
+  PETE_Return_t& operator*()
   {
     return (*Expr::Child)(Expr::Value.Arg);
   }
-  PETE_Return_t& offset(int i) 
+  PETE_Return_t& offset(int i)
   {
     return Expr::Child.offset(i)(Expr::Value.Arg);
   }
-  PETE_Return_t& offset(int i, int j) 
+  PETE_Return_t& offset(int i, int j)
   {
     return Expr::Child.offset(i,j)(Expr::Value.Arg);
   }
-  PETE_Return_t& offset(int i, int j, int k) 
+  PETE_Return_t& offset(int i, int j, int k)
   {
     return Expr::Child.offset(i,j,k)(Expr::Value.Arg);
   }
-  PETE_Return_t& unit_offset(int i) 
+  PETE_Return_t& unit_offset(int i)
   {
     return Expr::Child.unit_offset(i)(Expr::Value.Arg);
   }
-  PETE_Return_t& unit_offset(int i, int j) 
+  PETE_Return_t& unit_offset(int i, int j)
   {
     return Expr::Child.unit_offset(i,j)(Expr::Value.Arg);
   }
-  PETE_Return_t& unit_offset(int i, int j, int k) 
+  PETE_Return_t& unit_offset(int i, int j, int k)
   {
     return Expr::Child.unit_offset(i,j,k)(Expr::Value.Arg);
   }
@@ -228,10 +228,10 @@ public:
 
 template<class T1, unsigned Dim, class RHS, class OP>
 void
-assign(const IndexedBareField<T1,Dim,Dim> &aa, RHS b, OP op, ExprTag<true>, 
+assign(const IndexedBareField<T1,Dim,Dim> &aa, RHS b, OP op, ExprTag<true>,
        bool fillGC)
 {
-  IndexedBareField<T1,Dim,Dim> &a = 
+  IndexedBareField<T1,Dim,Dim> &a =
     const_cast<IndexedBareField<T1,Dim,Dim>&>(aa);
 
   // debugging output macros.  these are only enabled if DEBUG_ASSIGN is
@@ -262,8 +262,9 @@ assign(const IndexedBareField<T1,Dim,Dim> &aa, RHS b, OP op, ExprTag<true>,
   a.getBareField().setDirtyFlag();
 
   // Loop over all the local fields of the left hand side.
-  
+#ifdef DEBUG_ASSIGN
   int lfcount=0;
+#endif
   bool needFinalCompressCheck = false;
   while (la != aend)
     {
@@ -361,18 +362,20 @@ assign(const IndexedBareField<T1,Dim,Dim> &aa, RHS b, OP op, ExprTag<true>,
 	}
 
       ++la;
+#ifdef DEBUG_ASSIGN
       ++lfcount;
+#endif
     }
-  
+
 
   // If we are not deferring guard cell fills, and we need to do this
   // now, fill the guard cells.  This will also apply any boundary
   // conditions after the guards have been updated.
   if (fillGC) {
     ASSIGNMSG(msg << "Filling GC's at end if necessary ..." << endl);
-    
+
     a.getBareField().fillGuardCellsIfNotDirty();
-    
+
   }
 
   // Try to compress the result.
@@ -417,7 +420,7 @@ assign(PETE_TUTree<OpParens<TP>,A> lhs, RHS wrhs, OP op, Tag,
   typedef typename Expressionize<RHS>::type::Wrapped RHS_Wrapped;
   typename Expressionize<RHS>::type expr = Expressionize<RHS>::apply(wrhs);
   RHS_Wrapped & rhs = expr.PETE_unwrap();
-  
+
   // Get a reference to the BareField on the left hand side, and the
   // total domain we are modifying.
   BareField<T1,Dim>& bare = lhs.Child.getBareField();
@@ -438,7 +441,7 @@ assign(PETE_TUTree<OpParens<TP>,A> lhs, RHS wrhs, OP op, Tag,
   bare.setDirtyFlag();
 
   // Loop over all the local fields of the left hand side.
-  
+
   bool needFinalCompressCheck = false;
   while (la != aend)
     {
@@ -458,7 +461,7 @@ assign(PETE_TUTree<OpParens<TP>,A> lhs, RHS wrhs, OP op, Tag,
 	  // the whole expression.
 	  typedef typename LField<T1,Dim>::iterator LA;
 	  typedef PETE_TUTree<OpParens<TP>,LA> LHS;
-	  typedef BrickExpression<Dim,ParensIterator<LHS>,RHS_Wrapped,OP> 
+	  typedef BrickExpression<Dim,ParensIterator<LHS>,RHS_Wrapped,OP>
 	    ExprT;
 
 	  // First look and see if the arrays are sufficiently aligned
@@ -524,15 +527,15 @@ assign(PETE_TUTree<OpParens<TP>,A> lhs, RHS wrhs, OP op, Tag,
 	}
       ++la;
     }
-  
+
 
   // Fill the guard cells on the left hand side, if we are deferring
   // this operation until the next time it is needed.
   ASSIGNMSG(msg << "Filling GC's at end if necessary ..." << endl);
   if (fillGC) {
-    
+
     bare.fillGuardCellsIfNotDirty();
-    
+
   }
 
   // Compress the LHS.
@@ -590,8 +593,9 @@ assign(const BareField<T1,Dim>& ca, RHS b, OP op, ExprTag<true>)
   a.setDirtyFlag();
 
   // Loop over the LHS LFields, and assign from RHS LFields
-  
+#ifdef DEBUG_ASSIGN
   int lfcount = 0;
+#endif
   bool needFinalCompressCheck = false;
   while (la != aend)
   {
@@ -635,16 +639,18 @@ assign(const BareField<T1,Dim>& ca, RHS b, OP op, ExprTag<true>)
 
     ++la;
     for_each(bb,NextLField(),PETE_NullCombiner());
+#ifdef DEBUG_ASSIGN
     ++lfcount;
+#endif
   }
-  
+
 
   // Fill the guard cells on the left hand side, if we are deferring
   // this operation until the next time it is needed.
   ASSIGNMSG(msg << "Filling GC's at end if necessary ..." << endl);
-  
+
   a.fillGuardCellsIfNotDirty();
-  
+
 
   // Compress the LHS, if necessary
   if (needFinalCompressCheck) {

ippl/src/Field/BCond.hpp

@@ -1413,7 +1413,9 @@ void ParallelPeriodicFace<T,D,M,C>::apply( Field<T,D,M,C>& A )
   // Number of nodes that will send us messages.
 
   int receive_count = 0;
+#ifdef PRINT_DEBUG
   int send_count = 0;
+#endif
 
   // Communications tag
 
@@ -1744,7 +1746,9 @@ void ParallelPeriodicFace<T,D,M,C>::apply( Field<T,D,M,C>& A )
 #endif
 
 	      Ippl::Comm->send(messages[iproc], iproc, bc_comm_tag);
+#ifdef PRINT_DEBUG
 	      ++send_count;
+#endif
 
 	    }
 
@@ -2130,7 +2134,9 @@ void ParallelInterpolationFace<T,D,M,C>::apply( Field<T,D,M,C>& A )
   // Number of nodes that will send us messages.
 
   int receive_count = 0;
+#ifdef PRINT_DEBUG
   int send_count = 0;
+#endif
 
   // Communications tag
 
@@ -2467,8 +2473,9 @@ void ParallelInterpolationFace<T,D,M,C>::apply( Field<T,D,M,C>& A )
 #endif
 
 	      Ippl::Comm->send(messages[iproc], iproc, bc_comm_tag);
+#ifdef PRINT_DEBUG
 	      ++send_count;
-
+#endif
 	    }
 
 	}
@@ -6194,4 +6201,3 @@ void PatchBC<T,D,M,C>::apply( Field<T,D,M,C>& A )
 //----------------------------------------------------------------------
 
 #undef COMPONENT_APPLY_BUILTIN
-

ippl/src/FieldLayout/VRB.cpp

@@ -209,12 +209,13 @@ recurseCoordinateVRB(int dim,
     {
       int d;
 
+#ifndef NOPAssert
       // Calculate the total number of vnodes.
       int totalVnodes = sizes[0];
       for (d=1; d<dim; ++d)
         totalVnodes *= sizes[d];
       PAssert_GE(totalVnodes, nprocs);
-
+#endif
       // Find the number of processors on each side.
       int leftProcs = nprocs/2;
       int rightProcs = nprocs-leftProcs;
@@ -375,4 +376,4 @@ main(int argc, char *argv[])
  * $RCSfile: VRB.cpp,v $   $Author: adelmann $
  * $Revision: 1.1.1.1 $   $Date: 2003/01/23 07:40:27 $
  * IPPL_VERSION_ID: $Id: VRB.cpp,v 1.1.1.1 2003/01/23 07:40:27 adelmann Exp $
- ***************************************************************************/
\ No newline at end of file
+ ***************************************************************************/

ippl/src/Message/CMakeLists.txt

 set (_SRCS
     CRC.cpp
-    Communicate.cpp
     CommCreator.cpp
     CommMPI.cpp
+    Communicate.cpp
+    Format.cpp
     MessageFunctions.cpp
-    Formatter.cpp
+    MsgBuffer.cpp
     )
 
 set (_HDRS
+    CRC.h
     CommCreator.h
     CommMPI.h
     Communicate.h
-    CRC.h
     DataTypes.h
-    Formatter.h
-    GlobalComm.hpp
+    Format.h
     GlobalComm.h
-    Message.hpp
+    GlobalComm.hpp
     Message.h
+    Message.hpp
+    MsgBuffer.h
     Operations.h
     TagMaker.h
     Tags.h
@@ -30,4 +32,4 @@ include_directories (
 add_ippl_sources (${_SRCS})
 add_ippl_headers (${_HDRS})
 
-install (FILES ${_HDRS} DESTINATION include/Message)
+install (FILES ${_HDRS} DESTINATION include/Message)
\ No newline at end of file

ippl/src/Message/CommMPI.cpp

-// -*- C++ -*-
-/***************************************************************************
- *
- * The IPPL Framework
- *
- * This program was prepared by PSI.
- * All rights in the program are reserved by PSI.
- * Neither PSI nor the author(s)
- * makes any warranty, express or implied, or assumes any liability or
- * responsibility for the use of this software
- *
- * Visit www.amas.web.psi for more details
- *
- ***************************************************************************/
-
-// -*- C++ -*-
-/***************************************************************************
- *
- * The IPPL Framework
- *
- *
- * Visit http://people.web.psi.ch/adelmann/ for more details
- *
- ***************************************************************************/
-
-// include files
+//
+// CommMPI - MPI-specific communications object for use with the
+// Ippl framework.
+// Allows user to establish id's for available nodes, establish connections,
+// and send/receive data.
+//
+// Copyright (c) 2008 - 2020, Paul Scherrer Institut, Villigen PSI, Switzerland
+//
+// All rights reserved
+//
+// This file is part of OPAL.
+//
+// OPAL is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// You should have received a copy of the GNU General Public License
+// along with OPAL. If not, see <https://www.gnu.org/licenses/>.
+//
 #include "Message/CommMPI.h"
 #include "Message/Message.h"
 #include "Utility/IpplInfo.h"
@@ -103,7 +97,7 @@ CommMPI::CommMPI(int& argc , char**& argv, int procs, bool mpiinit, MPI_Comm mpi
         MPI_Init_thread(&argc, &argv, MPI_THREAD_FUNNELED, &provided);
         INFOMSG("Ippl will be initialized with " <<
                 omp_get_max_threads() << " OMP threads\n");
-        
+
         if ( provided != MPI_THREAD_FUNNELED )
             ERRORMSG("CommMPI: Didn't get requested MPI-OpenMP setting.\n");
 #else
@@ -149,7 +143,7 @@ CommMPI::CommMPI(int& argc , char**& argv, int procs, bool mpiinit, MPI_Comm mpi
         std::vector<int> child_ready(TotalNodes);
         for (i = 0; i < TotalNodes; child_ready[i++] = 0)
             ;
-        INFOMSG("CommMPI: Parent process waiting for children ..." << endl);
+        INFOMSG(level5 << "CommMPI: Parent process waiting for children ..." << endl);
         reported = 1;		// since the parent is already ready
         while (reported < TotalNodes)
         {
@@ -160,7 +154,7 @@ CommMPI::CommMPI(int& argc , char**& argv, int procs, bool mpiinit, MPI_Comm mpi
             {
                 child_ready[rep_host] = 1;
                 reported++;
-                INFOMSG("CommMPI: Child " << rep_host << " ready." << endl);
+                INFOMSG(level5 << "CommMPI: Child " << rep_host << " ready." << endl);
             }
             else
             {
@@ -171,7 +165,7 @@ CommMPI::CommMPI(int& argc , char**& argv, int procs, bool mpiinit, MPI_Comm mpi
         }
 
         //~ delete [] child_ready;
-        INFOMSG("CommMPI: Initialization complete." << endl);
+        INFOMSG(level5 << "CommMPI: Initialization complete." << endl);
 
     }
     else  			// this is a child process; get data from pops
@@ -180,7 +174,7 @@ CommMPI::CommMPI(int& argc , char**& argv, int procs, bool mpiinit, MPI_Comm mpi
         ierror = MPI_Get_processor_name(host_name, &result_len);
         if (ierror >= 0)
         {
-            INFOMSG("CommMPI: Started job " << myHost << " on host `");
+            INFOMSG(level5 << "CommMPI: Started job " << myHost << " on host `");
             INFOMSG(host_name <<  "'." << endl);
         }
         else
@@ -623,4 +617,4 @@ int CommMPI::raw_probe_receive(char *&data, int &node, int &tag)
     MPI_Recv(data, count, MPI_BYTE, node, tag, communicator, &stat);
 
     return count;
-}
+}
\ No newline at end of file

ippl/src/Message/CommMPI.h

-// -*- C++ -*-
-/***************************************************************************
- *
- * The IPPL Framework
- *
- *
- * Visit http://people.web.psi.ch/adelmann/ for more details
- *
- ***************************************************************************/
-
+//
+// CommMPI - MPI-specific communications object for use with the
+// Ippl framework.
+// Allows user to establish id's for available nodes, establish connections,
+// and send/receive data.
+//
+// Copyright (c) 2008 - 2020, Paul Scherrer Institut, Villigen PSI, Switzerland
+//
+// All rights reserved
+//
+// This file is part of OPAL.
+//
+// OPAL is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// You should have received a copy of the GNU General Public License
+// along with OPAL. If not, see <https://www.gnu.org/licenses/>.
+//
 #ifndef COMM_MPI_H
 #define COMM_MPI_H
 
-/***************************************************************************
- * CommMPI.h - MPI-specific communications object for use with the
- * Ippl framework.
- * Allows user to establish id's for available nodes, establish connections,
- * and send/receive data.
- ***************************************************************************/
-
-// include files
 #include "Message/Communicate.h"
 #include <mpi.h>
 
@@ -31,7 +33,7 @@ public:
     // constructor arguments: command-line args, and number of processes
     // to start (if < 0, start the 'default' number, i.e. the number of
     // hosts in a MPI virtual machine, the number of nodes in an O2K, etc)
-    CommMPI(int& argc, char**& argv, int procs = (-1), bool mpiinit = true, 
+    CommMPI(int& argc, char**& argv, int procs = (-1), bool mpiinit = true,
             MPI_Comm mpicomm = MPI_COMM_WORLD);
     virtual ~CommMPI(void);
 
@@ -101,4 +103,4 @@ private:
 };
 
 
-#endif // COMM_MPI_H
+#endif // COMM_MPI_H
\ No newline at end of file

ippl/src/Message/Communicate.cpp

@@ -61,7 +61,7 @@ std::ostream& operator<<(std::ostream& o, const Communicate& c)
 // Constructor.
 // 	arguments: command-line args, and number of processes
 // to start (if < 0, start the 'default' number, i.e. the number of
-// hosts 
+// hosts
 // Note: The base-class constructor does not need the argument info or
 // the number of nodes, it just by default sets the number of nodes=1
 // Also note: the derived classes should erase Contexts and Processes, and
@@ -83,7 +83,7 @@ Communicate::Communicate(int, char **, int)
 // Destructor.  Nothing to do at present.
 Communicate::~Communicate(void)
 {
-    
+
 
     // delete the cached messages
     SentCache_t::iterator cachei = sentMsgCache.begin();
@@ -157,7 +157,7 @@ Message* Communicate::myreceive(int&, int&, int)
 // Default version of virtual barrier function ... here, does nothing.
 void Communicate::mybarrier(void)
 {
-    
+
 
     // just return NULL, since we cannot find a message with this function
     return;
@@ -239,7 +239,7 @@ bool Communicate::send(Message *msg, int node, int tag, bool delmsg)
 //      2. In receive queue
 Message* Communicate::receive(int& node, int& tag)
 {
-    
+
 
     //Inform dbgmsg("Comm::receive", INFORM_ALL_NODES);
     //dbgmsg << "Doing receive from node " << node << ", tag " << tag << endl;
@@ -299,10 +299,10 @@ Message* Communicate::receive(int& node, int& tag)
 // A blocking version of receive.
 Message *Communicate::receive_block(int& node, int &tag)
 {
-    
-    
-    
-    
+
+
+
+
 
     // process list of resend requests
     process_resend_requests();
@@ -313,13 +313,13 @@ Message *Communicate::receive_block(int& node, int &tag)
 
     // If we haven't already found a message, check the local messages
     //dbgmsg << "Checking for queued message ..." << endl;
-    
+
     Message *msg = find_msg(node, tag);
-    
+
     //dbgmsg << "Found one? " << (msg != 0 ? "yes" : "no") << endl;
 
     // keep checking for remote msgs until we get one
-    
+
     if (myNode() != node)
     {
         while (msg == 0)
@@ -362,7 +362,7 @@ Message *Communicate::receive_block(int& node, int &tag)
             }
         }
     }
-    
+
 
     // If we're on just one node, and we did not find a message, this is
     // a big problem.
@@ -438,7 +438,7 @@ int Communicate::broadcast_others(Message *msg, int tag, bool delmsg)
 // else to get here before returning to calling function).
 void Communicate::barrier()
 {
-    
+
 
     mybarrier();
     //INCIPPLSTAT(incBarriers);
@@ -864,7 +864,7 @@ void Communicate::remove_single_ok_message(MsgNum_t mnum)
 // process list of resend requests
 void Communicate::process_resend_requests()
 {
-    if (resendList.size() > 0)
+    if (!resendList.empty())
     {
         Inform dbgmsg("***Communicate::process_resend_reqs", INFORM_ALL_NODES);
         dbgmsg << "Clearing " << sentOKList.size() << " and resending ";
@@ -872,7 +872,7 @@ void Communicate::process_resend_requests()
     }
 
     // clear out OK messages
-    while (sentOKList.size() > 0)
+    while (!sentOKList.empty())
     {
         MsgNum_t mnum = *(sentOKList.begin());
         sentOKList.erase(sentOKList.begin());
@@ -880,7 +880,7 @@ void Communicate::process_resend_requests()
     }
 
     // resend a message, if necessary
-    while (resendList.size() > 0)
+    while (!resendList.empty())
     {
         MsgNum_t mnum = *(resendList.begin());
         resendList.erase(resendList.begin());
@@ -888,7 +888,7 @@ void Communicate::process_resend_requests()
     }
 
     // inform other nodes that we've received their messages ok
-    while (informOKList.size() > 0)
+    while (!informOKList.empty())
     {
         int node = (*(informOKList.begin())).first;
         MsgNum_t mnum = (*(informOKList.begin())).second;
@@ -897,11 +897,11 @@ void Communicate::process_resend_requests()
     }
 
     // request resends from other nodes
-    while (requestList.size() > 0)
+    while (!requestList.empty())
     {
         int node = (*(requestList.begin())).first;
         MsgNum_t mnum = (*(requestList.begin())).second;
         requestList.erase(requestList.begin());
         request_retransmission(node, mnum);
     }
-}
+}
\ No newline at end of file

ippl/src/Message/Format.cpp

+//
+// Class Format
+// Format class to allow serializing message objects into plain buffers
+// to send directly with mpi calls or similar means
+//
+// Copyright (c) 2008 - 2020, Paul Scherrer Institut, Villigen PSI, Switzerland
+// All rights reserved
+//
+// This file is part of OPAL.
+//
+// OPAL is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// You should have received a copy of the GNU General Public License
+// along with OPAL. If not, see <https://www.gnu.org/licenses/>.
+//
+#include "Message/Format.h"
+
+Format::Format(Message *msg)
+{
+    items = msg->size();
+    size = 0;
+
+    format_array.resize(2*items);
+    for (unsigned int i=0; i<items; ++i)
+    {
+        Message::MsgItem &msgitem = msg->item(i);
+        format_array[2*i+0] = msgitem.numElems();
+        format_array[2*i+1] = msgitem.numBytes();
+        size += format_array[2*i+1];
+    }
+}
+
+void Format::print()
+{
+	std::cout << "size: " << size << std::endl;
+	for (unsigned int i=0; i<items; ++i)
+    {
+		std::cout << "entry " << i << ": " << format_array[2*i+0]
+			<< " elements " << format_array[2*i+1] << " bytes\n";
+    }
+}
\ No newline at end of file

ippl/src/Message/Format.h

+//
+// Class Format
+// Format class to allow serializing message objects into plain buffers
+// to send directly with mpi calls or similar means
+//
+// Copyright (c) 2008 - 2020, Paul Scherrer Institut, Villigen PSI, Switzerland
+// All rights reserved
+//
+// This file is part of OPAL.
+//
+// OPAL is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// You should have received a copy of the GNU General Public License
+// along with OPAL. If not, see <https://www.gnu.org/licenses/>.
+//
+#ifndef FORMATTER_H
+#define FORMATTER_H
+
+#include <algorithm>
+#include <cstring>
+#include <vector>
+#include "Message/Message.h"
+
+class Format {
+public:
+    Format(Message*);
+    unsigned int getItemCount() {
+        return items;
+    }
+    unsigned int getSize() {
+        return size;
+    }
+    unsigned int getFormatSize() {
+        return 2 * items * sizeof(int);
+    }
+    unsigned int getItemElems(int i) {
+        return format_array[2 * i + 0];
+    }
+    unsigned int getItemBytes(int i) {
+        return format_array[2 * i + 1];
+    }
+
+    void print();
+
+private:
+    unsigned int items, size;
+    std::vector<unsigned int> format_array;
+};
+
+#endif
\ No newline at end of file