.clang-format

+---
+Language:        Cpp
+BasedOnStyle:  Google
+AccessModifierOffset: -4
+AlignAfterOpenBracket: AlwaysBreak
+AlignConsecutiveAssignments: true
+AllowShortFunctionsOnASingleLine: None
+AllowShortIfStatementsOnASingleLine: Never
+AllowShortLambdasOnASingleLine: Empty
+AllowShortLoopsOnASingleLine: false
+BreakBeforeBinaryOperators: NonAssignment
+ColumnLimit:     100
+DerivePointerAlignment: false
+IndentWidth:     4
+IncludeBlocks: Preserve
+IndentGotoLabels: false
+NamespaceIndentation: All 
+...

.dir-locals.el

+(
+ (nil . (
+	 (indent-tabs-mode . nil)
+	 (require-final-newline . nil)
+	 )
+      )
+ (cmake-mode . (
+           (cmake-tab-width . 4)
+           )
+        )
+ 
+ (c-mode . (
+	    (c-basic-offset . 4)
+	    )
+	 )
+ (c++-mode . (
+	      (c-basic-offset . 4)
+	      )
+	   )
+ )
+

.gitignore

@@ -2,7 +2,10 @@
 *~
 *.o
 *.a
+*.so
 *.aux
+__pycache__
+optimizer/Tests/*.exe
 build
 CMakeCache.txt
 CMakeFiles

.gitlab/issue_templates/Bug.md

+### Summary
+
+(Summarize the bug encountered concisely)
+
+
+### Steps to reproduce
+
+(How one can reproduce the issue - this is very important)
+
+
+### What is the current *bug* behavior?
+
+(What actually happens)
+
+
+### What is the expected *correct* behavior?
+
+(What you should see instead)
+
+
+### Relevant logs and/or screenshots
+
+(Paste any relevant logs - please use code blocks (```) to format console output,
+logs, and code as it's very hard to read otherwise.)
+
+
+### Possible fixes
+
+(If you can, link to the line of code that might be responsible for the problem)
+
+/label ~Bug

.gitlab/issue_templates/Feature.md

+### Summary
+
+(Summarize the feature request concisely)
+
+
+
+/label ~"Feature request"
\ No newline at end of file

.gitlab/issue_templates/Release.md

+**source code and binary**
+  * [ ] create branch YEAR.N
+  * [ ] create issue "Release version YEAR.N" and merge request
+  * [ ] update version string in Doxyfile
+  * [ ] update the version string in CMakeLists.txt and commit
+  * [ ] wait for approval of MR and merge
+  * [ ] tag version YEAR.N.0
+  * [ ] upload source tar-ball to `/afs/psi.ch/project/amas/webhosting/Downloads/OPAL/src`
+  * [ ] compile new binary for Linux
+  * [ ] upload Linux binary package to `/afs/psi.ch/project/amas/webhosting/Downloads/OPAL/package/`
+  * [ ] compile new binary for macOS
+  * [ ] upload macOS binary package to `/afs/psi.ch/project/amas/webhosting/Downloads/OPAL/package/`
+
+**manual/documentation**
+  * [ ] setup a new branch for the new version of the manual
+  * [ ] fix version, branches and links in `Manual.attributes`
+  * [ ] clone repository into `/afs/psi.ch/project/amas/webhosting/opal/Documentation/x.y` and checkout new branch (`git clone https://gitlab.psi.ch/OPAL/documentation/manual.git`)
+  * [ ] add links to the binaries in the wiki
+  * [ ] update https://gitlab.psi.ch/OPAL/src/wikis/For-Developers/Compile-OPAL
+  * [ ] compile the change log/release notes and publish it in the wiki: https://gitlab.psi.ch/OPAL/src/wikis/ReleaseNotes
+  * [ ] review the file `src/addToDoxygenMainPage.h`
+  * [ ] build Doxygen documentation
+  * [ ] update https://gitlab.psi.ch/OPAL/src/wikis/home
+  * [ ] update https://gitlab.psi.ch/OPAL/src/wikis/regression-tests
+
+**tracker**
+  * [ ] new milestone for `OPAL x.(y+1)`
+  * [ ] update labels and milestones in issues
+
+**regression-tests**
+
+  * [ ] create new branch x.y
+  * [ ] setup the regression-tests to run the new version on opalrunner.psi.ch
+
+**varia**
+  * [ ] PSI module
+  * [ ] write e-mail to mailing list

.gitlab/merge_request_templates/Bug.md

+### What does this MR do?
+
+### Does this MR meet the acceptance criteria?
+
+* [ ] [Coding style guidelines](https://gitlab.psi.ch/OPAL/src/wikis/For-Developers/CodingStyle)
+* [ ] [Change to Release Notes](https://gitlab.psi.ch/OPAL/src/wikis/ReleaseNotes/ReleaseNotes)
+
+
+/label ~Bug

.gitlab/merge_request_templates/Feature.md

+### What does this MR do?
+
+### Does this MR meet the acceptance criteria?
+
+* [ ] [Coding style guidelines](https://gitlab.psi.ch/OPAL/src/wikis/For-Developers/CodingStyle)
+* [ ] [Change to Release Notes](https://gitlab.psi.ch/OPAL/src/wikis/ReleaseNotes/ReleaseNotes)
+* [ ] Documentation added
+
+
+/label ~"Feature request"

.local.vimrc

+set et ts=4 sw=4 sts=4
\ No newline at end of file

CMakeLists.txt

-cmake_minimum_required (VERSION 3.1)
-project (OPAL VERSION 2.0.0)
+cmake_minimum_required (VERSION 3.12)
+cmake_policy(VERSION 3.12)
+
+project (OPAL VERSION 2024.2)
 set (PROJECT_BUGREPORT opal@lists.psi.ch)
 
 string (REGEX REPLACE "\\.([0-9])\\." ".0\\1." PACKAGE_VERSION ${PROJECT_VERSION})
 string (REGEX REPLACE "\\.([0-9])$" ".0\\1" PACKAGE_VERSION ${PACKAGE_VERSION})
 string (REGEX REPLACE "\\." "" PACKAGE_VERSION ${PACKAGE_VERSION})
 
-message (PACKAGE_VERSION=${PACKAGE_VERSION})
+message (STATUS PACKAGE_VERSION=${PACKAGE_VERSION})
 
 if (NOT CMAKE_CONFIGURATION_TYPES AND NOT CMAKE_BUILD_TYPE)
     set (CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING
         "Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel."
         FORCE
-	)
+        )
 endif ()
 
 cmake_host_system_information (RESULT HOSTNAME QUERY HOSTNAME)
 
-message ("Build type is: " ${CMAKE_BUILD_TYPE})
-message ("Host OS System: ${CMAKE_HOST_SYSTEM}")
-message ("Hostname:  ${HOSTNAME}")
+message (STATUS "Build type is: " ${CMAKE_BUILD_TYPE})
+message (STATUS "Host OS System: ${CMAKE_HOST_SYSTEM}")
+message (STATUS "Hostname: ${HOSTNAME}")
 
 # Select flags.
 set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -g ")
 set (CMAKE_CXX_FLAGS_RELEASE "-O3")
 set (CMAKE_CXX_FLAGS_DEBUG  "-O0 -g")
 
+string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE)
+if (uppercase_CMAKE_BUILD_TYPE MATCHES "^(DEBUG|RELWITHDEBINFO)$")
+    option (BUILD_FOR_PRODUCTION "Disable all assertion checks; Only use for production" OFF)
+else()
+    option (BUILD_FOR_PRODUCTION "Disable all assertion checks; Only use for production" ON)
+endif()
+
 # Resolve all library dependencies
 set (CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/CMakeModules")
 
 find_package (MPI REQUIRED)
-message ("The C++ compiler identification is: ${CMAKE_CXX_COMPILER_ID}")
-message ("The C++ compiler version is: ${CMAKE_CXX_COMPILER_VERSION}")
-message ("The MPI C++ compiler is: ${MPI_CXX_COMPILER}")
-message ("The underlying C++ compiler is: ${CMAKE_CXX_COMPILER}")
+message (STATUS "The C++ compiler identification is: ${CMAKE_CXX_COMPILER_ID}")
+message (STATUS "The C++ compiler version is: ${CMAKE_CXX_COMPILER_VERSION}")
+message (STATUS "The MPI C++ compiler is: ${MPI_CXX_COMPILER}")
+message (STATUS "The underlying C++ compiler is: ${CMAKE_CXX_COMPILER}")
 
 option (ENABLE_OpenMP "Use hybrid parallelism MPI-OpenMP" OFF)
 
-set (CMAKE_CXX_STANDARD 11)
+set (CMAKE_CXX_STANDARD 17)
 set (CMAKE_CXX_STANDARD_REQUIRED ON)
 set (CMAKE_CXX_EXTENSIONS OFF)
 
-add_definitions (-DIPPL_LINUX -DIPPL_STRINGSTREAM)
-add_definitions (-DIPPL_MPI -DMPICH_SKIP_MPICXX -DIPPL_DONT_POOL)
-add_definitions (-DIPPL_USE_XDIV_RNG -DPETE_BITWISE_COPY)
-add_definitions (-DIPPL_USE_PARTIAL_SPECIALIZATION)
-add_definitions (-Drestrict=__restrict__ -DNOCTAssert)
+# Disable compile time assert (used by IPPL)
+add_definitions (-DNOCTAssert)
 
 add_compile_options (-Wall)
-add_compile_options (-funroll-loops -fstrict-aliasing)
-option (USE_STATIC_LIBRARIES "Link with static libraries if available" ON)
-if (NOT ${USE_STATIC_LIBRARIES})
+add_compile_options (-Wunused)
+add_compile_options (-Wextra)
+add_compile_options (-Werror)
+
+add_compile_options (-funroll-loops)
+add_compile_options (-fstrict-aliasing)
+
+option (WILL_BUILD_STATIC_LIBRARY "Building static library" ON)
+option (BUILD_OPAL_PYTHON "Building Python interface for OPAL" OFF)
+option (WILL_BUILD_SHARED_LIBRARY "Building python hooks - requires dynamic libs" OFF)
+if (BUILD_OPAL_PYTHON)
+    if (NOT WILL_BUILD_SHARED_LIBRARY)
+        message (FATAL_ERROR "For building python hooks you have to enable WILL_BUILD_SHARED_LIBRARY")
+    endif()
+    find_package (Python3 COMPONENTS Development REQUIRED)
+endif ()
+
+if (${WILL_BUILD_SHARED_LIBRARY})
     add_compile_options (-fPIE -fPIC)
-    set (CMAKE_EXE_LINKER_FLAGS  "${CMAKE_EXE_LINKER_FLAGS} -pie")
+    add_link_options (-pie)
+    # force OPAL to link to shared libraries
+    if (UNIX)
+        set(CMAKE_FIND_LIBRARY_SUFFIXES ".so")
+    elseif (APPLE)
+        set(CMAKE_FIND_LIBRARY_SUFFIXES ".dylib")
+    else()
+        MESSAGE(FATAL_ERROR "Operating system not APPLE or UNIX - quitting")
+    endif()
 endif ()
 
 # compiler dependent flags
-message ("The underlying C++ compiler ID is: ${CMAKE_CXX_COMPILER_ID}")
+message (STATUS "The underlying C++ compiler ID is: ${CMAKE_CXX_COMPILER_ID}")
 if ( ${CMAKE_CXX_COMPILER_ID} STREQUAL "Intel" )
     add_compile_options ("-diag-disable 383")
     add_compile_options ("-diag-disable 981")
     if (ENABLE_OpenMP)
-	add_compile_options (-openmp)
+        add_compile_options (-openmp)
+        add_link_options (-openmp)
     endif ()
 
 elseif (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
-    add_compile_options (-Werror -Wsign-compare -Wunused-variable -Warray-bounds)
+    add_compile_options (-Warray-bounds-pointer-arithmetic)
     # Increase template depth for boost
     add_compile_options (-ftemplate-depth=1024)
-    # Allow overloaded virtual functions (instances to be fixed in OPAL)
-    add_compile_options (-Wno-overloaded-virtual)
-    add_definitions(-DIPPL_RESTRICT_BUG)
+    # get rid of linking warning with boost. Looks like that boost
+    # is using this visibility setting.
+    add_compile_options (-fvisibility=hidden)
+    add_compile_options (-fvisibility-inlines-hidden)
+    add_compile_options (-Wno-deprecated-declarations)
+    add_compile_options (-Wno-deprecated-builtins)
+    add_compile_options (-Wno-deprecated-copy)
     if (ENABLE_OpenMP)
-	    add_compile_options (-fopenmp)
+        if (CMAKE_HOST_APPLE)
+            message(FATAL_ERROR "Apple Clang does not support OpenMP!")
+        else ()
+            add_compile_options (-fopenmp)
+            add_link_options (-fopenmp)
+        endif ()
     endif ()
 
 elseif (${CMAKE_CXX_COMPILER_ID} MATCHES "GNU")
-    if (${CMAKE_CXX_COMPILER_VERSION} VERSION_LESS "4.7.3")
-        message (FATAL_ERROR "To build OPAL you need GCC version 4.7.3 or greater")
+    # An error in Trilinos/12.18.1 causes
+    # -Waggressive-loop-optimizations with -O3 optimization.
+    # See issue 587 and corresponding merge request 415.
+    # Clang does not support this flag!
+    add_compile_options (-fno-aggressive-loop-optimizations)
+    add_compile_options (-Wno-error=cpp)
+
+    if (${CMAKE_CXX_COMPILER_VERSION} VERSION_LESS "9.5.0")
+        message (FATAL_ERROR "To build OPAL you need GCC version 9.5.0 or greater")
     endif ()
-    # Warning flags, -Wno-unused-local-typedefs is needed for boost versions below 1.59
-    add_compile_options (-Wno-unused-local-typedefs -Werror=unused-variable)
+    # Warning flags
 
     if (ENABLE_OpenMP)
         add_compile_options (-fopenmp)
+        add_link_options (-fopenmp)
     endif ()
 
+    # Enables extra error checking in the form of precondition assertion, such
+    # as bounds checking and null pointer checks when dereferencing smart pointers
+    if (NOT ${BUILD_FOR_PRODUCTION})
+        add_compile_options(-Wp,-D_GLIBCXX_ASSERTIONS)
+    endif()
+    enable_language (Fortran OPTIONAL)
 else ()
-    message ("Unknown C++ compiler. Please use the GNU or Intel compiler, if you are having problems.")
+    message (STATUS "Unknown C++ compiler. Please use the GNU or Intel compiler, if you are having problems.")
 endif ()
 
-enable_language (Fortran OPTIONAL)
-message ("Fortran compilers: ${CMAKE_Fortran_COMPILER_NAMES}")
+message (STATUS "Fortran compilers: ${CMAKE_Fortran_COMPILER_NAMES}")
 if (CMAKE_Fortran_COMPILER)
-    message ("The Fortran compiler identification is: ${CMAKE_Fortran_COMPILER_ID}")
-    message ("The Fortran compiler version is: ${CMAKE_Fortran_COMPILER_VERSION}")
-    message ("The Fortran compiler is: ${CMAKE_Fortran_COMPILER}")
+    message (STATUS "The Fortran compiler identification is: ${CMAKE_Fortran_COMPILER_ID}")
+    message (STATUS "The Fortran compiler version is: ${CMAKE_Fortran_COMPILER_VERSION}")
+    message (STATUS "The Fortran compiler is: ${CMAKE_Fortran_COMPILER}")
     if (${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU")
         set (Fortran_LIBS "-lgfortran")
     endif ()
 else ()
-    message ("No Fortran compiler found (optional)")
+    message (STATUS "No Fortran compiler found (optional)")
     if (ENABLE_AMR)
-	message (FATAL_ERROR "AMR is enabled but no Fortran compiler found!")
+        message (FATAL_ERROR "AMR is enabled but no Fortran compiler found!")
     endif ()
 endif ()
 
 find_package (H5Hut REQUIRED)
 
-if (USE_STATIC_LIBRARIES)
+if (NOT ${WILL_BUILD_SHARED_LIBRARY})
     set (HDF5_USE_STATIC_LIBRARIES ON)
 endif ()
+set (HDF5_PREFER_PARALLEL ON)
 find_package (HDF5 REQUIRED)
-message ("Found HDF5 libraries: ${HDF5_C_LIBRARIES}")
+if(UNIX AND NOT APPLE)
+    set (HDF5_LIBRARIES ${HDF5_LIBRARIES} ${CMAKE_DL_LIBS} rt z)
+endif ()
+message (STATUS "Found HDF5 libraries: ${HDF5_LIBRARIES}")
 
 find_package (GSL REQUIRED)
-message ("Found gsl libraries: ${GSL_LIBRARIES}")
+message (STATUS "Found gsl libraries: ${GSL_LIBRARIES}")
 
 set (BOOSTROOT $ENV{BOOST_DIR})
-if (USE_STATIC_LIBRARIES)
+if (NOT ${WILL_BUILD_SHARED_LIBRARY})
     set (Boost_USE_STATIC_LIBS ON)
 endif ()
 set (Boost_USE_MULTITHREADED OFF)
 set (Boost_USE_STATIC_RUNTIME OFF)
-find_package (Boost
-    REQUIRED COMPONENTS chrono filesystem iostreams regex serialization system)
-if (Boost_INCLUDE_DIRS)
-    if (${Boost_VERSION} VERSION_LESS 106000)
-	message (FATAL_ERROR
-	       "Found Boost library is too old; required is version 1.60.0 or newer!")
-    endif ()
-    message ("Found Boost include dir: ${Boost_INCLUDE_DIR}")
-    message ("Found Boost library dir: ${Boost_LIBRARY_DIR}")
-    message ("Found Boost libraries: ${Boost_LIBRARIES}")
-    include_directories (${Boost_INCLUDE_DIRS})
-endif ()
-
-option (ENABLE_DKS "Enable DKS" OFF)
-if (ENABLE_DKS)
-    find_package (DKS 1.1.1 REQUIRED HINTS $ENV{DKS_PREFIX} $ENV{DKS_DIR} $ENV{DKS})
-    set (DKS_VERSION_OPAL \"${DKS_VERSION}\")
-    message ("Found DKS version: ${DKS_VERSION}")
-    message ("Found DKS library: ${DKS_LIBRARY}")
-    message ("Found DKS include dir: ${DKS_INCLUDE_DIR}")
-    add_definitions(-DDKS_MPI)
-
-    ### OpenCL compiler flags ###
-    #add_compile_options (-lOpenCL -pthread -DDKS_OPENCL)
-
-    ### CUDA compiler flags ###
-    add_definitions(-DDKS_CUDA)
-
-    ### if any accelerator enabled set flag to use DKS ###
-    add_definitions(-DIPPL_DKS -DIPPL_DKS_CUDA -DOPAL_DKS)
-
+set (Boost_LIBS filesystem iostreams regex serialization system timer)
+find_package (Boost 1.66.0
+    REQUIRED COMPONENTS ${Boost_LIBS})
+if(UNIX AND NOT APPLE)
+    set (Boost_LIBRARIES ${Boost_LIBRARIES} rt)
 endif ()
+message (STATUS "Found Boost include dirs: ${Boost_INCLUDE_DIRS}")
+message (STATUS "Found Boost library dirs: ${Boost_LIBRARY_DIRS}")
+message (STATUS "Found Boost libraries: ${Boost_LIBRARIES}")
+include_directories (SYSTEM ${Boost_INCLUDE_DIRS})
 
 # Handle options
 option (BUILD_OPAL_UNIT_TESTS "Unit tests" OFF)
@@ -164,91 +197,107 @@ if (BUILD_OPAL_UNIT_TESTS)
 endif ()
 
 option (ENABLE_AMR "Enable AMReX based AMR solver" OFF)
-IF (ENABLE_AMR)
-    message ("Enable AMR_SOLVER    " ${ENABLE_AMR})
+if (ENABLE_AMR)
+    message (STATUS "Enable AMR_SOLVER    " ${ENABLE_AMR})
     enable_language (Fortran)
 
     find_package (AMReX CONFIG REQUIRED
         HINTS $ENV{AMREX_HOME}/cmake $ENV{AMREX_DIR}/cmake $ENV{AMREX_PREFIX}/cmake)
 
-    message ("Found AMReX include dir: ${AMREX_INCLUDE_DIR}")
-    message ("Found AMReX library dir: ${AMREX_LIBRARY_DIR}")
-    message ("Found AMReX libraries: ${AMREX_LIBRARIES}")
-    message ("Found AMReX Fortran links: ${AMREX_EXTRA_Fortran_LINK_LINE}")
+    message (STATUS "Found AMReX include dir: ${AMREX_INCLUDE_DIR}")
+    message (STATUS "Found AMReX library dir: ${AMREX_LIBRARY_DIR}")
+    message (STATUS "Found AMReX libraries: ${AMREX_LIBRARIES}")
+    message (STATUS "Found AMReX Fortran links: ${AMREX_EXTRA_Fortran_LINK_LINE}")
     echo_amrex_config_options()
 
     add_definitions(${AMREX_DEFINES})
     add_definitions(-DENABLE_AMR)
-    add_compile_options (-Wno-unused-variable -Wno-unused-but-set-variable -Wno-maybe-uninitialized)
 endif ()
 
 option (ENABLE_AMR_MG_SOLVER "Enable AMR MG solver" OFF)
-option (ENABLE_SAAMG_SOLVER "Enable iteartive SA-AMG-PCG self field solver" OFF)
+option (ENABLE_SAAMG_SOLVER "Enable iterative SA-AMG-PCG self field solver" OFF)
 if (ENABLE_SAAMG_SOLVER OR ENABLE_AMR_MG_SOLVER)
     #
     # :FIXME: We have to add checks whether the required Trilinos modules
     #         are available. If there are problems of any kind with this
     #         configuration, either fix it in a general way or build your
     #         Trilinos. Host specific should be avoided in almost any case.
-    message ("Enable SAAMG_SOLVER: " ${ENABLE_SAAMG_SOLVER})
-    message ("Enable AMR_MG_SOLVER: " ${ENABLE_AMR_MG_SOLVER})
+    message (STATUS "Enable SAAMG_SOLVER: " ${ENABLE_SAAMG_SOLVER})
+    message (STATUS "Enable AMR_MG_SOLVER: " ${ENABLE_AMR_MG_SOLVER})
 
     find_package (Trilinos REQUIRED HINTS $ENV{TRILINOS_PREFIX} $ENV{TRILINOS_DIR} $ENV{TRILINOS})
 
-    message ("Found Trilinos: ${Trilinos_DIR}")
-    message ("      Trilinos version: ${Trilinos_VERSION}")
-    message ("      Trilinos package list: ${Trilinos_PACKAGE_LIST}")
-    message ("      Trilinos libraries: ${Trilinos_LIBRARIES}")
-    message ("      Trilinos TPL libraries: ${Trilinos_TPL_LIBRARIES}")
+    message (STATUS "Found Trilinos: ${Trilinos_DIR}")
+    message (STATUS "      Trilinos version: ${Trilinos_VERSION}")
+    message (STATUS "      Trilinos package list: ${Trilinos_PACKAGE_LIST}")
+    message (STATUS "      Trilinos libraries: ${Trilinos_LIBRARIES}")
+    message (STATUS "      Trilinos TPL libraries: ${Trilinos_TPL_LIBRARIES}")
 
     if (NOT ${MPI_CXX_COMPILER} STREQUAL ${Trilinos_CXX_COMPILER} )
-        message ("Compiler mismatch:")
-        message ("        Trilinos was compiled with:")
-        message ("            ${Trilinos_C_COMPILER}")
-        message ("            ${Trilinos_CXX_COMPILER}")
-        message ("            ${Trilinos_Fortran_COMPILER}")
-        message ("        You are using:")
-        message ("            ${CMAKE_C_COMPILER}")
-        message ("            ${CMAKE_CXX_COMPILER}")
-        message ("            ${CMAKE_Fortran_COMPILER}")
+        message (WARNING "Compiler mismatch:")
+        message (WARNING "        Trilinos was compiled with:")
+        message (WARNING "            ${Trilinos_C_COMPILER}")
+        message (WARNING "            ${Trilinos_CXX_COMPILER}")
+        message (WARNING "            ${Trilinos_Fortran_COMPILER}")
+        message (WARNING "        You are using:")
+        message (WARNING "            ${CMAKE_C_COMPILER}")
+        message (WARNING "            ${CMAKE_CXX_COMPILER}")
+        message (WARNING "            ${CMAKE_Fortran_COMPILER}")
         message (FATAL_ERROR  "")
     endif ()
 endif ()
 
+option (ENABLE_OPAL_FEL "Enable OPAL FEL" OFF)
+if (ENABLE_OPAL_FEL)
+    message (STATUS "Enable OPAL FEL: " ${ENABLE_OPAL_FEL})
+    find_package (MITHRA MODULE REQUIRED)
+    add_definitions (-DENABLE_OPAL_FEL)
+endif()
+
 option (DBG_SCALARFIELD "Enable dump of scalar field rho_m" OFF)
 if (DBG_SCALARFIELD)
-    message ("\nWrite scalar rho_m field is enabled ")
-    add_DEFINITIONS (-DDBG_SCALARFIELD)
+    message (STATUS "Write scalar rho_m field is enabled ")
+    add_definitions (-DDBG_SCALARFIELD)
 endif ()
 
 option (DBG_STENCIL "Dump stencil of iterative solver in matlab format" OFF)
 if (DBG_STENCIL)
-    MESSAGE ("\nWrite stencil is enabled ")
-    ADD_DEFINITIONS(-DDBG_STENCIL)
+    message (STATUS "Write stencil is enabled ")
+    add_definitions(-DDBG_STENCIL)
 endif ()
 
 option (NO_FIELD_ASSIGN_OPTIMIZATION "Disable compiler optimization of IPPL field assignment" OFF)
 if (NO_FIELD_ASSIGN_OPTIMIZATION)
-    MESSAGE ("\nCompiler will not try to optimize field assignment")
-    ADD_DEFINITIONS(-DdontOPTIMIZE_FIELD_ASSIGNMENT)
+    message (STATUS "Compiler will not try to optimize field assignment")
+    add_definitions(-DdontOPTIMIZE_FIELD_ASSIGNMENT)
 endif ()
 
 option (ENABLE_DOXYDOC "compile Doxygen documentation" OFF)
-
-include (FindGit)
-if (GIT_FOUND)
-    set (OPAL_WC_REVISION 0)
-    execute_PROCESS(
-        WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-        COMMAND git rev-parse HEAD
-        OUTPUT_VARIABLE GIT_OUT OUTPUT_STRIP_TRAILING_WHITESPACE
+if (ENABLE_DOXYDOC)
+    find_package(Doxygen REQUIRED)
+    configure_file (${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in
+        ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile @ONLY
+	)
+    add_custom_target (doxydoc
+        ${DOXYGEN_EXECUTABLE} ${CMAKE_BINARY_DIR}/Doxyfile
+        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+        COMMENT "Generating API documentation with Doxygen" VERBATIM
         )
-    set (OPAL_WC_REVISION \"${GIT_OUT}\")
-else ()
-    message ("could not find git")
-    set (OPAL_WC_REVISION \"\")
 endif ()
 
+add_custom_target(always_rebuild ALL
+    DEPENDS
+    src/OPALrevision.h
+    )
+
+add_custom_command(
+    OUTPUT src/OPALrevision.h
+    COMMAND ./gen_OPALrevision
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    DEPENDS
+    always_rebuild
+    )
+
 add_subdirectory (ippl)
 add_subdirectory (optimizer)
 add_subdirectory (src)
@@ -262,9 +311,6 @@ endif ()
 set (OPAL_CXX_FLAGS ${OPAL_CXX_FLAGS}
     CACHE INTERNAL "" FORCE
     )
-set (OPAL_LIBS ${OPAL_LIBS}
-    CACHE INTERNAL "" FORCE
-    )
 
 # configure file
 configure_file (
@@ -277,10 +323,3 @@ install (
     DESTINATION "${CMAKE_INSTALL_PREFIX}/lib/cmake/${PROJECT_NAME}"
     RENAME ${PROJECT_NAME}Config.cmake
     )
-
-# vi: set et ts=4 sw=4 sts=4:
-# Local Variables:
-# mode:cmake
-# cmake-tab-width: 4
-# indent-tabs-mode:nil
-# End:

CMakeModules/FindGSL.cmake

 #
-# Find GSL includes and library
+# Find GSL includes and libraries
 #
-# FindGSL() shipped with CMake is somehow broken in newer versions (>= 3.9,
-# maybe also 3.7 and 3.8). It works only if GSL_ROOT_DIR is set. Hints do
-# not work.
+# The FindGSL module shipped with CMake has some drawbacks. To be 
+# able to find GSL, the CMake variable GSL_ROOT_DIR *must* be set.
+# The environment variable C_INCLUDE_PATH and LIBRARY_PATH are
+# *not* taken into account.
 #
-# :FIXME: For the time being we use our own version.
+# The following variables will be set if GSL is found:
+#
+# GSL_INCLUDE_DIR	where to find GSL include files
+# GSL_LIBRARY		GSL library to link against.
+# GSL_CBLAS_LIBRARY 	GSL CBlas library to link against
+# GSL_LIBRARIES		GSL libraries required for linking
+# GSL_FOUND		set to True if GSL was found
 #
-# GSL_INCLUDE_DIR - where to find ippl.h
-# GSL_LIBRARY     - GSL library to link against.
-# GSL_CBLAS_LIBRARY - GSL CBlas library to link against
-# GSL_FOUND       - do not attempt to use if "no" or undefined.
 
-FIND_PATH (GSL_INCLUDE_DIR gsl/gsl_fft.h
-    HINTS $ENV{GSL_ROOT_DIR}/include $ENV{GSL_INCLUDE_PATH} $ENV{GSL_INCLUDE_DIR} $ENV{GSL_PREFIX}/include $ENV{GSL_DIR}/include $ENV{GSL}/include
-    PATHS ENV C_INCLUDE_PATH
-)
+if( DEFINED ENV{GSL_ROOT_DIR} )
+  set( GSL_ROOT_DIR $ENV{GSL_ROOT_DIR} )
+elseif( DEFINED ENV{GSL_DIR} )
+  set( GSL_ROOT_DIR $ENV{GSL_DIR} )
+elseif( DEFINED ENV{GSL_HOME} )
+  set( GSL_ROOT_DIR $ENV{GSL_HOME} )
+elseif( DEFINED ENV{GSL_PREFIX} )
+  set( GSL_ROOT_DIR $ENV{GSL_PREFIX} )
+elseif( DEFINED ENV{GSL} )
+  set( GSL_ROOT_DIR $ENV{GSL} )
+else()
+  set( GSL_ROOT_DIR "/usr" )
+endif()
 
+find_path( GSL_INCLUDE_DIR gsl/gsl_fft.h
+  HINTS ${GSL_ROOT_DIR}/include $ENV{GSL_INCLUDE_PATH} $ENV{GSL_INCLUDE_DIR}
+  PATHS ENV C_INCLUDE_PATH
+  )
 
-FIND_LIBRARY (GSL_LIBRARY gsl
-    HINTS $ENV{GSL_ROOT_DIR}/lib $ENV{GSL_LIBRARY_PATH} $ENV{GSL_LIBRARY_DIR} $ENV{GSL_PREFIX}/lib $ENV{GSL_DIR}/lib $ENV{GSL}/lib
-    PATHS ENV LIBRARY_PATH
-)
+find_library( GSL_LIBRARY gsl
+  HINTS ${GSL_ROOT_DIR}/lib $ENV{GSL_LIBRARY_PATH} $ENV{GSL_LIBRARY_DIR}
+  PATHS ENV LIBRARY_PATH
+  )
 
-FIND_LIBRARY (GSL_CBLAS_LIBRARY gslcblas
-    HINTS $ENV{GSL_ROOT_DIR}/lib $ENV{GSL_LIBRARY_PATH} $ENV{GSL_LIBRARY_DIR} $ENV{GSL_PREFIX}/lib $ENV{GSL_DIR}/lib $ENV{GSL}/lib
-    PATHS ENV LIBRARY_PATH
-)
+find_library (GSL_CBLAS_LIBRARY gslcblas
+  HINTS ${GSL_ROOT_DIR}/lib $ENV{GSL_LIBRARY_PATH} $ENV{GSL_LIBRARY_DIR}
+  PATHS ENV LIBRARY_PATH
+  )
 
-IF (GSL_INCLUDE_DIR AND GSL_LIBRARY AND GSL_CBLAS_LIBRARY)
-    SET( GSL_FOUND "YES" )
-ENDIF()
+if( GSL_INCLUDE_DIR AND GSL_LIBRARY AND GSL_CBLAS_LIBRARY)
+  set( GSL_FOUND "YES" )
+  set( GSL_LIBRARIES ${GSL_LIBRARY},${GSL_CBLAS_LIBRARY} )
+endif()
 
-IF (GSL_FOUND)
-   IF (NOT GSL_FIND_QUIETLY)
-      MESSAGE(STATUS "Found GSL libraries: ${GSL_LIBRARY}")
-      MESSAGE(STATUS "Found GSL include dir: ${GSL_INCLUDE_DIR}")
-   ENDIF (NOT GSL_FIND_QUIETLY)
-ELSE ()
-   IF (GSL_FIND_REQUIRED)
-      MESSAGE(FATAL_ERROR "Could not find GSL!")
-   ENDIF ()
-ENDIF ()
+if( GSL_FOUND )
+  if( NOT GSL_FIND_QUIETLY )
+    message( STATUS "Found GSL libraries: ${GSL_LIBRARY}")
+    message( STATUS "Found GSL include dir: ${GSL_INCLUDE_DIR}")
+  endif()
+else()
+  if( GSL_FIND_REQUIRED )
+    message( FATAL_ERROR "Could not find GSL!" )
+  endif()
+endif()

CMakeModules/FindGTest.cmake

 #
 # Find gtest Google Test includes and library
 #
-# gtest
-# It can be found at:
-#     http://amas.web.psi.ch/tools/GSL/index.html
-#
 # GTEST_INCLUDE_DIR - where to find gtest/gtest.h
 # GTEST_LIBRARY     - libgtest.a path
 # GTEST_MAIN_LIBRARY     - libgtest_main.a path

CMakeModules/FindH5Hut.cmake

@@ -43,9 +43,3 @@ IF (HAVE_API2_FUNCTIONS)
 ELSE (HAVE_API2_FUNCTIONS)
     MESSAGE (ERROR "H5hut >= 2 required")
 ENDIF (HAVE_API2_FUNCTIONS)
-
-# Local Variables:
-# mode:cmake
-# cmake-tab-width: 4
-# indent-tabs-mode:nil
-# End:

CMakeModules/FindMITHRA.cmake

+#
+# Find MITHRA package
+# https://github.com/aryafallahi/mithra
+#
+# MITHRA_INCLUDE_DIR
+# MITHRA_LIBRARY_DIR
+# MITHRA_FOUND
+
+find_path (MITHRA_INCLUDE_DIR mithra/classes.h
+    HINTS $ENV{MITHRA_INCLUDE_DIR} $ENV{MITHRA_INCLUDE_PATH} $ENV{MITHRA_PREFIX}/include $ENV{MITHRA}/include
+)
+
+find_path (MITHRA_LIBRARY_DIR libmithra.a
+    HINTS $ENV{MITHRA_LIBRARY_DIR} $ENV{MITHRA_LIBRARY_PATH} $ENV{MITHRA_PREFIX}/lib $ENV{MITHRA}/lib
+)
+
+if (MITHRA_INCLUDE_DIR AND MITHRA_LIBRARY_DIR)
+    set (MITHRA_FOUND "YES")
+endif ()
+
+if (MITHRA_FOUND)
+    if (NOT MITHRA_FIND_QUIETLY)
+        message (STATUS "Found MITHRA include dir: ${MITHRA_INCLUDE_DIR}")
+        message (STATUS "Found MITHRA library dir: ${MITHRA_LIBRARY_DIR}")
+    endif ()
+else (MITHRA_FOUND)
+    if (MITHRA_FIND_REQUIRED)
+        if (NOT MITHRA_INCLUDE_DIR)
+            message (WARNING 
+                "MITHRA include directory was not found! "
+                "Make sure that MITHRA is compiled and that "
+                "the directory mithra/include/mithra has been automatically created. "
+                "Also make sure that at least one of the following "
+                "environment variables is set: "
+                "MITHRA_INCLUDE_DIR, MITHRA_INCLUDE_PATH, MITHRA_PREFIX, or MITHRA.")
+        endif ()
+        if (NOT MITHRA_LIBRARY_DIR)
+            message (WARNING 
+                "MITHRA library was not found! "
+                "Make sure that MITHRA is compiled and that "
+                "the directory mithra/lib has been automatically created. "
+                "Also make sure that at least one of the following "
+                "environment variables is set: "
+                "MITHRA_LIBRARY_DIR, MITHRA_LIBRARY_PATH, MITHRA_PREFIX, or MITHRA.")
+        endif ()
+        message (STATUS "MITHRA can be downloaded and compiled from https://github.com/aryafallahi/mithra.git")
+        message (FATAL_ERROR "Could not find MITHRA!")
+    endif (MITHRA_FIND_REQUIRED)
+endif (MITHRA_FOUND)

Doxyfile → Doxyfile.in

@@ -38,7 +38,7 @@ PROJECT_NAME           = "OPAL (Object Oriented Parallel Accelerator Library)"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = "2.0.0"
+PROJECT_NUMBER         = "2024.2"
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -51,7 +51,7 @@ PROJECT_BRIEF          = "OPAL"
 # and the maximum width should not exceed 200 pixels. Doxygen will copy the logo
 # to the output directory.
 
-PROJECT_LOGO           = ./amas.png
+PROJECT_LOGO           =
 
 # The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path
 # into which the generated documentation will be written. If a relative path is
@@ -764,10 +764,9 @@ WARN_LOGFILE           =
 # spaces.
 # Note: If this tag is empty the current directory is searched.
 
-INPUT                  = ./src \
-                         ./src/Classic \
-                         ./ippl/src \
-			 ./optimizer
+INPUT                  = @CMAKE_SOURCE_DIR@/src \
+                         @CMAKE_SOURCE_DIR@/ippl/src \
+                         @CMAKE_SOURCE_DIR@/optimizer
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
@@ -787,15 +786,8 @@ INPUT_ENCODING         = UTF-8
 # *.md, *.mm, *.dox, *.py, *.f90, *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf,
 # *.qsf, *.as and *.js.
 
-FILE_PATTERNS          = *.c \
-                         *.cpp \
-                         *.cc \
-                         *.h \
-                         *.hh \
-			 *.H \
-			 *.f90 \
-			 *.hpp
-			 
+FILE_PATTERNS          = *.c* \
+                         *.h*
 
 # The RECURSIVE tag can be used to specify whether or not subdirectories should
 # be searched for input files as well.
@@ -1417,7 +1409,7 @@ EXT_LINKS_IN_WINDOW    = NO
 # Minimum value: 8, maximum value: 50, default value: 10.
 # This tag requires that the tag GENERATE_HTML is set to YES.
 
-FORMULA_FONTSIZE       = 10
+FORMULA_FONTSIZE       = 15
 
 # Use the FORMULA_TRANPARENT tag to determine whether or not the images
 # generated for formulas are transparent PNGs. Transparent PNGs are not
@@ -1439,7 +1431,7 @@ FORMULA_TRANSPARENT    = YES
 # The default value is: NO.
 # This tag requires that the tag GENERATE_HTML is set to YES.
 
-USE_MATHJAX            = NO
+USE_MATHJAX            = YES
 
 # When MathJax is enabled you can set the default output format to be used for
 # the MathJax output. See the MathJax site (see:
@@ -1469,7 +1461,7 @@ MATHJAX_RELPATH        = http://cdn.mathjax.org/mathjax/latest
 # MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols
 # This tag requires that the tag USE_MATHJAX is set to YES.
 
-MATHJAX_EXTENSIONS     =
+MATHJAX_EXTENSIONS     = AMSmath AMSsymbols
 
 # The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces
 # of code that will be used on startup of the MathJax code. See the MathJax site
@@ -1572,7 +1564,7 @@ EXTRA_SEARCH_MAPPINGS  =
 # If the GENERATE_LATEX tag is set to YES doxygen will generate LaTeX output.
 # The default value is: YES.
 
-GENERATE_LATEX         = YES
+GENERATE_LATEX         = NO
 
 # The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. If a
 # relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
@@ -1625,7 +1617,8 @@ PAPER_TYPE             = a4wide
 # This tag requires that the tag GENERATE_LATEX is set to YES.
 
 EXTRA_PACKAGES         = amsfonts, \
-                         amssymb
+                         amssymb, \
+                         amsmath
 
 # The LATEX_HEADER tag can be used to specify a personal LaTeX header for the
 # generated LaTeX document. The header should contain everything until the first

LICENSE

+LICENSES/OPAL/LICENSE
\ No newline at end of file

cmake/OPALConfig.cmake.in

-SET(${PROJECT_NAME}_CMAKE_CXX_FLAGS "${${PROJECT_NAME}_CXX_FLAGS}")
-SET(${PROJECT_NAME}_INCLUDE_DIR "${CMAKE_INSTALL_PREFIX}/include")
-SET(${PROJECT_NAME}_LIBRARY_DIR "${CMAKE_INSTALL_PREFIX}/lib")
-SET(${PROJECT_NAME}_LIBRARY "${CMAKE_INSTALL_PREFIX}/lib/libOPAL.a")
+set (${PROJECT_NAME}_CMAKE_CXX_FLAGS "${${PROJECT_NAME}_CXX_FLAGS}")
+set (${PROJECT_NAME}_INCLUDE_DIR "${CMAKE_INSTALL_PREFIX}/include")
+set (${PROJECT_NAME}_LIBRARY_DIR "${CMAKE_INSTALL_PREFIX}/lib")
+set (${PROJECT_NAME}_LIBRARY "${CMAKE_INSTALL_PREFIX}/lib/libOPAL.a")
 
-SET(IPPL_CMAKE_CXX_FLAGS "${IPPL_CMAKE_CXX_FLAGS}")
-
-SET(${PROJECT_NAME}_LIBRARIES
+set (${PROJECT_NAME}_LIBRARIES
     ${GSL_LIBRARY}
     ${GSL_CBLAS_LIBRARY}
     ${H5Hut_LIBRARY}
@@ -20,7 +18,7 @@ SET(${PROJECT_NAME}_LIBRARIES
     ${CMAKE_DL_LIBS}
 )
 
-SET(${PROJECT_NAME}_INCLUDE_DIRS
+set (${PROJECT_NAME}_INCLUDE_DIRS
     ${OPAL_SRC_DIR}
     ${CLASSIC_SRC_DIR}
     ${IPPL_INCLUDE_DIR}

gen_OPALrevision

+#!/bin/bash
+#
+# create/rewrite 'src/OPALrevision.h'
+#
+
+print () {
+    echo '#define GIT_VERSION '\"$1\" > src/OPALrevision-new.h
+    if [[ -e  src/OPALrevision.h ]] && cmp -s src/OPALrevision.h src/OPALrevision-new.h; then
+        rm src/OPALrevision-new.h
+    else
+        mv src/OPALrevision-new.h src/OPALrevision.h
+    fi
+}
+
+# if git(1) is not in $PATH
+if ! which -a git &> /dev/null; then
+    if  [[ -e  src/OPALrevision.h ]]; then
+        # do nothing if file exist
+        exit 0
+    else
+        # otherwise create file
+        print unknown
+        exit $?
+    fi
+fi
+
+# if we are in a work-tree
+if git rev-parse --is-inside-work-tree &> /dev/null; then
+    # create/rewrite file
+    revision=$(git rev-parse HEAD)
+    print ${revision}
+    exit $?
+fi
+
+# if we are not in a work tree and the file does not exist
+if [[ ! -e  src/OPALrevision.h ]]; then
+    # create file
+    print unknown
+    exit $?
+fi
+
+exit 0
\ No newline at end of file