Merge branch 'master' of gitlab.psi.ch:OPAL/opal-x/src

CMakeLists.txt

@@ -17,15 +17,11 @@ if (${CMAKE_BUILD_TYPE} STREQUAL "Release")
     add_definitions (-DNOPAssert)
 endif ()
 
-
-if (${CMAKE_BUILD_TYPE} STREQUAL "Debug")
+if (NOT "${IPPL_PLATFORMS}" STREQUAL "CUDA" AND ${CMAKE_BUILD_TYPE} STREQUAL "Debug")
     add_compile_options (-fsanitize=undefined,address)
     add_link_options (-fsanitize=undefined,address)
 endif ()
 
-
-
-
 add_compile_options (-Wall)
 add_compile_options (-Wunused)
 add_compile_options (-Wextra)
@@ -44,19 +40,15 @@ message (STATUS "Build type is: " ${CMAKE_BUILD_TYPE})
 set (CMAKE_CXX_STANDARD 20)
 set (CMAKE_CUDA_EXTENSIONS OFF)
 set (CMAKE_CXX_STANDARD_REQUIRED ON)
-set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "-g  -O3")
+set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "-g -O3")
 set (CMAKE_CXX_FLAGS_RELEASE "-g -O3 ")
-set (CMAKE_CXX_FLAGS_DEBUG  "-g")
+set (CMAKE_CXX_FLAGS_DEBUG  "-g -O0")
 
 if (NOT "${IPPL_PLATFORMS}" STREQUAL "CUDA" AND CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
     # Use sanitizer flags for GCC or Clang only
      set (CMAKE_CXX_FLAGS_DEBUG  "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address,undefined")
 endif ()
 
-#if ("${IPPL_PLATFORMS}" STREQUAL "CUDA")
-#    set (CMAKE_CXX_FLAGS_DEBUG  "${CMAKE_CXX_FLAGS_DEBUG} -G") #-diag-suppress=611")
-#endif()
-
 message (STATUS "CMAKE_CXX_FLAGS_DEBUG: ${CMAKE_CXX_FLAGS_DEBUG}")
 
 # Disable compile time assert (used by IPPL)

README.md

@@ -6,6 +6,7 @@
 
 ## Modules needed OPENMP build
 
+```
 cmake/3.25.2
 
 openmpi/4.1.5_slurm
@@ -25,20 +26,58 @@ gtest/1.13.0-1
 hdf5/1.10.8_slurm     
 
 gnutls/3.5.19
+```
+
 
 
 ## Clone repo and build opal-x with OPENMP 
 
-% git clone git@gitlab.psi.ch:OPAL/opal-x/src.git opal-x
+```
+$ git clone git@gitlab.psi.ch:OPAL/opal-x/src.git opal-x
+
+$ cd opal-x
+
+$ ./gen_OPALrevision
+```
+
+
+### To compile for OPENMP:
+```
+$ mkdir build_openmp && cd build_openmp
+$ cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_STANDARD=20  -DENABLE_SOLVERS=ON  -DENABLE_FFT=ON -DIPPL_PLATFORMS=openmp
+```
+
+
+### To compile for GPU, for example Amper80 on Gwendolen
+```
+$ mkdir build_cuda && cd build_cuda
+```
+
+in debug mode:
 
-% cd opal-x
+```
+$ cmake .. -DCMAKE_BUILD_TYPE=Debug -DIPPL_PLATFORMS=CUDA -DKokkos_ARCH_AMPERE80=ON -DCMAKE_CXX_STANDARD=20 -DENABLE_FFT=ON  -DENABLE_SOLVERS=ON
+```
 
-% ./gen_OPALrevision
+and release (optimized) mode:
+```
+$ cmake .. -DCMAKE_BUILD_TYPE=Release -DIPPL_PLATFORMS=CUDA -DKokkos_ARCH_AMPERE80=ON -DCMAKE_CXX_STANDARD=20 -DENABLE_FFT=ON  -DENABLE_SOLVERS=ON
+```
 
-% mkdir build_openmp && cd build_openmp
+### To compile for other GPU architecture, like Pascal on the Merlin's login node
+```
+$ mkdir build_cuda_login && cd build_cuda_login
+```
 
-% cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_STANDARD=20  -DENABLE_SOLVERS=ON  -DENABLE_FFT=ON -DIPPL_PLATFORMS=openmp
+in debug mode:
 
+```
+$ cmake .. -DCMAKE_BUILD_TYPE=Debug -DIPPL_PLATFORMS=CUDA -DKokkos_ARCH_PASCAL61=ON -DCMAKE_CXX_STANDARD=20 -DENABLE_FFT=ON  -DENABLE_SOLVERS=ON
+```
 
+and release (optimized) mode:
+```
+$ cmake .. -DCMAKE_BUILD_TYPE=Release -DIPPL_PLATFORMS=CUDA -DKokkos_ARCH_PASCAL61=ON -DCMAKE_CXX_STANDARD=20 -DENABLE_FFT=ON  -DENABLE_SOLVERS=ON
+```
 
 The documentation has been moved to the [Wiki](https://gitlab.psi.ch/OPAL/src/wikis/home).

src/Distribution/Gaussian.hpp

@@ -72,7 +72,7 @@ public:
         }
 
         Kokkos::parallel_reduce(
-            "calc moments of particle distr.", numberOfParticles,
+            "calc moments of particle distr.", nlocal,
             KOKKOS_LAMBDA(
                     const int k, double& cent0, double& cent1, double& cent2) {
                     cent0 += Rview(k)[0];
@@ -81,16 +81,16 @@ public:
             },
             Kokkos::Sum<double>(loc_meanR[0]), Kokkos::Sum<double>(loc_meanR[1]), Kokkos::Sum<double>(loc_meanR[2]));
         Kokkos::fence();
-        ippl::Comm->barrier();
 
         MPI_Allreduce(loc_meanR, meanR, 3, MPI_DOUBLE, MPI_SUM, ippl::Comm->getCommunicator());
+        ippl::Comm->barrier();
 
         for(int i=0; i<3; i++){
            meanR[i] = meanR[i]/(1.*numberOfParticles);
         }
 
         Kokkos::parallel_for(
-                numberOfParticles,KOKKOS_LAMBDA(
+                nlocal, KOKKOS_LAMBDA(
                     const int k) {
                     Rview(k)[0] -= meanR[0];
                     Rview(k)[1] -= meanR[1];
@@ -127,16 +127,16 @@ public:
             },
 	    Kokkos::Sum<double>(loc_meanP[0]), Kokkos::Sum<double>(loc_meanP[1]), Kokkos::Sum<double>(loc_meanP[2]));
         Kokkos::fence();
-        ippl::Comm->barrier();
 
         MPI_Allreduce(loc_meanP, meanP, 3, MPI_DOUBLE, MPI_SUM, ippl::Comm->getCommunicator());
+        ippl::Comm->barrier();
 
         for(int i=0; i<3; i++){
            meanP[i] = meanP[i]/(1.*numberOfParticles);
         }
 
         Kokkos::parallel_for(
-            numberOfParticles,KOKKOS_LAMBDA(
+            nlocal, KOKKOS_LAMBDA(
                     const int k) {
                     Pview(k)[0] -= meanP[0];
                     Pview(k)[1] -= meanP[1];
@@ -149,7 +149,7 @@ public:
         // correct the mean
         double avrgpz = opalDist_m->getAvrgpz();
         Kokkos::parallel_for(
-            numberOfParticles,KOKKOS_LAMBDA(
+            nlocal, KOKKOS_LAMBDA(
                     const int k) {
                     Pview(k)[2] += avrgpz;
             }