|
|
= Build OPAL on Piz Daint at CSCS
|
|
|
Matthias Frey
|
|
|
:doctype: article
|
|
|
:numbered:
|
|
|
:toc:
|
|
|
|
|
|
'''
|
|
|
== Account
|
|
|
|
|
|
'''
|
|
|
=== Creation
|
|
|
First you need a proper CSCS account. In order to do that click on http://www.cscs.ch/user_lab/applying_for_accounts/index.html[this link].
|
|
|
|
|
|
'''
|
|
|
=== Login
|
|
|
One has first to access http://www.cscs.ch/computers/ela/index.html[Ela] before being able to login to http://www.cscs.ch/computers/piz_daint/index.html[Piz Daint].
|
|
|
|
|
|
```
|
|
|
$ ssh username@ela.cscs.ch
|
|
|
> ssh daint
|
|
|
|
|
|
```
|
|
|
|
|
|
'''
|
|
|
== Modules
|
|
|
The hybrid architecture of Piz Daint allows to run on GPUs and CPUs. Therefore, it provides two modules in order to set up the
|
|
|
environment properly:
|
|
|
|
|
|
```
|
|
|
> module load daint-gpu
|
|
|
> module load daint-mc
|
|
|
```
|
|
|
This instruction only shows the installation of OPAL for the multicore architecture (i.e. the module ```daint-mc```).
|
|
|
|
|
|
'''
|
|
|
=== Load environment
|
|
|
```
|
|
|
> module load daint-mc
|
|
|
> module switch PrgEnv-cray/6.0.8 PrgEnv-gnu/6.0.8
|
|
|
> module load CMake/3.14.5
|
|
|
```
|
|
|
|
|
|
'''
|
|
|
=== Export environment variables for builds
|
|
|
```
|
|
|
> export TARGET_DIR=$HOME/gcc/8.3.0
|
|
|
> export DOWNLOADS_DIR=$SCRATCH
|
|
|
> export SRC_DIR=$SCRATCH
|
|
|
> export NJOBS=4
|
|
|
```
|
|
|
|
|
|
'''
|
|
|
=== Building OpenBLAS
|
|
|
|
|
|
'''
|
|
|
==== Installation script (openblas_install.sh)
|
|
|
```bash
|
|
|
#!/bin/bash
|
|
|
|
|
|
# recipe for:
|
|
|
P=OpenBLAS
|
|
|
V=0.2.20
|
|
|
|
|
|
# download
|
|
|
curl -L \
|
|
|
--output "${DOWNLOADS_DIR}/$P-$V.tar.gz" \
|
|
|
"http://github.com/xianyi/$P/archive/v$V.tar.gz"
|
|
|
|
|
|
# unpack
|
|
|
mkdir -p "${SRC_DIR}/$P" && cd "$_"
|
|
|
tar xvf "${DOWNLOADS_DIR}/$P-$V.tar.gz"
|
|
|
cd "$P-$V"
|
|
|
|
|
|
# configure
|
|
|
# nothing to configure
|
|
|
|
|
|
# compile & install
|
|
|
make TARGET=CORE2
|
|
|
make TARGET=CORE2 PREFIX="${PREFIX}" install
|
|
|
|
|
|
```
|
|
|
'''
|
|
|
==== Installation procedure
|
|
|
```
|
|
|
> export PREFIX=$TARGET_DIR/OpenBLAS/0.2.20
|
|
|
> ./openblas_install.sh
|
|
|
> export OPENBLAS_DIR=$PREFIX
|
|
|
> export OPENBLAS_HOME=$OPENBLAS_DIR
|
|
|
> export OPENBLAS_INCLUDE_DIR=$OPENBLAS_HOME/include
|
|
|
> export OPENBLAS_LIBRARY_DIR=$OPENBLAS_HOME/lib
|
|
|
> export OPENBLAS_PREFIX=$OPENBLAS_HOME
|
|
|
> export OPENBLAS_VERSION=0.2.20
|
|
|
> export BLASLIB=OPENBLAS_LIBRARY_DIR/libopenblas.a
|
|
|
```
|
|
|
|
|
|
'''
|
|
|
=== Building HDF5
|
|
|
|
|
|
'''
|
|
|
==== Installation script (hdf5_install.sh)
|
|
|
|
|
|
```
|
|
|
#!/bin/bash
|
|
|
|
|
|
# recipe for:
|
|
|
P=hdf5
|
|
|
V_MAJOR=1
|
|
|
V_MINOR=10
|
|
|
V=${V_MAJOR}.${V_MINOR}.6
|
|
|
|
|
|
# download
|
|
|
curl -L \
|
|
|
--output "${DOWNLOADS_DIR}/$P-$V.tar.gz" \
|
|
|
"https://support.hdfgroup.org/ftp/HDF5/releases/$P-$V_MAJOR.$V_MINOR/$P-$V/src/$P-$V.tar.gz"
|
|
|
|
|
|
#unpack
|
|
|
mkdir -p "${SRC_DIR}/$P" && cd "$_"
|
|
|
tar xvf "${DOWNLOADS_DIR}/$P-$V.tar.gz"
|
|
|
|
|
|
#configure
|
|
|
mkdir -p "${SRC_DIR}/$P/build" && cd "$_"
|
|
|
CXX=CC CC=cc Ftn=ftn \
|
|
|
"${SRC_DIR}/$P/$P-$V/configure" \
|
|
|
--prefix="${PREFIX}" \
|
|
|
--enable-shared \
|
|
|
--enable-parallel \
|
|
|
--enable-cxx \
|
|
|
--enable-unsupported \
|
|
|
--with-pic
|
|
|
|
|
|
# compile & install
|
|
|
make -j ${NJOBS}
|
|
|
make install
|
|
|
|
|
|
```
|
|
|
|
|
|
'''
|
|
|
==== Installation procedure
|
|
|
|
|
|
```
|
|
|
> export PREFIX=$TARGET_DIR/hdf5/1.10.6
|
|
|
> ./hdf5_install.sh
|
|
|
> export HDF5_DIR=$PREFIX
|
|
|
> export HDF5_HOME=$HDF5_DIR
|
|
|
> export HDF5_PREFIX=$HDF5_DIR
|
|
|
> export HDF5_INCLUDE_DIR=$HDF5_HOME/include
|
|
|
> export HDF5_LIBRARY_DIR=$HDF5_HOME/lib
|
|
|
> export HDF5_VERSION=1.10.6
|
|
|
```
|
|
|
|
|
|
'''
|
|
|
=== Building H5hut
|
|
|
'''
|
|
|
==== Installation script (h5hut_install.sh)
|
|
|
```
|
|
|
#!/bin/bash
|
|
|
|
|
|
# recipe for:
|
|
|
P=H5hut
|
|
|
V=2.0.0rc6
|
|
|
SHA=7e7a85f14fc3ee123c122859a641db624de41b2a
|
|
|
|
|
|
# download
|
|
|
curl -L \
|
|
|
--output "${DOWNLOADS_DIR}/$P-$V.tar.gz" \
|
|
|
"https://gitlab.psi.ch/$P/src/repository/archive.tar.gz?ref=$P-$V"
|
|
|
|
|
|
# unpack
|
|
|
mkdir -p "${SRC_DIR}/$P" && cd "$_"
|
|
|
tar xvf "${DOWNLOADS_DIR}/$P-$V.tar.gz"
|
|
|
|
|
|
# pre-configure
|
|
|
cd "src-$P-$V-$SHA"
|
|
|
./autogen.sh
|
|
|
cd ..
|
|
|
|
|
|
# configure
|
|
|
mkdir -p "${SRC_DIR}/$P/build" && cd "$_"
|
|
|
CXX=CC CC=cc FC=ftn \
|
|
|
"${SRC_DIR}/$P/src-$P-$V-$SHA/configure" \
|
|
|
--prefix="${PREFIX}" \
|
|
|
--enable-parallel \
|
|
|
--with-hdf5="${HDF5_PREFIX}" \
|
|
|
--with-pic \
|
|
|
--enable-python
|
|
|
|
|
|
|
|
|
# compile & install
|
|
|
make -j ${NJOBS}
|
|
|
make install
|
|
|
|
|
|
```
|
|
|
|
|
|
'''
|
|
|
==== Installation procedure
|
|
|
```
|
|
|
> export PREFIX=$TARGET_DIR/H5hut/2.0.0rc6
|
|
|
> ./h5hut_install.sh
|
|
|
> export H5HUT_DIR=$TARGET_DIR/H5hut/2.0.0rc6
|
|
|
> export H5HUT_HOME=$H5HUT_DIR
|
|
|
> export H5HUT_PREFIX=$H5HUT_HOME
|
|
|
> export H5HUT_INCLUDE_DIR=$H5HUT_HOME/include
|
|
|
> export H5HUT_LIBRARY_DIR=$H5HUT_HOME/lib
|
|
|
> export H5HUT_VERSION=2.0.0rc6
|
|
|
```
|
|
|
|
|
|
'''
|
|
|
=== Building Boost
|
|
|
|
|
|
'''
|
|
|
==== Installation script (boost_install.sh)
|
|
|
|
|
|
```
|
|
|
#!/bin/bash
|
|
|
|
|
|
# recipe for:
|
|
|
P=boost
|
|
|
V=1_70_0
|
|
|
|
|
|
# download
|
|
|
curl -L \
|
|
|
--output "${DOWNLOADS_DIR}/${P}_$V.tar.gz" \
|
|
|
"https://netcologne.dl.sourceforge.net/project/$P/$P/${V//_/.}/${P}_$V.tar.gz"
|
|
|
|
|
|
# unpack
|
|
|
mkdir -p "${SRC_DIR}/$P" && cd "$_"
|
|
|
tar xvf "${DOWNLOADS_DIR}/${P}_$V.tar.gz"
|
|
|
|
|
|
# configure
|
|
|
mkdir -p "${SRC_DIR}/$P/build"
|
|
|
cd "${SRC_DIR}/$P/${P}_$V"
|
|
|
./bootstrap.sh \
|
|
|
--prefix="${PREFIX}" \
|
|
|
--with-libraries=mpi,chrono,filesystem,iostreams,regex,serialization,system,timer
|
|
|
|
|
|
echo "using mpi : : " >> project-config.jam
|
|
|
echo "<include>\"${MPICH_DIR}/include\"" >> project-config.jam
|
|
|
echo "<library-path>\"${MPICH_DIR}/lib\"" >> project-config.jam
|
|
|
echo "<find-shared-library>mpich" >> project-config.jam
|
|
|
echo "<find-shared-library>mpichcxx" >> project-config.jam
|
|
|
echo "<find-shared-library>fmpich ;" >> project-config.jam
|
|
|
|
|
|
# compile & install
|
|
|
./b2 \
|
|
|
--build-dir=""${SRC_DIR}/$P/build"" \
|
|
|
--layout=system \
|
|
|
variant=release \
|
|
|
link=shared,static \
|
|
|
threading=multi \
|
|
|
stage
|
|
|
./b2 \
|
|
|
--build-dir=""${SRC_DIR}/$P/build"" \
|
|
|
--layout=system \
|
|
|
variant=release \
|
|
|
link=shared,static \
|
|
|
threading=multi \
|
|
|
install
|
|
|
```
|
|
|
|
|
|
'''
|
|
|
==== Installation procedure
|
|
|
```
|
|
|
> export PREFIX=$TARGET_DIR/boost/1.70.0
|
|
|
> ./boost_install.sh
|
|
|
> export BOOST_DIR=$PREFIX
|
|
|
> export BOOST_HOME=$BOOST_DIR
|
|
|
> export BOOST_INCLUDE_DIR=$BOOST_DIR/include
|
|
|
> export BOOST_LIBRARY_DIR=$BOOST_DIR/lib
|
|
|
> export BOOST_PREFIX=$BOOST_DIR
|
|
|
> export BOOST_VERSION=1.70.0
|
|
|
```
|
|
|
|
|
|
'''
|
|
|
=== Building GSL
|
|
|
|
|
|
'''
|
|
|
==== Installation script (gsl_install.sh)
|
|
|
```bash
|
|
|
#!/bin/bash
|
|
|
|
|
|
# recipe for:
|
|
|
P=gsl
|
|
|
V=${GSL_VERSION:-2.6}
|
|
|
|
|
|
FNAME="$P-$V.tar.gz"
|
|
|
DOWNLOAD_URL="ftp://ftp.gnu.org/gnu/$P/${FNAME}"
|
|
|
SRC_FILE="${DOWNLOADS_DIR}/${FNAME}"
|
|
|
|
|
|
# download
|
|
|
test -r "${SRC_FILE}" || curl -L --output "$_" "${DOWNLOAD_URL}"
|
|
|
|
|
|
# unpack
|
|
|
mkdir -p "${SRC_DIR}/$P" && cd "$_"
|
|
|
tar xvf "${SRC_FILE}"
|
|
|
|
|
|
# configure
|
|
|
mkdir -p "${SRC_DIR}/$P/build" && cd "$_"
|
|
|
"${SRC_DIR}/$P/$P-$V/configure" \
|
|
|
--prefix="${PREFIX}" \
|
|
|
--disable-shared
|
|
|
|
|
|
# compile & install
|
|
|
make -j ${NJOBS}
|
|
|
make install
|
|
|
```
|
|
|
|
|
|
'''
|
|
|
==== Installation procedure
|
|
|
```
|
|
|
> export PREFIX=$TARGET_DIR/gsl/2.6
|
|
|
> ./gsl_install.sh
|
|
|
> export GSL_DIR=$PREFIX
|
|
|
> export GSL_HOME=$GSL_DIR
|
|
|
> export GSL_INCLUDE_DIR=$GSL_HOME/include
|
|
|
> export GSL_LIBRARY_DIR=$GSL_HOME/lib
|
|
|
> export GSL_PREFIX=$GSL_DIR
|
|
|
> export GSL_VERSION=2.6
|
|
|
```
|
|
|
|
|
|
'''
|
|
|
=== Building OPAL
|
|
|
|
|
|
'''
|
|
|
==== Installation script (opal_install.sh)
|
|
|
```
|
|
|
#!/bin/bash
|
|
|
|
|
|
# recipe for:
|
|
|
P=OPAL
|
|
|
V=2.4.0
|
|
|
branch=master
|
|
|
|
|
|
# download
|
|
|
mkdir -p "${DOWNLOADS_DIR}/$P" && cd "$_"
|
|
|
curl -L \
|
|
|
--output "${DOWNLOADS_DIR}/${P}_$V.tar.gz" \
|
|
|
"https://gitlab.psi.ch/$P/src/repository/archive.tar.gz?ref=$P-$V"
|
|
|
|
|
|
# configure
|
|
|
mkdir -p "${SRC_DIR}/$P/build" && cd "$_"
|
|
|
CXX=CC CC=cc FC=ftn cmake \
|
|
|
-DCMAKE_INSTALL_PREFIX:STRING="${PREFIX}" \
|
|
|
-DCMAKE_BUILD_TYPE:STRING=Release \
|
|
|
-DCMAKE_C_COMPILER=cc \
|
|
|
-DCMAKE_CXX_COMPILER=CC \
|
|
|
"${DOWNLOADS_DIR}/$P/src"
|
|
|
|
|
|
# compile & install
|
|
|
make -j ${NJOBS}
|
|
|
```
|
|
|
'''
|
|
|
==== Installation procedure
|
|
|
```
|
|
|
> export PREFIX=$TARGET_DIR/opal/2.4.0
|
|
|
> ./opal_install.sh
|
|
|
```
|
|
|
|
|
|
You might add the executable path to the environment variable:
|
|
|
```
|
|
|
> export OPAL_EXE_PATH=$HOME/opal/build/src/
|
|
|
> export PATH=$OPAL_EXE_PATH:$PATH
|
|
|
```
|
|
|
Make sure you use the right directory. You can check if it worked with
|
|
|
```
|
|
|
> which opal
|
|
|
```
|
|
|
or
|
|
|
```
|
|
|
> whereis opal
|
|
|
```
|
|
|
|
|
|
'''
|
|
|
== Modification of ~/.bashrc
|
|
|
It's convenient to add a helper function to your `~/.bashrc` in order to make life easier:
|
|
|
|
|
|
```
|
|
|
load_opal_gcc830() {
|
|
|
module load daint-mc
|
|
|
module switch PrgEnv-cray/6.0.8 PrgEnv-gnu/6.0.8
|
|
|
module load CMake/3.14.5
|
|
|
|
|
|
TARGET_DIR=$HOME/gcc/8.3.0
|
|
|
|
|
|
export MPI_HOME=$MPICH_DIR
|
|
|
export MPI_PREFIX=$MPI_HOME
|
|
|
export MPI_INCLUDE_DIR=$MPI_HOME/include
|
|
|
export MPI_LIBRARY_DIR=$MPI_HOME/lib
|
|
|
|
|
|
export OPENBLAS_DIR=$TARGET_DIR/OpenBLAS/0.2.20
|
|
|
export OPENBLAS_HOME=$OPENBLAS_DIR
|
|
|
export OPENBLAS_INCLUDE_DIR=$OPENBLAS_HOME/include
|
|
|
export OPENBLAS_LIBRARY_DIR=$OPENBLAS_HOME/lib
|
|
|
export OPENBLAS_PREFIX=$OPENBLAS_HOME
|
|
|
export OPENBLAS_VERSION=0.2.20
|
|
|
export BLASLIB=$OPENBLAS_LIBRARY_DIR/libopenblas.a
|
|
|
|
|
|
export LD_LIBRARY_PATH=$OPENBLAS_LIBRARY_DIR/:$LD_LIBRARY_PATH
|
|
|
|
|
|
export BOOST_DIR=$TARGET_DIR/boost/1.70.0
|
|
|
export BOOST_HOME=$BOOST_DIR
|
|
|
export BOOST_INCLUDE_DIR=$BOOST_DIR/include
|
|
|
export BOOST_LIBRARY_DIR=$BOOST_DIR/lib
|
|
|
export BOOST_PREFIX=$BOOST_DIR
|
|
|
export BOOST_VERSION=1.70.0
|
|
|
|
|
|
export GSL_DIR=$TARGET_DIR/gsl/2.6
|
|
|
export GSL_HOME=$GSL_DIR
|
|
|
export GSL_INCLUDE_DIR=$GSL_HOME/include
|
|
|
export GSL_LIBRARY_DIR=$GSL_HOME/lib
|
|
|
export GSL_PREFIX=$GSL_DIR
|
|
|
export GSL_VERSION=2.6
|
|
|
|
|
|
export HDF5_DIR=$TARGET_DIR/hdf5/1.10.6
|
|
|
export HDF5_HOME=$HDF5_DIR
|
|
|
export HDF5_PREFIX=$HDF5_DIR
|
|
|
export HDF5_INCLUDE_DIR=$HDF5_HOME/include
|
|
|
export HDF5_LIBRARY_DIR=$HDF5_HOME/lib
|
|
|
export HDF5_VERSION=1.10.6
|
|
|
|
|
|
export H5HUT_DIR=$TARGET_DIR/H5hut/2.0.0rc6
|
|
|
export H5HUT_HOME=$H5HUT_DIR
|
|
|
export H5HUT_PREFIX=$H5HUT_HOME
|
|
|
export H5HUT_INCLUDE_DIR=$H5HUT_HOME/include
|
|
|
export H5HUT_LIBRARY_DIR=$H5HUT_HOME/lib
|
|
|
export H5HUT_VERSION=2.0.0rc6
|
|
|
|
|
|
export OPAL_EXE_PATH=$TARGET_DIR/opal/bin
|
|
|
export PATH=$OPAL_EXE_PATH:$PATH
|
|
|
|
|
|
export CRAYPE_LINK_TYPE=dynamic
|
|
|
}
|
|
|
```
|
|
|
|
|
|
At the next login (or by just doing `source ~/.bashrc`) you can load everything by executing
|
|
|
```
|
|
|
> load_opal_gcc830
|
|
|
``` |