Commit 8eb34d76 authored by gsell's avatar gsell
Browse files

Pmodules/modmanage.bash.in:

- refactor function names with namespace
- dependency resolution implemented for 'install' sub-command
parent 1ab70662
......@@ -17,12 +17,12 @@ source "${libdir}/libpmodules.bash"
PATH="${bindir}:${PATH}"
_exit () {
die 1 "Interrupted..."
std::die 1 "Interrupted..."
}
_err () {
info "Oops: got an error in function '${FUNCNAME[1]}', line ${BASH_LINENO[0]}"
die 1 "Aborting ..."
std::info "Oops: got an error in function '${FUNCNAME[1]}', line ${BASH_LINENO[0]}"
std::die 1 "Aborting ..."
}
trap '_exit' INT TERM
......@@ -221,7 +221,7 @@ subcommand_init() {
local target_prefixes=()
local user=''
local opts=''
opts=$(get_options -o h -l src: -l user: -l help -l version: -- "$@")
opts=$(pmodules::get_options -o h -l src: -l user: -l help -l version: -- "$@")
if [[ $? != 0 ]]; then
subcommand_help_init
exit 1
......@@ -255,30 +255,32 @@ subcommand_init() {
esac
shift
done
(( ${#target_prefixes[@]} != 0 )) || die 1 "Error: no target directory specified!"
(( ${#target_prefixes[@]} != 0 )) || \
std::die 1 "Error: no target directory specified!"
# if source directory is not passed as argument, derive it from script name
if [[ -z "${src}" ]]; then
src=$(cd "${bindir}/../../../.." && pwd)
fi
[[ -d "${src}" ]] || \
die 1 "Error: ${src}: source directory does not exist!"
std::die 1 "Error: ${src}: source directory does not exist!"
[[ -r "${src}/config/profile.bash" ]] || \
die 1 "Error: ${src}: shell profile does not exist or is not readable!"
std::die 1 "Error: ${src}: shell profile does not exist or is not readable!"
source "${src}/config/profile.bash"
local -i euid=$(id -u)
if (( euid == 0 )); then
[[ -n "${user}" ]] || \
die 1 "Error: --user parameter is required!"
std::die 1 "Error: --user parameter is required!"
id -u "${user}" > /dev/null 2>&1 || \
die 1 "Error: Unable to retrieve user id of user '${user}'"
std::die 1 "Error: Unable to retrieve user id of user '${user}'"
else
[[ -z "${user}" ]] || \
die 1 "Error: --user option is only allowed if running as root!"
std::die 1 "Error: --user option is only allowed if running as root!"
fi
check_pmodules_env || die 1 "Giving up..."
pmodules::check_env || \
std::die 1 "Giving up..."
echo "
Attempting to create a minimal module environment from the
......@@ -294,22 +296,25 @@ environment at '${PMODULES_ROOT}'
echo
if [[ -d "${target_prefix}" ]] && [[ ${force} == no ]]; then
echo "Warning: ${target_prefix} already exists."
get_YN_answer "Do you really want to re-run the initialization? (y/N) " || \
die 1 "Abort ..."
std::get_YN_answer "Do you really want to re-run the initialization? (y/N) " || \
std::die 1 "Abort ..."
fi
force='yes'
echo "Creating target directory '${target_prefix}'..."
$DRY mkdir -p "${target_prefix}" || die 1 "Error: make directory failed!"
$DRY mkdir -p "${target_prefix}" || \
std::die 1 "Error: make directory failed!"
echo
echo "Syncing configuration ..."
sync_config "${PMODULES_ROOT}" \
"${target_prefix}" || die 1 "Error: configuration synchronization failed!"
"${target_prefix}" || \
std::die 1 "Error: configuration synchronization failed!"
echo "Syncing Pmodules ${PMODULES_VERSION} from '${src_prefix}' to '${target_prefix}'..."
sync_module "Tools/Pmodules/${PMODULES_VERSION}" \
"${src_prefix}" \
"${target_prefix}" || die 1 "Error: sync Pmodules failed!"
"${target_prefix}" || \
std::die 1 "Error: sync Pmodules failed!"
mkdir -p "${target_prefix}/Tools/${PMODULES_MODULEFILES_DIR}"
echo
......@@ -321,7 +326,8 @@ environment at '${PMODULES_ROOT}'
if [[ -n "${user}" ]]; then
echo "Changing user of new module environment to '${user}'..."
$DRY chown -R "${user}" "${target_prefix}" || die 1 "Error: changing owner failed!"
$DRY chown -R "${user}" "${target_prefix}" || \
std::die 1 "Error: changing owner failed!"
echo
fi
echo "New minimal module environment created at '${target_prefix}'."
......@@ -337,70 +343,133 @@ environment at '${PMODULES_ROOT}'
}
declare -a Groups=()
declare -A HierarchyDepths
get_groups () {
local -r root="$1"
{
cd "${root}"
# for some unknown reason [A-Z]* doesn't work on (some?) SL6 systems
for f in [ABCDEFGHIJKLMNOPQRSTUVWXYZ]*; do
[[ -d ${f}/${PMODULES_MODULEFILES_DIR} ]] || continue
Groups+=( $f )
done
};
}
#
# $1: root of modulefile hierarchy
get_hierarchy_depth () {
local -r root="$1"
local -a modulefiles_dir
std::split_fname modulefiles_dir "${PMODULES_MODULEFILES_DIR}"
local -ir off=$(( ${#modulefiles_dir[@]} + 3 ))
{
cd "${root}"
local group
for group in "${Groups[@]}"; do
local fname=$(find "${group}/${PMODULES_MODULEFILES_DIR}" \
-depth \( -type f -o -type l \) -print -quit)
[[ -n ${fname} ]] || continue
#local -a tmp2=( ${fname//\// } )
local -a tmp
std::split_fname tmp "${fname}"
(( HierarchyDepths[$group]=${#tmp[@]}-off ))
done
};
}
subcommand_install() {
local opts=''
local -a with=()
local -a releases=()
local -a module_pattern=()
local -r src_prefix="${PMODULES_INSTALL_SOURCE}"
local -r target_prefix="${PMODULES_ROOT}"
local rel_modulefile=''
local -A modules_to_install
local -A _modules_to_group
local -A _hierarchy_nodes
while read output; do
local _group=${output%/modulefiles/*}
local _abs_modulename="${output#*/modulefiles/}"
_modules_to_group[${_abs_modulename}]=${_group}
IFS='/'
local -a _items=(${_abs_modulename})
IFS=${_saved_IFS}
local -i _n=${#_items[*]}
if (( _n >= 4 )); then
local _key=$( IFS='/'; echo "${_items[*]:$_n-4:2}" )
_hierarchy_nodes[$_key]=${_group}
fi
done < <({ cd "${src_prefix}" && find */modulefiles -type l -o -type f \! -name ".*"; } )
local opts=''
local -a with=()
local -a releases=()
local -a module_pattern=()
local src_prefix="${PMODULES_INSTALL_SOURCE}"
local -r target_prefix="${PMODULES_ROOT}"
local modulefile=''
local -A modules_to_install
local -A dependencies_to_install
local -A map_to_family
local -a initial_modulepath=()
#
# Resolve dependencies to given module
#
# $1: modulefile relativ to src prefix. Something like:
# MPI/modulefiles/gcc/4.9.2/openmpi/1.8.4/hdf5/1.8.14
#
# Notes:
# The variables
# initial_modulepath
# modules_to_install
# map_to_family
# from the calling function are used!
#
resolve_dependencies_of_module () {
local -r __rel_modulefile=$1
# :FIXME: This must be initialized with all "root" groups
local -a __modulepath=("${src_prefix}/Tools/modulefiles" "${src_prefix}/Programming/modulefiles")
IFS='/'
local -a _items=(${__rel_modulefile})
IFS=${_saved_IFS}
local -i _n=${#_items[@]}
local -i _i
local _prefix="${src_prefix}/${_items[0]}"
for (( _i = _n-2; _i >= 2; _i-=2 )); do
_prefix+="/${_items[$_i]}/${_items[_i+1]}"
local -r modulefile=$1
local -a modulepath=( "${initial_modulepath[@]}" )
# compute filename with dependencies of given module
local -i i=0 n=0
std::split_fname items n "${modulefile}"
local fname_dependencies="${src_prefix}/${items[0]}"
for (( i = n-2; i >= 2; i-=2 )); do
fname_dependencies+="/${items[$i]}/${items[i+1]}"
done
local __dep
while read __dep; do
[[ -z ${__dep} ]] && continue
local __long_module_name=$(find "${__modulepath[@]}" -path "*/${__dep}" | head -n 1 )
__long_module_name=${__long_module_name/${src_prefix}\/}
modules_to_install[${__long_module_name/}]='.'
if [[ -n ${_hierarchy_nodes[${__dep}]} ]]; then
local __path="${src_prefix}/${_hierarchy_nodes[${__dep}]}/modulefiles/"
__path+="${__long_module_name/*\/modulefiles\/}"
__modulepath+=( "${__path}" )
fname_dependencies+='/.dependencies'
[[ -r ${fname_dependencies} ]] || return 0
# loop over all dependecies
local dep
while read dep; do
# skip empty lines
# :FIXME: skip comments?!
[[ -z ${dep} ]] && continue
# search for module with current modulepath and remember
local modulename=$(find "${modulepath[@]}" -path "*/${dep}" 2>/dev/null | head -n 1 )
[[ -n ${modulename} ]] || \
std::die 3 "Oops: required module '${dep}' not found!"
modulename=${modulename/${src_prefix}\/}
dependencies_to_install[${modulename}]='.'
resolve_dependencies_of_module "${modulename}"
# append new node in hierarchy to modulepath
if [[ -n ${map_to_family[${dep}]} ]]; then
local path="${src_prefix}/${map_to_family[${dep}]}/${PMODULES_MODULEFILES_DIR}/"
path+="${modulename/*\/${PMODULES_MODULEFILES_DIR}\/}"
modulepath+=( "${path}" )
fi
done < "${_prefix}/.dependencies"
}
resolve_dependencies () {
local _saved_IFS=${IFS};
# a relative modulefile is something like:
# MPI/modulefiles/gcc/4.8.4/openmpi/1.8.4/hdf5/1.8.14
local _rel_modulefile
for _rel_modulefile in "${!modules_to_install[@]}"; do
resolve_dependencies_of_module "${_rel_modulefile}"
done
done < "${fname_dependencies}"
}
opts=$(get_options -o hf -l dry-run -l force -l with: -l release: -l help -l src: -- "$@")
print_modules_to_install() {
local modulefile
std::info "The following modules will be installed/updated:"
for modulefile in "${!modules_to_install[@]}"; do
if [[ -e "${target_prefix}/${modulefile}" ]]; then
std::info " Updating: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }"
else
std::info " Installing: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }"
fi
done 2>&1 | sort
if (( ${#dependencies_to_install[@]} > 0 )); then
std::info "\nThe following dependencies will be installed/updated:"
for modulefile in "${!dependencies_to_install[@]}"; do
if [[ -e "${target_prefix}/${modulefile}" ]]; then
std::info " Updating: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }"
else
std::info " Installing: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }"
fi
done 2>&1 | sort
fi
std::info ""
std::get_YN_answer "Do you want to continue? [n] " || \
std::die 1 "Aborting..."
std::info ""
}
opts=$(pmodules::get_options -o hf -l dry-run -l force -l with: -l release: -l help -l src: -- "$@")
if [[ $? != 0 ]]; then
subcommand_help_install
exit 1
......@@ -444,9 +513,52 @@ subcommand_install() {
esac
shift
done
[[ -n ${src_prefix} ]] \
|| std::die 3 "Oops: no installation source given."
[[ -d ${src_prefix} ]] \
|| std::die 3 "Oops: '${src_prefix}' is not a valid installation source."
# scan available groups and their depth
get_groups "${src_prefix}"
get_hierarchy_depth "${src_prefix}"
# set initial modulepath
local group
for group in "${!HierarchyDepths[@]}"; do
if (( ${HierarchyDepths[${group}]} == 0 )); then
initial_modulepath+=( "${src_prefix}/${group}/${PMODULES_MODULEFILES_DIR}" )
fi
done
#
# create a mapping from module name to their family.
# Examples:
# gcc/5.2.0 -> Compiler
# openmpi/1.8.4 -> MPI
local _fname=''
while read _fname; do
local _family="${_fname%/${PMODULES_MODULEFILES_DIR}/*}"
local -a items
std::split_fname items "${_fname#*/${PMODULES_MODULEFILES_DIR}/}"
local -i n=${#items[*]}
# We are only interested in families adding something to
# the modulepath.
if (( n >= 4 )); then
local _key=$( IFS='/'; echo "${items[*]:$n-4:2}" )
map_to_family[$_key]=${_family}
fi
done < <({ cd "${src_prefix}" && \
find */"${PMODULES_MODULEFILES_DIR}" \
\( -type l -o -type f \) \! -name ".*"; } 2>/dev/null )
#
# search for to be installed modules and their dependencies
#
local -i n=0
while read rel_modulefile; do
modules_to_install["${rel_modulefile}"]+='.'
while read modulefile; do
resolve_dependencies_of_module "${modulefile}"
modules_to_install["${modulefile}"]+='.'
let n+=1
done < <(${PMODULES_HOME}/bin/modulecmd bash search \
"${module_pattern[@]}" \
......@@ -454,37 +566,45 @@ subcommand_install() {
"${releases[@]/#/--release=}" \
--no-header --print-modulefiles \
--src="${src_prefix}" 2>&1 1>/dev/null)
(( n == 0 )) && die 0 "Nothing to install..."
resolve_dependencies
info "The following modules will be installed/updated:\n"
for rel_modulefile in "${!modules_to_install[@]}"; do
info " ${rel_modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }"
done
info ""
get_YN_answer "Do you want to continue? [n] " || die 1 "Aborting..."
info ""
for rel_modulefile in "${!modules_to_install[@]}"; do
if [[ -e "${target_prefix}/${rel_modulefile}" ]]; then
info " Updating: ${rel_modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }"
(( n == 0 )) && \
std::die 0 "No matching modules found ..."
print_modules_to_install
# install ...
for modulefile in "${!modules_to_install[@]}" "${!dependencies_to_install[@]}"; do
if [[ -e "${target_prefix}/${modulefile}" ]]; then
std::info " Updating: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }"
else
info " Installing: ${rel_modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }"
std::info " Installing: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }"
fi
sync_module "${rel_modulefile}" \
sync_module "${modulefile}" \
"${src_prefix}" \
"${target_prefix}"
done
info "\nDone!\n"
std::info "\nDone!\n"
}
subcommand_sync() {
[[ -z "${PMODULES_ROOT}" ]] && die 1 "Error: No current module environment is configured!"
#
# delete specified module(s)
#
subcommand_delete() {
:
}
#
# remove modules which have been removed in our source
#
subcommand_cleanup() {
:
}
subcommand_sync() {
local delete=false
local opts=''
local dst_prefix=''
local src_prefix=''
opts=$(get_options -o h -l dst: -l delete -l help -- "$@")
opts=$(pmodules::get_options -o h -l dst: -l delete -l help -- "$@")
if [[ $? != 0 ]]; then
subcommand_help_sync
exit 1
......@@ -509,7 +629,7 @@ subcommand_sync() {
;;
* )
[[ -n "${src_prefix}" ]] && \
die 1 "Error: Only one source is allowed!"
std::die 1 "Error: Only one source is allowed!"
src_prefix="$1"
;;
esac
......@@ -521,29 +641,29 @@ subcommand_sync() {
dst_prefix="${PMODULES_ROOT}"
fi
(
PMODULES_ROOT="${dst_prefix}" check_pmodules_env || \
die 1 "Error: invalid destination modules environment!"
) || die 1 "Giving up..."
PMODULES_ROOT="${dst_prefix}" pmodules::check_env || \
std::die 1 "Error: invalid destination modules environment!"
) || std::die 1 "Giving up..."
: ${src_prefix:=${PMODULES_INSTALL_SOURCE}}
if [[ -z "${src_prefix}" ]]; then
die 1 "Error: no source module environment was specified!"
std::die 1 "Error: no source module environment was specified!"
fi
(
PMODULES_ROOT="${src_prefix}" check_pmodules_env || \
die 1 "Error: invalid source modules environment!"
) || die 1 "Giving up..."
PMODULES_ROOT="${src_prefix}" pmodules::check_env || \
std::die 1 "Error: invalid source modules environment!"
) || std::die 1 "Giving up..."
[[ "$( cd "$src_prefix"; pwd -P )" == "$( cd "$dst_prefix"; pwd -P )" ]] && \
die 1 "Error: source and destination are equal!"
std::die 1 "Error: source and destination are equal!"
local modbin=${PMODULES_HOME#"${PMODULES_ROOT}/"}/bin/modulecmd.tcl
local file_type_src=$( file -b "${src_prefix}/${modbin}" 2>&1 || echo err1 )
local file_type_dst=$( file -b "${dst_prefix}/${modbin}" 2>&1 || echo err2 )
[[ "${file_type_src}" == "${file_type_dst}" ]] || \
die 1 "Error: The file signatures in the source and destination installation do not match!"
std::die 1 "Error: The file signatures in the source and destination installation do not match!"
unset -v file_type_src file_type_dst
local dialog_script="${PMODULES_HOME}/bin/dialog.bash"
[[ -r "$dialog_script" ]] || \
die 1 "Error: Unable to find dialog script of installation $dialog_script";
std::die 1 "Error: Unable to find dialog script of installation $dialog_script";
DIALOG_LIB=1 # use dialog script as a library
source "$dialog_script" # dialog functions
......@@ -595,11 +715,13 @@ subcommand_sync() {
local -a modules_copy=( $(set_difference "${selected_modules[*]}" "${destination_modules[*]}") )
if [[ -n $modules_copy ]]; then
echo "Syncing configuration ..."
sync_config "$src_prefix" "$dst_prefix" || die 1 "Error: syncing the configuration failed"
sync_config "$src_prefix" "$dst_prefix" || \
std::die 1 "Error: syncing the configuration failed"
fi
for m in "${modules_copy[@]}"; do
echo "Copying module $m ..."
sync_module "$m" "$src_prefix" "$dst_prefix" || die 1 "Error: syncing of module $m failed!"
sync_module "$m" "$src_prefix" "$dst_prefix" || \
std::die 1 "Error: syncing of module $m failed!"
done
unset modules_copy
}
......@@ -642,6 +764,8 @@ if [[ -z ${subcommand} ]]; then
usage
exit 1
fi
[[ -z "${PMODULES_ROOT}" ]] && \
std::die 1 "Error: No current module environment is configured!"
$subcommand "${sargs[@]}"
# Local Variables:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment