Commit 8eb34d76 authored by gsell's avatar gsell
Browse files

Pmodules/modmanage.bash.in:

- refactor function names with namespace
- dependency resolution implemented for 'install' sub-command
parent 1ab70662
...@@ -17,12 +17,12 @@ source "${libdir}/libpmodules.bash" ...@@ -17,12 +17,12 @@ source "${libdir}/libpmodules.bash"
PATH="${bindir}:${PATH}" PATH="${bindir}:${PATH}"
_exit () { _exit () {
die 1 "Interrupted..." std::die 1 "Interrupted..."
} }
_err () { _err () {
info "Oops: got an error in function '${FUNCNAME[1]}', line ${BASH_LINENO[0]}" std::info "Oops: got an error in function '${FUNCNAME[1]}', line ${BASH_LINENO[0]}"
die 1 "Aborting ..." std::die 1 "Aborting ..."
} }
trap '_exit' INT TERM trap '_exit' INT TERM
...@@ -221,7 +221,7 @@ subcommand_init() { ...@@ -221,7 +221,7 @@ subcommand_init() {
local target_prefixes=() local target_prefixes=()
local user='' local user=''
local opts='' local opts=''
opts=$(get_options -o h -l src: -l user: -l help -l version: -- "$@") opts=$(pmodules::get_options -o h -l src: -l user: -l help -l version: -- "$@")
if [[ $? != 0 ]]; then if [[ $? != 0 ]]; then
subcommand_help_init subcommand_help_init
exit 1 exit 1
...@@ -255,30 +255,32 @@ subcommand_init() { ...@@ -255,30 +255,32 @@ subcommand_init() {
esac esac
shift shift
done done
(( ${#target_prefixes[@]} != 0 )) || die 1 "Error: no target directory specified!" (( ${#target_prefixes[@]} != 0 )) || \
std::die 1 "Error: no target directory specified!"
# if source directory is not passed as argument, derive it from script name # if source directory is not passed as argument, derive it from script name
if [[ -z "${src}" ]]; then if [[ -z "${src}" ]]; then
src=$(cd "${bindir}/../../../.." && pwd) src=$(cd "${bindir}/../../../.." && pwd)
fi fi
[[ -d "${src}" ]] || \ [[ -d "${src}" ]] || \
die 1 "Error: ${src}: source directory does not exist!" std::die 1 "Error: ${src}: source directory does not exist!"
[[ -r "${src}/config/profile.bash" ]] || \ [[ -r "${src}/config/profile.bash" ]] || \
die 1 "Error: ${src}: shell profile does not exist or is not readable!" std::die 1 "Error: ${src}: shell profile does not exist or is not readable!"
source "${src}/config/profile.bash" source "${src}/config/profile.bash"
local -i euid=$(id -u) local -i euid=$(id -u)
if (( euid == 0 )); then if (( euid == 0 )); then
[[ -n "${user}" ]] || \ [[ -n "${user}" ]] || \
die 1 "Error: --user parameter is required!" std::die 1 "Error: --user parameter is required!"
id -u "${user}" > /dev/null 2>&1 || \ id -u "${user}" > /dev/null 2>&1 || \
die 1 "Error: Unable to retrieve user id of user '${user}'" std::die 1 "Error: Unable to retrieve user id of user '${user}'"
else else
[[ -z "${user}" ]] || \ [[ -z "${user}" ]] || \
die 1 "Error: --user option is only allowed if running as root!" std::die 1 "Error: --user option is only allowed if running as root!"
fi fi
check_pmodules_env || die 1 "Giving up..." pmodules::check_env || \
std::die 1 "Giving up..."
echo " echo "
Attempting to create a minimal module environment from the Attempting to create a minimal module environment from the
...@@ -294,22 +296,25 @@ environment at '${PMODULES_ROOT}' ...@@ -294,22 +296,25 @@ environment at '${PMODULES_ROOT}'
echo echo
if [[ -d "${target_prefix}" ]] && [[ ${force} == no ]]; then if [[ -d "${target_prefix}" ]] && [[ ${force} == no ]]; then
echo "Warning: ${target_prefix} already exists." echo "Warning: ${target_prefix} already exists."
get_YN_answer "Do you really want to re-run the initialization? (y/N) " || \ std::get_YN_answer "Do you really want to re-run the initialization? (y/N) " || \
die 1 "Abort ..." std::die 1 "Abort ..."
fi fi
force='yes' force='yes'
echo "Creating target directory '${target_prefix}'..." echo "Creating target directory '${target_prefix}'..."
$DRY mkdir -p "${target_prefix}" || die 1 "Error: make directory failed!" $DRY mkdir -p "${target_prefix}" || \
std::die 1 "Error: make directory failed!"
echo echo
echo "Syncing configuration ..." echo "Syncing configuration ..."
sync_config "${PMODULES_ROOT}" \ sync_config "${PMODULES_ROOT}" \
"${target_prefix}" || die 1 "Error: configuration synchronization failed!" "${target_prefix}" || \
std::die 1 "Error: configuration synchronization failed!"
echo "Syncing Pmodules ${PMODULES_VERSION} from '${src_prefix}' to '${target_prefix}'..." echo "Syncing Pmodules ${PMODULES_VERSION} from '${src_prefix}' to '${target_prefix}'..."
sync_module "Tools/Pmodules/${PMODULES_VERSION}" \ sync_module "Tools/Pmodules/${PMODULES_VERSION}" \
"${src_prefix}" \ "${src_prefix}" \
"${target_prefix}" || die 1 "Error: sync Pmodules failed!" "${target_prefix}" || \
std::die 1 "Error: sync Pmodules failed!"
mkdir -p "${target_prefix}/Tools/${PMODULES_MODULEFILES_DIR}" mkdir -p "${target_prefix}/Tools/${PMODULES_MODULEFILES_DIR}"
echo echo
...@@ -321,7 +326,8 @@ environment at '${PMODULES_ROOT}' ...@@ -321,7 +326,8 @@ environment at '${PMODULES_ROOT}'
if [[ -n "${user}" ]]; then if [[ -n "${user}" ]]; then
echo "Changing user of new module environment to '${user}'..." echo "Changing user of new module environment to '${user}'..."
$DRY chown -R "${user}" "${target_prefix}" || die 1 "Error: changing owner failed!" $DRY chown -R "${user}" "${target_prefix}" || \
std::die 1 "Error: changing owner failed!"
echo echo
fi fi
echo "New minimal module environment created at '${target_prefix}'." echo "New minimal module environment created at '${target_prefix}'."
...@@ -337,70 +343,133 @@ environment at '${PMODULES_ROOT}' ...@@ -337,70 +343,133 @@ environment at '${PMODULES_ROOT}'
} }
declare -a Groups=()
declare -A HierarchyDepths
get_groups () {
local -r root="$1"
{
cd "${root}"
# for some unknown reason [A-Z]* doesn't work on (some?) SL6 systems
for f in [ABCDEFGHIJKLMNOPQRSTUVWXYZ]*; do
[[ -d ${f}/${PMODULES_MODULEFILES_DIR} ]] || continue
Groups+=( $f )
done
};
}
#
# $1: root of modulefile hierarchy
get_hierarchy_depth () {
local -r root="$1"
local -a modulefiles_dir
std::split_fname modulefiles_dir "${PMODULES_MODULEFILES_DIR}"
local -ir off=$(( ${#modulefiles_dir[@]} + 3 ))
{
cd "${root}"
local group
for group in "${Groups[@]}"; do
local fname=$(find "${group}/${PMODULES_MODULEFILES_DIR}" \
-depth \( -type f -o -type l \) -print -quit)
[[ -n ${fname} ]] || continue
#local -a tmp2=( ${fname//\// } )
local -a tmp
std::split_fname tmp "${fname}"
(( HierarchyDepths[$group]=${#tmp[@]}-off ))
done
};
}
subcommand_install() { subcommand_install() {
local opts='' local opts=''
local -a with=() local -a with=()
local -a releases=() local -a releases=()
local -a module_pattern=() local -a module_pattern=()
local -r src_prefix="${PMODULES_INSTALL_SOURCE}" local src_prefix="${PMODULES_INSTALL_SOURCE}"
local -r target_prefix="${PMODULES_ROOT}" local -r target_prefix="${PMODULES_ROOT}"
local rel_modulefile='' local modulefile=''
local -A modules_to_install local -A modules_to_install
local -A _modules_to_group local -A dependencies_to_install
local -A _hierarchy_nodes local -A map_to_family
while read output; do local -a initial_modulepath=()
local _group=${output%/modulefiles/*}
local _abs_modulename="${output#*/modulefiles/}" #
_modules_to_group[${_abs_modulename}]=${_group} # Resolve dependencies to given module
IFS='/' #
local -a _items=(${_abs_modulename}) # $1: modulefile relativ to src prefix. Something like:
IFS=${_saved_IFS} # MPI/modulefiles/gcc/4.9.2/openmpi/1.8.4/hdf5/1.8.14
local -i _n=${#_items[*]} #
if (( _n >= 4 )); then # Notes:
local _key=$( IFS='/'; echo "${_items[*]:$_n-4:2}" ) # The variables
_hierarchy_nodes[$_key]=${_group} # initial_modulepath
fi # modules_to_install
done < <({ cd "${src_prefix}" && find */modulefiles -type l -o -type f \! -name ".*"; } ) # map_to_family
# from the calling function are used!
#
resolve_dependencies_of_module () { resolve_dependencies_of_module () {
local -r __rel_modulefile=$1 local -r modulefile=$1
# :FIXME: This must be initialized with all "root" groups local -a modulepath=( "${initial_modulepath[@]}" )
local -a __modulepath=("${src_prefix}/Tools/modulefiles" "${src_prefix}/Programming/modulefiles")
IFS='/' # compute filename with dependencies of given module
local -a _items=(${__rel_modulefile}) local -i i=0 n=0
IFS=${_saved_IFS} std::split_fname items n "${modulefile}"
local -i _n=${#_items[@]} local fname_dependencies="${src_prefix}/${items[0]}"
local -i _i for (( i = n-2; i >= 2; i-=2 )); do
local _prefix="${src_prefix}/${_items[0]}" fname_dependencies+="/${items[$i]}/${items[i+1]}"
for (( _i = _n-2; _i >= 2; _i-=2 )); do
_prefix+="/${_items[$_i]}/${_items[_i+1]}"
done done
local __dep fname_dependencies+='/.dependencies'
while read __dep; do [[ -r ${fname_dependencies} ]] || return 0
[[ -z ${__dep} ]] && continue
local __long_module_name=$(find "${__modulepath[@]}" -path "*/${__dep}" | head -n 1 ) # loop over all dependecies
__long_module_name=${__long_module_name/${src_prefix}\/} local dep
modules_to_install[${__long_module_name/}]='.' while read dep; do
if [[ -n ${_hierarchy_nodes[${__dep}]} ]]; then # skip empty lines
local __path="${src_prefix}/${_hierarchy_nodes[${__dep}]}/modulefiles/" # :FIXME: skip comments?!
__path+="${__long_module_name/*\/modulefiles\/}" [[ -z ${dep} ]] && continue
__modulepath+=( "${__path}" )
# search for module with current modulepath and remember
local modulename=$(find "${modulepath[@]}" -path "*/${dep}" 2>/dev/null | head -n 1 )
[[ -n ${modulename} ]] || \
std::die 3 "Oops: required module '${dep}' not found!"
modulename=${modulename/${src_prefix}\/}
dependencies_to_install[${modulename}]='.'
resolve_dependencies_of_module "${modulename}"
# append new node in hierarchy to modulepath
if [[ -n ${map_to_family[${dep}]} ]]; then
local path="${src_prefix}/${map_to_family[${dep}]}/${PMODULES_MODULEFILES_DIR}/"
path+="${modulename/*\/${PMODULES_MODULEFILES_DIR}\/}"
modulepath+=( "${path}" )
fi fi
done < "${_prefix}/.dependencies" done < "${fname_dependencies}"
}
resolve_dependencies () {
local _saved_IFS=${IFS};
# a relative modulefile is something like:
# MPI/modulefiles/gcc/4.8.4/openmpi/1.8.4/hdf5/1.8.14
local _rel_modulefile
for _rel_modulefile in "${!modules_to_install[@]}"; do
resolve_dependencies_of_module "${_rel_modulefile}"
done
} }
opts=$(get_options -o hf -l dry-run -l force -l with: -l release: -l help -l src: -- "$@") print_modules_to_install() {
local modulefile
std::info "The following modules will be installed/updated:"
for modulefile in "${!modules_to_install[@]}"; do
if [[ -e "${target_prefix}/${modulefile}" ]]; then
std::info " Updating: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }"
else
std::info " Installing: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }"
fi
done 2>&1 | sort
if (( ${#dependencies_to_install[@]} > 0 )); then
std::info "\nThe following dependencies will be installed/updated:"
for modulefile in "${!dependencies_to_install[@]}"; do
if [[ -e "${target_prefix}/${modulefile}" ]]; then
std::info " Updating: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }"
else
std::info " Installing: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }"
fi
done 2>&1 | sort
fi
std::info ""
std::get_YN_answer "Do you want to continue? [n] " || \
std::die 1 "Aborting..."
std::info ""
}
opts=$(pmodules::get_options -o hf -l dry-run -l force -l with: -l release: -l help -l src: -- "$@")
if [[ $? != 0 ]]; then if [[ $? != 0 ]]; then
subcommand_help_install subcommand_help_install
exit 1 exit 1
...@@ -444,9 +513,52 @@ subcommand_install() { ...@@ -444,9 +513,52 @@ subcommand_install() {
esac esac
shift shift
done done
[[ -n ${src_prefix} ]] \
|| std::die 3 "Oops: no installation source given."
[[ -d ${src_prefix} ]] \
|| std::die 3 "Oops: '${src_prefix}' is not a valid installation source."
# scan available groups and their depth
get_groups "${src_prefix}"
get_hierarchy_depth "${src_prefix}"
# set initial modulepath
local group
for group in "${!HierarchyDepths[@]}"; do
if (( ${HierarchyDepths[${group}]} == 0 )); then
initial_modulepath+=( "${src_prefix}/${group}/${PMODULES_MODULEFILES_DIR}" )
fi
done
#
# create a mapping from module name to their family.
# Examples:
# gcc/5.2.0 -> Compiler
# openmpi/1.8.4 -> MPI
local _fname=''
while read _fname; do
local _family="${_fname%/${PMODULES_MODULEFILES_DIR}/*}"
local -a items
std::split_fname items "${_fname#*/${PMODULES_MODULEFILES_DIR}/}"
local -i n=${#items[*]}
# We are only interested in families adding something to
# the modulepath.
if (( n >= 4 )); then
local _key=$( IFS='/'; echo "${items[*]:$n-4:2}" )
map_to_family[$_key]=${_family}
fi
done < <({ cd "${src_prefix}" && \
find */"${PMODULES_MODULEFILES_DIR}" \
\( -type l -o -type f \) \! -name ".*"; } 2>/dev/null )
#
# search for to be installed modules and their dependencies
#
local -i n=0 local -i n=0
while read rel_modulefile; do while read modulefile; do
modules_to_install["${rel_modulefile}"]+='.' resolve_dependencies_of_module "${modulefile}"
modules_to_install["${modulefile}"]+='.'
let n+=1 let n+=1
done < <(${PMODULES_HOME}/bin/modulecmd bash search \ done < <(${PMODULES_HOME}/bin/modulecmd bash search \
"${module_pattern[@]}" \ "${module_pattern[@]}" \
...@@ -454,37 +566,45 @@ subcommand_install() { ...@@ -454,37 +566,45 @@ subcommand_install() {
"${releases[@]/#/--release=}" \ "${releases[@]/#/--release=}" \
--no-header --print-modulefiles \ --no-header --print-modulefiles \
--src="${src_prefix}" 2>&1 1>/dev/null) --src="${src_prefix}" 2>&1 1>/dev/null)
(( n == 0 )) && die 0 "Nothing to install..." (( n == 0 )) && \
resolve_dependencies std::die 0 "No matching modules found ..."
info "The following modules will be installed/updated:\n" print_modules_to_install
for rel_modulefile in "${!modules_to_install[@]}"; do
info " ${rel_modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }" # install ...
done for modulefile in "${!modules_to_install[@]}" "${!dependencies_to_install[@]}"; do
info "" if [[ -e "${target_prefix}/${modulefile}" ]]; then
get_YN_answer "Do you want to continue? [n] " || die 1 "Aborting..." std::info " Updating: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }"
info ""
for rel_modulefile in "${!modules_to_install[@]}"; do
if [[ -e "${target_prefix}/${rel_modulefile}" ]]; then
info " Updating: ${rel_modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }"
else else
info " Installing: ${rel_modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }" std::info " Installing: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }"
fi fi
sync_module "${rel_modulefile}" \ sync_module "${modulefile}" \
"${src_prefix}" \ "${src_prefix}" \
"${target_prefix}" "${target_prefix}"
done done
info "\nDone!\n" std::info "\nDone!\n"
} }
subcommand_sync() { #
[[ -z "${PMODULES_ROOT}" ]] && die 1 "Error: No current module environment is configured!" # delete specified module(s)
#
subcommand_delete() {
:
}
#
# remove modules which have been removed in our source
#
subcommand_cleanup() {
:
}
subcommand_sync() {
local delete=false local delete=false
local opts='' local opts=''
local dst_prefix='' local dst_prefix=''
local src_prefix='' local src_prefix=''
opts=$(get_options -o h -l dst: -l delete -l help -- "$@") opts=$(pmodules::get_options -o h -l dst: -l delete -l help -- "$@")
if [[ $? != 0 ]]; then if [[ $? != 0 ]]; then
subcommand_help_sync subcommand_help_sync
exit 1 exit 1
...@@ -509,7 +629,7 @@ subcommand_sync() { ...@@ -509,7 +629,7 @@ subcommand_sync() {
;; ;;
* ) * )
[[ -n "${src_prefix}" ]] && \ [[ -n "${src_prefix}" ]] && \
die 1 "Error: Only one source is allowed!" std::die 1 "Error: Only one source is allowed!"
src_prefix="$1" src_prefix="$1"
;; ;;
esac esac
...@@ -521,29 +641,29 @@ subcommand_sync() { ...@@ -521,29 +641,29 @@ subcommand_sync() {
dst_prefix="${PMODULES_ROOT}" dst_prefix="${PMODULES_ROOT}"
fi fi
( (
PMODULES_ROOT="${dst_prefix}" check_pmodules_env || \ PMODULES_ROOT="${dst_prefix}" pmodules::check_env || \
die 1 "Error: invalid destination modules environment!" std::die 1 "Error: invalid destination modules environment!"
) || die 1 "Giving up..." ) || std::die 1 "Giving up..."
: ${src_prefix:=${PMODULES_INSTALL_SOURCE}} : ${src_prefix:=${PMODULES_INSTALL_SOURCE}}
if [[ -z "${src_prefix}" ]]; then if [[ -z "${src_prefix}" ]]; then
die 1 "Error: no source module environment was specified!" std::die 1 "Error: no source module environment was specified!"
fi fi
( (
PMODULES_ROOT="${src_prefix}" check_pmodules_env || \ PMODULES_ROOT="${src_prefix}" pmodules::check_env || \
die 1 "Error: invalid source modules environment!" std::die 1 "Error: invalid source modules environment!"
) || die 1 "Giving up..." ) || std::die 1 "Giving up..."
[[ "$( cd "$src_prefix"; pwd -P )" == "$( cd "$dst_prefix"; pwd -P )" ]] && \ [[ "$( cd "$src_prefix"; pwd -P )" == "$( cd "$dst_prefix"; pwd -P )" ]] && \
die 1 "Error: source and destination are equal!" std::die 1 "Error: source and destination are equal!"
local modbin=${PMODULES_HOME#"${PMODULES_ROOT}/"}/bin/modulecmd.tcl local modbin=${PMODULES_HOME#"${PMODULES_ROOT}/"}/bin/modulecmd.tcl
local file_type_src=$( file -b "${src_prefix}/${modbin}" 2>&1 || echo err1 ) local file_type_src=$( file -b "${src_prefix}/${modbin}" 2>&1 || echo err1 )
local file_type_dst=$( file -b "${dst_prefix}/${modbin}" 2>&1 || echo err2 ) local file_type_dst=$( file -b "${dst_prefix}/${modbin}" 2>&1 || echo err2 )
[[ "${file_type_src}" == "${file_type_dst}" ]] || \ [[ "${file_type_src}" == "${file_type_dst}" ]] || \
die 1 "Error: The file signatures in the source and destination installation do not match!" std::die 1 "Error: The file signatures in the source and destination installation do not match!"
unset -v file_type_src file_type_dst unset -v file_type_src file_type_dst
local dialog_script="${PMODULES_HOME}/bin/dialog.bash" local dialog_script="${PMODULES_HOME}/bin/dialog.bash"
[[ -r "$dialog_script" ]] || \ [[ -r "$dialog_script" ]] || \
die 1 "Error: Unable to find dialog script of installation $dialog_script"; std::die 1 "Error: Unable to find dialog script of installation $dialog_script";
DIALOG_LIB=1 # use dialog script as a library DIALOG_LIB=1 # use dialog script as a library
source "$dialog_script" # dialog functions source "$dialog_script" # dialog functions
...@@ -595,11 +715,13 @@ subcommand_sync() { ...@@ -595,11 +715,13 @@ subcommand_sync() {
local -a modules_copy=( $(set_difference "${selected_modules[*]}" "${destination_modules[*]}") ) local -a modules_copy=( $(set_difference "${selected_modules[*]}" "${destination_modules[*]}") )
if [[ -n $modules_copy ]]; then if [[ -n $modules_copy ]]; then
echo "Syncing configuration ..." echo "Syncing configuration ..."
sync_config "$src_prefix" "$dst_prefix" || die 1 "Error: syncing the configuration failed" sync_config "$src_prefix" "$dst_prefix" || \
std::die 1 "Error: syncing the configuration failed"
fi fi
for m in "${modules_copy[@]}"; do for m in "${modules_copy[@]}"; do
echo "Copying module $m ..." echo "Copying module $m ..."
sync_module "$m" "$src_prefix" "$dst_prefix" || die 1 "Error: syncing of module $m failed!" sync_module "$m" "$src_prefix" "$dst_prefix" || \
std::die 1 "Error: syncing of module $m failed!"
done done
unset modules_copy unset modules_copy
} }
...@@ -642,6 +764,8 @@ if [[ -z ${subcommand} ]]; then ...@@ -642,6 +764,8 @@ if [[ -z ${subcommand} ]]; then
usage usage
exit 1 exit 1
fi fi
[[ -z "${PMODULES_ROOT}" ]] && \
std::die 1 "Error: No current module environment is configured!"
$subcommand "${sargs[@]}" $subcommand "${sargs[@]}"
# Local Variables: # Local Variables:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment