hpcflow.sdk.submission.shells package

Submodules

hpcflow.sdk.submission.shells.base module

class hpcflow.sdk.submission.shells.base.Shell(executable=None, os_args=None)

Bases: ABC

Class to represent a shell and templates for jobscript composition.

This class represents a combination of a shell and an OS. For example, running bash on a POSIX OS, and provides snippets that are used to compose a jobscript for that combination.

property executable: List[str]

abstract get_version_info(exclude_os=False)

Get shell and operating system information.

Parameters:

exclude_os (bool | None) –

Return type:

Dict

prepare_JS_path(js_path)

Parameters:

js_path (Path) –

Return type:

str

prepare_element_run_dirs(run_dirs)

Parameters:

run_dirs (List[List[Path]]) –

Return type:

List[List[str]]

process_JS_header_args(header_args)

Parameters:

header_args (Dict) –

Return type:

Dict

property shebang_executable: List[str]

hpcflow.sdk.submission.shells.bash module

class hpcflow.sdk.submission.shells.bash.Bash(*args, **kwargs)

Bases: Shell

Class to represent using bash on a POSIX OS to generate and submit a jobscript.

DEFAULT_EXE = '/bin/bash'

JS_DIRECT_HEADER = '{shebang}\n\n{header}\n'

JS_ELEMENT_ARRAY = 'JS_elem_idx=$(({scheduler_array_item_var} - 1))\n{main}\ncd $WK_PATH\n'

JS_ELEMENT_LOOP = 'for ((JS_elem_idx=0;JS_elem_idx<{num_elements};JS_elem_idx++))\ndo\n{main}\ndone\ncd $WK_PATH\n'

JS_ENV_SETUP_INDENT = '    '

JS_EXT = '.sh'

JS_HEADER = '{workflow_app_alias} () {{\n(\n{env_setup}{app_invoc}\\\n    --config-dir "{config_dir}"\\\n    --config-invocation-key "{config_invoc_key}"\\\n    "$@"\n)\n}}\n\nWK_PATH=`pwd`\nWK_PATH_ARG=$WK_PATH\nSUB_IDX={sub_idx}\nJS_IDX={js_idx}\nEAR_ID_FILE="$WK_PATH/artifacts/submissions/${{SUB_IDX}}/{EAR_file_name}"\nELEM_RUN_DIR_FILE="$WK_PATH/artifacts/submissions/${{SUB_IDX}}/{element_run_dirs_file_path}"\n'

JS_INDENT = '  '

JS_MAIN = 'elem_need_EARs=`sed "$((${{JS_elem_idx}} + 1))q;d" $EAR_ID_FILE`\nelem_run_dirs=`sed "$((${{JS_elem_idx}} + 1))q;d" $ELEM_RUN_DIR_FILE`\n\nfor ((JS_act_idx=0;JS_act_idx<{num_actions};JS_act_idx++))\ndo\n\n  need_EAR="$(cut -d\'{EAR_files_delimiter}\' -f $(($JS_act_idx + 1)) <<< $elem_need_EARs)"\n  if [ "$need_act" = "0" ]; then\n      continue\n  fi\n\n  run_dir="$(cut -d\'{EAR_files_delimiter}\' -f $(($JS_act_idx + 1)) <<< $elem_run_dirs)"\n  cd $WK_PATH/$run_dir\n\n  {workflow_app_alias} internal workflow $WK_PATH_ARG write-commands $SUB_IDX $JS_IDX $JS_elem_idx $JS_act_idx\n  {workflow_app_alias} internal workflow $WK_PATH_ARG set-ear-start $SUB_IDX $JS_IDX $JS_elem_idx $JS_act_idx\n  . {commands_file_name}\n  {workflow_app_alias} internal workflow $WK_PATH_ARG set-ear-end $SUB_IDX $JS_IDX $JS_elem_idx $JS_act_idx\n\ndone\n'

JS_SCHEDULER_HEADER = '{shebang}\n\n{scheduler_options}\n{header}\n'

JS_SHEBANG = '#!{shebang_executable} {shebang_args}'

format_save_parameter(workflow_app_alias, param_name, shell_var_name)

format_stream_assignment(shell_var_name, command)

get_version_info(exclude_os=False)

Get bash version information.

Parameters:

exclude_os (bool | None) – If True, exclude operating system information.

Return type:

Dict

property linux_release_file

wrap_in_subshell(commands)

Format commands to run within a subshell.

This assumes commands ends in a newline.

Parameters:

commands (str) –

Return type:

str

class hpcflow.sdk.submission.shells.bash.WSLBash(WSL_executable=None, WSL_distribution=None, WSL_user=None, *args, **kwargs)

Bases: Bash

Parameters:

• WSL_executable (str | None) –

• WSL_distribution (str | None) –

• WSL_user (str | None) –

DEFAULT_WSL_EXE = 'wsl'

JS_HEADER = '{workflow_app_alias} () {{\n(\n{env_setup}{app_invoc}\\\n    --config-dir "{config_dir}"\\\n    --config-invocation-key "{config_invoc_key}"\\\n    "$@"\n)\n}}\n\nWK_PATH=`pwd`\nWK_PATH_ARG=`wslpath -m $WK_PATH`\nSUB_IDX={sub_idx}\nJS_IDX={js_idx}\nEAR_ID_FILE="$WK_PATH/artifacts/submissions/${{SUB_IDX}}/{EAR_file_name}"\nELEM_RUN_DIR_FILE="$WK_PATH/artifacts/submissions/${{SUB_IDX}}/{element_run_dirs_file_path}"\n'

property executable: List[str]

get_version_info(exclude_os=False)

Get WSL and bash version information.

Parameters:

exclude_os (bool | None) – If True, exclude operating system information.

Return type:

Dict

prepare_JS_path(js_path)

Parameters:

js_path (Path) –

Return type:

str

prepare_element_run_dirs(run_dirs)

Parameters:

run_dirs (List[List[Path]]) –

Return type:

List[List[str]]

process_JS_header_args(header_args)

property shebang_executable: List[str]

hpcflow.sdk.submission.shells.os_version module

hpcflow.sdk.submission.shells.os_version.get_OS_info()

Return type:

Dict

hpcflow.sdk.submission.shells.os_version.get_OS_info_POSIX(WSL_executable=None, use_py=True, linux_release_file=None)

Parameters:

• WSL_executable (List[str] | None) – Executable to run subprocess calls via WSL on Windows.

• use_py (bool | None) – If True, use the platform.uname Python function to get the OS information. Otherwise use subprocess to call uname. We set this to False when getting OS info in WSL on Windows, since we need to call the WSL executable.

• linux_release_file (str | None) – If on Linux, record the name and version fields from this file.

hpcflow.sdk.submission.shells.os_version.get_OS_info_windows()

Return type:

Dict

hpcflow.sdk.submission.shells.powershell module

class hpcflow.sdk.submission.shells.powershell.WindowsPowerShell(*args, **kwargs)

Bases: Shell

Class to represent using PowerShell on Windows to generate and submit a jobscript.

DEFAULT_EXE = 'powershell.exe'

JS_DIRECT_HEADER = '{shebang}\n\n{header}\n'

JS_ELEMENT_LOOP = 'for ($JS_elem_idx = 0; $JS_elem_idx -lt {num_elements}; $JS_elem_idx += 1) {{\n{main}\n}}\nSet-Location $WK_PATH\n'

JS_ENV_SETUP_INDENT = '        '

JS_EXT = '.ps1'

JS_HEADER = 'function {workflow_app_alias} {{\n    & {{\n{env_setup}{app_invoc} `\n            --config-dir "{config_dir}" `\n            --config-invocation-key "{config_invoc_key}" `\n            $args\n    }} @args\n}}\n\nfunction get_nth_line($file, $line) {{\n    Get-Content $file | Select-Object -Skip $line -First 1\n}}\n\nfunction JoinMultiPath {{\n    $numArgs = $args.Length\n    $path = $args[0]\n    for ($i = 1; $i -lt $numArgs; $i++) {{\n        $path = Join-Path $path $args[$i]\n    }}\n    return $path\n}}\n\n$WK_PATH = $(Get-Location)\n$SUB_IDX = {sub_idx}\n$JS_IDX = {js_idx}\n$EAR_ID_FILE = JoinMultiPath $WK_PATH artifacts submissions $SUB_IDX {EAR_file_name}\n$ELEM_RUN_DIR_FILE = JoinMultiPath $WK_PATH artifacts submissions $SUB_IDX {element_run_dirs_file_path}\n'

JS_INDENT = '    '

JS_MAIN = '$elem_need_EARs = get_nth_line $EAR_ID_FILE $JS_elem_idx\n$elem_run_dirs = get_nth_line $ELEM_RUN_DIR_FILE $JS_elem_idx\n\nfor ($JS_act_idx = 0; $JS_act_idx -lt {num_actions}; $JS_act_idx += 1) {{\n\n    $need_EAR = ($elem_need_EARs -split "{EAR_files_delimiter}")[$JS_act_idx]\n    if ($need_EAR -eq 0) {{\n        continue\n    }}\n\n    $run_dir = ($elem_run_dirs -split "{EAR_files_delimiter}")[$JS_act_idx]\n    $run_dir_abs = "$WK_PATH\\$run_dir"\n    Set-Location $run_dir_abs\n\n    {workflow_app_alias} internal workflow $WK_PATH write-commands $SUB_IDX $JS_IDX $JS_elem_idx $JS_act_idx\n    {workflow_app_alias} internal workflow $WK_PATH set-ear-start $SUB_IDX $JS_IDX $JS_elem_idx $JS_act_idx\n\n    . (Join-Path $run_dir_abs "{commands_file_name}")\n    {workflow_app_alias} internal workflow $WK_PATH set-ear-end $SUB_IDX $JS_IDX $JS_elem_idx $JS_act_idx\n\n}}\n'

JS_SHEBANG = ''

format_save_parameter(workflow_app_alias, param_name, shell_var_name)

format_stream_assignment(shell_var_name, command)

get_version_info(exclude_os=False)

Get powershell version information.

Parameters:

exclude_os (bool | None) – If True, exclude operating system information.

Return type:

Dict

wrap_in_subshell(commands)

Format commands to run within a child scope.

This assumes commands ends in a newline.

Parameters:

commands (str) –

Return type:

str

Module contents

hpcflow.sdk.submission.shells.get_shell(shell_name, os_name=None, **kwargs)

Parameters:

os_name (str | None) –

Return type:

Shell

hpcflow.sdk.submission.shells.get_supported_shells(os_name=None)

Parameters:

os_name (str | None) –

Return type:

Dict[str, Shell]