Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions libensemble/executors/executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,9 @@
# To change logging level for just this module
# logger.setLevel(logging.DEBUG)

# Placeholder for container support - replaced with simulation directory at runtime
LIBE_SIM_DIR_PLACEHOLDER = "%LIBENSEMBLE_SIM_DIR%"

STATES = """
UNKNOWN
CREATED
Expand Down Expand Up @@ -431,6 +434,7 @@ def __init__(self) -> None:
self.workerID = None
self.comm = None
self.last_task = 0
self.base_dir = os.getcwd()
Executor.executor = self

def __enter__(self):
Expand Down Expand Up @@ -522,6 +526,10 @@ def register_app(

precedent: str, Optional
Any str that should directly precede the application full path.
Supports the placeholder ``%LIBENSEMBLE_SIM_DIR%`` which is replaced
at runtime with the simulation directory as a relative path from
where the executor was created. This is useful for container exec
commands.
"""

if not app_name:
Expand Down Expand Up @@ -684,6 +692,16 @@ def _check_app_exists(self, app: Application) -> None:
if not os.path.isfile(app.full_path):
raise ExecutorException(f"Application does not exist {app.full_path}")

def _set_sim_dir_env(self, task: Task, run_cmd: list[str]) -> list[str]:
"""Replace simulation directory placeholder in run command if present.

Supports container-based execution where the simulation directory needs to be
passed to container exec commands (e.g., podman-hpc exec --workdir).
"""
sim_dir = os.path.relpath(task.workdir, self.base_dir)
task._add_to_env("LIBENSEMBLE_SIM_DIR", sim_dir)
return [arg.replace(LIBE_SIM_DIR_PLACEHOLDER, sim_dir) for arg in run_cmd]
Copy link
Collaborator

@ax3l ax3l Feb 7, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am not sure if it is applied to late or early or not at all, but my test using commit 140baf4 with

ev_main = TemplateEvaluator(
    sim_template="templates/warpx_input_script",
    analysis_func=analysis_func_main,
    executable="/templates/warpx",
    precedent="podman-hpc exec --workdir /data/%LIBENSEMBLE_SIM_DIR% warpx /opt/entrypoint.sh",
    n_gpus=1,  # GPUs per individual evaluation
    env_mpi="srun",
)

fails to replace it for the run line:

$ pip install -U \
  git+https://github.com/ax3l/optimas.git@fix-app-check-container-bp-0.8.1  \
  git+https://github.com/Libensemble/libensemble.git@140baf438ed55714fffa9660b1a391e935014966

  Running command git clone --filter=blob:none --quiet https://github.com/ax3l/optimas.git /tmp/pip-req-build-idhfo1de
  Running command git checkout -b fix-app-check-container-bp-0.8.1 --track origin/fix-app-check-container-bp-0.8.1
  Switched to a new branch 'fix-app-check-container-bp-0.8.1'
  branch 'fix-app-check-container-bp-0.8.1' set up to track 'origin/fix-app-check-container-bp-0.8.1'.
  Running command git clone --filter=blob:none --quiet https://github.com/Libensemble/libensemble.git /tmp/pip-req-build-cdcrxjz4
  Running command git rev-parse -q --verify 'sha^140baf438ed55714fffa9660b1a391e935014966'
  Running command git fetch -q https://github.com/Libensemble/libensemble.git 140baf438ed55714fffa9660b1a391e935014966
  Running command git checkout -q 140baf438ed55714fffa9660b1a391e935014966
[0]  2026-02-06 16:22:33,455 libensemble.libE (INFO): Logger initializing: [workerID] precedes each line. [0] = Manager
[0]  2026-02-06 16:22:33,456 libensemble.libE (INFO): libE version v1.5.0+dev
[0]  2026-02-06 16:22:33,460 libensemble.manager (INFO): Manager exit_criteria: {'sim_max': 1}
[1]  2026-02-06 16:22:40,313 libensemble.executors.mpi_executor (INFO): Launching task libe_task_sim_0_worker1_0: srun -w nid001508 --ntasks 1 --nodes 1 --ntasks-per-node 1 --exact podman-hpc exec --workdir /data/%LIBENSEMBLE_SIM_DIR%/ warpx /opt/entrypoint.sh prepare_simulation.py
...

->

Error: crun: chdir to `/data/%LIBENSEMBLE_SIM_DIR%/`: No such file or directory: OCI runtime attempted to invoke a command that was not found

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I will need to look more next week, maybe wrong place. But in the submit function, I can do this (just adding a precendent) in Optimas test_template_evaluator.py.

        print(f'runline 1: {runline}')
        runline = self._set_sim_dir_env(task, runline)
        task.runline = " ".join(runline)

        print(f'runline 2: {runline}')

Output:

runline 1: ['mpirun', '-hosts', 'shuds', '-np', '1', '--ppn', '1', 'podman-hpc', 'exec', '--workdir', '/data/%LIBENSEMBLE_SIM_DIR%', 'warpx', '/opt/entrypoint.sh', 'template_simulation_script.py']
runline 2: ['mpirun', '-hosts', 'shuds', '-np', '1', '--ppn', '1', 'podman-hpc', 'exec', '--workdir', '/data/tests_output/test_template_evaluator/evaluations/sim0005', 'warpx', '/opt/entrypoint.sh', 'template_simulation_script.py']


def submit(
self,
calc_type: str | None = None,
Expand Down Expand Up @@ -757,6 +775,9 @@ def submit(
if task.app_args is not None:
runline.extend(task.app_args.split())

runline = self._set_sim_dir_env(task, runline)
task.runline = " ".join(runline)

if dry_run:
logger.info(f"Test (No submit) Runline: {' '.join(runline)}")
else:
Expand Down
5 changes: 2 additions & 3 deletions libensemble/executors/mpi_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,6 @@ class MPIExecutor(Executor):

from libensemble.executors.mpi_executor import MPIExecutor
exctr = MPIExecutor(custom_info=customizer)


"""

def __init__(self, custom_info: dict = {}) -> None:
Expand Down Expand Up @@ -363,7 +361,8 @@ def submit(
if task.app_args is not None:
runline.extend(task.app_args.split())

task.runline = " ".join(runline) # Allow to be queried
runline = self._set_sim_dir_env(task, runline)
task.runline = " ".join(runline)

if env_script is not None:
run_cmd = Executor._process_env_script(task, runline, env_script)
Expand Down
4 changes: 3 additions & 1 deletion libensemble/tests/unit_tests/test_executor_gpus.py
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,9 @@ def run_check(exp_env, exp_cmd, **kwargs):
args_for_sim = "sleep 0"
exp_runline = exp_cmd + " simdir/my_simtask.x sleep 0"
task = exctr.submit(calc_type="sim", app_args=args_for_sim, dry_run=True, **kwargs)
assert task.env == exp_env, f"Task env does not match expected:\n Received: {task.env}\n Expected: {exp_env}"
for key, value in exp_env.items():
assert key in task.env, f"Expected env key '{key}' not found in task.env: {task.env}"
assert task.env[key] == value, f"Env key '{key}' has value '{task.env[key]}', expected '{value}'"
assert (
task.runline == exp_runline
), f"Run line does not match expected.\n Received: {task.runline}\n Expected: {exp_runline}"
Expand Down
16 changes: 13 additions & 3 deletions libensemble/tools/test_support.py
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,16 @@ def check_gpu_setting(task, assert_setting=True, print_setting=False, resources=
print(f"Worker {task.workerID}: {desc}GPU setting ({stype}): {gpu_setting} {addon}", flush=True)

if assert_setting:
assert (
gpu_setting == expected
), f"Worker {task.workerID}: Found GPU setting: {gpu_setting}, Expected: {expected}"
if isinstance(expected, dict):
for key, value in expected.items():
assert key in gpu_setting, (
f"Worker {task.workerID}: Expected env key '{key}' not found in GPU setting: {gpu_setting}"
)
assert gpu_setting[key] == value, (
f"Worker {task.workerID}: GPU setting key '{key}' has value '{gpu_setting[key]}', "
f"expected '{value}'"
)
else:
assert (
gpu_setting == expected
), f"Worker {task.workerID}: Found GPU setting: {gpu_setting}, Expected: {expected}"