Skip to content

Base runner

CLASS DESCRIPTION
BaseRunner

Base class for Fractal runners.

MultisubmitTaskType

Valid task types for BaseRunner.multisubmit.

SubmitTaskType

Valid task types for BaseRunner.submit.

ATTRIBUTE DESCRIPTION
TASK_TYPES_MULTISUBMIT

List of valid task types for BaseRunner.multisubmit.

TYPE: list[TaskType]

TASK_TYPES_SUBMIT

List of valid task types for BaseRunner.submit.

TYPE: list[TaskType]

Attributes

TASK_TYPES_MULTISUBMIT = [TaskType.COMPOUND, TaskType.CONVERTER_COMPOUND, TaskType.PARALLEL] module-attribute

List of valid task types for BaseRunner.multisubmit.

TASK_TYPES_SUBMIT = [TaskType.COMPOUND, TaskType.CONVERTER_COMPOUND, TaskType.NON_PARALLEL, TaskType.CONVERTER_NON_PARALLEL] module-attribute

List of valid task types for BaseRunner.submit.

Classes

BaseRunner

Base class for Fractal runners.

METHOD DESCRIPTION
multisubmit

Run a parallel fractal task.

submit

Run a single fractal task.

validate_multisubmit_parameters

Validate parameters for multisubmit method

validate_submit_parameters

Validate parameters for submit method

Source code in fractal_server/runner/executors/base_runner.py
class BaseRunner:
    """
    Base class for Fractal runners.
    """

    shared_config: JobRunnerConfigLocal | JobRunnerConfigSLURM
    fractal_job_id: int
    resource_id: int
    executor_error_log: str | None = None

    def submit(
        self,
        *,
        base_command: str,
        workflow_task_order: int,
        workflow_task_id: int,
        task_name: str,
        parameters: dict[str, Any],
        history_unit_id: int,
        task_type: SubmitTaskType,
        task_files: TaskFiles,
        user_id: int,
        config: Any,
    ) -> tuple[Any, BaseException | None]:
        """
        Run a single fractal task.

        Args:
            base_command:
            workflow_task_order:
            workflow_task_id:
            task_name:
            parameters: Dictionary of parameters.
            history_unit_id:
                Database ID of the corresponding `HistoryUnit` entry.
            task_type: Task type.
            task_files: `TaskFiles` object.
            config: Runner-specific parameters.
            user_id:
        """
        raise NotImplementedError()

    def multisubmit(
        self,
        *,
        base_command: str,
        workflow_task_order: int,
        workflow_task_id: int,
        task_name: str,
        list_parameters: list[dict],
        history_unit_ids: list[int],
        list_task_files: list[TaskFiles],
        task_type: MultisubmitTaskType,
        config: Any,
        user_id: int,
    ) -> tuple[dict[int, Any], dict[int, BaseException]]:
        """
        Run a parallel fractal task.

        Note: `list_parameters`, `list_task_files` and `history_unit_ids`
        have the same size. For parallel tasks, this is also the number of
        input images, while for compound tasks these can differ.

        Args:
            base_command:
            workflow_task_order:
            workflow_task_id:
            task_name:
            list_parameters:
                List of dictionaries of parameters (each one must include
                `zarr_urls` key).
            history_unit_ids:
                Database IDs of the corresponding `HistoryUnit` entries.
            list_task_files: `TaskFiles` objects.
            task_type: Task type.
            config: Runner-specific parameters.
            user_id:
        """
        raise NotImplementedError()

    def validate_submit_parameters(
        self,
        parameters: dict[str, Any],
        task_type: SubmitTaskType | MultisubmitTaskType,
    ) -> None:
        """
        Validate parameters for `submit` method

        Args:
            parameters: Parameters dictionary.
            task_type: Task type.
        """
        logger.info("[validate_submit_parameters] START")
        if task_type not in TASK_TYPES_SUBMIT:
            raise ValueError(f"Invalid {task_type=} for `submit`.")
        if not isinstance(parameters, dict):
            raise ValueError("`parameters` must be a dictionary.")
        if task_type in [
            TaskType.NON_PARALLEL,
            TaskType.COMPOUND,
        ]:
            if "zarr_urls" not in parameters.keys():
                raise ValueError(
                    f"No 'zarr_urls' key in in {list(parameters.keys())}"
                )
        elif task_type in [
            TaskType.CONVERTER_NON_PARALLEL,
            TaskType.CONVERTER_COMPOUND,
        ]:
            if "zarr_urls" in parameters.keys():
                raise ValueError(
                    f"Forbidden 'zarr_urls' key in {list(parameters.keys())}"
                )
        logger.info("[validate_submit_parameters] END")

    def validate_multisubmit_parameters(
        self,
        *,
        task_type: MultisubmitTaskType,
        list_parameters: list[dict[str, Any]],
        list_task_files: list[TaskFiles],
        history_unit_ids: list[int],
    ) -> None:
        """
        Validate parameters for `multisubmit` method

        Args:
            task_type: Task type.
            list_parameters: List of parameters dictionaries.
            list_task_files:
            history_unit_ids:
        """
        if task_type not in TASK_TYPES_MULTISUBMIT:
            raise ValueError(f"Invalid {task_type=} for `multisubmit`.")

        if not isinstance(list_parameters, list):
            raise ValueError("`parameters` must be a list.")

        if len(list_parameters) != len(list_task_files):
            raise ValueError(
                f"{len(list_task_files)=} differs from {len(list_parameters)=}."
            )
        if len(history_unit_ids) != len(list_parameters):
            raise ValueError(
                f"{len(history_unit_ids)=} differs from "
                f"{len(list_parameters)=}."
            )

        subfolders = {
            task_file.wftask_subfolder_local for task_file in list_task_files
        }
        if len(subfolders) != 1:
            raise ValueError(f"More than one subfolders: {subfolders}.")

        for single_kwargs in list_parameters:
            if not isinstance(single_kwargs, dict):
                raise ValueError("kwargs itemt must be a dictionary.")
            if "zarr_url" not in single_kwargs.keys():
                raise ValueError(
                    f"No 'zarr_url' key in in {list(single_kwargs.keys())}"
                )
        if task_type == TaskType.PARALLEL:
            zarr_urls = [kwargs["zarr_url"] for kwargs in list_parameters]
            if len(zarr_urls) != len(set(zarr_urls)):
                raise ValueError("Non-unique zarr_urls")

Methods:

multisubmit(*, base_command, workflow_task_order, workflow_task_id, task_name, list_parameters, history_unit_ids, list_task_files, task_type, config, user_id)

Run a parallel fractal task.

Note: list_parameters, list_task_files and history_unit_ids have the same size. For parallel tasks, this is also the number of input images, while for compound tasks these can differ.

PARAMETER DESCRIPTION
base_command

TYPE: str

workflow_task_order

TYPE: int

workflow_task_id

TYPE: int

task_name

TYPE: str

list_parameters

List of dictionaries of parameters (each one must include zarr_urls key).

TYPE: list[dict]

history_unit_ids

Database IDs of the corresponding HistoryUnit entries.

TYPE: list[int]

list_task_files

TaskFiles objects.

TYPE: list[TaskFiles]

task_type

Task type.

TYPE: MultisubmitTaskType

config

Runner-specific parameters.

TYPE: Any

user_id

TYPE: int

Source code in fractal_server/runner/executors/base_runner.py
def multisubmit(
    self,
    *,
    base_command: str,
    workflow_task_order: int,
    workflow_task_id: int,
    task_name: str,
    list_parameters: list[dict],
    history_unit_ids: list[int],
    list_task_files: list[TaskFiles],
    task_type: MultisubmitTaskType,
    config: Any,
    user_id: int,
) -> tuple[dict[int, Any], dict[int, BaseException]]:
    """
    Run a parallel fractal task.

    Note: `list_parameters`, `list_task_files` and `history_unit_ids`
    have the same size. For parallel tasks, this is also the number of
    input images, while for compound tasks these can differ.

    Args:
        base_command:
        workflow_task_order:
        workflow_task_id:
        task_name:
        list_parameters:
            List of dictionaries of parameters (each one must include
            `zarr_urls` key).
        history_unit_ids:
            Database IDs of the corresponding `HistoryUnit` entries.
        list_task_files: `TaskFiles` objects.
        task_type: Task type.
        config: Runner-specific parameters.
        user_id:
    """
    raise NotImplementedError()
submit(*, base_command, workflow_task_order, workflow_task_id, task_name, parameters, history_unit_id, task_type, task_files, user_id, config)

Run a single fractal task.

PARAMETER DESCRIPTION
base_command

TYPE: str

workflow_task_order

TYPE: int

workflow_task_id

TYPE: int

task_name

TYPE: str

parameters

Dictionary of parameters.

TYPE: dict[str, Any]

history_unit_id

Database ID of the corresponding HistoryUnit entry.

TYPE: int

task_type

Task type.

TYPE: SubmitTaskType

task_files

TaskFiles object.

TYPE: TaskFiles

config

Runner-specific parameters.

TYPE: Any

user_id

TYPE: int

Source code in fractal_server/runner/executors/base_runner.py
def submit(
    self,
    *,
    base_command: str,
    workflow_task_order: int,
    workflow_task_id: int,
    task_name: str,
    parameters: dict[str, Any],
    history_unit_id: int,
    task_type: SubmitTaskType,
    task_files: TaskFiles,
    user_id: int,
    config: Any,
) -> tuple[Any, BaseException | None]:
    """
    Run a single fractal task.

    Args:
        base_command:
        workflow_task_order:
        workflow_task_id:
        task_name:
        parameters: Dictionary of parameters.
        history_unit_id:
            Database ID of the corresponding `HistoryUnit` entry.
        task_type: Task type.
        task_files: `TaskFiles` object.
        config: Runner-specific parameters.
        user_id:
    """
    raise NotImplementedError()
validate_multisubmit_parameters(*, task_type, list_parameters, list_task_files, history_unit_ids)

Validate parameters for multisubmit method

PARAMETER DESCRIPTION
task_type

Task type.

TYPE: MultisubmitTaskType

list_parameters

List of parameters dictionaries.

TYPE: list[dict[str, Any]]

list_task_files

TYPE: list[TaskFiles]

history_unit_ids

TYPE: list[int]

Source code in fractal_server/runner/executors/base_runner.py
def validate_multisubmit_parameters(
    self,
    *,
    task_type: MultisubmitTaskType,
    list_parameters: list[dict[str, Any]],
    list_task_files: list[TaskFiles],
    history_unit_ids: list[int],
) -> None:
    """
    Validate parameters for `multisubmit` method

    Args:
        task_type: Task type.
        list_parameters: List of parameters dictionaries.
        list_task_files:
        history_unit_ids:
    """
    if task_type not in TASK_TYPES_MULTISUBMIT:
        raise ValueError(f"Invalid {task_type=} for `multisubmit`.")

    if not isinstance(list_parameters, list):
        raise ValueError("`parameters` must be a list.")

    if len(list_parameters) != len(list_task_files):
        raise ValueError(
            f"{len(list_task_files)=} differs from {len(list_parameters)=}."
        )
    if len(history_unit_ids) != len(list_parameters):
        raise ValueError(
            f"{len(history_unit_ids)=} differs from "
            f"{len(list_parameters)=}."
        )

    subfolders = {
        task_file.wftask_subfolder_local for task_file in list_task_files
    }
    if len(subfolders) != 1:
        raise ValueError(f"More than one subfolders: {subfolders}.")

    for single_kwargs in list_parameters:
        if not isinstance(single_kwargs, dict):
            raise ValueError("kwargs itemt must be a dictionary.")
        if "zarr_url" not in single_kwargs.keys():
            raise ValueError(
                f"No 'zarr_url' key in in {list(single_kwargs.keys())}"
            )
    if task_type == TaskType.PARALLEL:
        zarr_urls = [kwargs["zarr_url"] for kwargs in list_parameters]
        if len(zarr_urls) != len(set(zarr_urls)):
            raise ValueError("Non-unique zarr_urls")
validate_submit_parameters(parameters, task_type)

Validate parameters for submit method

PARAMETER DESCRIPTION
parameters

Parameters dictionary.

TYPE: dict[str, Any]

task_type

Task type.

TYPE: SubmitTaskType | MultisubmitTaskType

Source code in fractal_server/runner/executors/base_runner.py
def validate_submit_parameters(
    self,
    parameters: dict[str, Any],
    task_type: SubmitTaskType | MultisubmitTaskType,
) -> None:
    """
    Validate parameters for `submit` method

    Args:
        parameters: Parameters dictionary.
        task_type: Task type.
    """
    logger.info("[validate_submit_parameters] START")
    if task_type not in TASK_TYPES_SUBMIT:
        raise ValueError(f"Invalid {task_type=} for `submit`.")
    if not isinstance(parameters, dict):
        raise ValueError("`parameters` must be a dictionary.")
    if task_type in [
        TaskType.NON_PARALLEL,
        TaskType.COMPOUND,
    ]:
        if "zarr_urls" not in parameters.keys():
            raise ValueError(
                f"No 'zarr_urls' key in in {list(parameters.keys())}"
            )
    elif task_type in [
        TaskType.CONVERTER_NON_PARALLEL,
        TaskType.CONVERTER_COMPOUND,
    ]:
        if "zarr_urls" in parameters.keys():
            raise ValueError(
                f"Forbidden 'zarr_urls' key in {list(parameters.keys())}"
            )
    logger.info("[validate_submit_parameters] END")

MultisubmitTaskType

Bases: StrEnum

Valid task types for BaseRunner.multisubmit.

ATTRIBUTE DESCRIPTION
PARALLEL

Parallel task.

COMPOUND

Compound task.

CONVERTER_COMPOUND

Compound converter task.

Source code in fractal_server/runner/executors/base_runner.py
class MultisubmitTaskType(StrEnum):
    """
    Valid task types for `BaseRunner.multisubmit`.

    Attributes:
        PARALLEL: Parallel task.
        COMPOUND: Compound task.
        CONVERTER_COMPOUND: Compound converter task.
    """

    PARALLEL = TaskType.PARALLEL
    COMPOUND = TaskType.COMPOUND
    CONVERTER_COMPOUND = TaskType.CONVERTER_COMPOUND

SubmitTaskType

Bases: StrEnum

Valid task types for BaseRunner.submit.

ATTRIBUTE DESCRIPTION
NON_PARALLEL

Non-parallel task.

COMPOUND

Compound task.

CONVERTER_NON_PARALLEL

Non-parallel converter task.

CONVERTER_COMPOUND

Compound converter task.

Source code in fractal_server/runner/executors/base_runner.py
class SubmitTaskType(StrEnum):
    """
    Valid task types for `BaseRunner.submit`.

    Attributes:
        NON_PARALLEL: Non-parallel task.
        COMPOUND: Compound task.
        CONVERTER_NON_PARALLEL: Non-parallel converter task.
        CONVERTER_COMPOUND: Compound converter task.
    """

    NON_PARALLEL = TaskType.NON_PARALLEL
    COMPOUND = TaskType.COMPOUND
    CONVERTER_NON_PARALLEL = TaskType.CONVERTER_NON_PARALLEL
    CONVERTER_COMPOUND = TaskType.CONVERTER_COMPOUND

Functions: