8000 template: include container configs in nextflow.config by mashehu · Pull Request #4600 · nf-core/tools · GitHub
[go: up one dir, main page]

Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
197 changes: 186 additions & 11 deletions nf_core/pipeline-template/nextflow.config
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,9 @@ process {
maxErrors = '-1'
}
{%- endif %}
{%- if modules and container_configs %}
includeConfig 'conf/containers_docker_amd64.config'
{%- endif %}

profiles {
debug {
Expand All @@ -102,16 +105,48 @@ profiles {
nextflow.enable.configProcessNamesValidation = true
}
conda {
{%- if modules and container_configs %}
includeConfig 'conf/containers_conda_lock_files_amd64.config'
{%- endif %}
conda.enabled = true
conda.channels = ['conda-forge', 'bioconda']
docker.enabled = false
singularity.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = false
apptainer.enabled = false
}
{%- if modules and container_configs %}
conda_arm {
{%- if modules and container_configs %}
includeConfig 'conf/containers_conda_lock_files_arm64.config'
{%- endif %}
conda.enabled = true
conda.channels = ['conda-forge', 'bioconda']
docker.enabled = false
singularity.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = false
apptainer.enabled = false
}
// Use the environment.yml file next to each module's main.nf instead of conda lock files
conda_env {
conda.enabled = true
conda.channels = ['conda-forge', 'bioconda']
docker.enabled = false
singularity.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = false
apptainer.enabled = false
}
{%- endif %}
mamba {
{%- if modules and container_configs %}
includeConfig 'conf/containers_conda_lock_files_amd64.config'
{%- endif %}
conda.enabled = true
conda.useMamba = true
docker.enabled = false
Expand All @@ -121,31 +156,96 @@ profiles {
charliecloud.enabled = false
apptainer.enabled = false
}
{%- if modules and container_configs %}
mamba_arm {
{%- if modules and container_configs %}
includeConfig 'conf/containers_conda_lock_files_arm64.config'
{%- endif %}
conda.enabled = true
conda.useMamba = true
docker.enabled = false
singularity.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = false
apptainer.enabled = false
}
// Use the environment.yml file next to each module's main.nf instead of conda lock files
mamba_env {
conda.enabled = true
conda.useMamba = true
docker.enabled = false
singularity.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = false
apptainer.enabled = false
}
{%- endif %}
docker {
docker.enabled = true
docker.runOptions = '-u $(id -u):$(id -g)'
conda.enabled = false
singularity.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = false
apptainer.enabled = false
docker.runOptions = '-u $(id -u):$(id -g)'
}
< 7438 /td> arm64 {
process.arch = 'arm64'
// TODO https://github.com/nf-core/modules/issues/6694
// For now if you're using arm64 you have to use wave for the sake of the maintainers
// wave profile
apptainer.ociAutoPull = true
singularity.ociAutoPull = true
wave.enabled = true
wave.freeze = true
wave.strategy = 'conda,container'
{%- if modules and container_configs %}
docker_arm {
includeConfig 'conf/containers_docker_arm64.config'
docker.enabled = true
docker.runOptions = '-u $(id -u):$(id -g)'
conda.enabled = false
singularity.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = false
apptainer.enabled = false
}
{%- endif %}
emulate_amd64 {
docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64'
}
singularity {
{%- if modules and container_configs %}
includeConfig 'conf/containers_singularity_https_amd64.config'
{%- endif %}
singularity.enabled = true
singularity.autoMounts = true
conda.enabled = false
docker.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = false
apptainer.enabled = false
}
{%- if modules and container_configs %}
singularity_arm {
includeConfig 'conf/containers_singularity_https_arm64.config'
singularity.enabled = true
singularity.autoMounts = true
conda.enabled = false
docker.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = false
apptainer.enabled = false
}
singularity_oras {
includeConfig 'conf/containers_singularity_oras_amd64.config'
singularity.enabled = true
singularity.autoMounts = true
conda.enabled = false
docker.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = false
apptainer.enabled = false
}
singularity_oras_arm {
includeConfig 'conf/containers_singularity_oras_arm64.config'
singularity.enabled = true
singularity.autoMounts = true
conda.enabled = false
Expand All @@ -155,6 +255,7 @@ profiles {
charliecloud.enabled = false
apptainer.enabled = false
}
{%- endif %}
podman {
podman.enabled = true
conda.enabled = false
Expand All @@ -164,6 +265,18 @@ profiles {
charliecloud.enabled = false
apptainer.enabled = false
}
{%- if modules and container_configs %}
podman_arm {
includeConfig 'conf/containers_docker_arm64.config'
podman.enabled = true
conda.enabled = false
docker.enabled = false
singularity.enabled = false
shifter.enabled = false
charliecloud.enabled = false
apptainer.enabled = false
}
{%- endif %}
shifter {
shifter.enabled = true
conda.enabled = false
Expand All @@ -173,6 +286,18 @@ profiles {
charliecloud.enabled = false
apptainer.enabled = false
}
{%- if modules and container_configs %}
shifter_arm {
includeConfig 'conf/containers_docker_arm64.config'
shifter.enabled = true
conda.enabled = false
docker.enabled = false
singularity.enabled = false
podman.enabled = false
charliecloud.enabled = false
apptainer.enabled = false
}
{%- endif %}
charliecloud {
charliecloud.enabled = true
conda.enabled = false
Expand All @@ -182,7 +307,45 @@ profiles {
shifter.enabled = false
apptainer.enabled = false
}
{%- if modules and container_configs %}
charliecloud_arm {
includeConfig 'conf/containers_docker_arm64.config'
charliecloud.enabled = true
conda.enabled = false
docker.enabled = false
singularity.enabled = false
podman.enabled = false
shifter.enabled = false
apptainer.enabled = false
}
{%- endif %}
apptainer {
{%- if modules and container_configs %}
includeConfig 'conf/containers_singularity_https_amd64.config'
{%- endif %}
apptainer.enabled = true
apptainer.autoMounts = true
conda.enabled = false
docker.enabled = false
singularity.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = false
}
{%- if modules and container_configs %}
apptainer_arm {
includeConfig 'conf/containers_singularity_https_arm64.config'
apptainer.enabled = true
apptainer.autoMounts = true
conda.enabled = false
docker.enabled = false
singularity.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = false
}
apptainer_oras {
includeConfig 'conf/containers_singularity_oras_amd64.config'
apptainer.enabled = true
apptainer.autoMounts = true
conda.enabled = false
Expand All @@ -192,6 +355,18 @@ profiles {
shifter.enabled = false
charliecloud.enabled = false
}
apptainer_oras_arm {
includeConfig 'conf/containers_singularity_oras_arm64.config'
apptainer.enabled = true
apptainer.autoMounts = true
conda.enabled = false
docker.enabled = false
singularity.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = false
}
{%- endif %}
wave {
apptainer.ociAutoPull = true
singularity.ociAutoPull = true
Expand Down
59 changes: 37 additions & 22 deletions nf_core/pipelines/containers_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

import yaml

from nf_core.utils import read_module_name
from nf_core.utils import CONFIG_PATHS, get_first_available_path, read_module_name

log = logging.getLogger(__name__)

Expand All @@ -21,6 +21,9 @@
"conda_lock_files_arm64": ["conda", "linux/arm64", "lock_file"],
}

# Written when no module covers a platform: nextflow.config always includes every file, and a missing include fails parsing
EMPTY_CONFIG = "// No module in this pipeline provides containers for this platform\n"

_CONFIG_LINE_RE = re.compile(r"withName:\s*'(\w+)'\s*\{\s*(?:container|conda)\s*=\s*'([^']+)'")


Expand All @@ -39,20 +42,20 @@ def _parse_config_file(config_path: Path) -> dict[str, str]:
return result


def _write_platform_config(config_path: Path, entries: dict[str, str], key: str) -> bool:
"""Write *entries* to *config_path*, or delete it when empty.
def _write_platform_config(config_path: Path, entries: dict[str, str], key: str) -> None:
"""Write *entries* to *config_path*, or a placeholder comment when empty.

Skips the write when content is unchanged. Returns True if the file now exists.
Skips the write when content is unchanged.
"""
if not entries:
config_path.unlink(missing_ok=True)
return False
new_content = "".join(
f"process {{ withName: '{name}' {{ {key} = '{container}' }} }}\n" for name, container in sorted(entries.items())
new_content = (
"".join(
f"process {{ withName: '{name}' {{ {key} = '{container}' }} }}\n"
for name, container in sorted(entries.items())
)
or EMPTY_CONFIG
)
if not config_path.exists() or config_path.read_text() != new_content:
config_path.write_text(new_content)
return True


def _process_meta(meta_path: Path) -> tuple[str, dict[str, str]] | None:
Expand Down Expand Up @@ -121,6 +124,8 @@ def update_module_container_config(self, module_path: Path) -> None:
entries = _parse_config_file(config_path)
if platform in platform_containers:
entries[module_name] = platform_containers[platform]
else:
entries.pop(module_name, None)
_write_platform_config(config_path, entries, _container_key(platform))

def generate_container_configs(self) -> set[str]:
Expand All @@ -130,12 +135,11 @@ def generate_container_configs(self) -> set[str]:
modules may have changed.

Returns:
set[str]: Names of config files written (e.g. ``{'containers_docker_amd64.config'}``).
set[str]: Names of config files written (e.g. ``{'containers_docker_amd64.config'}``), one per platform.
"""
modules_dir = self.workflow_directory / "modules"
if not modules_dir.is_dir():
log.debug(f"No modules directory found at {modules_dir}, skipping")
return set()
# Write placeholders even without modules/: nextflow.config includes every platform file
(self.workflow_directory / "conf").mkdir(exist_ok=True)

containers: dict[str, dict[str, str]] = {platform: {} for platform in PLATFORMS}

Expand All @@ -149,25 +153,36 @@ def generate_container_configs(self) -> set[str]:
for platform_name, container in platform_containers.items():
containers[platform_name][module_name] = container

# Every platform is visited, so each config is fully rewritten (dropping removed
# modules) or deleted when empty; unchanged files are left untouched.
written: set[str] = set()
# Every platform is rewritten (dropping removed modules); unchanged files are left untouched.
for platform, module_containers in containers.items():
config_path = self._config_path(platform)
if _write_platform_config(config_path, module_containers, _container_key(platform)):
written.add(config_path.name)
_write_platform_config(self._config_path(platform), module_containers, _container_key(platform))

log.info("Generated container configs for the pipeline.")
return written
return {self._config_path(platform).name for platform in PLATFORMS}


def container_configs_skipped(directory: Path) -> bool:
"""Whether the pipeline in *directory* opted out of the ``container_configs`` template feature."""
# Read the YAML directly: load_tools_config runs `nextflow config`, which fails while includes are missing
config_fn = get_first_available_path(directory, CONFIG_PATHS)
if config_fn is None:
return False
with contextlib.suppress(OSError, yaml.YAMLError):
template = (yaml.safe_load(config_fn.read_text()) or {}).get("template") or {}
return "container_configs" in (template.get("skip_features") or template.get("skip") or [])
return False


def try_generate_container_configs(directory: Path, module_path: Path | None = None) -> None:
"""Regenerate container configs for *directory*.

If *module_path* is given, only that module's entries are updated (fast
path for single-module installs). Otherwise a full scan of ``modules/``
is performed.
is performed. Does nothing if the pipeline skips the ``container_configs`` feature.
"""
if container_configs_skipped(directory):
log.debug("Pipeline skips the `container_configs` feature, not generating container configs")
return
try:
configs = ContainerConfigs(directory)
if module_path is not None:
Expand Down
Loading
Loading
0