nf-core/configs Maestro (at Pateur Institute, Paris) Configuration
To use, run the pipeline with -profile maestro,<qos> (with qos being long or normal). This will download and launch the maestro.config which has been pre-configured with a setup suitable for the Maestro cluster on either the long or normal qos.
Using one of these profiles, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline
needed Modules
Please first load java, nextflow and singularity modules
module load java
module load nextflow
module load singularity
Also, do not forget to run nextflow using tmux or alike.
Other profiles at Pasteur
If you are using TARS cluster, please refer to the pasteur profile.
Please refer to docs/pasteur.md for installing and running nf-core instructions.
Config file
params {
config_profile_description = 'Institut Pasteur Maestro cluster profile'
config_profile_url = 'https://research.pasteur.fr/en/equipment/maestro-compute-cluster/'
config_profile_contact = 'Pierre Luisi (@pierrespc)'
}
singularity {
enabled = true
autoMounts = true
runOptions = '--home $HOME:/home/$USER --bind /pasteur'
}
profiles {
normal {
process {
resourceLimits = [
memory: 500.GB,
cpus: 96,
time: 24.h
]
executor = 'slurm'
scratch = false
queue = 'common'
queueSize = 20
clusterOptions = '--qos=normal'
}
params {
igenomes_ignore = true
igenomesIgnore = true
max_memory = 500.GB
max_cpus = 96
max_time = 24.h
}
}
long {
process {
resourceLimits = [
memory: 500.GB,
cpus: 5,
time: 8760.h
]
executor = 'slurm'
scratch = false
queue = 'long'
clusterOptions = '--qos=long -p long'
}
params {
igenomes_ignore = true
igenomesIgnore = true
max_memory = 500.GB
max_cpus = 5
max_time = 8760.h
}
}
}
Pipeline configs
/*
* -------------------------------------------------
* Nextflow config file for running nf-core eager on whole genome data or mitogenomes
* -------------------------------------------------
* nextflow run nf-core/eager -profile maestro,<qos>,maestro,<genome> (where <qos> is long or normal and <genome> is nuclear, mitocondrial or unlimitedtime)
*/
params {
config_profile_name = 'nf-core/eager nuclear/mitocondrial - human profiles'
config_profile_description = "Simple profiles for assessing computational ressources that fit human nuclear dna, human mitogenomes processing. unlimitedtime is also available "
}
profiles {
nuclear {
process {
errorStrategy = 'retry'
maxRetries = 2
withName: makeBWAIndex {
cpus = { check_max(8 * task.attempt, 'cpus') }
memory = { check_max(8.GB * task.attempt, 'memory') }
time = { check_max(12.h * task.attempt, 'time') }
}
withName: adapter_removal {
cpus = { check_max(8 * task.attempt, 'cpus') }
memory = { check_max(16.GB * task.attempt, 'memory') }
time = { check_max(12.h * task.attempt, 'time') }
}
withName: bwa {
cpus = { check_max(40 * task.attempt, 'cpus') }
memory = { check_max(40.GB * task.attempt, 'memory') }
time = 24.h
cache = 'deep'
}
withName: markduplicates {
errorStrategy = { task.exitStatus in [143, 137, 104, 134, 139] ? 'retry' : 'finish' }
cpus = { check_max(16 * task.attempt, 'cpus') }
memory = { check_max(16.GB * task.attempt, 'memory') }
time = { check_max(12.h * task.attempt, 'time') }
}
withName: damageprofiler {
cpus = 1
memory = { check_max(8.GB * task.attempt, 'memory') }
time = { check_max(6.h * task.attempt, 'time') }
}
withName: fastp {
cpus = 8
memory = { check_max(8.GB * task.attempt, 'memory') }
time = { check_max(6.h * task.attempt, 'time') }
}
withName: fastqc {
cpus = 2
memory = { check_max(8.GB * task.attempt, 'memory') }
time = { check_max(6.h * task.attempt, 'time') }
}
}
}
mitocondrial {
process {
errorStrategy = 'retry'
maxRetries = 2
withName: makeBWAIndex {
cpus = { check_max(8 * task.attempt, 'cpus') }
memory = { check_max(8.GB * task.attempt, 'memory') }
time = { check_max(12.h * task.attempt, 'time') }
}
withName: adapter_removal {
cpus = { check_max(8 * task.attempt, 'cpus') }
memory = { check_max(16.GB * task.attempt, 'memory') }
time = { check_max(12.h * task.attempt, 'time') }
}
withName: bwa {
cpus = { check_max(5 * task.attempt, 'cpus') }
memory = { check_max(5.GB * task.attempt, 'memory') }
time = 24.h
}
withName: markduplicates {
errorStrategy = { task.exitStatus in [143, 137, 104, 134, 139] ? 'retry' : 'finish' }
cpus = { check_max(5 * task.attempt, 'cpus') }
memory = { check_max(5.GB * task.attempt, 'memory') }
time = { check_max(6.h * task.attempt, 'time') }
}
withName: damageprofiler {
cpus = 1
memory = { check_max(5.GB * task.attempt, 'memory') }
time = { check_max(3.h * task.attempt, 'time') }
}
withName: fastp {
cpus = 8
memory = { check_max(5.GB * task.attempt, 'memory') }
time = { check_max(3.h * task.attempt, 'time') }
}
withName: fastqc {
cpus = 2
memory = { check_max(8.GB * task.attempt, 'memory') }
time = { check_max(6.h * task.attempt, 'time') }
}
}
}
unlimitedtime {
process {
errorStrategy = 'finish'
cpus = 5
memory = 200.GB
time = 8760.h
}
}
}
// Function to ensure that resource requirements don't go beyond
// a maximum limit
// FOR DSL1 PIPELINE ONLY!
def check_max(obj, type) {
if (type == 'memory') {
try {
if (obj.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1)
return params.max_memory as nextflow.util.MemoryUnit
else
return obj
} catch (all) {
println " ### ERROR ### Max memory '${params.max_memory}' is not valid! Using default value: $obj"
return obj
}
} else if (type == 'time') {
try {
if (obj.compareTo(params.max_time as nextflow.util.Duration) == 1)
return params.max_time as nextflow.util.Duration
else
return obj
} catch (all) {
println " ### ERROR ### Max time '${params.max_time}' is not valid! Using default value: $obj"
return obj
}
} else if (type == 'cpus') {
try {
return Math.min( obj, params.max_cpus as int )
} catch (all) {
println " ### ERROR ### Max cpus '${params.max_cpus}' is not valid! Using default value: $obj"
return obj
}
}
}
executor
slurm
contact
Last modified
2024-09-24