Changes between Version 4 and Version 5 of SOPs/nextflow

Timestamp:

05/30/25 16:58:16 (6 months ago)

Author:

xinlei.gao

Comment:

--

SOPs/nextflow

@@ -108 +108 @@
 Please refer to nf-core documentation for more details (https://nf-co.re/docs/usage/getting_started/introduction).
 == Run nf-core pipelines on the slurm cluster
+The slurm executor allows you to run your pipeline script using the SLURM resource manager.
 
+Nextflow manages each process as a separate job that is submitted to the cluster using the sbatch command. The jobs can be distributed across multiple nodes depending on your request computing resources.
+
+The pipeline must be launched from a node where the sbatch command is available, which is typically the cluster login node.
+
+To enable the SLURM executor, set process.executor = 'slurm' in the nextflow.config file.
+
+SLURM partitions can be specified with the queue directive.
+
+To submit the nf-core pipelines to the slurm cluster, you could provide a configure file like this (take nf-core/cutandrun pipeline as an example and save it as 'cutandrun.config'):
+
+
+{{{
+process{
+  executor = 'slurm'
+  queue = '20'
+  slurm.queueSize = 10
+  memory = '200 GB'
+  cpus = 36
+  
+  withName: 'NFCORE_CUTANDRUN:CUTANDRUN:DEDUPLICATE_PICARD:BAM_SORT_STATS_SAMTOOLS:SAMTOOLS_SORT' {
+    cpus = { 6 * task.attempt }
+    memory = { 15.GB * task.attempt }
+  }
+  withName: 'NFCORE_CUTANDRUN:CUTANDRUN:PREPARE_PEAKCALLING:BEDTOOLS_SORT' {
+    cpus = { 1 * task.attempt }
+    memory = { 12.GB * task.attempt }
+  }
+  withName: 'NFCORE_CUTANDRUN:CUTANDRUN:SAMTOOLS_CUSTOMVIEW' {
+    cpus = { 2 * task.attempt }
+    memory = { 8.GB * task.attempt }
+  }
+  withName: 'NFCORE_CUTANDRUN:CUTANDRUN:FRAG_LEN_HIST' {
+    cpus = { 4 * task.attempt }
+    memory = { 12.GB * task.attempt }
+  }
+  withName: 'NFCORE_CUTANDRUN:CUTANDRUN:DEEPTOOLS_PLOTHEATMAP_GENE_ALL' {
+    cpus = { 4 * task.attempt }
+    memory = { 32.GB * task.attempt }
+  }
+}
+
+}}}
+
+And then submit the job using the command line like below:
+
+
+{{{
+sbatch --partition=20 --job-name=NextF --output=NextF-%j.out  --mem=200gb   --nodes=1 --ntasks=1 --cpus-per-task=36 --wrap \
+" /nfs/BaRC_Public/apps/nextflow/nextflow run nf-core/cutandrun -profile singularity --normalisation_binsize 1  --input samplesheet.csv -c cutandrun.config  --normalisation_mode CPM \
+--peakcaller 'MACS2' --replicate_threshold 2  --end_to_end FALSE  --multiqc_title 'multiQCReport' --skip_removeduplicates true \
+--skip_preseq false   --skip_dt_qc false --skip_multiqc false --skip_reporting false --dump_scale_factors true --email 'username@wi.mit.edu' --genome GRCh38 \
+--extend_fragments false --macs2_qvalue 0.01 --outdir  ./nextFlow_CUTTAG "
+}}}
+
+
+Reference link:
+https://www.nextflow.io/docs/latest/executor.html
 
 == The recommendations of running individual nf-core pipelines