Gnu Parallel - Parallelize Serial Command Line Programs Without Changing Them

e.g.

(wget -O - pi.dk/3 || curl pi.dk/3/ || fetch -o - http://pi.dk/3) | bash

for i in *gz; do 
  zcat $i > $(basename $i .gz).unpacked
done

parallel 'zcat {} > {.}.unpacked' ::: *.gz

cat foo.fa | parallel --round-robin --pipe --recstart '>' 'blat -noHead genome.fa stdin >(cat) >&2' >foo.psl

@M10991:61:000000000-A7EML:1:1101:14011:1001 1:N:0:28
CTCCTAGGTCGGCATGATGGGGGAAGGAGAGCATGGGAAGAAATGAGAGAGTAGCAAGG
+
#8BCCGGGGGFEFECFGGGGGGGGG@;FFGGGEG@FF<EE<@FFC,CEGCCGGFF<FGF

@HWUSI-EAS100R:6:73:941:1973#0/1
@EAS139:136:FC706VJ:2:2104:15343:197393 1:Y:18:ATCACG
@EAS139:136:FC706VJ:2:2104:15343:197393 1:N:18:1

parallel --pipepart -a big.fq --block -1 --regexp \
       --recend '\n' --recstart '@.*(/1| 1:.*)\n[A-Za-z\n\.~]' \
       fastq-reader

cat 1gb.fasta | parallel --block 100k --recstart '>' --pipe blastp -evalue 0.01 -outfmt 6 -db db.fa -query - > results

cat 1gb.fasta | parallel -S :,server1,server2 --block 100k --recstart '>' --pipe blastp -evalue 0.01 -outfmt 6 -db db.fa -query - > result

parallel bigWigToWig -chrom=chr{} wgEncodeCrgMapabilityAlign36mer_mm9.bigWig mm9_36mer_chr{}.map ::: {1..19} X Y M

parallel 'read_fasta -i {} | extract_seq -l 5 | write_fasta -o {.}_trim.fna -x' ::: *.fna

experiment --age 18 --sex M --chr 22

parallel experiment --age {1} --sex {2} --chr {3} ::: {1..80} ::: M F ::: {1..22} X Y

parallel experiment --age {1} --sex {2} --chr {3} '>' output.{1}.{2}.{3} ::: {1..80} ::: M F ::: {1..22} X Y

parallel --results outputdir experiment --age {1} --sex {2} --chr {3} ::: {1..80} ::: M F ::: {1..22} X Y

parallel --result outputdir --header : experiment --age {AGE} --sex {SEX} --chr {CHR} ::: AGE {1..80} ::: SEX M F ::: CHR {1..22} X Y

parallel --result output.tsv --header : experiment --age {AGE} --sex {SEX} --chr {CHR} ::: AGE {1..80} ::: SEX M F ::: CHR {1..22} X Y

echo AGE > age_file
seq 1 80 >> age_file
parallel --results outputdir --header : experiment --age {AGE} --sex {SEX} --chr {CHR} :::: age_file ::: SEX M F ::: CHR {1..22} X Y

parallel --shuf --results outputdir --header : experiment --age {AGE} --sex {SEX} --chr {CHR} :::: age_file ::: SEX M F ::: CHR {1..22} X Y

launch ID

parallel launch ::: ID1 ID2 ...

launch ID1 ID2 ...

#!/usr/bin/env bash
#!/usr/bin/env perl
#!/usr/bin/env python

#!/usr/bin/parallel --shebang-wrap bash
#!/usr/bin/parallel --shebang-wrap perl
#!/usr/bin/parallel --shebang-wrap python

launch ID DIR

#!/usr/bin/parallel --shebang-wrap bash

launch ID1 ID2 ID3 ::: DIR

$ ./transparal_old.pl 
Could not open file! at ./transparal_old.pl line 5. ( did not give the input file name!)

#!/usr/bin/parallel --shebang-wrap perl

$ ./transparal.pl 
-bash: ./transparal.pl: /usr/bin/parallel: bad interpreter: No such file or directory

$ which parallel
/usr/local/bin/parallel

cat file | parallel --fifo --pipe wc {}

cat file | parallel --cat --pipe wc {}

parallel jobs 1
wait
parallel jobs 2
...etc

parallel 'sleep {};echo Jobslot {%} slept {} seconds' ::: 4 3 2 1
seq 5 -.1 0 | parallel 'sleep {};echo Jobslot {%} slept {} seconds'
seq 5 -.1 0 | parallel -j0 'sleep {};echo Jobslot {%} slept {} seconds'

parallel "do something" ::: seq.*
-bash: /usr/local/bin/parallel: Argument list too long

#split the multifasta into individual seqs
cat $NAME/file.fna | parallel --recstart '>' -N1 --pipe "cat - > $NAME/seq.{#}"
#do stuff with the split files
export -f blastFunction
parallel blastFunction ::: $NAME/seq.*

blastFunction() {
        BLAST=$(blastn -query $1 -subject $1 -outfmt 6 -perc_identity 100)

ls *.txt > myFiles

parallel "do something" :::: myFiles

-bash: /bin/ls: Argument list too long

find $NAME/ -type f -maxdepth 1 -iname "seq.*" | parallel blastFunction

printf "%s\0" seq.* | parallel -0 do something

parallel --keep-order --max-procs 11 "freebayes --fasta-reference $REF \
    --genotype-qualities --experimental-gls \
    --region {} $BAM  " ::: $seqnames \
    | vcffirstheader \
    | vt normalize -r $REF - > $VCF

--region {} $BAM

parallel --keep-order --max-procs 0 "freebayes --fasta-reference hg38.fa " ::: S1.bam S2.bam > output_1.vcf

parallel --keep-order --max-procs 11 "freebayes --fasta-reference hg38.fa " ::: S1.bam S2.bam | vcffirstheader | vt normalize -r hg38.fa - > output_2.vcf

./predict_binding.py [argA] [argB] [argC] ./file.txt

string_1 
string_2 
string_3
...
string_n

parallel --verbose ./predict_binding ::: argA ::: argBi ::: argC ::: ./file.txt

./predict_binding.py argA argBi argC ./file.txt

argA argB1 argC ./file.txt
argA argB2 argC ./file.txt
...
argA argBm argC ./file.txt

cat args.txt | parallel --verbose ./predict_binding.py {}

cat ./args.txt | parallel --verbose echo | ./predict_binding

binding_func ( ) { ./predict_binding argA $1 argC ./file.txt}

parallel binding_func ::: argB1

parallel binding_func ::: argB1 argB2

cat args.txt | parallel --verbose binding_func {}

sort_index_bam(){
    outfile=`echo $1 | sed -e 's/.bam/_sorted/g'`
    samtools sort $1 $outfile
    index_file="$outfile.bam"
    samtools index $outfile
}
export -f sort_index_bam
parallel -j10 --progress --xapply sort_index_bam ::: `ls -1 *.bam`

[E::hts_open] fail to open file 'HiC_LY1_1_NoIndex_L003_034_sorted.bam'
local:10/33/100%/42.8s [bam_sort_core] merging from 3 files...

sort_index_bam(){
    samtools sort "$1" "$2"
    samtools index "$2"
}
export -f sort_index_bam
parallel --bar sort_index_bam {} '{=s/.bam/_sorted.bam/=}' ::: *.bam

seq 22 | xargs -i echo samtools cmd -r chr{} aln.bam | parallel -j 5
ls *.bed | sed s,.bed,, | xargs -i echo mv {}.bed {}.gff | sh
ls *.psmcfa | sed s,.psmcfa,, | xargs -i echo psmc {}.psmcfa \> {}.psmc \& | sh
ls *.psmcfa | sed s,.psmcfa,, | xargs -i echo bsub 'psmc -o {}.psmc {}.psmcfa' | sh

ls *.psmcfa | sed s,.psmcfa,, | xargs -i echo psmc -o {}.psmc {}.psmcfa | asub

seq 22 | parallel -j5 samtools cmd -r chr{} aln.bam
ls *.bed | parallel mv {} {.}.gff
ls *.psmcfa | parallel psmc {} \> {.}.psmc
ls *.psmcfa | parallel bsub psmc -o {.}.psmc {}
ls *.psmcfa | parallel echo psmc -o {.}.psmc {} | asub

ls *.psmcfa | parallel --dry-run psmc {} \> {.}.psmc

$ PARALLEL_HOSTS=foo,bar,baz
$ bedextract --list-chr input.bed \
    | parallel \
        --sshlogin $PARALLEL_HOSTS \
        "bedextract {} input.bed | starch - > input.{}.starch"
$ starchcat input.*.starch > input.starch
$ rm input.*.starch

$ bedops --chrom chrN --element-of -1 input.starch another.bed
...

server1

/usr/bin/rsh server1

parallel -S "/usr/bin/rsh server1" do_stuff

seqnames=$(grep ">" $REF | awk '{ print substr($1, 2, length($1)) }')

parallel --keep-order --max-procs 11 "samtools mpileup -Euf $REF -r {} $BAM \
   | bcftools view -v -" ::: $seqnames \
   | vcffirstheader \
   | vt normalize -r $REF - > $VCF

parallel --keep-order --max-procs 11 "freebayes --fasta-reference $REF \
    --genotype-qualities --experimental-gls \
    --region {} $BAM  " ::: $seqnames \
    | vcffirstheader \
    | vt normalize -r $REF - > $VCF

my_freebayes() {
  freebayes --fasta-reference $REF --genotype-qualities --experimental-gls --region "$1" $BAM
}
export -f my_freebayes

parallel --keep-order --max-procs 11 my_freebayes ::: $seqnames \
    | vcffirstheader \
    | vt normalize -r $REF - > $VCF

$: seq 1 100 | parallel ./launch_single_cosi_iteration.sh {} outputfolder

find . -type f -maxdepth 1 -iname "*ped" | parallel "plink --make-bed --noweb --file {.} --out {.}"

sampleID,chr1, ...
sampleID,chr1, ...
:
sampleID,chr1, ...
sampleID,chr2, ...
:
sampleID,chr2, ...
sampleID,chr3, ...

cat file | parallel --group-by 2 --colsep , -N1 --pipe process_chr

cat file | parallel --group-by 2 --colsep , --pipe process_chr

>Sequence name1                                                                                                     
sequence                                                                                                            
sequence continued                                                                                                  
>Sequence name2                                                                                                     
sequence                                                                                                            
sequence continued                                                                                                  
more sequence

cat file.fasta |                                                                                                    
  parallel --pipe -N1 --recstart '>' --rrs \                                                                        
    'read a; echo Name: "$a"; myprog $(tr -d "\n")'

myprg /some/dir/foo_1.fastq.gz /some/dir/foo_2.fastq.gz foo_1.out
myprg /some/dir/bar_1.fastq.gz /some/dir/bar_2.fastq.gz bar_1.out

parallel --plus myprg {} {/_1.fastq/_2.fastq} {/..}.out ::: /some/dir/*_1.fastq.gz