uploaded transcriptome assembly scripts

Author: bethsheets

scripts/Trinity population specific assemblies/Trinity_Astroides_GM.sbatch

@@ -0,0 +1,25 @@
+#!/bin/bash 
+#SBATCH -p hns,spalumbi
+#SBATCH --mail-type=FAIL
+#################
+#set a job name  
+#SBATCH --job-name=Astroides_GM
+#################  
+#a file for job output, you can check job progress
+#SBATCH --output=Astroides_GM
+#################
+# a file for errors from the job
+#SBATCH --error=Astroides_GM.err
+#################
+#time limit, default is 2 hours
+#SBATCH --time 48:0:0
+#number of CPUs
+#SBATCH -c 16
+#amount of RAM, most hns have 124 or 191
+#SBATCH --mem=191G
+
+ml biology trinity/2.8.4/
+
+Trinity --seqType fq --max_memory 191G --CPU 16 --min_contig_length 300 --trimmomatic --monitoring --left /scratch/groups/spalumbi/beth/Astroides/raw/fastq/14283X1_1.fq,/scratch/groups/spalumbi/beth/Astroides/raw/fastq/14283X2_1.fq,/scratch/groups/spalumbi/beth/Astroides/raw/fastq/14283X3_1.fq,/scratch/groups/spalumbi/beth/Astroides/raw/fastq/14283X4_1.fq --right /scratch/groups/spalumbi/beth/Astroides/raw/fastq/14283X1_2.fq,/scratch/groups/spalumbi/beth/Astroides/raw/fastq/14283X2_2.fq,/scratch/groups/spalumbi/beth/Astroides/raw/fastq/14283X3_2.fq,/scratch/groups/spalumbi/beth/Astroides/raw/fastq/14283X4_2.fq
+
+ 

scripts/Trinity population specific assemblies/Trinity_Astroides_PV.sbatch

@@ -0,0 +1,24 @@
+#!/bin/bash 
+#SBATCH -p hns,spalumbi
+#################
+#set a job name  
+#SBATCH --job-name=Astroides_PV
+#################  
+#a file for job output, you can check job progress
+#SBATCH --output=Astroides_PV
+#################
+# a file for errors from the job
+#SBATCH --error=Astroides_PV.err
+#################
+#time limit, default is 2 hours
+#SBATCH --time 48:0:0
+#number of CPUs
+#SBATCH -c 16
+#amount of RAM, most hns have 124 or 191
+#SBATCH --mem=191G
+
+
+ml biology trinity/2.8.4/
+
+Trinity --seqType fq --max_memory 191G --CPU 16 --min_contig_length 300 --trimmomatic --monitoring --left /scratch/groups/spalumbi/beth/Astroides/raw/fastq/14283X8_1.fq,/scratch/groups/spalumbi/beth/Astroides/raw/fastq/14283X9_1.fq,/scratch/groups/spalumbi/beth/Astroides/raw/fastq/14283X10_1.fq --right /scratch/groups/spalumbi/beth/Astroides/raw/fastq/14283X8_2.fq,/scratch/groups/spalumbi/beth/Astroides/raw/fastq/14283X9_2.fq,/scratch/groups/spalumbi/beth/Astroides/raw/fastq/14283X10_2.fq
+ 

scripts/Trinity population specific assemblies/Trinity_Astroides_SA.sbatch

@@ -0,0 +1,24 @@
+#!/bin/bash 
+#SBATCH -p hns,spalumbi
+#################
+#set a job name  
+#SBATCH --job-name=Astroides_SA
+#################  
+#a file for job output, you can check job progress
+#SBATCH --output=Astroides_SA
+#################
+# a file for errors from the job
+#SBATCH --error=Astroides_SA.err
+#################
+#time limit, default is 2 hours
+#SBATCH --time 48:0:0
+#number of CPUs
+#SBATCH -c 16
+#amount of RAM, most hns have 124 or 191
+#SBATCH --mem=191G
+
+
+ml biology trinity/2.8.4/
+
+Trinity --seqType fq --max_memory 191G --CPU 16 --min_contig_length 300 --trimmomatic --monitoring --left /scratch/groups/spalumbi/beth/Astroides/raw/fastq/14283X5_1.fq,/scratch/groups/spalumbi/beth/Astroides/raw/fastq/14283X6_1.fq,/scratch/groups/spalumbi/beth/Astroides/raw/fastq/14283X7_1.fq --right /scratch/groups/spalumbi/beth/Astroides/raw/fastq/14283X5_2.fq,/scratch/groups/spalumbi/beth/Astroides/raw/fastq/14283X6_2.fq,/scratch/groups/spalumbi/beth/Astroides/raw/fastq/14283X7_2.fq
+ 

scripts/Trinity population specific assemblies/Trinity_Astroides_SP.sbatch

@@ -0,0 +1,24 @@
+#!/bin/bash 
+#SBATCH -p spalumbi
+#################
+#set a job name  
+#SBATCH --job-name=Astroides_SP
+#################  
+#a file for job output, you can check job progress
+#SBATCH --output=Astroides_SP
+#################
+# a file for errors from the job
+#SBATCH --error=Astroides_SP.err
+#################
+#time limit, default is 2 hours
+#SBATCH --time 48:0:0
+#number of CPUs
+#SBATCH -c 16
+#amount of RAM, most hns have 124 or 191
+#SBATCH --mem=191G
+
+
+ml biology trinity/2.8.4/
+
+Trinity --seqType fq --max_memory 191G --CPU 16 --min_contig_length 300 --trimmomatic --monitoring --left /scratch/groups/spalumbi/beth/Astroides/raw/14833R/14833X10_1.fq,/scratch/groups/spalumbi/beth/Astroides/raw/14833R/14833X29_1.fq,/scratch/groups/spalumbi/beth/Astroides/raw/14833R/14833X38_1.fq --right /scratch/groups/spalumbi/beth/Astroides/raw/14833R/14833X10_2.fq,/scratch/groups/spalumbi/beth/Astroides/raw/14833R/14833X29_2.fq,/scratch/groups/spalumbi/beth/Astroides/raw/14833R/14833X38_2.fq
+

scripts/batch-blast-uniprot.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+#SBATCH -p owners
+#This script will split an assembly into batches of 100 contigs and sumbit blastx-uniprot.sh for teach temp file to the cluster 
+
+#usage: bash batch-blast-uniprot.sh assembly.fa
+
+awk 'BEGIN {n_seq=0;} /^>/{if(n_seq%100==0){file=sprintf("TEMP_%d.fa",n_seq);} print >> file;n_seq++; next;} { print >> file; }' < $1
+
+
+for i in TEMP*.fa; do sbatch blastx-uniprot.sh $i ; done
+
+

scripts/batch-hisat2-fq-paired-eas.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+#USAGE: bash batch-hisat2-fq-paired.sh index 4 *_1.txt.gz
+#if you don't have a hisat2 index, build it with "hisat2-build <reference>.fa basename"
+module load java
+CHUNK=$2
+COUNTER=0
+FQ="${@:3}"
+for i in $FQ; do
+    if [ $COUNTER -eq 0 ]; then
+    echo -e "#!/bin/bash\n#SBATCH -p owners\n#SBATCH --ntasks=1\n#SBATCH --cpus-per-task=3\n#SBATCH -t 12:00:00\n#SBATCH --mem 24000" > TEMPBATCH.sbatch; fi
+    BASE=$(basename $(basename $(basename $( basename $i .gz) .txt) .fq) .fastq)
+    echo "srun hisat2 --end-to-end --very-sensitive --no-spliced-alignment -p 3 -X 1500 --rg-id $BASE --rg SM:$BASE -x $1 -1 ${BASE}_1.txt.gz -2 ${BASE}_2.txt.gz > $BASE.sam" >> TEMPBATCH.sbatch
+    echo "samtools view -bSq 10 ${BASE}.sam > ${BASE}_BTVS-UNSORTED.bam " >> TEMPBATCH.sbatch
+    echo "srun samtools sort ${BASE}_BTVS-UNSORTED.bam > ${BASE}_UNDEDUP.bam" >> TEMPBATCH.sbatch
+    echo "srun java -Xmx4g -jar /share/PI/spalumbi/programs/picard.jar MarkDuplicates REMOVE_DUPLICATES=true INPUT=${BASE}_UNDEDUP.bam OUTPUT=${BASE}.bam METRICS_FILE=${BASE}-metrics.txt VALIDATION_STRINGENCY=LENIENT" >> TEMPBATCH.sbatch 
+    echo "srun samtools index ${BASE}.bam" >> TEMPBATCH.sbatch
+    echo "rm ${BASE}.sam" >> TEMPBATCH.sbatch
+    echo "rm ${BASE}_BTVS-UNSORTED.bam" >> TEMPBATCH.sbatch
+    echo "rm ${BASE}_UNDEDUP.bam" >> TEMPBATCH.sbatch
+    let COUNTER=COUNTER+1
+    if [ $COUNTER -eq $CHUNK ]; then
+    sbatch TEMPBATCH.sbatch
+    COUNTER=0; fi
+done
+if [ $COUNTER -ne 0 ]; then
+sbatch TEMPBATCH.sbatch; fi 

scripts/bcftools_parallel_eas.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+#before running: mkdir vcfout
+#usage: bash bcftools_parallel_eas.sh ref.fa vcfout 24 args *.bam
+#args are other arguments, for instance "-t DP"
+
+REF=$1
+NCPU=$3
+VCFOUT=$2
+BAMS="${@:4}"
+echo $REF
+echo $VCFOUT
+echo $BAMS
+
+samtools faidx $1
+awk '{print $1,"0",$2-1}' ${1}.fai > $VCFOUT/REGIONS.bed
+nregions=($(wc -l $VCFOUT/REGIONS.bed))
+nlines=$(($nregions /  $NCPU))
+echo "Splitting into batches of "$nlines
+split $VCFOUT/REGIONS.bed -l $nlines $VCFOUT/TEMP-REGIONS
+
+for i in $VCFOUT/TEMP-REGIONS* ; do
+    echo sending out batch $i
+#    echo -e "#!/bin/bash\n#SBATCH --time=24:00:00 -p spalumbi,owners" > ${i}.sbatch
+    echo -e "#!/bin/bash\n#SBATCH --time=24:00:00 -p owners" > ${i}.sbatch
+    echo "samtools mpileup -l $i -t AD -ugf $REF $BAMS | bcftools call -vmO v  > ${i}.vcf" >> ${i}.sbatch 
+    sbatch ${i}.sbatch
+done

scripts/blastx-uniprot.sh

@@ -0,0 +1,19 @@
+#!/bin/bash 
+#SBATCH -p hns,owners
+#SBATCH --time=48:00:00
+#SBATCH --mem=32000
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=6
+########################
+# -outfmt 5
+# to call: sbatch blastx-uniprot.sh input.fa
+
+#this echoes your TEMP file name to the slurm output in case any files are aborted on owners nodes or have errors
+echo $1
+
+#blast against uniprot_db
+#remember to update the uniprot database on sherlock regularly
+
+ml biology ncbi-blast+/2.7.1
+blastx -db /scratch/groups/spalumbi/BLAST_db/uniprot_trembl/uniprot_sprot_trembl_April2019 -query $1 -out $1.blast.out -evalue 0.001 -max_hsps 1 -num_threads 6 -outfmt 5
+

scripts/busco.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+#SBATCH -p owners,spalumbi
+#SBATCH -c 4
+#SBATCH --time 24:0:0
+
+#created by Beth March 2019
+#before using this script, you need add this path to the configfile in your home .bashrc file: 
+# export BUSCO_CONFIG_FILE="/home/groups/spalumbi/programs/busco-master/config/config.ini"
+
+#usage
+#busco.sh inputfile outputfile genome
+
+#genome= genome, transcriptome, proteins
+# I didn't install the Augustus package needed for genome assessment yet. You will need to do that to use this script on a genome.
+
+
+ml biology py-busco
+run_BUSCO.py -i $1 -o $2 -m $3 -l /home/groups/spalumbi/programs/busco-master/metazoa_odb9/
+

scripts/cap3.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+#SBATCH --time=8:00:00
+#SBATCH -p spalumbi,hns,owners
+#SBATCH --mem=181GB
+
+### OPTIONS FOR SHORT READS
+
+# -i specify segment pair score cutoff N > 20 (40)
+#The -i option is used to specify a score cutoff on segment pairs (ungapped alignments).
+#The score of a segment pair with 19 base matches and 1 base mismatch is 2 * 19 + (-5) * 1 = 33,
+# where each base match is given a score of 2 and each mismatch is given a score of -5.
+
+# -j specify chain score cutoff N > 30 (80)
+#The -j option is used to specify a score cutoff on chains of segment pairs,
+#where the score of a chain is the sum of scores of each segment pair
+#minus penalties for gaps between segment pairs.
+#The score of a chain consisting of one segment pair is simply the score of
+#the segment pair.
+
+# -o specify overlap length cutoff > 15 (40)
+# -s specify overlap similarity score cutoff N > 250 (900)
+#The -o option is used to specify a length cutoff on overlaps,
+#whereas the -s option is used to specify a score cutoff (based on matches, mismatches, and gaps) on overlaps.
+
+# -p specify overlap percent identity cutoff N > 65 (90)
+
+#to call: sbatch cap3.sh <yourassembly>.fasta
+
+cap3 $1 
+
+#recommended for short reads
+#-i 30 -j 31 -o 18 -s 300