Wednesday, August 27, 2014

AWS AMI for Hadoop DataNode





##Create an AWS AMI as the snapshot for hadoop datanode
##This image should contain all necessary tools, packages and libraries
##to be used by the pipeline and your hadoop-application

##assume the we already started a Ubuntu14.04-64-PV instance
##Public IP for the instance is 12.34.56.78 and our private key file was saved as 
##/home/hadoop/.ssh/aws.pem

AWS_IP=12.34.56.78
KEYFILE=/home/hadoop/.ssh/aws.pem
USER=ubuntu
########################
#step1. log in our AWS instance
########################
ssh -i ~/.ssh/aws.pem ubuntu@${AWS_IP}

########################
#step2. install basic packages
########################
sudo apt-get install openjdk-7-jdk -y
sudo apt-get install make -y
sudo apt-get install cmake -y
sudo apt-get install gcc -y
sudo apt-get install g++ -y
sudo apt-get install zlib1g-dev -y
sudo apt-get install unzip -y
sudo apt-get install libncurses5-dev -y
sudo apt-get install r-base -y
sudo apt-get install python-dev -y
sudo apt-get install python-dateutil -y
sudo apt-get install python-psutil -y
sudo apt-get install python-pip -y
sudo apt-get install maven2 -y
sudo apt-get install libxml2-dev -y
sudo apt-get install gradle -y

#install R packages
sudo R
source('http://www.bioconductor.org/biocLite.R')
biocLite('edgeR')
biocLite('DESeq')
biocLite('limma')
#... all other necessary packages

########################
#step3. create folder structure
########################
TOOLS_HOME=~/tools
BIO_HOME=~/bioinformatics
APP=$BIO_HOME/app
DATA=$BIO_HOME/data

mkdir -p ${TOOLS_HOME}
mkdir -p ${APP}
mkdir -p ${DATA}

########################
#step4. install hadoop under ~/tools/
########################
cd $TOOLS_HOME 
wget http://apache.cs.utah.edu/hadoop/common/hadoop-2.5.0/hadoop-2.5.0.tar.gz
tar -zxvf hadoop-2.4.0.tar.gz &&  rm -f hadoop-2.5.0.tar.gz && cd

#install s3tools under ~/tools/
cd $TOOLS_HOME 
wget https://github.com/s3tools/s3cmd/archive/master.zip
unzip master.zip

########################
#step5. install bioinformatics applications
########################

#BWA
cd $APP
wget http://downloads.sourceforge.net/project/bio-bwa/bwa-0.7.9a.tar.bz2
tar -jxvf bwa-0.7.9a.tar.bz2 && rm -fr bwa-0.7.9a.tar.bz2
cd bwa-0.7.9a && make && cd $APP
mkdir -p $APP/bwa/0.7.9a
find bwa-0.7.9a -executable -type f -print0 | xargs -0 -I {} mv {} $APP/bwa/0.7.9a/
rm -fr bwa-0.7.9a

#bowtie1
cd ${APP}
wget http://downloads.sourceforge.net/project/bowtie-bio/bowtie/1.0.1/bowtie-1.0.1-linux-x86_64.zip
unzip bowtie-1.0.1-linux-x86_64.zip && rm bowtie-1.0.1-linux-x86_64.zip
mkdir -p ${APP}/bowtie/1.0.1 && mv bowtie-1.0.1/* ${APP}/bowtie/1.0.1/ && rm -fr bowtie-1.0.1/

#bowtie2
cd ${APP}
wget http://downloads.sourceforge.net/project/bowtie-bio/bowtie2/2.2.3/bowtie2-2.2.3-linux-x86_64.zip
unzip bowtie2-2.2.3-linux-x86_64.zip && rm bowtie2-2.2.3-linux-x86_64.zip
mkdir -p ${APP}/bowtie/2.2.3 && mv bowtie2-2.2.3/* ${APP}/bowtie/2.2.3/ && rm -fr bowtie2-2.2.3/

#SNAP
cd ${APP}
curl http://snap.cs.berkeley.edu/downloads/snap-1.0beta.10-linux.tar.gz | tar xvz
mkdir -p ${APP}/snap/1.0beta.10/
mv snap-1.0beta.10-linux/* ${APP}/snap/1.0beta.10/ && rm -fr snap-1.0beta.10-linux

#GSNAP
cd ${APP}
curl http://research-pub.gene.com/gmap/src/gmap-gsnap-2014-06-10.tar.gz | tar xvz
cd gmap-2014-06-10 && ./configure --prefix=${APP}/gmap/2014-06-10/ && make && make install
rm -fr gmap-2014-06-10

#STAR
cd ${APP}
curl https://rna-star.googlecode.com/files/STAR_2.3.0e.Linux_x86_64.tgz | tar xvz
mkdir -p ${APP}/star/2.3.0e/
mv STAR_2.3.0e.Linux_x86_64/* ${APP}/star/2.3.0e/ && rm -fr STAR_2.3.0e.Linux_x86_64 

#Tophat2
cd ${APP}
curl http://ccb.jhu.edu/software/tophat/downloads/tophat-2.0.11.Linux_x86_64.tar.gz | tar xvz 
mkdir -p ${APP}/tophat/2.0.11 && mv tophat-2.0.11.Linux_x86_64/* ${APP}/tophat/2.0.11/ && rm -fr tophat-2.0.11.Linux_x86_64

#cufflinks
cd ${APP}
curl http://cufflinks.cbcb.umd.edu/downloads/cufflinks-2.2.1.Linux_x86_64.tar.gz | tar xvz 
mkdir -p ${APP}/cufflinks/2.2.1 && mv cufflinks-2.2.1.Linux_x86_64/* ${APP}/cufflinks/2.2.1/ && rm -fr cufflinks-2.2.1.Linux_x86_64

#HTSeq
cd ${APP}
echo -e "y\n" | sudo apt-get install python-pip
sudo pip install numpy
sudo pip install scipy
curl https://pypi.python.org/packages/source/H/HTSeq/HTSeq-0.6.1p1.tar.gz | tar xvz 
cd HTSeq-0.6.1p1 &&  python setup.py build && sudo python setup.py install && cd -
sudo rm -fr HTSeq-0.6.1p1

#samtools
cd ${APP}
wget http://downloads.sourceforge.net/project/samtools/samtools/0.1.19/samtools-0.1.19.tar.bz2
tar -jxvf samtools-0.1.19.tar.bz2 && cd samtools-0.1.19/ && make && cd
mkdir -p $APP/samtools/0.1.19/
find  samtools-0.1.19 -executable -type f -print0 | xargs -0 -I {} mv {} $APP/samtools/0.1.19/
rm -fr samtools-0.1.19*

#picard
cd ${APP}
wget http://downloads.sourceforge.net/project/picard/picard-tools/1.114/picard-tools-1.114.zip
unzip picard-tools-1.114.zip
mkdir -p ${APP}/picard/1.114/ && mv picard-tools-1.114/* ${APP}/picard/1.114/
rm -fr picard-tools-1.114 picard-tools-1.114.zip


#bamtools
cd $APP
git clone https://github.com/pezmaster31/bamtools.git
cd $APP/bamtools && mkdir build && cd build && cmake .. && make
mkdir -p $APP/bamtools/2.3.0/ && mv $APP/bamtools/* $APP/bamtools/2.3.0/


########################
#step5. install bioinformatics annotation files
########################

#hg19
mkdir -p ${DATA}/fasta/hg19/ && cd ${DATA}/fasta/hg19/
for i in {1..22} X Y M; do wget http://hgdownload.cse.ucsc.edu/goldenPath/hg19/chromosomes/chr${i}.fa.gz; done
gunzip *.gz

#create index and dict for each chromosome files 
for i in *.fa
do
j=$(echo $i | cut -d"." -f1)
echo $j
java -jar ${APP}/picard/1.114/CreateSequenceDictionary.jar R=$j.fa O=$j.dict
${APP}/samtools/0.1.19/samtools faidx $j.fa
done

#build hg19 genome index for BWA
mkdir -p $DATA/index/hg19/bwa/
${APP}/bwa/0.7.9a/bwa index -p ${DATA}/index/hg19/bwa/hg19 ${DATA}/fasta/hg19/hg19.fa

#build hg19 genome index for novoalign
mkdir -p ${DATA}/index/novoalign/hg19
${APP}/novocraft/3.02.05/novoindex -k 14 -s 1 ${DATA}/index/hg19/novoalign/hg19.nix ${DATA}/fasta/hg19/hg19.fa

#build hg19 genome index for bowtie1
mkdir -p $DATA/index/hg19/bowtie1/
$APP/bowtie/1.0.1/bowtie-build $DATA/fasta/hg19/hg19.fa $DATA/index/hg19/bowtie1/hg19
cp $DATA/fasta/hg19/hg19.fa $DATA/index/hg19/bowtie1/

#build hg19 genome index for bowtie2
mkdir -p $DATA/index/hg19/bowtie2/
$APP/bowtie/2.2.3/bowtie2-build $DATA/fasta/hg19/hg19.fa $DATA/index/hg19/bowtie2/hg19
cp $DATA/fasta/hg19/hg19.fa $DATA/index/hg19/bowtie2/

#build hg19 genome index for SNAP -require at least 64 GB of memory
mkdir -p ${DATA}/index/snap/hg19
sudo sysctl vm.overcommit_memory=1
${APP}/snap/1.0beta.10/snap index ${DATA}/fasta/hg19/hg19.fa ${DATA}/index/snap/hg19
#{APP}/snap/1.0beta.10/snap paired ${DATA}/index/snap/hg19

#TODO: build hg19 genome index for GSNAP

#TODO: build hg19 genome index for STAR

#dbsnp
curl ftp://ftp.ncbi.nih.gov/snp/organisms/human_9606/VCF/All.vcf.gz | gunzip -c > dbsnp_hg19.vcf
mkdir -p ${DATA}/misc/ && cd ${DATA}/misc/
for chromosome in {1..22} X Y M; 
do
awk -v c="$chromosome" '/^#/{print $0;next} $1~"^chr"c {print $0}' dbsnp.vcf > dbsnp_hg19.chr${chromosome}.vcf
done

No comments:

Post a Comment