diff --git a/INSTALL.sh b/INSTALL.sh
index 607851de..688d5414 100755
--- a/INSTALL.sh
+++ b/INSTALL.sh
@@ -11,14 +11,15 @@ mkdir -p $rootdir/bin
export PATH=$PATH:$rootdir/bin/
-assembly_tools=( idba spades )
-annotation_tools=( prokka RATT tRNAscan barrnap BLAST+ blastall phageFinder glimmer aragorn prodigal tbl2asn )
-utility_tools=( bedtools R GNU_parallel tabix JBrowse primer3 samtools sratoolkit )
-alignments_tools=( hmmer infernal bowtie2 bwa mummer )
+assembly_tools=( idba spades megahit )
+annotation_tools=( prokka RATT tRNAscan barrnap BLAST+ blastall phageFinder glimmer aragorn prodigal tbl2asn ShortBRED )
+utility_tools=( bedtools R GNU_parallel tabix JBrowse primer3 samtools sratoolkit ea-utils Rpackages)
+alignments_tools=( hmmer infernal bowtie2 bwa mummer RAPSearch2 )
taxonomy_tools=( kraken metaphlan kronatools gottcha )
phylogeny_tools=( FastTree RAxML )
perl_modules=( perl_parallel_forkmanager perl_excel_writer perl_archive_zip perl_string_approx perl_pdf_api2 perl_html_template perl_html_parser perl_JSON perl_bio_phylo perl_xml_twig perl_cgi_session )
-all_tools=("${assembly_tools[@]}" "${annotation_tools[@]}" "${utility_tools[@]}" "${alignments_tools[@]}" "${taxonomy_tools[@]}" "${phylogeny_tools[@]}" "${perl_modules[@]}")
+python_packages=( Anaconda2 Anaconda3 )
+all_tools=( "${python_packages[@]}" "${assembly_tools[@]}" "${annotation_tools[@]}" "${utility_tools[@]}" "${alignments_tools[@]}" "${taxonomy_tools[@]}" "${phylogeny_tools[@]}" "${perl_modules[@]}")
### Install functions ###
install_idba()
@@ -53,16 +54,36 @@ echo "
install_spades(){
+local VER=3.7.1
echo "------------------------------------------------------------------------------
- Installing SPAdes 3.5.0
+ Installing SPAdes $VER
------------------------------------------------------------------------------
"
-tar xvzf SPAdes-3.5.0-Linux.tar.gz
-ln -sf $rootdir/thirdParty/SPAdes-3.5.0-Linux/bin/spades.py $rootdir/bin/spades.py
+tar xvzf SPAdes-$VER-Linux.tar.gz
+ln -sf $rootdir/thirdParty/SPAdes-$VER-Linux/bin/spades.py $rootdir/bin/spades.py
cd $rootdir/thirdParty
echo "
------------------------------------------------------------------------------
- SPAdes installed
+ SPAdes $VER installed
+------------------------------------------------------------------------------
+"
+}
+
+install_megahit(){
+local VER=1.0.3
+## --version MEGAHIT v1.0.3
+echo "------------------------------------------------------------------------------
+ Installing megahit $VER
+------------------------------------------------------------------------------
+"
+tar xvzf megahit-v$VER.tar.gz
+cd megahit-$VER
+make
+cp -f megahit* $rootdir/bin/
+cd $rootdir/thirdParty
+echo "
+------------------------------------------------------------------------------
+ megahit $VER installed
------------------------------------------------------------------------------
"
}
@@ -145,18 +166,48 @@ echo "
install_sratoolkit()
{
+local VER=2.5.4
echo "------------------------------------------------------------------------------
- Installing sratoolkit.2.4.4-linux64
+ Installing sratoolkit.$VER-linux64
+------------------------------------------------------------------------------
+"
+tar xvzf sratoolkit.$VER-linux64.tgz
+cd sratoolkit.$VER-linux64
+ln -sf $rootdir/thirdParty/sratoolkit.$VER-linux64/bin/fastq-dump $rootdir/bin/fastq-dump
+ln -sf $rootdir/thirdParty/sratoolkit.$VER-linux64/bin/vdb-dump $rootdir/bin/vdb-dump
+./bin/vdb-config --restore-defaults
+./bin/vdb-config -s /repository/user/default-path=$rootdir/edge_ui/ncbi
+./bin/vdb-config -s /repository/user/main/public/root=$rootdir/edge_ui/ncbi/public
+if [[ -n ${HTTP_PROXY} ]]; then
+ proxy_without_protocol=${HTTP_PROXY#http://}
+ ./bin/vdb-config --proxy $proxy_without_protocol
+fi
+if [[ -n ${http_proxy} ]]; then
+ proxy_without_protocol=${http_proxy#http://}
+ ./bin/vdb-config --proxy $proxy_without_protocol
+fi
+
+cd $rootdir/thirdParty
+echo "
+------------------------------------------------------------------------------
+ sratoolkit.$VER-linux64 installed
------------------------------------------------------------------------------
"
-tar xvzf sratoolkit.2.4.4-linux64.tgz
-cd sratoolkit.2.4.4-linux64
-ln -sf $rootdir/thirdParty/sratoolkit.2.4.4-linux64/bin/fastq-dump $rootdir/bin/fastq-dump
-ln -sf $rootdir/thirdParty/sratoolkit.2.4.4-linux64/bin/vdb-dump $rootdir/bin/vdb-dump
+}
+
+install_ea-utils(){
+echo "------------------------------------------------------------------------------
+ Installing ea-utils.1.1.2-537
+------------------------------------------------------------------------------
+"
+tar xvzf ea-utils.1.1.2-537.tar.gz
+cd ea-utils.1.1.2-537
+PREFIX=$rootdir make install
+
cd $rootdir/thirdParty
echo "
------------------------------------------------------------------------------
- sratoolkit.2.4.3-linux64 installed
+ ea-utils.1.1.2-537 installed
------------------------------------------------------------------------------
"
}
@@ -164,12 +215,12 @@ echo "
install_R()
{
echo "------------------------------------------------------------------------------
- Compiling R 2.15.3
+ Compiling R 3.2.2
------------------------------------------------------------------------------
"
-tar xvzf R-2.15.3.tar.gz
-cd R-2.15.3
-./configure --prefix=$rootdir --with-readline=no
+tar xvzf R-3.2.2.tar.gz
+cd R-3.2.2
+./configure --prefix=$rootdir
make
make install
cd $rootdir/thirdParty
@@ -179,6 +230,23 @@ echo "
------------------------------------------------------------------------------
"
}
+install_Rpackages()
+{
+echo "------------------------------------------------------------------------------
+ installing R packages
+------------------------------------------------------------------------------
+"
+echo "if(\"gridExtra\" %in% rownames(installed.packages()) == FALSE) {install.packages(c(\"gtable_0.1.2.tar.gz\",\"gridExtra_2.0.0.tar.gz\"), repos = NULL, type=\"source\")}" | $rootdir/bin/Rscript -
+# need internet for following R packages.
+echo "if(\"devtools\" %in% rownames(installed.packages()) == FALSE) {install.packages('devtools',repos='https://cran.rstudio.com/')}" | $rootdir/bin/Rscript -
+echo "if(\"phyloseq\" %in% rownames(installed.packages()) == FALSE) {source('https://bioconductor.org/biocLite.R'); biocLite('phyloseq')} " | $rootdir/bin/Rscript -
+echo "library(devtools); options(unzip='internal'); install_github(repo = 'seninp-bioinfo/MetaComp', ref = 'v1.1');" | $rootdir/bin/Rscript -
+echo "
+------------------------------------------------------------------------------
+ R packages installed
+------------------------------------------------------------------------------
+"
+}
install_GNU_parallel()
{
@@ -274,15 +342,27 @@ echo "--------------------------------------------------------------------------
------------------------------------------------------------------------------
"
tar xvzf JBrowse-1.11.6.tar.gz
-cd JBrowse-1.11.6
-./setup.sh
-mkdir -p -m 775 data
-cd $rootdir/thirdParty
+if [ -e $rootdir/edge_ui/JBrowse/data ]
+then
+ mv $rootdir/edge_ui/JBrowse/data $rootdir/edge_ui/JBrowse_olddata
+fi
if [ -e $rootdir/edge_ui/JBrowse ]
then
- rm $rootdir/edge_ui/JBrowse
+ rm -rf $rootdir/edge_ui/JBrowse
+fi
+
+mv JBrowse-1.11.6 $rootdir/edge_ui/JBrowse
+cd $rootdir/edge_ui/JBrowse
+./setup.sh
+if [ -e $rootdir/edge_ui/JBrowse_olddata ]
+then
+ mv $rootdir/edge_ui/JBrowse_olddata $rootdir/edge_ui/JBrowse/data
+else
+ mkdir -p -m 775 data
fi
-ln -sf $rootdir/thirdParty/JBrowse-1.11.6 $rootdir/edge_ui/JBrowse
+
+cd $rootdir/thirdParty
+#ln -sf $rootdir/thirdParty/JBrowse-1.11.6 $rootdir/edge_ui/JBrowse
echo "
------------------------------------------------------------------------------
JBrowse-1.11.6 installed
@@ -364,15 +444,13 @@ echo "
install_bowtie2()
{
echo "------------------------------------------------------------------------------
- Compiling bowtie2 2.1.0
+ Compiling bowtie2 2.2.6
------------------------------------------------------------------------------
"
-tar xvzf bowtie2-2.1.0.tar.gz
-cd bowtie2-2.1.0
+tar xvzf bowtie2-2.2.6.tar.gz
+cd bowtie2-2.2.6
make
-cp bowtie2 $rootdir/bin/.
-cp bowtie2-build $rootdir/bin/.
-cp bowtie2-align $rootdir/bin/.
+cp bowtie2* $rootdir/bin/.
cd $rootdir/thirdParty
echo "
------------------------------------------------------------------------------
@@ -399,6 +477,7 @@ echo "
"
}
+
install_metaphlan()
{
echo "------------------------------------------------------------------------------
@@ -491,6 +570,21 @@ echo "
"
}
+install_ShortBRED()
+{
+echo "------------------------------------------------------------------------------
+ Installing ShortBRED
+------------------------------------------------------------------------------
+"
+tar xvzf ShortBRED-0.9.4M.tgz
+ln -sf $rootdir/thirdParty/ShortBRED-0.9.4M $rootdir/bin/ShortBRED
+echo "
+------------------------------------------------------------------------------
+ ShortBRED installed
+------------------------------------------------------------------------------
+"
+}
+
install_tbl2asn()
{
echo "------------------------------------------------------------------------------
@@ -538,6 +632,26 @@ echo "
"
}
+install_RAPSearch2()
+{
+local VER=2.23
+echo "------------------------------------------------------------------------------
+ Compiling RAPSearch2 $VER
+------------------------------------------------------------------------------
+"
+tar xvzf RAPSearch${VER}_64bits.tar.gz
+cd RAPSearch${VER}_64bits
+./install
+cp bin/rapsearch $rootdir/bin/rapsearch2
+cp bin/prerapsearch $rootdir/bin/.
+cd $rootdir/thirdParty
+echo "
+------------------------------------------------------------------------------
+ RAPSearch2 $VER compiled
+------------------------------------------------------------------------------
+"
+}
+
install_mummer()
{
echo "------------------------------------------------------------------------------
@@ -587,19 +701,19 @@ echo "
install_kronatools()
{
echo "------------------------------------------------------------------------------
- Installing KronaTools-2.4
+ Installing KronaTools-2.6
------------------------------------------------------------------------------
"
-tar xvzf KronaTools-2.4.tar.gz
-cd KronaTools-2.4
+tar xvzf KronaTools-2.6.tar.gz
+cd KronaTools-2.6/KronaTools
perl install.pl --prefix $rootdir --taxonomy $rootdir/database/Krona_taxonomy
#./updateTaxonomy.sh --local
cp $rootdir/scripts/microbial_profiling/script/ImportBWA.pl scripts/
-ln -sf $rootdir/thirdParty/KronaTools-2.4/scripts/ImportBWA.pl $rootdir/bin/ktImportBWA
+ln -sf $rootdir/thirdParty/KronaTools-2.6/KronaTools/scripts/ImportBWA.pl $rootdir/bin/ktImportBWA
cd $rootdir/thirdParty
echo "
------------------------------------------------------------------------------
- KronaTools-2.4 Installed
+ KronaTools-2.6 Installed
------------------------------------------------------------------------------
"
}
@@ -612,7 +726,7 @@ echo "--------------------------------------------------------------------------
"
tar xvzf samtools-0.1.19.tar.gz
cd samtools-0.1.19
-make
+make CFLAGS='-g -fPIC -Wall -O2'
cp samtools $rootdir/bin/.
cp bcftools/bcftools $rootdir/bin/.
cd $rootdir/thirdParty
@@ -855,6 +969,7 @@ echo "
"
}
+
install_perl_cgi_session()
{
echo "------------------------------------------------------------------------------
@@ -874,6 +989,38 @@ echo "
"
}
+install_Anaconda2()
+{
+echo "------------------------------------------------------------------------------
+ Installing Python Anaconda2 4.1.1
+------------------------------------------------------------------------------
+"
+if [ ! -f $rootdir/thirdParty/Anaconda2/bin/python ]; then
+ bash Anaconda2-4.1.1-Linux-x86_64.sh -b -p $rootdir/thirdParty/Anaconda2/
+fi
+anacondabin=$rootdir/thirdParty/Anaconda2/bin/
+ln -fs $anacondabin/python $rootdir/bin
+ln -fs $anacondabin/pip $rootdir/bin
+ln -fs $anacondabin/conda $rootdir/bin
+wget -q --spider https://pypi.python.org/
+online=$?
+if [[ $online -eq 0 ]]; then
+ $anacondabin/conda install -y biopython
+ $anacondabin/conda install -yc anaconda mysql-connector-python=2.0.3
+ $anacondabin/pip install qiime xlsx2csv
+ $anacondabin/conda install -y --channel https://conda.anaconda.org/bioconda rgi
+else
+ $anacondabin/conda install biopython-1.67-np110py27_0.tar.bz2
+ echo "Unable to connect to the internet, not able to install qiime or xlsx2csv"
+fi
+echo "
+------------------------------------------------------------------------------
+ Python Anaconda2 4.1.1 Installed
+------------------------------------------------------------------------------
+"
+}
+
+
checkSystemInstallation()
{
IFS=:
@@ -967,6 +1114,11 @@ print_tools_list()
do
echo "* $i"
done
+ echo -e "\nPython_Packages"
+ for i in "${python_packages[@]}"
+ do
+ echo "* $i"
+ done
}
@@ -974,23 +1126,13 @@ print_tools_list()
if ( checkSystemInstallation csh )
then
#echo "csh is found"
- echo ""
+ echo -n ""
else
echo "csh is not found"
echo "Please Install csh first, then INSTALL the package"
exit 1
fi
-if perl -MBio::Root::Version -e 'print $Bio::Root::Version::VERSION,"\n"' >/dev/null 2>&1
-then
- #perl -MBio::Root::Version -e 'print "BioPerl Version ", $Bio::Root::Version::VERSION," is found\n"'
- echo ""
-else
- echo "Cannot find a perl Bioperl Module installed" 1>&2
- echo "Please install Bioperl (http://www.bioperl.org/)"
- exit 1
-fi
-
if [ "$#" -ge 1 ]
then
for f in $@
@@ -1050,22 +1192,29 @@ then
install_$tool
done
echo -e "Perl_Modules installed.\n"
- exit 0;;
+ exit 0 ;;
+ Python_Packages)
+ for tool in "${python_packages[@]}"
+ do
+ install_$tool
+ done
+ echo -e "Python_Packages installed.\n"
+ exit 0 ;;
force)
- for tool in "${all_tools[@]}"
+ for tool in "${all_tools[@]}"
do
install_$tool
done
;;
*)
- if ( containsElement "$f" "${assembly_tools[@]}" || containsElement "$f" "${annotation_tools[@]}" || containsElement "$f" "${alignments_tools[@]}" || containsElement "$f" "${taxonomy_tools[@]}" || containsElement "$f" "${phylogeny_tools[@]}" || containsElement "$f" "${utility_tools[@]}" || containsElement "$f" "${perl_modules[@]}" )
+ if ( containsElement "$f" "${assembly_tools[@]}" || containsElement "$f" "${annotation_tools[@]}" || containsElement "$f" "${alignments_tools[@]}" || containsElement "$f" "${taxonomy_tools[@]}" || containsElement "$f" "${phylogeny_tools[@]}" || containsElement "$f" "${utility_tools[@]}" || containsElement "$f" "${perl_modules[@]}" || containsElement "$f" "${python_packages[@]}" )
then
install_$f
else
echo "$f: no this tool in the list"
print_tools_list
fi
- exit 0;;
+ exit;;
esac
done
fi
@@ -1079,6 +1228,22 @@ else
# exit 1
fi
+if perl -MBio::Root::Version -e 'print $Bio::Root::Version::VERSION,"\n"' >/dev/null 2>&1
+then
+ perl -MBio::Root::Version -e 'print "BioPerl Version ", $Bio::Root::Version::VERSION," is found\n"'
+else
+ echo "Cannot find a perl Bioperl Module installed" 1>&2
+ echo "Please install Bioperl (http://www.bioperl.org/)"
+ exit 1
+fi
+
+if $rootdir/bin/python -c 'import Bio; print Bio.__version__' >/dev/null 2>&1
+then
+ $rootdir/bin/python -c 'import Bio; print "BioPython Version", Bio.__version__, "is found"'
+else
+ install_Anaconda2
+fi
+
if [[ "$OSTYPE" == "darwin"* ]]
then
{
@@ -1100,7 +1265,17 @@ else
if ( checkLocalInstallation R )
then
{
- echo "R is found"
+ R_VER=`$rootdir/bin/R --version | perl -nle 'print $& if m{version \d+\.\d+}'`;
+ if ( echo $R_VER | awk '{if($2>="3.2") exit 0; else exit 1}' )
+ then
+ {
+ echo "R $R_VER found"
+ }
+ else
+ {
+ install_R
+ }
+ fi
}
else
{
@@ -1110,7 +1285,7 @@ else
}
fi
-echo "if(\"gridExtra\" %in% rownames(installed.packages()) == FALSE) {install.packages(\"gridExtra_0.9.1.tar.gz\", repos = NULL, type=\"source\")}" | Rscript -
+install_Rpackages
if ( checkSystemInstallation bedtools )
then
@@ -1122,10 +1297,23 @@ fi
if ( checkSystemInstallation fastq-dump )
then
- echo "sratoolkit is found"
+ sratoolkit_VER=`fastq-dump --version | perl -nle 'print $& if m{\d\.\d\.\d}'`;
+ if ( echo $sratoolkit_VER | awk '{if($1>="2.5.4") exit 0; else exit 1}' )
+ then
+ echo "sratoolkit $sratoolkit_VER found"
+ else
+ install_sratoolkit
+ fi
else
echo "sratoolkit is not found"
- install_sratoolkit
+ install_sratoolkit
+fi
+
+if ( checkSystemInstallation fastq-join )
+then
+ echo "fastq-join is found"
+else
+ install_ea-utils
fi
if ( checkSystemInstallation parallel )
@@ -1162,7 +1350,13 @@ fi
if ( checkLocalInstallation ktImportBLAST )
then
- echo "KronaTools is found"
+ Krona_VER=`$rootdir/bin/ktGetLibPath | perl -nle 'print $& if m{KronaTools-\d\.\d}' | perl -nle 'print $& if m{\d\.\d}'`;
+ if ( echo $Krona_VER | awk '{if($1>="2.6") exit 0; else exit 1}' )
+ then
+ echo "KronaTools $Krona_VER found"
+ else
+ install_kronatools
+ fi
else
echo "KronaTools is not found"
install_kronatools
@@ -1241,6 +1435,14 @@ else
install_tbl2asn
fi
+if ( checkLocalInstallation ShortBRED/shortbred_quantify.py )
+then
+ echo "ShortBRED is found"
+else
+ echo "ShortBRED is not found"
+ install_ShortBRED
+fi
+
if ( checkLocalInstallation kraken )
then
@@ -1268,12 +1470,33 @@ fi
if ( checkSystemInstallation bowtie2 )
then
- echo "bowtie2 is found"
+ bowtie_VER=`bowtie2 --version | grep bowtie | perl -nle 'print $& if m{version \d+\.\d+\.\d+}'`;
+ if ( echo $bowtie_VER | awk '{if($1>="2.2.4") exit 0; else exit 1}' )
+ then
+ echo "bowtie2 $bowtie_VER found"
+ else
+ install_bowtie2
+ fi
else
echo "bowtie2 is not found"
install_bowtie2
fi
+if ( checkSystemInstallation rapsearch2 )
+then
+ rapsearch_VER=`rapsearch2 2>&1| grep 'rapsearch v2' | perl -nle 'print $& if m{\d+\.\d+}'`;
+ if ( echo $rapsearch_VER | awk '{if($1>="2.23") exit 0; else exit 1}' )
+ then
+ echo "RAPSearch2 $rapsearch_VER found"
+ else
+ install_RAPSearch2
+ fi
+
+else
+ echo "RAPSearch2 is not found"
+ install_RAPSearch2
+fi
+
if ( checkLocalInstallation bwa )
then
echo "bwa is found"
@@ -1298,7 +1521,7 @@ else
install_mummer
fi
-if ( checkSystemInstallation wigToBigWig )
+if ( checkLocalInstallation wigToBigWig )
then
echo "wigToBigWig is found"
else
@@ -1323,14 +1546,35 @@ else
install_idba
fi
-if ( checkLocalInstallation spades.py )
+if ( checkSystemInstallation spades.py )
then
- echo "SPAdes is found"
+ spades_VER=`spades.py 2>&1 | perl -nle 'print $& if m{\d\.\d\.\d}'`;
+ if ( echo $spades_VER | awk '{if($1>="3.7.1") exit 0; else exit 1}' )
+ then
+ echo "SPAdes $spades_VER found"
+ else
+ install_spades
+ fi
else
echo "SPAdes is not found"
install_spades
fi
+if ( checkSystemInstallation megahit )
+then
+ ## --version MEGAHIT v1.0.3
+ megahit_VER=`megahit --version | perl -nle 'print $& if m{\d\.\d.\d}'`;
+ if ( echo $megahit_VER | awk '{if($1>="1.0.3") exit 0; else exit 1}' )
+ then
+ echo "megahit $megahit_VER found"
+ else
+ install_megahit
+ fi
+else
+ echo "megahit is not found"
+ install_megahit
+fi
+
if [ -x $rootdir/thirdParty/phage_finder_v2.1/bin/phage_finder_v2.1.sh ]
then
echo "phage_finder_v2.1 is found"
@@ -1367,10 +1611,10 @@ if ( checkSystemInstallation FastTreeMP )
then
FastTree_VER=`FastTreeMP 2>&1 | perl -nle 'print $& if m{version \d+\.\d+\.\d+}'`;
if ( echo $FastTree_VER | awk '{if($1>="2.1.8") exit 0; else exit 1}' )
- then
+ then
echo "FastTreeMP is found"
else
- install_FastTree
+ install_FastTree
fi
else
echo "FastTreeMP is not found"
@@ -1481,7 +1725,7 @@ else
install_perl_cgi_session
fi
-if [ -x $rootdir/thirdParty/JBrowse-1.11.6/bin/prepare-refseqs.pl ]
+if [ -x $rootdir/edge_ui/JBrowse/bin/prepare-refseqs.pl ]
then
echo "JBrowse is found"
else
@@ -1492,11 +1736,8 @@ fi
if [[ "$OSTYPE" == "darwin"* ]]
then
ln -sf $rootdir/thirdParty/gottcha/bin/splitrim $rootdir/scripts/microbial_profiling/script/splitrim
- cp -fR $rootdir/start_edge_ui.sh $HOME/Desktop/EDGE_Python_Server_startup
else
ln -sf $rootdir/thirdParty/gottcha/bin/splitrim $rootdir/scripts/microbial_profiling/script/splitrim
- mkdir -p $HOME/Desktop
- sed -e 's,,'"$rootdir"',g' $rootdir/scripts/EDGE.desktop > $HOME/Desktop/EDGE.desktop
fi
cd $rootdir
@@ -1505,29 +1746,57 @@ mkdir -p $rootdir/edge_ui/data
perl $rootdir/edge_ui/cgi-bin/edge_build_list.pl $rootdir/edge_ui/data/Host/* > $rootdir/edge_ui/data/host_list.json
perl $rootdir/edge_ui/cgi-bin/edge_build_list.pl -sort_by_size -basename $rootdir/database/NCBI_genomes/ > $rootdir/edge_ui/data/Ref_list.json
+echo "Setting up EDGE_input"
+if [ -d $rootdir/edge_ui/EDGE_input/ ]
+then
+ rsync -a $rootdir/deployment/public $rootdir/edge_ui/EDGE_input/
+ ln -sf $rootdir/testData $rootdir/edge_ui/EDGE_input/public/data/
+else
+ mkdir $HOME/EDGE_input/
+ rsync -a $rootdir/deployment/public $rootdir/edge_ui/EDGE_input/
+ ln -sf $rootdir/testData $rootdir/edge_ui/EDGE_input/public/data/
+fi
+if [ ! -d $rootdir/edge_ui/EDGE_output/ ]
+then
+ echo "Setting up EDGE_output/"
+ mkdir $HOME/EDGE_output
+fi
+
+
+# this may need sudo access
+#matplotlibrc=`python -c 'import matplotlib as m; print m.matplotlib_fname()' 2>&1`
+#if [ -n $matplotlibrc ]
+#then
+ # echo ""
+ #perl -i.orig -nle 's/(backend\s+:\s+\w+)/\#${1}\nbackend : Agg/; print;' $matplotlibrc
+#fi
+
if [ -f $HOME/.bashrc ]
then
{
echo "#Added by EDGE pipeline installation" >> $HOME/.bashrc
echo "export EDGE_HOME=$rootdir" >> $HOME/.bashrc
- echo "export PATH=$rootdir/bin/:$PATH:$rootdir/scripts" >> $HOME/.bashrc
+ echo "export EDGE_PATH=$rootdir/bin/:$rootdir/bin/Anaconda2/bin/:$rootdir/scripts" >> $HOME/.bashrc
+ echo "export PATH=\$EDGE_PATH:\$PATH:" >> $HOME/.bashrc
}
else
{
echo "#Added by EDGE pipeline installation" >> $HOME/.bash_profile
echo "export EDGE_HOME=$rootdir" >> $HOME/.bash_profile
- echo "export PATH=$rootdir/bin/:$PATH:$rootdir/scripts" >> $HOME/.bash_profile
+ echo "export EDGE_PATH=$rootdir/bin/:$rootdir/bin/Anaconda2/bin/:$rootdir/scripts" >> $HOME/.bashrc
+ echo "export PATH=\$EDGE_PATH:\$PATH:" >> $HOME/.bashrc
}
fi
-sed -i.bak 's,%EDGE_HOME%,'"$rootdir"',g' $rootdir/edge_ui/cgi-bin/edge_config.tmpl
+sed -i.bak 's,%EDGE_HOME%,'"$rootdir"',g' $rootdir/edge_ui/sys.properties
sed -i.bak 's,%EDGE_HOME%,'"$rootdir"',g' $rootdir/edge_ui/apache_conf/edge_apache.conf
+sed -i.bak 's,%EDGE_HOME%,'"$rootdir"',g' $rootdir/edge_ui/apache_conf/edge_httpd.conf
TOLCPU=`cat /proc/cpuinfo | grep processor | wc -l`;
if [ $TOLCPU -gt 0 ]
then
{
- sed -i.bak 's,%TOTAL_NUM_CPU%,'"$TOLCPU"',g' $rootdir/edge_ui/cgi-bin/edge_config.tmpl
+ sed -i.bak 's,%TOTAL_NUM_CPU%,'"$TOLCPU"',g' $rootdir/edge_ui/sys.properties
DEFAULT_CPU=`echo -n $((TOLCPU/3))`;
if [ $DEFAULT_CPU -lt 1 ]
then
@@ -1542,6 +1811,9 @@ then
}
fi
+# set up a cronjob for project old files clena up
+echo "01 00 * * * perl $rootdir/edge_ui/cgi-bin/edge_data_cleanup.pl" | crontab -
+
echo "
All done! Please Restart the Terminal Session.
diff --git a/README.Appendix.pdf b/README.Appendix.pdf
new file mode 100644
index 00000000..8c21b11d
Binary files /dev/null and b/README.Appendix.pdf differ
diff --git a/README.md b/README.md
index 7496744f..331160ff 100644
--- a/README.md
+++ b/README.md
@@ -1,23 +1,22 @@
# EDGE Bioinformatics
-This is version 1.1.1 of EDGE Bioinformatics, a product of Los Alamos National Laboratory, Naval Medical Research Center, and the Defense Threat Reduction Agency.
+This is version 1.5 of EDGE Bioinformatics, a product of Los Alamos National Laboratory, Naval Medical Research Center, and the Defense Threat Reduction Agency.
EDGE is a highly adaptable bioinformatics platform that allows laboratories to quickly analyze and interpret genomic sequence data. The bioinformatics platform allows users to address a wide range of use cases including assay validation and the characterization of novel biological threats, clinical samples, and complex environmental samples.
-A demonstration version of EDGE is currently available at https://bioedge.lanl.gov/edge_ui/ with example data sets available to the public to view and/or re-run and also allows EDGE runs of SRA/ENA data. (This webserver does not currently support upload of data due in part to LANL security regulations, however local installations are meant to be fully functional.) To run EDGE follow the "GUI" instructions from our documentation, found at https://edge.readthedocs.org/en/v1.1/introduction.html. It is also linked at the bottom of the EDGE homepage.
-
-For basic information about EDGE, visit the EDGE ABC's, that provide a brief overview of EDGE, the various workflows, and the computational environment restraints for local use.
+To demo EDGE, go to https://bioedge.lanl.gov/edge_ui/ and follow the "GUI" instructions from our documentation, found at https://edge.readthedocs.io. It is also linked at the bottom of the EDGE homepage.
## Documentation
-[ReadTheDocs](http://edge.readthedocs.org)
+[ReadTheDocs](http://edge.readthedocs.io)
-[PDF](https://readthedocs.org/projects/edge/downloads/pdf/latest/)
+[PDF](https://media.readthedocs.org/pdf/edge/v1.5/edge.pdf)
## Contact Info
Chien-Chi Lo:
Paul Li:
-Joe Anderson (DTRA/NMRC):
+Joe Anderson (DTRA/NMRC):
+Logan Voegtly (HJF/NMRC):
## Citation
diff --git a/config_template.txt b/config_template.txt
index d203e6db..88c90a59 100755
--- a/config_template.txt
+++ b/config_template.txt
@@ -52,11 +52,16 @@ nanoporeFile=
[Reads Mapping To Contigs]
# Reads mapping to contigs
DoReadsMappingContigs=auto
+## support bowtie or bwa
+edge-r2c-aligner=bowtie
+edge-r2c-aligner-options=
[Reads Mapping To Reference]
# Reads mapping to reference
-DoReadsMappingReference=0
-bowtieOptions=
+DoReadsMappingReference=
+## support bowtie or bwa
+edge-r2g-aligner=bowtie
+edge-r2g-aligner-options=
# reference genbank or fasta file
reference=
MapUnmappedReads=0
@@ -66,7 +71,7 @@ MapUnmappedReads=0
DoReadsTaxonomy=1
## If reference genome exists, only use unmapped reads to do Taxonomy Classification. Turn on AllReads=1 will use all reads instead.
AllReads=0
-enabledTools=gottcha-genDB-b,gottcha-speDB-b,gottcha-strDB-b,gottcha-genDB-v,gottcha-speDB-v,gottcha-strDB-v,metaphlan,bwa,kraken_mini
+enabledTools=gottcha-genDB-b,gottcha-speDB-b,gottcha-strDB-b,gottcha-genDB-v,gottcha-speDB-v,gottcha-strDB-v,metaphlan,bwa,kraken_mini,metascope
[Contigs Mapping To Reference]
# Contig mapping to reference
@@ -99,7 +104,7 @@ annotateSourceGBK=
[ProPhage Detection]
DoProPhageDetection=1
-[SNP Phylogeny]
+[Phylogenetic Analysis]
DoSNPtree=1
## Availabe choices are Ecoli, Yersinia, Francisella, Brucella, Bacillus
SNPdbName=Ecoli
@@ -107,7 +112,7 @@ SNPdbName=Ecoli
SNPGenomes=
SNPGenomesFiles=
## A refrence genoem from above two options for reads/contigs mapping
-SNPrefGenome=
+SNPrefGenome=
## FastTree or RAxML
treeMaker=FastTree
## SRA accessions ByrRun, ByExp, BySample, ByStudy
diff --git a/deployment/apache2.conf b/deployment/apache2.conf
new file mode 100644
index 00000000..93c3628d
--- /dev/null
+++ b/deployment/apache2.conf
@@ -0,0 +1,284 @@
+# This is the main Apache server configuration file. It contains the
+# configuration directives that give the server its instructions.
+# See http://httpd.apache.org/docs/2.4/ for detailed information about
+# the directives and /usr/share/doc/apache2/README.Debian about Debian specific
+# hints.
+#
+#
+# Summary of how the Apache 2 configuration works in Debian:
+# The Apache 2 web server configuration in Debian is quite different to
+# upstream's suggested way to configure the web server. This is because Debian's
+# default Apache2 installation attempts to make adding and removing modules,
+# virtual hosts, and extra configuration directives as flexible as possible, in
+# order to make automating the changes and administering the server as easy as
+# possible.
+
+# It is split into several files forming the configuration hierarchy outlined
+# below, all located in the /etc/apache2/ directory:
+#
+# /etc/apache2/
+# |-- apache2.conf
+# | `-- ports.conf
+# |-- mods-enabled
+# | |-- *.load
+# | `-- *.conf
+# |-- conf-enabled
+# | `-- *.conf
+# `-- sites-enabled
+# `-- *.conf
+#
+#
+# * apache2.conf is the main configuration file (this file). It puts the pieces
+# together by including all remaining configuration files when starting up the
+# web server.
+#
+# * ports.conf is always included from the main configuration file. It is
+# supposed to determine listening ports for incoming connections which can be
+# customized anytime.
+#
+# * Configuration files in the mods-enabled/, conf-enabled/ and sites-enabled/
+# directories contain particular configuration snippets which manage modules,
+# global configuration fragments, or virtual host configurations,
+# respectively.
+#
+# They are activated by symlinking available configuration files from their
+# respective *-available/ counterparts. These should be managed by using our
+# helpers a2enmod/a2dismod, a2ensite/a2dissite and a2enconf/a2disconf. See
+# their respective man pages for detailed information.
+#
+# * The binary is called apache2. Due to the use of environment variables, in
+# the default configuration, apache2 needs to be started/stopped with
+# /etc/init.d/apache2 or apache2ctl. Calling /usr/bin/apache2 directly will not
+# work with the default configuration.
+
+
+# Global configuration
+#
+
+#
+# ServerRoot: The top of the directory tree under which the server's
+# configuration, error, and log files are kept.
+#
+# NOTE! If you intend to place this on an NFS (or otherwise network)
+# mounted filesystem then please read the Mutex documentation (available
+# at );
+# you will save yourself a lot of trouble.
+#
+# Do NOT add a slash at the end of the directory path.
+#
+#ServerRoot "/etc/apache2"
+
+#
+# The accept serialization lock file MUST BE STORED ON A LOCAL DISK.
+#
+Mutex file:${APACHE_LOCK_DIR} default
+
+#
+# PidFile: The file in which the server should record its process
+# identification number when it starts.
+# This needs to be set in /etc/apache2/envvars
+#
+PidFile ${APACHE_PID_FILE}
+
+#
+# Timeout: The number of seconds before receives and sends time out.
+#
+Timeout 300
+
+#
+# KeepAlive: Whether or not to allow persistent connections (more than
+# one request per connection). Set to "Off" to deactivate.
+#
+KeepAlive On
+
+#
+# MaxKeepAliveRequests: The maximum number of requests to allow
+# during a persistent connection. Set to 0 to allow an unlimited amount.
+# We recommend you leave this number high, for maximum performance.
+#
+MaxKeepAliveRequests 100
+
+#
+# KeepAliveTimeout: Number of seconds to wait for the next request from the
+# same client on the same connection.
+#
+KeepAliveTimeout 5
+
+
+# These need to be set in /etc/apache2/envvars
+User ${APACHE_RUN_USER}
+Group ${APACHE_RUN_GROUP}
+
+#
+# HostnameLookups: Log the names of clients or just their IP addresses
+# e.g., www.apache.org (on) or 204.62.129.132 (off).
+# The default is off because it'd be overall better for the net if people
+# had to knowingly turn this feature on, since enabling it means that
+# each client request will result in AT LEAST one lookup request to the
+# nameserver.
+#
+HostnameLookups Off
+
+# ErrorLog: The location of the error log file.
+# If you do not specify an ErrorLog directive within a
+# container, error messages relating to that virtual host will be
+# logged here. If you *do* define an error logfile for a
+# container, that host's errors will be logged there and not here.
+#
+ErrorLog ${APACHE_LOG_DIR}/error.log
+
+#
+# LogLevel: Control the severity of messages logged to the error_log.
+# Available values: trace8, ..., trace1, debug, info, notice, warn,
+# error, crit, alert, emerg.
+# It is also possible to configure the log level for particular modules, e.g.
+# "LogLevel info ssl:warn"
+#
+LogLevel warn
+
+# Include module configuration:
+IncludeOptional mods-enabled/*.load
+IncludeOptional mods-enabled/*.conf
+
+# Include list of ports to listen on
+Include ports.conf
+
+
+# Sets the default security model of the Apache2 HTTPD server. It does
+# not allow access to the root filesystem outside of /usr/share and /var/www.
+# The former is used by web applications packaged in Debian,
+# the latter may be used for local directories served by the web server. If
+# your system is serving content from a sub-directory in /srv you must allow
+# access here, or in any related virtual host.
+
+ Options FollowSymLinks
+ AllowOverride None
+ Require all denied
+
+
+
+ AllowOverride None
+ Require all granted
+
+
+
+ Options Indexes FollowSymLinks
+ AllowOverride None
+ Require all granted
+
+
+#
+# Options Indexes FollowSymLinks
+# AllowOverride None
+# Require all granted
+#
+
+ Alias / "/home/edge/edge/edge_ui/"
+
+ Options Indexes FollowSymlinks
+ AllowOverride None
+ Require all granted
+
+
+
+ ScriptAlias /cgi-bin/ /home/edge/edge/edge_ui/cgi-bin/
+
+ AllowOverride None
+ Options +ExecCGI -MultiViews +SymLinksIfOwnerMatch
+ AddHandler cgi-script .cgi .pl
+ Order allow,deny
+ allow from all
+
+
+#for JBrowse
+AddType application/octet-stream .bam .bami
+
+
+ mod_gzip_item_exclude "(\.jsonz|\.txtz)"
+
+
+ SetEnvIf Request_URI "(\.jsonz|\.txtz)" no-gzip dont-vary
+
+
+
+ Header onsuccess set Content-Encoding gzip
+
+
+
+
+ AllowOverride FileInfo
+
+
+
+ ScriptAlias /edge_ui/cgi-bin/ /home/edge/edge/edge_ui/cgi-bin/
+
+ AllowOverride None
+ Options +ExecCGI -MultiViews +SymLinksIfOwnerMatch
+ AddHandler cgi-script .cgi .pl
+ Order allow,deny
+ allow from all
+
+
+
+
+ Alias /edge_ui/ "/home/edge/edge/edge_ui/"
+
+ Options Indexes FollowSymlinks
+ AllowOverride None
+ Require all granted
+
+
+
+
+ Options -Indexes
+
+
+
+ Order allow,deny
+ Allow from all
+
+
+
+# AccessFileName: The name of the file to look for in each directory
+# for additional configuration directives. See also the AllowOverride
+# directive.
+#
+AccessFileName .htaccess
+
+#
+# The following lines prevent .htaccess and .htpasswd files from being
+# viewed by Web clients.
+#
+
+ Require all denied
+
+
+
+#
+# The following directives define some format nicknames for use with
+# a CustomLog directive.
+#
+# These deviate from the Common Log Format definitions in that they use %O
+# (the actual bytes sent including headers) instead of %b (the size of the
+# requested file), because the latter makes it impossible to detect partial
+# requests.
+#
+# Note that the use of %{X-Forwarded-For}i instead of %h is not recommended.
+# Use mod_remoteip instead.
+#
+LogFormat "%v:%p %h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" vhost_combined
+LogFormat "%h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" combined
+LogFormat "%h %l %u %t \"%r\" %>s %O" common
+LogFormat "%{Referer}i -> %U" referer
+LogFormat "%{User-agent}i" agent
+
+# Include of directories ignores editors' and dpkg's backup files,
+# see README.Debian for details.
+
+# Include generic snippets of statements
+IncludeOptional conf-enabled/*.conf
+
+# Include the virtual host configurations:
+IncludeOptional sites-enabled/*.conf
+
+# vim: syntax=apache ts=4 sw=4 sts=4 sr noet
diff --git a/deployment/createDatabase.sql b/deployment/createDatabase.sql
new file mode 100644
index 00000000..405e0cf3
--- /dev/null
+++ b/deployment/createDatabase.sql
@@ -0,0 +1,5 @@
+CREATE DATABASE IF NOT EXIST userManagement;
+LOAD ~/edge/userManagement/userManagement_schema.sql;
+LOAD ~/edge/userManagement/userManagement_constrains.sql;
+CREATE USER 'edge'@'localhost' IDENTIFIED BY 'changePassword';
+GRANT ALL PRIVILEGES ON userManagement.* TO 'edge'@'localhost';
diff --git a/deployment/edge_httpd.conf b/deployment/edge_httpd.conf
new file mode 100644
index 00000000..bf16a308
--- /dev/null
+++ b/deployment/edge_httpd.conf
@@ -0,0 +1,53 @@
+
+ Alias / "/home/edge/edge/edge_ui/"
+
+ Options Indexes FollowSymlinks
+ AllowOverride None
+ Require all granted
+
+
+
+
+ ScriptAlias /cgi-bin/ /home/edge/edge/edge_ui/cgi-bin/
+
+ AllowOverride None
+ Options +ExecCGI -MultiViews +SymLinksIfOwnerMatch
+ AddHandler cgi-script .cgi .pl .php
+ Order allow,deny
+ allow from all
+
+
+
+
+ ForceType application/octet-stream
+ Header set Content-Disposition attachment
+
+
+
+ Options -Indexes
+
+
+# for user management tomcat server to be visible on port 80
+ProxyPass /userManagement http://localhost:8080/userManagement
+ProxyPassReverse /userManagement http://localhost:8080/userManagement
+ProxyPass /userManagementWS http://localhost:8080/userManagementWS
+ProxyPassReverse /userManagementWS http://localhost:8080/userManagementWS
+
+#for JBrowse
+AddType application/octet-stream .bam .bami
+
+
+ mod_gzip_item_exclude "(\.jsonz|\.txtz)"
+
+
+ SetEnvIf Request_URI "(\.jsonz|\.txtz)" no-gzip dont-vary
+
+
+
+ Header onsuccess set Content-Encoding gzip
+
+
+
+
+ AllowOverride FileInfo
+
diff --git a/deployment/httpd.conf b/deployment/httpd.conf
new file mode 100644
index 00000000..5709660b
--- /dev/null
+++ b/deployment/httpd.conf
@@ -0,0 +1,356 @@
+#
+# This is the main Apache HTTP server configuration file. It contains the
+# configuration directives that give the server its instructions.
+# See for detailed information.
+# In particular, see
+#
+# for a discussion of each configuration directive.
+#
+# Do NOT simply read the instructions in here without understanding
+# what they do. They're here only as hints or reminders. If you are unsure
+# consult the online docs. You have been warned.
+#
+# Configuration and logfile names: If the filenames you specify for many
+# of the server's control files begin with "/" (or "drive:/" for Win32), the
+# server will use that explicit path. If the filenames do *not* begin
+# with "/", the value of ServerRoot is prepended -- so 'log/access_log'
+# with ServerRoot set to '/www' will be interpreted by the
+# server as '/www/log/access_log', where as '/log/access_log' will be
+# interpreted as '/log/access_log'.
+
+#
+# ServerRoot: The top of the directory tree under which the server's
+# configuration, error, and log files are kept.
+#
+# Do not add a slash at the end of the directory path. If you point
+# ServerRoot at a non-local disk, be sure to specify a local disk on the
+# Mutex directive, if file-based mutexes are used. If you wish to share the
+# same ServerRoot for multiple httpd daemons, you will need to change at
+# least PidFile.
+#
+ServerRoot "/etc/httpd"
+
+#
+# Listen: Allows you to bind Apache to specific IP addresses and/or
+# ports, instead of the default. See also the
+# directive.
+#
+# Change this to Listen on specific IP addresses as shown below to
+# prevent Apache from glomming onto all bound IP addresses.
+#
+#Listen 12.34.56.78:80
+Listen 80
+
+#
+# Dynamic Shared Object (DSO) Support
+#
+# To be able to use the functionality of a module which was built as a DSO you
+# have to place corresponding `LoadModule' lines at this location so the
+# directives contained in it are actually available _before_ they are used.
+# Statically compiled modules (those listed by `httpd -l') do not need
+# to be loaded here.
+#
+# Example:
+# LoadModule foo_module modules/mod_foo.so
+#
+Include conf.modules.d/*.conf
+
+#
+# If you wish httpd to run as a different user or group, you must run
+# httpd as root initially and it will switch.
+#
+# User/Group: The name (or #number) of the user/group to run httpd as.
+# It is usually good practice to create a dedicated user and group for
+# running httpd, as with most system services.
+#
+User edge
+Group edge
+
+# 'Main' server configuration
+#
+# The directives in this section set up the values used by the 'main'
+# server, which responds to any requests that aren't handled by a
+# definition. These values also provide defaults for
+# any containers you may define later in the file.
+#
+# All of these directives may appear inside containers,
+# in which case these default settings will be overridden for the
+# virtual host being defined.
+#
+
+# Added to close http TRACE XSS attack vulnerability
+TraceEnable Off
+
+#
+# ServerAdmin: Your address, where problems with the server should be
+# e-mailed. This address appears on some server-generated pages, such
+# as error documents. e.g. admin@your-domain.com
+#
+ServerAdmin root@localhost
+
+#
+# ServerName gives the name and port that the server uses to identify itself.
+# This can often be determined automatically, but we recommend you specify
+# it explicitly to prevent problems during startup.
+#
+# If your host doesn't have a registered DNS name, enter its IP address here.
+#
+ServerName localhost
+
+#
+# Deny access to the entirety of your server's filesystem. You must
+# explicitly permit access to web content directories in other
+# blocks below.
+#
+
+ AllowOverride none
+ Require all denied
+
+
+#
+# Note that from this point forward you must specifically allow
+# particular features to be enabled - so if something's not working as
+# you might expect, make sure that you have specifically enabled it
+# below.
+#
+
+#
+# DocumentRoot: The directory out of which you will serve your
+# documents. By default, all requests are taken from this directory, but
+# symbolic links and aliases may be used to point to other locations.
+#
+DocumentRoot "/var/www/html"
+
+#
+# Relax access to content within /var/www.
+#
+
+ AllowOverride None
+ # Allow open access:
+ Require all granted
+
+
+# Further relax access to the default document root:
+
+ #
+ # Possible values for the Options directive are "None", "All",
+ # or any combination of:
+ # Indexes Includes FollowSymLinks SymLinksifOwnerMatch ExecCGI MultiViews
+ #
+ # Note that "MultiViews" must be named *explicitly* --- "Options All"
+ # doesn't give it to you.
+ #
+ # The Options directive is both complicated and important. Please see
+ # http://httpd.apache.org/docs/2.4/mod/core.html#options
+ # for more information.
+ #
+ Options Indexes FollowSymLinks
+
+ #
+ # AllowOverride controls what directives may be placed in .htaccess files.
+ # It can be "All", "None", or any combination of the keywords:
+ # Options FileInfo AuthConfig Limit
+ #
+ AllowOverride None
+
+ #
+ # Controls who can get stuff from this server.
+ #
+ Require all granted
+
+
+#
+# DirectoryIndex: sets the file that Apache will serve if a directory
+# is requested.
+#
+
+ DirectoryIndex index.html
+
+
+#
+# The following lines prevent .htaccess and .htpasswd files from being
+# viewed by Web clients.
+#
+
+ Require all denied
+
+
+#
+# ErrorLog: The location of the error log file.
+# If you do not specify an ErrorLog directive within a
+# container, error messages relating to that virtual host will be
+# logged here. If you *do* define an error logfile for a
+# container, that host's errors will be logged there and not here.
+#
+ErrorLog "logs/error_log"
+
+#
+# LogLevel: Control the number of messages logged to the error_log.
+# Possible values include: debug, info, notice, warn, error, crit,
+# alert, emerg.
+#
+LogLevel warn
+
+
+ #
+ # The following directives define some format nicknames for use with
+ # a CustomLog directive (see below).
+ #
+ LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
+ LogFormat "%h %l %u %t \"%r\" %>s %b" common
+
+
+ # You need to enable mod_logio.c to use %I and %O
+ LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio
+
+
+ #
+ # The location and format of the access logfile (Common Logfile Format).
+ # If you do not define any access logfiles within a
+ # container, they will be logged here. Contrariwise, if you *do*
+ # define per- access logfiles, transactions will be
+ # logged therein and *not* in this file.
+ #
+ #CustomLog "logs/access_log" common
+
+ #
+ # If you prefer a logfile with access, agent, and referer information
+ # (Combined Logfile Format) you can use the following directive.
+ #
+ CustomLog "logs/access_log" combined
+
+
+
+ #
+ # Redirect: Allows you to tell clients about documents that used to
+ # exist in your server's namespace, but do not anymore. The client
+ # will make a new request for the document at its new location.
+ # Example:
+ # Redirect permanent /foo http://www.example.com/bar
+
+ #
+ # Alias: Maps web paths into filesystem paths and is used to
+ # access content that does not live under the DocumentRoot.
+ # Example:
+ # Alias /webpath /full/filesystem/path
+ #
+ # If you include a trailing / on /webpath then the server will
+ # require it to be present in the URL. You will also likely
+ # need to provide a section to allow access to
+ # the filesystem path.
+
+ #
+ # ScriptAlias: This controls which directories contain server scripts.
+ # ScriptAliases are essentially the same as Aliases, except that
+ # documents in the target directory are treated as applications and
+ # run by the server when requested rather than as documents sent to the
+ # client. The same rules about trailing "/" apply to ScriptAlias
+ # directives as to Alias.
+ #
+ #ScriptAlias /cgi-bin/ "/var/www/cgi-bin/"
+
+
+
+#
+# "/var/www/cgi-bin" should be changed to whatever your ScriptAliased
+# CGI directory exists, if you have that configured.
+#
+
+ AllowOverride None
+ Options None
+ Require all granted
+
+
+
+ #
+ # TypesConfig points to the file containing the list of mappings from
+ # filename extension to MIME-type.
+ #
+ TypesConfig /etc/mime.types
+
+ #
+ # AddType allows you to add to or override the MIME configuration
+ # file specified in TypesConfig for specific file types.
+ #
+ #AddType application/x-gzip .tgz
+ #
+ # AddEncoding allows you to have certain browsers uncompress
+ # information on the fly. Note: Not all browsers support this.
+ #
+ #AddEncoding x-compress .Z
+ #AddEncoding x-gzip .gz .tgz
+ #
+ # If the AddEncoding directives above are commented-out, then you
+ # probably should define those extensions to indicate media types:
+ #
+ AddType application/x-compress .Z
+ AddType application/x-gzip .gz .tgz
+
+ #
+ # AddHandler allows you to map certain file extensions to "handlers":
+ # actions unrelated to filetype. These can be either built into the server
+ # or added with the Action directive (see below)
+ #
+ # To use CGI scripts outside of ScriptAliased directories:
+ # (You will also need to add "ExecCGI" to the "Options" directive.)
+ #
+ AddHandler cgi-script .cgi
+
+ # For type maps (negotiated resources):
+ #AddHandler type-map var
+
+ #
+ # Filters allow you to process content before it is sent to the client.
+ #
+ # To parse .shtml files for server-side includes (SSI):
+ # (You will also need to add "Includes" to the "Options" directive.)
+ #
+ AddType text/html .shtml
+ AddOutputFilter INCLUDES .shtml
+
+
+#
+# Specify a default charset for all content served; this enables
+# interpretation of all content as UTF-8 by default. To use the
+# default browser choice (ISO-8859-1), or to allow the META tags
+# in HTML content to override this choice, comment out this
+# directive:
+#
+AddDefaultCharset UTF-8
+
+
+ #
+ # The mod_mime_magic module allows the server to use various hints from the
+ # contents of the file itself to determine its type. The MIMEMagicFile
+ # directive tells the module where the hint definitions are located.
+ #
+ MIMEMagicFile conf/magic
+
+
+#
+# Customizable error responses come in three flavors:
+# 1) plain text 2) local redirects 3) external redirects
+#
+# Some examples:
+#ErrorDocument 500 "The server made a boo boo."
+#ErrorDocument 404 /missing.html
+#ErrorDocument 404 "/cgi-bin/missing_handler.pl"
+#ErrorDocument 402 http://www.example.com/subscription_info.html
+#
+
+#
+# EnableMMAP and EnableSendfile: On systems that support it,
+# memory-mapping or the sendfile syscall may be used to deliver
+# files. This usually improves server performance, but must
+# be turned off when serving from networked-mounted
+# filesystems or if support for these functions is otherwise
+# broken on your system.
+# Defaults if commented: EnableMMAP On, EnableSendfile Off
+#
+#EnableMMAP off
+EnableSendfile on
+
+# Supplemental configuration
+#
+# Load config files in the "/etc/httpd/conf.d" directory, if any.
+IncludeOptional conf.d/*.conf
diff --git a/deployment/nmrc-deploy.sh b/deployment/nmrc-deploy.sh
new file mode 100644
index 00000000..46e372da
--- /dev/null
+++ b/deployment/nmrc-deploy.sh
@@ -0,0 +1,151 @@
+#!/bin/bash
+####################################
+## This is the EDGE Bioinformatics Deployment Configurator Script.
+## It assumes a bare install of CentOS 7 with Development Tools and GNOME desk-
+## top, rebooted through first boot. This is the deployment script used for DoD
+## installations.
+##
+## A network connection with port 80 open is required for installation. You
+## must be able to route to centos.org, cpan.org, and fedoraproject.org as a
+## minimum. On a low-speed connection, this script may take several hours.
+##
+## WARNING: If you run this script on a server that's already been configured to
+## do something else, it will probably break. If you're running this script
+## and you haven't met Joe from NMRC and Patrick from LANL, it's probably not
+## time to run this script yet.
+##
+## This is the work of an employee of the United States Government, performed as
+## part of their official duties. No copyright is claimed, this work is entered
+## into the public domain without restriction.
+##
+####################################
+
+## Update changePassword to the appropriate password for the system
+## Update changePassword in createDatabase.sql to the appropriate password
+
+## Install pre-reqs
+sudo yum install -y epel-release
+sudo yum install -y libX11-devel readline-devel libXt-devel ncurses-devel inkscape scipy expat expat-devel freetype freetype-devel zlib zlib-devel perl-App-cpanminus perl-Test-Most python-pip blas-devel atlas-devel lapack-devel numpy numpy-f2py libpng12 libpng12-devel perl-XML-Simple perl-JSON csh gcc gcc-c++ make binutils gd gsl-devel git graphviz java-1.7.0-openjdk perl-Archive-Zip perl-CGI perl-CGI-Session perl-CPAN-Meta-YAML perl-DBI perl-Data-Dumper perl-GD perl-IO-Compress perl-Module-Build perl-XML-LibXML perl-XML-Parser perl-XML-SAX perl-XML-SAX-Writer perl-XML-Twig perl-XML-Writer perl-YAML perl-PerlIO-gzip python-matplotlib python-six libstdc++-static
+
+
+
+## Update existing python and perl tools
+sudo pip install --upgrade six scipy matplotlib
+sudo cpanm App::cpanoutdated
+sudo su -
+cpan-outdated -p | cpanm
+exit
+
+## Install more Perl modules
+## Some of these may fail, that's okay. BioPerl needs to go smoothly.
+cpanm Graph Time::Piece BioPerl
+cpanm Algorithm::Munkres Archive::Tar Array::Compare Clone Convert::Binary::C
+cpanm HTML::Template HTML::TableExtract List::MoreUtils PostScript::TextBlock
+cpanm SOAP::Lite SVG SVG::Graph Set::Scalar Sort::Naturally Spreadsheet::ParseExcel
+cpanm CGI CGI::Simple GD Graph GraphViz XML::Parser::PerlSAX XML::SAX XML::SAX::Writer XML::Simple XML::Twig XML::Writer
+
+## Install Apache for the Web GUI
+sudo yum install -y httpd httpd-tools
+
+## Configure firewall for ssh, http, https, and smtp:
+sudo firewall-cmd --permanent --add-service=ssh
+sudo firewall-cmd --permanent --add-service=http
+sudo firewall-cmd --permanent --add-service=https
+sudo firewall-cmd --permanent --add-service=smtp
+
+## Try to clone from the EDGE repo if SSH works
+cd ~
+if [ -d /home/edge/edge ]; then
+ echo "EDGE directory exists at ~/edge, skipping."
+elif [ $1 == "SSH" ]; then
+ echo "SSH deployment from BitBucket, requires registered SSH key."
+ git clone git@bitbucket.org:nmrcjoe/edge.git
+elif [ $1 == "HTTPS" ]; then
+ echo "HTTPS deployment from BitBucket, expect timeouts."
+ git clone https://nmrcjoe@bitbucket.org/nmrcjoe/edge.git
+elif [ $1 == "NMRC" ]; then
+ echo "Will use rsync to copy from bigsilver."
+ rsync -avzhr --progress joe@192.168.20.3:~/edge ~/
+else
+ echo "Program directory not present, and no valid source specified. Exiting."
+ exit 0
+fi
+
+## Do the same for the database files (if not done already)
+if ![ -d /home/edge/database ]; then
+ echo "Retrieving database files from bigsilver."
+ rsync -avzhr --progress joe@192.168.20.3:~/database ~/
+fi
+
+
+## Disable SELINUX
+# sudo sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/sysconfig/selinux
+# sudo sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
+sudo setenforce 0
+
+## Set default NMRC EDGE params
+sed -i 's/opt\/apps/home\/edge/g' /home/edge/edge/edge_ui/cgi-bin/edge_config.tmpl
+sed -i 's/user_management=1/user_management=0/g' /home/edge/edge/edge_ui/cgi-bin/edge_config.tmpl
+
+
+## Fix the database directories (Assumes dbs are in ~/database)
+## Make sure the database is linked prior to running INSTALL.sh
+rm -rf ~/edge/database
+ln -s ~/database ~/edge/database
+sudo ln -s ~/database/ /database
+## This should be done by the INSTALL.sh
+#ln -s ~/database/Krona_taxonomy ~/edge/thirdParty/KronaTools-2.4/taxonomy
+
+## Install LANL EDGE
+~/edge/INSTALL.sh
+
+## Copy the EDGE httpd conf files to the appropriate directories
+## This should be done after INSTALL.sh, the script inserts the appropriate paths
+sudo cp ~/edge/edge_ui/apache_conf/edge_httpd.conf /etc/httpd/conf.d/
+sudo cp ~/edge/deployment/httpd.conf /etc/httpd/conf/
+
+## Setup userManagement
+## Install database
+sudo yum install mariadb-server mariadb
+sudo systemctl start mariadb.service && sudo systemctl enable mariadb.service
+## Setup root password on database
+sudo mysql_secure_installation
+
+## Create userManagement database and Load schema/constrains
+## Update changePassword within createDatabase.sql to the appropriate password
+echo createDatabase.sql | mysql -u root -p
+
+## Install php
+sudo yum install php php-pear
+sudo yum install php-mysql
+sudo httpd -k restart
+
+## Install and Configure tomcat
+sudo yum install tomcat
+sudo cp ~/userManagement/mariadb-java-client-1.2.0.jar /usr/share/tomcat/lib
+sudo sed -i 's@@\n\n@g' /usr/share/tomcat/conf/tomcat-users.xml
+sudo sed -i 's@.*@4320@g' /usr/share/tomcat/conf/web.xml
+sudo sed -i 's@#JAVA_OPTS@JAVA_OPTS="-Xms256m -Xmx1024m -XX:PermSize=256m -XX:MaxPermSize=512m"\n#JAVA_OPTS@g' /usr/share/tomcat/conf/tomcat.conf
+
+## Deploy userManagement to tomcat server
+sudo cp ~/userManagement/userManagement*.war /usr/share/tomcat/webapps/
+
+## Edit ~/edge/userManagement/userManagementWS.xml then deploy it to /usr/share/tomcat/conf/Catalina/localhost
+sed -i 's@username=.*$@username="edge"@' ~/edge/userManagement/userManagementWS.xml
+sed -i 's@password=.*$@password="changePasword"@' ~/edge/userManagement/userManagementWS.xml
+sed -i 's@driverClassName=.*$@driverClassName="org.mariadb.jdbc.Driver"@' ~/edge/userManagement/userManagementWS.xml
+sudo cp ~/userManagement/userManagementWS.xml /usr/share/tomcat/conf/Catalina/localhost/
+
+## Edit /usr/share/tomcat/webapps/userManagement/WEB-INF/classes/sys.properties to match the appropriate settings for the server
+sudo sed -i 's@host_url=.*$@host_url=http://localhost:8080/userManagement@g' /usr/share/tomcat/webapps/userManagement/WEB-INF/classes/sys.properties
+sudo sed -i 's@wsURL=.*$@wsURL=http://localhost:8080/userManagementWS@g' /usr/share/tomcat/webapps/userManagement/WEB-INF/classes/sys.properties
+sudo sed -i 's@email_notification=.*$@email_notification=off@g' /usr/share/tomcat/webapps/userManagement/WEB-INF/classes/sys.properties
+
+## Create admin account for EDGE
+## Should this script fail, something is not set up correctly
+perl ~/edge/userManagement/createAdminAccount.pl -e admin@edge.com -p changePassword -fn admin -ln edge
+
+## Enable userManagement in ~/edge/edge_ui/sys.properties
+sed -i 's@user_management=.*$@user_management=1@g' ~/edge/edge_ui/sys.properties
+sed -i 's@edge_user_management_url=.*$@edge_user_management_url=http://localhost:8080/userManagement@g' ~/edge/edge_ui/sys.properties
+
diff --git a/edge_ui/EDGE_input/public/data/Reference/Ebolavirus_complete_genomes_20seqs/Ebolavirus_complete_genomes_20seqs.gbk b/deployment/public/data/Reference/Ebolavirus_complete_genomes_20seqs/Ebolavirus_complete_genomes_20seqs.gbk
similarity index 100%
rename from edge_ui/EDGE_input/public/data/Reference/Ebolavirus_complete_genomes_20seqs/Ebolavirus_complete_genomes_20seqs.gbk
rename to deployment/public/data/Reference/Ebolavirus_complete_genomes_20seqs/Ebolavirus_complete_genomes_20seqs.gbk
diff --git a/edge_ui/EDGE_input/public/data/Reference/MERS-CoV/MERS_Complete_genomes_24seqs.gbk b/deployment/public/data/Reference/MERS-CoV/MERS_Complete_genomes_24seqs.gbk
similarity index 100%
rename from edge_ui/EDGE_input/public/data/Reference/MERS-CoV/MERS_Complete_genomes_24seqs.gbk
rename to deployment/public/data/Reference/MERS-CoV/MERS_Complete_genomes_24seqs.gbk
diff --git a/edge_ui/EDGE_input b/edge_ui/EDGE_input
new file mode 120000
index 00000000..7ad9cc32
--- /dev/null
+++ b/edge_ui/EDGE_input
@@ -0,0 +1 @@
+/home/edge/EDGE_input
\ No newline at end of file
diff --git a/edge_ui/EDGE_input/public/data/testData b/edge_ui/EDGE_input/public/data/testData
deleted file mode 120000
index 19563774..00000000
--- a/edge_ui/EDGE_input/public/data/testData
+++ /dev/null
@@ -1 +0,0 @@
-../../../../testData/
\ No newline at end of file
diff --git a/edge_ui/EDGE_input/public/projects/.gitignore b/edge_ui/EDGE_input/public/projects/.gitignore
deleted file mode 100644
index e69de29b..00000000
diff --git a/edge_ui/EDGE_output b/edge_ui/EDGE_output
new file mode 120000
index 00000000..d893ce50
--- /dev/null
+++ b/edge_ui/EDGE_output
@@ -0,0 +1 @@
+/home/edge/EDGE_output
\ No newline at end of file
diff --git a/edge_ui/apache_conf/README b/edge_ui/apache_conf/README
index f7353f5d..9ae5a447 100644
--- a/edge_ui/apache_conf/README
+++ b/edge_ui/apache_conf/README
@@ -12,7 +12,15 @@
> sudo a2enmod cgid
> sudo a2enmod proxy proxy_http
-3. Modify sample apache configuration file: $EDGE_HOME/edge_ui/apache_conf/edge_apache.conf alias directories to match install
+3. Modify sample apache/httpd configuration file:
+
+ For Ubuntu
+ > vim $EDGE_HOME/edge_ui/apache_conf/edge_apache.conf
+ Make sure lines 2,3,13,14,26,51 have the appropriate path
+
+ For CentOS
+ > vim $EDGE_HOME/edge_ui/apache_conf/edge_httpd.conf
+ Make sure lines 2,3,11,12,26,51 have the appropriate path
4. Copy edge_apache.conf to the apache or Insert content into httpd.conf
@@ -20,14 +28,26 @@
> cp $EDGE_HOME/edge_ui/apache_conf/edge_apache.conf /etc/apache2/conf-available/
> ln -s /etc/apache2/conf-available/edge_apache.conf /etc/apache2/conf-enabled/
For CentOS
- /etc/httpd/conf/httpd.conf
+ > sudo cp $EDGE_HOME/edge_ui/apache_conf/edge_httpd.conf /etc/httpd/conf.d/
-4. Modify permissions: mod permissions on installed directory to match apache user (ex: change from www-data to edge)
+4. Modify configuration file:
+
+ For Ubuntu
+ mod permissions on installed directory to match apache user (ex: change from www-data to edge)
+
+ For CentOS
+ > sudo vim /etc/httpd/conf/httpd.conf
+ Modify lines as follows:
+ 66 modify line: User edge
+ 67 modify line: Group edge
+ 95 modify line: ServerName localhost
+ 247 comment out line: ScriptAlias /cgi-bin/ "/var/www/cgi-bin/"
+ 294 uncomment line: AddHandler cgi-script .cgi
5. Restart the apache2 to active the new configuration
For Ubuntu
> service apache2 restart
For CentOS
- > service http restart
+ > sudo httpd -k restart
diff --git a/edge_ui/apache_conf/centos_httpd.conf b/edge_ui/apache_conf/centos_httpd.conf
new file mode 100644
index 00000000..5709660b
--- /dev/null
+++ b/edge_ui/apache_conf/centos_httpd.conf
@@ -0,0 +1,356 @@
+#
+# This is the main Apache HTTP server configuration file. It contains the
+# configuration directives that give the server its instructions.
+# See for detailed information.
+# In particular, see
+#
+# for a discussion of each configuration directive.
+#
+# Do NOT simply read the instructions in here without understanding
+# what they do. They're here only as hints or reminders. If you are unsure
+# consult the online docs. You have been warned.
+#
+# Configuration and logfile names: If the filenames you specify for many
+# of the server's control files begin with "/" (or "drive:/" for Win32), the
+# server will use that explicit path. If the filenames do *not* begin
+# with "/", the value of ServerRoot is prepended -- so 'log/access_log'
+# with ServerRoot set to '/www' will be interpreted by the
+# server as '/www/log/access_log', where as '/log/access_log' will be
+# interpreted as '/log/access_log'.
+
+#
+# ServerRoot: The top of the directory tree under which the server's
+# configuration, error, and log files are kept.
+#
+# Do not add a slash at the end of the directory path. If you point
+# ServerRoot at a non-local disk, be sure to specify a local disk on the
+# Mutex directive, if file-based mutexes are used. If you wish to share the
+# same ServerRoot for multiple httpd daemons, you will need to change at
+# least PidFile.
+#
+ServerRoot "/etc/httpd"
+
+#
+# Listen: Allows you to bind Apache to specific IP addresses and/or
+# ports, instead of the default. See also the
+# directive.
+#
+# Change this to Listen on specific IP addresses as shown below to
+# prevent Apache from glomming onto all bound IP addresses.
+#
+#Listen 12.34.56.78:80
+Listen 80
+
+#
+# Dynamic Shared Object (DSO) Support
+#
+# To be able to use the functionality of a module which was built as a DSO you
+# have to place corresponding `LoadModule' lines at this location so the
+# directives contained in it are actually available _before_ they are used.
+# Statically compiled modules (those listed by `httpd -l') do not need
+# to be loaded here.
+#
+# Example:
+# LoadModule foo_module modules/mod_foo.so
+#
+Include conf.modules.d/*.conf
+
+#
+# If you wish httpd to run as a different user or group, you must run
+# httpd as root initially and it will switch.
+#
+# User/Group: The name (or #number) of the user/group to run httpd as.
+# It is usually good practice to create a dedicated user and group for
+# running httpd, as with most system services.
+#
+User edge
+Group edge
+
+# 'Main' server configuration
+#
+# The directives in this section set up the values used by the 'main'
+# server, which responds to any requests that aren't handled by a
+# definition. These values also provide defaults for
+# any containers you may define later in the file.
+#
+# All of these directives may appear inside containers,
+# in which case these default settings will be overridden for the
+# virtual host being defined.
+#
+
+# Added to close http TRACE XSS attack vulnerability
+TraceEnable Off
+
+#
+# ServerAdmin: Your address, where problems with the server should be
+# e-mailed. This address appears on some server-generated pages, such
+# as error documents. e.g. admin@your-domain.com
+#
+ServerAdmin root@localhost
+
+#
+# ServerName gives the name and port that the server uses to identify itself.
+# This can often be determined automatically, but we recommend you specify
+# it explicitly to prevent problems during startup.
+#
+# If your host doesn't have a registered DNS name, enter its IP address here.
+#
+ServerName localhost
+
+#
+# Deny access to the entirety of your server's filesystem. You must
+# explicitly permit access to web content directories in other
+# blocks below.
+#
+
+ AllowOverride none
+ Require all denied
+
+
+#
+# Note that from this point forward you must specifically allow
+# particular features to be enabled - so if something's not working as
+# you might expect, make sure that you have specifically enabled it
+# below.
+#
+
+#
+# DocumentRoot: The directory out of which you will serve your
+# documents. By default, all requests are taken from this directory, but
+# symbolic links and aliases may be used to point to other locations.
+#
+DocumentRoot "/var/www/html"
+
+#
+# Relax access to content within /var/www.
+#
+
+ AllowOverride None
+ # Allow open access:
+ Require all granted
+
+
+# Further relax access to the default document root:
+
+ #
+ # Possible values for the Options directive are "None", "All",
+ # or any combination of:
+ # Indexes Includes FollowSymLinks SymLinksifOwnerMatch ExecCGI MultiViews
+ #
+ # Note that "MultiViews" must be named *explicitly* --- "Options All"
+ # doesn't give it to you.
+ #
+ # The Options directive is both complicated and important. Please see
+ # http://httpd.apache.org/docs/2.4/mod/core.html#options
+ # for more information.
+ #
+ Options Indexes FollowSymLinks
+
+ #
+ # AllowOverride controls what directives may be placed in .htaccess files.
+ # It can be "All", "None", or any combination of the keywords:
+ # Options FileInfo AuthConfig Limit
+ #
+ AllowOverride None
+
+ #
+ # Controls who can get stuff from this server.
+ #
+ Require all granted
+
+
+#
+# DirectoryIndex: sets the file that Apache will serve if a directory
+# is requested.
+#
+
+ DirectoryIndex index.html
+
+
+#
+# The following lines prevent .htaccess and .htpasswd files from being
+# viewed by Web clients.
+#
+
+ Require all denied
+
+
+#
+# ErrorLog: The location of the error log file.
+# If you do not specify an ErrorLog directive within a
+# container, error messages relating to that virtual host will be
+# logged here. If you *do* define an error logfile for a
+# container, that host's errors will be logged there and not here.
+#
+ErrorLog "logs/error_log"
+
+#
+# LogLevel: Control the number of messages logged to the error_log.
+# Possible values include: debug, info, notice, warn, error, crit,
+# alert, emerg.
+#
+LogLevel warn
+
+
+ #
+ # The following directives define some format nicknames for use with
+ # a CustomLog directive (see below).
+ #
+ LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
+ LogFormat "%h %l %u %t \"%r\" %>s %b" common
+
+
+ # You need to enable mod_logio.c to use %I and %O
+ LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio
+
+
+ #
+ # The location and format of the access logfile (Common Logfile Format).
+ # If you do not define any access logfiles within a
+ # container, they will be logged here. Contrariwise, if you *do*
+ # define per- access logfiles, transactions will be
+ # logged therein and *not* in this file.
+ #
+ #CustomLog "logs/access_log" common
+
+ #
+ # If you prefer a logfile with access, agent, and referer information
+ # (Combined Logfile Format) you can use the following directive.
+ #
+ CustomLog "logs/access_log" combined
+
+
+
+ #
+ # Redirect: Allows you to tell clients about documents that used to
+ # exist in your server's namespace, but do not anymore. The client
+ # will make a new request for the document at its new location.
+ # Example:
+ # Redirect permanent /foo http://www.example.com/bar
+
+ #
+ # Alias: Maps web paths into filesystem paths and is used to
+ # access content that does not live under the DocumentRoot.
+ # Example:
+ # Alias /webpath /full/filesystem/path
+ #
+ # If you include a trailing / on /webpath then the server will
+ # require it to be present in the URL. You will also likely
+ # need to provide a section to allow access to
+ # the filesystem path.
+
+ #
+ # ScriptAlias: This controls which directories contain server scripts.
+ # ScriptAliases are essentially the same as Aliases, except that
+ # documents in the target directory are treated as applications and
+ # run by the server when requested rather than as documents sent to the
+ # client. The same rules about trailing "/" apply to ScriptAlias
+ # directives as to Alias.
+ #
+ #ScriptAlias /cgi-bin/ "/var/www/cgi-bin/"
+
+
+
+#
+# "/var/www/cgi-bin" should be changed to whatever your ScriptAliased
+# CGI directory exists, if you have that configured.
+#
+
+ AllowOverride None
+ Options None
+ Require all granted
+
+
+
+ #
+ # TypesConfig points to the file containing the list of mappings from
+ # filename extension to MIME-type.
+ #
+ TypesConfig /etc/mime.types
+
+ #
+ # AddType allows you to add to or override the MIME configuration
+ # file specified in TypesConfig for specific file types.
+ #
+ #AddType application/x-gzip .tgz
+ #
+ # AddEncoding allows you to have certain browsers uncompress
+ # information on the fly. Note: Not all browsers support this.
+ #
+ #AddEncoding x-compress .Z
+ #AddEncoding x-gzip .gz .tgz
+ #
+ # If the AddEncoding directives above are commented-out, then you
+ # probably should define those extensions to indicate media types:
+ #
+ AddType application/x-compress .Z
+ AddType application/x-gzip .gz .tgz
+
+ #
+ # AddHandler allows you to map certain file extensions to "handlers":
+ # actions unrelated to filetype. These can be either built into the server
+ # or added with the Action directive (see below)
+ #
+ # To use CGI scripts outside of ScriptAliased directories:
+ # (You will also need to add "ExecCGI" to the "Options" directive.)
+ #
+ AddHandler cgi-script .cgi
+
+ # For type maps (negotiated resources):
+ #AddHandler type-map var
+
+ #
+ # Filters allow you to process content before it is sent to the client.
+ #
+ # To parse .shtml files for server-side includes (SSI):
+ # (You will also need to add "Includes" to the "Options" directive.)
+ #
+ AddType text/html .shtml
+ AddOutputFilter INCLUDES .shtml
+
+
+#
+# Specify a default charset for all content served; this enables
+# interpretation of all content as UTF-8 by default. To use the
+# default browser choice (ISO-8859-1), or to allow the META tags
+# in HTML content to override this choice, comment out this
+# directive:
+#
+AddDefaultCharset UTF-8
+
+
+ #
+ # The mod_mime_magic module allows the server to use various hints from the
+ # contents of the file itself to determine its type. The MIMEMagicFile
+ # directive tells the module where the hint definitions are located.
+ #
+ MIMEMagicFile conf/magic
+
+
+#
+# Customizable error responses come in three flavors:
+# 1) plain text 2) local redirects 3) external redirects
+#
+# Some examples:
+#ErrorDocument 500 "The server made a boo boo."
+#ErrorDocument 404 /missing.html
+#ErrorDocument 404 "/cgi-bin/missing_handler.pl"
+#ErrorDocument 402 http://www.example.com/subscription_info.html
+#
+
+#
+# EnableMMAP and EnableSendfile: On systems that support it,
+# memory-mapping or the sendfile syscall may be used to deliver
+# files. This usually improves server performance, but must
+# be turned off when serving from networked-mounted
+# filesystems or if support for these functions is otherwise
+# broken on your system.
+# Defaults if commented: EnableMMAP On, EnableSendfile Off
+#
+#EnableMMAP off
+EnableSendfile on
+
+# Supplemental configuration
+#
+# Load config files in the "/etc/httpd/conf.d" directory, if any.
+IncludeOptional conf.d/*.conf
diff --git a/edge_ui/apache_conf/edge_apache.conf b/edge_ui/apache_conf/edge_apache.conf
index 160d11be..f1c78085 100644
--- a/edge_ui/apache_conf/edge_apache.conf
+++ b/edge_ui/apache_conf/edge_apache.conf
@@ -27,6 +27,11 @@
Options -Indexes
+
+Order allow,deny
+Deny from all
+
+
# for user management tomcat server to be visible on port 80
ProxyPass /userManagement http://localhost:8080/userManagement
ProxyPassReverse /userManagement http://localhost:8080/userManagement
diff --git a/edge_ui/apache_conf/edge_httpd.conf b/edge_ui/apache_conf/edge_httpd.conf
new file mode 100644
index 00000000..a0dee9b3
--- /dev/null
+++ b/edge_ui/apache_conf/edge_httpd.conf
@@ -0,0 +1,58 @@
+
+ ScriptAlias /cgi-bin/ %EDGE_HOME%/edge_ui/cgi-bin/
+
+ AllowOverride None
+ Options +ExecCGI -MultiViews +SymLinksIfOwnerMatch
+ AddHandler cgi-script .cgi .pl .php
+ Order allow,deny
+ allow from all
+
+
+
+
+ Alias / "%EDGE_HOME%/edge_ui/"
+
+ Options Indexes FollowSymlinks
+ AllowOverride None
+ Require all granted
+
+
+
+
+ ForceType application/octet-stream
+ Header set Content-Disposition attachment
+
+
+
+ Options -Indexes
+
+
+
+Order allow,deny
+Deny from all
+
+
+# for user management tomcat server to be visible on port 80
+ProxyPass /userManagement http://localhost:8080/userManagement
+ProxyPassReverse /userManagement http://localhost:8080/userManagement
+ProxyPass /userManagementWS http://localhost:8080/userManagementWS
+ProxyPassReverse /userManagementWS http://localhost:8080/userManagementWS
+
+#for JBrowse
+AddType application/octet-stream .bam .bami
+
+
+ mod_gzip_item_exclude "(\.jsonz|\.txtz)"
+
+
+ SetEnvIf Request_URI "(\.jsonz|\.txtz)" no-gzip dont-vary
+
+
+
+ Header onsuccess set Content-Encoding gzip
+
+
+
+
+ AllowOverride FileInfo
+
diff --git a/edge_ui/apache_conf/httpd.conf b/edge_ui/apache_conf/httpd.conf
new file mode 100644
index 00000000..15dcdf14
--- /dev/null
+++ b/edge_ui/apache_conf/httpd.conf
@@ -0,0 +1,1074 @@
+#
+# This is the main Apache server configuration file. It contains the
+# configuration directives that give the server its instructions.
+# See for detailed information.
+# In particular, see
+#
+# for a discussion of each configuration directive.
+#
+#
+# Do NOT simply read the instructions in here without understanding
+# what they do. They're here only as hints or reminders. If you are unsure
+# consult the online docs. You have been warned.
+#
+# The configuration directives are grouped into three basic sections:
+# 1. Directives that control the operation of the Apache server process as a
+# whole (the 'global environment').
+# 2. Directives that define the parameters of the 'main' or 'default' server,
+# which responds to requests that aren't handled by a virtual host.
+# These directives also provide default values for the settings
+# of all virtual hosts.
+# 3. Settings for virtual hosts, which allow Web requests to be sent to
+# different IP addresses or hostnames and have them handled by the
+# same Apache server process.
+#
+# Configuration and logfile names: If the filenames you specify for many
+# of the server's control files begin with "/" (or "drive:/" for Win32), the
+# server will use that explicit path. If the filenames do *not* begin
+# with "/", the value of ServerRoot is prepended -- so "logs/foo.log"
+# with ServerRoot set to "/etc/httpd" will be interpreted by the
+# server as "/etc/httpd/logs/foo.log".
+#
+
+### Section 1: Global Environment
+#
+# The directives in this section affect the overall operation of Apache,
+# such as the number of concurrent requests it can handle or where it
+# can find its configuration files.
+#
+
+#
+# Don't give away too much information about all the subcomponents
+# we are running. Comment out this line if you don't mind remote sites
+# finding out what major optional modules you are running
+ServerTokens OS
+
+#
+# ServerRoot: The top of the directory tree under which the server's
+# configuration, error, and log files are kept.
+#
+# NOTE! If you intend to place this on an NFS (or otherwise network)
+# mounted filesystem then please read the LockFile documentation
+# (available at );
+# you will save yourself a lot of trouble.
+#
+# Do NOT add a slash at the end of the directory path.
+#
+ServerRoot "/etc/httpd"
+
+#
+# PidFile: The file in which the server should record its process
+# identification number when it starts. Note the PIDFILE variable in
+# /etc/sysconfig/httpd must be set appropriately if this location is
+# changed.
+#
+PidFile run/httpd.pid
+
+#
+# Timeout: The number of seconds before receives and sends time out.
+#
+Timeout 60
+
+#
+# KeepAlive: Whether or not to allow persistent connections (more than
+# one request per connection). Set to "Off" to deactivate.
+#
+KeepAlive Off
+
+#
+# MaxKeepAliveRequests: The maximum number of requests to allow
+# during a persistent connection. Set to 0 to allow an unlimited amount.
+# We recommend you leave this number high, for maximum performance.
+#
+MaxKeepAliveRequests 100
+
+#
+# KeepAliveTimeout: Number of seconds to wait for the next request from the
+# same client on the same connection.
+#
+KeepAliveTimeout 15
+
+##
+## Server-Pool Size Regulation (MPM specific)
+##
+
+# prefork MPM
+# StartServers: number of server processes to start
+# MinSpareServers: minimum number of server processes which are kept spare
+# MaxSpareServers: maximum number of server processes which are kept spare
+# ServerLimit: maximum value for MaxClients for the lifetime of the server
+# MaxClients: maximum number of server processes allowed to start
+# MaxRequestsPerChild: maximum number of requests a server process serves
+
+StartServers 8
+MinSpareServers 5
+MaxSpareServers 20
+ServerLimit 256
+MaxClients 256
+MaxRequestsPerChild 4000
+
+
+# worker MPM
+# StartServers: initial number of server processes to start
+# MaxClients: maximum number of simultaneous client connections
+# MinSpareThreads: minimum number of worker threads which are kept spare
+# MaxSpareThreads: maximum number of worker threads which are kept spare
+# ThreadsPerChild: constant number of worker threads in each server process
+# MaxRequestsPerChild: maximum number of requests a server process serves
+
+StartServers 4
+MaxClients 300
+MinSpareThreads 25
+MaxSpareThreads 75
+ThreadsPerChild 25
+MaxRequestsPerChild 0
+
+
+#
+# Listen: Allows you to bind Apache to specific IP addresses and/or
+# ports, in addition to the default. See also the
+# directive.
+#
+# Change this to Listen on specific IP addresses as shown below to
+# prevent Apache from glomming onto all bound IP addresses (0.0.0.0)
+#
+#Listen 12.34.56.78:80
+Listen 0.0.0.0:80
+
+#
+# Dynamic Shared Object (DSO) Support
+#
+# To be able to use the functionality of a module which was built as a DSO you
+# have to place corresponding `LoadModule' lines at this location so the
+# directives contained in it are actually available _before_ they are used.
+# Statically compiled modules (those listed by `httpd -l') do not need
+# to be loaded here.
+#
+# Example:
+# LoadModule foo_module modules/mod_foo.so
+#
+LoadModule auth_basic_module modules/mod_auth_basic.so
+LoadModule auth_digest_module modules/mod_auth_digest.so
+LoadModule authn_file_module modules/mod_authn_file.so
+LoadModule authn_alias_module modules/mod_authn_alias.so
+LoadModule authn_anon_module modules/mod_authn_anon.so
+LoadModule authn_dbm_module modules/mod_authn_dbm.so
+LoadModule authn_default_module modules/mod_authn_default.so
+LoadModule authz_host_module modules/mod_authz_host.so
+LoadModule authz_user_module modules/mod_authz_user.so
+LoadModule authz_owner_module modules/mod_authz_owner.so
+LoadModule authz_groupfile_module modules/mod_authz_groupfile.so
+LoadModule authz_dbm_module modules/mod_authz_dbm.so
+LoadModule authz_default_module modules/mod_authz_default.so
+LoadModule ldap_module modules/mod_ldap.so
+LoadModule authnz_ldap_module modules/mod_authnz_ldap.so
+LoadModule include_module modules/mod_include.so
+LoadModule log_config_module modules/mod_log_config.so
+LoadModule logio_module modules/mod_logio.so
+LoadModule env_module modules/mod_env.so
+LoadModule ext_filter_module modules/mod_ext_filter.so
+LoadModule mime_magic_module modules/mod_mime_magic.so
+LoadModule expires_module modules/mod_expires.so
+LoadModule deflate_module modules/mod_deflate.so
+LoadModule headers_module modules/mod_headers.so
+LoadModule usertrack_module modules/mod_usertrack.so
+LoadModule setenvif_module modules/mod_setenvif.so
+LoadModule mime_module modules/mod_mime.so
+LoadModule dav_module modules/mod_dav.so
+LoadModule status_module modules/mod_status.so
+LoadModule autoindex_module modules/mod_autoindex.so
+LoadModule info_module modules/mod_info.so
+LoadModule dav_fs_module modules/mod_dav_fs.so
+LoadModule vhost_alias_module modules/mod_vhost_alias.so
+LoadModule negotiation_module modules/mod_negotiation.so
+LoadModule dir_module modules/mod_dir.so
+LoadModule actions_module modules/mod_actions.so
+LoadModule speling_module modules/mod_speling.so
+LoadModule userdir_module modules/mod_userdir.so
+LoadModule alias_module modules/mod_alias.so
+LoadModule substitute_module modules/mod_substitute.so
+LoadModule rewrite_module modules/mod_rewrite.so
+LoadModule proxy_module modules/mod_proxy.so
+LoadModule proxy_balancer_module modules/mod_proxy_balancer.so
+LoadModule proxy_ftp_module modules/mod_proxy_ftp.so
+LoadModule proxy_http_module modules/mod_proxy_http.so
+LoadModule proxy_ajp_module modules/mod_proxy_ajp.so
+LoadModule proxy_connect_module modules/mod_proxy_connect.so
+LoadModule cache_module modules/mod_cache.so
+LoadModule suexec_module modules/mod_suexec.so
+LoadModule disk_cache_module modules/mod_disk_cache.so
+LoadModule cgi_module modules/mod_cgi.so
+LoadModule version_module modules/mod_version.so
+
+#
+# The following modules are not loaded by default:
+#
+#LoadModule asis_module modules/mod_asis.so
+#LoadModule authn_dbd_module modules/mod_authn_dbd.so
+#LoadModule cern_meta_module modules/mod_cern_meta.so
+LoadModule cgid_module modules/mod_cgid.so
+#LoadModule dbd_module modules/mod_dbd.so
+#LoadModule dumpio_module modules/mod_dumpio.so
+#LoadModule filter_module modules/mod_filter.so
+#LoadModule ident_module modules/mod_ident.so
+#LoadModule log_forensic_module modules/mod_log_forensic.so
+#LoadModule unique_id_module modules/mod_unique_id.so
+#
+
+#
+# Load config files from the config directory "/etc/httpd/conf.d".
+#
+Include conf.d/*.conf
+
+#
+# ExtendedStatus controls whether Apache will generate "full" status
+# information (ExtendedStatus On) or just basic information (ExtendedStatus
+# Off) when the "server-status" handler is called. The default is Off.
+#
+#ExtendedStatus On
+
+#
+# If you wish httpd to run as a different user or group, you must run
+# httpd as root initially and it will switch.
+#
+# User/Group: The name (or #number) of the user/group to run httpd as.
+# . On SCO (ODT 3) use "User nouser" and "Group nogroup".
+# . On HPUX you may not be able to use shared memory as nobody, and the
+# suggested workaround is to create a user www and use that user.
+# NOTE that some kernels refuse to setgid(Group) or semctl(IPC_SET)
+# when the value of (unsigned)Group is above 60000;
+# don't use Group #-1 on these systems!
+#
+User edge
+Group edge
+
+### Section 2: 'Main' server configuration
+#
+# The directives in this section set up the values used by the 'main'
+# server, which responds to any requests that aren't handled by a
+# definition. These values also provide defaults for
+# any containers you may define later in the file.
+#
+# All of these directives may appear inside containers,
+# in which case these default settings will be overridden for the
+# virtual host being defined.
+#
+
+#
+# ServerAdmin: Your address, where problems with the server should be
+# e-mailed. This address appears on some server-generated pages, such
+# as error documents. e.g. admin@your-domain.com
+#
+ServerAdmin root@localhost
+
+#
+# ServerName gives the name and port that the server uses to identify itself.
+# This can often be determined automatically, but we recommend you specify
+# it explicitly to prevent problems during startup.
+#
+# If this is not set to valid DNS name for your host, server-generated
+# redirections will not work. See also the UseCanonicalName directive.
+#
+# If your host doesn't have a registered DNS name, enter its IP address here.
+# You will have to access it by its address anyway, and this will make
+# redirections work in a sensible way.
+#
+#ServerName www.example.com:80
+
+#
+# UseCanonicalName: Determines how Apache constructs self-referencing
+# URLs and the SERVER_NAME and SERVER_PORT variables.
+# When set "Off", Apache will use the Hostname and Port supplied
+# by the client. When set "On", Apache will use the value of the
+# ServerName directive.
+#
+UseCanonicalName Off
+
+#
+# DocumentRoot: The directory out of which you will serve your
+# documents. By default, all requests are taken from this directory, but
+# symbolic links and aliases may be used to point to other locations.
+#
+DocumentRoot "/data/edge_v1.0/edge_ui"
+
+#
+# Each directory to which Apache has access can be configured with respect
+# to which services and features are allowed and/or disabled in that
+# directory (and its subdirectories).
+#
+# First, we configure the "default" to be a very restrictive set of
+# features.
+#
+
+ Options FollowSymLinks
+ AllowOverride None
+
+
+#
+# Note that from this point forward you must specifically allow
+# particular features to be enabled - so if something's not working as
+# you might expect, make sure that you have specifically enabled it
+# below.
+#
+
+#
+# This should be changed to whatever you set DocumentRoot to.
+#
+
+
+#
+# Possible values for the Options directive are "None", "All",
+# or any combination of:
+# Indexes Includes FollowSymLinks SymLinksifOwnerMatch ExecCGI MultiViews
+#
+# Note that "MultiViews" must be named *explicitly* --- "Options All"
+# doesn't give it to you.
+#
+# The Options directive is both complicated and important. Please see
+# http://httpd.apache.org/docs/2.2/mod/core.html#options
+# for more information.
+#
+ Options Indexes FollowSymLinks
+
+#
+# AllowOverride controls what directives may be placed in .htaccess files.
+# It can be "All", "None", or any combination of the keywords:
+# Options FileInfo AuthConfig Limit
+#
+ AllowOverride None
+
+#
+# Controls who can get stuff from this server.
+#
+ Order allow,deny
+ Allow from all
+
+
+
+#
+# UserDir: The name of the directory that is appended onto a user's home
+# directory if a ~user request is received.
+#
+# The path to the end user account 'public_html' directory must be
+# accessible to the webserver userid. This usually means that ~userid
+# must have permissions of 711, ~userid/public_html must have permissions
+# of 755, and documents contained therein must be world-readable.
+# Otherwise, the client will only receive a "403 Forbidden" message.
+#
+# See also: http://httpd.apache.org/docs/misc/FAQ.html#forbidden
+#
+
+ #
+ # UserDir is disabled by default since it can confirm the presence
+ # of a username on the system (depending on home directory
+ # permissions).
+ #
+ UserDir disabled
+
+ #
+ # To enable requests to /~user/ to serve the user's public_html
+ # directory, remove the "UserDir disabled" line above, and uncomment
+ # the following line instead:
+ #
+ #UserDir public_html
+
+
+
+#
+# Control access to UserDir directories. The following is an example
+# for a site where these directories are restricted to read-only.
+#
+#
+# AllowOverride FileInfo AuthConfig Limit
+# Options MultiViews Indexes SymLinksIfOwnerMatch IncludesNoExec
+#
+# Order allow,deny
+# Allow from all
+#
+#
+# Order deny,allow
+# Deny from all
+#
+#
+
+#
+# DirectoryIndex: sets the file that Apache will serve if a directory
+# is requested.
+#
+# The index.html.var file (a type-map) is used to deliver content-
+# negotiated documents. The MultiViews Option can be used for the
+# same purpose, but it is much slower.
+#
+DirectoryIndex index.html index.html.var
+
+#
+# AccessFileName: The name of the file to look for in each directory
+# for additional configuration directives. See also the AllowOverride
+# directive.
+#
+AccessFileName .htaccess
+
+#
+# The following lines prevent .htaccess and .htpasswd files from being
+# viewed by Web clients.
+#
+
+ Order allow,deny
+ Deny from all
+ Satisfy All
+
+
+#
+# TypesConfig describes where the mime.types file (or equivalent) is
+# to be found.
+#
+TypesConfig /etc/mime.types
+
+#
+# DefaultType is the default MIME type the server will use for a document
+# if it cannot otherwise determine one, such as from filename extensions.
+# If your server contains mostly text or HTML documents, "text/plain" is
+# a good value. If most of your content is binary, such as applications
+# or images, you may want to use "application/octet-stream" instead to
+# keep browsers from trying to display binary files as though they are
+# text.
+#
+DefaultType text/plain
+
+#
+# The mod_mime_magic module allows the server to use various hints from the
+# contents of the file itself to determine its type. The MIMEMagicFile
+# directive tells the module where the hint definitions are located.
+#
+
+# MIMEMagicFile /usr/share/magic.mime
+ MIMEMagicFile conf/magic
+
+
+#
+# HostnameLookups: Log the names of clients or just their IP addresses
+# e.g., www.apache.org (on) or 204.62.129.132 (off).
+# The default is off because it'd be overall better for the net if people
+# had to knowingly turn this feature on, since enabling it means that
+# each client request will result in AT LEAST one lookup request to the
+# nameserver.
+#
+HostnameLookups Off
+
+#
+# EnableMMAP: Control whether memory-mapping is used to deliver
+# files (assuming that the underlying OS supports it).
+# The default is on; turn this off if you serve from NFS-mounted
+# filesystems. On some systems, turning it off (regardless of
+# filesystem) can improve performance; for details, please see
+# http://httpd.apache.org/docs/2.2/mod/core.html#enablemmap
+#
+#EnableMMAP off
+
+#
+# EnableSendfile: Control whether the sendfile kernel support is
+# used to deliver files (assuming that the OS supports it).
+# The default is on; turn this off if you serve from NFS-mounted
+# filesystems. Please see
+# http://httpd.apache.org/docs/2.2/mod/core.html#enablesendfile
+#
+#EnableSendfile off
+
+#
+# ErrorLog: The location of the error log file.
+# If you do not specify an ErrorLog directive within a
+# container, error messages relating to that virtual host will be
+# logged here. If you *do* define an error logfile for a
+# container, that host's errors will be logged there and not here.
+#
+ErrorLog logs/error_log
+
+#
+# LogLevel: Control the number of messages logged to the error_log.
+# Possible values include: debug, info, notice, warn, error, crit,
+# alert, emerg.
+#
+LogLevel warn
+
+#
+# The following directives define some format nicknames for use with
+# a CustomLog directive (see below).
+#
+LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
+LogFormat "%h %l %u %t \"%r\" %>s %b" common
+LogFormat "%{Referer}i -> %U" referer
+LogFormat "%{User-agent}i" agent
+
+# "combinedio" includes actual counts of actual bytes received (%I) and sent (%O); this
+# requires the mod_logio module to be loaded.
+#LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio
+
+#
+# The location and format of the access logfile (Common Logfile Format).
+# If you do not define any access logfiles within a
+# container, they will be logged here. Contrariwise, if you *do*
+# define per- access logfiles, transactions will be
+# logged therein and *not* in this file.
+#
+#CustomLog logs/access_log common
+
+#
+# If you would like to have separate agent and referer logfiles, uncomment
+# the following directives.
+#
+#CustomLog logs/referer_log referer
+#CustomLog logs/agent_log agent
+
+#
+# For a single logfile with access, agent, and referer information
+# (Combined Logfile Format), use the following directive:
+#
+CustomLog logs/access_log combined
+
+#
+# Optionally add a line containing the server version and virtual host
+# name to server-generated pages (internal error documents, FTP directory
+# listings, mod_status and mod_info output etc., but not CGI generated
+# documents or custom error documents).
+# Set to "EMail" to also include a mailto: link to the ServerAdmin.
+# Set to one of: On | Off | EMail
+#
+ServerSignature On
+
+#
+# Aliases: Add here as many aliases as you need (with no limit). The format is
+# Alias fakename realname
+#
+# Note that if you include a trailing / on fakename then the server will
+# require it to be present in the URL. So "/icons" isn't aliased in this
+# example, only "/icons/". If the fakename is slash-terminated, then the
+# realname must also be slash terminated, and if the fakename omits the
+# trailing slash, the realname must also omit it.
+#
+# We include the /icons/ alias for FancyIndexed directory listings. If you
+# do not use FancyIndexing, you may comment this out.
+#
+Alias /icons/ "/var/www/icons/"
+
+
+ Options Indexes MultiViews FollowSymLinks
+ AllowOverride None
+ Order allow,deny
+ Allow from all
+
+
+#
+# WebDAV module configuration section.
+#
+
+ # Location of the WebDAV lock database.
+ DAVLockDB /var/lib/dav/lockdb
+
+
+#
+# ScriptAlias: This controls which directories contain server scripts.
+# ScriptAliases are essentially the same as Aliases, except that
+# documents in the realname directory are treated as applications and
+# run by the server when requested rather than as documents sent to the client.
+# The same rules about trailing "/" apply to ScriptAlias directives as to
+# Alias.
+#
+#ScriptAlias /cgi-bin/ "/var/www/cgi-bin/"
+
+#
+# "/var/www/cgi-bin" should be changed to whatever your ScriptAliased
+# CGI directory exists, if you have that configured.
+#
+
+ AllowOverride None
+ Options None
+ Order allow,deny
+ Allow from all
+
+
+#
+# Redirect allows you to tell clients about documents which used to exist in
+# your server's namespace, but do not anymore. This allows you to tell the
+# clients where to look for the relocated document.
+# Example:
+# Redirect permanent /foo http://www.example.com/bar
+
+#
+# Directives controlling the display of server-generated directory listings.
+#
+
+#
+# IndexOptions: Controls the appearance of server-generated directory
+# listings.
+#
+IndexOptions FancyIndexing VersionSort NameWidth=* HTMLTable Charset=UTF-8
+
+#
+# AddIcon* directives tell the server which icon to show for different
+# files or filename extensions. These are only displayed for
+# FancyIndexed directories.
+#
+AddIconByEncoding (CMP,/icons/compressed.gif) x-compress x-gzip
+
+AddIconByType (TXT,/icons/text.gif) text/*
+AddIconByType (IMG,/icons/image2.gif) image/*
+AddIconByType (SND,/icons/sound2.gif) audio/*
+AddIconByType (VID,/icons/movie.gif) video/*
+
+AddIcon /icons/binary.gif .bin .exe
+AddIcon /icons/binhex.gif .hqx
+AddIcon /icons/tar.gif .tar
+AddIcon /icons/world2.gif .wrl .wrl.gz .vrml .vrm .iv
+AddIcon /icons/compressed.gif .Z .z .tgz .gz .zip
+AddIcon /icons/a.gif .ps .ai .eps
+AddIcon /icons/layout.gif .html .shtml .htm .pdf
+AddIcon /icons/text.gif .txt
+AddIcon /icons/c.gif .c
+AddIcon /icons/p.gif .pl .py
+AddIcon /icons/f.gif .for
+AddIcon /icons/dvi.gif .dvi
+AddIcon /icons/uuencoded.gif .uu
+AddIcon /icons/script.gif .conf .sh .shar .csh .ksh .tcl
+AddIcon /icons/tex.gif .tex
+AddIcon /icons/bomb.gif core
+
+AddIcon /icons/back.gif ..
+AddIcon /icons/hand.right.gif README
+AddIcon /icons/folder.gif ^^DIRECTORY^^
+AddIcon /icons/blank.gif ^^BLANKICON^^
+
+#
+# DefaultIcon is which icon to show for files which do not have an icon
+# explicitly set.
+#
+DefaultIcon /icons/unknown.gif
+
+#
+# AddDescription allows you to place a short description after a file in
+# server-generated indexes. These are only displayed for FancyIndexed
+# directories.
+# Format: AddDescription "description" filename
+#
+#AddDescription "GZIP compressed document" .gz
+#AddDescription "tar archive" .tar
+#AddDescription "GZIP compressed tar archive" .tgz
+
+#
+# ReadmeName is the name of the README file the server will look for by
+# default, and append to directory listings.
+#
+# HeaderName is the name of a file which should be prepended to
+# directory indexes.
+ReadmeName README.html
+HeaderName HEADER.html
+
+#
+# IndexIgnore is a set of filenames which directory indexing should ignore
+# and not include in the listing. Shell-style wildcarding is permitted.
+#
+IndexIgnore .??* *~ *# HEADER* README* RCS CVS *,v *,t
+
+#
+# DefaultLanguage and AddLanguage allows you to specify the language of
+# a document. You can then use content negotiation to give a browser a
+# file in a language the user can understand.
+#
+# Specify a default language. This means that all data
+# going out without a specific language tag (see below) will
+# be marked with this one. You probably do NOT want to set
+# this unless you are sure it is correct for all cases.
+#
+# * It is generally better to not mark a page as
+# * being a certain language than marking it with the wrong
+# * language!
+#
+# DefaultLanguage nl
+#
+# Note 1: The suffix does not have to be the same as the language
+# keyword --- those with documents in Polish (whose net-standard
+# language code is pl) may wish to use "AddLanguage pl .po" to
+# avoid the ambiguity with the common suffix for perl scripts.
+#
+# Note 2: The example entries below illustrate that in some cases
+# the two character 'Language' abbreviation is not identical to
+# the two character 'Country' code for its country,
+# E.g. 'Danmark/dk' versus 'Danish/da'.
+#
+# Note 3: In the case of 'ltz' we violate the RFC by using a three char
+# specifier. There is 'work in progress' to fix this and get
+# the reference data for rfc1766 cleaned up.
+#
+# Catalan (ca) - Croatian (hr) - Czech (cs) - Danish (da) - Dutch (nl)
+# English (en) - Esperanto (eo) - Estonian (et) - French (fr) - German (de)
+# Greek-Modern (el) - Hebrew (he) - Italian (it) - Japanese (ja)
+# Korean (ko) - Luxembourgeois* (ltz) - Norwegian Nynorsk (nn)
+# Norwegian (no) - Polish (pl) - Portugese (pt)
+# Brazilian Portuguese (pt-BR) - Russian (ru) - Swedish (sv)
+# Simplified Chinese (zh-CN) - Spanish (es) - Traditional Chinese (zh-TW)
+#
+AddLanguage ca .ca
+AddLanguage cs .cz .cs
+AddLanguage da .dk
+AddLanguage de .de
+AddLanguage el .el
+AddLanguage en .en
+AddLanguage eo .eo
+AddLanguage es .es
+AddLanguage et .et
+AddLanguage fr .fr
+AddLanguage he .he
+AddLanguage hr .hr
+AddLanguage it .it
+AddLanguage ja .ja
+AddLanguage ko .ko
+AddLanguage ltz .ltz
+AddLanguage nl .nl
+AddLanguage nn .nn
+AddLanguage no .no
+AddLanguage pl .po
+AddLanguage pt .pt
+AddLanguage pt-BR .pt-br
+AddLanguage ru .ru
+AddLanguage sv .sv
+AddLanguage zh-CN .zh-cn
+AddLanguage zh-TW .zh-tw
+
+#
+# LanguagePriority allows you to give precedence to some languages
+# in case of a tie during content negotiation.
+#
+# Just list the languages in decreasing order of preference. We have
+# more or less alphabetized them here. You probably want to change this.
+#
+LanguagePriority en ca cs da de el eo es et fr he hr it ja ko ltz nl nn no pl pt pt-BR ru sv zh-CN zh-TW
+
+#
+# ForceLanguagePriority allows you to serve a result page rather than
+# MULTIPLE CHOICES (Prefer) [in case of a tie] or NOT ACCEPTABLE (Fallback)
+# [in case no accepted languages matched the available variants]
+#
+ForceLanguagePriority Prefer Fallback
+
+#
+# Specify a default charset for all content served; this enables
+# interpretation of all content as UTF-8 by default. To use the
+# default browser choice (ISO-8859-1), or to allow the META tags
+# in HTML content to override this choice, comment out this
+# directive:
+#
+AddDefaultCharset UTF-8
+
+#
+# AddType allows you to add to or override the MIME configuration
+# file mime.types for specific file types.
+#
+#AddType application/x-tar .tgz
+
+#
+# AddEncoding allows you to have certain browsers uncompress
+# information on the fly. Note: Not all browsers support this.
+# Despite the name similarity, the following Add* directives have nothing
+# to do with the FancyIndexing customization directives above.
+#
+#AddEncoding x-compress .Z
+#AddEncoding x-gzip .gz .tgz
+
+# If the AddEncoding directives above are commented-out, then you
+# probably should define those extensions to indicate media types:
+#
+AddType application/x-compress .Z
+AddType application/x-gzip .gz .tgz
+
+#
+# MIME-types for downloading Certificates and CRLs
+#
+AddType application/x-x509-ca-cert .crt
+AddType application/x-pkcs7-crl .crl
+
+#
+# AddHandler allows you to map certain file extensions to "handlers":
+# actions unrelated to filetype. These can be either built into the server
+# or added with the Action directive (see below)
+#
+# To use CGI scripts outside of ScriptAliased directories:
+# (You will also need to add "ExecCGI" to the "Options" directive.)
+#
+#AddHandler cgi-script .cgi
+
+#
+# For files that include their own HTTP headers:
+#
+#AddHandler send-as-is asis
+
+#
+# For type maps (negotiated resources):
+# (This is enabled by default to allow the Apache "It Worked" page
+# to be distributed in multiple languages.)
+#
+AddHandler type-map var
+
+#
+# Filters allow you to process content before it is sent to the client.
+#
+# To parse .shtml files for server-side includes (SSI):
+# (You will also need to add "Includes" to the "Options" directive.)
+#
+AddType text/html .shtml
+AddOutputFilter INCLUDES .shtml
+
+#
+# Action lets you define media types that will execute a script whenever
+# a matching file is called. This eliminates the need for repeated URL
+# pathnames for oft-used CGI file processors.
+# Format: Action media/type /cgi-script/location
+# Format: Action handler-name /cgi-script/location
+#
+
+#
+# Customizable error responses come in three flavors:
+# 1) plain text 2) local redirects 3) external redirects
+#
+# Some examples:
+#ErrorDocument 500 "The server made a boo boo."
+#ErrorDocument 404 /missing.html
+#ErrorDocument 404 "/cgi-bin/missing_handler.pl"
+#ErrorDocument 402 http://www.example.com/subscription_info.html
+#
+
+#
+# Putting this all together, we can internationalize error responses.
+#
+# We use Alias to redirect any /error/HTTP_.html.var response to
+# our collection of by-error message multi-language collections. We use
+# includes to substitute the appropriate text.
+#
+# You can modify the messages' appearance without changing any of the
+# default HTTP_.html.var files by adding the line:
+#
+# Alias /error/include/ "/your/include/path/"
+#
+# which allows you to create your own set of files by starting with the
+# /var/www/error/include/ files and
+# copying them to /your/include/path/, even on a per-VirtualHost basis.
+#
+
+Alias /error/ "/var/www/error/"
+
+
+
+
+ AllowOverride None
+ Options IncludesNoExec
+ AddOutputFilter Includes html
+ AddHandler type-map var
+ Order allow,deny
+ Allow from all
+ LanguagePriority en es de fr
+ ForceLanguagePriority Prefer Fallback
+
+
+# ErrorDocument 400 /error/HTTP_BAD_REQUEST.html.var
+# ErrorDocument 401 /error/HTTP_UNAUTHORIZED.html.var
+# ErrorDocument 403 /error/HTTP_FORBIDDEN.html.var
+# ErrorDocument 404 /error/HTTP_NOT_FOUND.html.var
+# ErrorDocument 405 /error/HTTP_METHOD_NOT_ALLOWED.html.var
+# ErrorDocument 408 /error/HTTP_REQUEST_TIME_OUT.html.var
+# ErrorDocument 410 /error/HTTP_GONE.html.var
+# ErrorDocument 411 /error/HTTP_LENGTH_REQUIRED.html.var
+# ErrorDocument 412 /error/HTTP_PRECONDITION_FAILED.html.var
+# ErrorDocument 413 /error/HTTP_REQUEST_ENTITY_TOO_LARGE.html.var
+# ErrorDocument 414 /error/HTTP_REQUEST_URI_TOO_LARGE.html.var
+# ErrorDocument 415 /error/HTTP_UNSUPPORTED_MEDIA_TYPE.html.var
+# ErrorDocument 500 /error/HTTP_INTERNAL_SERVER_ERROR.html.var
+# ErrorDocument 501 /error/HTTP_NOT_IMPLEMENTED.html.var
+# ErrorDocument 502 /error/HTTP_BAD_GATEWAY.html.var
+# ErrorDocument 503 /error/HTTP_SERVICE_UNAVAILABLE.html.var
+# ErrorDocument 506 /error/HTTP_VARIANT_ALSO_VARIES.html.var
+
+
+
+
+#
+# The following directives modify normal HTTP response behavior to
+# handle known problems with browser implementations.
+#
+BrowserMatch "Mozilla/2" nokeepalive
+BrowserMatch "MSIE 4\.0b2;" nokeepalive downgrade-1.0 force-response-1.0
+BrowserMatch "RealPlayer 4\.0" force-response-1.0
+BrowserMatch "Java/1\.0" force-response-1.0
+BrowserMatch "JDK/1\.0" force-response-1.0
+
+#
+# The following directive disables redirects on non-GET requests for
+# a directory that does not include the trailing slash. This fixes a
+# problem with Microsoft WebFolders which does not appropriately handle
+# redirects for folders with DAV methods.
+# Same deal with Apple's DAV filesystem and Gnome VFS support for DAV.
+#
+BrowserMatch "Microsoft Data Access Internet Publishing Provider" redirect-carefully
+BrowserMatch "MS FrontPage" redirect-carefully
+BrowserMatch "^WebDrive" redirect-carefully
+BrowserMatch "^WebDAVFS/1.[0123]" redirect-carefully
+BrowserMatch "^gnome-vfs/1.0" redirect-carefully
+BrowserMatch "^XML Spy" redirect-carefully
+BrowserMatch "^Dreamweaver-WebDAV-SCM1" redirect-carefully
+
+#
+# Allow server status reports generated by mod_status,
+# with the URL of http://servername/server-status
+# Change the ".example.com" to match your domain to enable.
+#
+#
+# SetHandler server-status
+# Order deny,allow
+# Deny from all
+# Allow from .example.com
+#
+
+#
+# Allow remote server configuration reports, with the URL of
+# http://servername/server-info (requires that mod_info.c be loaded).
+# Change the ".example.com" to match your domain to enable.
+#
+#
+# SetHandler server-info
+# Order deny,allow
+# Deny from all
+# Allow from .example.com
+#
+
+#
+# Proxy Server directives. Uncomment the following lines to
+# enable the proxy server:
+#
+#
+#ProxyRequests On
+#
+#
+# Order deny,allow
+# Deny from all
+# Allow from .example.com
+#
+
+#
+# Enable/disable the handling of HTTP/1.1 "Via:" headers.
+# ("Full" adds the server version; "Block" removes all outgoing Via: headers)
+# Set to one of: Off | On | Full | Block
+#
+#ProxyVia On
+
+#
+# To enable a cache of proxied content, uncomment the following lines.
+# See http://httpd.apache.org/docs/2.2/mod/mod_cache.html for more details.
+#
+#
+# CacheEnable disk /
+# CacheRoot "/var/cache/mod_proxy"
+#
+#
+
+#
+# End of proxy directives.
+
+### Section 3: Virtual Hosts
+#
+# VirtualHost: If you want to maintain multiple domains/hostnames on your
+# machine you can setup VirtualHost containers for them. Most configurations
+# use only name-based virtual hosts so the server doesn't need to worry about
+# IP addresses. This is indicated by the asterisks in the directives below.
+#
+# Please see the documentation at
+#
+# for further details before you try to setup virtual hosts.
+#
+# You may use the command line option '-S' to verify your virtual host
+# configuration.
+
+#
+# Use name-based virtual hosting.
+#
+#NameVirtualHost *:80
+#
+# NOTE: NameVirtualHost cannot be used without a port specifier
+# (e.g. :80) if mod_ssl is being used, due to the nature of the
+# SSL protocol.
+#
+
+#
+# VirtualHost example:
+# Almost any Apache directive may go into a VirtualHost container.
+# The first VirtualHost section is used for requests without a known
+# server name.
+#
+#
+# ServerAdmin webmaster@dummy-host.example.com
+# DocumentRoot /www/docs/dummy-host.example.com
+# ServerName dummy-host.example.com
+# ErrorLog logs/dummy-host.example.com-error_log
+# CustomLog logs/dummy-host.example.com-access_log common
+#
+
+
+
+#Added the edge_apache_conf file right here
+
+
+ ScriptAlias /edge_ui/cgi-bin/ /data/edge_v1.0/edge_ui/cgi-bin/
+
+ AllowOverride None
+ Options +ExecCGI -MultiViews +SymLinksIfOwnerMatch
+ AddHandler cgi-script .cgi .pl
+ Order allow,deny
+ allow from all
+
+
+
+
+ Alias /edge_ui/ "/data/edge_v1.0/edge_ui/"
+
+ Options Indexes FollowSymlinks
+ AllowOverride None
+ #Require all granted
+
+
+
+
+ ForceType application/octet-stream
+ Header set Content-Disposition attachment
+
+
+
+ Options -Indexes
+
+
+
+ Order allow,deny
+ Allow from all
+
+
+# for user management tomcat server to be visible on port 80
+ProxyPass /userManagement http://localhost:8080/userManagement
+ProxyPassReverse /userManagement http://localhost:8080/userManagement
+ProxyPass /userManagementWS http://localhost:8080/userManagementWS
+ProxyPassReverse /userManagementWS http://localhost:8080/userManagementWS
+
+#for JBrowse
+AddType application/octet-stream .bam .bami
+
+
+ mod_gzip_item_exclude "(\.jsonz|\.txtz)"
+
+
+ SetEnvIf Request_URI "(\.jsonz|\.txtz)" no-gzip dont-vary
+
+
+
+ Header onsuccess set Content-Encoding gzip
+
+
+
+
+ AllowOverride FileInfo
+
+
+
diff --git a/edge_ui/cgi-bin/batch_run_sra.pl b/edge_ui/cgi-bin/batch_run_sra.pl
new file mode 100755
index 00000000..ca6426b9
--- /dev/null
+++ b/edge_ui/cgi-bin/batch_run_sra.pl
@@ -0,0 +1,153 @@
+#!/usr/bin/env perl
+
+use strict;
+use FindBin qw($RealBin);
+use POSIX qw(strftime);
+use LWP::Simple; # from CPAN
+use JSON qw( decode_json ); # from CPAN
+
+
+if (!@ARGV){ print "$0 [ bsve_sra_tsv_dir | file | ebi ]\n"; exit;}
+
+my $date_string = strftime "%Y-%m-%d", localtime;
+my $proxy = $ENV{HTTP_PROXY} || $ENV{http_proxy};
+$proxy = "--proxy $proxy " if ($proxy);
+
+my $file;
+if ( -d $ARGV[0]){
+ my $dir = $ARGV[0];
+ $file= "$dir/bsve_$date_string.tsv";
+}elsif( -f $ARGV[0]){
+ $file=$ARGV[0];
+}else{
+ `mkdir -p $ENV{HOME}/sra_runs`;
+ $file = "$ENV{HOME}/sra_runs/bsve_$date_string.tsv";
+ my $cmd = "/usr/bin/curl $proxy -o $file \"http://www.ebi.ac.uk/ena/data/warehouse/search?query=tax_tree(408169)%20AND%20first_public=$date_string%20AND%20library_strategy=WGS%20AND%20library_selection=RANDOM%20AND%20(library_source=GENOMIC%20OR%20library_source=METAGENOMIC)&result=read_run&fields=run_accession,sample_accession,study_accession,study_title,experiment_title,scientific_name,instrument_model,library_layout,base_count&limit=10000&display=report\" ";
+ print "Query ebi using REST URL ...\n";
+ #print $cmd,"\n";
+ system($cmd);
+}
+
+if ( ! -e $file or -z $file){
+ print "The $file not exist or empty\n";
+}
+
+open (my $fh,$file) or die "Cannot open $file. $!\n";
+while(<$fh>){
+ chomp;
+ next if ($_ =~ /run_accession/);
+ next if ($_ =~ /^\s*$/);
+ my @array= split /\t/,$_;
+ my $sra_id = $array[0];
+ my $study_id = $array[2];
+ my $study = $array[3];
+ my $experiment = $array[4];
+ my $instrument = $array[6];
+ my $projname = "bsve_$sra_id";
+ my $projdesc = "$sra_id $study";
+ $projdesc =~ s/(['"])/\\$1/g;
+ #$projdesc =~ s/\(/\\(/g;
+ #$projdesc =~ s/\)/\\)/g;
+ next if ( -d "$RealBin/../EDGE_output/$projname");
+
+ #get sample metadata from ebi by sample id
+ my $sample_id = $array[1];
+ my $cmd = "mkdir -p $ENV{HOME}/sra_runs";
+ system($cmd);
+ my $tmp = "$ENV{HOME}/sra_runs/bsve_metadata_tmp.txt";
+ $cmd = "/usr/bin/curl $proxy -o $tmp \"http://www.ebi.ac.uk/ena/data/warehouse/search?query=accession=$sample_id&result=sample&fields=accession,collection_date,country,description,first_public,isolation_source,location,scientific_name,sample_alias,center_name,environment_material,host,host_status,host_sex&display=report\" ";
+ #print "$cmd\n";
+ system($cmd);
+ open TMP, $tmp;
+ my $sequencer = $array[6];
+ my ($sampleType, $host, $collectionDate, $city, $state, $country, $lat, $lng,$seqPlatform, $gender, $hostCondition, $source, $sampleName, $center, $seqDate, $location);
+ while() {
+ chomp;
+ next if($_ =~ /^accession/);
+ next if ($_ =~ /^\s*$/);
+ my @parts = split /\t/, $_;
+ $collectionDate = $parts[1];
+ if($collectionDate =~ /\//) {
+ my @its = split /\//, $collectionDate;
+ $collectionDate = $its[1];
+ }
+ $location = $parts[2];
+ $country = $location;
+ if($country =~ /(.*):\s*(.*?),\s*(.*)\s*/) {
+ $country = $1;
+ $city = $2;
+ $state = $3;
+ }
+ if($country =~ /(.*):\s*(.*)\s*:\s*(.*)\s*/) {
+ $country = $1;
+ $city = $2;
+ }
+ elsif($country =~ /(.*):\s*(.*)\s*/) {
+ $country = $1;
+ $state= $2;
+ }
+ $sampleName = $parts[3];
+ $sampleName = $parts[8] unless $sampleName;
+ $seqDate = $parts[4];
+ if($seqDate =~ /\//) {
+ my @its = split /\//, $seqDate;
+ $seqDate = $its[1];
+ }
+ $source = $parts[5];
+ $source = $parts[10] unless $source;
+ my $latlng = $parts[6];
+ $latlng =~ s/^\s+//;
+ my @its = split /\s+/, $latlng;
+ $lat = $its[0];
+ $lng = $its[2];
+ if($its[1] eq "S") {
+ $lat = -$lat;
+ }
+ if($its[3] eq "W") {
+ $lng = -$lng;
+ }
+ $host = $parts[11];
+ $sampleType = "environmental";
+ my $stype = lc $parts[7];
+ if($stype =~ /human/ || lc($host) =~ /human/ || lc($host) =~ /homo/) {
+ $sampleType = "human";
+ }
+ elsif($stype =~ /mouse|rat|pig|fish|ant|chicken|bee|frog/ || lc($host) =~ /mouse|rat|pig|fish|ant|chicken|bee|frog/) {
+ $sampleType = "animal";
+ }
+ $center = $parts[9];
+ $hostCondition = $parts[12];
+ $gender = $parts[13];
+ #get lat,lng from location
+ if(!$lat && !$lng && $location) {
+ ($lat,$lng) = getLatLong($location);
+ }
+ }
+ close TMP;
+
+ chdir $RealBin;
+ $cmd = "$RealBin/edge_submit.cgi $projname $sra_id \"$projdesc\"";
+ $cmd .= " \"$sampleType\" \"$source\" \"$collectionDate\" \"$city\" \"$state\" \"$country\" \"$lat\" \"$lng\" \"$seqPlatform\" \"$sequencer\" \"$seqDate\" \"$host\" \"$hostCondition\" \"$gender\"";
+ $cmd .= " SRA \"$sra_id\" \"$study\" \"$sampleName\" \"$experiment\" \"$location\" \"$instrument\" \"$center\" \"$study_id\"";
+ print $cmd,"\n";
+ system($cmd);
+}
+close $fh;
+
+
+#################
+sub getLatLong($){
+ my ($address) = @_;
+ my $format = "json"; #can also to 'xml'
+ my $geocodeapi = "https://maps.googleapis.com/maps/api/geocode/";
+ my $url = $geocodeapi . $format . "?address=" . $address;
+ my $json = get($url);
+ my $d_json = decode_json( $json );
+
+ my $lat = $d_json->{results}->[0]->{geometry}->{location}->{lat};
+ my $lng = $d_json->{results}->[0]->{geometry}->{location}->{lng};
+
+ return ($lat, $lng);
+}
+
+1;
diff --git a/edge_ui/cgi-bin/edge_action.cgi b/edge_ui/cgi-bin/edge_action.cgi
index 7e31ab68..f606fb4d 100755
--- a/edge_ui/cgi-bin/edge_action.cgi
+++ b/edge_ui/cgi-bin/edge_action.cgi
@@ -16,19 +16,35 @@ use HTTP::Request::Common;
use POSIX qw(strftime);
use Data::Dumper;
use Digest::MD5 qw(md5_hex);
+use Tie::File;
+
require "edge_user_session.cgi";
+require "../cluster/clusterWrapper.pl";
+##sample metadata
+require "../metadata_scripts/metadata_api.pl";
+#END
my $cgi = CGI->new;
my %opt = $cgi->Vars();
my $pname = $opt{proj};
my $username = $opt{username};
my $password = $opt{password};
+my $new_proj_name = $opt{rename_project}; #getting project name input value from edge.js
+my $new_proj_desc = $opt{project_description}; #getting project description input value from edge.js
my $action = lc($opt{action});
my $shareEmail = $opt{shareEmail};
my $userType = $opt{userType}||"user";
my $protocol = $opt{protocol}||"http:";
my $sid = $opt{sid};
-my $domain = $ENV{'HTTP_HOST'};
+my $taxa_for_contig_extract = $opt{taxa};
+my $cptool_for_reads_extract = $opt{cptool};
+my $contig_id = $opt{contigID};
+my $blast_params = $opt{"edge-contig-blast-params"} || " -num_alignments 10 -num_descriptions 10 -evalue 1e-10 " ;
+my $domain = $ENV{'HTTP_HOST'}|| 'edge-bsve.lanl.gov';
+my ($webhostname) = $domain =~ /^(\S+?)\./;
+my $EDGE_HOME = $ENV{EDGE_HOME};
+$EDGE_HOME ||= "$RealBin/../..";
+$ENV{PATH} = "$EDGE_HOME/bin:$ENV{PATH}";
$pname ||= $ARGV[0];
$action ||= $ARGV[1];
@@ -37,38 +53,66 @@ $password ||= $ARGV[3];
$shareEmail ||= $ARGV[4];
$sid ||= $ARGV[5];
$domain ||= $ARGV[6];
+my $umSystemStatus = $ARGV[7];
-# read system params from config template
-my $config_tmpl = "$RealBin/edge_config.tmpl";
-my $sys = &getSysParamFromConfig($config_tmpl);
+# read system params from sys.properties
+my $sysconfig = "$RealBin/../sys.properties";
+my $sys = &getSysParamFromConfig($sysconfig);
+$sys->{edgeui_output} = "$sys->{edgeui_output}"."/$webhostname" if ( -d "$sys->{edgeui_output}/$webhostname");
+$sys->{edgeui_input} = "$sys->{edgeui_input}"."/$webhostname" if ( -d "$sys->{edgeui_input}/$webhostname");
my $out_dir = $sys->{edgeui_output};
my $input_dir = $sys->{edgeui_input};
+my $www_root = $sys->{edgeui_wwwroot};
my $um_url = $sys->{edge_user_management_url};
+my $keep_days = $sys->{edgeui_proj_store_days};
$domain ||= "edgeset.lanl.gov";
$um_url ||= "$protocol//$domain/userManagement";
$out_dir ||= "/tmp"; #for security
+$umSystemStatus ||= $sys->{user_management} if (! @ARGV);
my $info;
my $proj_dir = abs_path("$out_dir/$pname");
my $list;
my $permission;
+#cluster
+my $cluster = $sys->{cluster};
+my $cluster_job_prefix = $sys->{cluster_job_prefix};
+my $cluster_qsub_options= $sys->{cluster_qsub_options};
+
#check projects vital
-my ($vital, $name2pid) = &checkProjVital();
+my ($vital, $name2pid, $error);
+if($cluster) {
+ ($vital, $name2pid, $error) = checkProjVital_cluster($cluster_job_prefix);
+ if($error) {
+ $info->{INFO} = "ERROR: $error";
+ }
+} else {
+ ($vital, $name2pid) = &checkProjVital();
+}
+
+
my $time = strftime "%F %X", localtime;
+my ($memUsage, $cpuUsage, $diskUsage) = &getSystemUsage();
$info->{STATUS} = "FAILURE";
#$info->{INFO} = "Project $pname not found.";
-
+if ( ($memUsage > 99 or $cpuUsage > 99) and $action ne 'interrupt' and !$cluster){
+ $info->{INFO} = "No enough CPU/MEM resource to perform action. Please wait or contact system administrator.";
+ &returnStatus();
+}
#session check
my $real_name = $pname;
-my $user_proj_dir;
-if ( $sys->{user_management} )
+my $projCode;
+my $projStatus;
+my @projCodes = split /,/,$opt{proj} if ($action eq 'compare');
+my $user_proj_dir = "$input_dir/tmp";
+if ( $umSystemStatus )
{
my $valid = verifySession($sid);
unless($valid){
$info->{INFO} = "ERROR: Invalid session found.";
- &returnStatus();
+ &returnStatus() if (!@ARGV);
}
else{
($username,$password) = getCredentialsFromSession($sid);
@@ -76,8 +120,8 @@ if ( $sys->{user_management} )
$list = &getUserProjFromDB("owner");
- $real_name=getProjNameFromDB($pname);
-
+ ($real_name,$projCode,$projStatus)= &getProjNameFromDB($pname) if ($action ne 'compare');
+
$user_proj_dir = "$input_dir/". md5_hex($username)."/MyProjects/$real_name"."_".$pname;
#separate permission for future uses. A permission module can be added potentially..
if( defined $list->{$pname} || $userType =~ /admin/){
@@ -91,8 +135,23 @@ if ( $sys->{user_management} )
$permission->{unshare} = 1;
$permission->{publish} = 1;
$permission->{unpublish} = 1;
+ $permission->{tarproj} = 1;
+ $permission->{getcontigbytaxa} = 1;
+ $permission->{getreadsbytaxa} = 1;
+ $permission->{metadata} = 1;
}
#print STDERR "User: $username; Sid: $sid; Valid: $valid; Pname: $pname; Realname: $real_name; List:",Dumper($list),"\n";
+}else{
+ ($real_name,$projCode,$projStatus)= &scanProjToList($out_dir,$pname) if ($action ne 'compare');
+ if (!$real_name){
+ $info->{INFO} = "ERROR: No project with ID $pname.";
+ &returnStatus();
+ }
+}
+ $proj_dir = abs_path("$out_dir/$projCode") if ( -d "$out_dir/$projCode");
+
+if ($action eq 'rename' ){
+ renameProject();
}
if( $action eq 'empty' ){
@@ -101,7 +160,7 @@ if( $action eq 'empty' ){
&returnStatus();
}
- if( $name2pid->{$pname} ){
+ if( $name2pid->{$pname} || $name2pid->{$projCode}){
$info->{INFO} = "ERROR: Project $real_name is running.";
&returnStatus();
}
@@ -111,6 +170,7 @@ if( $action eq 'empty' ){
while( defined (my $file = readdir BIN) ) {
next if $file eq '.' or $file eq '..';
`rm -rf $proj_dir/$file` if -d "$proj_dir/$file";
+ `rm -f $proj_dir/\.run*`;
}
closedir(BIN);
@@ -168,26 +228,35 @@ elsif( $action eq 'delete' ){
$info->{STATUS} = "FAILURE";
$info->{INFO} = "Failed to delete the output directory.";
- my $pid = $name2pid->{$pname};
+ my $pid = $name2pid->{$pname} || $name2pid->{$projCode};
if( $pid ){
- my $invalid = &killProcess($pid);
- if( $invalid ){
- $info->{INFO} = "Failed to kill the running process. (PID: $pid)";
+ my $invalid;
+ if($cluster) {
+ $invalid = clusterDeleteJob($pid);
+ if( $invalid ){
+ $info->{INFO} = "Failed to kill the running cluster job. (Job ID: $pid)";
+ }
+ } else {
+ $invalid = &killProcess($pid);
+ if( $invalid ){
+ $info->{INFO} = "Failed to kill the running process. (PID: $pid)";
+ }
}
}
- `rm -rf $proj_dir`;
- `rm -rf $out_dir/$pname`;
- if( !-e $proj_dir && !-e "$out_dir/$pname" ){
- $info->{STATUS} = "SUCCESS";
- $info->{INFO} = "Project $real_name has been deleted.";
- }
if ($username && $password){
&updateDBProjectStatus($pname,"delete");
`rm -f $user_proj_dir`;
`rm -f $input_dir/public/projects/${real_name}_$pname`;
`rm -f $input_dir/*/SharedProjects/${real_name}_$pname`;
}
+ `rm -rf $proj_dir`;
+ `rm -rf $out_dir/$pname`;
+ `rm -f $input_dir/../JBrowse/data/$pname $input_dir/../JBrowse/data/$projCode`;
+ if( !-e $proj_dir && !-e "$out_dir/$pname" ){
+ $info->{STATUS} = "SUCCESS";
+ $info->{INFO} = "Project $real_name has been deleted.";
+ }
}
else{
@@ -204,14 +273,24 @@ elsif( $action eq 'interrupt' ){
$info->{STATUS} = "FAILURE";
$info->{INFO} = "Failed to stop EDGE process.";
- my $pid = $name2pid->{$pname};
+ my $pid = $name2pid->{$pname} || $name2pid->{$projCode} || $projStatus eq "unstarted";
if( $pid ){
- my $invalid = &killProcess($pid);
- if( !$invalid ){
- `echo "\n*** [$time] EDGE_UI: This project has been interrupted. ***" |tee -a $proj_dir/process.log >> $proj_dir/process_current.log`;
- $info->{STATUS} = "SUCCESS";
- $info->{INFO} = "The process (PID: $pid) has been stopped.";
+ my $invalid;
+ if($cluster) {
+ $invalid = clusterDeleteJob($pid);
+ if( !$invalid ){
+ `echo "\n*** [$time] EDGE_UI: This project has been interrupted. ***" |tee -a $proj_dir/process.log >> $proj_dir/process_current.log`;
+ $info->{STATUS} = "SUCCESS";
+ $info->{INFO} = "The cluster job (JOB ID: $pid) has been stopped.";
+ }
+ } else {
+ $invalid = &killProcess($pid);
+ if( !$invalid || $projStatus eq "unstarted"){
+ `echo "\n*** [$time] EDGE_UI: This project has been interrupted. ***" |tee -a $proj_dir/process.log >> $proj_dir/process_current.log`;
+ $info->{STATUS} = "SUCCESS";
+ $info->{INFO} = "The process (PID: $pid) has been stopped.";
+ }
}
}
else{
@@ -219,7 +298,7 @@ elsif( $action eq 'interrupt' ){
}
}
elsif( $action eq 'rerun' ){
- if( $sys->{user_management} && !$permission->{$action} ){
+ if( $umSystemStatus && !$permission->{$action} ){
$info->{INFO} = "ERROR: Permission denied. Only project owner can perform this action.";
&returnStatus();
}
@@ -227,45 +306,72 @@ elsif( $action eq 'rerun' ){
$info->{STATUS} = "FAILURE";
$info->{INFO} = "Failed to rerun project $real_name.";
- my $pid = $name2pid->{$pname};
+ if( -e "$proj_dir/config.txt.bak"){
+ cleanProjectForNewConfig();
+ }
+
+ my $pid = $name2pid->{$pname} || $name2pid->{$projCode};
if( ! defined $pid ){
- my $cmd = "";
- open LOG, "$proj_dir/process.log" or die "Can't open process log:$!.";
- foreach(){
- chomp;
- if( /runPipeline -c / ){
- $cmd = $_;
+ if($cluster) {
+ my $cluster_job_script = "$proj_dir/clusterSubmit.sh";
+ if(!-e $cluster_job_script) {
+ $info->{INFO} = "Failed to restart this project. File $cluster_job_script not found.";
+ } else {
+ &updateDBProjectStatus($pname,"running") if ($username && $password);
+ my ($job_id,$error) = clusterSubmitJob($cluster_job_script,$cluster_qsub_options);
+ if($error) {
+ $info->{INFO} = "Failed to restart this project: $error";
+ } else {
+ $info->{STATUS} = "SUCCESS";
+ $info->{INFO} = "Project $real_name has been restarted (JOB ID: $job_id).";
+ $info->{PID} = $job_id;
+ `echo "\n*** [$time] EDGE_UI: This project has been restarted. (unstarted) ***" |tee -a $proj_dir/process.log >> $proj_dir/process_current.log`;
+ }
}
- }
- close LOG;
- my ($numcpu) = $cmd =~ /-cpu (\d+)/;
- my $run = &availableToRun($numcpu);
- if (!$run){
- my $time = strftime "%F %X", localtime;
- `echo "\n*** [$time] EDGE_UI: This project is queued. ***" |tee -a $proj_dir/process.log >> $proj_dir/process_current.log`;
- `echo "$cmd" >> $proj_dir/process.log`;
- `echo "\n*** [$time] EDGE_UI: Project unstarted ***" >> $proj_dir/process.log`;
- $info->{INFO} = "The server does not have enough CPU available to run this job. The job is queued";
- &returnStatus();
- }
- if( $cmd ){
- chdir($proj_dir);
- my $newpid = open RUNPIPLINE, "-|", "$cmd > $proj_dir/process_current.log 2>&1 &" or die $!;
- close RUNPIPLINE;
- if( $newpid ){
- $newpid++;
- $info->{STATUS} = "SUCCESS";
- $info->{INFO} = "Project $real_name has been restarted (PID: $newpid).";
- $info->{PID} = $newpid;
+ } else {
+ my $cmd = "";
+ open LOG, "$proj_dir/process.log" or die "Can't open process log:$!.";
+ foreach(){
+ chomp;
+ if( /runPipeline -c / ){
+ $cmd = $_;
+ }
+ }
+ close LOG;
+ my ($numcpu) = $cmd =~ /-cpu (\d+)/;
+ my $run = &availableToRun($numcpu);
+ if (!$run){
+ my $time = strftime "%F %X", localtime;
+ `echo "\n*** [$time] EDGE_UI: This project is queued. ***" |tee -a $proj_dir/process.log >> $proj_dir/process_current.log`;
+ `echo "$cmd" >> $proj_dir/process.log`;
+ `echo "\n*** [$time] EDGE_UI: Project unstarted ***" >> $proj_dir/process.log`;
+ $info->{INFO} = "The server does not have enough CPU available to run this job. The job is queued";
+ &updateDBProjectStatus($pname,"unstarted") if ($username && $password);
+ &returnStatus();
+ }
+ if( $cmd ){
+ chdir($proj_dir);
+ #remove cached report/status
+ #`rm -f $proj_dir/.run.complete.status.json`;
+ `rm -f $proj_dir/HTML_Report/.complete_report_web`;
+ my $newpid = open RUNPIPLINE, "-|", "$cmd > $proj_dir/process_current.log 2>&1 &" or die $!;
+ close RUNPIPLINE;
+ if( $newpid ){
+ $newpid++;
+ $info->{STATUS} = "SUCCESS";
+ $info->{INFO} = "Project $real_name has been restarted (PID: $newpid).";
+ $info->{PID} = $newpid;
+ &updateDBProjectStatus($pname,"running") if ($username && $password);
+ }
+ else{
+ $info->{INFO} = "Failed to restart this project.";
+ }
}
else{
- $info->{INFO} = "Failed to restart this project.";
+ $info->{INFO} = "Failed to restart this project. No runPipeline command found.";
}
}
- else{
- $info->{INFO} = "Failed to restart this project. No runPipeline command found.";
- }
}
else{
$info->{INFO} = "Project $real_name can't be restarted because it's still running (PID: $pid).";
@@ -305,33 +411,294 @@ elsif( $action eq 'archive' ){
$info->{INFO} = "Start archiving project $real_name.";
}
if ($username && $password){
- &updateDBProjectStatus($pname,"archive");
+ &updateDBProjectStatus($pname,"archived");
`rm -f $user_proj_dir`;
`rm -f $input_dir/public/projects/${real_name}_$pname`;
`rm -f $input_dir/*/SharedProjects/${real_name}_$pname`;
}
}
}
+elsif( $action eq 'tarproj'){
+ if( $sys->{user_management} && !$permission->{$action} ){
+ $info->{INFO} = "ERROR: Permission denied. Only project owner can perform this action.";
+ &returnStatus();
+ }
+
+ my $tarFile = "$proj_dir/$real_name.tgz";
+ my $tarDir = "$proj_dir";
+ if ($username && $password){
+ $tarFile = "$proj_dir/${real_name}_$pname.tgz";
+ $tarDir = "$out_dir/${real_name}_$pname";
+ }
+ $tarDir =~ s/$out_dir//;
+ $tarDir =~ s/^\///;
+ (my $tarLink = $tarFile ) =~ s/$www_root//;
+ $tarLink =~ s/^\///;
+ $info->{STATUS} = "FAILURE";
+ $info->{INFO} = "Failed to tar project $real_name $tarLink";
+ chdir $out_dir;
+ if ( ! -e "$proj_dir/.tarfinished" || ! -e $tarLink){
+ `ln -s $proj_dir $tarDir` if ($username && $password);
+ my $cmd = "tar --exclude=\"*gz\" --exclude=\"*.sam\" --exclude=\"*.bam\" --exclude=\"*.fastq\" -cvzf $tarFile $tarDir/* ; touch $proj_dir/.tarfinished ";
+ my $pid;
+ if (@ARGV){
+ $pid=`$cmd`;
+ }else{
+ $pid = open TARPROJ, "-|", $cmd or die $!;
+ close TARPROJ;
+ $pid++;
+ }
+ if( $pid ){
+ unlink $tarDir if ($username && $password);
+ $info->{STATUS} = "SUCCESS";
+ $info->{INFO} = "$real_name compressed file is ready to ";
+ $info->{LINK} = "download";
+ }else{
+ $info->{INFO} = "Project $real_name tar file existed";
+ }
+ }
+}
+elsif( $action eq 'getcontigbytaxa'){
+ if( $sys->{user_management} && !$permission->{$action} ){
+ $info->{INFO} = "ERROR: Permission denied. Only project owner can perform this action.";
+ &returnStatus();
+ }
+ my $assemble_outdir="$proj_dir/AssemblyBasedAnalysis";
+ my $taxa_outdir="$assemble_outdir/Taxonomy";
+ (my $relative_taxa_outdir=$taxa_outdir) =~ s/$www_root//;
+ (my $out_fasta_name = $taxa_for_contig_extract) =~ s/[ .']/_/;
+ $out_fasta_name = "$real_name"."_"."$out_fasta_name.fasta";
+ my $cmd = "$EDGE_HOME/scripts/contig_classifier_by_bwa/extract_fasta_by_taxa.pl -fasta $assemble_outdir/${real_name}_contigs.fa -csv $taxa_outdir/$real_name.ctg_class.top.csv -taxa \"$taxa_for_contig_extract\" -rank genus > $taxa_outdir/$out_fasta_name";
+ $info->{STATUS} = "FAILURE";
+ $info->{INFO} = "Failed to extract $taxa_for_contig_extract contig fasta";
+
+ if ( -s "$taxa_outdir/$out_fasta_name.fasta"){
+ $info->{STATUS} = "SUCCESS";
+ $info->{PATH} = "$relative_taxa_outdir/$out_fasta_name";
+ }else{
+ my $pid = open EXTRACTCONTIG, "-|", $cmd or die $!;
+ close EXTRACTCONTIG;
+ $pid++;
+
+ if( $pid ){
+ $info->{STATUS} = "SUCCESS";
+ $info->{PATH} = "$relative_taxa_outdir/$out_fasta_name";
+ }
+ }
+}
+elsif( $action eq 'getreadsbytaxa'){
+ if( $sys->{user_management} && !$permission->{$action} ){
+ $info->{INFO} = "ERROR: Permission denied. Only project owner can perform this action.";
+ &returnStatus();
+ }
+ my $read_type="allReads";
+ my $reads_fastq="$proj_dir/ReadsBasedAnalysis/Taxonomy/$read_type.fastq";
+ my $readstaxa_outdir="$proj_dir/ReadsBasedAnalysis/Taxonomy/report/1_$read_type/$cptool_for_reads_extract";
+
+ if ( -e "$proj_dir/ReadsBasedAnalysis/UnmappedReads/Taxonomy"){
+ $read_type="UnmappedReads";
+ $reads_fastq="$proj_dir/ReadsBasedAnalysis/$read_type/Taxonomy/$read_type.fastq";
+ $readstaxa_outdir="$proj_dir/ReadsBasedAnalysis/$read_type/Taxonomy/report/1_$read_type/$cptool_for_reads_extract";
+
+ }
+ (my $relative_taxa_outdir=$readstaxa_outdir) =~ s/$www_root//;
+ (my $out_fasta_name = $taxa_for_contig_extract) =~ s/[ .']/_/g;
+ my $extract_from_original_fastq = ($cptool_for_reads_extract =~ /gottcha/i)? " -fastq $reads_fastq " : "";
+ $out_fasta_name = "$real_name"."_"."$cptool_for_reads_extract"."_"."$out_fasta_name";
+ my $cmd = "$EDGE_HOME/scripts/microbial_profiling/script/bam_to_fastq_by_taxa.pl -rank species -name \"$taxa_for_contig_extract\" -prefix $readstaxa_outdir/$out_fasta_name -se -zip $extract_from_original_fastq $readstaxa_outdir/${read_type}-$cptool_for_reads_extract.bam 1>>$readstaxa_outdir/ReadsExtractLog.txt";
+ #GOTTCHA2 Only
+ if( $cptool_for_reads_extract =~ /gottcha2/i ){
+ $cmd = "$EDGE_HOME/thirdParty/gottcha2/gottcha.py -s $readstaxa_outdir/*.sam -m extract -x $taxa_for_contig_extract -c > $readstaxa_outdir/$out_fasta_name.fastq; cd $readstaxa_outdir; zip $out_fasta_name.fastq.zip $out_fasta_name.fastq; rm $out_fasta_name.fastq";
+ }
+ $info->{STATUS} = "FAILURE";
+ $info->{INFO} = "Failed to extract $taxa_for_contig_extract reads fastq";
+
+ if ( -s "$readstaxa_outdir/$out_fasta_name.fastq.zip"){
+ $info->{STATUS} = "SUCCESS";
+ $info->{PATH} = "$relative_taxa_outdir/$out_fasta_name.fastq.zip";
+ }elsif ( ! -e "$readstaxa_outdir/${read_type}-$cptool_for_reads_extract.bam" && ! glob("$readstaxa_outdir/*.sam") ){
+ $info->{INFO} = "The result bam does not exist.";
+ $info->{INFO} .= "If the project is older than $keep_days days, it has been deleted." if ($keep_days);
+ }else
+ {
+ my $pid = open EXTRACTREADS, "-|", $cmd or die $!;
+ close EXTRACTREADS;
+ $pid++;
+
+ if( $pid ){
+ $info->{STATUS} = "SUCCESS";
+ $info->{PATH} = "$relative_taxa_outdir/$out_fasta_name.fastq.zip";
+ }
+ }
+}
elsif( $action eq 'share' || $action eq 'unshare' ){
if( $sys->{user_management} && !$permission->{$action} ){
$info->{INFO} = "ERROR: Permission denied. Only project owner can perform this action.";
&returnStatus();
}
- &shareProject($pname,$shareEmail,$action);
+ &shareProject($pname,$proj_dir,$shareEmail,$action);
+ my $owner = $list->{$pname}->{OWNER};
+ if ($action eq 'share'){
+ my $msg = "$owner has shared EDGE project $real_name to you. You can login to $protocol//$domain/edge_ui/ and see the project. Or click link below.\n\n $protocol//$domain/edge_ui/?proj=$projCode\n";
+ my $subject = "EDGE project $real_name";
+ &sendMail($username,$shareEmail,$subject,$msg);
+ }
}
elsif( $action eq 'publish' || $action eq 'unpublish'){
- print STDERR "USERMANAGMENT: $sys->{user_management}; $action: $permission->{$action}";
+ #print STDERR "USERMANAGMENT: $sys->{user_management}; $action: $permission->{$action}";
if( $sys->{user_management} && !$permission->{$action} ){
$info->{INFO} = "ERROR: Permission denied. Only project owner can perform this action.";
&returnStatus();
}
&publishProject($pname,$action);
my $public_proj_dir = "$input_dir/public/projects/${real_name}_$pname";
- `ln -sf $out_dir/$pname $public_proj_dir` if ($action eq 'publish' && ! -e "$public_proj_dir");
+ `ln -sf $proj_dir $public_proj_dir` if ($action eq 'publish' && ! -e "$public_proj_dir");
`rm -f $public_proj_dir` if ($action eq 'unpublish');
}
+elsif( $action eq 'compare'){
+ my $compare_out_dir = "$out_dir/ProjectComparison/". md5_hex(join ('',@projCodes));
+ my $projects = join(",",map { "$out_dir/$_" } @projCodes);
+ (my $relative_outdir=$compare_out_dir) =~ s/$www_root//;
+ $info->{PATH} = "$relative_outdir/compare_project.html";
+ $info->{INFO} = "The comparison result is available here";
+ if ( -s "$compare_out_dir/compare_project.html"){
+ $info->{STATUS} = "SUCCESS";
+ }else{
+ my $cmd = "$EDGE_HOME/scripts/compare_projects/compare_projects.pl -out_dir $compare_out_dir -projects $projects";
+ my $pid = open COMPARE, "-|", $cmd or die $!;
+ close COMPARE;
+ $pid++;
+ if( $pid ){
+ my $err = `grep "No Taxonomy Classification" $compare_out_dir/log.txt`;
+ if ($err){
+ $info->{INFO} = "Error: $err";
+ `rm -rf $compare_out_dir`;
+ &returnStatus();
+ }else{
+ $info->{STATUS} = "SUCCESS";
+ }
+ }
+ }
+}elsif($action eq 'contigblast'){
+ my $blast_out_dir="$proj_dir/AssemblyBasedAnalysis/ContigBlast";
+ my $contig_file="$proj_dir/AssemblyBasedAnalysis/${real_name}_contigs.fa";
+ my $nt_db="$EDGE_HOME/database/nt/nt";
+ my $cpu = `grep -a "cpu=" $proj_dir/config.txt | awk -F"=" '{print \$2}'`;
+ chomp $cpu;
+ $blast_params =~ s/-num_threads\s+\d+//;
+ `mkdir -p $blast_out_dir`;
+ (my $relative_outdir=$blast_out_dir) =~ s/$www_root//;
+ $info->{PATH} = "$relative_outdir/$contig_id.blastNT.html";
+ $info->{INFO} = "The comparison result is available here";
+ if ( -s "$blast_out_dir/$contig_id.blastNT.html"){
+ $info->{STATUS} = "SUCCESS";
+ }else{
+ my $cmd = "$EDGE_HOME/scripts/get_seqs.pl $contig_id $contig_file | blastn -query - -db $nt_db $blast_params -out $blast_out_dir/$contig_id.blastNT.html -num_threads $cpu -html ";
+
+ my $pid = open BLAST, "-|", $cmd or die $!;
+ close BLAST;
+
+ if( $pid ){
+ #parent
+ $pid++;
+ $info->{STATUS} = "SUCCESS";
+ }else{
+ #child
+ close STDOUT;
+ }
+ }
+}elsif( $action eq 'metadata-delete' ){
+##sample metatdata
+ $info->{STATUS} = "SUCCESS";
+ $info->{INFO} = "Project $real_name sample metadata has been deleted.";
+ my $metadata = "$proj_dir/sample_metadata.txt";
+ if( -e $metadata ){
+ if(-w $metadata) {
+ my $bsveId = `grep -a "bsve_id=" $metadata | awk -F'=' '{print \$2}'`;
+ chomp $bsveId;
+ `rm -f $metadata`;
+ if($bsveId) {#keep bsve_id
+ open OUT, ">$metadata";
+ print OUT "bsve_id=$bsveId"."\n";
+ close OUT;
+ }
+ } else {
+ $info->{STATUS} = "FAILURE";
+ $info->{INFO} = "Failed to delete the sample metadata.";
+ }
+ }
+} elsif($action eq 'metadata-bsveadd') {
+ if( $sys->{user_management} && !$permission->{metadata} ){
+ $info->{INFO} = "ERROR: Permission denied. Only project owner can perform this action.";
+ &returnStatus();
+ }
+ if(pushSampleMetadata("add", $proj_dir, $sys)) {
+ $info->{STATUS} = "SUCCESS";
+ $info->{INFO} = "Project $real_name sample metadata has been submitted to the BSVE server.";
+ } else {
+ $info->{STATUS} = "FAILURE";
+ $info->{INFO} = "Failed to submit the sample metadata to the BSVE server";
+ }
+
+} elsif($action eq 'metadata-bsveupdate') {
+ if(pushSampleMetadata("update", $proj_dir, $sys)) {
+ $info->{STATUS} = "SUCCESS";
+ $info->{INFO} = "Project $real_name sample metadata has been updated in the BSVE server.";
+ } else {
+ $info->{STATUS} = "FAILURE";
+ $info->{INFO} = "Failed to update the sample metadata in the BSVE server";
+ }
+
+} elsif($action eq 'metadata-bsvedelete') {
+ if(pushSampleMetadata( "delete", $proj_dir, $sys)) {
+ $info->{STATUS} = "SUCCESS";
+ $info->{INFO} = "Project $real_name sample metadata has been deleted from the BSVE server.";
+ } else {
+ $info->{STATUS} = "FAILURE";
+ $info->{INFO} = "Failed to delete the sample metadata from the BSVE server.";
+ }
+}
+#END sample metadata
+elsif($action eq 'define-gap-depth'){
+ my $gap_depth_cutoff = ($opt{"gap-depth-cutoff"})? $opt{"gap-depth-cutoff"}:0;
+ my $gap_out_dir="$proj_dir/ReferenceBasedAnalysis/readsMappingToRef";
+ my $gap_outfile="$gap_out_dir/readsToRef_d$gap_depth_cutoff.gaps";
+ my $gff_file="$proj_dir/Reference/reference.gff";
+ my $gap_analysisOutfile="$gap_out_dir/Gap_d${gap_depth_cutoff}VSReference.report.txt";
+ my $gap_analysisOutfile_json="$gap_out_dir/Gap_d${gap_depth_cutoff}VSReference.report.json";
+ (my $relative_gap_out_dir=$gap_out_dir) =~ s/$www_root//;
+
+ if ( $gap_depth_cutoff == 0 ){
+ $info->{STATUS} = "SUCCESS";
+ $info->{PATH} = "$relative_gap_out_dir/GapVSReference.report.json";
+ &returnStatus();
+ }
+ if ( -s $gap_analysisOutfile_json){
+ $info->{STATUS} = "SUCCESS";
+ $info->{PATH} = "$relative_gap_out_dir/Gap_d${gap_depth_cutoff}VSReference.report.json";
+ &returnStatus();
+ }
+ my $cmd;
+ unless ( -s $gap_outfile){
+ opendir( my $dh, $gap_out_dir);
+ my @coverage_file = grep { /coverage$/ && -f "$gap_out_dir/$_" } readdir($dh);
+ closedir $dh;
+ foreach my $file(@coverage_file){
+ $cmd .= "$EDGE_HOME/scripts/gap_count.pl $gap_out_dir/$file $gap_depth_cutoff >> $gap_outfile ;";
+ }
+ }
+ $cmd .= "$EDGE_HOME/scripts/gap_analysis.pl -gff $gff_file -gap $gap_outfile > $gap_analysisOutfile;";
+ $cmd .= "$EDGE_HOME/scripts/tab2Json_for_dataTable.pl -project_dir $proj_dir -limit 0 -mode ref_gap $gap_analysisOutfile > $gap_analysisOutfile_json";
+ my $pid = open GAP, "-|",$cmd or die $!;
+ close GAP;
+ if( $pid ){
+ $pid++;
+ $info->{STATUS} = "SUCCESS";
+ $info->{PATH} = "$relative_gap_out_dir/Gap_d${gap_depth_cutoff}VSReference.report.json";
+ }
+}
&returnStatus();
######################################################
@@ -378,18 +745,19 @@ sub checkProjVital {
}
sub availableToRun {
- my $num_cpu = shift;
- my $cpu_been_used = 0;
- return 0 if ($num_cpu > $sys->{edgeui_tol_cpu});
- if( $sys->{edgeui_auto_queue} && $sys->{edgeui_tol_cpu} ){
- foreach my $pid ( keys %$vital ){
- $cpu_been_used += $vital->{$pid}->{CPU};
- return 0 if (($cpu_been_used + $num_cpu) > $sys->{edgeui_tol_cpu});
- }
- }
- return 1;
+ my $num_cpu = shift;
+ my $cpu_been_used = 0;
+ if( $sys->{edgeui_auto_queue} && $sys->{edgeui_tol_cpu} ){
+ foreach my $pid ( keys %$vital ){
+ $cpu_been_used += $vital->{$pid}->{CPU};
+ return 0 if (($cpu_been_used + $num_cpu) > $sys->{edgeui_tol_cpu});
+ }
+ return 0 if ($num_cpu > $sys->{edgeui_tol_cpu});
+ }
+ return 1;
}
+
sub returnStatus {
my $json = "{}";
$json = to_json($info) if $info;
@@ -427,10 +795,50 @@ sub getProjNameFromDB{
$info->{INFO} .= $result->{error_msg}."\n";;
}
else{
- return $result->{name};
+ return ($result->{name} , $result->{code}, $result->{status});
}
}
+sub renameProject{
+ my $project_name = $new_proj_name;
+ my $project_description = $new_proj_desc;
+ my $pnameID = $pname;
+ #adjust txt files. (config.txt and process.log )
+ my $config_file = $proj_dir."/config.txt";
+ tie my @array, 'Tie::File', $config_file or die;
+
+ my %data = (
+ email => $username,
+ password => $password,
+ project_id => $pnameID,
+ new_project_name => $project_name,
+ new_description => $project_description,
+ );
+
+ #Encode the data structure to JSON
+ # #interacts with the java api to access the sql DB tables
+ my $data = to_json(\%data);
+ #w Set the request parameters
+ my $url = $um_url ."WS/project/update";
+ my $browser = LWP::UserAgent->new;
+ my $req = PUT $url;
+ $req->header('Content-Type' => 'application/json');
+ $req->header('Accept' => 'application/json');
+ #must set this, otherwise, will get 'Content-Length header value was wrong, fixed at...' warning
+ $req->header( "Content-Length" => length($data) );
+ $req->content($data);
+
+ my $response = $browser->request($req);
+ my $result_json = $response->decoded_content;
+ my $result = from_json($result_json);
+
+ if ($result->{status})
+ {
+ $info->{STATUS} = "SUCCESS";
+ $info->{INFO} .= " Project has been ${action}d to $project_name.";
+ }
+}
+
sub updateDBProjectStatus{
my $project = shift;
my $status = shift;
@@ -461,8 +869,28 @@ sub updateDBProjectStatus{
}
}
+sub sendMail{
+ my $sender=shift;
+ my $recipients=shift;
+ my $subject=shift;
+ my $msg=shift;
+ $recipients =~ s/ //g;
+ $recipients = join(',', grep (!/$sender/, split(',',$recipients)));
+ if (`which sendmail`){
+ open(MAIL, "|sendmail -t") or die "$!\n";
+ print MAIL "To: $recipients\n";
+ print MAIL "From: $sender\n";
+ print MAIL "Subject: $subject\n\n";
+ # print MAIL "Content-Type: text/html; charset=ISO-8859-1\n";
+ # print MAIL "Content-Disposition: inline\n";
+ print MAIL "$msg";
+ close MAIL;
+ }
+}
+
sub shareProject{
my $project=shift;
+ my $proj_dir=shift;
my $email=shift;
my $action =shift;
$email =~ s/ //g;
@@ -498,7 +926,7 @@ sub shareProject{
my $shared_proj_dir = "$user_dir/SharedProjects/${real_name}_$project";
if ( $action eq "share"){
`mkdir -p $user_dir/SharedProjects`;
- `ln -sf $out_dir/$project $shared_proj_dir` if (!-e $shared_proj_dir);
+ `ln -sf $proj_dir $shared_proj_dir` if (!-e $shared_proj_dir);
}else{# unshare
`rm -f $shared_proj_dir` if ( -e $shared_proj_dir);
}
@@ -593,16 +1021,128 @@ sub getUserProjFromDB{
{
my $id = $hash_ref->{id};
my $project_name = $hash_ref->{name};
+ my $projCode = $hash_ref->{code};
my $status = $hash_ref->{status};
next if ($status =~ /delete/i);
- next if (! -r "$out_dir/$id/process.log");
+ next if (! -r "$out_dir/$id/process.log" && ! -r "$out_dir/$projCode/process.log");
$list->{$id}->{PROJNAME} = $id;
$list->{$id}->{REAL_PROJNAME} = $project_name;
- $list->{$id}->{OWNER} = $hash_ref->{owner_firstname};
+ $list->{$id}->{PROJCODE} = $projCode;
+ $list->{$id}->{OWNER} = "$hash_ref->{owner_firstname} $hash_ref->{owner_lastname}";
$list->{$id}->{OWNER_EMAIL} = $hash_ref->{owner_email};
$list->{$id}->{PROJ_TYPE} = $hash_ref->{type};
}
return $list;
}
+sub scanProjToList{
+ my $out_dir = shift;
+ my $pname = shift;
+ my $config_file;
+ my $processLog;
+ my ($projid,$projCode,$projName,$projStatus);
+ if ($pname && -d "$out_dir/$pname"){
+ $config_file = "$out_dir/$pname/config.txt";
+ $processLog = "$out_dir/$pname/process_current.log";
+ }else{
+ $config_file = `grep -a "projid=$pname" $out_dir/*/config.txt | awk -F':' '{print \$1}'`;
+ }
+ chomp $config_file;
+ return ($projName,$projCode,$projStatus) if ( ! -e $config_file);
+ if ( -r "$processLog"){
+ open (my $fh, $processLog);
+ while(<$fh>){
+ if (/queued/){
+ $projStatus="unstarted";
+ last;
+ }
+ if (/^All Done/){
+ $projStatus="finished";
+ }
+ }
+ close $fh;
+ }
+ open (my $fh, $config_file) or die "Cannot read $config_file\n";
+ while(<$fh>){
+ last if (/^\[Down/);
+ $projid=$1 if (/^projid=(\S+)/);
+ $projCode=$1 if (/^projcode=(\S+)/);
+ $projName=$1 if (/^projname=(\S+)/);
+
+ }
+ close $fh;
+ return ($projName,$projid,$projStatus);
+}
+
+sub getSystemUsage {
+ my $mem = `vmstat -s | awk '\$0 ~/total memory/ {total=\$1 } \$0 ~/free memory/ {free=\$1} \$0 ~/buffer memory/ {buffer=\$1} \$0 ~/cache/ {cache=\$1} END{print (total-free-buffer-cache)/total*100}'`;
+ my $cpu = `top -bn1 | grep load | awk '{printf "%.1f", \$(NF-2)}'`;
+ my $disk = `df -h $out_dir | tail -1 | awk '{print \$5}'`;
+ $disk= `df -h $out_dir | tail -1 | awk '{print \$4}'` if ($disk !~ /\%/);
+ $cpu = $cpu/$sys->{edgeui_tol_cpu}*100;
+ $disk =~ s/\%//;
+ if( $mem || $cpu || $disk ){
+ $mem = sprintf "%.1f", $mem;
+ $cpu = sprintf "%.1f", $cpu;
+ $disk = sprintf "%.1f", $disk;
+ return ($mem,$cpu,$disk);
+ }
+ else{
+ return (0,0,0);
+ }
+}
+
+sub cleanProjectForNewConfig {
+ my $module_ctl;
+ $module_ctl->{"Qiime analysis"} ->{"general"} = "$proj_dir/QiimeAnalysis/runQiimeAnalysis.finished";
+ $module_ctl->{"Download SRA"} ->{"general"} = "$proj_dir/SRA_Download/DownloadSRA.finished";
+ $module_ctl->{"ProPhage Detection"} ->{"general"} = "$proj_dir/AssemblyBasedAnalysis/Prophage/phageFinder.finished";
+ $module_ctl->{"Count Fastq"} ->{"general"} = "$proj_dir/QcReads/countFastq.finished";
+ $module_ctl->{"Quality Trim and Filter"} ->{"general"} = "$proj_dir/QcReads/runQC.finished";
+ $module_ctl->{"Host Removal"} ->{"general"} = "$proj_dir/HostRemoval/*/run*.finished"; # system("rm -f $outputDir/run${prefix}Removal.finished");
+ $module_ctl->{"Assembly"} ->{"Provided"} = "$proj_dir/AssemblyBasedAnalysis/processProvideContigs.finished";
+ $module_ctl->{"Assembly"} ->{"SPAdes"} = "$proj_dir/AssemblyBasedAnalysis/runSPAdesAssembly.finished"; #system("rm -f $outputDir/runAPAdesAssembly.finished");
+ $module_ctl->{"Assembly"} ->{"Idba"} = "$proj_dir/AssemblyBasedAnalysis/runIdbaAssembly.finished";
+ $module_ctl->{"Assembly"} ->{"Idba"} = "$proj_dir/AssemblyBasedAnalysis/runIdbaAssembly.finished";
+ $module_ctl->{"Assembly"} ->{"general"} = "$proj_dir/AssemblyBasedAnalysis/runAssembly.finished";
+ $module_ctl->{"Reads Mapping To Contigs"} ->{"general"} = "$proj_dir/AssemblyBasedAnalysis/readsMappingToContig/runReadsToContig.finished";
+ $module_ctl->{"Reads Mapping To Reference"} ->{"general"} = "$proj_dir/ReadsBasedAnalysis/readsMappingToRef/runReadsToGenome.finished";
+ $module_ctl->{"Reads Mapping To Reference"} ->{"UnmappedReads"} = "$proj_dir/ReferenceBasedAnalysis/UnmappedReads/retrieveUnmappedReads.finished";
+ $module_ctl->{"Reads Taxonomy Classification"} ->{"AllReads"} = "$proj_dir/ReadsBasedAnalysis/Taxonomy/taxonomyAssignment.finished";
+ $module_ctl->{"Reads Taxonomy Classification"} ->{"UnmappedReads"} = "$proj_dir/ReadsBasedAnalysis/UnmappedReads/Taxonomy/taxonomyAssignment.finished";
+ $module_ctl->{"Contigs Mapping To Reference"} ->{"general"} = "$proj_dir/AssemblyBasedAnalysis/contigMappingToRef/runContigToGenome.finished";
+ $module_ctl->{"Variant Analysis"} ->{"ReadsBased"} = "$proj_dir/ReadsBasedAnalysis/contigMappingToRef/variantAnalysis.finished";
+ $module_ctl->{"Variant Analysis"} ->{"AssemblyBased"} = "$proj_dir/AssemblyBasedAnalysis/contigMappingToRef/variantAnalysis.finished";
+ $module_ctl->{"Contigs Taxonomy Classification"}->{"general"} = "$proj_dir/AssemblyBasedAnalysis/Taxonomy/ContigsTaxonomy.finished";
+ $module_ctl->{"Contigs Blast"} ->{"general"} = "$proj_dir/AssemblyBasedAnalysis/Blast/ContigsBlast.finished";
+ $module_ctl->{"Contigs Annotation"} ->{"general"} = "$proj_dir/AssemblyBasedAnalysis/Annotation/runAnnotation.finished";
+ $module_ctl->{"Phylogenetic Analysis"} ->{"general"} = "$proj_dir/SNP_Phylogeny/SNPtree.finished";
+ $module_ctl->{"Phylogenetic Analysis"} ->{"SRA"} = "$proj_dir/SNP_Phylogeny/SRAreads/download.finished";
+ $module_ctl->{"Specialty Genes Profiling"} ->{"general"} = "$proj_dir/.runSpecialtyGenesProfiling.finished";
+ $module_ctl->{"Specialty Genes Profiling"} ->{"ReadsBased"} = "$proj_dir/ReadsBasedAnalysis/SpecialtyGenes/runSpecialtyGenesProfiling.finished";
+ $module_ctl->{"Specialty Genes Profiling"} ->{"AssemblyBased"} = "$proj_dir/AssemblyBasedAnalysis/SpecialtyGenes/runSpecialtyGenesProfiling.finished";
+ $module_ctl->{"Primer Validation"} ->{"general"} = "$proj_dir/AssayCheck/pcrDesign.finished";
+ $module_ctl->{"Primer Design"} ->{"general"} = "$proj_dir/AssayCheck/pcrDesign.finished";
+ $module_ctl->{"Generate JBrowse Tracks"} ->{"general"} = "$proj_dir/JBrowse/writeJBrowseInfo.finished";
+ $module_ctl->{"HTML Report"} ->{"general"} = "$proj_dir/HTML_Report/writeHTMLReport.finished";
+ my $new_config = &getSysParamFromConfig( "$proj_dir/config.txt" );
+ my $old_config = &getSysParamFromConfig( "$proj_dir/config.txt.bak" );
+
+ foreach my $module ( keys %$new_config ){
+ foreach my $param ( keys %{$new_config->{$module}} ){
+ #if one of the parameter in the module changed, reset the whole module
+ if( $new_config->{$module}->{$param} ne $old_config->{$module}->{$param} ){
+ foreach my $task ( keys %{$module_ctl->{$module}} ){
+ `rm -f $module_ctl->{$module}->{$task}`;
+ }
+ last;
+ }
+ }
+ }
+
+ #remove old config file
+ `rm -f "$proj_dir/config.txt.bak"`;
+
+ return;
+}
diff --git a/edge_ui/cgi-bin/edge_config.tmpl b/edge_ui/cgi-bin/edge_config.tmpl
index 4bb98bd5..682f9ea8 100755
--- a/edge_ui/cgi-bin/edge_config.tmpl
+++ b/edge_ui/cgi-bin/edge_config.tmpl
@@ -1,29 +1,14 @@
-[system]
-##edge_ui environmental vars
-edgeui_wwwroot=%EDGE_HOME%/edge_ui
-edgeui_input=%EDGE_HOME%/edge_ui/EDGE_input
-edgeui_output=%EDGE_HOME%/edge_ui/EDGE_output
-edgeui_archive=
-edgeui_tol_cpu=%TOTAL_NUM_CPU%
-edgeui_auto_queue=1
-edgeui_auto_run=1
-debug=0
+[project]
##number of cpu
cpu=
##store output to [outpath]/[projname]
outpath=
projname=
projdesc=
-## configure user management system 1: with 0: without
-user_management=0
-## if user management system is not in the same domain with edge
-## ex: http://www.someother.com/userManagement
-edge_user_management_url=
-## configure user upload function 1: able 0: disable
-user_upload=1
-## max file size for user upload
-user_upload_maxFileSize='5gb'
-user_social_login=0
+projid=
+projcode=
+projowner=
+projrunhost=
[Download Fastq]
DoFastqDownload=auto
@@ -72,30 +57,41 @@ similarity=
DoAssembly=
assembledContigs=
minContigSize=
-## spades or idba_ud
+## spades or idba_ud or megahit
assembler=
-idbaOptions="--pre_correction --mink --maxk --step --min_contig "
+idbaOptions="--pre_correction --mink --maxk --step --min_contig "
## for spades
-singleCellMode=
-pacbioFile=
-nanoporeFile=
+SpadesSingleCellMode=
+SpadesMetagenomeMode=
+SpadesPacbioFile=
+SpadesNanoporeFile=
+## for megahit
+megahit_preset=
[Reads Mapping To Contigs]
# Reads mapping to contigs
DoReadsMappingContigs=auto
+## support bowtie or bwa
+r2c_aligner=
+r2c_aligner_options=
+r2c_extract_unmapped=
[Reads Mapping To Reference]
# Reads mapping to reference
DoReadsMappingReference=
-bowtieOptions=
+## support bowtie or bwa
+r2g_aligner=
+r2g_aligner_options=
window_size=1000
step_size=200
reference=
MapUnmappedReads=
+r2g_extract_mapped=
+r2g_extract_unmapped=
[Reads Taxonomy Classification]
## boolean, 1=yes, 0=no
-DoTaxonomy=
+DoReadsTaxonomy=
## If reference genome exists, only use unmapped reads to do Taxonomy Classification. Turn on AllReads=1 will use all reads instead.
AllReads=
enabledTools=
@@ -130,7 +126,7 @@ annotateProgram=
annotateSourceGBK=
[ProPhage Detection]
-DoProPhageDetection=
+DoProPhageDetection=auto
[Phylogenetic Analysis]
DoSNPtree=
@@ -146,6 +142,18 @@ SNPrefGenome=
treeMaker=
## SRA accessions ByrRun, ByExp, BySample, ByStudy
SNP_SRA_ids=
+PhaMEbootstrap=
+PhaMEbootstrap_num=
+
+
+[Specialty Genes Profiling]
+DoReadsSpecialtyGenes=
+DoORFsSpecialtyGenes=
+SpecialtyGenesSearchTool=
+ShortBREDMinPercIdentity=
+ShortBREDMinPercLength=
+
+
[Primer Validation]
DoPrimerValidation=
@@ -169,7 +177,28 @@ tm_diff=
top=
[Generate JBrowse Tracks]
-DoJBrowse=1
+DoJBrowse=
+
+
+[Qiime analysis]
+DoQiimeAnalysis=
+qiime_amplicon_type=
+qiime_input_dir=
+qiime_paired_reads_orientation=
+qiime_mapping_file=
+qiime_barcode_file=
+qiime_barcode_len=
+qiime_q_threshold=
+qiime_max_n=
+qiime_min_per_read_length_fraction=
+qiime_UPARSE=
+qiime_minimum_otu_size=
+qiime_similarity=
+qiime_sampling_depth=
+qiime_filter_taxa=
+qiime_substract_ntc=
+
+
[HTML Report]
DoHTMLReport=1
diff --git a/edge_ui/cgi-bin/edge_data_cleanup.pl b/edge_ui/cgi-bin/edge_data_cleanup.pl
new file mode 100755
index 00000000..ed741882
--- /dev/null
+++ b/edge_ui/cgi-bin/edge_data_cleanup.pl
@@ -0,0 +1,63 @@
+#! /usr/bin/env perl
+
+use strict;
+use File::Find;
+use FindBin qw($RealBin);
+use CGI qw(:standard);
+
+# read system params from sys.properties
+my $sysconfig = "$RealBin/../sys.properties";
+my $sys = &getSysParamFromConfig($sysconfig);
+my $keep_days = $sys->{edgeui_proj_store_days};
+my $out_dir = $sys->{edgeui_output};
+my $edgeui_wwwroot = $sys->{edgeui_wwwroot};
+my $input_dir = $sys->{edgeui_input};
+exit if ( $ENV{"REQUEST_METHOD"} );
+exit if (!$keep_days or $keep_days =~ /\D+/);
+#my @dirs = ($out_dir,$input_dir);
+my $ncbi_tmp = "$edgeui_wwwroot/ncbi";
+my @dirs = ($out_dir);
+push @dirs, $ncbi_tmp if (-d $ncbi_tmp);
+
+my $keep_secs = $keep_days * 24 * 60 * 60;
+my $now = time();
+
+if (@ARGV){
+ find( \&printOldFiles, @dirs);
+}else{
+ find( \&unwanted, @dirs);
+}
+
+sub unwanted {
+ $File::Find::name =~ /\.(sam|bam|fastq|fq|gz|tgz|cache)$/i &&
+ $File::Find::name !~ /JBrowse|public/i &&
+ ($now-(stat $_)[9]) > $keep_secs &&
+ unlink $File::Find::name;
+}
+
+sub printOldFiles {
+ $File::Find::name =~ /\.(sam|bam|fastq|fq|gz|tgz|cache)$/i &&
+ $File::Find::name !~ /JBrowse|public/i &&
+ ($now-(stat $_)[9]) > $keep_secs &&
+ print $File::Find::name;
+}
+
+sub getSysParamFromConfig {
+ my $config = shift;
+ my $sys;
+ open CONF, $config or die "Can't open $config: $!";
+ while(){
+ if( /^\[system\]/ ){
+ while(){
+ chomp;
+ last if /^\[/;
+ if ( /^([^=]+)=([^=]+)/ ){
+ $sys->{$1}=$2;
+ }
+ }
+ }
+ last;
+ }
+ close CONF;
+ return $sys;
+}
diff --git a/edge_ui/cgi-bin/edge_dynamic_selection.cgi b/edge_ui/cgi-bin/edge_dynamic_selection.cgi
index dae81137..4cfd6176 100755
--- a/edge_ui/cgi-bin/edge_dynamic_selection.cgi
+++ b/edge_ui/cgi-bin/edge_dynamic_selection.cgi
@@ -11,9 +11,9 @@ my $cgi = CGI->new;
my %opt = $cgi->Vars();
my $queries = $opt{'query'} || $ARGV[0];
-# read system params from config template
-my $config_tmpl = "$RealBin/edge_config.tmpl";
-my $sys = &getSysParamFromConfig($config_tmpl);
+# read system params from sys.properties
+my $sysconfig = "$RealBin/../sys.properties";
+my $sys = &getSysParamFromConfig($sysconfig);
my $edgeui_wwwroot = $sys->{edgeui_wwwroot};
my $ref_json_file = "$edgeui_wwwroot/data/Ref_list.json";
diff --git a/edge_ui/cgi-bin/edge_info.cgi b/edge_ui/cgi-bin/edge_info.cgi
index ccf3acef..db99f918 100755
--- a/edge_ui/cgi-bin/edge_info.cgi
+++ b/edge_ui/cgi-bin/edge_info.cgi
@@ -1,5 +1,4 @@
#!/usr/bin/env perl
-#
# Po-E (Paul) Li
# Los Alamos National Lab.
# 2014-08-07
@@ -17,7 +16,9 @@ use POSIX qw(strftime);
use Data::Dumper;
use LWP::UserAgent;
use HTTP::Request::Common;
+use Digest::MD5 qw(md5_hex);
require "edge_user_session.cgi";
+require "../cluster/clusterWrapper.pl";
######################################################################################
# DATA STRUCTURE:
@@ -34,11 +35,14 @@ require "edge_user_session.cgi";
# ->{STATUS} // [unfinished|skip|already|running|done|failed]
# ->{2}...
#
+# $info->{INFO}->{CPUU} ...
+#
######################################################################################
my $cgi = CGI->new;
my %opt = $cgi->Vars();
my $pname = $opt{proj};
+my $init = $opt{init};
$pname ||= $ARGV[0];
my $username = $opt{'username'}|| $ARGV[1];
my $password = $opt{'password'}|| $ARGV[2];
@@ -48,18 +52,31 @@ my $protocol = $opt{protocol} || 'http:';
my $sid = $opt{'sid'}|| $ARGV[4];
my $ip = $ARGV[5];
$ENV{REMOTE_ADDR} = $ip if $ip;
-
-# read system params from config template
-my $config_tmpl = "$RealBin/edge_config.tmpl";
-my $sys = &getSysParamFromConfig($config_tmpl);
+my $domain = $ENV{'HTTP_HOST'} || 'edge-dev-master.lanl.gov';
+my ($webhostname) = $domain =~ /^(\S+?)\./;
+
+# read system params from sys.properties
+my $sysconfig = "$RealBin/../sys.properties";
+my $sys = &getSysParamFromConfig($sysconfig);
+$sys->{edgeui_output} = "$sys->{edgeui_output}"."/$webhostname" if ( -d "$sys->{edgeui_output}/$webhostname");
+$sys->{edgeui_input} = "$sys->{edgeui_input}"."/$webhostname" if ( -d "$sys->{edgeui_input}/$webhostname");
my $um_url = $sys->{edge_user_management_url};
my $out_dir = $sys->{edgeui_output};
-my $domain = $ENV{'HTTP_HOST'};
+my $www_root = $sys->{edgeui_wwwroot};
+my $edge_total_cpu = $sys->{"edgeui_tol_cpu"};
+my $max_num_jobs = $sys->{"max_num_jobs"};
+my $edge_projlist_num = $sys->{"edgeui_project_list_num"};
my $hideProjFromList = 0;
-$domain ||= "edgeset.lanl.gov";
$um_url ||= "$protocol//$domain/userManagement";
+$umSystemStatus ||= $sys->{user_management} if (!@ARGV);
+$umSystemStatus = ($umSystemStatus eq "false")?0:$umSystemStatus;
+#cluster
+my $cluster = $sys->{cluster};
+my $cluster_job_prefix = $sys->{cluster_job_prefix};
+my $cluster_job_max_cpu= $sys->{cluster_job_max_cpu};
my $list; # ref for project list
+my @projlist; # project list index
my $prog; # progress for latest job
my $info; # info to return
@@ -68,21 +85,51 @@ $info->{INFO}->{CPUU} = $cpuUsage;
$info->{INFO}->{MEMU} = $memUsage;
$info->{INFO}->{DISKU} = $diskUsage;
-$info->{INFO}->{UPLOAD} = "true" if ( $sys->{user_upload} );
-
+my $runcpu = ($cluster)? int(($cluster_job_max_cpu-1)/$max_num_jobs): int(($edge_total_cpu-1)/$max_num_jobs);
+$info->{INFO}->{RUNCPU} = ($runcpu>1)? $runcpu :1;
+
+$info->{INFO}->{PROJLISTNUM} = $edge_projlist_num;
+
+# module on/off
+$info->{INFO}->{UMSYSTEM}= ( $sys->{user_management} )? "true":"false";
+$info->{INFO}->{UPLOAD} = ( $sys->{user_upload} )?"true":"false";
+$info->{INFO}->{ARCHIVE} = ( -w $sys->{edgeui_archive} ) ? "true":"false";
+$info->{INFO}->{MQC} = ( $sys->{m_qc} )?"true":"false";
+$info->{INFO}->{MAA} = ( $sys->{m_assembly_annotation} )?"true":"false";
+$info->{INFO}->{MRBA} = ( $sys->{m_reference_based_analysis} )?"true":"false";
+$info->{INFO}->{MTC} = ( $sys->{m_taxonomy_classfication} )?"true":"false";
+$info->{INFO}->{MPA} = ( $sys->{m_phylogenetic_analysis} )?"true":"false";
+$info->{INFO}->{MSGP} = ( $sys->{m_specialty_genes_profiling} )?"true":"false";
+$info->{INFO}->{MPPA} = ( $sys->{m_pcr_primer_analysis} )?"true":"false";
+$info->{INFO}->{MQIIME} = ( $sys->{m_qiime} )?"true":"false";
+
+&returnStatus() if ($init);
#($umSystemStatus =~ /true/i)? &getUserProjFromDB():&scanNewProjToList();
+#check projects vital
+my ($vital, $name2pid, $error);
+if($cluster) {
+ ($vital, $name2pid, $error) = checkProjVital_cluster($cluster_job_prefix);
+ if($error) {
+ $info->{INFO}->{ERROR}= "CLUSTER ERROR: $error";
+ }
+} else {
+ ($vital, $name2pid) = &checkProjVital();
+}
+
# session check
-if( $sys->{user_management} ){
+if( $umSystemStatus ){
my $valid = verifySession($sid);
if($valid){
($username,$password,$viewType) = getCredentialsFromSession($sid);
+ my $user_config = $sys->{edgeui_input}."/". md5_hex($username)."/user.properties";
&getUserProjFromDB();
- &getProjInfoFromDB($pname) if ! defined $list->{$pname};
+ &getProjInfoFromDB($pname) if ($pname and ! defined $list->{$pname});
$info->{INFO}->{SESSION_STATUS} = "valid";
}
else{
&getUserProjFromDB();
+ &getProjInfoFromDB($pname) if ($pname and ! defined $list->{$pname});
$info->{INFO}->{SESSION_STATUS} = "invalid";
}
}
@@ -90,55 +137,76 @@ else{
&scanNewProjToList();
}
-
-#check projects vital
-my ($vital, $name2pid) = &checkProjVital();
-
my $time = strftime "%F %X", localtime;
-
-if( scalar keys %$list ){
- my $idx;
+@projlist = sort {$list->{$b}->{TIME} cmp $list->{$a}->{TIME}} keys %$list;
+my $idx;
+if( scalar @projlist ){
my $progs;
-
- foreach my $i ( keys %$list ) {
+ my $count=0;
+
+ # retrive progress info of a project that is selected by the following priorities:
+ # 1. assigned project
+ # 2. latest running project
+ # 3. lastest project
+ $idx= ($pname)? (grep $list->{$_}->{NAME} eq $pname, @projlist)[0] : $projlist[0];
+ my @running_idxs = grep { $list->{$_}->{STATUS} eq "running" or $list->{$_}->{STATUS} =~ /unstarted|interrupted|in process/ and $list->{$_}->{NAME} ne $pname } @projlist;
+ $idx = shift @running_idxs if (scalar(@running_idxs) && !$pname);
+ $idx = $projlist[0] if (!$idx); # when given $pname does not exist.
+ @projlist = ($idx,@running_idxs); # update running projects and focus project program info.
+
+ foreach my $i ( @projlist ) {
+ last if ($edge_projlist_num && ++$count > $edge_projlist_num);
my $lproj = $list->{$i}->{NAME};
+ my $lprojc = $list->{$i}->{PROJCODE};
my $lcpu = $list->{$i}->{CPU};
my $lstatus = $list->{$i}->{STATUS};
my $lpid = $list->{$i}->{PID};
- my $realpid = $name2pid->{$lproj};
+ my $realpid = $name2pid->{$lproj}|| $name2pid->{$lprojc};
my $proj_dir = "$out_dir/$lproj";
+ $proj_dir = "$out_dir/$lprojc" if ( $lprojc && -d "$out_dir/$lprojc");
my $log = "$proj_dir/process.log";
+ my $sjson = "$proj_dir/.run.complete.status.json";
+ my $current_log = "$proj_dir/process_current.log";
my $config = "$proj_dir/config.txt";
- $idx = $i if $lproj eq $pname;
#remove project from list if output directory has been removed
- unless( -e $log ){
+ unless( -e $log || -e $config){
delete $list->{$i};
next;
}
-
+
+ #status JSON
+ #if( -e $sjson ){
+ # my $storedStatus = readListFromJson($sjson);
+ # $list->{$i} = $storedStatus->{LIST};
+ # $progs->{$i} = $storedStatus->{PROG};
+ # next;
+ #}
+
# update current project status
if( -r $log ){
- my ($p_status,$prog,$proj_start,$numcpu,$proj_desc,$proj_name) = &parseProcessLog($log);
- $list->{$i}->{TIME} = $proj_start;
+ my ($p_status,$prog,$proj_start,$numcpu,$proj_desc,$proj_name,$proj_id) = &parseProcessLog($log);
+ $list->{$i}->{TIME} ||= $proj_start;
$list->{$i}->{TIME} ||= strftime "%F %X", localtime;
$list->{$i}->{PID} = $realpid;
$list->{$i}->{CPU} = $numcpu;
$list->{$i}->{DESC} = $proj_desc;
+ ($list->{$i}->{PROJLOG} = $current_log) =~ s/$www_root//;
#for unstarted project, read steps from config file
- (my $tmp,$prog,$proj_start,$numcpu,$proj_desc,$proj_name) = &parseProcessLog($config) if $p_status eq "unstarted";
+ $p_status = "unknown" if (!$p_status);
+ (my $tmp,$prog,$proj_start,$numcpu,$proj_desc,$proj_name,$proj_id) = &parseProcessLog($config) if $p_status =~ /unstarted|unknown|interrupted/;
$list->{$i}->{CPU} = $numcpu;
$list->{$i}->{PROJNAME} = $proj_name;
- if( defined $name2pid->{$lproj} ){ #running
+ if( defined $name2pid->{$lproj} || defined $name2pid->{$lprojc} ){ #running
$list->{$i}->{STATUS} = "running";
}
elsif( $p_status =~ /running/ ){
# the process log reports it's running, but can't find vital
# Unexpected exit detected
$list->{$i}->{STATUS} = "failed";
- `echo "\n*** [$time] EDGE_UI: Pipeline failed (PID:$lpid). Unexpected exit detected! ***" |tee -a $log >> $proj_dir/process_current.log`;
+ `echo "\n*** [$time] EDGE_UI: Pipeline failed (PID:$realpid). Unexpected exit detected! ***" |tee -a $log >> $proj_dir/process_current.log`;
}
else{
$list->{$i}->{STATUS} = $p_status;
@@ -148,43 +216,49 @@ if( scalar keys %$list ){
if ( $list->{$i}->{DBSTATUS} && ($list->{$i}->{STATUS} ne $list->{$i}->{DBSTATUS})){
&updateDBProjectStatus($i, $list->{$i}->{STATUS});
}
+
+ #if( $list->{$i}->{STATUS} eq "finished" && !-e $sjson ){
+ # my $storedStatus;
+ # $storedStatus->{LIST} = $list->{$i};
+ # $storedStatus->{PROG} = $progs->{$i};
+ # saveListToJason($storedStatus, $sjson);
+ #}
}
}
- # retrive progress info of a project that is selected by the following priorities:
- # 1. assigned project
- # 2. latest running project
- # 3. lastest project
-
- unless( $idx ){
- my @idxs1 = grep { $list->{$_}->{STATUS} eq "running" } sort {$list->{$b}->{TIME} cmp $list->{$a}->{TIME}} keys %$list;
- my @idxs2 = sort {$list->{$b}->{TIME} cmp $list->{$a}->{TIME}} keys %$list;
- my @idxs = (@idxs1,@idxs2);
- $idx = shift @idxs;
- }
$info->{PROG} = $progs->{$idx};
# with user management, NAME becomes unique project id
$info->{INFO}->{NAME} = $list->{$idx}->{NAME};
$info->{INFO}->{PROJNAME} = $list->{$idx}->{PROJNAME};
+ $info->{INFO}->{PROJCODE} = $list->{$idx}->{PROJCODE};
+ $info->{INFO}->{PROJLOG} = $list->{$idx}->{PROJLOG};;
$info->{INFO}->{STATUS} = $list->{$idx}->{STATUS};
$info->{INFO}->{TIME} = strftime "%F %X", localtime;
$info->{INFO}->{PROJTYPE} = $list->{$idx}->{PROJTYPE} if ($list->{$idx}->{PROJTYPE});
+
+ ## sample metadata
+ $info->{INFO}->{SHOWMETA} = $list->{$idx}->{SHOWMETA} if ($list->{$idx}->{SHOWMETA});
+ $info->{INFO}->{ISOWNER} = $list->{$idx}->{ISOWNER} if($list->{$idx}->{ISOWNER});
+ $info->{INFO}->{HASMETA} = $list->{$idx}->{HASMETA} if ($list->{$idx}->{HASMETA});
+ $info->{INFO}->{METABSVE} = $list->{$idx}->{METABSVE} if ($list->{$idx}->{METABSVE});
+ ## END sample metadata
}
#autorun
if( scalar keys %$list && $sys->{edgeui_auto_run} ){
- my ( $progs, $proj, $p_status, $proj_start, $proj_dir, $log, $config );
+ my ( $progs, $proj, $projCode, $p_status, $proj_start, $proj_dir, $log, $config );
my $num_cpu_used = 0;
foreach my $i ( sort {$list->{$a}->{TIME} cmp $list->{$b}->{TIME}} keys %$list ) {
$proj = $list->{$i}->{NAME};
- $proj_dir = "$out_dir/$proj";
+ $projCode = $list->{$i}->{PROJCODE};
+ $proj_dir = "$out_dir/$projCode";
my $run=0;
$run = &availableToRun($list->{$i}->{CPU}, $num_cpu_used ) if $list->{$i}->{STATUS} eq "unstarted";
- if($run){
- my $json = `$RealBin/edge_action.cgi $proj rerun "" "" "" $sid $domain 2>> $proj_dir/error.log`;
+ if($run && ($umSystemStatus eq 'ture'?$sid:1) ){
+ my $json = `$RealBin/edge_action.cgi $proj rerun "" "" "" $sid $domain $umSystemStatus 2>> $proj_dir/error.log`;
#print STDERR "$json";
my $info = decode_json($json);
- $list->{$i}->{STATUS} = "running" if $info->{STATUS} == "SUCCESS";
+ $list->{$i}->{STATUS} = "running" if $info->{STATUS} eq "SUCCESS";
$num_cpu_used += $list->{$i}->{CPU};
}
}
@@ -196,6 +270,27 @@ $info->{LIST} = $list if $list;
######################################################
+sub readListFromJson {
+ my $json = shift;
+ my $list = {};
+ if( -r $json ){
+ open JSON, $json;
+ flock(JSON, 1);
+ local $/ = undef;
+ $list = decode_json();
+ close JSON;
+ }
+ return $list;
+}
+
+sub saveListToJason {
+ my ($list, $file) = @_;
+ open JSON, ">$file" or die "Can't write to file: $file\n";
+ my $json = encode_json($list);
+ print JSON $json;
+ close JSON;
+}
+
sub getSysParamFromConfig {
my $config = shift;
my $sys;
@@ -225,6 +320,7 @@ sub parseProcessLog {
my $proj_start;
my $proj_desc;
my $proj_name;
+ my $proj_id;
my $numcpu;
my ($step,$ord,$do,$status);
my %map;
@@ -256,9 +352,12 @@ sub parseProcessLog {
elsif( /^projname=(.*)/){
$proj_name=$1;
}
+ elsif( /^projid=(.*)/){
+ $proj_id=$1;
+ }
elsif( /^\[(.*)\]/ ){
my $step = $1;
- next if $step eq "system";
+ next if $step eq "system" or $step eq "project";
if( defined $map{"$step"} ){
$ord = $map{"$step"};
@@ -272,9 +371,11 @@ sub parseProcessLog {
}
elsif( /^Do.*=(.*)$/ ){
my $do = $1;
- $prog->{$cnt}->{DO}=$do;
- $prog->{$cnt}->{STATUS}="unfinished";
- $prog->{$cnt}->{STATUS}="skip" if $do eq 0;
+ $prog->{$cnt}->{DO}= 'auto' if ($do eq 'auto');
+ $prog->{$cnt}->{DO}= 1 if ($do eq 1);
+ $prog->{$cnt}->{DO}= 0 if ($do eq 0 && !$prog->{$cnt}->{DO});
+ $prog->{$cnt}->{STATUS}="skip";
+ $prog->{$cnt}->{STATUS}="unfinished" if ($prog->{$cnt}->{DO});
}
elsif( /Finished/ ){
$prog->{$ord}->{STATUS} = "finished";
@@ -305,6 +406,7 @@ sub parseProcessLog {
#unstarted project
$proj_status = "unstarted" if $lastline =~ /EDGE_UI.*unstarted/;
$proj_status = "interrupted" if $lastline =~ /EDGE_UI.*interrupted/;
+ $proj_status = "archived" if $lastline =~ /EDGE_UI.*archived/;
$proj_start = $1 if $lastline =~ /\[(\S+ \S+)\] EDGE_UI/;
$prog->{$ord}->{STATUS} = "unfinished" if $proj_status eq "interrupted"; #turn last step to unfinished
@@ -319,25 +421,57 @@ sub parseProcessLog {
}
}
- return ($proj_status,$prog,$proj_start,$numcpu,$proj_desc,$proj_name);
+ return ($proj_status,$prog,$proj_start,$numcpu,$proj_desc,$proj_name,$proj_id);
}
sub scanNewProjToList {
- my $list_idx;
- my $cnt = (sort {$b<=>$a} keys %$list)[0];
-
- foreach my $i ( keys %$list ) {
- my $n = $list->{$i}->{NAME};
- $list_idx->{$n}=$i;
- }
+ my $cnt = 1;
opendir(BIN, $out_dir) or die "Can't open $out_dir: $!";
- while( defined (my $file = readdir BIN) ) {
- next if $file eq '.' or $file eq '..';
- if ( -d "$out_dir/$file" ) {
- next if defined $list_idx->{$file};
- $list->{++$cnt}->{NAME} = $file if -r "$out_dir/$file/process.log";
+ my @dirfiles = readdir(BIN);
+
+ foreach my $file (@dirfiles) {
+ next if ($file eq '.' || $file eq '..' || ! -d "$out_dir/$file/");
+ my $config = "$out_dir/$file/config.txt";
+ my $processLog = "$out_dir/$file/process_current.log";
+ $cnt++;
+ if (-r "$config"){
+ $list->{$cnt}->{NAME} = $file ;
+ $list->{$cnt}->{TIME} = (stat("$out_dir/$file"))[10];
+ $list->{$cnt}->{STATUS} eq "running" if $name2pid->{$file};
+ if ( -r "$processLog"){
+ open (my $fh, $processLog);
+ while(<$fh>){
+ if (/queued/){
+ $list->{$cnt}->{STATUS} = "unstarted";
+ }
+ if (/All Done/){
+ $list->{$cnt}->{STATUS} = "finished";
+ }
+ if (/failed/i){
+ $list->{$cnt}->{STATUS} = "failed";
+ }
+ }
+ close $fh;
+ }
+ my $projname = $file;
+ chomp $projname;
+ $list->{$cnt}->{PROJNAME} = $projname;
}
+
+ ## sample metadata
+ if($sys->{edge_sample_metadata}) {
+ $list->{$cnt}->{SHOWMETA} = 1;
+ $list->{$cnt}->{ISOWNER} = 1;
+ }
+ my $metaFile = "$out_dir/$file/sample_metadata.txt";
+ if(-r $metaFile) {
+ $list->{$cnt}->{HASMETA} = 1;
+ my $bsveId = `grep -a "bsve_id=" $metaFile | awk -F'=' '{print \$2}'`;
+ chomp $bsveId;
+ $list->{$cnt}->{METABSVE} = $bsveId;
+ }
+ ## END sample metadata
}
closedir(BIN);
}
@@ -363,12 +497,14 @@ sub availableToRun {
sub getSystemUsage {
my $mem = `vmstat -s | awk '\$0 ~/total memory/ {total=\$1 } \$0 ~/free memory/ {free=\$1} \$0 ~/buffer memory/ {buffer=\$1} \$0 ~/cache/ {cache=\$1} END{print (total-free-buffer-cache)/total*100}'`;
my $cpu = `top -bn1 | grep load | awk '{printf "%.1f", \$(NF-2)}'`;
- my $disk = `df -h $out_dir | tail -1 | awk '{printf "%.1f", \$5}'`;
+ my $disk = `df -h $out_dir | tail -1 | awk '{print \$5}'`;
+ $disk= `df -h $out_dir | tail -1 | awk '{print \$4}'` if ($disk !~ /\%/);
$cpu = $cpu/$sys->{edgeui_tol_cpu}*100;
$disk =~ s/\%//;
if( $mem || $cpu || $disk ){
$mem = sprintf "%.1f", $mem;
$cpu = sprintf "%.1f", $cpu;
+ $disk = sprintf "%.1f", $disk;
return ($mem,$cpu,$disk);
}
else{
@@ -464,22 +600,47 @@ sub getUserProjFromDB{
foreach my $hash_ref (@$array_ref)
{
my $id = $hash_ref->{id};
+ my $projCode = $hash_ref->{code};
my $project_name = $hash_ref->{name};
my $status = $hash_ref->{status};
- next if (! -r "$out_dir/$id/process.log");
+ my $created = $hash_ref->{created};
+ next if (! -r "$out_dir/$id/process.log" && ! -r "$out_dir/$projCode/process.log" && !$cluster);
next if ( $status =~ /delete/i);
$list->{$id}->{NAME} = $id;
$list->{$id}->{PROJNAME} = $project_name;
+ $list->{$id}->{PROJCODE} = $projCode;
$list->{$id}->{DBSTATUS} = $status;
+ $list->{$id}->{STATUS} = $status;
+ $list->{$id}->{TIME} = $created;
$list->{$id}->{OWNER_EMAIL} = $hash_ref->{owner_email};
$list->{$id}->{OWNER_FisrtN} = $hash_ref->{owner_firstname};
$list->{$id}->{OWNER_LastN} = $hash_ref->{owner_lastname};
$list->{$id}->{PROJTYPE} = $hash_ref->{type} if ($username && $password);
+
+ ## sample metadata
+ if($sys->{edge_sample_metadata}) {
+ $list->{$id}->{SHOWMETA} = 1;
+ }
+ if($username eq $hash_ref->{owner_email}) {
+ $list->{$id}->{ISOWNER} = 1;
+ }
+ my $metaFile = "$out_dir/$id/sample_metadata.txt";
+ if(!-e $metaFile) {
+ $metaFile = "$out_dir/$projCode/sample_metadata.txt";
+ }
+ if(-r $metaFile) {
+ $list->{$id}->{HASMETA} = 1;
+ my $bsveId = `grep -a "bsve_id=" $metaFile | awk -F'=' '{print \$2}'`;
+ chomp $bsveId;
+ $list->{$id}->{METABSVE} = $bsveId;
+ }
+ ## END sample metadata
}
}
sub getProjInfoFromDB{
- my $project=shift;
+ my $project=shift;
+ $project = &getProjID($project);
my %data = (
email => $username,
password => $password,
@@ -499,25 +660,45 @@ sub getProjInfoFromDB{
my $response = $browser->request($req);
my $result_json = $response->decoded_content;
- print $result_json if (@ARGV);
+ #print $result_json,"\n" if (@ARGV);
my $hash_ref = from_json($result_json);
my $id = $hash_ref->{id};
my $project_name = $hash_ref->{name};
+ my $projCode = $hash_ref->{code};
my $status = $hash_ref->{status};
+ my $created = $hash_ref->{created};
my $projtype = ($hash_ref->{isPublished})?"publish":"false";
#next if (! -r "$out_dir/$id/process.log");
#next if ( $status =~ /delete/i);
$list->{$id}->{NAME} = $id;
$list->{$id}->{PROJNAME} = $project_name;
+ $list->{$id}->{PROJCODE} = $projCode;
$list->{$id}->{DBSTATUS} = $status;
+ $list->{$id}->{STATUS} = $status;
$list->{$id}->{PROJTYPE} = $projtype;
+ $list->{$id}->{TIME} = $created;
+}
+
+sub getProjID {
+ my $project=shift;
+ my $projID = $project;
+ if ( -d "$out_dir/$project"){ # use ProjCode as dir
+ open (my $fh, "$out_dir/$project/config.txt") or die "Cannot open $out_dir/$project/config.txt\n";
+ while(<$fh>){
+ if (/^projid=(\S+)/){
+ $projID = $1;
+ last;
+ }
+ }
+ }
+ return $projID;
}
sub returnStatus {
my $json;
$json = to_json( $info ) if $info;
- $json = to_json( $info, { ascii => 1, pretty => 1 } ) if $info && $ARGV[0];
+ $json = to_json( $info, { ascii => 1, pretty => 1 } ) if $info && $ARGV[1];
print $cgi->header('application/json'), $json;
exit;
}
diff --git a/edge_ui/cgi-bin/edge_projectspage.cgi b/edge_ui/cgi-bin/edge_projectspage.cgi
index 03da6e55..632f08b0 100755
--- a/edge_ui/cgi-bin/edge_projectspage.cgi
+++ b/edge_ui/cgi-bin/edge_projectspage.cgi
@@ -10,7 +10,7 @@ use CGI qw(:standard);
#use CGI::Carp qw(fatalsToBrowser);
use POSIX qw(strftime);
use Data::Dumper;
-use CGI::Pretty;
+#use CGI::Pretty;
require "edge_user_session.cgi";
my $cgi = CGI->new;
@@ -22,15 +22,16 @@ my $userType = $opt{'userType'}|| $ARGV[3];
my $viewType = $opt{'view'}|| $ARGV[4];
my $protocol = $opt{protocol}||'http:';
my $sid = $opt{'sid'}|| $ARGV[5];
+my $domain = $ENV{'HTTP_HOST'} || 'edge-bsve.lanl.gov';
+my ($webhostname) = $domain =~ /^(\S+?)\./;
-# read system params from config template
-my $config_tmpl = "$RealBin/edge_config.tmpl";
-my $sys = &getSysParamFromConfig($config_tmpl);
+# read system params from sys.properties
+my $sysconfig = "$RealBin/../sys.properties";
+my $sys = &getSysParamFromConfig($sysconfig);
+$sys->{edgeui_output} = "$sys->{edgeui_output}"."/$webhostname" if ( -d "$sys->{edgeui_output}/$webhostname");
my $out_dir = $sys->{edgeui_output};
my $um_config = $sys->{user_management};
my $um_url = $sys->{edge_user_management_url};
-my $domain = $ENV{'HTTP_HOST'};
-$domain ||= "edgeset.lanl.gov";
$um_url ||= "$protocol//$domain/userManagement";
# session check
@@ -43,89 +44,78 @@ if( $sys->{user_management} ){
#print Dumper ($list);
print $cgi->header( "text/html" );
-if ($userType =~ /admin/i && $viewType !~ /admin/i){
- print "
EDGE bioinformatics is intended to help truly democratize the use of Next Generation Sequencing for exploring genomes and metagenomes. Given that bioinformatic analysis is now the rate limiting factor in genomics, we developed EDGE bioinformatics with a user-friendly interface that allows scientists to perform a number of tailored analyses using many cutting-edge tools.
This website is available for use with publicly available data, while local stand-alone implementations can accommodate data generated on-site for immediate analysis, eliminating the need to move large datasets.
Allow users to address a wide range of use cases including the assembly/annotation and comparison of novel genomes, and the characterization of complex clinical or environmental samples
-
Can present the results of several taxonomy classification tools for easy comparison
-
Focus on accurate and rapid analysis
-
Enables sequencing as a solution in facilities where human-resources, space, bandwidth, and time are limited
-
-
+
+
+
No need for high-level bioinformaticists
+
Allow users to address a wide range of use cases including the assembly/annotation and comparison of novel genomes, and the characterization of complex clinical or environmental samples
+
Can present the results of several taxonomy classification tools for easy comparison
+
Focus on accurate and rapid analysis
+
Enables sequencing as a solution in facilities where human-resources, space, bandwidth, and time are limited
EDGE provides many modules to do various analyses. You can choose to run or skip a specific process. Parameters/options are provided for most of the analyses.
+
Choose Processes / Analyses
+
EDGE provides many modules to do various analyses. You can choose to run or skip a specific process. Parameters/options are provided for most of the analyses.
You can click here to
turn all on,
expand all sections or
close all sections.
IDBA_UD performs well on isolates as well as metagenomes but it may not work well on very large genomes.
@@ -429,7 +740,16 @@
-
SPAdes performs well on isolates as well as single cell data but it may not work on larger genome, and it takes more computational resource. PacBio CLR and Oxford Nanopore reads are used for gap closure and repeat resolution.
+
SPAdes performs well on isolates as well as single cell data but it may not work on larger genomes, and it takes more computational resource. PacBio CLR and Oxford Nanopore reads are used for gap closure and repeat resolution.
+
+
+
+
+
MEGAHIT is an ultra-fast single-node solution for large and complex metagenomics assembly via succinct de Bruijn graph which achieves low memory assembly.
Please provide the reference/source annotation (Genbank file), EDGE will use RATT to transfer the annotation from the reference genome. The reference genome must be close relative to the sample.
EDGE will use all reads by default. You can change the behavior to use reads that are unmapped to the reference if Reference-based Analysis is on.
-
EDGE uses multiple tools for taxonomy classification including GOTTCHA (bacterial & viral databases), MetaPhlAn, MetaPhyler (short read version), Kraken, MetaScope and reads mapping to NCBI RefSeq using BWA.
+
EDGE uses multiple tools for taxonomy classification including GOTTCHA (bacterial & viral databases), MetaPhlAn, Kraken and reads mapping to NCBI RefSeq using BWA.
Contig-based Taxonomy Classification
-
EDGE will map contigs to NCBI genomes and make taxonomy inference to each contigs.
+
EDGE will map contigs to NCBI genomes and make a taxonomic inference for each contig.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Read-based Specialty Gene Analysis
+
EDGE will use ShortBRED to search the reads for Antibiotic Resistance genes from ARDB and Resfams and for Virulence genes from VFDB.
+
+
+
+
Contig-based (ORF) Specialty Gene Analysis
+
EDGE will use ShortBRED to search the ORFs on the contigs for Virulence genes from VFDB.