a-flip

#!/bin/bash


#default user
user=($USER)

#read static variables
inst_root="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source $inst_root/a_env_conf
source $inst_root/allas-lib

if [[ $# -lt 1 ]]
then
  echo "Please give the name of a file to be uploaded to allas as an argument of this command."
  echo ""
  exit 1
fi

# local variables
tmp_file=("not_defined")
bucket_name=("not_defined")
print_help=(0)
os_project_name=("$OS_PROJECT_NAME")
input_def=("")
mode=("swift")
#tmp_dir=("${tmp_root}/a_put_$$_tmp")


# read customer defaults
if [[ -e $HOME/.a_tools_conf ]]; then
   customized=1
   source $HOME/.a_tools_conf
else
   customized=0
fi

#Process command line
while [[ $# -ge 1 ]]
do
  case "$1" in
             '--project' | '-p' )
                  os_project_name=($2)
                  shift
                  shift
                ;;
             '-h' | '--help' )
                  print_help=(1)
                  shift
                ;;
             *)
                   input_def=("$input_def $1")
                   shift                       # No more switches
                ;;
    esac
done

if [ $print_help -eq 1 ]; then
cat <<EOF
a-flip is a tool to make individual files temporary available in the internet.

a-flip copies a file to Allas into a bucket that can be publicly accessed. Thus, anyone with the address (URL) of the
uploaded data object can read and download the data with a web browser or tools like *wget* and *curl*.
a-flip works mostly like a-publish but there are some differences:
1) only the pre-defined bucket name ( _username-projectNumber_-flip ) can be used
2) When the command is executed it automatically deletes objects that are older than two days

The basic syntax of the command is:

  a-flip file_name

The file is uploaded to a bucket _username-projectNumber_-flip. You can define other bucket names can't be used.
The URL to the uploaded object is:

https://a3s.fi/username-projectNumber-flip/object_name

After uploading the file to the public flip bucket, it checks up the content of the bucket and
removes object that were uploaded more than two days ago.

EOF

exit

fi

#Check if connection works
if [[ $mode == "swift" ]]
then
  test=$(rclone about ${storage_server}: 2> /dev/null | wc -l)
  #test=$(swift stat 2> /dev/null | grep -c "Account:")
  if [[ $test -lt 1 ]]
  then
    echo "No connection to Allas!"
    echo "Please try setting up the connection again."
    exit 1
  fi
fi

if [[ $mode == "s3cmd" ]]
then
  echo "a-flip is not available s3allas."
  exit
fi

if [[ $mode == "lumi" ]]
then
  storage_server="lumi-pub"
fi




#source /appl/opt/allas_conf
#input=("$1")

#check free space in $WRKDIR
#quota_s=($(lfs quota -q -u $USER $WRKDIR))
#free_space=$(expr ${quota_s[2]} - ${quota_s[1]})

echo "Files to be uploaded: $input_def"

#set default bucket
project_label=$(echo ${os_project_name} |  sed -e s/"project_"/""/g)
bucket_name=("${user}-${project_label}-flip")

if [[ $mode == "lumi" ]]; then
   echo "rclone mkdir ${storage_server}:${bucket_name}"
   rclone mkdir ${storage_server}:${bucket_name}
fi


make_temp_dir

for input in $input_def
do
  echo "Processing: $input"
  if [ ! -e $input ] ; then
    echo "File: $input does not exist!"
    exit 1
  fi

  if [[ $(file -b $input | grep -c directory ) -eq 1 ]]
  then
    echo "This command can only be used to publish files, not directories."
    exit 1
  fi

  #check that file name does not end with _ameta
  if [[ ${input:(-5):5} == "_ameta" ]]; then
    echo "Found a file/directoryname which ends with _ameta"
    echo "  $input"
    echo ""
    echo "Please rename this file as it will mix up a the metadata management of a-put"
    exit 1
  fi

  file_path=$(abspath $input)

  echo "Checking total size of $input. Please wait."
  tot_size=$(du -s $input | cut -f1)
  #echo $tot_size

  #tmp file name. Depends on compression and is file is a directory
  if [  $tmp_file == "not_defined" ]
  then
     tmp_file=($(basename $input | tr " " "_" ))
  fi


  #Check if stored file already exist
  #echo "rclone ls ${storage_server}:${bucket_name}/$tmp_file"

  if [[ $(rclone ls ${storage_server}:${bucket_name}/$tmp_file 2> /dev/null | wc -c) -gt 2 ]]
  then
    echo ""
    echo "A file/directory with the same name has already been uploaded into"
    echo "bucket $bucket_name in $storage_server"
    echo ""
    rclone lsl ${storage_server}:${bucket_name}/$tmp_file
    echo ""
    echo "Do you wish to overwrite the existing old object? [y/n]"
    read overwrite_old
    if [[ $overwrite_old == "y" ]] || [[ $overwrite_old == "yes" ]]; then
      echo "Overwriting ${bucket_name}/$tmp_file"
    else
      exit 1
    fi
  fi

  #collect and count metadata
  echo "user: $user" >> ${tmp_dir}/${tmp_file}_ameta
  echo "host: $(hostname)" >> ${tmp_dir}/${tmp_file}_ameta
  echo "original_location: $file_path" >>  ${tmp_dir}/${tmp_file}_ameta
  echo ""
  ls -l $input >>  ${tmp_dir}/${tmp_file}_ameta

  if [[ $tot_size -gt $max_size ]]
  then
    echo "This file or directory is too big for this tool"
    echo "Total size: ${tot_size}K"
    echo "Please use swift or rclone command to upload the data to allas"
    rm -f ${tmp_dir}/${tmp_file}_ameta
    clean_temp_dir
    exit 1
  fi

#  if [[ $tot_size -gt $free_space ]]
#  then
#    echo "There is not enough space for the temporary files."
#    echo "$input contains $num_files files or directories that take ${tot_size}K of disk space"
#    echo "Available free space is ${free_space}K"
#    echo ""
#    rm -f ${tmp_dir}/${tmp_file}_ameta
#    rmdir ${tmp_dir}
#    exit 1
#  fi

  #this is not a good approach.
  #test if a symbolic link would do
  ln -s ${file_path} ${tmp_dir}/$tmp_file

  #upload

  #if [[ $mode == "swift" ]]; then
  # For less than 5GB files rclone is used for uploading

  echo "Uploading data to allas."
  echo "rclone copy --progress ${tmp_dir}/$tmp_file ${storage_server}:${bucket_name}"
  rclone copy -L --progress ${tmp_dir}/$tmp_file ${storage_server}:${bucket_name}
  exitcode=$?
  if [ $exitcode -ne 0 ]; then
       echo ""
       echo "File upload for $infile failed"

       rclone deletefile ${storage_server}:${bucket_name}/$tmp_file
       rm -f ${tmp_dir}/$tmp_file
       rm -f ${tmp_dir}/${tmp_file}_ameta
       clean_temp_dir
       exit 1
  fi

  # rclone md5sums can be calculated only for files that are smaller than 5GB
  if [[ $tot_size -lt 5000000 ]]
  then
     echo "Confirming upload..."
     #checksums for local and allas files
     sum1=$(checksum_tmpdir_tmpfile)
     sum2=($(rclone md5sum  ${storage_server}:${bucket_name}/$tmp_file))

     #check is checksums match
     if [[ ${sum1[0]} !=  ${sum2[0]} ]]
     then
       echo "Upload of $input was not successful!"
       echo "Cleaning the failed upload"
       rclone deletefile ${storage_server}:${bucket_name}/$tmp_file
       rm -f ${tmp_dir}/$tmp_file
       rm -f ${tmp_dir}/${tmp_file}_ameta
       clean_temp_dir
       exit 1
     fi
     echo "$input OK"
  fi


  #echo "Removing temporary file"
  rm -f  ${tmp_dir}/$tmp_file

  #update metadata
  echo ""
  echo "Adding metadata for uploaded $input"
  #echo "rclone copy ./${tmp_file}_ameta ${storage_server}:${bucket_name}/${partial_path}"
  rclone copy  ${tmp_dir}/${tmp_file}_ameta ${storage_server}:${bucket_name}/${partial_path}

  rm -f ${tmp_dir}/${tmp_file}_ameta

  echo "$input uploaded to ${bucket_name}"
  if [[ $mode == "swift" ]]; then
     echo "Public link: https://a3s.fi/${bucket_name}/$tmp_file"
  fi
  if [[ $mode == "lumi" ]]; then
     echo "Public link:  https://${os_project_name}.lumidata.eu/${bucket_name}/$tmp_file"
  fi

  tmp_file=("not_defined")

done

#Clean old files from flip bucket

for ameta_name in $(rclone lsl ${storage_server}:${bucket_name} | grep "_ameta$" | grep -v $(date +%Y-%m-%d ) | grep -v $(date -d "yesterday 13:00" +%Y-%m-%d) | awk '{ print $NF}')
do
  #  echo "removing: ${object_name}."
  #remove ameta file
  rclone deletefile ${storage_server}:${bucket_name}/${ameta_name}
  #remove corresponding object or file
  rclone deletefile ${storage_server}:${bucket_name}/${ameta_name%_ameta}
done

#Publish and rebuild index file

echo '<html>' >  ${tmp_dir}/index.html
#for i in $(swift list $bucket_name | grep -v '_ameta$' )
for i in $(rclone ls  ${storage_server}:${bucket_name} | grep -v '_ameta$' | awk '{print "X "$0}' | tr -s " " | cut -d " " -f 3- )
do
  echo '<li><a href="'$i'">'$i'</a></li>' >> ${tmp_dir}/index.html
done
echo '</html>' >>  ${tmp_dir}/index.html

rclone copy  ${tmp_dir}/index.html ${storage_server}:${bucket_name}/
rm -f   ${tmp_dir}/index.html

#set access control in case of Allas/swift
if [[ $mode == "swift" ]]; then
  swift post ${bucket_name} --read-acl ".r:*,.rlistings"
  swift post ${bucket_name}_segments --read-acl ".r:*,.rlistings"
fi

clean_temp_dir
echo ""
echo "Upload ready."
echo "The public link will be valid for at least one day."
exit 0