europepmc

#! /bin/bash
#------------------------------------------------------------------------------
#$Author$
#$Date$
#$Revision$
#$URL$
#------------------------------------------------------------------------------
#*
# This script is made to collect search results on europepmc website :
# User enters the search terms as arguments of the script, using the option 
# '--search-terms', or writes them in a file (url encoded beforehand, or using
# '--url-encode' option), given as an argument as well. The script makes the
# query on the website and returns the results on the standard output.
#**
TMP_DIR="${TMPDIR}"

set -ue

script() { echo "# $*" >&2 ; }

INFO='$Id$'
FILES=""
SEARCH_TERMS=""
BASENAME="$(basename $0)"
HITCOUNT_ONLY=false
RESULT_TYPE=lite
OPEN_ACCESS=''
URLENCODE=false
LIMIT=false
PROGRESS=false
serverURL=https://www.ebi.ac.uk
resourcePATH=europepmc/webservices/rest/search
pageSize=1000
format=json
script=""
req_hit=""

case $0 in
    *-xml)  format=xml ;;
    *-json) format=json ;;
    *)      format=unknown ;;
esac

#** USAGE:
#**   $0 --options files*.inp
#**   $0 --search-terms 'search_term_1+search_term_2' files*.inp
#**
#** OPTIONS:
#**  -c, --count-only   Prints the number of results.
#**
#**  --json-output      Output results in JSON format.
#**
#**  -l, --limit 10     Limit the number of pages to 10.
#**
#**  -p, --page-size 10
#**                     Modify the number of results per page (default is 1000).
#**
#**  --progress         Prints a progress indicator for each search term.
#**
#**  --script           Prints out the command lines as comment without
#**                     executing them.
#**
#**  -s, --search-terms 'crystal+structure'
#**                     Add search terms as arguments after this option.
#**
#**  -U, --url-encode   URL-encode all search terms.
#**
#**  --version          Prints name, version, date and author of the script.
#**
#**  Result types:
#**  --idlist           Return only the list of identifiers
#**  --light            Light result type (default)
#**  --core             Core result type (contains references to OA texts)
#**
#**  Free text availability (possible values are 'y', 'n' and '' for 'any'):
#**  --open-access y    Specify whether only OA papers should be returned
#**
#**  --xml-output       Output results in XML format.
#**
#**  --help             Prints short help message (this message) and exit.

while [ $# -gt 0 ]
do
    case $1 in
        --help|--hel|--he|--h)
            awk '/#\*/,/#\*\*/ {
                    sub("^ *#[*]?[*]?", ""); \
                    gsub("\\$0","'$(basename $0)'"); \
                    print $0
                }' $0
            exit 0
            ;;
        -c|--count-only|--count-onl|--count-on|--count-o|--count|--coun|--cou| \
        --co|--c)
            HITCOUNT_ONLY=true
            ;;
        --json-output|--json-outpu|--json-outp|--json-out|--json-ou|--json-o|\
        --json|--jso|--js|--j)
            format=json;
            ;;
        --xml-output|--xml-outpu|--xml-outp|--xml-out|--xml-ou|--xml-o|\
        --xml|--xm|--x)
            format=xml
            ;;
        --unknown-format)
            # Just for debugging and testing:
            format=xyz
            ;;
        -l|--limit|--limi|--lim|--li|--l)
            LIMIT=true
            req_pages="$2"
            shift
            ;;
        --core|--cor|--co|--c)
            RESULT_TYPE=core
            ;;
        --idlist|--idlis|--idli|--idl|--id|--i)
            RESULT_TYPE=idlist
            ;;
        --light|--ligh|--lig|--li|--l)
            RESULT_TYPE=light
            ;;
        --open-access|--open-acces|--open-acce|--open-acc|--open-ac|--open-a|\
        --open|--ope|--op|--o)
            OPEN_ACCESS="$2"
            if [ "${OPEN_ACCESS}" != 'y' -a \
                 "${OPEN_ACCESS}" != 'n' -a \
                 "${OPEN_ACCESS}" != '' ]
            then
                echo "$0: WARNING, value '${OPEN_ACCESS}' for the option " \
                     "'--open-access' is not understood, switching to ''" >&2
                OPEN_ACCESS=''
            fi
            shift
            ;;
        --options|--option|--optio|--opti|--opt|--op|--o)
            echo "$0: WARNING, '--options' is a place-holder; " \
                 "please use '$0 --help' to get the list of "\
                 "available options." >&2
            ;;
        -p|--page-size|--page-siz|--page-si|--page-s|--page|--pag|--pa|--p)
            pageSize="$2"
            shift
            ;;
        --progress|--progres|--progre|--progr|--prog|--pro|--pr)
            PROGRESS=true
            ;;
        --script)
            script=script
            ;;
        -s|--search-terms|--search-term|--search-ter|--search-te|--search-t| \
        --search|--searc|--sear|--sea|--se|--s)
            SEARCH_TERMS="$2"
            if [ -z "${SEARCH_TERMS}" ]
            then
                echo "$0: WARNING, search term is empty for '--search-term'" >&2
            fi
            shift
            ;;
        --url-encode|--url-encod|--url-enco|--url-enc|--url-en|--url-e|--url| \
        --ur|--u|--URL|-U|--encode)
            URLENCODE=true
            ;;
        --version|--versio|--versi|--vers|--ver|--ve|--v)
            echo bibliophile version $($(dirname $0)/bibliophile-version)
            echo $INFO
            exit 0
            ;;
        -*) echo "$0: ERROR, unknown option '$1', "\
            "please use '$0 --help' to get the list of available options." >&2
            exit 1
            ;;
        *)  FILES="${FILES} '$1'" ;;
    esac
    shift
done

OA_OPTION=''

if [ "${OPEN_ACCESS}" != "" ]
then
    OA_OPTION='+AND+open_access:'${OPEN_ACCESS}
fi

echo "$0: NOTE, files : "${FILES} >&2
echo "$0: NOTE, search terms : "${SEARCH_TERMS} >&2

eval set -- "${FILES}"

test -z "${TMP_DIR}" && TMP_DIR="/tmp"
TMP_DIR="${TMP_DIR}/tmp-${BASENAME}-$$"
mkdir "${TMP_DIR}"

## trap "rm -rf '${TMP_DIR}'" 1 2 3 15
trap "rm -rf '${TMP_DIR}'" HUP INT QUIT TERM

## set -x

if [ "${SEARCH_TERMS}" = "" ]
then 
    cat "$@"
else
    echo "${SEARCH_TERMS}"
    if [ $# -gt 0 ]
    then
        cat "$@"
    fi
fi \
| grep -vP '^\s*$|^#' \
| if [ $URLENCODE  = true ]
then
    perl -MURI::Encode -nle \
        'print URI::Encode::uri_encode($_,{encode_reserved=>1})'\
    | sed 's/%20/+/g'
else
    cat
fi \
| while read SEARCH_TERMS
do
    echo "$0: NOTE, search term :" ${SEARCH_TERMS} >&2

    if [[ "${SEARCH_TERMS}" =~ \ |\' ]] 
    then
        echo "$0: WARNING, the search term '${SEARCH_TERMS}' contains" \
             "spaces and will not work" >&2
        echo "$0: WARNING, please URL-encode your search term or" \
             "use '--url-encode' option" >&2
        echo "$0: WARNING, will skip the '${SEARCH_TERMS}' search term" >&2
        continue
    fi

    request=query="${SEARCH_TERMS}${OA_OPTION}"
    queryURL="$serverURL/$resourcePATH?$request"
    result="$(curl -sSL "$queryURL&format=json&pageSize=1")"

    n=$(echo "$result" | jq . | awk '/hitCount/{print $2}' | sed 's/,//')

    if [ -z "$n" ]
    then
        echo "$0: ERROR, could not determine the number of hits" >&2
        exit 2
    fi
    
    if [ $HITCOUNT_ONLY = true ]
    then
        echo -e $n"\t"${request}
    else
        if [ "$n" -lt $pageSize ]
        then
            j=1
        else
            j=$(expr $n / $pageSize)
            if [ $(expr $n % $pageSize) -gt 0 ]
            then
                j=$(expr $j + 1)
            fi
            
            if [ $LIMIT = true ]
            then
                if [ $j -gt $req_pages ]
                then
                    j=$req_pages
                fi
            fi
        fi

        options="format=$format&pageSize=$pageSize&resultType=$RESULT_TYPE"
        cursorMark='*'
        for i in $(seq 1 $j) ;
        do
            result="$($script curl -sSL \
            "$queryURL&cursorMark=$cursorMark&$options")"
            if [ "$script" = "" ]
            then
                echo "$result"
                if [ $PROGRESS = true ]
                then
                    echo "$0: NOTE, search term = ${request} : page $i out of $j" >&2
                fi
                if [ $format = "json" ]
                then
                    cursorMark=$(echo "$result" \
                        | jq . \
                        | awk '/nextCursorMark/{print $2}' \
                        | sed 's/[",]//g')
                elif [ $format = "xml" ]
                then
                    cursorMark=$(echo "$result" \
                        | perl -077 -ne \
                           'print $1
                                if /<nextCursorMark>(.*?)<\/nextCursorMark>/ms')                    
                else
                    echo "$0: ERROR, unknown format '$format' requested" >&2
                    exit 3
                fi
            fi
        done
    fi
done

## sleep 6

rm -rf "${TMP_DIR}"