Merge branch 'esa-presentation' into dont-panic

Instructions-flask.md

@@ -18,6 +18,7 @@ or with venv... in our local machine
 
 ```sh
 python3 -m venv ~/proyectos/flsk
+# or python3 -m venv ~/venv/flsk
 source ~/proyectos/flsk/bin/activate
 ## or source ~/proyectos/venv/flsk/bin/activate
 
@@ -45,6 +46,9 @@ pip install folium
 pip install pandas
 pip install datetime
 pip install openpyxl
+pip install pillow  ipyplot
+pip install pyinaturalist
+pip install pickle5
 
 ```
 

requirements.txt

@@ -1,70 +1,27 @@
-alembic==1.7.6
-aniso8601==9.0.1
-attrs==21.4.0
-branca==0.4.2
-certifi==2021.10.8
-charset-normalizer==2.0.12
-click==8.0.3
-dnspython==2.2.0
-email-validator==1.1.3
+blinker==1.6.2
+branca==0.6.0
+certifi==2023.5.7
+charset-normalizer==3.1.0
+click==8.1.3
+DateTime==5.1
 et-xmlfile==1.1.0
-Flask==2.1.2
-Flask-Login==0.5.0
-Flask-Migrate==3.1.0
-flask-restx==0.5.1
-Flask-SQLAlchemy==2.5.1
-Flask-WTF==1.0.0
-folium==0.12.1.post1
-GDAL==3.5.0
-greenlet==1.1.2
-gunicorn==20.1.0
-idna==3.3
-importlib-metadata==4.12.0
-itsdangerous==2.0.1
-Jinja2==3.0.3
-jsonschema==4.4.0
-Mako==1.1.6
-MarkupSafe==2.0.1
-numpy==1.23.0
-openpyxl==3.0.9
-# packaging @ file:///private/tmp/sip--packaging-20220416-83003-1hqovwb/packaging-21.3
-pandas==1.4.1
-Pillow==9.1.1
-ply==3.11
-postgis==1.0.4
-protobuf==3.19.4
-psycopg-postgis==0.4.0
-psycopg2==2.9.3
-psycopg2-binary==2.9.3
-pybind11==2.9.2
-pycairo==1.21.0
-pycountry==22.3.5
-PyGObject==3.42.1
-PyQt6==6.2.0
-PyQt6-3D==6.2.0
-PyQt6-Charts==6.2.0
-PyQt6-DataVisualization==6.2.0
-PyQt6-NetworkAuth==6.2.0
-PyQt6-sip==13.1.0
-PyQt6-WebEngine==6.2.0
-pyrsistent==0.18.1
-PySide6==6.2.4
+Flask==2.3.2
+Flask-WTF==1.1.1
+folium==0.14.0
+idna==3.4
+itsdangerous==2.1.2
+Jinja2==3.1.2
+MarkupSafe==2.1.3
+numpy==1.25.0
+openpyxl==3.1.2
+pandas==2.0.2
+psycopg2-binary==2.9.6
 python-dateutil==2.8.2
-python-decouple==3.5
-python-dotenv==0.19.2
-pytz==2021.3
-reportlab==3.6.10
-requests==2.27.1
-scipy==1.8.1
-shiboken6==6.2.4
-shiboken6-generator==6.2.4
-sip==6.6.1
+pytz==2023.3
+requests==2.31.0
 six==1.16.0
-SQLAlchemy==1.4.29
-TBB==0.2
-toml==0.10.2
-urllib3==1.26.8
-Werkzeug==2.0.2
+tzdata==2023.3
+urllib3==2.0.3
+Werkzeug==2.3.6
 WTForms==3.0.1
-wxPython==4.1.1
-zipp==3.8.0
+zope.interface==6.0

webapp/sites.py

@@ -18,7 +18,7 @@
 def survey_list():
     pg = get_pg_connection()
     cur = pg.cursor()
-    cur.execute('SELECT survey_name,count(distinct visit_id), count(*),min(visit_date),max(visit_date) FROM form.field_visit GROUP BY survey_name;')
+    cur.execute('SELECT survey_name,survey_description,count(distinct visit_id), count(*),min(visit_date),max(visit_date) FROM form.field_visit LEFT JOIN form.surveys USING (survey_name) GROUP BY survey_name, survey_description;')
     survey_list = cur.fetchall()
     cur.close()
     return render_template('sites/survey_list.html', pairs=survey_list, the_title="List of surveys")

webapp/species.py

@@ -10,6 +10,17 @@
 from psycopg2.extras import DictCursor
 import pandas as pd
 
+
+from datetime import datetime, timedelta
+import ipyplot
+from dateutil.relativedelta import relativedelta
+from pyinaturalist import (
+    Observation,
+    get_observations,
+    pprint,
+)
+from rich import print
+
 bp = Blueprint('species', __name__, url_prefix='/species')
 create_spp_trait_table="""
 CREATE TEMP TABLE species_traits (species_code,trait_codes) AS (
@@ -55,8 +66,11 @@ def threat_list():
     cur.execute(create_spp_trait_table)
     cur.execute('SELECT \"stateConservation\" AS fam,count(distinct "speciesID"), count(distinct s.species_code), count(distinct q.species_code) FROM species.caps LEFT JOIN species_traits s ON "speciesCode_Synonym"=s.species_code::text LEFT JOIN form.quadrat_samples q ON "speciesCode_Synonym"=q.species_code::text  GROUP BY fam;')
     fam_list = cur.fetchall()
+    cur.execute('SELECT count(distinct "speciesID"), count(distinct s.species_code), count(distinct q.species_code) FROM species.caps LEFT JOIN species_traits s ON "speciesCode_Synonym"=s.species_code::text LEFT JOIN form.quadrat_samples q ON "speciesCode_Synonym"=q.species_code::text;')
+    fam_total = cur.fetchall()
     cur.close()
-    return render_template('species/threat-list.html', pairs=fam_list, the_title="Species per family")
+
+    return render_template('species/threat-list.html', pairs=fam_list, ttl=fam_total, the_title="Species per family")
 
 @bp.route('/family/<id>', methods=['GET', 'POST'])
 @login_required
@@ -130,7 +144,7 @@ def sp_info(id):
             column="speciesCode_Synonym"
         else:
             column="speciesID"
-        qryspp=f"SELECT \"scientificName\", \"speciesID\"::int, family, \"taxonRank\", family, \"speciesCode_Synonym\", \"scientificNameAuthorship\", \"vernacularName\", \"establishmentMeans\", \"primaryGrowthFormGroup\", \"secondaryGrowthFormGroups\", \"stateConservation\", \"protectedInNSW\", \"countryConservation\" from species.caps WHERE \"{column}\"=%s"
+        qryspp=f"SELECT \"scientificName\", \"speciesID\"::int, family, \"taxonRank\", family, \"speciesCode_Synonym\", \"scientificNameAuthorship\", \"vernacularName\", \"establishmentMeans\", \"primaryGrowthFormGroup\", \"secondaryGrowthFormGroups\", \"stateConservation\", \"protectedInNSW\", \"countryConservation\", \"TSProfileID\" from species.caps WHERE \"{column}\"=%s"
 
         cur.execute(qryspp, (id,))
         try:
@@ -196,6 +210,14 @@ def sp_info(id):
         vag_info = cur.fetchone()
 
         cur.close()
-        return render_template('species/info.html', info=spp_info, fsamp=samples, traits=traits, mainrefs=ref_list, addrefs=add_list, check=synonym, vag=vag_info)
+
+        raw_obs = get_observations(taxon_name=spp_info[0], per_page=1)
+        iNobs = Observation.from_json_list(raw_obs)
+        #images = [obs.photos[0].small_url for obs in iNobs[:3]]
+        #labels = [str(obs) for obs in iNobs[:3]]
+
+        return render_template('species/info.html', info=spp_info, inat_obs=iNobs, fsamp=samples, traits=traits, mainrefs=ref_list, addrefs=add_list, check=synonym, vag=vag_info)
     else:
         return redirect(url_for('.search_list', id=request.form['speciesname']))
+
+## use TSProfileID to link to https://www.environment.nsw.gov.au/threatenedspeciesapp/profile.aspx?id=<ID>

webapp/templates/sites/survey_list.html

@@ -9,6 +9,7 @@ <h1>{% block title %}List of field surveys{% endblock %}</h1>
     <table>
         <tr>
           <th>Survey name</th>
+          <th>Survey description</th>
           <th colspan=2>Nr. of distinct sites</th>
           <th colspan=2>Nr. of distinct visits</th>
           <th>Dates</th>
@@ -17,17 +18,18 @@ <h1>{% block title %}List of field surveys{% endblock %}</h1>
         <tr>
 
               {% if pair[0] == 'TO BE CLASSIFIED' %}
-                <td class='empty'>Not assigned to survey</td>
+              <td class='empty' colspan="2">Not assigned to survey</td>
               {% else %}
                 <td><a href='{{ url_for( 'sites.sites_list', survey=pair[0] ) }}'>{{ pair[0] }}</a>  </td>
+                <td>{{ pair[1] }}</td>
               {% endif %}
-
-          <td>{{ pair[1] }}</td>
-          <td><a href='{{ url_for( 'sites.sites_map', survey=pair[0] ) }}'>Map</a> </td>
+
           <td>{{ pair[2] }}</td>
+          <td><a href='{{ url_for( 'sites.sites_map', survey=pair[0] ) }}'>Map</a> </td>
+          <td>{{ pair[3] }}</td>
           <td><a href='{{ url_for( 'visits.visits_list', survey=pair[0] ) }}'>List</a> </td>
 
-          <td>{{ pair[3] }} - {{ pair[4] }}</td>
+          <td>{{ pair[4] }} - {{ pair[5] }}</td>
         </tr>
     {% endfor %}
     </table>

webapp/templates/species/info.html

@@ -23,7 +23,7 @@ <h1><i>{% block title %}{{ info[0] }}{% endblock %}</i>  </h1>
 
 {% block content %}
 Check: {{check}}
-<div>
+<div style="position:relative">
   <h2>Species information</h2>
   <h4 class='credits'>(from BioNET export)</h4>
   <table>
@@ -57,7 +57,14 @@ <h4 class='credits'>(from BioNET export)</h4>
     {% if info[11] == None %}
     <td class='empty'>No value recorded.</td>
     {% else %}
-    <td>{{ info[11] }}</td>
+    <td>
+      {% if info[14]  %}
+        <a href="https://www.environment.nsw.gov.au/threatenedspeciesapp/profile.aspx?id={{ info[14] | int }}" target="TSprofile">{{ info[11] }}</a>
+      {% else %}
+        {{ info[11] }}
+
+      {% endif %}
+    </td>
     {% endif %}
   </tr>
 
@@ -83,8 +90,21 @@ <h4 class='credits'>(from BioNET export)</h4>
     {% endif %}
       </tr>
 </table>
+
+<div style="position:absolute; right:10%; top:0%">
+{% for obs in inat_obs %}
+
+  <img src="{{ obs.photos[0].small_url }}" alt="{{ obs.place_guess }}">
+  <h4 style="font-size: 10px; word-wrap: break-word;">
+    {{ obs.place_guess }} <br>
+    {{ obs.observed_on }} <br>
+    By: <a href="{{obs.uri}}">{{ obs.user.name }} @ iNaturalist</a>
+    </h4>
+
+{% endfor %}
 </div>
 
+
 <p>Click tab to display information on:</p>
 
 <button class='selecttab' onclick="openTab('Occurrence Map (from ALA)')">Occurrence Map (from ALA)</button>
@@ -354,4 +374,7 @@ <h3>Additional references</h3>
 </div>
 {%endif%}
 
+
+</div> 
+
 {% endblock %}

webapp/templates/species/threat-list.html

@@ -32,6 +32,12 @@ <h1>{% block title %}List of species{% endblock %}</h1>
           {% endif %}
         </tr>
     {% endfor %}
+    <tr>
+      <th>Totals </th>
+      <th>{{ttl[0][0]}}</th>
+      <th>{{ttl[0][1]}}</th>
+      <th>{{ttl[0][2]}}</th>
+    </tr>
     </table>
 
 {% endblock %}

webapp/templates/visits/list.html

@@ -26,6 +26,14 @@ <h1>{% block title %}
   {% endif %}
 
 
+  {% if ttl == None %}
+  <p>
+  No survey has been selected, so this list shows all sites. If you want to narrow down the list, go to the list of <a href="{{ url_for('sites.survey_list') }}">surveys</a>.
+  </p>
+  {% else %}
+  <p> This survey includes {{ttl[0]}} sites, {{ttl[1]}} visits, {{ttl[2]}} subplots, {{ttl[3]}} species and {{ttl[4]}} main observers.</p>
+  {% endif %}
+
 <p>Sites are identified by a site label. One site can be visited multiple times. Click any site's label to see more information about site description and coordinates. Click on the visit's date to see more information regarding the visit event.</p>
 
     <table>

webapp/visits.py

@@ -24,16 +24,21 @@ def visits_list(survey):
         qry="SELECT * FROM form.surveys where survey_name=%s"
         cur.execute(qry,(survey,))
         survinfo=cur.fetchone()
-
+
+        qry='SELECT count(distinct visit_id) as nlocs, count(distinct concat(visit_id,visit_date)) as nvisits, count(distinct CONCAT(visit_date, sample_nr)) as nsamples, count(distinct species_code) as nspp, count(distinct CONCAT(givennames,\' \',surname)) as main_observers FROM form.field_visit v LEFT JOIN form.field_samples s USING(visit_id,visit_date) LEFT JOIN form.quadrat_samples q USING (visit_id,visit_date,sample_nr) LEFT JOIN form.observerid ON mainobserver=userkey WHERE survey_name=%s ;'
+        cur.execute(qry, (survey,))
+        visit_total=cur.fetchone()
+
         qry='SELECT visit_id,visit_date,count(distinct sample_nr),count(distinct species_code),survey_name,CONCAT(givennames,\' \',surname) as main_observer, vegetation_formation, vegetation_class  FROM form.field_visit v LEFT JOIN form.field_samples s USING(visit_id,visit_date) LEFT JOIN form.quadrat_samples q USING (visit_id,visit_date,sample_nr) LEFT JOIN form.observerid ON mainobserver=userkey LEFT JOIN form.field_visit_veg_description USING(visit_id,visit_date)  WHERE survey_name=%s GROUP BY survey_name,visit_id,visit_date, givennames, surname, vegetation_formation, vegetation_class ORDER BY survey_name,visit_id;'
         cur.execute(qry, (survey,))
+
     visit_list = cur.fetchall()
     cur.close()
 
     if survey == None:
         return render_template('visits/list.html', visits=visit_list, survey=survey)
     else:
-        return render_template('visits/list.html', visits=visit_list, survey=survinfo)
+        return render_template('visits/list.html', visits=visit_list, survey=survinfo, ttl=visit_total)
 
 @bp.route('/<path:id>/<dt>')
 @login_required

webapp/xlinit.py

@@ -11,6 +11,9 @@
 from webapp.pg import get_pg_connection
 from psycopg2.extras import DictCursor
 
+# https://stackoverflow.com/questions/63329657/python-3-7-error-unsupported-pickle-protocol-5
+#import pickle5 as pickle
+
 import pickle
 
 ## First, load SQL queries