Merge remote-tracking branch 'origin/master' into endpoints

Author: ptormene

doc/getting-started/installation-instructions/development.rst

@@ -113,12 +113,17 @@ convenience you can also add it as an ``alias`` in your ``~/.bashrc``
 Multiple installations
 ----------------------
 
-If any other installation of the Engine exists on the same machine, like
-a system-wide installation made with packages, you must change the
+If any other installation of the Engine exists on the same machine - like
+a system-wide installation made with packages - you should change the
 DbServer port from the default one (1908) to any other unused port.
-Using a DbServer started from a different codebase (which may be
-out-of-sync) could lead to unexpected behaviours and errors. To change
-the DbServer port ``oq-engine/openquake/engine/openquake.cfg`` must be
+
+If the two installations have the same structure of the database and
+the codebase is not that different (which is the normal case) everything will
+work even without changing the DbServer port.
+
+However, in general using a DbServer started from a different installation
+could lead to unexpected behaviours and errors. To change
+the DbServer port the file ``openquake.cfg`` must be
 updated:
 
 ::
@@ -129,6 +134,8 @@ updated:
    authkey = changeme  |  authkey = changeme
    ## cut ##           |  ## cut ##
 
+The file should be put in your virtual environment.
+
 Running the tests
 -----------------
 

doc/release-notes/whats-new-2.4.md

@@ -479,7 +479,7 @@ the amount of dependencies needed by `python-oq-engine` when installed on
 a single node. Specific configurations for _master_ and _workers_ nodes
 are provided by dedicated packages.
 This setup will be ported to Ubuntu packages too in the next release.
-See the [documentation](cluster) for further information.
+See the documentation for further information.
 
 Deprecations
 ------------------------------

doc/release-notes/whats-new-3.3.md

@@ -665,7 +665,7 @@ filesystem and to configure the `shared_dir` parameter in the file
 `openquake.cfg`. Without that, classical calculations will fail during
 the calculation of statistics and event based calculations will fail
 during the computations of GMFs, with some kind of "File not found"
-error. The documentation is [here](cluster).
+error.
 
 5. Docker support has been improved and now works properly on a
 multi-node, multi-host deployment. We now also have a mechanism to

openquake/calculators/classical.py

@@ -738,10 +738,6 @@ def _create_hcurves_maps(self):
         oq = self.oqparam
         N = len(self.sitecol)
         R = len(self.datastore['weights'])
-        if oq.individual_rlzs is None:  # not specified in the job.ini
-            individual_rlzs = (N == 1) * (R > 1)
-        else:
-            individual_rlzs = oq.individual_rlzs
         hstats = oq.hazard_stats()
         # initialize datasets
         P = len(oq.poes)
@@ -753,7 +749,7 @@ def _create_hcurves_maps(self):
             L = oq.imtls.size
         L1 = self.L1 = L // M
         S = len(hstats)
-        if R == 1 or individual_rlzs:
+        if R == 1 or oq.individual_rlzs:
             self.datastore.create_dset('hcurves-rlzs', F32, (N, R, M, L1))
             self.datastore.set_shape_descr(
                 'hcurves-rlzs', site_id=N, rlz_id=R, imt=imts, lvl=L1)
@@ -772,7 +768,7 @@ def _create_hcurves_maps(self):
                 self.datastore.set_shape_descr(
                     'hmaps-stats', site_id=N, stat=list(hstats),
                     imt=list(oq.imtls), poe=oq.poes)
-        return N, S, M, P, L1, individual_rlzs
+        return N, S, M, P, L1
 
     # called by execute before post_execute
     def build_curves_maps(self):
@@ -781,15 +777,15 @@ def build_curves_maps(self):
         """
         oq = self.oqparam
         hstats = oq.hazard_stats()
-        N, S, M, P, L1, individual = self._create_hcurves_maps()
+        N, S, M, P, L1 = self._create_hcurves_maps()
         if '_rates' in set(self.datastore) or not self.datastore.parent:
             dstore = self.datastore
         else:
             dstore = self.datastore.parent
         wget = self.full_lt.wget
-        allargs = [(getter, wget, hstats, individual, oq.max_sites_disagg,
-                    self.amplifier) for getter in getters.map_getters(
-                        dstore, self.full_lt)]
+        allargs = [(getter, wget, hstats, oq.individual_rlzs,
+                    oq.max_sites_disagg, self.amplifier)
+                   for getter in getters.map_getters(dstore, self.full_lt)]
         if not config.directory.custom_tmp and not allargs:  # case_60
             logging.warning('No rates were generated')
             return

openquake/calculators/postproc/median_spectrum.py

@@ -100,7 +100,8 @@ def check_rup_unique(spec_disagg):
 
 
 # NB: we are ignoring IMT-dependent weight
-def compute_median_spectrum(cmaker, context, uhs, monitor=performance.Monitor()):
+def compute_median_spectrum(
+        cmaker, context, uhs, monitor=performance.Monitor()):
     """
     For a given group, computes the median hazard spectrum using a weighted
     mean based on the poes.

openquake/commonlib/readinput.py

@@ -208,7 +208,7 @@ def update(params, items, base_path):
     for key, value in items:
         if key in ('hazard_curves_csv', 'hazard_curves_file',
                    'gmfs_csv', 'gmfs_file',
-                   'site_model_csv', 'site_model_file',
+                   'site_model_csv', 'site_model_file', 'source_model_file',
                    'exposure_csv', 'exposure_file'):
             input_type, fnames = _normalize(key, value.split(), base_path)
             params['inputs'][input_type] = fnames
@@ -1184,8 +1184,8 @@ def get_station_data(oqparam, sitecol, duplicates_strategy='error'):
         logging.error('Conditioned scenarios are not meant to be run '
                       ' on a cluster')
     # Read the station data and associate the site ID from longitude, latitude
-    df = read_df(oqparam.inputs['station_data'], 'LONGITUDE', 'LATITUDE', 'STATION_ID',
-                 duplicates_strategy=duplicates_strategy)
+    df = read_df(oqparam.inputs['station_data'], 'LONGITUDE', 'LATITUDE',
+                 'STATION_ID', duplicates_strategy=duplicates_strategy)
     lons = df['LONGITUDE'].to_numpy()
     lats = df['LATITUDE'].to_numpy()
     nsites = len(sitecol.complete)
@@ -1300,7 +1300,8 @@ def impact_tmap(oqparam, taxidx):
         for key in exp['tmap']:
             # tmap has fields conversion, taxonomy, weight
             df = exp.read_df('tmap/' + key)
-            for taxo, risk_id, weight in zip(df.taxonomy, df.conversion, df.weight):
+            for taxo, risk_id, weight in zip(
+                    df.taxonomy, df.conversion, df.weight):
                 if taxo in taxidx:
                     acc['country'].append(key)
                     acc['peril'].append('groundshaking')

openquake/hazardlib/contexts.py

@@ -554,6 +554,7 @@ class ContextMaker(object):
     deltagetter = None
     fewsites = False
     tom = None
+    cluster = None  # set in PmapMaker
 
     def __init__(self, trt, gsims, oq, monitor=Monitor(), extraparams=()):
         self.trt = trt
@@ -1191,9 +1192,9 @@ def get_pmap(self, ctxs, tom=None, rup_mutex={}):
         rup_indep = not rup_mutex
         sids = numpy.unique(ctxs[0].sids)
         pmap = MapArray(sids, size(self.imtls), len(self.gsims)).fill(rup_indep)
-        ptom = PoissonTOM(self.investigation_time)
+        self.tom = tom or PoissonTOM(self.investigation_time)
         for ctx in ctxs:
-            self.update(pmap, ctx, tom or ptom, rup_mutex)
+            self.update(pmap, ctx, rup_mutex)
         return ~pmap if rup_indep else pmap
 
     def ratesNLG(self, srcgroup, sitecol):