Rename dashboard -> analytics; remove cruft

IATI · Bjwebb · Mar 5, 2018 · Mar 5, 2018 · Jun 11, 2018 · Jul 20, 2018
commit a383c06396cb45443416fd2aa8fcabe205b7ec18
diff --git a/README.rst b/README.rst
@@ -13,7 +13,7 @@ Introduction
 
 IATI-Stats is a python application for generating JSON stats files from IATI data. An example of this outputted JSON can be found at https://github.com/codeforIATI/IATI-Stats/tree/gh-pages
 
-These stats are used to build the `IATI Dashboard <http://dashboard.iatistandard.org/>`_, and also to produce some of the stats for the Transparency Indicator and the IATI Annual report.
+These stats are used to build `Code for IATI Analytics <https://analytics.codeforiati.org/>`_.
 
 Requirements
 ------------
@@ -39,44 +39,12 @@ For example, on Ubuntu these requirements can be installed by running:
 Getting some data to run stats on
 ---------------------------------
 
-This stats code expects a ``data/`` directory, containing a subdirectory for each publisher. Each publisher subdirectory contains that publisher's raw XML files.  All the data on the registry can be downloaded in this structure using the `IATI-Registry-Refresher <https://github.com/IATI/IATI-Registry-Refresher/>`__.
-
-The IATI Tech Team maintains an archive with a snapshot of this data taken every night, from which aggregate stats are produced for the dashboard, using the code in this repository. For political and security reasons this snapshot archive is not publicly available, but is available on request to others wishing to use it for aggregate calculations. Please email code [at] iatistandard [dot] org
+This stats code expects a ``data/`` directory, containing a subdirectory for each publisher. Each publisher subdirectory contains that publisher's raw XML files.  All the data on the registry can be downloaded in this structure using Code for IATI’s `IATI Data Dump <https://iati-data-dump.codeforiati.org/>`__.
 
 Getting started
 ---------------
 
-.. code-block:: bash
-
-    # Get the code
-    git clone https://github.com/codeforIATI/IATI-Stats.git
-    cd IATI-Stats
-
-    # Put some IATI data in the 'data' directory
-    # (see previous section)
-
-    # Create a virtual environment (recommended)
-    python3 -m venv pyenv
-    source pyenv/bin/activate
-    # Install python depencies
-    pip install -r requirements.txt
-
-    # Fetch helper data
-    cd helpers
-    git clone --branch version-2.03 https://github.com/IATI/IATI-Rulesets.git
-    ln -s IATI-Rulesets/rulesets .
-    ./get_codelist_mapping.sh
-    ./get_codelists.sh
-    ./get_schemas.sh
-    wget "http://dashboard.iatistandard.org/stats/ckan.json"
-    wget "https://raw.githubusercontent.com/codeforIATI/IATI-Dashboard/main/registry_id_relationships.csv"
-    cd ..
-
-    # Calculate some stats
-    python calculate_stats.py loop [--folder publisher-registry-id]
-    python calculate_stats.py aggregate
-    python calculate_stats.py invert
-    # You will now have some JSON stats in the out/ directory
+Take a look at `this Github Action <https://github.com/codeforIATI/IATI-Stats/blob/main/.github/workflows/run.yml>`__ to see how this code is used.
 
 You can run ``python calculate_stats.py --help`` for a full list of command line options.
 
@@ -97,7 +65,7 @@ for each stat.
 Structure of stats functions
 ----------------------------
 
-Stats definitions are located in a python module, by default ``stats.dashboard`` (``stats/dashboard.py``). This can be changed with the ``--stats-module`` flag. This module must contain the following classes:
+Stats definitions are located in a python module, by default ``stats.analytics`` (``stats/analytics.py``). This can be changed with the ``--stats-module`` flag. This module must contain the following classes:
 
 -  ``PublisherStats``
 -  ``ActivityStats``
@@ -117,35 +85,7 @@ purposes. The ``returns_numberdict`` and ``returns_number`` decorators are
 provided for this purpose.
 
 To calculate a new stat, add a function to the appropriate class in
-``stats/dashboard.py`` (or a different stats module).
-
-
-Running for every commit in the data directory
-----------------------------------------------
-
-If the data directory is a git repository (e.g. as a result of running `IATI-Registry-Refresher's git.sh <https://github.com/IATI/IATI-Registry-Refresher#creating-a-git-data-snapshot>`__), you can run the code:
-
-.. code-block:: bash
-
-    # WARNING: This takes a long time (hours) and produces a lot of data (GBs)
-    mkdir gitout
-    ALL_COMMITS=1 ./git.sh
-
-Environment variables for git.sh
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The behaviour of `git.sh` can be modified using environment variables. `git_dashboard.sh` contains the two different runs of `git.sh` that are now used to generate data for the dashboard, each run with different environment variables.
-
-The availible environment variables are:
-
-GITOUT_DIR
-    This is the output directory for git.sh (note that it uses the out directory for each commit, and then moves that to the appropriate place). Defaults to "gitout".
-ALL_COMMITS
-    By default git.sh only computes stats for the most recent commit. To override this, set this environment variable to any non-empty value.
-GITOUT_SKIP_INCOMMITSDIR
-    If this evironment variable has a non-empty value, a commit will be skipped if a directory already exists in $GITOUT_DIR/commits
-COMMIT_SKIP_FILE
-    The name of a file that will be grepped for the commit hash. If the hash exists in the file, the commit will be skipped. Defaults to "$GITOUT_DIR/gitaggregate/activities.json".
+``stats/analytics.py`` (or a different stats module).
 
 License
 -------

diff --git a/get_stats.sh b/get_stats.sh
diff --git a/git.sh b/git.sh
diff --git a/git_dashboard.sh b/git_dashboard.sh
diff --git a/pytest.ini b/pytest.ini
@@ -1,4 +1,4 @@
 [pytest]
-testpaths = stats/tests statsrunner tests
+testpaths = stats/tests statsrunner
 norecursedirs = *__pycache__* *.pytest_cache*
 console_output_style = count
diff --git a/stats/dashboard.py → stats/analytics.py b/stats/dashboard.py → stats/analytics.py
@@ -80,7 +80,7 @@ def is_number(v):
         Return: True if v is a number
                 False if v is not a number
         NOTE: Any changes to this function should be replicated in:
-              https://github.com/IATI/IATI-Dashboard/blob/master/coverage.py#L7
+              https://github.com/codeforIATI/analytics/blob/881f950c/coverage.py#L10
     """
     try:
         float(v)
@@ -94,7 +94,7 @@ def convert_to_float(x):
         Input: x - a variable
         Return: x as a float, or zero if x is not a number
         NOTE: Any changes to this function should be replicated in:
-              https://github.com/IATI/IATI-Dashboard/blob/master/coverage.py#L19
+              https://github.com/codeforIATI/analytics/blob/881f950c/coverage.py#L23
     """
     if is_number(x):
         return float(x)
@@ -740,7 +740,7 @@ def activities_secondary_reported(self):
 
     @returns_numberdictdict
     def forwardlooking_currency_year(self):
-        # Note this is not currently displayed on the dashboard
+        # Note this is not currently displayed on Analytics
         # As the forwardlooking page now only displays counts,
         # not the sums that this function calculates.
         out = defaultdict(lambda: defaultdict(Decimal))

diff --git a/stats/tests/test_budget_not_provided.py b/stats/tests/test_budget_not_provided.py
@@ -1,6 +1,6 @@
 from lxml import etree
 
-from stats.dashboard import ActivityStats
+from stats.analytics import ActivityStats
 
 
 class MockActivityStats(ActivityStats):

diff --git a/stats/tests/test_comprehensiveness.py b/stats/tests/test_comprehensiveness.py
@@ -4,7 +4,7 @@
 import datetime
 import pytest
 
-from stats.dashboard import ActivityStats
+from stats.analytics import ActivityStats
 
 
 class MockActivityStats(ActivityStats):

diff --git a/stats/tests/test_forward_looking.py b/stats/tests/test_forward_looking.py
@@ -2,7 +2,7 @@
 from lxml import etree
 import pytest
 
-from stats.dashboard import ActivityStats
+from stats.analytics import ActivityStats
 from .test_comprehensiveness import MockActivityStats
 
 

diff --git a/stats/tests/test_generic_file_stats.py b/stats/tests/test_generic_file_stats.py
@@ -3,7 +3,7 @@
 from lxml import etree
 import pytest
 
-from stats.dashboard import ActivityStats
+from stats.analytics import ActivityStats
 
 
 @pytest.mark.parametrize('version', ['1.01', '1.02', '1.03', '1.04', '1.05', '2.01', '2.02', '2.03'])

diff --git a/stats/tests/test_hierarchy.py b/stats/tests/test_hierarchy.py
@@ -1,6 +1,6 @@
 from lxml import etree
 
-from stats.dashboard import ActivityStats, PublisherStats
+from stats.analytics import ActivityStats, PublisherStats
 
 
 def test_hierarchies():

diff --git a/stats/tests/test_humanitarian.py b/stats/tests/test_humanitarian.py
@@ -2,7 +2,7 @@
 from lxml import etree
 import pytest
 
-from stats.dashboard import ActivityStats
+from stats.analytics import ActivityStats
 
 
 class MockActivityStats(ActivityStats):

diff --git a/stats/tests/test_valid_functions.py b/stats/tests/test_valid_functions.py
@@ -1,6 +1,7 @@
-from stats.dashboard import valid_coords, valid_date, valid_url, valid_value
 from lxml import etree
 
+from stats.analytics import valid_coords, valid_date, valid_url, valid_value
+
 
 def test_valid_coords():
     """