transaction calculations: fixes #9

iatiflattener/__init__.py

@@ -81,17 +81,35 @@ def setup_organisations(self):
 
 
     def process_transaction(self, csvwriter, activity, transaction):
+        """Called once per <transaction> element in IATI activity XML file to process the transaction
+
+        :param activity: the parent <iati-activity> element
+        :type activity: lxml.etree._Element
+        :param transaction: a <transaction> element
+        :type transaction: lxml.etree._Element
+        """
+
+        # type(activity) is lxml.etree._Element; type(transaction) is lxml.etree._Element
         _transaction = model.Transaction(activity, transaction, self.activity_cache,
-            self.exchange_rates, self.countries_currencies, True, self.organisations, self.langs,
-            self.reporting_organisation_groups)
+                                         self.exchange_rates, self.countries_currencies,
+                                         True, self.organisations, self.langs,
+                                         self.reporting_organisation_groups)
+
+        # the generate() method returns 'self' if successful, otherwise False; so on success,
+        # `generated` refers to the same object as `_transaction`.
+        # after `generate()` has been called, its `value_local` value is a dictionary indexed by country code, where
+        # each value--e.g., generated.value_local['AT']--is the value of the entire transaction in the currency of the
+        # country specified.
         generated = _transaction.generate()
+
         if generated is not False:
-            _flat_transaction = model.FlatTransaction(_transaction, self.category_group).flatten()
-            for _part_flat_transaction in _flat_transaction:
-                transaction_csv = model.FlatTransactionBudgetCSV(
-                countries=self.countries,
-                csv_writer=csvwriter,
-                flat_transaction_budget=_part_flat_transaction).output()
+            _flat_transaction = model.FlatTransaction(_transaction, self.category_group)
+            _flattened_transaction = _flat_transaction.flatten()
+            for _part_flat_transaction in _flattened_transaction:
+                transaction_csv = model.FlatTransactionBudgetCSV(countries=self.countries,
+                                                                 csv_writer=csvwriter,
+                                                                 flat_transaction_budget=_part_flat_transaction)
+                transaction_csv.output()
 
 
     def process_activity_for_budgets(self, csvwriter, activity):
@@ -121,7 +139,13 @@ def process_activity(self, csvwriter, activity):
 
 
     def process_package(self, publisher, package, root_dir):
-        """Read the activity elements from XML and write out flattened rows to transaction-NN.csv and budget-NN.csv"""
+        """Read the activity elements from XML and write out flattened rows to transaction-NN.csv and budget-NN.csv
+
+        :param publisher: the publisher of the package being processed
+        :type publisher: str
+        :param package: the filename of the XML file (the package) to be processed
+        :type package: str
+        """
 
         doc = etree.parse(os.path.join(root_dir, "{}".format(package)))
         if doc.getroot().get("version") not in ['2.01', '2.02', '2.03']: return
@@ -130,15 +154,21 @@ def process_package(self, publisher, package, root_dir):
         activity_csvwriter = model.ActivityCSVFilesWriter(self.output_dir, headers=self.activity_csv_headers)
         activities = doc.xpath("//iati-activity")
         for activity in activities:
+            # type(activity) is lxml.etree._Element
             self.process_activity(activity_csvwriter, activity)
 
         activity_csvwriter.write()
 
+        # csvwriter is an instance of CSVFilesWriter; csvwriter.csvfiles is a dictionary indexed by country code
+        # which contains a CSV file handle for writing to the CSV ('file'), a csv writer object ('csv'), and a list
+        # rows, which is populated below with all the rows to be added to the CSV ('rows')
         csvwriter = model.CSVFilesWriter(budget_transaction='transaction',
                                          headers=self.csv_headers,
                                          output_dir=self.output_dir)
+
         transactions = doc.xpath("//transaction")
         for transaction in transactions:
+            # transaction.getparent() gets the <iati-activity> element
             self.process_transaction(csvwriter, transaction.getparent(), transaction)
 
         csvwriter.write()
@@ -167,7 +197,7 @@ def run_for_publishers(self):
                     try:
                         if package.endswith(".xml"):
                             self.process_package(publisher, package,
-                                os.path.join(self.iatikitcache_dir, "data", publisher))
+                                                 os.path.join(self.iatikitcache_dir, "data", publisher))
                     except BdbQuit:
                         raise
                     except Exception as e:

iatiflattener/model.py

@@ -50,8 +50,16 @@ def __init__(self, iati_identifier):
             setattr(self, field, None)
 
 
-class ActivityCache():
+class ActivityCache:
+    """Keeps a cache of ActivityCacheActivity of objects"""
+
     def get(self, iati_identifier):
+        """Gets (*or adds*) an ActivityCacheActivity from (to) the cache
+
+        :param iati_identifier: activity identifier
+        :type iati_identifier: str
+        """
+
         activity = self.data.get(iati_identifier)
         if activity is None:
             self.data[iati_identifier] = ActivityCacheActivity(iati_identifier)
@@ -122,9 +130,8 @@ def __init__(self, output_dir='output', headers=[]):
         self.csv_headers = headers
 
 
-class FlatBudget():
-    def make_flattened(self, country, sector, aid_type,
-            finance_type, flow_type, budget):
+class FlatBudget:
+    def make_flattened(self, country, sector, aid_type, finance_type, flow_type, budget):
         for k, v in budget.items():
             self.flat_budget[k] = v
         self.flat_budget['country_code'] = country.get('code')
@@ -145,7 +152,9 @@ def make_flattened(self, country, sector, aid_type,
         self.flat_budget['value_eur'] = (
             budget['value_eur'] * pct_adjustment
         )
-        self.flat_budget['value_local'] = dict([(country, (value_local * pct_adjustment)) for country, value_local in self.flat_budget['value_local'].items()])
+
+        self.flat_budget['value_local'] = dict([(country, (value_local * pct_adjustment)) for \
+                                                 country, value_local in self.flat_budget['value_local'].items()])
         return dict([(k, v) for k, v in self.flat_budget.items() if k not in ['countries', 'sectors']])
 
     def flatten(self):
@@ -183,18 +192,26 @@ def __init__(self, budget, sector_categories={}):
         self.flat_budget = self.budget_dict
 
 
-class FlatTransactionBudgetCSV():
+class FlatTransactionBudgetCSV:
+    """Class takes a flattened transaction/budget obj, sets 'value_local' to value for current country, writes to CSV"""
+
     def get_local_currency(self, country, flat_transaction_budget):
+        """Reduces the 'value_local' list to a single value for just the current country"""
         flat_transaction_budget['value_local'] = flat_transaction_budget['value_local'].get(country)
         return flat_transaction_budget
 
     def output(self):
         country = self.flat_transaction_budget['country_code']
         if country in self.countries:
             self.csv_writer.append(country=country,
-                flat_transaction_budget=self.get_local_currency(country, self.flat_transaction_budget))
+                                   flat_transaction_budget=self.get_local_currency(country, self.flat_transaction_budget))
 
     def __init__(self, countries, csv_writer, flat_transaction_budget):
+        """
+
+        :param flat_transaction_budget:
+        :type flat_transaction_budget: dictionary
+        """
         self.countries = countries
         self.csv_writer = csv_writer
         self.flat_transaction_budget = flat_transaction_budget
@@ -213,25 +230,38 @@ def __init__(self, organisations, csv_writer, activity_data):
         self.activity_data = activity_data
 
 
-class FlatTransaction():
+class FlatTransaction:
     def make_flattened(self, sector, country):
+        """
+        :param sector: dictionary with sector code and percentage
+        :type sector: {'code':str, 'percentage': float}
+        :example: {'code': '32182', 'percentage': 72.0}
+
+        :param country: dictionary with country code and percentage
+        :type country: {'country': str, 'percentage': float}
+        """
         self.flat_transaction['country_code'] = country.get('code')
         self.flat_transaction['sector_code'] = sector.get('code')
         self.flat_transaction['sector_category'] = get_sector_category(sector.get('code'), self.sector_categories)
         sector_pct_adjustment = (country['percentage']/100) * (sector['percentage']/100)
-        self.flat_transaction['value_original'] = (
-            self.transaction.value_original.value * sector_pct_adjustment
-        )
-        self.flat_transaction['value_usd'] = (
-            self.transaction.value_usd.value * sector_pct_adjustment
-        )
-        self.flat_transaction['value_eur'] = (
-            self.transaction.value_eur.value * sector_pct_adjustment
-        )
-        self.flat_transaction['value_local'] = dict([(country, (value_local * sector_pct_adjustment)) for country, value_local in self.flat_transaction['value_local'].items()])
+
+        self.flat_transaction['value_original'] = (self.transaction.value_original.value * sector_pct_adjustment)
+        self.flat_transaction['value_usd'] = (self.transaction.value_usd.value * sector_pct_adjustment)
+        self.flat_transaction['value_eur'] = (self.transaction.value_eur.value * sector_pct_adjustment)
+
+        # the `value_local` dict has the total value of the transaction listed in each country's local currency; this
+        # adjusts it according to proportion of total allocated to sector/country
+        self.flat_transaction['value_local'] = dict([(country, (value_local * sector_pct_adjustment)) for
+                                                      country, value_local in self.transaction.value_local.value.items()])
+
+        # this returns a new dictionary which is self.flat_transaction but without 'countries' and 'sectors'
         return dict([(k, v) for k, v in self.flat_transaction.items() if k not in ['countries', 'sectors']])
 
     def flatten(self):
+        """
+        :return:
+        :rtype: generator
+        """
         for sector in self.transaction.sectors.value:
             for country in self.transaction.countries.value:
                 yield self.make_flattened(sector, country)
@@ -271,9 +301,7 @@ def _values_local(self):
         for country in self.countries.value:
             currency_code = self.currencies.get(country.get('code'))
             try:
-                closest_exchange_rate = self.exchange_rates.closest_rate(
-                    currency_code, self.value_date.value
-                )
+                closest_exchange_rate = self.exchange_rates.closest_rate(currency_code, self.value_date.value)
                 exchange_rate = closest_exchange_rate.get('conversion_rate')
                 value = self.value_usd.value * exchange_rate
                 out[country.get('code')] = value
@@ -637,6 +665,8 @@ def generate(self):
     def __init__(self, activity, activity_cache,
             organisations_cache={}, langs=['en'],
             reporting_organisation_groups={}):
+
+        # type(activity) = lxml.etree._Element
         self.activity = activity
         self.activity_cache = activity_cache.get(
             self._iati_identifier().value
@@ -721,6 +751,8 @@ def generate(self):
     def __init__(self, activity, transaction, activity_cache, exchange_rates,
             currencies, limit_transaction_types=True, organisations_cache={},
             langs=['en'], reporting_organisation_groups={}):
+
+        # type(transaction) is lxml.etree._Element
         self.transaction = transaction
         self.activity = activity
         self.activity_cache = activity_cache.get(

iatiflattener/tests/artefacts/fcdo-transaction-flat.json

@@ -1 +1 @@
-{"iati_identifier": "GB-1-103662-101", "title#en": "PROCOFSERVICES and P0220 for Civil Ser. Cap. Bldng. Liberia", "reporting_org_group": null, "reporting_org#en": "Foreign, Commonwealth and Development Office [GB-GOV-1]", "reporting_org_type": "10", "multi_country": 0, "humanitarian": 0, "aid_type": "C01", "finance_type": "110", "flow_type": "10", "provider_org#en": "UK - Foreign, Commonwealth and Development Office (FCDO) [GB-GOV-1]", "provider_org_type": null, "receiver_org#en": "Adam Smith International [GB-COH-2732176]", "receiver_org_type": "70", "transaction_type": "3", "value_original": 1232.0, "currency_original": "GBP", "value_date": "2010-05-27", "transaction_date": "2010-05-27", "fiscal_year": 2010, "fiscal_quarter": "Q2", "fiscal_year_quarter": "2010 Q2", "exchange_rate": 0.726269155, "exchange_rate_date": "2021-08-31", "value_usd": 1696.3408008150918, "value_eur": 1433.446680400464, "value_local": {"LR": 0.0}, "url": "https://d-portal.org/q.html?aid=GB-1-103662-101", "country_code": "LR", "sector_code": "15110", "sector_category": ""}
+{"iati_identifier": "GB-1-103662-101", "title#en": "PROCOFSERVICES and P0220 for Civil Ser. Cap. Bldng. Liberia", "reporting_org_group": null, "reporting_org#en": "Foreign, Commonwealth and Development Office [GB-GOV-1]", "reporting_org_type": "10", "multi_country": 0, "humanitarian": 0, "aid_type": "C01", "finance_type": "110", "flow_type": "10", "provider_org#en": "UK - Foreign, Commonwealth and Development Office (FCDO) [GB-GOV-1]", "provider_org_type": null, "receiver_org#en": "Adam Smith International [GB-COH-2732176]", "receiver_org_type": "70", "transaction_type": "3", "value_original": 1232.0, "currency_original": "GBP", "value_date": "2010-05-27", "transaction_date": "2010-05-27", "fiscal_year": 2010, "fiscal_quarter": "Q2", "fiscal_year_quarter": "2010 Q2", "exchange_rate": 0.726269155, "exchange_rate_date": "2021-08-31", "value_usd": 1696.3408008150918, "value_eur": 1433.446680400464, "value_local": {"LR": 291426.5998257905}, "url": "https://d-portal.org/q.html?aid=GB-1-103662-101", "country_code": "LR", "sector_code": "15110", "sector_category": ""}

iatiflattener/tests/artefacts/fcdo-transaction.json

@@ -1 +1 @@
-{"iati_identifier": "GB-1-103662-101", "title": {"en": "PROCOFSERVICES and P0220 for Civil Ser. Cap. Bldng. Liberia"}, "reporting_org_group": null, "reporting_org": {"en": {"text": "Foreign, Commonwealth and Development Office", "type": "10", "ref": "GB-GOV-1", "display": "Foreign, Commonwealth and Development Office [GB-GOV-1]"}}, "reporting_org_type": "10", "countries": [{"percentage": 100.0, "code": "LR"}], "sectors": [{"percentage": 100.0, "code": "15110"}], "multi_country": 0, "humanitarian": 0, "aid_type": "C01", "finance_type": "110", "flow_type": "10", "provider_org": {"en": {"text": "UK - Foreign, Commonwealth and Development Office (FCDO)", "ref": "GB-GOV-1", "type": null, "display": "UK - Foreign, Commonwealth and Development Office (FCDO) [GB-GOV-1]"}}, "provider_org_type": null, "receiver_org": {"en": {"text": "Adam Smith International", "ref": "GB-COH-2732176", "type": "70", "display": "Adam Smith International [GB-COH-2732176]"}}, "receiver_org_type": "70", "transaction_type": "3", "value_original": 1232.0, "currency_original": "GBP", "value_date": "2010-05-27", "transaction_date": "2010-05-27", "fiscal_year": 2010, "fiscal_quarter": "Q2", "fiscal_year_quarter": "2010 Q2", "exchange_rate": 0.726269155, "exchange_rate_date": "2021-08-31", "value_usd": 1696.3408008150918, "value_eur": 1433.446680400464, "value_local": {"LR": 0.0}, "url": "https://d-portal.org/q.html?aid=GB-1-103662-101"}
+{"iati_identifier": "GB-1-103662-101", "title": {"en": "PROCOFSERVICES and P0220 for Civil Ser. Cap. Bldng. Liberia"}, "reporting_org_group": null, "reporting_org": {"en": {"text": "Foreign, Commonwealth and Development Office", "type": "10", "ref": "GB-GOV-1", "display": "Foreign, Commonwealth and Development Office [GB-GOV-1]"}}, "reporting_org_type": "10", "countries": [{"percentage": 100.0, "code": "LR"}], "sectors": [{"percentage": 100.0, "code": "15110"}], "multi_country": 0, "humanitarian": 0, "aid_type": "C01", "finance_type": "110", "flow_type": "10", "provider_org": {"en": {"text": "UK - Foreign, Commonwealth and Development Office (FCDO)", "ref": "GB-GOV-1", "type": null, "display": "UK - Foreign, Commonwealth and Development Office (FCDO) [GB-GOV-1]"}}, "provider_org_type": null, "receiver_org": {"en": {"text": "Adam Smith International", "ref": "GB-COH-2732176", "type": "70", "display": "Adam Smith International [GB-COH-2732176]"}}, "receiver_org_type": "70", "transaction_type": "3", "value_original": 1232.0, "currency_original": "GBP", "value_date": "2010-05-27", "transaction_date": "2010-05-27", "fiscal_year": 2010, "fiscal_quarter": "Q2", "fiscal_year_quarter": "2010 Q2", "exchange_rate": 0.726269155, "exchange_rate_date": "2021-08-31", "value_usd": 1696.3408008150918, "value_eur": 1433.446680400464, "value_local": {"LR": 291426.5998257905}, "url": "https://d-portal.org/q.html?aid=GB-1-103662-101"}

iatiflattener/tests/artefacts/ifad-transaction-flat.json

@@ -1 +1 @@
-{"iati_identifier": "XM-DAC-41108-1100001761", "title#en": "Rural Development: Tree Crops Extension Project", "reporting_org_group": null, "reporting_org#en": "International Fund for Agricultural Development [XM-DAC-41108]", "reporting_org_type": "40", "multi_country": 0, "humanitarian": 0, "aid_type": "C01", "finance_type": "421", "flow_type": "20", "provider_org#en": "International Fund for Agricultural Development [XM-DAC-41108]", "provider_org_type": "40", "receiver_org#en": null, "receiver_org_type": null, "transaction_type": "2", "value_original": 9480000.0, "currency_original": "XDR", "value_date": "2015-12-10", "transaction_date": "2015-12-10", "fiscal_year": 2015, "fiscal_quarter": "Q4", "fiscal_year_quarter": "2015 Q4", "exchange_rate": 0.702121, "exchange_rate_date": "2021-08-31", "value_usd": 13501946.245732574, "value_eur": 11409452.638049567, "value_local": {"LR": 0.0}, "url": "https://d-portal.org/q.html?aid=XM-DAC-41108-1100001761", "country_code": "LR", "sector_code": "", "sector_category": ""}
+{"iati_identifier": "XM-DAC-41108-1100001761", "title#en": "Rural Development: Tree Crops Extension Project", "reporting_org_group": null, "reporting_org#en": "International Fund for Agricultural Development [XM-DAC-41108]", "reporting_org_type": "40", "multi_country": 0, "humanitarian": 0, "aid_type": "C01", "finance_type": "421", "flow_type": "20", "provider_org#en": "International Fund for Agricultural Development [XM-DAC-41108]", "provider_org_type": "40", "receiver_org#en": null, "receiver_org_type": null, "transaction_type": "2", "value_original": 9480000.0, "currency_original": "XDR", "value_date": "2015-12-10", "transaction_date": "2015-12-10", "fiscal_year": 2015, "fiscal_quarter": "Q4", "fiscal_year_quarter": "2015 Q4", "exchange_rate": 0.702121, "exchange_rate_date": "2021-08-31", "value_usd": 13501946.245732574, "value_eur": 11409452.638049567, "value_local": {"LR": 2319596559.567368}, "url": "https://d-portal.org/q.html?aid=XM-DAC-41108-1100001761", "country_code": "LR", "sector_code": "", "sector_category": ""}