diff --git a/git.sh b/git.sh index 2a5e1a35b..2d8b69d30 100755 --- a/git.sh +++ b/git.sh @@ -13,7 +13,7 @@ echo "LOG: `date '+%Y-%m-%d %H:%M:%S'` - Running make_plots.py" python -m iati_dashboard.make_plots || exit 1 echo "LOG: `date '+%Y-%m-%d %H:%M:%S'` - Running make_csv.py" -python -m iati_dashboard.make_csv || exit 1 +python manage.py make_csv || exit 1 echo "LOG: `date '+%Y-%m-%d %H:%M:%S'` - Running speakers_kit.py" python -m iati_dashboard.speakers_kit || exit 1 diff --git a/iati_dashboard/make_csv.py b/iati_dashboard/make_csv.py index 05eb29f37..eaa4cb2ce 100644 --- a/iati_dashboard/make_csv.py +++ b/iati_dashboard/make_csv.py @@ -1,13 +1,12 @@ """Generates CSV files from data in the 'stats-calculated' folder and using additional logic """ -import argparse import csv import logging import os import sys -from . import comprehensiveness, data, filepaths, forwardlooking, humanitarian, summary_stats, timeliness +from . import comprehensiveness, data, filepaths, forwardlooking, models, summary_stats, timeliness from .ui.jinja2 import round_nicely logger = logging.getLogger(__name__) @@ -38,14 +37,9 @@ def publisher_dicts(): } -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--verbose", action="store_true", help="Output progress to stdout") - args = parser.parse_args() - - if args.verbose: - logger.setLevel(logging.INFO) - logger.addHandler(logging.StreamHandler(sys.stdout)) +def make_csv(verbose=False): + logger.setLevel(logging.INFO) + logger.addHandler(logging.StreamHandler(sys.stdout)) logger.info("Generating CSV files") os.makedirs(filepaths.join_out_path("data/csv"), exist_ok=True) @@ -131,16 +125,13 @@ def main(): writer.writerow( ["Publisher Name", "Publisher Registry Id"] + previous_months + ["Frequency", "First published"] ) - for ( - publisher, - publisher_title, - per_month, - assessment, - hft, - first_published_band, - ) in timeliness.publisher_frequency_sorted(): + for publisher in models.Publisher.objects.all(): + per_month = publisher.timeliness_frequency["updates_per_month"] + first_published_band = publisher.timeliness_frequency["first_published_band"] + assessment = publisher.timeliness_frequency["frequency"] + # hft=publisher.has_future_transactions writer.writerow( - [publisher_title, publisher] + [publisher.human_readable_name, publisher.short_name] + [per_month.get(x) or 0 for x in previous_months] + [assessment, first_published_band] ) @@ -149,9 +140,15 @@ def main(): with open(filepaths.join_out_path("data/csv/timeliness_timelag.csv"), "w") as fp: writer = csv.writer(fp) writer.writerow(["Publisher Name", "Publisher Registry Id"] + previous_months + ["Time lag"]) - for publisher, publisher_title, per_month, assessment, hft in timeliness.publisher_timelag_sorted(): + for publisher in models.Publisher.objects.all(): + per_month = publisher.stats_json["transaction_months_with_year"] + # hft=publisher.has_future_transactions + previous_months = timeliness.previous_months_reversed + assessment = publisher.stats_json["timelag"] writer.writerow( - [publisher_title, publisher] + [per_month.get(x) or 0 for x in previous_months] + [assessment] + [publisher.human_readable_name, publisher.short_name] + + [per_month.get(x) or 0 for x in previous_months] + + [assessment] ) logger.info("Generating forwardlooking.csv") @@ -225,15 +222,20 @@ def main(): writer.writerow( ["Publisher Name", "Publisher Registry Id"] + [header for slug, header in summary_stats.columns] ) - for row in summary_stats.table(): + for publisher in models.Publisher.objects.all(): # Write each row - writer.writerow( - [row["publisher_title"], row["publisher"]] - + [ - row[slug] if header == "Publisher Type" else round_nicely(row[slug]) - for slug, header in summary_stats.columns - ] - ) + if publisher.summary_stats: + writer.writerow( + [publisher.human_readable_name, publisher.short_name] + + [ + ( + publisher.summary_stats[column_slug] + if header == "Publisher Type" + else round_nicely(publisher.summary_stats[column_slug]) + ) + for column_slug, header in summary_stats.columns + ] + ) logger.info("Generating humanitarian.csv") with open(filepaths.join_out_path("data/csv/humanitarian.csv"), "w") as fp: @@ -252,21 +254,19 @@ def main(): "Humanitarian Score", ] ) - for row in humanitarian.table(): - writer.writerow( - [ - row["publisher_title"], - row["publisher"], - row["publisher_type"], - row["num_activities"], - round_nicely(row["publishing_humanitarian"]), - round_nicely(row["humanitarian_attrib"]), - round_nicely(row["appeal_emergency"]), - round_nicely(row["clusters"]), - round_nicely(row["average"]), - ] - ) - - -if __name__ == "__main__": - main() + for publisher in models.Publisher.objects.all(): + row = publisher.humanitarian + if row: + writer.writerow( + [ + publisher.human_readable_name, + publisher.short_name, + row["publisher_type"], + row["num_activities"], + round_nicely(row["publishing_humanitarian"]), + round_nicely(row["humanitarian_attrib"]), + round_nicely(row["appeal_emergency"]), + round_nicely(row["clusters"]), + round_nicely(row["average"]), + ] + ) diff --git a/iati_dashboard/management/commands/make_csv.py b/iati_dashboard/management/commands/make_csv.py new file mode 100644 index 000000000..5078f5cb0 --- /dev/null +++ b/iati_dashboard/management/commands/make_csv.py @@ -0,0 +1,10 @@ +from django.core.management.base import BaseCommand +from iati_dashboard.make_csv import make_csv + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument("--verbose", action="store_true", help="Output progress to stdout") + + def handle(self, *args, **options): + make_csv(options.get("verbose", False))