Calculate historic daily calorie supply

dag/agriculture.yml

@@ -73,6 +73,62 @@ steps:
   data://grapher/wb/2024-03-26/food_prices_for_nutrition:
     - data://garden/wb/2024-03-26/food_prices_for_nutrition
 
+  #
+  # Harris et al. (2015) - Daily calories in England and Wales, according to various studies.
+  #
+  data://meadow/agriculture/2024-05-23/harris_et_al_2015:
+    - snapshot://agriculture/2024-05-23/harris_et_al_2015.csv
+  #
+  # Floud et al. (2011) - Daily calories in United States and Western Europe.
+  #
+  data://meadow/agriculture/2024-05-23/floud_et_al_2011:
+  - snapshot://agriculture/2024-05-23/floud_et_al_2011_daily_calories_europe.csv
+  - snapshot://agriculture/2024-05-23/floud_et_al_2011_daily_calories_us.csv
+  #
+  # Jonsson (1998) - Daily calories in Iceland.
+  #
+  data://meadow/agriculture/2024-05-23/jonsson_1998:
+    - snapshot://agriculture/2024-05-23/jonsson_1998.csv
+  #
+  # Grigg (1995) - Daily calories in Western Europe.
+  #
+  data://meadow/agriculture/2024-05-23/grigg_1995:
+    - snapshot://agriculture/2024-05-23/grigg_1995.csv
+  #
+  # Fogel (2004) - Daily calories in France and Great Britain.
+  #
+  data://meadow/agriculture/2024-05-23/fogel_2004:
+    - snapshot://agriculture/2024-05-23/fogel_2004.csv
+  #
+  # FAO (2000) - The State of Food and Agriculture 2000.
+  #
+  data://meadow/agriculture/2024-05-23/fao_2000:
+    - snapshot://agriculture/2024-05-23/fao_2000.csv
+  #
+  # FAO (1949) - The State of Food and Agriculture 1949.
+  #
+  data://meadow/agriculture/2024-05-23/fao_1949:
+    - snapshot://agriculture/2024-05-23/fao_1949.csv
+  #
+  # USDA/ERS - Food availability.
+  #
+  data://meadow/usda_ers/2024-05-23/food_availability:
+    - snapshot://usda_ers/2024-05-23/food_availability.xls
+  #
+  # Agriculture - Long-run daily calorie supply per person.
+  #
+  data://garden/agriculture/2024-05-23/daily_calories_per_person:
+    - data://garden/faostat/2024-03-14/faostat_fbsc
+    - data://meadow/agriculture/2024-05-23/harris_et_al_2015
+    - data://meadow/agriculture/2024-05-23/floud_et_al_2011
+    - data://meadow/agriculture/2024-05-23/jonsson_1998
+    - data://meadow/agriculture/2024-05-23/grigg_1995
+    - data://meadow/agriculture/2024-05-23/fogel_2004
+    - data://meadow/agriculture/2024-05-23/fao_2000
+    - data://meadow/agriculture/2024-05-23/fao_1949
+    - data://meadow/usda_ers/2024-05-23/food_availability
+  data://grapher/agriculture/2024-05-23/daily_calories_per_person:
+    - data://garden/agriculture/2024-05-23/daily_calories_per_person
   ######################################################################################################################
   # Older versions to be archived once they are not used by any other steps.
   ######################################################################################################################

etl/steps/data/garden/agriculture/2024-05-23/daily_calories_per_person.countries.json

@@ -0,0 +1,247 @@
+{
+  "Belgium": "Belgium",
+  "Finland": "Finland",
+  "Germany": "Germany",
+  "Iceland": "Iceland",
+  "Italy": "Italy",
+  "Netherlands": "Netherlands",
+  "Norway": "Norway",
+  "Algeria": "Algeria",
+  "Argentina": "Argentina",
+  "Australia": "Australia",
+  "Austria": "Austria",
+  "Brazil": "Brazil",
+  "Burma": "Myanmar",
+  "Cambodia": "Cambodia",
+  "Canada": "Canada",
+  "Chile": "Chile",
+  "China": "China",
+  "Colombia": "Colombia",
+  "Cuba": "Cuba",
+  "Czechoslovakia": "Czechoslovakia",
+  "Denmark": "Denmark",
+  "Egypt": "Egypt",
+  "El Salvador": "El Salvador",
+  "Ethiopia": "Ethiopia",
+  "France": "France",
+  "Greece": "Greece",
+  "Hungary": "Hungary",
+  "India": "India",
+  "Ireland": "Ireland",
+  "Japan": "Japan",
+  "Luxembourg": "Luxembourg",
+  "Madagascar": "Madagascar",
+  "Mexico": "Mexico",
+  "Morocco": "Morocco",
+  "New Zealand": "New Zealand",
+  "Peru": "Peru",
+  "Philippines": "Philippines",
+  "Poland": "Poland",
+  "Portugal": "Portugal",
+  "Spain": "Spain",
+  "Sweden": "Sweden",
+  "Switzerland": "Switzerland",
+  "Thailand": "Thailand",
+  "Tunisia": "Tunisia",
+  "Turkey": "Turkey",
+  "Uganda": "Uganda",
+  "United Kingdom": "United Kingdom",
+  "United States": "United States",
+  "Uruguay": "Uruguay",
+  "Yugoslavia": "Yugoslavia",
+  "Ceylon": "Sri Lanka",
+  "England": "United Kingdom",
+  "England and Wales": "United Kingdom",
+  "Great Britain": "United Kingdom",
+  "Tanganyika": "Tanzania",
+  "Union of South Africa": "South Africa",
+  "Net Food Importing Developing Countries (FAO)": "Net Food Importing Developing Countries (FAO)",
+  "Serbia": "Serbia",
+  "Bermuda": "Bermuda",
+  "Bosnia and Herzegovina": "Bosnia and Herzegovina",
+  "Georgia": "Georgia",
+  "Taiwan": "Taiwan",
+  "North Korea": "North Korea",
+  "Liberia": "Liberia",
+  "Saint Kitts and Nevis": "Saint Kitts and Nevis",
+  "Romania": "Romania",
+  "Africa (FAO)": "Africa (FAO)",
+  "Haiti": "Haiti",
+  "Senegal": "Senegal",
+  "Croatia": "Croatia",
+  "Brunei": "Brunei",
+  "Guinea": "Guinea",
+  "Jamaica": "Jamaica",
+  "Syria": "Syria",
+  "Albania": "Albania",
+  "Southern Africa (FAO)": "Southern Africa (FAO)",
+  "Papua New Guinea": "Papua New Guinea",
+  "Sudan": "Sudan",
+  "Niger": "Niger",
+  "China (FAO)": "China (FAO)",
+  "Southern Asia (FAO)": "Southern Asia (FAO)",
+  "Antigua and Barbuda": "Antigua and Barbuda",
+  "High-income countries": "High-income countries",
+  "Small Island Developing States (FAO)": "Small Island Developing States (FAO)",
+  "Togo": "Togo",
+  "Honduras": "Honduras",
+  "Nicaragua": "Nicaragua",
+  "Lower-middle-income countries": "Lower-middle-income countries",
+  "Middle Africa (FAO)": "Middle Africa (FAO)",
+  "Paraguay": "Paraguay",
+  "South Africa": "South Africa",
+  "Indonesia": "Indonesia",
+  "Western Europe (FAO)": "Western Europe (FAO)",
+  "Northern America (FAO)": "Northern America (FAO)",
+  "Burundi": "Burundi",
+  "Gambia": "Gambia",
+  "Russia": "Russia",
+  "Belarus": "Belarus",
+  "Kenya": "Kenya",
+  "Slovenia": "Slovenia",
+  "Cameroon": "Cameroon",
+  "Micronesia (country)": "Micronesia (country)",
+  "Namibia": "Namibia",
+  "Vanuatu": "Vanuatu",
+  "Bolivia": "Bolivia",
+  "Samoa": "Samoa",
+  "Sri Lanka": "Sri Lanka",
+  "South America": "South America",
+  "Bahamas": "Bahamas",
+  "Nepal": "Nepal",
+  "Zambia": "Zambia",
+  "Mauritius": "Mauritius",
+  "Turkmenistan": "Turkmenistan",
+  "Sudan (former)": "Sudan (former)",
+  "Least Developed Countries (FAO)": "Least Developed Countries (FAO)",
+  "Czechia": "Czechia",
+  "Afghanistan": "Afghanistan",
+  "Sierra Leone": "Sierra Leone",
+  "Asia (FAO)": "Asia (FAO)",
+  "Netherlands Antilles": "Netherlands Antilles",
+  "Mongolia": "Mongolia",
+  "Saudi Arabia": "Saudi Arabia",
+  "Yemen": "Yemen",
+  "Grenada": "Grenada",
+  "Lebanon": "Lebanon",
+  "Cyprus": "Cyprus",
+  "Kiribati": "Kiribati",
+  "North Macedonia": "North Macedonia",
+  "South America (FAO)": "South America (FAO)",
+  "Malaysia": "Malaysia",
+  "Eswatini": "Eswatini",
+  "Barbados": "Barbados",
+  "Uzbekistan": "Uzbekistan",
+  "Angola": "Angola",
+  "French Polynesia": "French Polynesia",
+  "Rwanda": "Rwanda",
+  "Oceania": "Oceania",
+  "Vietnam": "Vietnam",
+  "Slovakia": "Slovakia",
+  "Fiji": "Fiji",
+  "Botswana": "Botswana",
+  "Lithuania": "Lithuania",
+  "Caribbean (FAO)": "Caribbean (FAO)",
+  "Micronesia (FAO)": "Micronesia (FAO)",
+  "Southern Europe (FAO)": "Southern Europe (FAO)",
+  "Guinea-Bissau": "Guinea-Bissau",
+  "Bhutan": "Bhutan",
+  "Dominica": "Dominica",
+  "Land Locked Developing Countries (FAO)": "Land Locked Developing Countries (FAO)",
+  "Asia": "Asia",
+  "Central America (FAO)": "Central America (FAO)",
+  "South-eastern Asia (FAO)": "South-eastern Asia (FAO)",
+  "Azerbaijan": "Azerbaijan",
+  "Malta": "Malta",
+  "Low Income Food Deficit Countries (FAO)": "Low Income Food Deficit Countries (FAO)",
+  "Qatar": "Qatar",
+  "East Timor": "East Timor",
+  "Libya": "Libya",
+  "Comoros": "Comoros",
+  "Cote d'Ivoire": "Cote d'Ivoire",
+  "Nauru": "Nauru",
+  "Armenia": "Armenia",
+  "Upper-middle-income countries": "Upper-middle-income countries",
+  "Tanzania": "Tanzania",
+  "Burkina Faso": "Burkina Faso",
+  "Iran": "Iran",
+  "Eastern Asia (FAO)": "Eastern Asia (FAO)",
+  "Central African Republic": "Central African Republic",
+  "New Caledonia": "New Caledonia",
+  "Europe (FAO)": "Europe (FAO)",
+  "European Union (27)": "European Union (27)",
+  "Moldova": "Moldova",
+  "Belgium-Luxembourg (FAO)": "Belgium-Luxembourg (FAO)",
+  "Benin": "Benin",
+  "Israel": "Israel",
+  "Democratic Republic of Congo": "Democratic Republic of Congo",
+  "Nigeria": "Nigeria",
+  "Africa": "Africa",
+  "Belize": "Belize",
+  "Suriname": "Suriname",
+  "South Sudan": "South Sudan",
+  "Ecuador": "Ecuador",
+  "Cape Verde": "Cape Verde",
+  "Western Asia (FAO)": "Western Asia (FAO)",
+  "Sao Tome and Principe": "Sao Tome and Principe",
+  "Eastern Africa (FAO)": "Eastern Africa (FAO)",
+  "Tajikistan": "Tajikistan",
+  "Low-income countries": "Low-income countries",
+  "Oman": "Oman",
+  "Montenegro": "Montenegro",
+  "Latvia": "Latvia",
+  "Lesotho": "Lesotho",
+  "Bahrain": "Bahrain",
+  "North America": "North America",
+  "Myanmar": "Myanmar",
+  "Seychelles": "Seychelles",
+  "Trinidad and Tobago": "Trinidad and Tobago",
+  "Bulgaria": "Bulgaria",
+  "Western Africa (FAO)": "Western Africa (FAO)",
+  "Panama": "Panama",
+  "Saint Vincent and the Grenadines": "Saint Vincent and the Grenadines",
+  "Chad": "Chad",
+  "World": "World",
+  "Serbia and Montenegro": "Serbia and Montenegro",
+  "Maldives": "Maldives",
+  "Bangladesh": "Bangladesh",
+  "Kyrgyzstan": "Kyrgyzstan",
+  "Northern Europe (FAO)": "Northern Europe (FAO)",
+  "Central Asia (FAO)": "Central Asia (FAO)",
+  "Congo": "Congo",
+  "Europe": "Europe",
+  "Zimbabwe": "Zimbabwe",
+  "Mali": "Mali",
+  "Iraq": "Iraq",
+  "Jordan": "Jordan",
+  "Ghana": "Ghana",
+  "Solomon Islands": "Solomon Islands",
+  "Mozambique": "Mozambique",
+  "Costa Rica": "Costa Rica",
+  "Americas (FAO)": "Americas (FAO)",
+  "Hong Kong": "Hong Kong",
+  "Saint Lucia": "Saint Lucia",
+  "Malawi": "Malawi",
+  "Guyana": "Guyana",
+  "Eastern Europe (FAO)": "Eastern Europe (FAO)",
+  "Pakistan": "Pakistan",
+  "Estonia": "Estonia",
+  "Oceania (FAO)": "Oceania (FAO)",
+  "Djibouti": "Djibouti",
+  "Kuwait": "Kuwait",
+  "Ukraine": "Ukraine",
+  "Gabon": "Gabon",
+  "Mauritania": "Mauritania",
+  "Guatemala": "Guatemala",
+  "South Korea": "South Korea",
+  "Kazakhstan": "Kazakhstan",
+  "Laos": "Laos",
+  "Macao": "Macao",
+  "USSR": "USSR",
+  "United Arab Emirates": "United Arab Emirates",
+  "Dominican Republic": "Dominican Republic",
+  "Ethiopia (former)": "Ethiopia (former)",
+  "Northern Africa (FAO)": "Northern Africa (FAO)",
+  "Venezuela": "Venezuela",
+  "European Union (27) (FAO)": "European Union (27) (FAO)"
+}

etl/steps/data/garden/agriculture/2024-05-23/daily_calories_per_person.meta.yml

@@ -0,0 +1,41 @@
+definitions:
+  common:
+    processing_level: major
+    presentation:
+      topic_tags:
+        - Food Supply
+
+dataset:
+  update_period_days: 365
+
+tables:
+  daily_calories_per_person:
+    title: Daily calorie supply per person
+    variables:
+      daily_calories:
+        title: Daily calorie supply per person
+        unit: kilocalories per day
+        short_unit: kcal
+        description_key:
+          - This data shows per capita daily calorie supply, which is the amount of calories available to an average person, and does necessarily correspond to the calories actually consumed by that person.
+          - Calorie supply is always larger than actual calorie consumption, since there may be waste at the household level.
+          - For historical data, daily calorie supply and consumption are sometimes used interchangeably, due to poor data availability.
+          - "This data does not give a complete picture of nutrition - for a healthy diet [we need much more](https://ourworldindata.org/micronutrient-deficiency) than just energy. But as the most basic criteria of food security, getting enough calories is an important measure. It is used as input for the most important metrics used to assess global malnutrition: [undernourishment](https://ourworldindata.org/undernourishment-definition)."
+        description_processing: |-
+          - For all countries, the data after 1960 is taken from FAOSTAT Food Balances datasets ([old](https://www.fao.org/faostat/en/#data/FBSH) and [new](https://www.fao.org/faostat/en/#data/FBS) methodologies combined).
+          - For the UK: We load Appendix Table from [Harris et al. (2015)](https://www.emerald.com/insight/content/doi/10.1108/S0363-326820150000031003/full/html). From that table, we select values from [Broadberry et al. (2015)](https://www.cambridge.org/core/books/british-economic-growth-12701870/A270234C137117C8E0F1D1E7E6F0DA56) and the corrected values from [Floud et al (2011)](https://www.cambridge.org/core/books/changing-body/DE3BB0E3577205AC26823CF2120D8B7E) (taking the average value of Estimates (A) and (B)).
+          - For the US: For years 1800-1900, we use Table 6.6 of [Floud et al. (2011)](https://www.cambridge.org/core/books/changing-body/DE3BB0E3577205AC26823CF2120D8B7E). For years 1900-1960, we use [the archived table of food supply from USDA](https://www.ers.usda.gov/webdocs/DataFiles/50472/nutrients.xls?v=6096.1).
+          - For Iceland: We use Table 5 of [Jonsson (1994)](https://www.tandfonline.com/doi/abs/10.1080/03585522.1998.10414677).
+          - For Finland, Germany, Italy, Norway: We use Table 1 from [Grigg (1995)](https://www.sciencedirect.com/science/article/abs/pii/S0305748885700187), which is a compilation of many sources.
+          - For France: We use Table 1 from Grigg (1995).
+            - We include the two additional data points (1705 and 1785) from [Fogel (2004)](https://www.cambridge.org/core/books/escape-from-hunger-and-premature-death-17002100/384C6032DE4E73C90EF6C9D1E55009CA).
+          - For Belgium and Netherlands: We use Table 5.5 of Floud et al. (2011).
+          - For Uganda, Cambodia, China, India, Brazil, Mexico, and Peru for 1936 and 1947: We use Table 11 of [FAO (2000)](https://www.fao.org/4/x4400e/x4400e.pdf) (The State of Food and Agriculture).
+          - For many countries (including some of the above) for 1947 and 1948: We use values from Table 15 from [FAO (1949)](https://www.fao.org/4/ap637e/ap637e.pdf).
+          - Note that prior to 1961, data for the UK may correspond to England, or England and Wales; and Tanzania refers to Tanganyika.
+        display:
+          numDecimalPlaces: 0
+      source:
+        title: Source of the data point
+        unit: ""
+        short_unit: ""