Skip to content

Commit

Permalink
Merge pull request #24 from materials-data-facility/forge-dev
Browse files Browse the repository at this point in the history 
Forge dev
  • Loading branch information
@jgaff
jgaff authored Apr 11, 2018
2 parents 5d9a022 + a55b52f commit d63020b
Show file tree
Hide file tree
Showing 17 changed files with 1,555 additions and 2,018 deletions.
4 changes: 2 additions & 2 deletions .travis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ before_install:
- openssl aes-256-cbc -K $encrypted_39a50b90a369_key -iv $encrypted_39a50b90a369_iv
-in travis.tar.enc -out travis.tar -d
- tar xvf travis.tar
- mkdir -p ~/mdf/credentials/
- mv MDF_Forge_tokens.json ~/mdf/credentials/MDF_Forge_tokens.json
- mkdir -p ~/.mdf/credentials/
- mv MDF_Forge_tokens.json ~/.mdf/credentials/MDF_Forge_tokens.json
after_success:
- coveralls
notifications:
Expand Down
248 changes: 150 additions & 98 deletions docs/examples/Example Aggregations.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
"metadata": {},
"source": [
"## aggregate_source - NIST XPS DB\n",
"Example: We want to collect all records from the NIST XPS Database and analyze the quality metrics. This database has almost 30,000 records, so we have to use `aggregate()`."
"Example: We want to collect all records from the NIST XPS Database and analyze the binding energies. This database has almost 30,000 records, so we have to use `aggregate()`."
]
},
{
Expand All @@ -50,7 +50,7 @@
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 29189/29189 [00:27<00:00, 1067.83it/s]"
"100%|██████████| 29189/29189 [00:27<00:00, 1056.63it/s]"
]
},
{
Expand All @@ -70,34 +70,82 @@
],
"source": [
"# First, let's aggregate all the nist_xps_db data.\n",
"all_entries = mdf.aggregate_source(\"nist_xps_db\")\n",
"all_entries = mdf.aggregate_sources(\"nist_xps_db\")\n",
"print(len(all_entries))"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'': 15940, 'good': 4, 'Good': 1615, 'Adequate': 11630}\n"
"{\n",
" \"\": 24344,\n",
" \"0.001\": 2,\n",
" \"0.003\": 2,\n",
" \"0.005\": 12,\n",
" \"0.01\": 27,\n",
" \"0.010\": 3,\n",
" \"0.012\": 2,\n",
" \"0.015\": 2,\n",
" \"0.02\": 159,\n",
" \"0.020\": 6,\n",
" \"0.025\": 10,\n",
" \"0.026\": 1,\n",
" \"0.03\": 122,\n",
" \"0.030\": 14,\n",
" \"0.04\": 56,\n",
" \"0.042\": 1,\n",
" \"0.05\": 416,\n",
" \"0.050\": 1,\n",
" \"0.06\": 12,\n",
" \"0.07\": 33,\n",
" \"0.070\": 1,\n",
" \"0.075\": 1,\n",
" \"0.08\": 14,\n",
" \"0.1\": 1501,\n",
" \"0.10\": 14,\n",
" \"0.100\": 2,\n",
" \"0.12\": 4,\n",
" \"0.13\": 1,\n",
" \"0.15\": 220,\n",
" \"0.17\": 1,\n",
" \"0.2\": 1660,\n",
" \"0.20\": 2,\n",
" \"0.200\": 1,\n",
" \"0.25\": 24,\n",
" \"0.3\": 266,\n",
" \"0.30\": 4,\n",
" \"0.4\": 117,\n",
" \"0.40\": 4,\n",
" \"0.5\": 108,\n",
" \"0.6\": 9,\n",
" \"0.7\": 2,\n",
" \"0.8\": 4,\n",
" \"0.9\": 2,\n",
" \"1.2\": 1,\n",
" \"2.0\": 1\n",
"}\n"
]
}
],
"source": [
"# Now, let's parse out the \"Quality of Data\" and print te results for analysis.\n",
"qualities = {}\n",
"# Now, let's parse out the enery_uncertainty_ev and print the results for analysis.\n",
"uncertainties = {}\n",
"for record in all_entries:\n",
" if record[\"mdf\"][\"resource_type\"] == \"record\":\n",
" raw = json.loads(record[\"mdf\"][\"raw\"])\n",
" if raw[\"Quality of Data\"] in qualities.keys():\n",
" qualities[raw[\"Quality of Data\"]] += 1\n",
" unc = record.get(\"nist_xps_db_v1\", {}).get(\"energy_uncertainty_ev\", 0)\n",
" if not uncertainties.get(unc):\n",
" uncertainties[unc] = 1\n",
" else:\n",
" qualities[raw[\"Quality of Data\"]] = 1\n",
"print(qualities)"
" uncertainties[unc] += 1\n",
"print(json.dumps(uncertainties, sort_keys=True, indent=4, separators=(',', ': ')))"
]
},
{
Expand All @@ -117,14 +165,14 @@
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 29168/29168 [01:07<00:00, 298.34it/s]"
"100%|██████████| 25582/25582 [01:03<00:00, 381.77it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"29168\n"
"25582\n"
]
},
{
Expand All @@ -137,106 +185,110 @@
],
"source": [
"# First, let's aggregate everything that has \"Ga\" in the list of elements.\n",
"all_results = mdf.aggregate(\"mdf.elements:Ga\")\n",
"all_results = mdf.aggregate(\"material.elements:Ga\")\n",
"print(len(all_results))"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{\n",
" \"Ac\": 651,\n",
" \"Ag\": 588,\n",
" \"Al\": 576,\n",
" \"Ag\": 550,\n",
" \"Al\": 556,\n",
" \"Ar\": 2,\n",
" \"As\": 1330,\n",
" \"Au\": 649,\n",
" \"B\": 681,\n",
" \"Ba\": 802,\n",
" \"As\": 1296,\n",
" \"Au\": 589,\n",
" \"B\": 528,\n",
" \"Ba\": 670,\n",
" \"Be\": 496,\n",
" \"Bi\": 583,\n",
" \"Br\": 127,\n",
" \"C\": 1843,\n",
" \"Ca\": 682,\n",
" \"Cd\": 581,\n",
" \"Ce\": 639,\n",
" \"Cl\": 672,\n",
" \"Co\": 954,\n",
" \"Cr\": 712,\n",
" \"Cs\": 552,\n",
" \"Cu\": 817,\n",
" \"Dy\": 600,\n",
" \"Er\": 670,\n",
" \"Eu\": 614,\n",
" \"F\": 344,\n",
" \"Fe\": 793,\n",
" \"Ga\": 29168,\n",
" \"Gd\": 612,\n",
" \"Ge\": 776,\n",
" \"H\": 1933,\n",
" \"Hf\": 659,\n",
" \"Hg\": 534,\n",
" \"Ho\": 598,\n",
" \"I\": 187,\n",
" \"In\": 603,\n",
" \"Ir\": 557,\n",
" \"K\": 698,\n",
" \"La\": 960,\n",
" \"Li\": 990,\n",
" \"Lu\": 527,\n",
" \"Mg\": 955,\n",
" \"Mn\": 715,\n",
" \"Mo\": 606,\n",
" \"N\": 1402,\n",
" \"Na\": 840,\n",
" \"Nb\": 564,\n",
" \"Nd\": 616,\n",
" \"Ni\": 836,\n",
" \"Np\": 506,\n",
" \"O\": 4031,\n",
" \"Os\": 668,\n",
" \"P\": 676,\n",
" \"Pa\": 607,\n",
" \"Pb\": 544,\n",
" \"Pd\": 653,\n",
" \"Bi\": 550,\n",
" \"Br\": 52,\n",
" \"C\": 85,\n",
" \"Ca\": 613,\n",
" \"Cd\": 562,\n",
" \"Ce\": 599,\n",
" \"Cl\": 75,\n",
" \"Co\": 875,\n",
" \"Cr\": 678,\n",
" \"Cs\": 501,\n",
" \"Cu\": 741,\n",
" \"Dy\": 578,\n",
" \"Er\": 641,\n",
" \"Eu\": 561,\n",
" \"F\": 105,\n",
" \"Fe\": 708,\n",
" \"Ga\": 25582,\n",
" \"Gd\": 575,\n",
" \"Ge\": 643,\n",
" \"H\": 167,\n",
" \"Hf\": 630,\n",
" \"Hg\": 526,\n",
" \"Ho\": 567,\n",
" \"I\": 59,\n",
" \"In\": 585,\n",
" \"Ir\": 543,\n",
" \"K\": 583,\n",
" \"La\": 719,\n",
" \"Li\": 851,\n",
" \"Lu\": 513,\n",
" \"Mg\": 1004,\n",
" \"Mn\": 608,\n",
" \"Mo\": 635,\n",
" \"N\": 150,\n",
" \"Na\": 722,\n",
" \"Nb\": 539,\n",
" \"Nd\": 573,\n",
" \"Ni\": 754,\n",
" \"Np\": 503,\n",
" \"O\": 2514,\n",
" \"On\": 6,\n",
" \"Os\": 665,\n",
" \"Ox\": 39,\n",
" \"P\": 160,\n",
" \"Pa\": 609,\n",
" \"Pb\": 519,\n",
" \"Pd\": 604,\n",
" \"Pm\": 624,\n",
" \"Pr\": 714,\n",
" \"Pt\": 743,\n",
" \"Pu\": 535,\n",
" \"Rb\": 588,\n",
" \"Re\": 498,\n",
" \"Rh\": 584,\n",
" \"Ru\": 562,\n",
" \"S\": 512,\n",
" \"Sb\": 632,\n",
" \"Sc\": 715,\n",
" \"Se\": 374,\n",
" \"Si\": 1075,\n",
" \"Sm\": 773,\n",
" \"Sn\": 606,\n",
" \"Sr\": 711,\n",
" \"Ta\": 558,\n",
" \"Tb\": 591,\n",
" \"Pr\": 692,\n",
" \"Pt\": 699,\n",
" \"Pu\": 527,\n",
" \"Rb\": 544,\n",
" \"Re\": 496,\n",
" \"Rh\": 554,\n",
" \"Ru\": 533,\n",
" \"S\": 186,\n",
" \"Sb\": 568,\n",
" \"Sc\": 680,\n",
" \"Se\": 167,\n",
" \"Si\": 782,\n",
" \"Sm\": 732,\n",
" \"Sn\": 540,\n",
" \"Sr\": 605,\n",
" \"Ta\": 543,\n",
" \"Tb\": 566,\n",
" \"Tc\": 508,\n",
" \"Te\": 758,\n",
" \"Th\": 519,\n",
" \"Ti\": 693,\n",
" \"Tl\": 687,\n",
" \"Tm\": 590,\n",
" \"U\": 507,\n",
" \"V\": 714,\n",
" \"W\": 563,\n",
" \"Te\": 682,\n",
" \"Th\": 511,\n",
" \"Ti\": 638,\n",
" \"Tl\": 680,\n",
" \"Tm\": 571,\n",
" \"U\": 484,\n",
" \"V\": 676,\n",
" \"W\": 616,\n",
" \"Xe\": 1,\n",
" \"Y\": 724,\n",
" \"Yb\": 741,\n",
" \"Zn\": 672,\n",
" \"Zr\": 648\n",
" \"Y\": 668,\n",
" \"Yb\": 702,\n",
" \"Zn\": 614,\n",
" \"Zr\": 618\n",
"}\n"
]
}
Expand All @@ -246,7 +298,7 @@
"elements = {}\n",
"for record in all_results:\n",
" if record[\"mdf\"][\"resource_type\"] == \"record\":\n",
" elems = record[\"mdf\"][\"elements\"]\n",
" elems = record[\"material\"][\"elements\"]\n",
" for elem in elems:\n",
" if elem in elements.keys():\n",
" elements[elem] += 1\n",
Expand Down Expand Up @@ -279,7 +331,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.2"
"version": "3.6.4"
}
},
"nbformat": 4,
Expand Down
Loading

0 comments on commit d63020b

Please sign in to comment.