Merge pull request #223 from Matico-Platform/develop

Develop

.editorconfig

@@ -0,0 +1,8 @@
+root = true
+
+[*]
+indent_style = space
+indent_size = 2
+end_of_line = lf
+charset = utf-8
+insert_final_newline = true

.gitignore

@@ -2,6 +2,7 @@ target
 node_modules
 dist/
 .cache
+.DS_Store
 
 .yarn/*
 !.yarn/cache

.husky/pre-commit

@@ -0,0 +1,5 @@
+#!/usr/bin/env sh
+. "$(dirname -- "$0")/_/husky.sh"
+
+yarn prettier-write
+yarn rust-format

.prettierignore

@@ -0,0 +1,8 @@
+# Ignore artifacts:
+build
+coverage
+
+# Ignore these file directories
+node_modules/*
+.yarn/
+editor/

.prettierrc.json

@@ -0,0 +1 @@
+{}

data_prep/process_covid.py

@@ -0,0 +1,51 @@
+import pyarrow.parquet as pq
+import s3fs
+import pandas as pd 
+from zipfile import ZipFile
+import numpy as np
+import json
+s3 = s3fs.S3FileSystem()
+
+with open('state_lookup.json','r') as f:
+    states = json.load(f)
+
+states_inv = {v: k for k, v in states.items()} 
+
+with ZipFile("./us_covid_atlas_data_2022-10-11.zip") as zf:
+    cases = pd.read_csv(zf.open("data/covid_confirmed_nyt-2022-10-11.csv"),dtype={"fips":'str'})
+    cases.fips = cases.fips.astype('str').str.pad(width=5, side='left', fillchar='0')
+    cases = cases.melt(id_vars="fips", var_name="date", value_name='cases').fillna(0)
+    cases.cases= cases.cases.astype(np.int32)
+
+    deaths = pd.read_csv(zf.open("data/covid_deaths_nyt-2022-10-11.csv"))
+    deaths.fips = deaths.fips.astype("str").str.pad(width=5, side='left', fillchar='0')
+    deaths = deaths.melt(id_vars="fips", var_name="date", value_name='deaths').fillna(0)
+    deaths.deaths = deaths.deaths.astype(np.int32)
+
+    combined = pd.merge(cases,deaths, on=['fips','date'])
+
+    vaccinations = pd.read_csv(zf.open('data/vaccination_fully_vaccinated_cdc-2022-10-11.csv'))
+    vaccinations.fips = vaccinations.fips.astype('str').str.pad(width=5, side='left', fillchar='0')
+    vaccinations = vaccinations.melt(id_vars="fips", var_name="date", value_name='vaccinations').fillna(0)
+    vaccinations.vaccinations = vaccinations.vaccinations.astype(np.int32)
+
+    combined = pd.merge(combined,vaccinations, on=['fips','date'])
+    combined = combined.assign(state = combined.fips.str[0:2].apply(lambda x: states_inv[x]) )
+    combined.fips = combined.fips.astype(np.int32)
+    combined.date = pd.to_datetime(combined.date)
+
+for (state,df) in combined.groupby('state'):
+    result = df.reset_index().drop(columns=['index'])
+    result = result.sort_values('date')
+    result.cases = result.cases - result.groupby('fips').cases.shift(1)
+    result.deaths = result.deaths- result.groupby('fips').deaths.shift(1)
+    result.vaccinations= result.vaccinations- result.groupby('fips').vaccinations.shift(1)
+    result = result.fillna(0)
+    result.reset_index().to_feather(f"covid/{state}.feather", compression='uncompressed')
+
+
+combined.cases = combined.cases - combined.groupby('fips').cases.shift(1)
+combined.deaths = combined.deaths- combined.groupby('fips').deaths.shift(1)
+combined.vaccinations= combined.vaccinations- combined.groupby('fips').vaccinations.shift(1)
+combined=  combined.fillna(0)
+combined.reset_index().to_feather(f"covid/all.feather", compression='uncompressed')

data_prep/state_lookup.json

@@ -0,0 +1,57 @@
+{
+  "AK": "02",
+  "AL": "01",
+  "AR": "05",
+  "AS": "60",
+  "AZ": "04",
+  "CA": "06",
+  "CO": "08",
+  "CT": "09",
+  "DC": "11",
+  "DE": "10",
+  "FL": "12",
+  "GA": "13",
+  "GU": "66",
+  "HI": "15",
+  "IA": "19",
+  "ID": "16",
+  "IL": "17",
+  "IN": "18",
+  "KS": "20",
+  "KY": "21",
+  "LA": "22",
+  "MA": "25",
+  "MD": "24",
+  "ME": "23",
+  "MI": "26",
+  "MN": "27",
+  "MO": "29",
+  "MS": "28",
+  "MT": "30",
+  "NC": "37",
+  "ND": "38",
+  "NE": "31",
+  "NH": "33",
+  "NJ": "34",
+  "NM": "35",
+  "NV": "32",
+  "NY": "36",
+  "OH": "39",
+  "OK": "40",
+  "OR": "41",
+  "PA": "42",
+  "PR": "72",
+  "RI": "44",
+  "SC": "45",
+  "SD": "46",
+  "TN": "47",
+  "TX": "48",
+  "UT": "49",
+  "VA": "51",
+  "VI": "78",
+  "VT": "50",
+  "WA": "53",
+  "WI": "55",
+  "WV": "54",
+  "WY": "56"
+}

matico_admin/ExternalTypes/deck.d.ts

@@ -1,4 +1,4 @@
-import * as DeckTypings from '@danmarshall/deckgl-typings';
-declare module 'deck.gl' {
-    export namespace DeckTypings {}
+import * as DeckTypings from "@danmarshall/deckgl-typings";
+declare module "deck.gl" {
+  export namespace DeckTypings {}
 }

matico_admin/components/ColumnDetails.tsx

@@ -8,7 +8,7 @@ import {
   Grid,
 } from "@adobe/react-spectrum";
 import MoreVertical from "@spectrum-icons/workflow/MoreSmallListVert";
-import React, {useState} from "react";
+import React, { useState } from "react";
 import { useColumnStat } from "../hooks/useColumnStat";
 import { useDatasetColumn } from "../hooks/useDatasetColumns";
 import { Source } from "../hooks/useTableData";
@@ -20,8 +20,8 @@ interface ColumnDetailProps {
 }
 
 const statForDataType = (dataType: any) => {
-  const histogram = { type:'histogram', noBins: 20} ;
-  const categories = {type:"valueCounts" };
+  const histogram = { type: "histogram", noBins: 20 };
+  const categories = { type: "valueCounts" };
 
   switch (dataType) {
     case "INT4":
@@ -90,12 +90,12 @@ export const ColumnDetails: React.FC<ColumnDetailProps> = ({
   source,
   colName,
 }) => {
-  const [isOpen, setIsOpen] = useState(false)
-  const { column , columnError} = useDatasetColumn(source, colName);
+  const [isOpen, setIsOpen] = useState(false);
+  const { column, columnError } = useDatasetColumn(source, colName);
 
   const stat = statForDataType(column?.colType);
 
-  console.log("ATTEMPTING TO GET STAT ", stat, column?.colType, column)
+  console.log("ATTEMPTING TO GET STAT ", stat, column?.colType, column);
   const { data: dataSummary, error: dataSummaryError } = useColumnStat(
     isOpen ? source : null,
     colName,
@@ -110,7 +110,7 @@ export const ColumnDetails: React.FC<ColumnDetailProps> = ({
       <Dialog isDismissable>
         <Heading>{colName}</Heading>
         <Content>
-          {(dataSummary && isOpen) && chartForSummary(dataSummary, colName)}
+          {dataSummary && isOpen && chartForSummary(dataSummary, colName)}
         </Content>
       </Dialog>
     </DialogTrigger>

matico_admin/components/DataTable.tsx

@@ -14,17 +14,17 @@ import {
 import { Page, useTableData } from "../hooks/useTableData";
 import { ColumnDetails } from "./ColumnDetails";
 import MapView from "@spectrum-icons/workflow/MapView";
-import {Source} from '../utils/api'
+import { Source } from "../utils/api";
 
 export interface DataTableProps {
   source: Source;
   filters: Array<any>;
   onError?: (error: string) => void;
-  selection?:string | number | null | undefined;
-  onSelectionChange?: (featureId : string | number | null) => void;
+  selection?: string | number | null | undefined;
+  onSelectionChange?: (featureId: string | number | null) => void;
   onVizualizeCol?: (col: string | null) => void;
-  visCol? : string | null,
-  idCol : string  | null
+  visCol?: string | null;
+  idCol: string | null;
 }
 
 export const DataTable: React.FC<DataTableProps> = ({
@@ -35,52 +35,50 @@ export const DataTable: React.FC<DataTableProps> = ({
   visCol,
   selection,
   onSelectionChange,
-  idCol
+  idCol,
 }) => {
   const [sort, setSort] = useState<null | any>(null);
-  const [page, setPage] = useState<Page>({limit:15, offset:0})
+  const [page, setPage] = useState<Page>({ limit: 15, offset: 0 });
 
   const {
     data: tableData,
     error,
     mutate: updateTableData,
-  } = useTableData(source, filters, sort,page);
+  } = useTableData(source, filters, sort, page);
 
-  const tData = tableData && tableData.data ? tableData.data : tableData
+  const tData = tableData && tableData.data ? tableData.data : tableData;
 
   // This insures we update the table whenever the various options change
-  useEffect(()=>{
-    updateTableData()
-  },[source,filters,sort])
+  useEffect(() => {
+    updateTableData();
+  }, [source, filters, sort]);
 
   const columns = tData
     ? Object.keys(tData[0]).map((item) => ({ id: item, name: item }))
     : [];
 
-  const updateSelection = (selection : "all" | Set<Key>)=>{
-    if(onSelectionChange){
-      onSelectionChange(Array.from(selection)[0] as string)
+  const updateSelection = (selection: "all" | Set<Key>) => {
+    if (onSelectionChange) {
+      onSelectionChange(Array.from(selection)[0] as string);
     }
-  }
+  };
 
-  const setVisCol = (column: string)=>{
-    if(onVizualizeCol){
-      if(visCol === column){
-        onVizualizeCol(null)
-      }
-      else{
-        onVizualizeCol(column)
+  const setVisCol = (column: string) => {
+    if (onVizualizeCol) {
+      if (visCol === column) {
+        onVizualizeCol(null);
+      } else {
+        onVizualizeCol(column);
       }
     }
-  }
+  };
 
   useEffect(() => {
     if (onError) {
       onError(error);
     }
   });
 
-
   if (!tData) {
     return <div>Loading</div>;
   }
@@ -101,13 +99,18 @@ export const DataTable: React.FC<DataTableProps> = ({
             <Flex direction="row" alignItems="center">
               {col.name}
               <TooltipTrigger delay={0}>
-                <ColumnDetails  source={source} colName={col.name} />
+                <ColumnDetails source={source} colName={col.name} />
                 <Tooltip>Column stats and controls</Tooltip>
               </TooltipTrigger>
               <TooltipTrigger delay={0}>
-              <ToggleButton isEmphasized isSelected={visCol === col.name} onChange={()=> setVisCol(col.name)} isQuiet>
-                <MapView size="XXS" />
-              </ToggleButton>
+                <ToggleButton
+                  isEmphasized
+                  isSelected={visCol === col.name}
+                  onChange={() => setVisCol(col.name)}
+                  isQuiet
+                >
+                  <MapView size="XXS" />
+                </ToggleButton>
                 <Tooltip>Visualize on map</Tooltip>
               </TooltipTrigger>
             </Flex>
@@ -116,7 +119,9 @@ export const DataTable: React.FC<DataTableProps> = ({
       </TableHeader>
       <TableBody items={tData}>
         {(param: { [key: string]: any }) => (
-          <Row key={ idCol && param[idCol] ? param[idCol] : JSON.stringify(param)}>
+          <Row
+            key={idCol && param[idCol] ? param[idCol] : JSON.stringify(param)}
+          >
             {(columnKey) => (
               <Cell>
                 {typeof param[columnKey] === "object"
@@ -130,4 +135,3 @@ export const DataTable: React.FC<DataTableProps> = ({
     </TableView>
   );
 };
-