From 801ef77a8128ce65b8ba67fbf3f56036d6d4f20a Mon Sep 17 00:00:00 2001 From: Scott Kirkland Date: Thu, 12 May 2022 11:49:36 -0700 Subject: [PATCH] update cluster info to match new shapefile --- package-lock.json | 95 +++++++++++++++++++++++++++++++---------------- shapes.ts | 20 ++++++---- 2 files changed, 75 insertions(+), 40 deletions(-) diff --git a/package-lock.json b/package-lock.json index 19b4b9e..09612b7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -104,21 +104,16 @@ } }, "@types/pg": { - "version": "7.14.1", - "resolved": "https://registry.npmjs.org/@types/pg/-/pg-7.14.1.tgz", - "integrity": "sha512-gQgg4bLuykokypx4O1fwEzl5e6UjjyaBtN3znn5zhm0YB9BnKyHDw+e4cQY9rAPzpdM2qpJbn9TNzUazbmTsdw==", + "version": "8.6.5", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.6.5.tgz", + "integrity": "sha512-tOkGtAqRVkHa/PVZicq67zuujI4Oorfglsr2IbKofDwBSysnaqSx7W1mDqFqdkGE6Fbgh+PZAl0r/BWON/mozw==", "dev": true, "requires": { "@types/node": "*", - "@types/pg-types": "*" + "pg-protocol": "*", + "pg-types": "^2.2.0" } }, - "@types/pg-types": { - "version": "1.11.5", - "resolved": "https://registry.npmjs.org/@types/pg-types/-/pg-types-1.11.5.tgz", - "integrity": "sha512-L8ogeT6vDzT1vxlW3KITTCt+BVXXVkLXfZ/XNm6UqbcJgxf+KPO7yjWx7dQQE8RW07KopL10x2gNMs41+IkMGQ==", - "dev": true - }, "@types/shpjs": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/@types/shpjs/-/shpjs-3.4.0.tgz", @@ -140,6 +135,28 @@ "knex": "^0.16.3", "pg": "^7.8.1", "swagger-ui-express": "^4.0.7" + }, + "dependencies": { + "pg": { + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/pg/-/pg-7.18.2.tgz", + "integrity": "sha512-Mvt0dGYMwvEADNKy5PMQGlzPudKcKKzJds/VbOeZJpb6f/pI3mmoXX0JksPgI3l3JPP/2Apq7F36O63J7mgveA==", + "requires": { + "buffer-writer": "2.0.0", + "packet-reader": "1.0.0", + "pg-connection-string": "0.1.3", + "pg-packet-stream": "^1.1.0", + "pg-pool": "^2.0.10", + "pg-types": "^2.1.0", + "pgpass": "1.x", + "semver": "4.3.2" + } + }, + "semver": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-4.3.2.tgz", + "integrity": "sha1-x6BxWKgL7dBSNVt3DYLWZA+AO+c=" + } } }, "abbrev": { @@ -3526,24 +3543,28 @@ "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA=" }, "pg": { - "version": "7.17.1", - "resolved": "https://registry.npmjs.org/pg/-/pg-7.17.1.tgz", - "integrity": "sha512-SYWEip6eADsgDQIZk0bmB2JDOrC8Xu6z10KlhlXl03NSomwVmHB6ZTVyDCwOfT6bXHI8QndJdk5XxSSRXikaSA==", + "version": "8.7.3", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.7.3.tgz", + "integrity": "sha512-HPmH4GH4H3AOprDJOazoIcpI49XFsHCe8xlrjHkWiapdbHK+HLtbm/GQzXYAZwmPju/kzKhjaSfMACG+8cgJcw==", "requires": { "buffer-writer": "2.0.0", "packet-reader": "1.0.0", - "pg-connection-string": "0.1.3", - "pg-packet-stream": "^1.1.0", - "pg-pool": "^2.0.9", + "pg-connection-string": "^2.5.0", + "pg-pool": "^3.5.1", + "pg-protocol": "^1.5.0", "pg-types": "^2.1.0", - "pgpass": "1.x", - "semver": "4.3.2" + "pgpass": "1.x" }, "dependencies": { - "semver": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-4.3.2.tgz", - "integrity": "sha1-x6BxWKgL7dBSNVt3DYLWZA+AO+c=" + "pg-connection-string": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.5.0.tgz", + "integrity": "sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ==" + }, + "pg-pool": { + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.5.1.tgz", + "integrity": "sha512-6iCR0wVrro6OOHFsyavV+i6KYL4lVNyYAB9RD18w66xSzN+d8b66HiwuP30Gp1SH5O9T82fckkzsRjlrhD0ioQ==" } } }, @@ -3563,9 +3584,14 @@ "integrity": "sha512-kRBH0tDIW/8lfnnOyTwKD23ygJ/kexQVXZs7gEyBljw4FYqimZFxnMMx50ndZ8In77QgfGuItS5LLclC2TtjYg==" }, "pg-pool": { - "version": "2.0.9", - "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-2.0.9.tgz", - "integrity": "sha512-gNiuIEKNCT3OnudQM2kvgSnXsLkSpd6mS/fRnqs6ANtrke6j8OY5l9mnAryf1kgwJMWLg0C1N1cYTZG1xmEYHQ==" + "version": "2.0.10", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-2.0.10.tgz", + "integrity": "sha512-qdwzY92bHf3nwzIUcj+zJ0Qo5lpG/YxchahxIN8+ZVmXqkahKXsnl2aiJPHLYN9o5mB/leG+Xh6XKxtP7e0sjg==" + }, + "pg-protocol": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.5.0.tgz", + "integrity": "sha512-muRttij7H8TqRNu/DxrAJQITO4Ac7RmX3Klyr/9mJEOBeIpgnF8f9jAfRz5d3XwQZl5qBjF9gLsUtMPJE0vezQ==" }, "pg-types": { "version": "2.2.0", @@ -3580,11 +3606,11 @@ } }, "pgpass": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.2.tgz", - "integrity": "sha1-Knu0G2BltnkH6R2hsHwYR8h3swY=", + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", "requires": { - "split": "^1.0.0" + "split2": "^4.1.0" } }, "pify": { @@ -3609,9 +3635,9 @@ "integrity": "sha1-AntTPAqokOJtFy1Hz5zOzFIazTU=" }, "postgres-date": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.4.tgz", - "integrity": "sha512-bESRvKVuTrjoBluEcpv2346+6kgB7UlnqWZsnbnCccTNq/pqfj1j6oBaN5+b/NrDXepYUT/HKadqv3iS9lJuVA==" + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==" }, "postgres-interval": { "version": "1.2.0", @@ -4134,6 +4160,11 @@ "extend-shallow": "^3.0.0" } }, + "split2": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.1.0.tgz", + "integrity": "sha512-VBiJxFkxiXRlUIeyMQi8s4hgvKCSjtknJv/LVYbrgALPwf5zSKmEwV9Lst25AkvMDnvxODugjdl6KZgwKM1WYQ==" + }, "sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", diff --git a/shapes.ts b/shapes.ts index e048db1..7a715d9 100644 --- a/shapes.ts +++ b/shapes.ts @@ -9,6 +9,8 @@ interface TreatedClusterInfo { county_name: string; } +const cluster_no_property_name = 'DEM360'; + const process = (fileName: string): Promise => { return new Promise((resolve, reject) => { const file = fs.readFileSync(fileName); @@ -17,21 +19,20 @@ const process = (fileName: string): Promise => { shp(file) .then((geojson: any) => { - // see bellow for whats here this internally call shp.parseZip() console.log('processed'); - // console.log(geojson.features); const structuredResults: TreatedClusterInfo[] = geojson.features.map((result: any) => ({ - cluster_no: result.properties.cluster_no, + cluster_no: result.properties[cluster_no_property_name], geography: { type: 'Feature', - id: result.properties.cluster_no, + id: result.properties[cluster_no_property_name], geometry: result.geometry, }, county_name: 'NA', })); - console.log('example row', structuredResults[0]); + console.log('total clusters: ' + structuredResults.length); + console.log('example rows', structuredResults.slice(3)); resolve(structuredResults); }) @@ -43,11 +44,12 @@ const upload = async (clusterInfo: TreatedClusterInfo[]) => { const db = knex({ client: 'pg', connection: { - host: 'cecdss.postgres.database.azure.com', - user: 'cec@cecdss', + host: 'cecdssdb.postgres.database.azure.com', + user: 'cec', password: '', database: 'cecdss', port: 5432, + ssl: true, }, }); @@ -60,6 +62,7 @@ const upload = async (clusterInfo: TreatedClusterInfo[]) => { // try with batch insert await txn.batchInsert('treatedclustersInfo', clusterInfo); + await txn.commit(); }); } catch (error) { console.error(error); @@ -68,6 +71,7 @@ const upload = async (clusterInfo: TreatedClusterInfo[]) => { console.log('insert complete'); }; -process('./data/Sierra_Nevada_shapes.zip') +process('./data/CaliforniaClusters.zip') +// process('./data/Sierra_Nevada_shapes.zip') .then(upload) .then(() => console.log('done'));