-
Notifications
You must be signed in to change notification settings - Fork 0
/
app.py
143 lines (101 loc) · 6.13 KB
/
app.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
__author__ = 'djvdorp'
from collections import OrderedDict
from progressbar import *
import shapefile
import pyproj
import csv
import logging
import pandas
HECTOPUNTEN_OUTPUT_FIELDS = ['HECTOMTRNG', 'AFSTAND', 'WVK_ID', 'WVK_BEGDAT']
WEGVAKKEN_OUTPUT_FIELDS = ['WVK_ID', 'WVK_BEGDAT', 'JTE_ID_BEG', 'JTE_ID_END', 'WEGBEHSRT', 'WEGNUMMER', 'WEGDEELLTR', 'HECTO_LTTR', 'BAANSUBSRT', 'RPE_CODE', 'ADMRICHTNG', 'RIJRICHTNG', 'STT_NAAM', 'WPSNAAMNEN', 'GME_ID', 'GME_NAAM', 'HNRSTRLNKS', 'HNRSTRRHTS', 'E_HNR_LNKS', 'E_HNR_RHTS', 'L_HNR_LNKS', 'L_HNR_RHTS', 'BEGAFSTAND', 'ENDAFSTAND', 'BEGINKM', 'EINDKM', 'POS_TV_WOL']
MERGED_OUTPUT_FIELDS = ['ID', 'WEGNUMMER', 'HECTOMTRNG', 'LONGITUDE', 'LATITUDE', 'STT_NAAM', 'GME_NAAM', 'WEGBEHSRT', 'RPE_CODE', 'POS_TV_WOL', 'WEGDEELLTR', 'HECTO_LTTR', 'BAANSUBSRT']
MERGED_RENAME_FIELDS_MAPPING = {'ID': 'HP_ID', 'WEGNUMMER': 'WEGNR','HECTOMTRNG': 'HECTONR'}
logging.basicConfig(level=logging.INFO)
widgets = ['Processing: ', Percentage(), ' ', Bar(marker=RotatingMarker()), ' ', ETA()]
def shp_transform_to_different_projection(input_path, input_fields, src_projection, dest_projection, output_filename):
logging.info("START processing shapefile '{}' to '{}'".format(input_path, output_filename))
csv_file = open(output_filename, 'wb')
writer = csv.writer(csv_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_NONNUMERIC)
r = shapefile.Reader(input_path)
input_shapes = r.shapeRecords()
nr_of_shapes_in_file = len(input_shapes)
logging.info("{} shapes in file '{}' will be transformed".format(nr_of_shapes_in_file, input_path))
field_names = [str(i[0]) for i in r.fields]
field_names.remove('DeletionFlag') # of moet dit zijn: del field_names[0]
logging.info("fieldNames in shapefile: {}".format(field_names))
input_projection = pyproj.Proj(src_projection)
output_projection = pyproj.Proj(dest_projection)
# 3 = shapefile.POLYLINE = wegvakken
# 8 = shapefile.MULTIPOINT = hectopunten
logging.info("shapeType read: {}".format(r.shapeType))
counter = 0
pbar = ProgressBar(widgets=widgets, maxval=nr_of_shapes_in_file).start()
for input_shape in input_shapes:
nr_of_points_in_shape = len(input_shape.shape.points)
result_entry = OrderedDict()
for input_field in input_fields:
key = (field_names.index(input_field))
input_record = input_shape.record
input_entry = input_record[key]
# Lists (voor datum) platslaan tot een string
if isinstance(input_entry, list):
input_entry = int_array_to_string(input_entry)
# HECTOMTRNG in Hectopunten.shp moet gedeeld worden door 10
if input_field == 'HECTOMTRNG':
input_entry = (input_record[key] / 10.)
result_entry[input_field] = input_entry
if nr_of_points_in_shape == 1:
input_x = input_shape.shape.points[0][0]
input_y = input_shape.shape.points[0][1]
# Convert input_x, input_y from Rijksdriehoekstelsel_New to WGS84
x, y = pyproj.transform(input_projection, output_projection, input_x, input_y)
logging.debug(field_names)
logging.debug([str(i) for i in input_record])
logging.debug('Rijksdriehoekstelsel_New ({:-f}, {:-f}) becomes WGS84 ({:-f}, {:-f})'.format(input_x, input_y, x, y))
result_entry['LONGITUDE'] = x
result_entry['LATITUDE'] = y
else:
logging.debug("number of points for this shape was >1, it was: {}".format(nr_of_points_in_shape))
headers = result_entry.keys()
if counter == 0:
writer.writerow(headers)
line = []
for field in headers:
line.append(result_entry[field])
writer.writerow(line)
counter += 1
pbar.update(counter)
csv_file.close()
pbar.finish()
logging.info("FINISHED processing - saved file '{}'".format(output_filename))
def int_array_to_string(input_array):
return "-".join(str(i) for i in input_array)
def merge_shapefile_csvs(input_hectopunten, input_wegvakken, merge_on_field, fields_to_keep, fields_rename_mapping, output_filename):
logging.info("START merging csv files '{}' and '{}' to file '{}'".format(input_hectopunten, input_wegvakken, output_filename))
hectopunten_df = pandas.read_csv(input_hectopunten)
wegvakken_df = pandas.read_csv(input_wegvakken)
# Join de 2 input files samen, left=hectopunten en right=wegvakken
merged_df = pandas.merge(hectopunten_df, wegvakken_df, on=merge_on_field)
# Voeg een ID field toe per regel
merged_df['ID'] = merged_df.index
# Bewaar alleen de meegegeven velden om te bewaren
result_df = merged_df[fields_to_keep]
# Hernoem columns zodat deze af kunnen wijken van de input columns
result_df = result_df.rename(columns=fields_rename_mapping)
# Exporteer dit naar een merged csv
result_df.to_csv(output_filename, mode='wb', index=False, header=True, delimiter=',', quotechar='"', quoting=csv.QUOTE_NONNUMERIC)
logging.info("FINISHED merging csv files - saved file '{}'".format(output_filename))
# Real action here
input_projection_string = "+init=EPSG:28992" # Dit is Rijksdriehoekstelsel_New vanuit de .prj files, officieel EPSG:28992 Amersfoort / RD New
output_projection_string = "+init=EPSG:4326" # LatLon with WGS84 datum used by GPS units and Google Earth, officieel EPSG:4326
# Bestanden kunnen worden gevonden op: http://www.jigsaw.nl/nwb/downloads/NWB_01-07-2014.zip
shp_hectopunten = "input/Hectopunten/Hectopunten"
shp_wegvakken = "input/Wegvakken/Wegvakken"
# CSV files van de SHP files
csv_hectopunten = "output/Hectopunten.csv"
csv_wegvakken = "output/Wegvakken.csv"
# CSV output na mergen
csv_merged = "output/merged.csv"
shp_transform_to_different_projection(shp_hectopunten, HECTOPUNTEN_OUTPUT_FIELDS, input_projection_string, output_projection_string, csv_hectopunten)
shp_transform_to_different_projection(shp_wegvakken, WEGVAKKEN_OUTPUT_FIELDS, input_projection_string, output_projection_string, csv_wegvakken)
merge_shapefile_csvs(csv_hectopunten, csv_wegvakken, 'WVK_ID', MERGED_OUTPUT_FIELDS, MERGED_RENAME_FIELDS_MAPPING, csv_merged)