diff --git a/osm_fieldwork/ODKInstance.py b/osm_fieldwork/ODKInstance.py
index 7ad8317c..626d3134 100755
--- a/osm_fieldwork/ODKInstance.py
+++ b/osm_fieldwork/ODKInstance.py
@@ -18,14 +18,13 @@
#
import argparse
+import json
import logging
import os
import re
import sys
-# from shapely.geometry import Point, LineString, Polygon
-from collections import OrderedDict
-
+import flatdict
import xmltodict
# Instantiate logger
@@ -38,8 +37,8 @@ def __init__(
filespec: str = None,
data: str = None,
):
- """This class imports a ODK Instance file, which is in XML into a data
- structure.
+ """This class imports a ODK Instance file, which is in XML into a
+ data structure.
Args:
filespec (str): The filespec to the ODK XML Instance file
@@ -50,6 +49,7 @@ def __init__(
"""
self.data = data
self.filespec = filespec
+ self.ignore = ["today", "start", "deviceid", "nodel", "instanceID"]
if filespec:
self.data = self.parse(filespec=filespec)
elif data:
@@ -59,7 +59,7 @@ def parse(
self,
filespec: str,
data: str = None,
- ):
+ ) -> dict:
"""Import an ODK XML Instance file ito a data structure. The input is
either a filespec to the Instance file copied off your phone, or
the XML that has been read in elsewhere.
@@ -69,9 +69,9 @@ def parse(
data (str): The XML data
Returns:
- (list): All the entries in the IOPDK XML Instance file
+ (dict): All the entries in the OSM XML Instance file
"""
- rows = list()
+ row = dict()
if filespec:
logging.info("Processing instance file: %s" % filespec)
file = open(filespec, "rb")
@@ -80,47 +80,29 @@ def parse(
elif data:
xml = data
doc = xmltodict.parse(xml)
- import json
json.dumps(doc)
tags = dict()
data = doc["data"]
- for i, j in data.items():
- if j is None or i == "meta":
+ flattened = flatdict.FlatDict(data)
+ rows = list()
+ pat = re.compile("[0-9.]* [0-9.-]* [0-9.]* [0-9.]*")
+ for key, value in flattened.items():
+ if key[0] == "@" or value is None:
continue
- print(f"tag: {i} == {j}")
- pat = re.compile("[0-9.]* [0-9.-]* [0-9.]* [0-9.]*")
- if pat.match(str(j)):
- if i == "warmup":
- continue
- gps = j.split(" ")
- tags["lat"] = gps[0]
- tags["lon"] = gps[1]
+ if re.search(pat, value):
+ gps = value.split(" ")
+ row["lat"] = gps[0]
+ row["lon"] = gps[1]
continue
- if type(j) == OrderedDict or type(j) == dict:
- for ii, jj in j.items():
- pat = re.compile("[0-9.]* [0-9.-]* [0-9.]* [0-9.]*")
- if pat.match(str(jj)):
- gps = jj.split(" ")
- tags["lat"] = gps[0]
- tags["lon"] = gps[1]
- continue
- if jj is None:
- continue
- print(f"tag: {i} == {j}")
- if type(jj) == OrderedDict or type(jj) == dict:
- for iii, jjj in jj.items():
- if jjj is not None:
- tags[iii] = jjj
- # print(iii, jjj)
- else:
- print(ii, jj)
- tags[ii] = jj
- else:
- if i[0:1] != "@":
- tags[i] = j
- rows.append(tags)
- return rows
+
+ # print(key, value)
+ tmp = key.split(":")
+ if tmp[len(tmp) - 1] in self.ignore:
+ continue
+ row[tmp[len(tmp) - 1]] = value
+
+ return row
if __name__ == "__main__":
@@ -147,3 +129,4 @@ def parse(
inst = ODKInstance(args.infile)
data = inst.parse(args.infile)
+ # print(data)
diff --git a/osm_fieldwork/convert.py b/osm_fieldwork/convert.py
index 42fa9991..cd55b7b7 100755
--- a/osm_fieldwork/convert.py
+++ b/osm_fieldwork/convert.py
@@ -100,7 +100,7 @@ def privateData(
self,
keyword: str,
) -> bool:
- """Search he private data category for a keyword.
+ """Search the private data category for a keyword.
Args:
keyword (str): The keyword to search for
@@ -207,14 +207,14 @@ def convertEntry(
# If the tag is in the config file, convert it.
if self.convertData(newtag):
newtag = self.convertTag(newtag)
- if newtag != tag:
- logging.debug(f"Converted Tag for entry {tag} to {newtag}")
+ # if newtag != tag:
+ # logging.debug(f"Converted Tag for entry {tag} to {newtag}")
# Truncate the elevation, as it's really long
if newtag == "ele":
value = value[:7]
newval = self.convertValue(newtag, value)
- logging.debug("Converted Value for entry '%s' to '%s'" % (value, newval))
+ # logging.debug("Converted Value for entry '%s' to '%s'" % (value, newval))
# there can be multiple new tag/value pairs for some values from ODK
if type(newval) == str:
all.append({newtag: newval})
@@ -287,7 +287,7 @@ def convertTag(
if low in self.convert:
newtag = self.convert[low]
if type(newtag) is str:
- logging.debug("\tTag '%s' converted tag to '%s'" % (tag, newtag))
+ # logging.debug("\tTag '%s' converted tag to '%s'" % (tag, newtag))
tmp = newtag.split("=")
if len(tmp) > 1:
newtag = tmp[0]
@@ -315,18 +315,20 @@ def convertMultiple(
Returns:
(list): The new tags
"""
- tags = list()
+ tags = dict()
for tag in value.split(" "):
low = tag.lower()
if self.convertData(low):
newtag = self.convert[low]
- # tags.append({newtag}: {value})
if newtag.find("=") > 0:
tmp = newtag.split("=")
- tags.append({tmp[0]: tmp[1]})
+ if tmp[0] in tags:
+ tags[tmp[0]] = f"{tags[tmp[0]]};{tmp[1]}"
+ else:
+ tags.update({tmp[0]: tmp[1]})
else:
- tags.append({low: "yes"})
- logging.debug(f"\tConverted multiple to {tags}")
+ tags.update({low: "yes"})
+ # logging.debug(f"\tConverted multiple to {tags}")
return tags
def parseXLS(
@@ -396,6 +398,8 @@ def createEntry(
"action",
)
+ if key in self.ignore:
+ continue
# When using existing OSM data, there's a special geometry field.
# Otherwise use the GPS coordinates where you are.
if key == "geometry" and len(value) > 0:
@@ -412,8 +416,18 @@ def createEntry(
attrs[key] = value
# log.debug("Adding attribute %s with value %s" % (key, value))
continue
-
if value is not None and value != "no" and value != "unknown":
+ if key == "username":
+ tags["user"] = value
+ continue
+ items = self.convertEntry(key, value)
+ if key in self.types:
+ if self.types[key] == "select_multiple":
+ vals = self.convertMultiple(value)
+ if len(vals) > 0:
+ for tag in vals:
+ tags.update(tag)
+ continue
if key == "track" or key == "geoline":
# refs.append(tags)
# log.debug("Adding reference %s" % tags)
diff --git a/osm_fieldwork/CSVDump.py b/osm_fieldwork/csvdump.py
similarity index 65%
rename from osm_fieldwork/CSVDump.py
rename to osm_fieldwork/csvdump.py
index 7bf64f28..d68e3695 100755
--- a/osm_fieldwork/CSVDump.py
+++ b/osm_fieldwork/csvdump.py
@@ -25,10 +25,8 @@
import sys
from datetime import datetime
-from geojson import Feature, FeatureCollection, Point, dump
-
from osm_fieldwork.convert import Convert
-from osm_fieldwork.osmfile import OsmFile
+from osm_fieldwork.support import basename
from osm_fieldwork.xlsforms import xlsforms_path
# Instantiate logger
@@ -59,124 +57,6 @@ def __init__(
self.entries = dict()
self.types = dict()
- def lastSaved(
- self,
- keyword: str,
- ) -> str:
- """Get the last saved value for a question.
-
- Args:
- keyword (str): The keyword to search for
-
- Returns:
- (str): The last saved value for the question
-
- """
- if keyword is not None and len(keyword) > 0:
- return self.saved[keyword]
- return None
-
- def updateSaved(
- self,
- keyword: str,
- value: str,
- ) -> bool:
- """Update the last saved value for a question.
-
- Args:
- keyword (str): The keyword to search for
- value (str): The new value
-
- Returns:
- (bool): If the new value got saved
-
- """
- if keyword is not None and value is not None and len(value) > 0:
- self.saved[keyword] = value
- return True
- else:
- return False
-
- def createOSM(
- self,
- filespec: str,
- ):
- """Create an OSM XML output files.
-
- Args:
- filespec (str): The output file name
- """
- log.debug("Creating OSM XML file: %s" % filespec)
- self.osm = OsmFile(filespec)
- # self.osm.header()
-
- def writeOSM(
- self,
- feature: dict,
- ):
- """Write a feature to an OSM XML output file.
-
- Args:
- feature (dict): The OSM feature to write to
- """
- out = ""
- if "id" in feature["tags"]:
- feature["id"] = feature["tags"]["id"]
- if "lat" not in feature["attrs"] or "lon" not in feature["attrs"]:
- return None
- if "refs" not in feature:
- out += self.osm.createNode(feature)
- else:
- out += self.osm.createWay(feature)
- self.osm.write(out)
-
- def finishOSM(self):
- """Write the OSM XML file footer and close it."""
- # This is now handled by a destructor in the OsmFile class
- # self.osm.footer()
-
- def createGeoJson(
- self,
- filespec: str = "tmp.geojson",
- ):
- """Create a GeoJson output file.
-
- Args:
- filespec (str): The output file name
- """
- log.debug("Creating GeoJson file: %s" % filespec)
- self.json = open(filespec, "w")
-
- def writeGeoJson(
- self,
- feature: dict,
- ):
- """Write a feature to a GeoJson output file.
-
- Args:
- feature (dict): The OSM feature to write to
- """
- # These get written later when finishing , since we have to create a FeatureCollection
- if "lat" not in feature["attrs"] or "lon" not in feature["attrs"]:
- return None
- self.features.append(feature)
-
- def finishGeoJson(self):
- """Write the GeoJson FeatureCollection to the output file and close it."""
- features = list()
- for item in self.features:
- if len(item["attrs"]["lon"]) == 0 or len(item["attrs"]["lat"]) == 0:
- log.warning("Bad location data in entry! %r", item["attrs"])
- continue
- poi = Point((float(item["attrs"]["lon"]), float(item["attrs"]["lat"])))
- if "private" in item:
- props = {**item["tags"], **item["private"]}
- else:
- props = item["tags"]
- features.append(Feature(geometry=poi, properties=props))
- collection = FeatureCollection(features)
- dump(collection, self.json)
-
def parse(
self,
filespec: str,
@@ -201,9 +81,9 @@ def parse(
tags = dict()
# log.info(f"ROW: {row}")
for keyword, value in row.items():
- if keyword is None or (value and len(value) == 0):
+ if keyword is None or len(value) == 0:
continue
- base = self.basename(keyword).lower()
+ base = basename(keyword).lower()
# There's many extraneous fields in the input file which we don't need.
if base is None or base in self.ignore or value is None:
continue
@@ -228,7 +108,6 @@ def parse(
if base == "longitude" and len(value) == 0:
value = row["warmup-Longitude"]
items = self.convertEntry(base, value)
-
# log.info(f"ROW: {base} {value}")
if len(items) > 0:
if base in self.saved:
@@ -253,24 +132,6 @@ def parse(
all_tags.append(tags)
return all_tags
- def basename(
- self,
- line: str,
- ) -> str:
- """Extract the basename of a path after the last -.
-
- Args:
- line (str): The path from the json file entry
-
- Returns:
- (str): The last node of the path
- """
- tmp = line.split("-")
- if len(tmp) == 0:
- return line
- base = tmp[len(tmp) - 1]
- return base
-
def main():
"""Run conversion directly from the terminal."""
diff --git a/osm_fieldwork/json2osm.py b/osm_fieldwork/json2osm.py
old mode 100755
new mode 100644
index 03bb2757..1a31feec
--- a/osm_fieldwork/json2osm.py
+++ b/osm_fieldwork/json2osm.py
@@ -43,7 +43,8 @@ def __init__(
self,
yaml: str = None,
):
- """A class to convert the JSON file from ODK Central, or the GeoJson
+ """
+ A class to convert the JSON file from ODK Central, or the GeoJson
file created by the odk2geojson utility.
Args:
@@ -59,6 +60,99 @@ def __init__(
self.json = None
self.features = list()
self.config = super().__init__(yaml)
+ self.saved = dict()
+ self.defaults = dict()
+ self.entries = dict()
+ self.types = dict()
+
+ def createOSM(
+ self,
+ filespec: str = "tmp.osm",
+ ):
+ """Create an OSM XML output files.
+
+ Args:
+ filespec (str): The filespec for the output OSM XML file
+
+ Returns:
+ (OsmFile): An instance of the OSM XML output file
+ """
+ log.debug(f"Creating OSM XML file: {filespec}")
+ self.osm = OsmFile(filespec)
+ return self.osm
+
+ def writeOSM(
+ self,
+ feature: dict,
+ ):
+ """Write a feature to an OSM XML output file.
+
+ Args:
+ feature (dict): The feature to write to the OSM XML output file
+ """
+ out = ""
+ if "id" in feature["tags"]:
+ feature["id"] = feature["tags"]["id"]
+ if "lat" not in feature["attrs"] or "lon" not in feature["attrs"]:
+ return None
+ if "user" in feature["tags"] and "user" not in feature["attrs"]:
+ feature["attrs"]["user"] = feature["tags"]["user"]
+ del feature["tags"]["user"]
+ if "uid" in feature["tags"] and "uid" not in ["attrs"]:
+ feature["attrs"]["uid"] = feature["tags"]["uid"]
+ del feature["tags"]["uid"]
+ if "refs" not in feature:
+ out += self.osm.createNode(feature, True)
+ else:
+ out += self.osm.createWay(feature, True)
+ self.osm.write(out)
+
+ def finishOSM(self):
+ """Write the OSM XML file footer and close it. The destructor in the
+ OsmFile class should do this, but this is the manual way.
+ """
+ self.osm.footer()
+
+ def createGeoJson(
+ self,
+ file="tmp.geojson",
+ ):
+ """Create a GeoJson output file.
+
+ Args:
+ file (str): The filespec of the output GeoJson file
+ """
+ log.debug("Creating GeoJson file: %s" % file)
+ self.json = open(file, "w")
+
+ def writeGeoJson(
+ self,
+ feature: dict,
+ ):
+ """Write a feature to a GeoJson output file.
+
+ Args:
+ feature (dict): The feature to write to the GeoJson output file
+ """
+ # These get written later when finishing , since we have to create a FeatureCollection
+ if "lat" not in feature["attrs"] or "lon" not in feature["attrs"]:
+ return None
+ self.features.append(feature)
+
+ def finishGeoJson(self):
+ """Write the GeoJson FeatureCollection to the output file and close it."""
+ features = list()
+ for item in self.features:
+ # poi = Point()
+ poi = Point((float(item["attrs"]["lon"]), float(item["attrs"]["lat"])))
+ if "private" in item:
+ props = {**item["tags"], **item["private"]}
+ else:
+ props = item["tags"]
+ features.append(Feature(geometry=poi, properties=props))
+ collection = FeatureCollection(features)
+ dump(collection, self.json)
+=======
def createOSM(
self,
@@ -147,13 +241,15 @@ def finishGeoJson(self):
features.append(Feature(geometry=poi, properties=props))
collection = FeatureCollection(features)
dump(collection, self.json)
+>>>>>>> main
def parse(
self,
filespec: str = None,
data: str = None,
) -> list:
- """Parse the JSON file from ODK Central and convert it to a data structure.
+ """
+ Parse the JSON file from ODK Central and convert it to a data structure.
The input is either a filespec to open, or the data itself.
Args:
@@ -265,6 +361,64 @@ def parse(
# log.debug(f"Finished parsing JSON file {filespec}")
return total
+def json2osm(input_file, yaml_file=None):
+ """Process the JSON file from ODK Central or the GeoJSON file to OSM XML format.
+
+ Args:
+ input_file (str): The path to the input JSON or GeoJSON file.
+ yaml_file (str): The path to the YAML config file (optional).
+
+ Returns:
+ osmoutfile (str): Path to the converted OSM XML file.
+ """
+ log.info(f"Converting JSON file to OSM: {input_file}")
+ if yaml_file:
+ jsonin = JsonDump(yaml_file)
+ else:
+ jsonin = JsonDump()
+
+ # jsonin.parseXLS(args.xlsfile)
+
+ # Modify the input file name for the 2 output files, which will get written
+ # to the current directory.
+
+ base = Path(input_file).stem
+ osmoutfile = f"{base}-out.osm"
+ jsonin.createOSM(osmoutfile)
+
+ data = jsonin.parse(input_file)
+ # This OSM XML file only has OSM appropriate tags and values
+
+ for entry in data:
+ feature = jsonin.createEntry(entry)
+
+ # Sometimes bad entries, usually from debugging XForm design, sneak in
+ if len(feature) == 0:
+ continue
+
+ if len(feature) > 0:
+ if "lat" not in feature["attrs"]:
+ if "geometry" in feature["tags"]:
+ if isinstance(feature["tags"]["geometry"], str):
+ coords = list(feature["tags"]["geometry"])
+ # del feature['tags']['geometry']
+ elif "coordinates" in feature["tags"]:
+ coords = feature["tags"]["coordinates"]
+ feature["attrs"] = {"lat": coords[1], "lon": coords[0]}
+ else:
+ log.warning(f"Bad record! {feature}")
+ continue # Skip bad records
+
+ jsonin.writeOSM(feature)
+ # log.debug("Writing final OSM XML file...")
+
+ # jsonin.finishOSM()
+ log.info(f"Wrote OSM XML file: {osmoutfile}")
+
+ return osmoutfile
+
+
+=======
# def json2osm(
# cmdln: dict,
@@ -323,6 +477,7 @@ def parse(
# return osmoutfile
+>>>>>>> main
def main():
"""Run conversion directly from the terminal."""
parser = argparse.ArgumentParser(description="convert JSON from ODK Central to OSM XML")
@@ -393,7 +548,55 @@ def main():
jsonin.finishGeoJson()
log.info("Wrote OSM XML file: %r" % osmoutfile)
log.info("Wrote GeoJson file: %r" % jsonoutfile)
+>>>>>>> main
+
+ jsonin.parseXLS(args.xlsfile)
+
+ base = Path(args.infile).stem
+ osmoutfile = f"{base}.osm"
+ jsonin.createOSM(osmoutfile)
+
+ jsonoutfile = f"{base}.geojson"
+ jsonin.createGeoJson(jsonoutfile)
+
+ log.debug("Parsing json files %r" % args.infile)
+ data = jsonin.parse(args.infile)
+
+ # This OSM XML file only has OSM appropriate tags and values
+ nodeid = -1000
+ for entry in data:
+ feature = jsonin.createEntry(entry)
+ if len(feature) == 0:
+ continue
+ if "refs" in feature:
+ refs = list()
+ for ref in feature["refs"]:
+ now = datetime.now().strftime("%Y-%m-%dT%TZ")
+ if len(ref) == 0:
+ continue
+ coords = ref.split(" ")
+ print(coords)
+ node = {"attrs": {"id": nodeid, "version": 1, "timestamp": now, "lat": coords[0], "lon": coords[1]}, "tags": dict()}
+ jsonin.writeOSM(node)
+ refs.append(nodeid)
+ nodeid -= 1
+
+ feature["refs"] = refs
+ jsonin.writeOSM(feature)
+ else:
+ # Sometimes bad entries, usually from debugging XForm design, sneak in
+ if "lat" not in feature["attrs"]:
+ log.warning("Bad record! %r" % feature)
+ continue
+ jsonin.writeOSM(feature)
+ # This GeoJson file has all the data values
+ jsonin.writeGeoJson(feature)
+ # print("TAGS: %r" % feature['tags'])
+ jsonin.finishOSM()
+ jsonin.finishGeoJson()
+ log.info("Wrote OSM XML file: %r" % osmoutfile)
+ log.info("Wrote GeoJson file: %r" % jsonoutfile)
if __name__ == "__main__":
"""This is just a hook so this file can be run standlone during development."""
diff --git a/osm_fieldwork/jsondump.py b/osm_fieldwork/jsondump.py
new file mode 100755
index 00000000..2bde04a3
--- /dev/null
+++ b/osm_fieldwork/jsondump.py
@@ -0,0 +1,253 @@
+#!/usr/bin/python3
+
+# Copyright (c) 2023, 2024 Humanitarian OpenStreetMap Team
+#
+# This file is part of OSM-Fieldwork.
+#
+# OSM-Fieldwork is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# OSM-Fieldwork is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with OSM-Fieldwork. If not, see .
+#
+
+import argparse
+import json
+import logging
+
+# import pandas as pd
+import sys
+from pathlib import Path
+
+import flatdict
+import geojson
+
+from osm_fieldwork.convert import Convert
+
+log = logging.getLogger(__name__)
+
+
+class JsonDump(Convert):
+ """A class to parse the JSON files from ODK Central or odk2geojson."""
+
+ def __init__(
+ self,
+ yaml: str = None,
+ ):
+ """A class to convert the JSON file from ODK Central, or the GeoJson
+ file created by the odk2geojson utility.
+
+ Args:
+ yaml (str): The filespec of the YAML config file
+
+ Returns:
+ (JsonDump): An instance of this object
+ """
+ self.fields = dict()
+ self.nodesets = dict()
+ self.data = list()
+ self.osm = None
+ self.json = None
+ self.features = list()
+ self.config = super().__init__(yaml)
+
+ def parse(
+ self,
+ filespec: str = None,
+ data: str = None,
+ ) -> list:
+ """Parse the JSON file from ODK Central and convert it to a data structure.
+ The input is either a filespec to open, or the data itself.
+
+ Args:
+ filespec (str): The JSON or GeoJson input file to convert
+ data (str): The data to convert
+
+ Returns:
+ (list): A list of all the features in the input file
+ """
+ log.debug(f"Parsing JSON file {filespec}")
+ total = list()
+ if not data:
+ file = open(filespec, "r")
+ infile = Path(filespec)
+ if infile.suffix == ".geojson":
+ reader = geojson.load(file)
+ elif infile.suffix == ".json":
+ reader = json.load(file)
+ else:
+ log.error("Need to specify a JSON or GeoJson file!")
+ return total
+ elif isinstance(data, str):
+ reader = geojson.loads(data)
+ elif isinstance(data, list):
+ reader = data
+
+ # JSON files from Central use value as the keyword, whereas
+ # GeoJSON uses features for the same thing.
+ if "value" in reader:
+ data = reader["value"]
+ elif "features" in reader:
+ data = reader["features"]
+ else:
+ data = reader
+ for row in data:
+ # log.debug(f"ROW: {row}\n")
+ tags = dict()
+ # Extract the location regardless of what the tag is
+ # called.
+ # pat = re.compile("[-0-9.]*, [0-9.-]*, [0-9.]*")
+ # gps = re.findall(pat, str(row))
+ # tmp = list()
+ # if len(gps) == 0:
+ # log.error(f"No location data in: {row}")
+ # continue
+ # elif len(gps) == 1:
+ # # Only the warmup has any coordinates.
+ # tmp = gps[0].split(" ")
+ # elif len(gps) == 2:
+ # # both the warmup and the coordinates have values
+ # tmp = gps[1].split(" ")
+
+ # if len(tmp) > 0:
+ # lat = float(tmp[0][:-1])
+ # lon = float(tmp[1][:-1])
+ # geom = Point([lon, lat])
+ # row["geometry"] = geom
+ # # tags["geometry"] = row["geometry"]
+
+ if "properties" in row:
+ row["properties"] # A GeoJson formatted file
+ else:
+ pass # A JOSM file from ODK Central
+
+ # flatten all the groups into a sodk2geojson.pyingle data structure
+ flattened = flatdict.FlatDict(row)
+ for k, v in flattened.items():
+ last = k.rfind(":") + 1
+ key = k[last:]
+ # a JSON file from ODK Central always uses coordinates as
+ # the keyword
+ if key is None or key in self.ignore or v is None:
+ continue
+ log.debug(f"Processing tag {key} = {v}")
+ if key == "coordinates":
+ if isinstance(v, list):
+ tags["lat"] = v[1]
+ tags["lon"] = v[0]
+ # poi = Point(float(lon), float(lat))
+ # tags["geometry"] = poi
+ continue
+
+ if key in self.types:
+ if self.types[key] == "select_multiple":
+ # log.debug(f"Found key '{self.types[key]}'")
+ if v is None:
+ continue
+ vals = self.convertMultiple(v)
+ if len(vals) > 0:
+ for tag in vals:
+ tags.update(tag)
+ # print(f"BASE {tags}")
+ continue
+
+ items = self.convertEntry(key, v)
+ if items is None or len(items) == 0:
+ continue
+
+ if type(items) == str:
+ log.debug(f"string Item {items}")
+ else:
+ log.debug(f"dict Item {items}")
+ if len(items) == 0:
+ tags.update(items[0])
+ # log.debug(f"TAGS: {tags}")
+ if len(tags) > 0:
+ total.append(tags)
+
+ # log.debug(f"Finished parsing JSON file {filespec}")
+ return total
+
+
+def main():
+ """Run conversion directly from the terminal."""
+ parser = argparse.ArgumentParser(description="convert JSON from ODK Central to OSM XML")
+ parser.add_argument("-v", "--verbose", action="store_true", help="verbose output")
+ parser.add_argument("-y", "--yaml", help="Alternate YAML file")
+ parser.add_argument("-x", "--xlsfile", help="Source XLSFile")
+ parser.add_argument("-i", "--infile", required=True, help="The input file downloaded from ODK Central")
+ args = parser.parse_args()
+
+ # if verbose, dump to the terminal.
+ if args.verbose is not None:
+ logging.basicConfig(
+ level=logging.DEBUG,
+ format=("%(threadName)10s - %(name)s - %(levelname)s - %(message)s"),
+ datefmt="%y-%m-%d %H:%M:%S",
+ stream=sys.stdout,
+ )
+ logging.getLogger("urllib3").setLevel(logging.DEBUG)
+
+ if args.yaml:
+ jsonvin = JsonDump(args.yaml)
+ else:
+ jsonin = JsonDump()
+
+ jsonin.parseXLS(args.xlsfile)
+
+ base = Path(args.infile).stem
+ osmoutfile = f"{base}.osm"
+ jsonin.createOSM(osmoutfile)
+
+ jsonoutfile = f"{base}.geojson"
+ jsonin.createGeoJson(jsonoutfile)
+
+ log.debug("Parsing json files %r" % args.infile)
+ data = jsonin.parse(args.infile)
+ # This OSM XML file only has OSM appropriate tags and values
+ nodeid = -1000
+ for entry in data:
+ feature = jsonin.createEntry(entry)
+ if len(feature) == 0:
+ continue
+ if "refs" in feature:
+ refs = list()
+ for ref in feature["refs"]:
+ now = datetime.now().strftime("%Y-%m-%dT%TZ")
+ if len(ref) == 0:
+ continue
+ coords = ref.split(" ")
+ print(coords)
+ node = {"attrs": {"id": nodeid, "version": 1, "timestamp": now, "lat": coords[0], "lon": coords[1]}, "tags": dict()}
+ jsonin.writeOSM(node)
+ refs.append(nodeid)
+ nodeid -= 1
+
+ feature["refs"] = refs
+ jsonin.writeOSM(feature)
+ else:
+ # Sometimes bad entries, usually from debugging XForm design, sneak in
+ if "lat" not in feature["attrs"]:
+ log.warning("Bad record! %r" % feature)
+ continue
+ jsonin.writeOSM(feature)
+ # This GeoJson file has all the data values
+ jsonin.writeGeoJson(feature)
+ # print("TAGS: %r" % feature['tags'])
+
+ jsonin.finishOSM()
+ jsonin.finishGeoJson()
+ log.info("Wrote OSM XML file: %r" % osmoutfile)
+ log.info("Wrote GeoJson file: %r" % jsonoutfile)
+
+
+if __name__ == "__main__":
+ """This is just a hook so this file can be run standlone during development."""
+ main()
diff --git a/osm_fieldwork/odk2csv.py b/osm_fieldwork/odk2csv.py
index 25d97dda..91d451ef 100755
--- a/osm_fieldwork/odk2csv.py
+++ b/osm_fieldwork/odk2csv.py
@@ -1,7 +1,9 @@
#!/usr/bin/python3
+# This file has been replaced by ODKParsers(), and will be delete in the next release.
+
#
-# Copyright (C) 2020, 2021, 2022, 2023 Humanitarian OpenstreetMap Team
+# Copyright (C) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenstreetMap Team
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
diff --git a/osm_fieldwork/odk2geojson.py b/osm_fieldwork/odk2geojson.py
index bb916386..2c3f75ef 100755
--- a/osm_fieldwork/odk2geojson.py
+++ b/osm_fieldwork/odk2geojson.py
@@ -1,7 +1,9 @@
#!/usr/bin/python3
+# This file has been replaced by ODKParsers(), and will be delete in the next release.
+
#
-# Copyright (C) 2023 Humanitarian OpenstreetMap Team
+# Copyright (C) 2023, 2024 Humanitarian OpenstreetMap Team
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
diff --git a/osm_fieldwork/odk2osm.py b/osm_fieldwork/odk2osm.py
index 19d1bf3d..663bd3fc 100755
--- a/osm_fieldwork/odk2osm.py
+++ b/osm_fieldwork/odk2osm.py
@@ -1,7 +1,7 @@
#!/usr/bin/python3
#
-# Copyright (C) 2020, 2021, 2022, 2023 Humanitarian OpenstreetMap Team
+# Copyright (C) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenstreetMap Team
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -18,16 +18,14 @@
#
import argparse
-import csv
+import glob
import logging
import os
-import re
import sys
-from collections import OrderedDict
-from datetime import datetime
from pathlib import Path
-import xmltodict
+from osm_fieldwork.parsers import ODKParsers
+from osm_fieldwork.support import OutSupport
# Instantiate logger
log = logging.getLogger(__name__)
@@ -39,7 +37,9 @@ def main():
"""
parser = argparse.ArgumentParser(description="Convert ODK XML instance file to OSM XML format")
parser.add_argument("-v", "--verbose", nargs="?", const="0", help="verbose output")
- parser.add_argument("-i", "--instance", required=True, help="The instance file(s) from ODK Collect")
+ parser.add_argument("-y", "--yaml", help="Alternate YAML file")
+ parser.add_argument("-x", "--xlsfile", help="Source XLSFile")
+ parser.add_argument("-i", "--infile", required=True, help="The input file")
# parser.add_argument("-o","--outfile", default='tmp.csv', help='The output file for JOSM')
args = parser.parse_args()
@@ -52,98 +52,45 @@ def main():
stream=sys.stdout,
)
+ toplevel = Path(args.infile)
+ odk = ODKParsers(args.yaml)
+ odk.parseXLS(args.xlsfile)
+ out = OutSupport()
xmlfiles = list()
- if args.instance.find("*") >= 0:
- toplevel = Path()
- for dir in toplevel.glob(args.instance):
- if dir.is_dir():
- xml = os.listdir(dir)
- # There is always only one XML file per instance
- full = os.path.join(dir, xml[0])
- xmlfiles.append(full)
- else:
- toplevel = Path(args.instance)
- if toplevel.is_dir():
- # There is always only one XML file per instance
- full = os.path.join(toplevel, os.path.basename(toplevel))
- xmlfiles.append(full + ".xml")
-
- # print(xmlfiles)
-
- # These are all generated by Collect, and can be ignored
- rows = list()
- for instance in xmlfiles:
- logging.info("Processing instance file: %s" % instance)
- with open(instance, "rb") as file:
- # Instances are small, read the whole file
- xml = file.read(os.path.getsize(instance))
- doc = xmltodict.parse(xml)
- fields = list()
- tags = dict()
- data = doc["data"]
- for i, j in data.items():
- if j is None or i == "meta":
- continue
- # print(f"tag: {i} == {j}")
- pat = re.compile("[0-9.]* [0-9.-]* [0-9.]* [0-9.]*")
- if pat.match(str(j)):
- if i == "warmup":
- continue
- gps = j.split(" ")
- tags["lat"] = gps[0]
- tags["lon"] = gps[1]
- continue
- if type(j) == OrderedDict or type(j) == dict:
- for ii, jj in j.items():
- pat = re.compile("[0-9.]* [0-9.-]* [0-9.]* [0-9.]*")
- if pat.match(str(jj)):
- gps = jj.split(" ")
- tags["lat"] = gps[0]
- tags["lon"] = gps[1]
- continue
- if jj is None:
- continue
- print(f"tag2: {i} == {j}")
- if type(jj) == OrderedDict or type(jj) == dict:
- for iii, jjj in jj.items():
- if jjj is not None:
- pat = re.compile("[0-9.]* [0-9.-]* [0-9.]* [0-9.]*")
- if pat.match(str(jjj)):
- gps = jjj.split(" ")
- tags["lat"] = gps[0]
- tags["lon"] = gps[1]
- continue
- else:
- tags[iii] = jjj
- # print(f"FOO {iii}, {jjj}")
- else:
- # print(f"WHERE {ii}, {jj}")
- fields.append(ii)
- tags[ii] = jj
- else:
- if i[0:1] != "@":
- tags[i] = j
- rows.append(tags)
-
- xml = os.path.basename(xmlfiles[0])
- tmp = xml.replace(" ", "").split("_")
- now = datetime.now()
- timestamp = f"_{now.year}_{now.hour}_{now.minute}"
-
- outfile = tmp[0] + timestamp + ".csv"
-
- with open(outfile, "w", newline="") as csvfile:
- fields = list()
- for row in rows:
- for key in row.keys():
- if key not in fields:
- fields.append(key)
- out = csv.DictWriter(csvfile, dialect="excel", fieldnames=fields)
- out.writeheader()
- for row in rows:
- out.writerow(row)
-
- print("Wrote: %s" % outfile)
+ data = list()
+ # It's a wildcard, used for XML instance files
+ if args.infile.find("*") >= 0:
+ log.debug(f"Parsing multiple ODK XML files {args.infile}")
+ toplevel = Path(args.infile[:-1])
+ for dirs in glob.glob(args.infile):
+ xml = os.listdir(dirs)
+ full = os.path.join(dirs, xml[0])
+ xmlfiles.append(full)
+ for infile in xmlfiles:
+ tmp = odk.XMLparser(infile)
+ entry = odk.createEntry(tmp[0])
+ data.append(entry)
+ elif toplevel.suffix == ".xml":
+ # It's an instance file from ODK Collect
+ log.debug(f"Parsing ODK XML files {args.infile}")
+ # There is always only one XML file per infile
+ full = os.path.join(toplevel, os.path.basename(toplevel))
+ xmlfiles.append(full + ".xml")
+ tmp = odk.XMLparser(args.infile)
+ # odki = ODKInstance(filespec=args.infile, yaml=args.yaml)
+ entry = odk.createEntry(tmp)
+ data.append(entry)
+ elif toplevel.suffix == ".csv":
+ log.debug(f"Parsing csv files {args.infile}")
+ for entry in odk.CSVparser(args.infile):
+ data.append(odk.createEntry(entry))
+ elif toplevel.suffix == ".json":
+ log.debug(f"Parsing json files {args.infile}")
+ for entry in odk.JSONparser(args.infile):
+ data.append(odk.createEntry(entry))
+
+ # Write the data
+ out.WriteData(toplevel.stem, data)
if __name__ == "__main__":
diff --git a/osm_fieldwork/odk_merge.py b/osm_fieldwork/odk_merge.py
index 5b4a6e8c..b75f4451 100755
--- a/osm_fieldwork/odk_merge.py
+++ b/osm_fieldwork/odk_merge.py
@@ -1,6 +1,8 @@
#!/usr/bin/python3
-# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team
+# This file has been replaced by ODKParsers(), and will be delete in the next release.
+
+# Copyright (c) 2022, 2023, 2024 Humanitarian OpenStreetMap Team
#
# This program is free software: you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
diff --git a/osm_fieldwork/osmfile.py b/osm_fieldwork/osmfile.py
index 73ea39c0..ac1eec96 100755
--- a/osm_fieldwork/osmfile.py
+++ b/osm_fieldwork/osmfile.py
@@ -81,7 +81,7 @@ def __init__(
def __del__(self):
"""Close the OSM XML file automatically."""
- log.debug("Closing output file")
+ # log.debug("Closing output file")
self.footer()
def isclosed(self):
diff --git a/osm_fieldwork/parsers.py b/osm_fieldwork/parsers.py
new file mode 100644
index 00000000..77ab1569
--- /dev/null
+++ b/osm_fieldwork/parsers.py
@@ -0,0 +1,301 @@
+#!/usr/bin/python3
+
+# Copyright (c) 2024 Humanitarian OpenStreetMap Team
+#
+# This file is part of OSM-Fieldwork.
+#
+# OSM-Fieldwork is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# OSM-Fieldwork is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with OSM-Fieldwork. If not, see .
+#
+
+import csv
+import json
+import logging
+import os
+import re
+from pathlib import Path
+
+import flatdict
+import xmltodict
+
+from osm_fieldwork.convert import Convert
+from osm_fieldwork.support import basename
+from osm_fieldwork.xlsforms import xlsforms_path
+
+# Instantiate logger
+log = logging.getLogger(__name__)
+
+
+class ODKParsers(Convert):
+ """A class to parse the CSV files from ODK Central."""
+
+ def __init__(
+ self,
+ yaml: str = None,
+ ):
+ self.fields = dict()
+ self.nodesets = dict()
+ self.data = list()
+ self.osm = None
+ self.json = None
+ self.features = list()
+ xlsforms_path.replace("xlsforms", "")
+ if yaml:
+ pass
+ else:
+ pass
+ self.config = super().__init__(yaml)
+ self.saved = dict()
+ self.defaults = dict()
+ self.entries = dict()
+ self.types = dict()
+
+ def CSVparser(
+ self,
+ filespec: str,
+ data: str = None,
+ ) -> list:
+ """Parse the CSV file from ODK Central and convert it to a data structure.
+
+ Args:
+ filespec (str): The file to parse.
+ data (str): Or the data to parse.
+
+ Returns:
+ (list): The list of features with tags
+ """
+ all_tags = list()
+ if not data:
+ f = open(filespec, newline="")
+ reader = csv.DictReader(f, delimiter=",")
+ else:
+ reader = csv.DictReader(data, delimiter=",")
+ for row in reader:
+ tags = dict()
+ # log.info(f"ROW: {row}")
+ for keyword, value in row.items():
+ if keyword is None or value is None:
+ continue
+ if len(value) == 0:
+ continue
+ base = basename(keyword).lower()
+ # There's many extraneous fields in the input file which we don't need.
+ if base is None or base in self.ignore or value is None:
+ continue
+ else:
+ # log.info(f"ITEM: {keyword} = {value}")
+ if base in self.types:
+ if self.types[base] == "select_multiple":
+ vals = self.convertMultiple(value)
+ if len(vals) > 0:
+ tags.update(vals)
+ continue
+ # When using geopoint warmup, once the display changes to the map
+
+ # location, there is not always a value if the accuracy is way
+ # off. In this case use the warmup value, which is where we are
+ # hopefully standing anyway.
+ if base == "latitude" and len(value) == 0:
+ if "warmup-Latitude" in row:
+ value = row["warmup-Latitude"]
+ if base == "longitude" and len(value) == 0:
+ value = row["warmup-Longitude"]
+ items = self.convertEntry(base, value)
+ # log.info(f"ROW: {base} {value}")
+ if len(items) > 0:
+ if base in self.saved:
+ if str(value) == "nan" or len(value) == 0:
+ # log.debug(f"FIXME: {base} {value}")
+ val = self.saved[base]
+ if val and len(value) == 0:
+ log.warning(f'Using last saved value for "{base}"! Now "{val}"')
+ value = val
+ else:
+ self.saved[base] = value
+ log.debug(f'Updating last saved value for "{base}" with "{value}"')
+ # Handle nested dict in list
+ if isinstance(items, list):
+ items = items[0]
+ for k, v in items.items():
+ tags[k] = v
+ else:
+ tags[base] = value
+ # log.debug(f"\tFIXME1: {tags}")
+ all_tags.append(tags)
+ return all_tags
+
+ def JSONparser(
+ self,
+ filespec: str = None,
+ data: str = None,
+ ) -> list:
+ """Parse the JSON file from ODK Central and convert it to a data structure.
+ The input is either a filespec to open, or the data itself.
+
+ Args:
+ filespec (str): The JSON or GeoJson input file to convert
+ data (str): The data to convert
+
+ Returns:
+ (list): A list of all the features in the input file
+ """
+ log.debug(f"Parsing JSON file {filespec}")
+ total = list()
+ if not data:
+ file = open(filespec, "r")
+ infile = Path(filespec)
+ if infile.suffix == ".geojson":
+ reader = geojson.load(file)
+ elif infile.suffix == ".json":
+ reader = json.load(file)
+ else:
+ log.error("Need to specify a JSON or GeoJson file!")
+ return total
+ elif isinstance(data, str):
+ reader = geojson.loads(data)
+ elif isinstance(data, list):
+ reader = data
+
+ # JSON files from Central use value as the keyword, whereas
+ # GeoJSON uses features for the same thing.
+ if "value" in reader:
+ data = reader["value"]
+ elif "features" in reader:
+ data = reader["features"]
+ else:
+ data = reader
+ for row in data:
+ # log.debug(f"ROW: {row}\n")
+ tags = dict()
+ if "properties" in row:
+ row["properties"] # A GeoJson formatted file
+ else:
+ pass # A JOSM file from ODK Central
+
+ # flatten all the groups into a sodk2geojson.pyingle data structure
+ flattened = flatdict.FlatDict(row)
+ # log.debug(f"FLAT: {flattened}\n")
+ for k, v in flattened.items():
+ last = k.rfind(":") + 1
+ key = k[last:]
+ # a JSON file from ODK Central always uses coordinates as
+ # the keyword
+ if key is None or key in self.ignore or v is None:
+ continue
+ # log.debug(f"Processing tag {key} = {v}")
+ if key == "coordinates":
+ if isinstance(v, list):
+ tags["lat"] = v[1]
+ tags["lon"] = v[0]
+ # poi = Point(float(lon), float(lat))
+ # tags["geometry"] = poi
+ continue
+
+ if key in self.types:
+ if self.types[key] == "select_multiple":
+ # log.debug(f"Found key '{self.types[key]}'")
+ if v is None:
+ continue
+ vals = self.convertMultiple(v)
+ if len(vals) > 0:
+ tags.update(vals)
+ continue
+ items = self.convertEntry(key, v)
+ if items is None or len(items) == 0:
+ continue
+
+ if type(items) == str:
+ log.debug(f"string Item {items}")
+ elif type(items) == list:
+ # log.debug(f"list Item {items}")
+ tags.update(items[0])
+ elif type(items) == dict:
+ # log.debug(f"dict Item {items}")
+ tags.update(items)
+ # log.debug(f"TAGS: {tags}")
+ if len(tags) > 0:
+ total.append(tags)
+
+ # log.debug(f"Finished parsing JSON file {filespec}")
+ return total
+
+ def XMLparser(
+ self,
+ filespec: str,
+ data: str = None,
+ ) -> list:
+ """Import an ODK XML Instance file ito a data structure. The input is
+ either a filespec to the Instance file copied off your phone, or
+ the XML that has been read in elsewhere.
+
+ Args:
+ filespec (str): The filespec to the ODK XML Instance file
+ data (str): The XML data
+
+ Returns:
+ (list): All the entries in the OSM XML Instance file
+ """
+ row = dict()
+ if filespec:
+ logging.info("Processing instance file: %s" % filespec)
+ file = open(filespec, "rb")
+ # Instances are small, read the whole file
+ xml = file.read(os.path.getsize(filespec))
+ elif data:
+ xml = data
+ doc = xmltodict.parse(xml)
+
+ json.dumps(doc)
+ tags = dict()
+ data = doc["data"]
+ flattened = flatdict.FlatDict(data)
+ # total = list()
+ # log.debug(f"FLAT: {flattened}")
+ pat = re.compile("[0-9.]* [0-9.-]* [0-9.]* [0-9.]*")
+ for key, value in flattened.items():
+ if key[0] == "@" or value is None:
+ continue
+ # Get the last element deliminated by a dash
+ # for CSV & JSON, or a colon for ODK XML.
+ base = basename(key)
+ log.debug(f"FLAT: {base} = {value}")
+ if base in self.ignore:
+ continue
+ if re.search(pat, value):
+ gps = value.split(" ")
+ row["lat"] = gps[0]
+ row["lon"] = gps[1]
+ continue
+
+ if base in self.types:
+ if self.types[base] == "select_multiple":
+ # log.debug(f"Found key '{self.types[base]}'")
+ vals = self.convertMultiple(value)
+ if len(vals) > 0:
+ tags.update(vals)
+ continue
+ else:
+ item = self.convertEntry(base, value)
+ if item is None or len(item) == 0:
+ continue
+ if len(tags) == 0:
+ tags = item[0]
+ else:
+ if type(item) == list:
+ # log.debug(f"list Item {item}")
+ tags.update(item[0])
+ elif type(item) == dict:
+ # log.debug(f"dict Item {item}")
+ tags.update(item)
+ row.update(tags)
+ return [row]
diff --git a/osm_fieldwork/support.py b/osm_fieldwork/support.py
new file mode 100644
index 00000000..6fa50379
--- /dev/null
+++ b/osm_fieldwork/support.py
@@ -0,0 +1,222 @@
+#!/usr/bin/python3
+
+# Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team
+#
+# This file is part of OSM-Fieldwork.
+#
+# OSM-Fieldwork is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# OSM-Fieldwork is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with OSM-Fieldwork. If not, see .
+#
+
+import logging
+from datetime import datetime
+from pathlib import Path
+
+from geojson import Feature, FeatureCollection, Point, dump
+
+from osm_fieldwork.osmfile import OsmFile
+
+# Instantiate logger
+log = logging.getLogger(__name__)
+
+
+def basename(
+ line: str,
+) -> str:
+ """Extract the basename of a path after the last -.
+
+ Args:
+ line (str): The path from the json file entry
+
+ Returns:
+ (str): The last node of the path
+ """
+ if line.find("-") > 0:
+ tmp = line.split("-")
+ if len(tmp) > 0:
+ return tmp[len(tmp) - 1]
+ elif line.find(":") > 0:
+ tmp = line.split(":")
+ if len(tmp) > 0:
+ return tmp[len(tmp) - 1]
+ else:
+ # return tmp[len(tmp) - 1]
+ return line
+
+
+class OutSupport(object):
+ def __init__(
+ self,
+ filespec: str = None,
+ ):
+ self.osm = None
+ self.filespec = filespec
+ self.features = list()
+ if filespec:
+ path = Path(filespec)
+ if path.suffix == ".osm":
+ self.createOSM(filespec)
+ elif path.suffix == ".geojson":
+ self.createGeoJson(filespec)
+ else:
+ log.error(f"{filespec} is not a valid file!")
+
+ def createOSM(
+ self,
+ filespec: str = None,
+ ) -> bool:
+ """Create an OSM XML output files.
+
+ Args:
+ filespec (str): The output file name
+ """
+ if filespec is not None:
+ log.debug("Creating OSM XML file: %s" % filespec)
+ self.osm = OsmFile(filespec)
+ elif self.filespec is not None:
+ log.debug("Creating OSM XML file: %s" % self.filespec)
+ self.osm = OsmFile(self.filespec)
+
+ return True
+
+ def writeOSM(
+ self,
+ feature: dict,
+ ) -> bool:
+ """Write a feature to an OSM XML output file.
+
+ Args:
+ feature (dict): The OSM feature to write to
+ """
+ out = ""
+ if "tags" in feature:
+ if "id" in feature["tags"]:
+ feature["id"] = feature["tags"]["id"]
+ else:
+ return True
+ if "lat" not in feature["attrs"] or "lon" not in feature["attrs"]:
+ return None
+ if "refs" not in feature:
+ out += self.osm.createNode(feature)
+ else:
+ out += self.osm.createWay(feature)
+ self.osm.write(out)
+
+ return True
+
+ def finishOSM(self):
+ """Write the OSM XML file footer and close it."""
+ # This is now handled by a destructor in the OsmFile class
+ # self.osm.footer()
+
+ def createGeoJson(
+ self,
+ filespec: str = "tmp.geojson",
+ ) -> bool:
+ """Create a GeoJson output file.
+
+ Args:
+ filespec (str): The output file name
+ """
+ log.debug("Creating GeoJson file: %s" % filespec)
+ self.json = open(filespec, "w")
+
+ return True
+
+ def writeGeoJson(
+ self,
+ feature: dict,
+ ) -> bool:
+ """Write a feature to a GeoJson output file.
+
+ Args:
+ feature (dict): The OSM feature to write to
+ """
+ # These get written later when finishing , since we have to create a FeatureCollection
+ if "lat" not in feature["attrs"] or "lon" not in feature["attrs"]:
+ return None
+ self.features.append(feature)
+
+ return True
+
+ def finishGeoJson(self):
+ """Write the GeoJson FeatureCollection to the output file and close it."""
+ features = list()
+ for item in self.features:
+ if len(item["attrs"]["lon"]) == 0 or len(item["attrs"]["lat"]) == 0:
+ log.warning("Bad location data in entry! %r", item["attrs"])
+ continue
+ poi = Point((float(item["attrs"]["lon"]), float(item["attrs"]["lat"])))
+ if "private" in item:
+ props = {**item["tags"], **item["private"]}
+ else:
+ props = item["tags"]
+ features.append(Feature(geometry=poi, properties=props))
+ collection = FeatureCollection(features)
+ dump(collection, self.json)
+
+ def WriteData(
+ self,
+ base: str,
+ data: dict(),
+ ) -> bool:
+ """Write the data to the output files.
+
+ Args:
+ base (str): The base of the input file name
+ data (dict): The data to write
+
+ Returns:
+ (bool): Whether the data got written
+ """
+ osmoutfile = f"{base}.osm"
+ self.createOSM(osmoutfile)
+
+ jsonoutfile = f"{base}.geojson"
+ self.createGeoJson(jsonoutfile)
+
+ nodeid = -1000
+ for feature in data:
+ if len(feature) == 0:
+ continue
+ if "refs" in feature:
+ # it's a way
+ refs = list()
+ for ref in feature["refs"]:
+ now = datetime.now().strftime("%Y-%m-%dT%TZ")
+ if len(ref) == 0:
+ continue
+ coords = ref.split(" ")
+ node = {
+ "attrs": {"id": nodeid, "version": 1, "timestamp": now, "lat": coords[0], "lon": coords[1]},
+ "tags": dict(),
+ }
+ self.writeOSM(node)
+ self.writeGeoJson(node)
+ refs.append(nodeid)
+ nodeid -= 1
+ feature["refs"] = refs
+ else:
+ # it's a node
+ if "lat" not in feature["attrs"]:
+ # Sometimes bad entries, usually from debugging XForm design, sneak in
+ log.warning("Bad record! %r" % feature)
+ continue
+ self.writeOSM(feature)
+
+ self.finishOSM()
+ log.info("Wrote OSM XML file: %r" % osmoutfile)
+ self.finishGeoJson()
+ log.info("Wrote GeoJson file: %r" % jsonoutfile)
+
+ return True
diff --git a/pyproject.toml b/pyproject.toml
index 4345be67..2ac2e6f0 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -124,13 +124,8 @@ ignore = ["N805", "B008"]
convention = "google"
[project.scripts]
-json2osm = "osm_fieldwork.json2osm:main"
basemapper = "osm_fieldwork.basemapper:main"
osm2favorites = "osm_fieldwork.osm2favorities:main"
-csv2osm = "osm_fieldwork.CSVDump:main"
-odk2csv = "osm_fieldwork.odk2csv:main"
odk2osm = "osm_fieldwork.odk2osm:main"
-odk2geojson = "osm_fieldwork.odk2geojson:main"
-odk_merge = "osm_fieldwork.odk_merge:main"
odk_client = "osm_fieldwork.odk_client:main"
make_data_extract = "osm_fieldwork.make_data_extract:main"
diff --git a/tests/test_central.py b/tests/test_central.py
old mode 100644
new mode 100755
diff --git a/tests/test_convert.py b/tests/test_convert.py
index 5adb0046..5643bf8a 100755
--- a/tests/test_convert.py
+++ b/tests/test_convert.py
@@ -80,8 +80,8 @@ def test_multiple_value():
"""Test tag value conversion."""
hits = 0
vals = csv.convertMultiple("picnic_table fire_pit parking")
- print(vals)
- if len(vals) > 0 and vals[0]["leisure"] == "picnic_table" and vals[1]["leisure"] == "firepit":
+ # print(vals)
+ if len(vals) > 0 and vals["leisure"] == "picnic_table;firepit":
hits += 1
assert hits == 1
diff --git a/tests/test_csv.py b/tests/test_csv.py
index 7509cbad..8ee2125f 100755
--- a/tests/test_csv.py
+++ b/tests/test_csv.py
@@ -21,7 +21,8 @@
import argparse
import os
-from osm_fieldwork.CSVDump import CSVDump
+from osm_fieldwork.parsers import ODKParsers
+from osm_fieldwork.support import OutSupport
# find the path of root tests dir
rootdir = os.path.dirname(os.path.abspath(__file__))
@@ -30,20 +31,21 @@
def test_csv():
"""Make sure the CSV file got loaded and parsed."""
# FIXME use fixture
- csv = CSVDump()
- data = csv.parse(f"{rootdir}/testdata/test.csv")
+ csv = ODKParsers()
+ data = csv.CSVparser(f"{rootdir}/testdata/test.csv")
assert len(data) > 0
def test_init():
"""Make sure the YAML file got loaded."""
- csv = CSVDump()
+ csv = ODKParsers()
assert len(csv.yaml.yaml) > 0
def test_osm_entry(infile=f"{rootdir}/testdata/test.csv"):
- csv = CSVDump()
- csv.createOSM(infile)
+ csv = ODKParsers()
+ out = OutSupport()
+ out.createOSM(infile)
line = {
"timestamp": "2021-09-25T14:27:43.862Z",
"end": "2021-09-24T17:55:26.194-06:00",