Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for reading pbf #44

Merged
merged 3 commits into from
Aug 1, 2021
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Project.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,13 @@ authors = ["Przemyslaw Szufel <[email protected]>", "Bartosz Pankratz <bartosz.pa
version = "0.2.3"

[deps]
CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193"
DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
LibExpat = "522f3ed2-3f36-55e3-b6df-e94fee9b0c07"
LightGraphs = "093fc24a-ae57-5d10-9952-331d41423f4d"
ProtoBuf = "3349acd9-ac6a-5e09-bcdb-63829b23a429"
Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
Expand Down
6 changes: 6 additions & 0 deletions src/OSMPBF.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# Automatically generated by ProtoBuf.jl
module OSMPBF
const _ProtoBuf_Top_ = @static isdefined(parentmodule(@__MODULE__), :_ProtoBuf_Top_) ? (parentmodule(@__MODULE__))._ProtoBuf_Top_ : parentmodule(@__MODULE__)
include("fileformat_pb.jl")
include("osmformat_pb.jl")
end
1 change: 1 addition & 0 deletions src/OpenStreetMapX.jl
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ include("bounds.jl") #bounds of the map
include("conversion.jl") #conversion of geographical coordinates

include("parseMap.jl") #map parsing funcions
include("pbf.jl")
include("classification.jl") #highways, features and buildings classification functions
include("crop.jl") #cropping nodes and ways

Expand Down
104 changes: 104 additions & 0 deletions src/fileformat_pb.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
# Automatically generated by ProtoBuf.jl
# syntax: proto2
using ProtoBuf
import ProtoBuf.meta

mutable struct Blob <: ProtoType
__protobuf_jl_internal_meta::ProtoMeta
__protobuf_jl_internal_values::Dict{Symbol,Any}
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I do not know whether this is just controlled by ProtoBuf, but if we control it could we, for performance reasons, use some concrete type or a type Union perhaps?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is generated by ProtoBuf.jl, I was also surprised that it would not create concrete type fields instead of a Dict, I thought the whole idea behind ProtoBuf was to generate concrete specialized structs with fields. I suggest we stay with this with this PR and then try to change it to concrete fields and see what's the performance speedup in another PR. This separate PR could be a nice story to report to ProtoBuf.jl if we get a significant speedup

__protobuf_jl_internal_defaultset::Set{Symbol}

function Blob(; kwargs...)
obj = new(meta(Blob), Dict{Symbol,Any}(), Set{Symbol}())
values = obj.__protobuf_jl_internal_values
symdict = obj.__protobuf_jl_internal_meta.symdict
for nv in kwargs
fldname, fldval = nv
fldtype = symdict[fldname].jtyp
(fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
if fldval !== nothing
values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
end
end
obj
end
end # mutable struct Blob
const __meta_Blob = Ref{ProtoMeta}()
function meta(::Type{Blob})
ProtoBuf.metalock() do
if !isassigned(__meta_Blob)
__meta_Blob[] = target = ProtoMeta(Blob)
fnum = Int[2,1,3,4,5,6,7]
allflds = Pair{Symbol,Union{Type,String}}[:raw_size => Int32, :raw => Vector{UInt8}, :zlib_data => Vector{UInt8}, :lzma_data => Vector{UInt8}, :OBSOLETE_bzip2_data => Vector{UInt8}, :lz4_data => Vector{UInt8}, :zstd_data => Vector{UInt8}]
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Again I do not use ProtoBuf, but id we control it Union{Type,Symbol} would look better to me.

Copy link
Contributor Author

@blegat blegat Jul 12, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You mean Pair{Symbol,Type} ?

oneofs = Int[0,1,1,1,1,1,1]
oneof_names = Symbol[Symbol("data")]
meta(target, Blob, allflds, ProtoBuf.DEF_REQ, fnum, ProtoBuf.DEF_VAL, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, oneofs, oneof_names)
end
__meta_Blob[]
end
end
function Base.getproperty(obj::Blob, name::Symbol)
if name === :raw_size
return (obj.__protobuf_jl_internal_values[name])::Int32
elseif name === :raw
return (obj.__protobuf_jl_internal_values[name])::Vector{UInt8}
elseif name === :zlib_data
return (obj.__protobuf_jl_internal_values[name])::Vector{UInt8}
elseif name === :lzma_data
return (obj.__protobuf_jl_internal_values[name])::Vector{UInt8}
elseif name === :OBSOLETE_bzip2_data
return (obj.__protobuf_jl_internal_values[name])::Vector{UInt8}
elseif name === :lz4_data
return (obj.__protobuf_jl_internal_values[name])::Vector{UInt8}
elseif name === :zstd_data
return (obj.__protobuf_jl_internal_values[name])::Vector{UInt8}
else
getfield(obj, name)
end
end

mutable struct BlobHeader <: ProtoType
__protobuf_jl_internal_meta::ProtoMeta
__protobuf_jl_internal_values::Dict{Symbol,Any}
__protobuf_jl_internal_defaultset::Set{Symbol}

function BlobHeader(; kwargs...)
obj = new(meta(BlobHeader), Dict{Symbol,Any}(), Set{Symbol}())
values = obj.__protobuf_jl_internal_values
symdict = obj.__protobuf_jl_internal_meta.symdict
for nv in kwargs
fldname, fldval = nv
fldtype = symdict[fldname].jtyp
(fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
if fldval !== nothing
values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
end
end
obj
end
end # mutable struct BlobHeader
const __meta_BlobHeader = Ref{ProtoMeta}()
function meta(::Type{BlobHeader})
ProtoBuf.metalock() do
if !isassigned(__meta_BlobHeader)
__meta_BlobHeader[] = target = ProtoMeta(BlobHeader)
req = Symbol[:_type,:datasize]
allflds = Pair{Symbol,Union{Type,String}}[:_type => AbstractString, :indexdata => Vector{UInt8}, :datasize => Int32]
meta(target, BlobHeader, allflds, req, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
end
__meta_BlobHeader[]
end
end
function Base.getproperty(obj::BlobHeader, name::Symbol)
if name === :_type
return (obj.__protobuf_jl_internal_values[name])::AbstractString
elseif name === :indexdata
return (obj.__protobuf_jl_internal_values[name])::Vector{UInt8}
elseif name === :datasize
return (obj.__protobuf_jl_internal_values[name])::Int32
else
getfield(obj, name)
end
end

export Blob, BlobHeader
Loading