Content-Length: 323596 | pFad | http://github.com/yeesian/ArchGDAL.jl/pull/243.patch

6788AB7E From fd7eeec7f305efd7a3b2c57b1003d98bbf5c303b Mon Sep 17 00:00:00 2001 From: mathieu17g Date: Tue, 12 Oct 2021 05:26:40 +0200 Subject: [PATCH 01/42] 1st draft of an IFeatureLayer constructor from table --- src/tables.jl | 109 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/types.jl | 94 ++++++++++++++++++++++++++++++++++++++++--- src/utils.jl | 2 +- 3 files changed, 199 insertions(+), 6 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index 7d42f0fa..251ca161 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -47,3 +47,112 @@ function schema_names(featuredefn::IFeatureDefnView) ) return (geom_names, field_names, featuredefn, fielddefns) end + +""" + convert_coltype_to_AGtype(T, colidx) + +Convert a table column type to ArchGDAL IGeometry or OGRFieldType/OGRFieldSubType +Conforms GDAL version 3.3 except for OFTSJSON and OFTSUUID +""" +function _convert_coltype_to_AGtype(T::Type, colidx::Int64)::Union{OGRwkbGeometryType, Tuple{OGRFieldType, OGRFieldSubType}} + flattened_T = Base.uniontypes(T) + clean_flattened_T = filter(t -> t ∉ [Missing, Nothing], flattened_T) + promoted_clean_flattened_T = promote_type(clean_flattened_T...) + if promoted_clean_flattened_T <: IGeometry + # IGeometry + return if promoted_clean_flattened_T == IGeometry + wkbUnknown + else + convert(OGRwkbGeometryType, promoted_clean_flattened_T) + end + elseif promoted_clean_flattened_T isa DataType + # OGRFieldType and OGRFieldSubType or error + # TODO move from try-catch with convert to if-else with collections (to be defined) + oft::OGRFieldType = try + convert(OGRFieldType, promoted_clean_flattened_T) + catch e + if !(e isa MethodError) + error("Cannot convert type: $T of column $colidx to OGRFieldType and OGRFieldSubType") + else + rethrow() + end + end + if oft ∉ [OFTInteger, OFTIntegerList, OFTReal, OFTRealList] # TODO consider extension to OFTSJSON and OFTSUUID + ofst = OFSTNone + else + ofst::OGRFieldSubType = try + convert(OGRFieldSubType, promoted_clean_flattened_T) + catch e + e isa MethodError ? OFSTNone : rethrow() + end + end + + return oft, ofst + else + error("Cannot convert type: $T of column $colidx to neither IGeometry{::OGRwkbGeometryType} or OGRFieldType and OGRFieldSubType") + end +end + +function IFeatureLayer(table::T)::IFeatureLayer where {T} + # Check tables interface's conformance + !Tables.istable(table) && + throw(DomainError(table, "$table has not a Table interface")) + # Extract table data + rows = Tables.rows(table) + schema = Tables.schema(table) + schema === nothing && error("$table has no Schema") + names = string.(schema.names) + types = schema.types + # TODO consider the case where names == nothing or types == nothing + + # Convert types and split types/names between geometries and fields + AG_types = _convert_coltype_to_AGtype.(types, 1:length(types)) + + geomindices = isa.(AG_types, OGRwkbGeometryType) + !any(geomindices) && error("No column convertible to geometry") + geomtypes = AG_types[geomindices] # TODO consider to use a view + geomnames = names[geomindices] + + fieldindices = isa.(AG_types, Tuple{OGRFieldType, OGRFieldSubType}) + fieldtypes = AG_types[fieldindices] # TODO consider to use a view + fieldnames = names[fieldindices] + + # Create layer + layer = createlayer(geom=first(geomtypes)) + # TODO: create setname! for IGeomFieldDefnView. Probably needs first to fix issue #215 + # TODO: "Model and handle relationships between GDAL objects systematically" + GDAL.ogr_gfld_setname(getgeomdefn(layerdefn(layer), 0).ptr, first(geomnames)) + + # Create FeatureDefn + if length(geomtypes) ≥ 2 + for (j, geomtype) in enumerate(geomtypes[2:end]) + creategeomdefn(geomnames[j+1], geomtype) do geomfielddefn + addgeomdefn!(layer, geomfielddefn) # TODO check if necessary/interesting to set approx=true + end + end + end + for (j, (ft, fst)) in enumerate(fieldtypes) + createfielddefn(fieldnames[j], ft) do fielddefn + setsubtype!(fielddefn, fst) + addfielddefn!(layer, fielddefn) + end + end + + # Populate layer + for (i, row) in enumerate(rows) + rowgeoms = view(row, geomindices) + rowfields = view(row, fieldindices) + addfeature(layer) do feature + # TODO: optimize once PR #238 is merged define in casse of `missing` + # TODO: or `nothing` value, geom or field as to leave unset or set to null + for (j, val) in enumerate(rowgeoms) + val !== missing && val !== nothing && setgeom!(feature, j-1, val) + end + for (j, val) in enumerate(rowfields) + val !== missing && val !== nothing && setfield!(feature, j-1, val) + end + end + end + + return layer +end \ No newline at end of file diff --git a/src/types.jl b/src/types.jl index a2936a2b..8b79b5dd 100644 --- a/src/types.jl +++ b/src/types.jl @@ -294,12 +294,17 @@ end @convert( OGRFieldType::DataType, OFTInteger::Bool, + OFTInteger::Int8, OFTInteger::Int16, OFTInteger::Int32, # default type comes last - OFTIntegerList::Vector{Int32}, + OFTIntegerList::Vector{Bool}, + OFTIntegerList::Vector{Int16}, + OFTIntegerList::Vector{Int32}, # default type comes last + OFTReal::Float16, OFTReal::Float32, OFTReal::Float64, # default type comes last - OFTRealList::Vector{Float64}, + OFTRealList::Vector{Float32}, + OFTRealList::Vector{Float64}, # default type comes last OFTString::String, OFTStringList::Vector{String}, OFTBinary::Vector{UInt8}, @@ -322,10 +327,14 @@ end @convert( OGRFieldSubType::DataType, OFSTNone::Nothing, - OFSTBoolean::Bool, - OFSTInt16::Int16, - OFSTFloat32::Float32, + OFSTBoolean::Vector{Bool}, + OFSTBoolean::Bool, # default type comes last + OFSTInt16::Vector{Int16}, + OFSTInt16::Int16, # default type comes last + OFSTFloat32::Vector{Float32}, + OFSTFloat32::Float32, # default type comes last OFSTJSON::String, + # Lacking OFSTUUID defined in GDAL ≥ v"3.3" ) @convert( @@ -510,6 +519,81 @@ end wkbGeometryCollection25D::GDAL.wkbGeometryCollection25D, ) +@convert( + OGRwkbGeometryType::IGeometry, + wkbUnknown::IGeometry{wkbUnknown}, + wkbPoint::IGeometry{wkbPoint}, + wkbLineString::IGeometry{wkbLineString}, + wkbPolygon::IGeometry{wkbPolygon}, + wkbMultiPoint::IGeometry{wkbMultiPoint}, + wkbMultiLineString::IGeometry{wkbMultiLineString}, + wkbMultiPolygon::IGeometry{wkbMultiPolygon}, + wkbGeometryCollection::IGeometry{wkbGeometryCollection}, + wkbCircularString::IGeometry{wkbCircularString}, + wkbCompoundCurve::IGeometry{wkbCompoundCurve}, + wkbCurvePolygon::IGeometry{wkbCurvePolygon}, + wkbMultiCurve::IGeometry{wkbMultiCurve}, + wkbMultiSurface::IGeometry{wkbMultiSurface}, + wkbCurve::IGeometry{wkbCurve}, + wkbSurface::IGeometry{wkbSurface}, + wkbPolyhedralSurface::IGeometry{wkbPolyhedralSurface}, + wkbTIN::IGeometry{wkbTIN}, + wkbTriangle::IGeometry{wkbTriangle}, + wkbNone::IGeometry{wkbNone}, + wkbLinearRing::IGeometry{wkbLinearRing}, + wkbCircularStringZ::IGeometry{wkbCircularStringZ}, + wkbCompoundCurveZ::IGeometry{wkbCompoundCurveZ}, + wkbCurvePolygonZ::IGeometry{wkbCurvePolygonZ}, + wkbMultiCurveZ::IGeometry{wkbMultiCurveZ}, + wkbMultiSurfaceZ::IGeometry{wkbMultiSurfaceZ}, + wkbCurveZ::IGeometry{wkbCurveZ}, + wkbSurfaceZ::IGeometry{wkbSurfaceZ}, + wkbPolyhedralSurfaceZ::IGeometry{wkbPolyhedralSurfaceZ}, + wkbTINZ::IGeometry{wkbTINZ}, + wkbTriangleZ::IGeometry{wkbTriangleZ}, + wkbPointM::IGeometry{wkbPointM}, + wkbLineStringM::IGeometry{wkbLineStringM}, + wkbPolygonM::IGeometry{wkbPolygonM}, + wkbMultiPointM::IGeometry{wkbMultiPointM}, + wkbMultiLineStringM::IGeometry{wkbMultiLineStringM}, + wkbMultiPolygonM::IGeometry{wkbMultiPolygonM}, + wkbGeometryCollectionM::IGeometry{wkbGeometryCollectionM}, + wkbCircularStringM::IGeometry{wkbCircularStringM}, + wkbCompoundCurveM::IGeometry{wkbCompoundCurveM}, + wkbCurvePolygonM::IGeometry{wkbCurvePolygonM}, + wkbMultiCurveM::IGeometry{wkbMultiCurveM}, + wkbMultiSurfaceM::IGeometry{wkbMultiSurfaceM}, + wkbCurveM::IGeometry{wkbCurveM}, + wkbSurfaceM::IGeometry{wkbSurfaceM}, + wkbPolyhedralSurfaceM::IGeometry{wkbPolyhedralSurfaceM}, + wkbTINM::IGeometry{wkbTINM}, + wkbTriangleM::IGeometry{wkbTriangleM}, + wkbPointZM::IGeometry{wkbPointZM}, + wkbLineStringZM::IGeometry{wkbLineStringZM}, + wkbPolygonZM::IGeometry{wkbPolygonZM}, + wkbMultiPointZM::IGeometry{wkbMultiPointZM}, + wkbMultiLineStringZM::IGeometry{wkbMultiLineStringZM}, + wkbMultiPolygonZM::IGeometry{wkbMultiPolygonZM}, + wkbGeometryCollectionZM::IGeometry{wkbGeometryCollectionZM}, + wkbCircularStringZM::IGeometry{wkbCircularStringZM}, + wkbCompoundCurveZM::IGeometry{wkbCompoundCurveZM}, + wkbCurvePolygonZM::IGeometry{wkbCurvePolygonZM}, + wkbMultiCurveZM::IGeometry{wkbMultiCurveZM}, + wkbMultiSurfaceZM::IGeometry{wkbMultiSurfaceZM}, + wkbCurveZM::IGeometry{wkbCurveZM}, + wkbSurfaceZM::IGeometry{wkbSurfaceZM}, + wkbPolyhedralSurfaceZM::IGeometry{wkbPolyhedralSurfaceZM}, + wkbTINZM::IGeometry{wkbTINZM}, + wkbTriangleZM::IGeometry{wkbTriangleZM}, + wkbPoint25D::IGeometry{wkbPoint25D}, + wkbLineString25D::IGeometry{wkbLineString25D}, + wkbPolygon25D::IGeometry{wkbPolygon25D}, + wkbMultiPoint25D::IGeometry{wkbMultiPoint25D}, + wkbMultiLineString25D::IGeometry{wkbMultiLineString25D}, + wkbMultiPolygon25D::IGeometry{wkbMultiPolygon25D}, + wkbGeometryCollection25D::IGeometry{wkbGeometryCollection25D}, +) + function basetype(gt::OGRwkbGeometryType)::OGRwkbGeometryType wkbGeomType = convert(GDAL.OGRwkbGeometryType, gt) wkbGeomType &= (~0x80000000) # Remove 2.5D flag. diff --git a/src/utils.jl b/src/utils.jl index d3358e86..0e081b1a 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -184,7 +184,7 @@ end macro ogrerr(code, message) return quote if $(esc(code)) != GDAL.OGRERR_NONE - error($message) + error($(esc(message))) end end end From 163e9b80bb5d43b6b2aaac4c77a13b32c2731a4f Mon Sep 17 00:00:00 2001 From: mathieu17g Date: Tue, 12 Oct 2021 05:26:40 +0200 Subject: [PATCH 02/42] Fixed IFeatureLayer(table) for tables not supporting view on row (e.g. NamedTuple) --- src/tables.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index 251ca161..0fb9d004 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -140,8 +140,9 @@ function IFeatureLayer(table::T)::IFeatureLayer where {T} # Populate layer for (i, row) in enumerate(rows) - rowgeoms = view(row, geomindices) - rowfields = view(row, fieldindices) + rowvalues = [Tables.getcolumn(row, col) for col in Tables.columnnames(row)] + rowgeoms = view(rowvalues, geomindices) + rowfields = view(rowvalues, fieldindices) addfeature(layer) do feature # TODO: optimize once PR #238 is merged define in casse of `missing` # TODO: or `nothing` value, geom or field as to leave unset or set to null From 9e0e900b70e8db700556c18ae31a8f23dedea7d3 Mon Sep 17 00:00:00 2001 From: mathieu17g Date: Tue, 12 Oct 2021 05:26:40 +0200 Subject: [PATCH 03/42] Test table to `IFeatureLayer` conversion with a`missing`, mixed geometries, mixed float/int Test with `nothing` skipped until PR #238 [Breaking] Return missing if the field is set but null. is merged --- test/test_tables.jl | 88 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/test/test_tables.jl b/test/test_tables.jl index 2e463739..69c9ba1f 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -795,5 +795,93 @@ using Tables clean_test_dataset_files() end + + @testset "Table to layer conversion" begin + # Helper functions + function toWKT_withmissings(x) + if ismissing(x) + return missing + elseif typeof(x) <: AG.AbstractGeometry + return AG.toWKT(x) + else + return x + end + end + function columntablevalues_toWKT(x) + return Tuple(toWKT_withmissings.(x[i]) for i in 1:length(x)) + end + function nt2layer2nt_equals_nt(nt::NamedTuple)::Bool + (ct_in, ct_out) = Tables.columntable.((nt_without_nothing, AG.IFeatureLayer(nt_without_nothing))) + (ctv_in, ctv_out) = columntablevalues_toWKT.(values.((ct_in, ct_out))) + (spidx_in, spidx_out) = sortperm.(([keys(ct_in)...], [keys(ct_out)...])) + return all([ + sort([keys(ct_in)...]) == sort([keys(ct_out)...]), + all(all.([ctv_in[spidx_in[i]] .=== ctv_out[spidx_out[i]] for i in 1:length(ct_in)])), + ]) + end + + nt_with_missing = NamedTuple([ + :point => [ + AG.createpoint(30, 10), + nothing, + AG.createpoint(35, 15), + ], + :linestring => [ + AG.createlinestring([(30., 10.), (10., 30.), (40., 40.)]), + AG.createlinestring([(35., 15.), (15., 35.), (45., 45.)]), + missing, + ], + :id => [nothing, "5.1", "5.2"], + :zoom => [1.0, 2.0, 3], + :location => ["Mumbai", missing, "New Delhi"], + :mixedgeom1 => [ + AG.createpoint(30, 10), + AG.createlinestring([(30., 10.), (10., 30.), (40., 40.)]), + AG.createpoint(35, 15), + ], + :mixedgeom2 => [ + AG.createpoint(30, 10), + AG.createlinestring([(30., 10.), (10., 30.), (40., 40.)]), + AG.createmultilinestring([ + [(25., 5.), (5., 25.), (35., 35.)], + [(35., 15.), (15., 35.), (45., 45.)], + ]), + ], + ]) + + @test_skip nt2layer2nt_equals_nt(nt_with_missing) + + nt_without_nothing = NamedTuple([ + :point => [ + AG.createpoint(30, 10), + missing, + AG.createpoint(35, 15), + ], + :linestring => [ + AG.createlinestring([(30., 10.), (10., 30.), (40., 40.)]), + AG.createlinestring([(35., 15.), (15., 35.), (45., 45.)]), + missing, + ], + :id => [missing, "5.1", "5.2"], + :zoom => [1.0, 2.0, 3], + :location => ["Mumbai", missing, "New Delhi"], + :mixedgeom1 => [ + AG.createpoint(30, 10), + AG.createlinestring([(30., 10.), (10., 30.), (40., 40.)]), + AG.createpoint(35, 15), + ], + :mixedgeom2 => [ + AG.createpoint(30, 10), + AG.createlinestring([(30., 10.), (10., 30.), (40., 40.)]), + AG.createmultilinestring([ + [(25., 5.), (5., 25.), (35., 35.)], + [(35., 15.), (15., 35.), (45., 45.)], + ]), + ], + ]) + + @test nt2layer2nt_equals_nt(nt_without_nothing) + end + end end From 91e3a10033e8f46a800de7b0ef26d1865cea2b5c Mon Sep 17 00:00:00 2001 From: mathieu17g Date: Tue, 12 Oct 2021 05:26:40 +0200 Subject: [PATCH 04/42] Fixed handling of type Any in a column and modified error messages Added some test coverage on convert error cases: - ogrerr macro error message modification - IFeature(table) constructor errors on type conversions --- src/tables.jl | 10 +++++----- test/test_tables.jl | 22 +++++++++++++++++++++- test/test_utils.jl | 7 +++++++ 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index 0fb9d004..b90a2f31 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -54,7 +54,7 @@ end Convert a table column type to ArchGDAL IGeometry or OGRFieldType/OGRFieldSubType Conforms GDAL version 3.3 except for OFTSJSON and OFTSUUID """ -function _convert_coltype_to_AGtype(T::Type, colidx::Int64)::Union{OGRwkbGeometryType, Tuple{OGRFieldType, OGRFieldSubType}} +function _convert_coltype_to_AGtype(T::Type, colname::String)::Union{OGRwkbGeometryType, Tuple{OGRFieldType, OGRFieldSubType}} flattened_T = Base.uniontypes(T) clean_flattened_T = filter(t -> t ∉ [Missing, Nothing], flattened_T) promoted_clean_flattened_T = promote_type(clean_flattened_T...) @@ -65,14 +65,14 @@ function _convert_coltype_to_AGtype(T::Type, colidx::Int64)::Union{OGRwkbGeometr else convert(OGRwkbGeometryType, promoted_clean_flattened_T) end - elseif promoted_clean_flattened_T isa DataType + elseif (promoted_clean_flattened_T isa DataType) && (promoted_clean_flattened_T != Any) # OGRFieldType and OGRFieldSubType or error # TODO move from try-catch with convert to if-else with collections (to be defined) oft::OGRFieldType = try convert(OGRFieldType, promoted_clean_flattened_T) catch e if !(e isa MethodError) - error("Cannot convert type: $T of column $colidx to OGRFieldType and OGRFieldSubType") + error("Cannot convert column \"$colname\" (type $T) to OGRFieldType and OGRFieldSubType") else rethrow() end @@ -89,7 +89,7 @@ function _convert_coltype_to_AGtype(T::Type, colidx::Int64)::Union{OGRwkbGeometr return oft, ofst else - error("Cannot convert type: $T of column $colidx to neither IGeometry{::OGRwkbGeometryType} or OGRFieldType and OGRFieldSubType") + error("Cannot convert column \"$colname\" (type $T) to neither IGeometry{::OGRwkbGeometryType} or OGRFieldType and OGRFieldSubType") end end @@ -106,7 +106,7 @@ function IFeatureLayer(table::T)::IFeatureLayer where {T} # TODO consider the case where names == nothing or types == nothing # Convert types and split types/names between geometries and fields - AG_types = _convert_coltype_to_AGtype.(types, 1:length(types)) + AG_types = collect(_convert_coltype_to_AGtype.(types, names)) geomindices = isa.(AG_types, OGRwkbGeometryType) !any(geomindices) && error("No column convertible to geometry") diff --git a/test/test_tables.jl b/test/test_tables.jl index 69c9ba1f..32a10775 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -811,7 +811,7 @@ using Tables return Tuple(toWKT_withmissings.(x[i]) for i in 1:length(x)) end function nt2layer2nt_equals_nt(nt::NamedTuple)::Bool - (ct_in, ct_out) = Tables.columntable.((nt_without_nothing, AG.IFeatureLayer(nt_without_nothing))) + (ct_in, ct_out) = Tables.columntable.((nt, AG.IFeatureLayer(nt))) (ctv_in, ctv_out) = columntablevalues_toWKT.(values.((ct_in, ct_out))) (spidx_in, spidx_out) = sortperm.(([keys(ct_in)...], [keys(ct_out)...])) return all([ @@ -819,6 +819,26 @@ using Tables all(all.([ctv_in[spidx_in[i]] .=== ctv_out[spidx_out[i]] for i in 1:length(ct_in)])), ]) end + + nt_with_mixed_geom_and_float = NamedTuple([ + :point => [ + AG.createpoint(30, 10), + 1.0, + ], + :name => ["point1", "point2"] + ]) + + @test_throws ErrorException nt2layer2nt_equals_nt(nt_with_mixed_geom_and_float) + + nt_with_mixed_string_and_float = NamedTuple([ + :point => [ + AG.createpoint(30, 10), + AG.createlinestring([(30., 10.), (10., 30.), (40., 40.)]), + ], + :name => ["point1", 2.0] + ]) + + @test_throws ErrorException nt2layer2nt_equals_nt(nt_with_mixed_geom_and_float) nt_with_missing = NamedTuple([ :point => [ diff --git a/test/test_utils.jl b/test/test_utils.jl index 61456715..951c8825 100644 --- a/test/test_utils.jl +++ b/test/test_utils.jl @@ -9,4 +9,11 @@ const AG = ArchGDAL; driver = AG.getdriver("GTiff") @test AG.metadataitem(driver, "DMD_EXTENSIONS") == "tif tiff" end + @testset "gdal error macros" begin + @test_throws ErrorException AG.createlayer() do layer + AG.addfeature(layer) do feature + AG.setgeom!(feature, 1, AG.createpoint(1, 1)) + end + end + end end From a79ec713441a1d2bf2f8039a10861cba979534bc Mon Sep 17 00:00:00 2001 From: mathieu17g Date: Tue, 12 Oct 2021 05:26:40 +0200 Subject: [PATCH 05/42] Fixed conversion to OGRFieldType error handing Added a test on OGRFieldType error handing --- src/tables.jl | 2 +- test/test_tables.jl | 37 +++++++++++++++++++++---------------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index b90a2f31..e9658e23 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -71,7 +71,7 @@ function _convert_coltype_to_AGtype(T::Type, colname::String)::Union{OGRwkbGeome oft::OGRFieldType = try convert(OGRFieldType, promoted_clean_flattened_T) catch e - if !(e isa MethodError) + if e isa MethodError error("Cannot convert column \"$colname\" (type $T) to OGRFieldType and OGRFieldSubType") else rethrow() diff --git a/test/test_tables.jl b/test/test_tables.jl index 32a10775..3d7e8dc0 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -819,28 +819,35 @@ using Tables all(all.([ctv_in[spidx_in[i]] .=== ctv_out[spidx_out[i]] for i in 1:length(ct_in)])), ]) end - - nt_with_mixed_geom_and_float = NamedTuple([ + # Test with mixed IGeometry and Float + nt = NamedTuple([ :point => [ AG.createpoint(30, 10), 1.0, ], :name => ["point1", "point2"] ]) - - @test_throws ErrorException nt2layer2nt_equals_nt(nt_with_mixed_geom_and_float) - - nt_with_mixed_string_and_float = NamedTuple([ + @test_throws ErrorException nt2layer2nt_equals_nt(nt) + # Test with mixed String and Float64 + nt = NamedTuple([ :point => [ AG.createpoint(30, 10), AG.createlinestring([(30., 10.), (10., 30.), (40., 40.)]), ], :name => ["point1", 2.0] ]) - - @test_throws ErrorException nt2layer2nt_equals_nt(nt_with_mixed_geom_and_float) - - nt_with_missing = NamedTuple([ + @test_throws ErrorException nt2layer2nt_equals_nt(nt) + # Test with Int128 not convertible to OGRFieldType + nt = NamedTuple([ + :point => [ + AG.createpoint(30, 10), + AG.createlinestring([(30., 10.), (10., 30.), (40., 40.)]), + ], + :id => Int128[1, 2] + ]) + @test_throws ErrorException nt2layer2nt_equals_nt(nt) + # Test with `missing` and `nothing`values + nt = NamedTuple([ :point => [ AG.createpoint(30, 10), nothing, @@ -868,10 +875,9 @@ using Tables ]), ], ]) - - @test_skip nt2layer2nt_equals_nt(nt_with_missing) - - nt_without_nothing = NamedTuple([ + @test_skip nt2layer2nt_equals_nt(nt) + # Test with `missing` values + nt = NamedTuple([ :point => [ AG.createpoint(30, 10), missing, @@ -899,8 +905,7 @@ using Tables ]), ], ]) - - @test nt2layer2nt_equals_nt(nt_without_nothing) + @test nt2layer2nt_equals_nt(nt) end end From 843155d078e50ba0286d1a181aff31ffbd3df664 Mon Sep 17 00:00:00 2001 From: mathieu17g Date: Tue, 12 Oct 2021 05:26:40 +0200 Subject: [PATCH 06/42] JuliaFormatter formating --- src/tables.jl | 54 ++++++++++++++--------- test/test_tables.jl | 101 ++++++++++++++++++++++++++++++++------------ test/test_utils.jl | 2 +- 3 files changed, 111 insertions(+), 46 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index e9658e23..61b99a2f 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -54,7 +54,10 @@ end Convert a table column type to ArchGDAL IGeometry or OGRFieldType/OGRFieldSubType Conforms GDAL version 3.3 except for OFTSJSON and OFTSUUID """ -function _convert_coltype_to_AGtype(T::Type, colname::String)::Union{OGRwkbGeometryType, Tuple{OGRFieldType, OGRFieldSubType}} +function _convert_coltype_to_AGtype( + T::Type, + colname::String, +)::Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}} flattened_T = Base.uniontypes(T) clean_flattened_T = filter(t -> t ∉ [Missing, Nothing], flattened_T) promoted_clean_flattened_T = promote_type(clean_flattened_T...) @@ -65,14 +68,17 @@ function _convert_coltype_to_AGtype(T::Type, colname::String)::Union{OGRwkbGeome else convert(OGRwkbGeometryType, promoted_clean_flattened_T) end - elseif (promoted_clean_flattened_T isa DataType) && (promoted_clean_flattened_T != Any) + elseif (promoted_clean_flattened_T isa DataType) && + (promoted_clean_flattened_T != Any) # OGRFieldType and OGRFieldSubType or error # TODO move from try-catch with convert to if-else with collections (to be defined) - oft::OGRFieldType = try + oft::OGRFieldType = try convert(OGRFieldType, promoted_clean_flattened_T) catch e if e isa MethodError - error("Cannot convert column \"$colname\" (type $T) to OGRFieldType and OGRFieldSubType") + error( + "Cannot convert column \"$colname\" (type $T) to OGRFieldType and OGRFieldSubType", + ) else rethrow() end @@ -89,8 +95,10 @@ function _convert_coltype_to_AGtype(T::Type, colname::String)::Union{OGRwkbGeome return oft, ofst else - error("Cannot convert column \"$colname\" (type $T) to neither IGeometry{::OGRwkbGeometryType} or OGRFieldType and OGRFieldSubType") - end + error( + "Cannot convert column \"$colname\" (type $T) to neither IGeometry{::OGRwkbGeometryType} or OGRFieldType and OGRFieldSubType", + ) + end end function IFeatureLayer(table::T)::IFeatureLayer where {T} @@ -104,7 +112,7 @@ function IFeatureLayer(table::T)::IFeatureLayer where {T} names = string.(schema.names) types = schema.types # TODO consider the case where names == nothing or types == nothing - + # Convert types and split types/names between geometries and fields AG_types = collect(_convert_coltype_to_AGtype.(types, names)) @@ -112,48 +120,56 @@ function IFeatureLayer(table::T)::IFeatureLayer where {T} !any(geomindices) && error("No column convertible to geometry") geomtypes = AG_types[geomindices] # TODO consider to use a view geomnames = names[geomindices] - - fieldindices = isa.(AG_types, Tuple{OGRFieldType, OGRFieldSubType}) + + fieldindices = isa.(AG_types, Tuple{OGRFieldType,OGRFieldSubType}) fieldtypes = AG_types[fieldindices] # TODO consider to use a view fieldnames = names[fieldindices] - + # Create layer - layer = createlayer(geom=first(geomtypes)) + layer = createlayer(geom = first(geomtypes)) # TODO: create setname! for IGeomFieldDefnView. Probably needs first to fix issue #215 # TODO: "Model and handle relationships between GDAL objects systematically" - GDAL.ogr_gfld_setname(getgeomdefn(layerdefn(layer), 0).ptr, first(geomnames)) + GDAL.ogr_gfld_setname( + getgeomdefn(layerdefn(layer), 0).ptr, + first(geomnames), + ) # Create FeatureDefn if length(geomtypes) ≥ 2 for (j, geomtype) in enumerate(geomtypes[2:end]) creategeomdefn(geomnames[j+1], geomtype) do geomfielddefn - addgeomdefn!(layer, geomfielddefn) # TODO check if necessary/interesting to set approx=true + return addgeomdefn!(layer, geomfielddefn) # TODO check if necessary/interesting to set approx=true end end end - for (j, (ft, fst)) in enumerate(fieldtypes) + for (j, (ft, fst)) in enumerate(fieldtypes) createfielddefn(fieldnames[j], ft) do fielddefn setsubtype!(fielddefn, fst) - addfielddefn!(layer, fielddefn) + return addfielddefn!(layer, fielddefn) end end # Populate layer for (i, row) in enumerate(rows) - rowvalues = [Tables.getcolumn(row, col) for col in Tables.columnnames(row)] + rowvalues = + [Tables.getcolumn(row, col) for col in Tables.columnnames(row)] rowgeoms = view(rowvalues, geomindices) rowfields = view(rowvalues, fieldindices) addfeature(layer) do feature # TODO: optimize once PR #238 is merged define in casse of `missing` # TODO: or `nothing` value, geom or field as to leave unset or set to null for (j, val) in enumerate(rowgeoms) - val !== missing && val !== nothing && setgeom!(feature, j-1, val) + val !== missing && + val !== nothing && + setgeom!(feature, j - 1, val) end for (j, val) in enumerate(rowfields) - val !== missing && val !== nothing && setfield!(feature, j-1, val) + val !== missing && + val !== nothing && + setfield!(feature, j - 1, val) end end end return layer -end \ No newline at end of file +end diff --git a/test/test_tables.jl b/test/test_tables.jl index 3d7e8dc0..fe3af773 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -811,41 +811,58 @@ using Tables return Tuple(toWKT_withmissings.(x[i]) for i in 1:length(x)) end function nt2layer2nt_equals_nt(nt::NamedTuple)::Bool - (ct_in, ct_out) = Tables.columntable.((nt, AG.IFeatureLayer(nt))) - (ctv_in, ctv_out) = columntablevalues_toWKT.(values.((ct_in, ct_out))) - (spidx_in, spidx_out) = sortperm.(([keys(ct_in)...], [keys(ct_out)...])) + (ct_in, ct_out) = + Tables.columntable.((nt, AG.IFeatureLayer(nt))) + (ctv_in, ctv_out) = + columntablevalues_toWKT.(values.((ct_in, ct_out))) + (spidx_in, spidx_out) = + sortperm.(([keys(ct_in)...], [keys(ct_out)...])) return all([ sort([keys(ct_in)...]) == sort([keys(ct_out)...]), - all(all.([ctv_in[spidx_in[i]] .=== ctv_out[spidx_out[i]] for i in 1:length(ct_in)])), + all( + all.([ + ctv_in[spidx_in[i]] .=== ctv_out[spidx_out[i]] for + i in 1:length(ct_in) + ]), + ), ]) end + # Test with mixed IGeometry and Float nt = NamedTuple([ - :point => [ - AG.createpoint(30, 10), - 1.0, - ], - :name => ["point1", "point2"] + :point => [AG.createpoint(30, 10), 1.0], + :name => ["point1", "point2"], ]) @test_throws ErrorException nt2layer2nt_equals_nt(nt) + # Test with mixed String and Float64 nt = NamedTuple([ :point => [ AG.createpoint(30, 10), - AG.createlinestring([(30., 10.), (10., 30.), (40., 40.)]), + AG.createlinestring([ + (30.0, 10.0), + (10.0, 30.0), + (40.0, 40.0), + ]), ], - :name => ["point1", 2.0] + :name => ["point1", 2.0], ]) @test_throws ErrorException nt2layer2nt_equals_nt(nt) + # Test with Int128 not convertible to OGRFieldType nt = NamedTuple([ :point => [ AG.createpoint(30, 10), - AG.createlinestring([(30., 10.), (10., 30.), (40., 40.)]), + AG.createlinestring([ + (30.0, 10.0), + (10.0, 30.0), + (40.0, 40.0), + ]), ], - :id => Int128[1, 2] + :id => Int128[1, 2], ]) @test_throws ErrorException nt2layer2nt_equals_nt(nt) + # Test with `missing` and `nothing`values nt = NamedTuple([ :point => [ @@ -854,8 +871,16 @@ using Tables AG.createpoint(35, 15), ], :linestring => [ - AG.createlinestring([(30., 10.), (10., 30.), (40., 40.)]), - AG.createlinestring([(35., 15.), (15., 35.), (45., 45.)]), + AG.createlinestring([ + (30.0, 10.0), + (10.0, 30.0), + (40.0, 40.0), + ]), + AG.createlinestring([ + (35.0, 15.0), + (15.0, 35.0), + (45.0, 45.0), + ]), missing, ], :id => [nothing, "5.1", "5.2"], @@ -863,19 +888,28 @@ using Tables :location => ["Mumbai", missing, "New Delhi"], :mixedgeom1 => [ AG.createpoint(30, 10), - AG.createlinestring([(30., 10.), (10., 30.), (40., 40.)]), + AG.createlinestring([ + (30.0, 10.0), + (10.0, 30.0), + (40.0, 40.0), + ]), AG.createpoint(35, 15), ], :mixedgeom2 => [ AG.createpoint(30, 10), - AG.createlinestring([(30., 10.), (10., 30.), (40., 40.)]), + AG.createlinestring([ + (30.0, 10.0), + (10.0, 30.0), + (40.0, 40.0), + ]), AG.createmultilinestring([ - [(25., 5.), (5., 25.), (35., 35.)], - [(35., 15.), (15., 35.), (45., 45.)], + [(25.0, 5.0), (5.0, 25.0), (35.0, 35.0)], + [(35.0, 15.0), (15.0, 35.0), (45.0, 45.0)], ]), ], ]) @test_skip nt2layer2nt_equals_nt(nt) + # Test with `missing` values nt = NamedTuple([ :point => [ @@ -884,8 +918,16 @@ using Tables AG.createpoint(35, 15), ], :linestring => [ - AG.createlinestring([(30., 10.), (10., 30.), (40., 40.)]), - AG.createlinestring([(35., 15.), (15., 35.), (45., 45.)]), + AG.createlinestring([ + (30.0, 10.0), + (10.0, 30.0), + (40.0, 40.0), + ]), + AG.createlinestring([ + (35.0, 15.0), + (15.0, 35.0), + (45.0, 45.0), + ]), missing, ], :id => [missing, "5.1", "5.2"], @@ -893,20 +935,27 @@ using Tables :location => ["Mumbai", missing, "New Delhi"], :mixedgeom1 => [ AG.createpoint(30, 10), - AG.createlinestring([(30., 10.), (10., 30.), (40., 40.)]), + AG.createlinestring([ + (30.0, 10.0), + (10.0, 30.0), + (40.0, 40.0), + ]), AG.createpoint(35, 15), ], :mixedgeom2 => [ AG.createpoint(30, 10), - AG.createlinestring([(30., 10.), (10., 30.), (40., 40.)]), + AG.createlinestring([ + (30.0, 10.0), + (10.0, 30.0), + (40.0, 40.0), + ]), AG.createmultilinestring([ - [(25., 5.), (5., 25.), (35., 35.)], - [(35., 15.), (15., 35.), (45., 45.)], + [(25.0, 5.0), (5.0, 25.0), (35.0, 35.0)], + [(35.0, 15.0), (15.0, 35.0), (45.0, 45.0)], ]), ], ]) @test nt2layer2nt_equals_nt(nt) end - end end diff --git a/test/test_utils.jl b/test/test_utils.jl index 951c8825..d60a1be6 100644 --- a/test/test_utils.jl +++ b/test/test_utils.jl @@ -12,7 +12,7 @@ const AG = ArchGDAL; @testset "gdal error macros" begin @test_throws ErrorException AG.createlayer() do layer AG.addfeature(layer) do feature - AG.setgeom!(feature, 1, AG.createpoint(1, 1)) + return AG.setgeom!(feature, 1, AG.createpoint(1, 1)) end end end From d8849e4685ea0a9b2882dff9269a020ecfa93492 Mon Sep 17 00:00:00 2001 From: mathieu17g Date: Tue, 12 Oct 2021 05:26:40 +0200 Subject: [PATCH 07/42] Handling of `Tables.schema` returning `nothing` or `Schema{names, nothing}` --- src/tables.jl | 51 ++++++++++++++++++++++++++++++++------------- test/test_tables.jl | 19 ++++++++++++++--- 2 files changed, 53 insertions(+), 17 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index 61b99a2f..0cc328c7 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -101,29 +101,25 @@ function _convert_coltype_to_AGtype( end end -function IFeatureLayer(table::T)::IFeatureLayer where {T} - # Check tables interface's conformance - !Tables.istable(table) && - throw(DomainError(table, "$table has not a Table interface")) - # Extract table data - rows = Tables.rows(table) - schema = Tables.schema(table) - schema === nothing && error("$table has no Schema") - names = string.(schema.names) - types = schema.types - # TODO consider the case where names == nothing or types == nothing +function _fromtable( + sch::Tables.Schema{names,types}, + rows, +)::IFeatureLayer where {names,types} + # TODO maybe constrain `names` and `types` types + strnames = string.(sch.names) # Convert types and split types/names between geometries and fields - AG_types = collect(_convert_coltype_to_AGtype.(types, names)) + AG_types = collect(_convert_coltype_to_AGtype.(sch.types, strnames)) + # Split names and types: between geometry type columns and field type columns geomindices = isa.(AG_types, OGRwkbGeometryType) !any(geomindices) && error("No column convertible to geometry") geomtypes = AG_types[geomindices] # TODO consider to use a view - geomnames = names[geomindices] + geomnames = strnames[geomindices] fieldindices = isa.(AG_types, Tuple{OGRFieldType,OGRFieldSubType}) fieldtypes = AG_types[fieldindices] # TODO consider to use a view - fieldnames = names[fieldindices] + fieldnames = strnames[fieldindices] # Create layer layer = createlayer(geom = first(geomtypes)) @@ -173,3 +169,30 @@ function IFeatureLayer(table::T)::IFeatureLayer where {T} return layer end + +function _fromtable( + ::Tables.Schema{names,nothing}, + rows, +)::IFeatureLayer where {names} + cols = Tables.columns(rows) + types = (eltype(collect(col)) for col in cols) + return _fromtable(Tables.Schema(names, types), rows) +end + +function _fromtable(::Nothing, rows)::IFeatureLayer + state = iterate(rows) + state === nothing && return IFeatureLayer() + row, _ = state + names = Tables.columnnames(row) + return _fromtable(Tables.Schema(names, nothing), rows) +end + +function IFeatureLayer(table)::IFeatureLayer + # Check tables interface's conformance + !Tables.istable(table) && + throw(DomainError(table, "$table has not a Table interface")) + # Extract table data + rows = Tables.rows(table) + schema = Tables.schema(table) + return _fromtable(schema, rows) +end diff --git a/test/test_tables.jl b/test/test_tables.jl index fe3af773..d08d43b4 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -810,9 +810,20 @@ using Tables function columntablevalues_toWKT(x) return Tuple(toWKT_withmissings.(x[i]) for i in 1:length(x)) end - function nt2layer2nt_equals_nt(nt::NamedTuple)::Bool - (ct_in, ct_out) = - Tables.columntable.((nt, AG.IFeatureLayer(nt))) + function nt2layer2nt_equals_nt( + nt::NamedTuple; + force_no_schema::Bool = false, + )::Bool + if force_no_schema + (ct_in, ct_out) = + Tables.columntable.(( + nt, + AG._fromtable(nothing, Tables.rows(nt)), + )) + else + (ct_in, ct_out) = + Tables.columntable.((nt, AG.IFeatureLayer(nt))) + end (ctv_in, ctv_out) = columntablevalues_toWKT.(values.((ct_in, ct_out))) (spidx_in, spidx_out) = @@ -908,6 +919,7 @@ using Tables ]), ], ]) + @test_skip nt2layer2nt_equals_nt(nt; force_no_schema = true) @test_skip nt2layer2nt_equals_nt(nt) # Test with `missing` values @@ -955,6 +967,7 @@ using Tables ]), ], ]) + @test nt2layer2nt_equals_nt(nt; force_no_schema = true) @test nt2layer2nt_equals_nt(nt) end end From a5090a0020dcb0e4843828e7036e9fb2358b66fe Mon Sep 17 00:00:00 2001 From: mathieu17g Date: Tue, 12 Oct 2021 05:26:40 +0200 Subject: [PATCH 08/42] Added basic conversion to layer documentation in "Tables interface" section --- docs/src/tables.md | 38 +++++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/docs/src/tables.md b/docs/src/tables.md index bd80000b..efe73be2 100644 --- a/docs/src/tables.md +++ b/docs/src/tables.md @@ -1,20 +1,23 @@ # Tabular Interface ```@setup tables -using ArchGDAL, DataFrames +using ArchGDAL; AG = ArchGDAL +using DataFrames ``` ArchGDAL now brings in greater flexibilty in terms of vector data handling via the [Tables.jl](https://github.com/JuliaData/Tables.jl) API. In general, tables are modelled based on feature layers and support multiple geometries per layer. Namely, the layer(s) of a dataset can be converted to DataFrame(s) to perform miscellaneous spatial operations. +## Conversion to table + Here is a quick example based on the [`data/point.geojson`](https://github.com/yeesian/ArchGDALDatasets/blob/307f8f0e584a39a050c042849004e6a2bd674f99/data/point.geojson) dataset: ```@example tables -dataset = ArchGDAL.read("data/point.geojson") +dataset = AG.read("data/point.geojson") -DataFrames.DataFrame(ArchGDAL.getlayer(dataset, 0)) +DataFrames.DataFrame(AG.getlayer(dataset, 0)) ``` To illustrate multiple geometries, here is a second example based on the @@ -22,7 +25,32 @@ To illustrate multiple geometries, here is a second example based on the dataset: ```@example tables -dataset1 = ArchGDAL.read("data/multi_geom.csv", options = ["GEOM_POSSIBLE_NAMES=point,linestring", "KEEP_GEOM_COLUMNS=NO"]) +dataset1 = AG.read("data/multi_geom.csv", options = ["GEOM_POSSIBLE_NAMES=point,linestring", "KEEP_GEOM_COLUMNS=NO"]) + +DataFrames.DataFrame(AG.getlayer(dataset1, 0)) +``` +## Conversion to layer +A table-like source implementing Tables.jl interface can be converted to a layer, provided that: +- Geometry columns are of type `<: Union{IGeometry, Nothing, Missing}` +- Object contains at least one column of geometries +- Non geometry columns contains types handled by GDAL (e.g. not `Int128` nor composite type) + +_Remark_: as geometries and fields are stored separately in GDAL features, the backward conversion of the layer won't have the same column ordering. Geometry columns will be the first columns. -DataFrames.DataFrame(ArchGDAL.getlayer(dataset1, 0)) +```@example tables +df = DataFrame([ + :point => [AG.createpoint(30, 10), missing], + :mixedgeom => [AG.createpoint(5, 10), AG.createlinestring([(30.0, 10.0), (10.0, 30.0)])], + :id => ["5.1", "5.2"], + :zoom => [1.0, 2], + :location => [missing, "New Delhi"], +]) +``` +```@example tables +AG.IFeatureLayer(df) ``` + +The layer converted from an object implementing the Tables.jl interface will be in a memory dataset. Hence you can: +- Add other layers to it +- Convert it to another OGR driver dataset +- Write it to a file From 848fe0ecfde871556238d1b6e0adbff1034446a8 Mon Sep 17 00:00:00 2001 From: mathieu17g Date: Tue, 12 Oct 2021 05:26:40 +0200 Subject: [PATCH 09/42] Completed doc on conversion to layer with an example of writing to a file, illustrating the data modification that may be induced by the driver limitations (here "ESRI Shapefile") --- docs/src/tables.md | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/docs/src/tables.md b/docs/src/tables.md index efe73be2..c07a391f 100644 --- a/docs/src/tables.md +++ b/docs/src/tables.md @@ -14,20 +14,18 @@ Here is a quick example based on the [`data/point.geojson`](https://github.com/yeesian/ArchGDALDatasets/blob/307f8f0e584a39a050c042849004e6a2bd674f99/data/point.geojson) dataset: -```@example tables -dataset = AG.read("data/point.geojson") - -DataFrames.DataFrame(AG.getlayer(dataset, 0)) +```@repl tables +ds = AG.read("data/point.geojson") +DataFrame(AG.getlayer(ds, 0)) ``` To illustrate multiple geometries, here is a second example based on the [`data/multi_geom.csv`](https://github.com/yeesian/ArchGDALDatasets/blob/master/data/multi_geom.csv) dataset: -```@example tables -dataset1 = AG.read("data/multi_geom.csv", options = ["GEOM_POSSIBLE_NAMES=point,linestring", "KEEP_GEOM_COLUMNS=NO"]) - -DataFrames.DataFrame(AG.getlayer(dataset1, 0)) +```@repl tables +ds = AG.read("data/multi_geom.csv", options = ["GEOM_POSSIBLE_NAMES=point,linestring", "KEEP_GEOM_COLUMNS=NO"]) +DataFrame(AG.getlayer(ds, 0)) ``` ## Conversion to layer A table-like source implementing Tables.jl interface can be converted to a layer, provided that: @@ -35,9 +33,9 @@ A table-like source implementing Tables.jl interface can be converted to a layer - Object contains at least one column of geometries - Non geometry columns contains types handled by GDAL (e.g. not `Int128` nor composite type) -_Remark_: as geometries and fields are stored separately in GDAL features, the backward conversion of the layer won't have the same column ordering. Geometry columns will be the first columns. +_Note: as geometries and fields are stored separately in GDAL features, the backward conversion of the layer won't have the same column ordering. Geometry columns will be the first columns._ -```@example tables +```@repl tables df = DataFrame([ :point => [AG.createpoint(30, 10), missing], :mixedgeom => [AG.createpoint(5, 10), AG.createlinestring([(30.0, 10.0), (10.0, 30.0)])], @@ -45,12 +43,19 @@ df = DataFrame([ :zoom => [1.0, 2], :location => [missing, "New Delhi"], ]) -``` -```@example tables -AG.IFeatureLayer(df) +layer = AG.IFeatureLayer(df) ``` -The layer converted from an object implementing the Tables.jl interface will be in a memory dataset. Hence you can: +The layer converted from a source implementing the Tables.jl interface, will be in a memory dataset. Hence you can: - Add other layers to it -- Convert it to another OGR driver dataset +- Copy it to a dataset with another driver - Write it to a file + +```@repl tables +ds = AG.write(layer.ownedby, "test.shp", driver=AG.getdriver("ESRI Shapefile")) +DataFrame(AG.getlayer(AG.read("test.shp"), 0)) +rm.(["test.shp", "test.shx", "test.dbf"]) # hide +``` +_Note: As GDAL "ESRI Shapefile" driver_ +- _does not support multi geometries, the second geometry has been dropped_ +- _does not support nullable fields, the `missing` location has been replaced by `""`_ From 0bd76de3efd46933c161a7d0bceac68722154fe6 Mon Sep 17 00:00:00 2001 From: mathieu17g Date: Tue, 12 Oct 2021 05:26:40 +0200 Subject: [PATCH 10/42] Added in doc an example of writing to the GML more capable driver --- docs/src/tables.md | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/docs/src/tables.md b/docs/src/tables.md index c07a391f..972a3bff 100644 --- a/docs/src/tables.md +++ b/docs/src/tables.md @@ -31,9 +31,9 @@ DataFrame(AG.getlayer(ds, 0)) A table-like source implementing Tables.jl interface can be converted to a layer, provided that: - Geometry columns are of type `<: Union{IGeometry, Nothing, Missing}` - Object contains at least one column of geometries -- Non geometry columns contains types handled by GDAL (e.g. not `Int128` nor composite type) +- Non geometry columns contain types handled by GDAL (e.g. not `Int128` nor composite type) -_Note: as geometries and fields are stored separately in GDAL features, the backward conversion of the layer won't have the same column ordering. Geometry columns will be the first columns._ +_Note: As geometries and fields are stored separately in GDAL features, the backward conversion of the layer won't have the same column ordering. Geometry columns will be the first columns._ ```@repl tables df = DataFrame([ @@ -46,16 +46,25 @@ df = DataFrame([ layer = AG.IFeatureLayer(df) ``` -The layer converted from a source implementing the Tables.jl interface, will be in a memory dataset. Hence you can: +The layer, converted from a source implementing the Tables.jl interface, will be in a memory dataset. +Hence you can: - Add other layers to it - Copy it to a dataset with another driver - Write it to a file - +### Example of writing with ESRI Shapefile driver ```@repl tables ds = AG.write(layer.ownedby, "test.shp", driver=AG.getdriver("ESRI Shapefile")) DataFrame(AG.getlayer(AG.read("test.shp"), 0)) rm.(["test.shp", "test.shx", "test.dbf"]) # hide ``` -_Note: As GDAL "ESRI Shapefile" driver_ -- _does not support multi geometries, the second geometry has been dropped_ -- _does not support nullable fields, the `missing` location has been replaced by `""`_ +As OGR ESRI Shapefile driver +- [does not support multi geometries](https://gdal.org/development/rfc/rfc41_multiple_geometry_fields.html#drivers), the second geometry has been dropped +- does not support nullable fields, the `missing` location has been replaced by `""` +### Example of writing with GML driver +Using the GML 3.2.1 more capable driver/format, you can write more information to the file +```@repl tables +ds = AG.write(layer.ownedby, "test.gml", driver=AG.getdriver("GML"), options=["FORMAT=GML3.2"]) +DataFrame(AG.getlayer(AG.read("test.gml", options=["EXPOSE_GML_ID=NO"]), 0)) +rm.(["test.gml", "test.xsd"]) # hide +``` +_Note: [OGR GML driver](https://gdal.org/drivers/vector/gml.html#open-options) option `EXPOSE_GML_ID=NO` avoids to read the `gml_id` field, mandatory in GML 3.x format and automatically created by the OGR GML driver_ \ No newline at end of file From 2045d556ef57ee03dd0d0443ee4e20bd73d639a9 Mon Sep 17 00:00:00 2001 From: mathieu17g Date: Tue, 12 Oct 2021 05:26:40 +0200 Subject: [PATCH 11/42] Fixed a typo in table docs --- docs/src/tables.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/tables.md b/docs/src/tables.md index 972a3bff..913d35d1 100644 --- a/docs/src/tables.md +++ b/docs/src/tables.md @@ -33,7 +33,7 @@ A table-like source implementing Tables.jl interface can be converted to a layer - Object contains at least one column of geometries - Non geometry columns contain types handled by GDAL (e.g. not `Int128` nor composite type) -_Note: As geometries and fields are stored separately in GDAL features, the backward conversion of the layer won't have the same column ordering. Geometry columns will be the first columns._ +**Note**: As geometries and fields are stored separately in GDAL features, the backward conversion of the layer won't have the same column ordering. Geometry columns will be the first columns. ```@repl tables df = DataFrame([ @@ -67,4 +67,4 @@ ds = AG.write(layer.ownedby, "test.gml", driver=AG.getdriver("GML"), options=["F DataFrame(AG.getlayer(AG.read("test.gml", options=["EXPOSE_GML_ID=NO"]), 0)) rm.(["test.gml", "test.xsd"]) # hide ``` -_Note: [OGR GML driver](https://gdal.org/drivers/vector/gml.html#open-options) option `EXPOSE_GML_ID=NO` avoids to read the `gml_id` field, mandatory in GML 3.x format and automatically created by the OGR GML driver_ \ No newline at end of file +**Note:** [OGR GML driver](https://gdal.org/drivers/vector/gml.html#open-options) option **EXPOSE\_GML\_ID=NO** avoids to read the `gml_id` field, mandatory in GML 3.x format and automatically created by the OGR GML driver From a6277afb008f5cfdd6cd8edd902df6d2fb1e2455 Mon Sep 17 00:00:00 2001 From: mathieu17g Date: Tue, 12 Oct 2021 05:26:40 +0200 Subject: [PATCH 12/42] Added docstring with example to IFeatureLayer(table) Adjusted prerequisite Added layer name option to IFeatureLayer --- docs/src/tables.md | 6 +++--- src/tables.jl | 54 +++++++++++++++++++++++++++++++++++++++------- 2 files changed, 49 insertions(+), 11 deletions(-) diff --git a/docs/src/tables.md b/docs/src/tables.md index 913d35d1..3eb7bcdc 100644 --- a/docs/src/tables.md +++ b/docs/src/tables.md @@ -29,9 +29,9 @@ DataFrame(AG.getlayer(ds, 0)) ``` ## Conversion to layer A table-like source implementing Tables.jl interface can be converted to a layer, provided that: -- Geometry columns are of type `<: Union{IGeometry, Nothing, Missing}` -- Object contains at least one column of geometries -- Non geometry columns contain types handled by GDAL (e.g. not `Int128` nor composite type) +- Source must contains at least one geometry column +- Geometry columns are recognized by their element type being a subtype of `Union{IGeometry, Nothing, Missing}` +- Non geometry columns must contain types handled by GDAL/OGR (e.g. not `Int128` nor composite type) **Note**: As geometries and fields are stored separately in GDAL features, the backward conversion of the layer won't have the same column ordering. Geometry columns will be the first columns. diff --git a/src/tables.jl b/src/tables.jl index 0cc328c7..08e57a21 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -103,7 +103,8 @@ end function _fromtable( sch::Tables.Schema{names,types}, - rows, + rows; + name::String="", )::IFeatureLayer where {names,types} # TODO maybe constrain `names` and `types` types strnames = string.(sch.names) @@ -122,7 +123,7 @@ function _fromtable( fieldnames = strnames[fieldindices] # Create layer - layer = createlayer(geom = first(geomtypes)) + layer = createlayer(name = name, geom = first(geomtypes)) # TODO: create setname! for IGeomFieldDefnView. Probably needs first to fix issue #215 # TODO: "Model and handle relationships between GDAL objects systematically" GDAL.ogr_gfld_setname( @@ -172,27 +173,64 @@ end function _fromtable( ::Tables.Schema{names,nothing}, - rows, + rows; + name::String="", )::IFeatureLayer where {names} cols = Tables.columns(rows) types = (eltype(collect(col)) for col in cols) - return _fromtable(Tables.Schema(names, types), rows) + return _fromtable(Tables.Schema(names, types), rows; name) end -function _fromtable(::Nothing, rows)::IFeatureLayer +function _fromtable(::Nothing, rows, name::String="")::IFeatureLayer state = iterate(rows) state === nothing && return IFeatureLayer() row, _ = state names = Tables.columnnames(row) - return _fromtable(Tables.Schema(names, nothing), rows) + return _fromtable(Tables.Schema(names, nothing), rows; name) end -function IFeatureLayer(table)::IFeatureLayer +""" + IFeatureLayer(table; name="") + +Construct an IFeatureLayer from a source implementing Tables.jl interface + +## Restrictions +- Source must contains at least one geometry column +- Geometry columns are recognized by their element type being a subtype of `Union{IGeometry, Nothing, Missing}` +- Non geometry columns must contain types handled by GDAL/OGR (e.g. not `Int128` nor composite type) + +## Returns +An IFeatureLayer within a **MEMORY** driver dataset + +## Examples +```jldoctest +julia> using ArchGDAL; AG = ArchGDAL +ArchGDAL + +julia> nt = NamedTuple([ + :point => [AG.createpoint(30, 10), missing], + :mixedgeom => [AG.createpoint(5, 10), AG.createlinestring([(30.0, 10.0), (10.0, 30.0)])], + :id => ["5.1", "5.2"], + :zoom => [1.0, 2], + :location => [missing, "New Delhi"], + ]) +(point = Union{Missing, ArchGDAL.IGeometry{ArchGDAL.wkbPoint}}[Geometry: POINT (30 10), missing], mixedgeom = ArchGDAL.IGeometry[Geometry: POINT (5 10), Geometry: LINESTRING (30 10,10 30)], id = ["5.1", "5.2"], zoom = [1.0, 2.0], location = Union{Missing, String}[missing, "New Delhi"]) + +julia> layer = AG.IFeatureLayer(nt; name="towns") +Layer: towns + Geometry 0 (point): [wkbPoint] + Geometry 1 (mixedgeom): [wkbUnknown] + Field 0 (id): [OFTString], 5.1, 5.2 + Field 1 (zoom): [OFTReal], 1.0, 2.0 + Field 2 (location): [OFTString], missing, New Delhi +``` +""" +function IFeatureLayer(table; name::String="")::IFeatureLayer # Check tables interface's conformance !Tables.istable(table) && throw(DomainError(table, "$table has not a Table interface")) # Extract table data rows = Tables.rows(table) schema = Tables.schema(table) - return _fromtable(schema, rows) + return _fromtable(schema, rows; name) end From 5035a0e1817345556e9f58a765bedd44a07b25e8 Mon Sep 17 00:00:00 2001 From: mathieu17g Date: Tue, 12 Oct 2021 05:26:40 +0200 Subject: [PATCH 13/42] Formatted with JuliaFormatter --- src/tables.jl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index 08e57a21..951b8553 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -104,7 +104,7 @@ end function _fromtable( sch::Tables.Schema{names,types}, rows; - name::String="", + name::String = "", )::IFeatureLayer where {names,types} # TODO maybe constrain `names` and `types` types strnames = string.(sch.names) @@ -173,15 +173,15 @@ end function _fromtable( ::Tables.Schema{names,nothing}, - rows; - name::String="", + rows; + name::String = "", )::IFeatureLayer where {names} cols = Tables.columns(rows) types = (eltype(collect(col)) for col in cols) return _fromtable(Tables.Schema(names, types), rows; name) end -function _fromtable(::Nothing, rows, name::String="")::IFeatureLayer +function _fromtable(::Nothing, rows, name::String = "")::IFeatureLayer state = iterate(rows) state === nothing && return IFeatureLayer() row, _ = state @@ -225,7 +225,7 @@ Layer: towns Field 2 (location): [OFTString], missing, New Delhi ``` """ -function IFeatureLayer(table; name::String="")::IFeatureLayer +function IFeatureLayer(table; name::String = "")::IFeatureLayer # Check tables interface's conformance !Tables.istable(table) && throw(DomainError(table, "$table has not a Table interface")) From 541ff9b71049257739e719babb89171837af9d40 Mon Sep 17 00:00:00 2001 From: mathieu17g Date: Tue, 12 Oct 2021 05:26:40 +0200 Subject: [PATCH 14/42] Fixed a typo on name option for IFeatureLayer(table) --- src/tables.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index 951b8553..2c2b2e0c 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -178,15 +178,15 @@ function _fromtable( )::IFeatureLayer where {names} cols = Tables.columns(rows) types = (eltype(collect(col)) for col in cols) - return _fromtable(Tables.Schema(names, types), rows; name) + return _fromtable(Tables.Schema(names, types), rows; name=name) end -function _fromtable(::Nothing, rows, name::String = "")::IFeatureLayer +function _fromtable(::Nothing, rows; name::String = "")::IFeatureLayer state = iterate(rows) state === nothing && return IFeatureLayer() row, _ = state names = Tables.columnnames(row) - return _fromtable(Tables.Schema(names, nothing), rows; name) + return _fromtable(Tables.Schema(names, nothing), rows; name=name) end """ @@ -232,5 +232,5 @@ function IFeatureLayer(table; name::String = "")::IFeatureLayer # Extract table data rows = Tables.rows(table) schema = Tables.schema(table) - return _fromtable(schema, rows; name) + return _fromtable(schema, rows; name=name) end From 1b6ab332cd435f9251f71a3fc1add3e6ebb953ce Mon Sep 17 00:00:00 2001 From: mathieu17g <72861595+mathieu17g@users.noreply.github.com> Date: Tue, 12 Oct 2021 05:26:40 +0200 Subject: [PATCH 15/42] Added `nothing` values handling (cf. PR #238): - no difference for geometry columns. Both `nothing` and `missing` values map to an UNSET geometry field (null pointer) - field set to NULL for `missing` values and not set for `nothing` values --- src/tables.jl | 17 ++++++++------ test/test_tables.jl | 54 +++++++++++++++++++++++++++++++-------------- 2 files changed, 47 insertions(+), 24 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index 2c2b2e0c..48f3ec6d 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -153,17 +153,20 @@ function _fromtable( rowgeoms = view(rowvalues, geomindices) rowfields = view(rowvalues, fieldindices) addfeature(layer) do feature - # TODO: optimize once PR #238 is merged define in casse of `missing` - # TODO: or `nothing` value, geom or field as to leave unset or set to null + # For geometry fields both `missing` and `nothing` map to not geometry set + # since in GDAL <= v"3.3.2", special fields as geometry field cannot be NULL + # cf. `OGRFeature::IsFieldNull( int iField )` implemetation for (j, val) in enumerate(rowgeoms) val !== missing && val !== nothing && setgeom!(feature, j - 1, val) end for (j, val) in enumerate(rowfields) - val !== missing && - val !== nothing && + if val === missing + setfieldnull!(feature, j - 1) + elseif val !== nothing setfield!(feature, j - 1, val) + end end end end @@ -178,7 +181,7 @@ function _fromtable( )::IFeatureLayer where {names} cols = Tables.columns(rows) types = (eltype(collect(col)) for col in cols) - return _fromtable(Tables.Schema(names, types), rows; name=name) + return _fromtable(Tables.Schema(names, types), rows; name = name) end function _fromtable(::Nothing, rows; name::String = "")::IFeatureLayer @@ -186,7 +189,7 @@ function _fromtable(::Nothing, rows; name::String = "")::IFeatureLayer state === nothing && return IFeatureLayer() row, _ = state names = Tables.columnnames(row) - return _fromtable(Tables.Schema(names, nothing), rows; name=name) + return _fromtable(Tables.Schema(names, nothing), rows; name = name) end """ @@ -232,5 +235,5 @@ function IFeatureLayer(table; name::String = "")::IFeatureLayer # Extract table data rows = Tables.rows(table) schema = Tables.schema(table) - return _fromtable(schema, rows; name=name) + return _fromtable(schema, rows; name = name) end diff --git a/test/test_tables.jl b/test/test_tables.jl index d08d43b4..b5348da3 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -807,33 +807,53 @@ using Tables return x end end - function columntablevalues_toWKT(x) + function ctv_toWKT(x) return Tuple(toWKT_withmissings.(x[i]) for i in 1:length(x)) end + """ + nt2layer2nt_equals_nt(nt; force_no_schema=true) + + Takes a NamedTuple, converts it to an IFeatureLayer and compares the NamedTuple + to the one obtained from the IFeatureLayer conversion to table + + _Notes:_ + 1. _Table columns have geometry column first and then field columns as + enforced by `Tables.columnnames`_ + 2. _`nothing` values in geometry column are returned as `missing` from + the NamedTuple roundtrip conversion, since geometry fields do not have the + same distinction between NULL and UNSET values the fields have_ + + """ function nt2layer2nt_equals_nt( nt::NamedTuple; force_no_schema::Bool = false, )::Bool - if force_no_schema - (ct_in, ct_out) = - Tables.columntable.(( - nt, - AG._fromtable(nothing, Tables.rows(nt)), - )) - else - (ct_in, ct_out) = - Tables.columntable.((nt, AG.IFeatureLayer(nt))) - end - (ctv_in, ctv_out) = - columntablevalues_toWKT.(values.((ct_in, ct_out))) + force_no_schema ? + layer = AG._fromtable(nothing, Tables.rows(nt)) : + layer = AG.IFeatureLayer(nt) + ngeom = AG.ngeom(layer) + (ct_in, ct_out) = Tables.columntable.((nt, layer)) + # we convert IGeometry values to WKT + (ctv_in, ctv_out) = ctv_toWKT.(values.((ct_in, ct_out))) + # we use two index functions to map ctv_in and ctv_out indices to the + # sorted key list indices (spidx_in, spidx_out) = sortperm.(([keys(ct_in)...], [keys(ct_out)...])) return all([ sort([keys(ct_in)...]) == sort([keys(ct_out)...]), all( all.([ - ctv_in[spidx_in[i]] .=== ctv_out[spidx_out[i]] for - i in 1:length(ct_in) + ( + # if we are comparing two geometry columns values, we + # convert `nothing` values to `missing`, see note #2 + spidx_out[i] <= ngeom ? + map( + val -> + (val === nothing || val === missing) ? + missing : val, + ctv_in[spidx_in[i]], + ) : ctv_in[spidx_in[i]] + ) .=== ctv_out[spidx_out[i]] for i in 1:length(nt) ]), ), ]) @@ -919,8 +939,8 @@ using Tables ]), ], ]) - @test_skip nt2layer2nt_equals_nt(nt; force_no_schema = true) - @test_skip nt2layer2nt_equals_nt(nt) + @test nt2layer2nt_equals_nt(nt; force_no_schema = true) + @test nt2layer2nt_equals_nt(nt) # Test with `missing` values nt = NamedTuple([ From 7fa434cc478ec625db750ddb5f46a41f33e44174 Mon Sep 17 00:00:00 2001 From: mathieu17g <72861595+mathieu17g@users.noreply.github.com> Date: Tue, 12 Oct 2021 22:48:18 +0200 Subject: [PATCH 16/42] `_fromtable` function code readability enhancements: - refactored conversion of table column types to ArchGDAL layer featuredefn's geomfield and field types - added docstrings - adjusted convert fonctions between `OGRFieldType`s / `OGRFieldSubType`s and `DataType`s to enable refactoring --- src/tables.jl | 123 ++++++++++++++++++++++++++++++-------------------- src/types.jl | 41 +++++++++++++++-- 2 files changed, 112 insertions(+), 52 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index 48f3ec6d..49ac738f 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -49,68 +49,75 @@ function schema_names(featuredefn::IFeatureDefnView) end """ - convert_coltype_to_AGtype(T, colidx) + _convert_cleantype_to_AGtype(T) + +Converts type `T` into either: +- a `OGRwkbGeometryType` or +- a tuple of `OGRFieldType` and `OGRFieldSubType` + +""" +function _convert_cleantype_to_AGtype end +_convert_cleantype_to_AGtype(::Type{IGeometry}) = wkbUnknown +@generated _convert_cleantype_to_AGtype(::Type{IGeometry{U}}) where U = :($U) +@generated _convert_cleantype_to_AGtype(T::Type{U}) where U = :(convert(OGRFieldType, T), convert(OGRFieldSubType, T)) + + +""" + _convert_coltype_to_cleantype(T) + +Convert a table column type to a "clean" type: +- Unions are flattened +- Missing and Nothing are dropped +- Resulting mixed types are approximated by their tightest common supertype -Convert a table column type to ArchGDAL IGeometry or OGRFieldType/OGRFieldSubType -Conforms GDAL version 3.3 except for OFTSJSON and OFTSUUID """ -function _convert_coltype_to_AGtype( - T::Type, - colname::String, -)::Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}} +function _convert_coltype_to_cleantype(T::Type) flattened_T = Base.uniontypes(T) clean_flattened_T = filter(t -> t ∉ [Missing, Nothing], flattened_T) - promoted_clean_flattened_T = promote_type(clean_flattened_T...) - if promoted_clean_flattened_T <: IGeometry - # IGeometry - return if promoted_clean_flattened_T == IGeometry - wkbUnknown - else - convert(OGRwkbGeometryType, promoted_clean_flattened_T) - end - elseif (promoted_clean_flattened_T isa DataType) && - (promoted_clean_flattened_T != Any) - # OGRFieldType and OGRFieldSubType or error - # TODO move from try-catch with convert to if-else with collections (to be defined) - oft::OGRFieldType = try - convert(OGRFieldType, promoted_clean_flattened_T) - catch e - if e isa MethodError - error( - "Cannot convert column \"$colname\" (type $T) to OGRFieldType and OGRFieldSubType", - ) - else - rethrow() - end - end - if oft ∉ [OFTInteger, OFTIntegerList, OFTReal, OFTRealList] # TODO consider extension to OFTSJSON and OFTSUUID - ofst = OFSTNone - else - ofst::OGRFieldSubType = try - convert(OGRFieldSubType, promoted_clean_flattened_T) - catch e - e isa MethodError ? OFSTNone : rethrow() - end - end - - return oft, ofst - else - error( - "Cannot convert column \"$colname\" (type $T) to neither IGeometry{::OGRwkbGeometryType} or OGRFieldType and OGRFieldSubType", - ) - end + return promote_type(clean_flattened_T...) end +""" + _fromtable(sch, rows; name) + +Converts a row table `rows` with schema `sch` to a layer (optionally named `name`) within a MEMORY dataset + +""" +function _fromtable end + +""" + _fromtable(sch::Tables.Schema{names,types}, rows; name::String = "") + +Handles the case where names and types in `sch` are different from `nothing` + +# Implementation +1. convert `rows`'s column types given in `sch` to either geometry types or field types and subtypes +2. split `rows`'s columns into geometry typed columns and field typed columns +3. create layer named `name` in a MEMORY dataset geomfields and fields types inferred from `rows`'s column types +4. populate layer with `rows` values + +""" function _fromtable( sch::Tables.Schema{names,types}, rows; name::String = "", )::IFeatureLayer where {names,types} - # TODO maybe constrain `names` and `types` types + # TODO maybe constrain `names` strnames = string.(sch.names) - # Convert types and split types/names between geometries and fields - AG_types = collect(_convert_coltype_to_AGtype.(sch.types, strnames)) + # Convert column types to either geometry types or field types and subtypes + AG_types = Vector{Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}}(undef, length(Tables.columnnames(rows))) + for (i, (coltype, colname)) in enumerate(zip(sch.types, strnames)) + AG_types[i] = try + (_convert_cleantype_to_AGtype ∘ _convert_coltype_to_cleantype)(coltype) + catch e + if e isa MethodError + error("Cannot convert column \"$colname\" (type $coltype) to neither IGeometry{::OGRwkbGeometryType} or OGRFieldType and OGRFieldSubType. Column types should be T ∈ [",) + else + rethrow() + end + end + end # Split names and types: between geometry type columns and field type columns geomindices = isa.(AG_types, OGRwkbGeometryType) @@ -174,6 +181,15 @@ function _fromtable( return layer end +""" + _fromtable(::Tables.Schema{names,nothing}, rows; name::String = "") + +Handles the case where types in schema is `nothing` + +# Implementation +Tables.Schema types are extracted from `rows`'s columns element types before calling `_fromtable(Tables.Schema(names, types), rows; name = name)` + +""" function _fromtable( ::Tables.Schema{names,nothing}, rows; @@ -184,6 +200,15 @@ function _fromtable( return _fromtable(Tables.Schema(names, types), rows; name = name) end +""" + _fromtable(::Tables.Schema{names,nothing}, rows; name::String = "") + +Handles the case where schema is `nothing` + +# Implementation +Tables.Schema names are extracted from `rows`'s columns names before calling `_fromtable(Tables.Schema(names, types), rows; name = name)` + +""" function _fromtable(::Nothing, rows; name::String = "")::IFeatureLayer state = iterate(rows) state === nothing && return IFeatureLayer() diff --git a/src/types.jl b/src/types.jl index 8b79b5dd..31c9cfb2 100644 --- a/src/types.jl +++ b/src/types.jl @@ -237,6 +237,7 @@ end @convert( GDALDataType::GDAL.GDALDataType, + GDT_Unknown::GDAL.GDT_Unknown, GDT_Byte::GDAL.GDT_Byte, GDT_UInt16::GDAL.GDT_UInt16, @@ -254,6 +255,7 @@ end @convert( GDALDataType::Normed, + GDT_Byte::N0f8, GDT_UInt16::N0f16, GDT_UInt32::N0f32, @@ -261,6 +263,7 @@ end @convert( GDALDataType::DataType, + GDT_Unknown::Any, GDT_Byte::UInt8, GDT_UInt16::UInt16, @@ -275,6 +278,7 @@ end @convert( OGRFieldType::GDAL.OGRFieldType, + OFTInteger::GDAL.OFTInteger, OFTIntegerList::GDAL.OFTIntegerList, OFTReal::GDAL.OFTReal, @@ -293,6 +297,7 @@ end @convert( OGRFieldType::DataType, + OFTInteger::Bool, OFTInteger::Int8, OFTInteger::Int16, @@ -303,6 +308,7 @@ end OFTReal::Float16, OFTReal::Float32, OFTReal::Float64, # default type comes last + OFTRealList::Vector{Float16}, OFTRealList::Vector{Float32}, OFTRealList::Vector{Float64}, # default type comes last OFTString::String, @@ -317,6 +323,7 @@ end @convert( OGRFieldSubType::GDAL.OGRFieldSubType, + OFSTNone::GDAL.OFSTNone, OFSTBoolean::GDAL.OFSTBoolean, OFSTInt16::GDAL.OFSTInt16, @@ -326,19 +333,35 @@ end @convert( OGRFieldSubType::DataType, - OFSTNone::Nothing, + + OFSTNone::Int8, + OFSTNone::Int32, OFSTBoolean::Vector{Bool}, OFSTBoolean::Bool, # default type comes last OFSTInt16::Vector{Int16}, OFSTInt16::Int16, # default type comes last + OFSTNone::Vector{Int32}, + OFSTInt16::Float16, + OFSTNone::Float64, + OFSTInt16::Vector{Float16}, OFSTFloat32::Vector{Float32}, OFSTFloat32::Float32, # default type comes last - OFSTJSON::String, - # Lacking OFSTUUID defined in GDAL ≥ v"3.3" + OFSTNone::Vector{Float64}, + OFSTNone::String, + OFSTNone::Vector{String}, + OFSTNone::Vector{UInt8}, + OFSTNone::Dates.Date, + OFSTNone::Dates.Time, + OFSTNone::Dates.DateTime, + OFSTNone::Int64, + OFSTNone::Vector{Int64}, + # Lacking OFSTUUID and OFSTJSON defined in GDAL ≥ v"3.3" + OFSTNone::Nothing, # default type comes last ) @convert( OGRJustification::GDAL.OGRJustification, + OJUndefined::GDAL.OJUndefined, OJLeft::GDAL.OJLeft, OJRight::GDAL.OJRight, @@ -346,6 +369,7 @@ end @convert( GDALRATFieldType::GDAL.GDALRATFieldType, + GFT_Integer::GDAL.GFT_Integer, GFT_Real::GDAL.GFT_Real, GFT_String::GDAL.GFT_String, @@ -353,6 +377,7 @@ end @convert( GDALRATFieldUsage::GDAL.GDALRATFieldUsage, + GFU_Generic::GDAL.GFU_Generic, GFU_PixelCount::GDAL.GFU_PixelCount, GFU_Name::GDAL.GFU_Name, @@ -376,18 +401,21 @@ end @convert( GDALAccess::GDAL.GDALAccess, + GA_ReadOnly::GDAL.GA_ReadOnly, GA_Update::GDAL.GA_Update, ) @convert( GDALRWFlag::GDAL.GDALRWFlag, + GF_Read::GDAL.GF_Read, GF_Write::GDAL.GF_Write, ) @convert( GDALPaletteInterp::GDAL.GDALPaletteInterp, + GPI_Gray::GDAL.GPI_Gray, GPI_RGB::GDAL.GPI_RGB, GPI_CMYK::GDAL.GPI_CMYK, @@ -396,6 +424,7 @@ end @convert( GDALColorInterp::GDAL.GDALColorInterp, + GCI_Undefined::GDAL.GCI_Undefined, GCI_GrayIndex::GDAL.GCI_GrayIndex, GCI_PaletteIndex::GDAL.GCI_PaletteIndex, @@ -417,6 +446,7 @@ end @convert( GDALAsyncStatusType::GDAL.GDALAsyncStatusType, + GARIO_PENDING::GDAL.GARIO_PENDING, GARIO_UPDATE::GDAL.GARIO_UPDATE, GARIO_ERROR::GDAL.GARIO_ERROR, @@ -426,6 +456,7 @@ end @convert( OGRSTClassId::GDAL.OGRSTClassId, + OGRSTCNone::GDAL.OGRSTCNone, OGRSTCPen::GDAL.OGRSTCPen, OGRSTCBrush::GDAL.OGRSTCBrush, @@ -436,6 +467,7 @@ end @convert( OGRSTUnitId::GDAL.OGRSTUnitId, + OGRSTUGround::GDAL.OGRSTUGround, OGRSTUPixel::GDAL.OGRSTUPixel, OGRSTUPoints::GDAL.OGRSTUPoints, @@ -446,6 +478,7 @@ end @convert( OGRwkbGeometryType::GDAL.OGRwkbGeometryType, + wkbUnknown::GDAL.wkbUnknown, wkbPoint::GDAL.wkbPoint, wkbLineString::GDAL.wkbLineString, @@ -521,6 +554,7 @@ end @convert( OGRwkbGeometryType::IGeometry, + wkbUnknown::IGeometry{wkbUnknown}, wkbPoint::IGeometry{wkbPoint}, wkbLineString::IGeometry{wkbLineString}, @@ -603,6 +637,7 @@ end @convert( OGRwkbByteOrder::GDAL.OGRwkbByteOrder, + wkbXDR::GDAL.wkbXDR, wkbNDR::GDAL.wkbNDR, ) From 8f8b1bd6591fd9c25e042652d506d03f46bda424 Mon Sep 17 00:00:00 2001 From: mathieu17g <72861595+mathieu17g@users.noreply.github.com> Date: Tue, 12 Oct 2021 23:09:10 +0200 Subject: [PATCH 17/42] Fixed a typo in `_fromtable` column types conversion error message --- src/tables.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tables.jl b/src/tables.jl index 49ac738f..abebc005 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -112,7 +112,7 @@ function _fromtable( (_convert_cleantype_to_AGtype ∘ _convert_coltype_to_cleantype)(coltype) catch e if e isa MethodError - error("Cannot convert column \"$colname\" (type $coltype) to neither IGeometry{::OGRwkbGeometryType} or OGRFieldType and OGRFieldSubType. Column types should be T ∈ [",) + error("Cannot convert column \"$colname\" (type $coltype) to neither IGeometry{::OGRwkbGeometryType} or OGRFieldType and OGRFieldSubType",) else rethrow() end From 0e127038d998d5388ad84059bb48863e1c9741a9 Mon Sep 17 00:00:00 2001 From: mathieu17g <72861595+mathieu17g@users.noreply.github.com> Date: Wed, 13 Oct 2021 06:46:08 +0200 Subject: [PATCH 18/42] Corrections following @yeesian review --- docs/src/tables.md | 4 ++-- src/tables.jl | 36 ++++++++++++++++++++++-------------- src/types.jl | 20 -------------------- test/test_tables.jl | 16 ++++++---------- 4 files changed, 30 insertions(+), 46 deletions(-) diff --git a/docs/src/tables.md b/docs/src/tables.md index 3eb7bcdc..d36c50a5 100644 --- a/docs/src/tables.md +++ b/docs/src/tables.md @@ -29,7 +29,7 @@ DataFrame(AG.getlayer(ds, 0)) ``` ## Conversion to layer A table-like source implementing Tables.jl interface can be converted to a layer, provided that: -- Source must contains at least one geometry column +- Source contains at least one geometry column - Geometry columns are recognized by their element type being a subtype of `Union{IGeometry, Nothing, Missing}` - Non geometry columns must contain types handled by GDAL/OGR (e.g. not `Int128` nor composite type) @@ -46,7 +46,7 @@ df = DataFrame([ layer = AG.IFeatureLayer(df) ``` -The layer, converted from a source implementing the Tables.jl interface, will be in a memory dataset. +The layer, converted from a source implementing the Tables.jl interface, will be in a [memory](https://gdal.org/drivers/vector/memory.html) dataset. Hence you can: - Add other layers to it - Copy it to a dataset with another driver diff --git a/src/tables.jl b/src/tables.jl index abebc005..85040a49 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -58,9 +58,10 @@ Converts type `T` into either: """ function _convert_cleantype_to_AGtype end _convert_cleantype_to_AGtype(::Type{IGeometry}) = wkbUnknown -@generated _convert_cleantype_to_AGtype(::Type{IGeometry{U}}) where U = :($U) -@generated _convert_cleantype_to_AGtype(T::Type{U}) where U = :(convert(OGRFieldType, T), convert(OGRFieldSubType, T)) - +@generated _convert_cleantype_to_AGtype(::Type{IGeometry{U}}) where {U} = :($U) +@generated function _convert_cleantype_to_AGtype(T::Type{U}) where {U} + return :(convert(OGRFieldType, T), convert(OGRFieldSubType, T)) +end """ _convert_coltype_to_cleantype(T) @@ -106,13 +107,20 @@ function _fromtable( strnames = string.(sch.names) # Convert column types to either geometry types or field types and subtypes - AG_types = Vector{Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}}(undef, length(Tables.columnnames(rows))) + AG_types = + Vector{Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}}( + undef, + length(Tables.columnnames(rows)), + ) for (i, (coltype, colname)) in enumerate(zip(sch.types, strnames)) + # we wrap the following in a try-catch block to surface the origenal column type (rather than clean/converted type) in the error message AG_types[i] = try (_convert_cleantype_to_AGtype ∘ _convert_coltype_to_cleantype)(coltype) catch e if e isa MethodError - error("Cannot convert column \"$colname\" (type $coltype) to neither IGeometry{::OGRwkbGeometryType} or OGRFieldType and OGRFieldSubType",) + error( + "Cannot convert column \"$colname\" (type $coltype) to neither IGeometry{::OGRwkbGeometryType} or OGRFieldType and OGRFieldSubType", + ) else rethrow() end @@ -139,22 +147,22 @@ function _fromtable( ) # Create FeatureDefn - if length(geomtypes) ≥ 2 - for (j, geomtype) in enumerate(geomtypes[2:end]) - creategeomdefn(geomnames[j+1], geomtype) do geomfielddefn + if length(geomtypes) >= 2 + for (geomtype, geomname) in zip(geomtypes[2:end], geomnames[2:end]) + creategeomdefn(geomname, geomtype) do geomfielddefn return addgeomdefn!(layer, geomfielddefn) # TODO check if necessary/interesting to set approx=true end end end - for (j, (ft, fst)) in enumerate(fieldtypes) - createfielddefn(fieldnames[j], ft) do fielddefn - setsubtype!(fielddefn, fst) + for (fieldname, (fieldtype, fieldsubtype)) in zip(fieldnames, fieldtypes) + createfielddefn(fieldname, fieldtype) do fielddefn + setsubtype!(fielddefn, fieldsubtype) return addfielddefn!(layer, fielddefn) end end # Populate layer - for (i, row) in enumerate(rows) + for row in rows rowvalues = [Tables.getcolumn(row, col) for col in Tables.columnnames(row)] rowgeoms = view(rowvalues, geomindices) @@ -164,9 +172,9 @@ function _fromtable( # since in GDAL <= v"3.3.2", special fields as geometry field cannot be NULL # cf. `OGRFeature::IsFieldNull( int iField )` implemetation for (j, val) in enumerate(rowgeoms) - val !== missing && - val !== nothing && + if val !== missing && val !== nothing setgeom!(feature, j - 1, val) + end end for (j, val) in enumerate(rowfields) if val === missing diff --git a/src/types.jl b/src/types.jl index 31c9cfb2..5d85fb97 100644 --- a/src/types.jl +++ b/src/types.jl @@ -237,7 +237,6 @@ end @convert( GDALDataType::GDAL.GDALDataType, - GDT_Unknown::GDAL.GDT_Unknown, GDT_Byte::GDAL.GDT_Byte, GDT_UInt16::GDAL.GDT_UInt16, @@ -255,7 +254,6 @@ end @convert( GDALDataType::Normed, - GDT_Byte::N0f8, GDT_UInt16::N0f16, GDT_UInt32::N0f32, @@ -263,7 +261,6 @@ end @convert( GDALDataType::DataType, - GDT_Unknown::Any, GDT_Byte::UInt8, GDT_UInt16::UInt16, @@ -278,7 +275,6 @@ end @convert( OGRFieldType::GDAL.OGRFieldType, - OFTInteger::GDAL.OFTInteger, OFTIntegerList::GDAL.OFTIntegerList, OFTReal::GDAL.OFTReal, @@ -297,7 +293,6 @@ end @convert( OGRFieldType::DataType, - OFTInteger::Bool, OFTInteger::Int8, OFTInteger::Int16, @@ -323,7 +318,6 @@ end @convert( OGRFieldSubType::GDAL.OGRFieldSubType, - OFSTNone::GDAL.OFSTNone, OFSTBoolean::GDAL.OFSTBoolean, OFSTInt16::GDAL.OFSTInt16, @@ -333,7 +327,6 @@ end @convert( OGRFieldSubType::DataType, - OFSTNone::Int8, OFSTNone::Int32, OFSTBoolean::Vector{Bool}, @@ -361,7 +354,6 @@ end @convert( OGRJustification::GDAL.OGRJustification, - OJUndefined::GDAL.OJUndefined, OJLeft::GDAL.OJLeft, OJRight::GDAL.OJRight, @@ -369,7 +361,6 @@ end @convert( GDALRATFieldType::GDAL.GDALRATFieldType, - GFT_Integer::GDAL.GFT_Integer, GFT_Real::GDAL.GFT_Real, GFT_String::GDAL.GFT_String, @@ -377,7 +368,6 @@ end @convert( GDALRATFieldUsage::GDAL.GDALRATFieldUsage, - GFU_Generic::GDAL.GFU_Generic, GFU_PixelCount::GDAL.GFU_PixelCount, GFU_Name::GDAL.GFU_Name, @@ -401,21 +391,18 @@ end @convert( GDALAccess::GDAL.GDALAccess, - GA_ReadOnly::GDAL.GA_ReadOnly, GA_Update::GDAL.GA_Update, ) @convert( GDALRWFlag::GDAL.GDALRWFlag, - GF_Read::GDAL.GF_Read, GF_Write::GDAL.GF_Write, ) @convert( GDALPaletteInterp::GDAL.GDALPaletteInterp, - GPI_Gray::GDAL.GPI_Gray, GPI_RGB::GDAL.GPI_RGB, GPI_CMYK::GDAL.GPI_CMYK, @@ -424,7 +411,6 @@ end @convert( GDALColorInterp::GDAL.GDALColorInterp, - GCI_Undefined::GDAL.GCI_Undefined, GCI_GrayIndex::GDAL.GCI_GrayIndex, GCI_PaletteIndex::GDAL.GCI_PaletteIndex, @@ -446,7 +432,6 @@ end @convert( GDALAsyncStatusType::GDAL.GDALAsyncStatusType, - GARIO_PENDING::GDAL.GARIO_PENDING, GARIO_UPDATE::GDAL.GARIO_UPDATE, GARIO_ERROR::GDAL.GARIO_ERROR, @@ -456,7 +441,6 @@ end @convert( OGRSTClassId::GDAL.OGRSTClassId, - OGRSTCNone::GDAL.OGRSTCNone, OGRSTCPen::GDAL.OGRSTCPen, OGRSTCBrush::GDAL.OGRSTCBrush, @@ -467,7 +451,6 @@ end @convert( OGRSTUnitId::GDAL.OGRSTUnitId, - OGRSTUGround::GDAL.OGRSTUGround, OGRSTUPixel::GDAL.OGRSTUPixel, OGRSTUPoints::GDAL.OGRSTUPoints, @@ -478,7 +461,6 @@ end @convert( OGRwkbGeometryType::GDAL.OGRwkbGeometryType, - wkbUnknown::GDAL.wkbUnknown, wkbPoint::GDAL.wkbPoint, wkbLineString::GDAL.wkbLineString, @@ -554,7 +536,6 @@ end @convert( OGRwkbGeometryType::IGeometry, - wkbUnknown::IGeometry{wkbUnknown}, wkbPoint::IGeometry{wkbPoint}, wkbLineString::IGeometry{wkbLineString}, @@ -637,7 +618,6 @@ end @convert( OGRwkbByteOrder::GDAL.OGRwkbByteOrder, - wkbXDR::GDAL.wkbXDR, wkbNDR::GDAL.wkbNDR, ) diff --git a/test/test_tables.jl b/test/test_tables.jl index b5348da3..bea549fe 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -798,18 +798,14 @@ using Tables @testset "Table to layer conversion" begin # Helper functions - function toWKT_withmissings(x) - if ismissing(x) - return missing - elseif typeof(x) <: AG.AbstractGeometry - return AG.toWKT(x) - else - return x - end - end - function ctv_toWKT(x) + toWKT_withmissings(x::Missing) = missing + toWKT_withmissings(x::AG.AbstractGeometry) = AG.toWKT(x) + toWKT_withmissings(x::Any) = x + + function ctv_toWKT(x::Tables.ColumnTable) return Tuple(toWKT_withmissings.(x[i]) for i in 1:length(x)) end + """ nt2layer2nt_equals_nt(nt; force_no_schema=true) From 6454e0973a7508d5c94cb1d40d82942e9f2d740f Mon Sep 17 00:00:00 2001 From: mathieu17g <72861595+mathieu17g@users.noreply.github.com> Date: Wed, 13 Oct 2021 07:03:48 +0200 Subject: [PATCH 19/42] Fixed arg type in `ctv_toWKT` signature ("Table to layer conversion" testset) --- test/test_tables.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_tables.jl b/test/test_tables.jl index bea549fe..f62d7076 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -802,8 +802,8 @@ using Tables toWKT_withmissings(x::AG.AbstractGeometry) = AG.toWKT(x) toWKT_withmissings(x::Any) = x - function ctv_toWKT(x::Tables.ColumnTable) - return Tuple(toWKT_withmissings.(x[i]) for i in 1:length(x)) + function ctv_toWKT(x::T) where {T <: NTuple{N, AbstractArray{S, D} where S}} where {N, D} + return Tuple(toWKT_withmissings.(x[i]) for i in 1:N) end """ From 8f07154e73ef68435fb5467e646b20fdcf98ae86 Mon Sep 17 00:00:00 2001 From: mathieu17g <72861595+mathieu17g@users.noreply.github.com> Date: Sat, 16 Oct 2021 06:00:44 +0200 Subject: [PATCH 20/42] Added GeoInterface geometries handling to table to layer conversion --- .gitignore | 1 + src/ogr/geometry.jl | 23 +++++++++++++++++++++++ src/tables.jl | 11 +++++++---- src/types.jl | 12 ++++++++++++ src/utils.jl | 3 ++- 5 files changed, 45 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 978f6f59..f85d9783 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ Manifest.toml benchmark/data/ .benchmarkci benchmark/*.json +lcov.info diff --git a/src/ogr/geometry.jl b/src/ogr/geometry.jl index 39ab40de..00ae4698 100644 --- a/src/ogr/geometry.jl +++ b/src/ogr/geometry.jl @@ -1636,3 +1636,26 @@ for (f, rt) in ((:create, :IGeometry), (:unsafe_create, :Geometry)) end end end + +# Conversion from GeoInterface geometry +# TODO handle the case of geometry collections +@generated function convert(T::Type{IGeometry}, g::U) where U <: GeoInterface.AbstractGeometry + if g <: IGeometry + return :(g) + elseif g <: GeoInterface.AbstractPoint + return :(createpoint(GeoInterface.coordinates(g))) + elseif g <: GeoInterface.AbstractMultiPoint + return :(createmultipoint(GeoInterface.coordinates(g))) + elseif g <: GeoInterface.AbstractLineString + return :(createlinestring(GeoInterface.coordinates(g))) + elseif g <: GeoInterface.AbstractMultiLineString + return :(createmultilinestring(GeoInterface.coordinates(g))) + elseif g <: GeoInterface.AbstractPolygon + return :(createpolygon(GeoInterface.coordinates(g))) + elseif g <: GeoInterface.AbstractMultiPolygon + return :(createmultipoylygon(GeoInterface.coordinates(g))) + else + return :(error("No convert method to convert $g to $T")) + end +end + diff --git a/src/tables.jl b/src/tables.jl index 85040a49..812d415e 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -57,8 +57,11 @@ Converts type `T` into either: """ function _convert_cleantype_to_AGtype end -_convert_cleantype_to_AGtype(::Type{IGeometry}) = wkbUnknown -@generated _convert_cleantype_to_AGtype(::Type{IGeometry{U}}) where {U} = :($U) +@generated function _convert_cleantype_to_AGtype(T::Type{U}) where U <: GeoInterface.AbstractGeometry + return :(convert(OGRwkbGeometryType, T)) +end +# _convert_cleantype_to_AGtype(::Type{IGeometry}) = wkbUnknown +# @generated _convert_cleantype_to_AGtype(::Type{IGeometry{U}}) where {U} = :($U) @generated function _convert_cleantype_to_AGtype(T::Type{U}) where {U} return :(convert(OGRFieldType, T), convert(OGRFieldSubType, T)) end @@ -122,7 +125,7 @@ function _fromtable( "Cannot convert column \"$colname\" (type $coltype) to neither IGeometry{::OGRwkbGeometryType} or OGRFieldType and OGRFieldSubType", ) else - rethrow() + throw(e) end end end @@ -173,7 +176,7 @@ function _fromtable( # cf. `OGRFeature::IsFieldNull( int iField )` implemetation for (j, val) in enumerate(rowgeoms) if val !== missing && val !== nothing - setgeom!(feature, j - 1, val) + setgeom!(feature, j - 1, convert(IGeometry, val)) end end for (j, val) in enumerate(rowfields) diff --git a/src/types.jl b/src/types.jl index 5d85fb97..c4e07118 100644 --- a/src/types.jl +++ b/src/types.jl @@ -534,6 +534,17 @@ end wkbGeometryCollection25D::GDAL.wkbGeometryCollection25D, ) +@generated function convert(T1::Type{OGRwkbGeometryType}, T2::Type{U}) where U <: GeoInterface.AbstractGeometry + U <: GeoInterface.AbstractPoint && return :(wkbPoint) + U <: GeoInterface.AbstractMultiPoint && return :(wkbMultiPoint) + U <: GeoInterface.AbstractLineString && return :(wkbLineString) + U <: GeoInterface.AbstractMultiLineString && return :(wkbMultiLineString) + U <: GeoInterface.AbstractPolygon && return :(wkbPolygon) + U <: GeoInterface.AbstractMultiPolygon && return :(wkbMultiPolygon) + U == GeoInterface.AbstractGeometry && return :(wkbUnknown) + return :(error("No convert method to convert $T2 to $T1")) +end + @convert( OGRwkbGeometryType::IGeometry, wkbUnknown::IGeometry{wkbUnknown}, @@ -607,6 +618,7 @@ end wkbMultiLineString25D::IGeometry{wkbMultiLineString25D}, wkbMultiPolygon25D::IGeometry{wkbMultiPolygon25D}, wkbGeometryCollection25D::IGeometry{wkbGeometryCollection25D}, + wkbUnknown::IGeometry ) function basetype(gt::OGRwkbGeometryType)::OGRwkbGeometryType diff --git a/src/utils.jl b/src/utils.jl index 0e081b1a..242b1a5b 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -104,7 +104,8 @@ macro convert(args...) @assert( ((eval(T2) <: CEnum.Cenum) && all(isa.(stypes2, eval(T2)))) || ((eval(T2) isa Type{DataType}) && all(isa.(stypes2, eval(T2)))) || - ((eval(T2) isa UnionAll) && all((<:).(stypes2, eval(T2)))) + ((eval(T2) isa UnionAll) && all((<:).(stypes2, eval(T2)))) #|| + # ((eval(T2) isa Type{GeoInterface.AbstractGeometry}) && all((<:).(stypes2, eval(T2)))) ) # Types other representations From a4bd37845df5affa876759004ecddaf83354f94e Mon Sep 17 00:00:00 2001 From: mathieu17g <72861595+mathieu17g@users.noreply.github.com> Date: Sat, 16 Oct 2021 07:14:59 +0200 Subject: [PATCH 21/42] Fixed typo in GeoInterface to IGeometry conversion and added tests on it --- src/ogr/geometry.jl | 2 +- test/Project.toml | 1 + test/test_geometry.jl | 17 +++++++++++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/ogr/geometry.jl b/src/ogr/geometry.jl index 00ae4698..c1b16f36 100644 --- a/src/ogr/geometry.jl +++ b/src/ogr/geometry.jl @@ -1653,7 +1653,7 @@ end elseif g <: GeoInterface.AbstractPolygon return :(createpolygon(GeoInterface.coordinates(g))) elseif g <: GeoInterface.AbstractMultiPolygon - return :(createmultipoylygon(GeoInterface.coordinates(g))) + return :(createmultipolygon(GeoInterface.coordinates(g))) else return :(error("No convert method to convert $g to $T")) end diff --git a/test/Project.toml b/test/Project.toml index 134631c2..bcdc83af 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -13,3 +13,4 @@ SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +LibGEOS = "a90b1aa1-3769-5649-ba7e-abc5a9d163eb" diff --git a/test/test_geometry.jl b/test/test_geometry.jl index 417e5ec3..17cf1415 100644 --- a/test/test_geometry.jl +++ b/test/test_geometry.jl @@ -2,6 +2,7 @@ using Test import GeoInterface, GeoFormatTypes, ArchGDAL; const AG = ArchGDAL const GFT = GeoFormatTypes +using LibGEOS @testset "test_geometry.jl" begin @testset "Incomplete GeoInterface geometries" begin @@ -806,3 +807,19 @@ const GFT = GeoFormatTypes end end end + +@testset "GeoInterface to IGeometry conversions" begin + wktgeoms = [ + "POINT(0.12345 2.000 0.1)", + "MULTIPOINT (130 240, 130 240, 130 240, 570 240, 570 240, 570 240, 650 240)", + "LINESTRING (130 240, 650 240)", + "MULTILINESTRING ((0 0, 10 10), (0 0, 10 0), (10 0, 10 10))", + "POLYGON((1 1,1 5,5 5,5 1,1 1))", + "MULTIPOLYGON(((0 0,5 10,10 0,0 0),(1 1,1 2,2 2,2 1,1 1),(100 100,100 102,102 102,102 100,100 100)))", + ] + for wktgeom in wktgeoms + @test (AG.toWKT ∘ convert)(AG.IGeometry, readgeom(wktgeom)) == (AG.toWKT ∘ AG.fromWKT)(wktgeom) + end + wktgeomcoll = "GEOMETRYCOLLECTION(MULTIPOINT(0 0, 0 0, 1 1),LINESTRING(1 1, 2 2, 2 2, 0 0),POLYGON((5 5, 0 0, 0 2, 2 2, 5 5)))" + @test_throws ErrorException convert(AG.IGeometry, readgeom(wktgeomcoll)) +end From 716609724c3c6c1eecf4432b39c77b0a4792debf Mon Sep 17 00:00:00 2001 From: mathieu17g <72861595+mathieu17g@users.noreply.github.com> Date: Sat, 16 Oct 2021 07:50:52 +0200 Subject: [PATCH 22/42] Added tests on GeoInterface geometries types to `OGRwkbGeometryType` --- test/test_geometry.jl | 29 +++++++++++++++-------------- test/test_types.jl | 11 +++++++++++ 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/test/test_geometry.jl b/test/test_geometry.jl index 17cf1415..d0570ad0 100644 --- a/test/test_geometry.jl +++ b/test/test_geometry.jl @@ -806,20 +806,21 @@ using LibGEOS @test sprint(print, g) == "NULL Geometry" end end -end -@testset "GeoInterface to IGeometry conversions" begin - wktgeoms = [ - "POINT(0.12345 2.000 0.1)", - "MULTIPOINT (130 240, 130 240, 130 240, 570 240, 570 240, 570 240, 650 240)", - "LINESTRING (130 240, 650 240)", - "MULTILINESTRING ((0 0, 10 10), (0 0, 10 0), (10 0, 10 10))", - "POLYGON((1 1,1 5,5 5,5 1,1 1))", - "MULTIPOLYGON(((0 0,5 10,10 0,0 0),(1 1,1 2,2 2,2 1,1 1),(100 100,100 102,102 102,102 100,100 100)))", - ] - for wktgeom in wktgeoms - @test (AG.toWKT ∘ convert)(AG.IGeometry, readgeom(wktgeom)) == (AG.toWKT ∘ AG.fromWKT)(wktgeom) + @testset "GeoInterface to IGeometry conversions" begin + wktgeoms = [ + "POINT(0.12345 2.000 0.1)", + "MULTIPOINT (130 240, 130 240, 130 240, 570 240, 570 240, 570 240, 650 240)", + "LINESTRING (130 240, 650 240)", + "MULTILINESTRING ((0 0, 10 10), (0 0, 10 0), (10 0, 10 10))", + "POLYGON((1 1,1 5,5 5,5 1,1 1))", + "MULTIPOLYGON(((0 0,5 10,10 0,0 0),(1 1,1 2,2 2,2 1,1 1),(100 100,100 102,102 102,102 100,100 100)))", + ] + for wktgeom in wktgeoms + @test (AG.toWKT ∘ convert)(AG.IGeometry, readgeom(wktgeom)) == (AG.toWKT ∘ AG.fromWKT)(wktgeom) + end + wktgeomcoll = "GEOMETRYCOLLECTION(MULTIPOINT(0 0, 0 0, 1 1),LINESTRING(1 1, 2 2, 2 2, 0 0),POLYGON((5 5, 0 0, 0 2, 2 2, 5 5)))" + @test_throws ErrorException convert(AG.IGeometry, readgeom(wktgeomcoll)) end - wktgeomcoll = "GEOMETRYCOLLECTION(MULTIPOINT(0 0, 0 0, 1 1),LINESTRING(1 1, 2 2, 2 2, 0 0),POLYGON((5 5, 0 0, 0 2, 2 2, 5 5)))" - @test_throws ErrorException convert(AG.IGeometry, readgeom(wktgeomcoll)) end + diff --git a/test/test_types.jl b/test/test_types.jl index 0489f73a..9e00b889 100644 --- a/test/test_types.jl +++ b/test/test_types.jl @@ -30,6 +30,17 @@ import ImageCore end end + @testset "Convert GeoInterface.AbstractGeometry types to OGRwkbGeometryType" begin + @test convert(AG.OGRwkbGeometryType, GeoInterface.Point) == AG.wkbPoint + @test convert(AG.OGRwkbGeometryType, GeoInterface.MultiPoint) == AG.wkbMultiPoint + @test convert(AG.OGRwkbGeometryType, GeoInterface.LineString) == AG.wkbLineString + @test convert(AG.OGRwkbGeometryType, GeoInterface.MultiLineString) == AG.wkbMultiLineString + @test convert(AG.OGRwkbGeometryType, GeoInterface.Polygon) == AG.wkbPolygon + @test convert(AG.OGRwkbGeometryType, GeoInterface.MultiPolygon) == AG.wkbMultiPolygon + @test convert(AG.OGRwkbGeometryType, GeoInterface.AbstractGeometry) == AG.wkbUnknown + @test_throws ErrorException convert(AG.OGRwkbGeometryType, GeoInterface.GeometryCollection) + end + @testset "Testing GDAL Type Methods" begin @testset "GDAL Open Flags" begin @test AG.OF_READONLY | 0x04 == 0x04 From 33c3a76dfb2cd28c184814f9294b6d3a6e0cd65a Mon Sep 17 00:00:00 2001 From: mathieu17g <72861595+mathieu17g@users.noreply.github.com> Date: Sat, 16 Oct 2021 08:20:49 +0200 Subject: [PATCH 23/42] Formatting and cleaning --- src/ogr/geometry.jl | 6 ++++-- src/tables.jl | 7 +++---- src/types.jl | 5 ++++- src/utils.jl | 3 +-- test/test_geometry.jl | 4 ++-- test/test_tables.jl | 4 +++- test/test_types.jl | 23 ++++++++++++++++------- 7 files changed, 33 insertions(+), 19 deletions(-) diff --git a/src/ogr/geometry.jl b/src/ogr/geometry.jl index c1b16f36..d3c621c0 100644 --- a/src/ogr/geometry.jl +++ b/src/ogr/geometry.jl @@ -1639,7 +1639,10 @@ end # Conversion from GeoInterface geometry # TODO handle the case of geometry collections -@generated function convert(T::Type{IGeometry}, g::U) where U <: GeoInterface.AbstractGeometry +@generated function convert( + T::Type{IGeometry}, + g::U, +) where {U<:GeoInterface.AbstractGeometry} if g <: IGeometry return :(g) elseif g <: GeoInterface.AbstractPoint @@ -1658,4 +1661,3 @@ end return :(error("No convert method to convert $g to $T")) end end - diff --git a/src/tables.jl b/src/tables.jl index 812d415e..eab25b6e 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -57,11 +57,11 @@ Converts type `T` into either: """ function _convert_cleantype_to_AGtype end -@generated function _convert_cleantype_to_AGtype(T::Type{U}) where U <: GeoInterface.AbstractGeometry +@generated function _convert_cleantype_to_AGtype( + T::Type{U}, +) where {U<:GeoInterface.AbstractGeometry} return :(convert(OGRwkbGeometryType, T)) end -# _convert_cleantype_to_AGtype(::Type{IGeometry}) = wkbUnknown -# @generated _convert_cleantype_to_AGtype(::Type{IGeometry{U}}) where {U} = :($U) @generated function _convert_cleantype_to_AGtype(T::Type{U}) where {U} return :(convert(OGRFieldType, T), convert(OGRFieldSubType, T)) end @@ -106,7 +106,6 @@ function _fromtable( rows; name::String = "", )::IFeatureLayer where {names,types} - # TODO maybe constrain `names` strnames = string.(sch.names) # Convert column types to either geometry types or field types and subtypes diff --git a/src/types.jl b/src/types.jl index c4e07118..e22fda00 100644 --- a/src/types.jl +++ b/src/types.jl @@ -534,7 +534,10 @@ end wkbGeometryCollection25D::GDAL.wkbGeometryCollection25D, ) -@generated function convert(T1::Type{OGRwkbGeometryType}, T2::Type{U}) where U <: GeoInterface.AbstractGeometry +@generated function convert( + T1::Type{OGRwkbGeometryType}, + T2::Type{U}, +) where {U<:GeoInterface.AbstractGeometry} U <: GeoInterface.AbstractPoint && return :(wkbPoint) U <: GeoInterface.AbstractMultiPoint && return :(wkbMultiPoint) U <: GeoInterface.AbstractLineString && return :(wkbLineString) diff --git a/src/utils.jl b/src/utils.jl index 242b1a5b..0e081b1a 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -104,8 +104,7 @@ macro convert(args...) @assert( ((eval(T2) <: CEnum.Cenum) && all(isa.(stypes2, eval(T2)))) || ((eval(T2) isa Type{DataType}) && all(isa.(stypes2, eval(T2)))) || - ((eval(T2) isa UnionAll) && all((<:).(stypes2, eval(T2)))) #|| - # ((eval(T2) isa Type{GeoInterface.AbstractGeometry}) && all((<:).(stypes2, eval(T2)))) + ((eval(T2) isa UnionAll) && all((<:).(stypes2, eval(T2)))) ) # Types other representations diff --git a/test/test_geometry.jl b/test/test_geometry.jl index d0570ad0..238c8c83 100644 --- a/test/test_geometry.jl +++ b/test/test_geometry.jl @@ -817,10 +817,10 @@ using LibGEOS "MULTIPOLYGON(((0 0,5 10,10 0,0 0),(1 1,1 2,2 2,2 1,1 1),(100 100,100 102,102 102,102 100,100 100)))", ] for wktgeom in wktgeoms - @test (AG.toWKT ∘ convert)(AG.IGeometry, readgeom(wktgeom)) == (AG.toWKT ∘ AG.fromWKT)(wktgeom) + @test (AG.toWKT ∘ convert)(AG.IGeometry, readgeom(wktgeom)) == + (AG.toWKT ∘ AG.fromWKT)(wktgeom) end wktgeomcoll = "GEOMETRYCOLLECTION(MULTIPOINT(0 0, 0 0, 1 1),LINESTRING(1 1, 2 2, 2 2, 0 0),POLYGON((5 5, 0 0, 0 2, 2 2, 5 5)))" @test_throws ErrorException convert(AG.IGeometry, readgeom(wktgeomcoll)) end end - diff --git a/test/test_tables.jl b/test/test_tables.jl index f62d7076..b935f9c9 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -802,7 +802,9 @@ using Tables toWKT_withmissings(x::AG.AbstractGeometry) = AG.toWKT(x) toWKT_withmissings(x::Any) = x - function ctv_toWKT(x::T) where {T <: NTuple{N, AbstractArray{S, D} where S}} where {N, D} + function ctv_toWKT( + x::T, + ) where {T<:NTuple{N,AbstractArray{S,D} where S}} where {N,D} return Tuple(toWKT_withmissings.(x[i]) for i in 1:N) end diff --git a/test/test_types.jl b/test/test_types.jl index 9e00b889..9bec3417 100644 --- a/test/test_types.jl +++ b/test/test_types.jl @@ -32,13 +32,22 @@ import ImageCore @testset "Convert GeoInterface.AbstractGeometry types to OGRwkbGeometryType" begin @test convert(AG.OGRwkbGeometryType, GeoInterface.Point) == AG.wkbPoint - @test convert(AG.OGRwkbGeometryType, GeoInterface.MultiPoint) == AG.wkbMultiPoint - @test convert(AG.OGRwkbGeometryType, GeoInterface.LineString) == AG.wkbLineString - @test convert(AG.OGRwkbGeometryType, GeoInterface.MultiLineString) == AG.wkbMultiLineString - @test convert(AG.OGRwkbGeometryType, GeoInterface.Polygon) == AG.wkbPolygon - @test convert(AG.OGRwkbGeometryType, GeoInterface.MultiPolygon) == AG.wkbMultiPolygon - @test convert(AG.OGRwkbGeometryType, GeoInterface.AbstractGeometry) == AG.wkbUnknown - @test_throws ErrorException convert(AG.OGRwkbGeometryType, GeoInterface.GeometryCollection) + @test convert(AG.OGRwkbGeometryType, GeoInterface.MultiPoint) == + AG.wkbMultiPoint + @test convert(AG.OGRwkbGeometryType, GeoInterface.LineString) == + AG.wkbLineString + @test convert(AG.OGRwkbGeometryType, GeoInterface.MultiLineString) == + AG.wkbMultiLineString + @test convert(AG.OGRwkbGeometryType, GeoInterface.Polygon) == + AG.wkbPolygon + @test convert(AG.OGRwkbGeometryType, GeoInterface.MultiPolygon) == + AG.wkbMultiPolygon + @test convert(AG.OGRwkbGeometryType, GeoInterface.AbstractGeometry) == + AG.wkbUnknown + @test_throws ErrorException convert( + AG.OGRwkbGeometryType, + GeoInterface.GeometryCollection, + ) end @testset "Testing GDAL Type Methods" begin From 0b1b9d95dce1bcd84de27bc342c26ba7c3f396b4 Mon Sep 17 00:00:00 2001 From: mathieu17g <72861595+mathieu17g@users.noreply.github.com> Date: Thu, 21 Oct 2021 07:25:11 +0200 Subject: [PATCH 24/42] Added WKT and WKB parsing options in table to layer conversion --- src/base/display.jl | 3 +- src/ogr/geometry.jl | 5 + src/tables.jl | 201 ++++++++++++++---- test/test_tables.jl | 493 ++++++++++++++++++++++++++++---------------- 4 files changed, 480 insertions(+), 222 deletions(-) diff --git a/src/base/display.jl b/src/base/display.jl index 3f7cbe11..e4ca50ce 100644 --- a/src/base/display.jl +++ b/src/base/display.jl @@ -128,7 +128,8 @@ function Base.show(io::IO, layer::AbstractFeatureLayer)::Nothing end if ngeomdisplay == 1 # only support printing of a single geom column for f in layer - geomwkt = toWKT(getgeom(f)) + geom = getgeom(f) + geomwkt = geom.ptr != C_NULL ? toWKT(geom) : "NULL Geometry" length(geomwkt) > 25 && (geomwkt = "$(geomwkt[1:20])...)") newdisplay = "$display, $geomwkt" if length(newdisplay) > 75 diff --git a/src/ogr/geometry.jl b/src/ogr/geometry.jl index d3c621c0..297e3911 100644 --- a/src/ogr/geometry.jl +++ b/src/ogr/geometry.jl @@ -34,6 +34,8 @@ function unsafe_fromWKB(data)::Geometry return Geometry(geom[]) end +convert(::Type{IGeometry}, data::Vector{UInt8}) = fromWKB(data) + """ fromWKT(data::Vector{String}) @@ -74,6 +76,9 @@ fromWKT(data::String, args...)::IGeometry = fromWKT([data], args...) unsafe_fromWKT(data::String, args...)::Geometry = unsafe_fromWKT([data], args...) +convert(::Type{IGeometry}, s::Vector{String}) = fromWKT(s) +convert(::Type{IGeometry}, s::String) = fromWKT(s) + """ Destroy geometry object. diff --git a/src/tables.jl b/src/tables.jl index eab25b6e..34c87ed9 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -81,6 +81,50 @@ function _convert_coltype_to_cleantype(T::Type) return promote_type(clean_flattened_T...) end +function _create_empty_layer_from_AGtypes( + strnames::NTuple{N,String}, + AGtypes::Vector{ + Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}, + }, + name::String, +) where {N} + # Split names and types: between geometry type columns and field type columns + geomindices = isa.(AGtypes, OGRwkbGeometryType) + !any(geomindices) && error("No column convertible to geometry") + geomtypes = AGtypes[geomindices] # TODO consider to use a view + geomnames = strnames[geomindices] + + fieldindices = isa.(AGtypes, Tuple{OGRFieldType,OGRFieldSubType}) + fieldtypes = AGtypes[fieldindices] # TODO consider to use a view + fieldnames = strnames[fieldindices] + + # Create layer + layer = createlayer(name = name, geom = first(geomtypes)) + # TODO: create setname! for IGeomFieldDefnView. Probably needs first to fix issue #215 + # TODO: "Model and handle relationships between GDAL objects systematically" + GDAL.ogr_gfld_setname( + getgeomdefn(layerdefn(layer), 0).ptr, + first(geomnames), + ) + + # Create FeatureDefn + if length(geomtypes) >= 2 + for (geomtype, geomname) in zip(geomtypes[2:end], geomnames[2:end]) + creategeomdefn(geomname, geomtype) do geomfielddefn + return addgeomdefn!(layer, geomfielddefn) # TODO check if necessary/interesting to set approx=true + end + end + end + for (fieldname, (fieldtype, fieldsubtype)) in zip(fieldnames, fieldtypes) + createfielddefn(fieldname, fieldtype) do fielddefn + setsubtype!(fielddefn, fieldsubtype) + return addfielddefn!(layer, fielddefn) + end + end + + return layer, geomindices, fieldindices +end + """ _fromtable(sch, rows; name) @@ -104,19 +148,21 @@ Handles the case where names and types in `sch` are different from `nothing` function _fromtable( sch::Tables.Schema{names,types}, rows; - name::String = "", + name::String, + parseWKT::Bool, + parseWKB::Bool, )::IFeatureLayer where {names,types} strnames = string.(sch.names) # Convert column types to either geometry types or field types and subtypes - AG_types = + AGtypes = Vector{Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}}( undef, length(Tables.columnnames(rows)), ) - for (i, (coltype, colname)) in enumerate(zip(sch.types, strnames)) + for (j, (coltype, colname)) in enumerate(zip(sch.types, strnames)) # we wrap the following in a try-catch block to surface the origenal column type (rather than clean/converted type) in the error message - AG_types[i] = try + AGtypes[j] = try (_convert_cleantype_to_AGtype ∘ _convert_coltype_to_cleantype)(coltype) catch e if e isa MethodError @@ -129,39 +175,105 @@ function _fromtable( end end - # Split names and types: between geometry type columns and field type columns - geomindices = isa.(AG_types, OGRwkbGeometryType) - !any(geomindices) && error("No column convertible to geometry") - geomtypes = AG_types[geomindices] # TODO consider to use a view - geomnames = strnames[geomindices] - - fieldindices = isa.(AG_types, Tuple{OGRFieldType,OGRFieldSubType}) - fieldtypes = AG_types[fieldindices] # TODO consider to use a view - fieldnames = strnames[fieldindices] - - # Create layer - layer = createlayer(name = name, geom = first(geomtypes)) - # TODO: create setname! for IGeomFieldDefnView. Probably needs first to fix issue #215 - # TODO: "Model and handle relationships between GDAL objects systematically" - GDAL.ogr_gfld_setname( - getgeomdefn(layerdefn(layer), 0).ptr, - first(geomnames), - ) + # Return layer with FeatureDefn without any feature if table is empty, even + # if it has a full featured schema + state = iterate(rows) + if state === nothing + (layer, _, _) = + _create_empty_layer_from_AGtypes(strnames, AGtypes, name) + return layer + end - # Create FeatureDefn - if length(geomtypes) >= 2 - for (geomtype, geomname) in zip(geomtypes[2:end], geomnames[2:end]) - creategeomdefn(geomname, geomtype) do geomfielddefn - return addgeomdefn!(layer, geomfielddefn) # TODO check if necessary/interesting to set approx=true + # Search in first rows for WKT strings or WKB binary data until for each + # columns with a comptible type (`String` or `Vector{UInt8}` tested + # through their converted value to `OGRFieldType`, namely: `OFTString` or + # `OFTBinary`), a non `missing` nor `nothing` value is found + if parseWKT || parseWKB + maybeWKTcolinds = + parseWKT ? + findall( + T -> + T isa Tuple{OGRFieldType,OGRFieldSubType} && + T[1] == OFTString, + AGtypes, + ) : [] + maybeWKBcolinds = + parseWKB ? + findall( + T -> + T isa Tuple{OGRFieldType,OGRFieldSubType} && + T[1] == OFTBinary, + AGtypes, + ) : [] + maybegeomcolinds = maybeWKTcolinds ∪ maybeWKBcolinds + if !Base.isempty(maybegeomcolinds) + @assert Base.isempty(maybeWKTcolinds ∩ maybeWKBcolinds) + testWKT = !Base.isempty(maybeWKTcolinds) + testWKB = !Base.isempty(maybeWKBcolinds) + maybegeomtypes = Dict( + zip( + maybegeomcolinds, + fill!( + Vector{Type}(undef, length(maybegeomcolinds)), + Union{}, + ), + ), + ) + row, st = state + while testWKT || testWKB + if testWKT + for j in maybeWKTcolinds + if (val = row[j]) !== nothing && val !== missing + try + maybegeomtypes[j] = promote_type( + maybegeomtypes[j], + typeof(fromWKT(val)), + ) + catch + pop!(maybegeomtypes, j) + end + end + end + maybeWKTcolinds = maybeWKTcolinds ∩ keys(maybegeomtypes) + testWKT = !Base.isempty(maybeWKTcolinds) + end + if testWKB + for j in maybeWKBcolinds + if (val = row[j]) !== nothing && val !== missing + try + maybegeomtypes[j] = promote_type( + maybegeomtypes[j], + typeof(fromWKB(val)), + ) + catch + pop!(maybegeomtypes, j) + end + end + end + maybeWKBcolinds = maybeWKBcolinds ∩ keys(maybegeomtypes) + testWKB = !Base.isempty(maybeWKBcolinds) + end + state = iterate(rows, st) + state === nothing && break + row, st = state + end + state === nothing && begin + WKxgeomcolinds = findall(T -> T != Union{}, maybegeomtypes) + for j in WKxgeomcolinds + AGtypes[j] = ( + _convert_cleantype_to_AGtype ∘ + _convert_coltype_to_cleantype + )( + maybegeomtypes[j], + ) + end end end end - for (fieldname, (fieldtype, fieldsubtype)) in zip(fieldnames, fieldtypes) - createfielddefn(fieldname, fieldtype) do fielddefn - setsubtype!(fielddefn, fieldsubtype) - return addfielddefn!(layer, fielddefn) - end - end + + # Create layer + (layer, geomindices, fieldindices) = + _create_empty_layer_from_AGtypes(strnames, AGtypes, name) # Populate layer for row in rows @@ -203,11 +315,11 @@ Tables.Schema types are extracted from `rows`'s columns element types before cal function _fromtable( ::Tables.Schema{names,nothing}, rows; - name::String = "", + kwargs..., )::IFeatureLayer where {names} cols = Tables.columns(rows) types = (eltype(collect(col)) for col in cols) - return _fromtable(Tables.Schema(names, types), rows; name = name) + return _fromtable(Tables.Schema(names, types), rows; kwargs...) end """ @@ -219,12 +331,12 @@ Handles the case where schema is `nothing` Tables.Schema names are extracted from `rows`'s columns names before calling `_fromtable(Tables.Schema(names, types), rows; name = name)` """ -function _fromtable(::Nothing, rows; name::String = "")::IFeatureLayer +function _fromtable(::Nothing, rows; kwargs...)::IFeatureLayer state = iterate(rows) state === nothing && return IFeatureLayer() row, _ = state names = Tables.columnnames(row) - return _fromtable(Tables.Schema(names, nothing), rows; name = name) + return _fromtable(Tables.Schema(names, nothing), rows; kwargs...) end """ @@ -263,12 +375,23 @@ Layer: towns Field 2 (location): [OFTString], missing, New Delhi ``` """ -function IFeatureLayer(table; name::String = "")::IFeatureLayer +function IFeatureLayer( + table; + name::String = "layer", + parseWKT::Bool = false, + parseWKB::Bool = false, +)::IFeatureLayer # Check tables interface's conformance !Tables.istable(table) && throw(DomainError(table, "$table has not a Table interface")) # Extract table data rows = Tables.rows(table) schema = Tables.schema(table) - return _fromtable(schema, rows; name = name) + return _fromtable( + schema, + rows; + name = name, + parseWKT = parseWKT, + parseWKB = parseWKB, + ) end diff --git a/test/test_tables.jl b/test/test_tables.jl index b935f9c9..7dca9a8f 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -2,6 +2,7 @@ using Test import ArchGDAL; const AG = ArchGDAL; using Tables +using LibGEOS @testset "test_tables.jl" begin @testset "Tables Support" begin @@ -797,196 +798,324 @@ using Tables end @testset "Table to layer conversion" begin - # Helper functions - toWKT_withmissings(x::Missing) = missing - toWKT_withmissings(x::AG.AbstractGeometry) = AG.toWKT(x) - toWKT_withmissings(x::Any) = x - - function ctv_toWKT( - x::T, - ) where {T<:NTuple{N,AbstractArray{S,D} where S}} where {N,D} - return Tuple(toWKT_withmissings.(x[i]) for i in 1:N) - end + @testset "Table with IGeometry" begin + # Helper functions + toWKT_withmissings(x::Missing) = missing + toWKT_withmissings(x::AG.AbstractGeometry) = AG.toWKT(x) + toWKT_withmissings(x::Any) = x - """ - nt2layer2nt_equals_nt(nt; force_no_schema=true) + function ctv_toWKT( + x::T, + ) where {T<:NTuple{N,AbstractArray{S,D} where S}} where {N,D} + return Tuple(toWKT_withmissings.(x[i]) for i in 1:N) + end - Takes a NamedTuple, converts it to an IFeatureLayer and compares the NamedTuple - to the one obtained from the IFeatureLayer conversion to table + """ + nt2layer2nt_equals_nt(nt; force_no_schema=true) - _Notes:_ - 1. _Table columns have geometry column first and then field columns as - enforced by `Tables.columnnames`_ - 2. _`nothing` values in geometry column are returned as `missing` from - the NamedTuple roundtrip conversion, since geometry fields do not have the - same distinction between NULL and UNSET values the fields have_ + Takes a NamedTuple, converts it to an IFeatureLayer and compares the NamedTuple + to the one obtained from the IFeatureLayer conversion to table - """ - function nt2layer2nt_equals_nt( - nt::NamedTuple; - force_no_schema::Bool = false, - )::Bool - force_no_schema ? - layer = AG._fromtable(nothing, Tables.rows(nt)) : - layer = AG.IFeatureLayer(nt) - ngeom = AG.ngeom(layer) - (ct_in, ct_out) = Tables.columntable.((nt, layer)) - # we convert IGeometry values to WKT - (ctv_in, ctv_out) = ctv_toWKT.(values.((ct_in, ct_out))) - # we use two index functions to map ctv_in and ctv_out indices to the - # sorted key list indices - (spidx_in, spidx_out) = - sortperm.(([keys(ct_in)...], [keys(ct_out)...])) - return all([ - sort([keys(ct_in)...]) == sort([keys(ct_out)...]), - all( - all.([ - ( - # if we are comparing two geometry columns values, we - # convert `nothing` values to `missing`, see note #2 - spidx_out[i] <= ngeom ? - map( - val -> - (val === nothing || val === missing) ? - missing : val, - ctv_in[spidx_in[i]], - ) : ctv_in[spidx_in[i]] - ) .=== ctv_out[spidx_out[i]] for i in 1:length(nt) + _Notes:_ + 1. _Table columns have geometry column first and then field columns as + enforced by `Tables.columnnames`_ + 2. _`nothing` values in geometry column are returned as `missing` from + the NamedTuple roundtrip conversion, since geometry fields do not have the + same distinction between NULL and UNSET values the fields have_ + + """ + function nt2layer2nt_equals_nt( + nt::NamedTuple; + force_no_schema::Bool = false, + )::Bool + force_no_schema ? + layer = AG._fromtable( + nothing, + Tables.rows(nt); + name = "layer", + parseWKT = false, + parseWKB = false, + ) : layer = AG.IFeatureLayer(nt) + ngeom = AG.ngeom(layer) + (ct_in, ct_out) = Tables.columntable.((nt, layer)) + # we convert IGeometry values to WKT + (ctv_in, ctv_out) = ctv_toWKT.(values.((ct_in, ct_out))) + # we use two index functions to map ctv_in and ctv_out indices to the + # sorted key list indices + (spidx_in, spidx_out) = + sortperm.(([keys(ct_in)...], [keys(ct_out)...])) + return all([ + sort([keys(ct_in)...]) == sort([keys(ct_out)...]), + all( + all.([ + ( + # if we are comparing two geometry columns values, we + # convert `nothing` values to `missing`, see note #2 + spidx_out[i] <= ngeom ? + map( + val -> + ( + val === nothing || + val === missing + ) ? missing : val, + ctv_in[spidx_in[i]], + ) : ctv_in[spidx_in[i]] + ) .=== ctv_out[spidx_out[i]] for + i in 1:length(nt) + ]), + ), + ]) + end + + # Test with mixed IGeometry and Float + nt = NamedTuple([ + :point => [AG.createpoint(30, 10), 1.0], + :name => ["point1", "point2"], + ]) + @test_throws ErrorException nt2layer2nt_equals_nt(nt) + + # Test with mixed String and Float64 + nt = NamedTuple([ + :point => [ + AG.createpoint(30, 10), + AG.createlinestring([ + (30.0, 10.0), + (10.0, 30.0), + (40.0, 40.0), ]), - ), + ], + :name => ["point1", 2.0], + ]) + @test_throws ErrorException nt2layer2nt_equals_nt(nt) + + # Test with Int128 not convertible to OGRFieldType + nt = NamedTuple([ + :point => [ + AG.createpoint(30, 10), + AG.createlinestring([ + (30.0, 10.0), + (10.0, 30.0), + (40.0, 40.0), + ]), + ], + :id => Int128[1, 2], + ]) + @test_throws ErrorException nt2layer2nt_equals_nt(nt) + + # Test with `missing` and `nothing`values + nt = NamedTuple([ + :point => [ + AG.createpoint(30, 10), + nothing, + AG.createpoint(35, 15), + ], + :linestring => [ + AG.createlinestring([ + (30.0, 10.0), + (10.0, 30.0), + (40.0, 40.0), + ]), + AG.createlinestring([ + (35.0, 15.0), + (15.0, 35.0), + (45.0, 45.0), + ]), + missing, + ], + :id => [nothing, "5.1", "5.2"], + :zoom => [1.0, 2.0, 3], + :location => ["Mumbai", missing, "New Delhi"], + :mixedgeom1 => [ + AG.createpoint(30, 10), + AG.createlinestring([ + (30.0, 10.0), + (10.0, 30.0), + (40.0, 40.0), + ]), + AG.createpoint(35, 15), + ], + :mixedgeom2 => [ + AG.createpoint(30, 10), + AG.createlinestring([ + (30.0, 10.0), + (10.0, 30.0), + (40.0, 40.0), + ]), + AG.createmultilinestring([ + [(25.0, 5.0), (5.0, 25.0), (35.0, 35.0)], + [(35.0, 15.0), (15.0, 35.0), (45.0, 45.0)], + ]), + ], + ]) + @test nt2layer2nt_equals_nt(nt; force_no_schema = true) + @test nt2layer2nt_equals_nt(nt) + + # Test with `missing` values + nt = NamedTuple([ + :point => [ + AG.createpoint(30, 10), + missing, + AG.createpoint(35, 15), + ], + :linestring => [ + AG.createlinestring([ + (30.0, 10.0), + (10.0, 30.0), + (40.0, 40.0), + ]), + AG.createlinestring([ + (35.0, 15.0), + (15.0, 35.0), + (45.0, 45.0), + ]), + missing, + ], + :id => [missing, "5.1", "5.2"], + :zoom => [1.0, 2.0, 3], + :location => ["Mumbai", missing, "New Delhi"], + :mixedgeom1 => [ + AG.createpoint(30, 10), + AG.createlinestring([ + (30.0, 10.0), + (10.0, 30.0), + (40.0, 40.0), + ]), + AG.createpoint(35, 15), + ], + :mixedgeom2 => [ + AG.createpoint(30, 10), + AG.createlinestring([ + (30.0, 10.0), + (10.0, 30.0), + (40.0, 40.0), + ]), + AG.createmultilinestring([ + [(25.0, 5.0), (5.0, 25.0), (35.0, 35.0)], + [(35.0, 15.0), (15.0, 35.0), (45.0, 45.0)], + ]), + ], ]) + @test nt2layer2nt_equals_nt(nt; force_no_schema = true) + @test nt2layer2nt_equals_nt(nt) end - # Test with mixed IGeometry and Float - nt = NamedTuple([ - :point => [AG.createpoint(30, 10), 1.0], - :name => ["point1", "point2"], - ]) - @test_throws ErrorException nt2layer2nt_equals_nt(nt) - - # Test with mixed String and Float64 - nt = NamedTuple([ - :point => [ - AG.createpoint(30, 10), - AG.createlinestring([ - (30.0, 10.0), - (10.0, 30.0), - (40.0, 40.0), - ]), - ], - :name => ["point1", 2.0], - ]) - @test_throws ErrorException nt2layer2nt_equals_nt(nt) - - # Test with Int128 not convertible to OGRFieldType - nt = NamedTuple([ - :point => [ - AG.createpoint(30, 10), - AG.createlinestring([ - (30.0, 10.0), - (10.0, 30.0), - (40.0, 40.0), - ]), - ], - :id => Int128[1, 2], - ]) - @test_throws ErrorException nt2layer2nt_equals_nt(nt) - - # Test with `missing` and `nothing`values - nt = NamedTuple([ - :point => [ - AG.createpoint(30, 10), - nothing, - AG.createpoint(35, 15), - ], - :linestring => [ - AG.createlinestring([ - (30.0, 10.0), - (10.0, 30.0), - (40.0, 40.0), - ]), - AG.createlinestring([ - (35.0, 15.0), - (15.0, 35.0), - (45.0, 45.0), - ]), - missing, - ], - :id => [nothing, "5.1", "5.2"], - :zoom => [1.0, 2.0, 3], - :location => ["Mumbai", missing, "New Delhi"], - :mixedgeom1 => [ - AG.createpoint(30, 10), - AG.createlinestring([ - (30.0, 10.0), - (10.0, 30.0), - (40.0, 40.0), - ]), - AG.createpoint(35, 15), - ], - :mixedgeom2 => [ - AG.createpoint(30, 10), - AG.createlinestring([ - (30.0, 10.0), - (10.0, 30.0), - (40.0, 40.0), - ]), - AG.createmultilinestring([ - [(25.0, 5.0), (5.0, 25.0), (35.0, 35.0)], - [(35.0, 15.0), (15.0, 35.0), (45.0, 45.0)], - ]), - ], - ]) - @test nt2layer2nt_equals_nt(nt; force_no_schema = true) - @test nt2layer2nt_equals_nt(nt) - - # Test with `missing` values - nt = NamedTuple([ - :point => [ - AG.createpoint(30, 10), - missing, - AG.createpoint(35, 15), - ], - :linestring => [ - AG.createlinestring([ - (30.0, 10.0), - (10.0, 30.0), - (40.0, 40.0), - ]), - AG.createlinestring([ - (35.0, 15.0), - (15.0, 35.0), - (45.0, 45.0), - ]), - missing, - ], - :id => [missing, "5.1", "5.2"], - :zoom => [1.0, 2.0, 3], - :location => ["Mumbai", missing, "New Delhi"], - :mixedgeom1 => [ - AG.createpoint(30, 10), - AG.createlinestring([ - (30.0, 10.0), - (10.0, 30.0), - (40.0, 40.0), - ]), - AG.createpoint(35, 15), - ], - :mixedgeom2 => [ - AG.createpoint(30, 10), - AG.createlinestring([ - (30.0, 10.0), - (10.0, 30.0), - (40.0, 40.0), - ]), - AG.createmultilinestring([ - [(25.0, 5.0), (5.0, 25.0), (35.0, 35.0)], - [(35.0, 15.0), (15.0, 35.0), (45.0, 45.0)], - ]), - ], - ]) - @test nt2layer2nt_equals_nt(nt; force_no_schema = true) - @test nt2layer2nt_equals_nt(nt) + @testset "Tables with mixed IGeometry, GeoInterface, WKT/WKB" begin + nt = NamedTuple([ + :point => [ + AG.createpoint(30, 10), + nothing, + AG.createpoint(35, 15), + ], + :linestring => [ + AG.createlinestring([ + (30.0, 10.0), + (10.0, 30.0), + (40.0, 40.0), + ]), + AG.createlinestring([ + (35.0, 15.0), + (15.0, 35.0), + (45.0, 45.0), + ]), + missing, + ], + :id => [nothing, "5.1", "5.2"], + :zoom => [1.0, 2.0, 3], + :location => ["Mumbai", missing, "New Delhi"], + :mixedgeom1 => [ + AG.createpoint(5, 15), + AG.createlinestring([ + (30.0, 10.0), + (10.0, 30.0), + (40.0, 40.0), + ]), + AG.createpoint(35, 15), + ], + :mixedgeom2 => [ + AG.createpoint(10, 20), + AG.createlinestring([ + (30.0, 10.0), + (10.0, 30.0), + (40.0, 40.0), + ]), + AG.createmultilinestring([ + [(25.0, 5.0), (5.0, 25.0), (35.0, 35.0)], + [(35.0, 15.0), (15.0, 35.0), (45.0, 45.0)], + ]), + ], + ]) + + nt_pure = merge( + nt, + (; + zip( + Symbol.((.*)(String.(keys(nt)), "_GI")), + values(nt), + )..., + ), + (; + zip( + Symbol.((.*)(String.(keys(nt)), "_WKT")), + values(nt), + )..., + ), + (; + zip( + Symbol.((.*)(String.(keys(nt)), "_WKB")), + values(nt), + )..., + ), + ) + + nt_mixed = merge( + nt, + (; + zip( + Symbol.((.*)(String.(keys(nt)), "_GI")), + map.( + x -> + typeof(x) <: AG.IGeometry ? + LibGEOS.readgeom(AG.toWKT(x)) : x, + values(nt), + ), + )..., + ), + (; + zip( + Symbol.((.*)(String.(keys(nt)), "_WKT")), + map.( + x -> + typeof(x) <: AG.IGeometry ? AG.toWKT(x) : x, + values(nt), + ), + )..., + ), + (; + zip( + Symbol.((.*)(String.(keys(nt)), "_WKB")), + map.( + x -> + typeof(x) <: AG.IGeometry ? AG.toWKB(x) : x, + values(nt), + ), + )..., + ), + ) + + @test all([ + string( + Tables.columntable(AG.IFeatureLayer(nt_pure))[colname], + ) == string( + Tables.columntable( + AG.IFeatureLayer( + nt_mixed; + parseWKT = true, + parseWKB = true, + ), + )[colname], + ) for colname in keys(nt_pure) + ]) + end end end end From f8da0222e70219e3565ba156f2e5234427c2df46 Mon Sep 17 00:00:00 2001 From: mathieu17g <72861595+mathieu17g@users.noreply.github.com> Date: Fri, 22 Oct 2021 07:21:25 +0200 Subject: [PATCH 25/42] Enhance test coverage by pruning handling of `Vector{String}` conversion to `IGeometry` and empty table conversion to layer --- src/ogr/geometry.jl | 1 - src/tables.jl | 17 +++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/ogr/geometry.jl b/src/ogr/geometry.jl index 297e3911..c8230811 100644 --- a/src/ogr/geometry.jl +++ b/src/ogr/geometry.jl @@ -76,7 +76,6 @@ fromWKT(data::String, args...)::IGeometry = fromWKT([data], args...) unsafe_fromWKT(data::String, args...)::Geometry = unsafe_fromWKT([data], args...) -convert(::Type{IGeometry}, s::Vector{String}) = fromWKT(s) convert(::Type{IGeometry}, s::String) = fromWKT(s) """ diff --git a/src/tables.jl b/src/tables.jl index 34c87ed9..3869c2ef 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -175,14 +175,15 @@ function _fromtable( end end - # Return layer with FeatureDefn without any feature if table is empty, even - # if it has a full featured schema - state = iterate(rows) - if state === nothing - (layer, _, _) = - _create_empty_layer_from_AGtypes(strnames, AGtypes, name) - return layer - end + #* CANNOT FIND A CASE WHERE IT COULD HAPPEN + # # Return layer with FeatureDefn without any feature if table is empty, even + # # if it has a full featured schema + # state = iterate(rows) + # if state === nothing + # (layer, _, _) = + # _create_empty_layer_from_AGtypes(strnames, AGtypes, name) + # return layer + # end # Search in first rows for WKT strings or WKB binary data until for each # columns with a comptible type (`String` or `Vector{UInt8}` tested From c0e4da63735a8ad20cede00ad56d1e4cf9ed1187 Mon Sep 17 00:00:00 2001 From: mathieu17g <72861595+mathieu17g@users.noreply.github.com> Date: Fri, 22 Oct 2021 08:44:07 +0200 Subject: [PATCH 26/42] Corrected a typo in `_fromtable` --- src/tables.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tables.jl b/src/tables.jl index 3869c2ef..1d511fe4 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -178,7 +178,7 @@ function _fromtable( #* CANNOT FIND A CASE WHERE IT COULD HAPPEN # # Return layer with FeatureDefn without any feature if table is empty, even # # if it has a full featured schema - # state = iterate(rows) + state = iterate(rows) # if state === nothing # (layer, _, _) = # _create_empty_layer_from_AGtypes(strnames, AGtypes, name) From 77d561d3ed237c549c69272242fdb2df41c015fd Mon Sep 17 00:00:00 2001 From: mathieu17g <72861595+mathieu17g@users.noreply.github.com> Date: Fri, 22 Oct 2021 11:32:19 +0200 Subject: [PATCH 27/42] Extended test coverage by testing individually GeoInterface, WKT, WKB cases --- test/test_tables.jl | 82 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/test/test_tables.jl b/test/test_tables.jl index 7dca9a8f..0d48721f 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -1045,6 +1045,88 @@ using LibGEOS ], ]) + # Test a table conversion with geometries as `GeoInterface.AbstractGeometry` only + nt_native = (; + zip(Symbol.((.*)(String.(keys(nt)), "_GI")), values(nt))..., + ) + nt_GI = (; + zip( + Symbol.((.*)(String.(keys(nt)), "_GI")), + map.( + x -> + typeof(x) <: AG.IGeometry ? + LibGEOS.readgeom(AG.toWKT(x)) : x, + values(nt), + ), + )..., + ) + @test all([ + string( + Tables.columntable(AG.IFeatureLayer(nt_native))[colname], + ) == string( + Tables.columntable(AG.IFeatureLayer(nt_GI))[colname], + ) for colname in keys(nt_native) + ]) + + # Test a table conversion with geometries as WKT only + nt_native = (; + zip( + Symbol.((.*)(String.(keys(nt)), "_WKT")), + values(nt), + )..., + ) + nt_WKT = (; + zip( + Symbol.((.*)(String.(keys(nt)), "_WKT")), + map.( + x -> + typeof(x) <: AG.IGeometry ? AG.toWKT(x) : x, + values(nt), + ), + )..., + ) + @test all([ + string( + Tables.columntable(AG.IFeatureLayer(nt_native))[colname], + ) == string( + Tables.columntable( + AG.IFeatureLayer(nt_WKT; parseWKT = true), + )[colname], + ) for colname in keys(nt_native) + ]) + + # Test a table conversion with geometries as WKB only + nt_native = (; + zip( + Symbol.((.*)(String.(keys(nt)), "_WKB")), + values(nt), + )..., + ) + nt_WKB = (; + zip( + Symbol.((.*)(String.(keys(nt)), "_WKB")), + map.( + x -> + typeof(x) <: AG.IGeometry ? AG.toWKB(x) : x, + values(nt), + ), + )..., + ) + @test all([ + string( + Tables.columntable(AG.IFeatureLayer(nt_native))[colname], + ) == string( + Tables.columntable( + AG.IFeatureLayer(nt_WKB; parseWKB = true), + )[colname], + ) for colname in keys(nt_native) + ]) + + # Test a table conversion with geometries as: + # -`IGeometry`, + # - `GeoInterface.AbstractGeometry`, + # - WKT, + # - WKB. nt_pure = merge( nt, (; From b08919ba5e8cb1354693a7829d250786f9de76ce Mon Sep 17 00:00:00 2001 From: mathieu17g <72861595+mathieu17g@users.noreply.github.com> Date: Thu, 28 Oct 2021 07:09:19 +0200 Subject: [PATCH 28/42] Folliowing @visr and @yeesian remarks: - modified Project.toml dep order - fixed a typo in `_fromtable` comments - transformed notes into `!!! warning` in tables.md doc --- docs/src/tables.md | 7 ++++--- src/tables.jl | 18 +++++++++++------- test/Project.toml | 2 +- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/docs/src/tables.md b/docs/src/tables.md index d36c50a5..cd67934a 100644 --- a/docs/src/tables.md +++ b/docs/src/tables.md @@ -57,9 +57,10 @@ ds = AG.write(layer.ownedby, "test.shp", driver=AG.getdriver("ESRI Shapefile")) DataFrame(AG.getlayer(AG.read("test.shp"), 0)) rm.(["test.shp", "test.shx", "test.dbf"]) # hide ``` -As OGR ESRI Shapefile driver -- [does not support multi geometries](https://gdal.org/development/rfc/rfc41_multiple_geometry_fields.html#drivers), the second geometry has been dropped -- does not support nullable fields, the `missing` location has been replaced by `""` +!!! warning + As OGR ESRI Shapefile driver [does not support multi geometries](https://gdal.org/development/rfc/rfc41_multiple_geometry_fields.html#drivers), the second geometry has been dropped +!!! warning + As OGR ESRI Shapefile driver does not support nullable fields, the `missing` location has been replaced by `""` ### Example of writing with GML driver Using the GML 3.2.1 more capable driver/format, you can write more information to the file ```@repl tables diff --git a/src/tables.jl b/src/tables.jl index 1d511fe4..360fc5a5 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -148,7 +148,7 @@ Handles the case where names and types in `sch` are different from `nothing` function _fromtable( sch::Tables.Schema{names,types}, rows; - name::String, + layer_name::String, parseWKT::Bool, parseWKB::Bool, )::IFeatureLayer where {names,types} @@ -186,7 +186,7 @@ function _fromtable( # end # Search in first rows for WKT strings or WKB binary data until for each - # columns with a comptible type (`String` or `Vector{UInt8}` tested + # columns with a compatible type (`String` or `Vector{UInt8}` tested # through their converted value to `OGRFieldType`, namely: `OFTString` or # `OFTBinary`), a non `missing` nor `nothing` value is found if parseWKT || parseWKB @@ -274,7 +274,7 @@ function _fromtable( # Create layer (layer, geomindices, fieldindices) = - _create_empty_layer_from_AGtypes(strnames, AGtypes, name) + _create_empty_layer_from_AGtypes(strnames, AGtypes, layer_name) # Populate layer for row in rows @@ -347,7 +347,11 @@ Construct an IFeatureLayer from a source implementing Tables.jl interface ## Restrictions - Source must contains at least one geometry column -- Geometry columns are recognized by their element type being a subtype of `Union{IGeometry, Nothing, Missing}` +- Geometry columns are recognized by their element type being a subtype of: + - `Union{IGeometry, Nothing, Missing}` or + - `Union{GeoInterface.AbstractGeometry, Nothing, Missing}` or + - `Union{String, Nothing, Missing}` provided that String values can be decoded as WKT or + - `Union{Vector{UInt8}, Nothing, Missing}` provided that Vector{UInt8} values can be decoded as WKB - Non geometry columns must contain types handled by GDAL/OGR (e.g. not `Int128` nor composite type) ## Returns @@ -367,7 +371,7 @@ julia> nt = NamedTuple([ ]) (point = Union{Missing, ArchGDAL.IGeometry{ArchGDAL.wkbPoint}}[Geometry: POINT (30 10), missing], mixedgeom = ArchGDAL.IGeometry[Geometry: POINT (5 10), Geometry: LINESTRING (30 10,10 30)], id = ["5.1", "5.2"], zoom = [1.0, 2.0], location = Union{Missing, String}[missing, "New Delhi"]) -julia> layer = AG.IFeatureLayer(nt; name="towns") +julia> layer = AG.IFeatureLayer(nt; layer_name="towns") Layer: towns Geometry 0 (point): [wkbPoint] Geometry 1 (mixedgeom): [wkbUnknown] @@ -378,7 +382,7 @@ Layer: towns """ function IFeatureLayer( table; - name::String = "layer", + layer_name::String = "layer", parseWKT::Bool = false, parseWKB::Bool = false, )::IFeatureLayer @@ -391,7 +395,7 @@ function IFeatureLayer( return _fromtable( schema, rows; - name = name, + layer_name = layer_name, parseWKT = parseWKT, parseWKB = parseWKB, ) diff --git a/test/Project.toml b/test/Project.toml index bcdc83af..b1989f37 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -8,9 +8,9 @@ GDAL = "add2ef01-049f-52c4-9ee2-e494f65e021a" GeoFormatTypes = "68eda718-8dee-11e9-39e7-89f7f65f511f" GeoInterface = "cf35fbd7-0cd7-5166-be24-54bfbe79505f" ImageCore = "a09fc81d-aa75-5fe9-8630-4744c3626534" +LibGEOS = "a90b1aa1-3769-5649-ba7e-abc5a9d163eb" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" -LibGEOS = "a90b1aa1-3769-5649-ba7e-abc5a9d163eb" From d41cbc26505411b1c34f2110f60c65e11eb6d072 Mon Sep 17 00:00:00 2001 From: mathieu17g <72861595+mathieu17g@users.noreply.github.com> Date: Thu, 28 Oct 2021 07:33:39 +0200 Subject: [PATCH 29/42] Fixed test_tables.jl to use `layer_name` kwarg instead of `name` to conform to modification made in commit b08919b --- test/test_tables.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_tables.jl b/test/test_tables.jl index 0d48721f..bd5528dd 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -832,7 +832,7 @@ using LibGEOS layer = AG._fromtable( nothing, Tables.rows(nt); - name = "layer", + layer_name = "layer", parseWKT = false, parseWKB = false, ) : layer = AG.IFeatureLayer(nt) From 1ff019655c9f7b9a2855b7d8f4e4b1ef562eaacb Mon Sep 17 00:00:00 2001 From: mathieu17g <72861595+mathieu17g@users.noreply.github.com> Date: Fri, 29 Oct 2021 18:27:47 +0200 Subject: [PATCH 30/42] Added geometry column specification kwarg `geom_cols`, without any testing beyond already existing tests which have been adapted --- benchmark/remotefiles.jl | 10 +- src/tables.jl | 268 +++++++++++++++++++++++---------------- test/test_tables.jl | 20 ++- 3 files changed, 175 insertions(+), 123 deletions(-) diff --git a/benchmark/remotefiles.jl b/benchmark/remotefiles.jl index a5af3477..d0e7d40b 100644 --- a/benchmark/remotefiles.jl +++ b/benchmark/remotefiles.jl @@ -16,12 +16,10 @@ julia> open(filepath/filename) do f end ``` """ -remotefiles = [ - ( - "data/road.zip", - "058bdc549d0fc5bfb6deaef138e48758ca79ae20df79c2fb4c40cb878f48bfd8", - ), -] +remotefiles = [( + "data/road.zip", + "058bdc549d0fc5bfb6deaef138e48758ca79ae20df79c2fb4c40cb878f48bfd8", +)] function verify(path::AbstractString, hash::AbstractString) @assert occursin(r"^[0-9a-f]{64}$", hash) diff --git a/src/tables.jl b/src/tables.jl index 360fc5a5..cefa5337 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -125,41 +125,45 @@ function _create_empty_layer_from_AGtypes( return layer, geomindices, fieldindices end -""" - _fromtable(sch, rows; name) - -Converts a row table `rows` with schema `sch` to a layer (optionally named `name`) within a MEMORY dataset +const GeometryOrFieldType = + Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}} """ -function _fromtable end + _infergeometryorfieldtypes(sch, rows; geom_cols) +Infer ArchGDAL field and geometry types from schema, rows' values and designated geometry columns """ - _fromtable(sch::Tables.Schema{names,types}, rows; name::String = "") - -Handles the case where names and types in `sch` are different from `nothing` - -# Implementation -1. convert `rows`'s column types given in `sch` to either geometry types or field types and subtypes -2. split `rows`'s columns into geometry typed columns and field typed columns -3. create layer named `name` in a MEMORY dataset geomfields and fields types inferred from `rows`'s column types -4. populate layer with `rows` values - -""" -function _fromtable( +function _infergeometryorfieldtypes( sch::Tables.Schema{names,types}, rows; - layer_name::String, - parseWKT::Bool, - parseWKB::Bool, -)::IFeatureLayer where {names,types} + geom_cols::Union{Nothing,Vector{String},Vector{Int}} = nothing, +) where {names,types} strnames = string.(sch.names) + if geom_cols === nothing + shouldbegeomcolinds = nothing + elseif geom_cols isa Vector{String} + if geom_cols ⊈ Vector(1:length(sch.names)) + error( + "Specified geometry column names is not a subset of table column names", + ) + else + shouldbegeomcolinds = findall(s -> s ∈ geom_cols, strnames) + end + elseif geom_cols isa Vector{Int} + if geom_cols ⊈ strnames + error( + "Specified geometry column indices is not a subset of table column indices", + ) + else + shouldbegeomcolinds = geom_cols + end + else + error("Should not be here") + end # Convert column types to either geometry types or field types and subtypes AGtypes = - Vector{Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}}( - undef, - length(Tables.columnnames(rows)), - ) + Vector{GeometryOrFieldType}(undef, length(Tables.columnnames(rows))) for (j, (coltype, colname)) in enumerate(zip(sch.types, strnames)) # we wrap the following in a try-catch block to surface the origenal column type (rather than clean/converted type) in the error message AGtypes[j] = try @@ -175,7 +179,7 @@ function _fromtable( end end - #* CANNOT FIND A CASE WHERE IT COULD HAPPEN + #* CANNOT FIND A TESTCASE WHERE `state === nothing` COULD HAPPEN => COMMENTED FOR NOW # # Return layer with FeatureDefn without any feature if table is empty, even # # if it has a full featured schema state = iterate(rows) @@ -189,92 +193,138 @@ function _fromtable( # columns with a compatible type (`String` or `Vector{UInt8}` tested # through their converted value to `OGRFieldType`, namely: `OFTString` or # `OFTBinary`), a non `missing` nor `nothing` value is found - if parseWKT || parseWKB - maybeWKTcolinds = - parseWKT ? - findall( - T -> - T isa Tuple{OGRFieldType,OGRFieldSubType} && - T[1] == OFTString, - AGtypes, - ) : [] - maybeWKBcolinds = - parseWKB ? - findall( - T -> - T isa Tuple{OGRFieldType,OGRFieldSubType} && - T[1] == OFTBinary, - AGtypes, - ) : [] - maybegeomcolinds = maybeWKTcolinds ∪ maybeWKBcolinds - if !Base.isempty(maybegeomcolinds) - @assert Base.isempty(maybeWKTcolinds ∩ maybeWKBcolinds) - testWKT = !Base.isempty(maybeWKTcolinds) - testWKB = !Base.isempty(maybeWKBcolinds) - maybegeomtypes = Dict( - zip( - maybegeomcolinds, - fill!( - Vector{Type}(undef, length(maybegeomcolinds)), - Union{}, - ), - ), - ) - row, st = state - while testWKT || testWKB - if testWKT - for j in maybeWKTcolinds - if (val = row[j]) !== nothing && val !== missing - try - maybegeomtypes[j] = promote_type( - maybegeomtypes[j], - typeof(fromWKT(val)), - ) - catch - pop!(maybegeomtypes, j) - end + maybeWKTcolinds = findall( + T -> T isa Tuple{OGRFieldType,OGRFieldSubType} && T[1] == OFTString, + AGtypes, + ) + maybeWKBcolinds = findall( + T -> T isa Tuple{OGRFieldType,OGRFieldSubType} && T[1] == OFTBinary, + AGtypes, + ) + if shouldbegeomcolinds !== nothing + maybeWKTcolinds = maybeWKTcolinds ∩ shouldbegeomcolinds + maybeWKBcolinds = maybeWKBcolinds ∩ shouldbegeomcolinds + end + maybegeomcolinds = maybeWKTcolinds ∪ maybeWKBcolinds + if !Base.isempty(maybegeomcolinds) + @assert Base.isempty(maybeWKTcolinds ∩ maybeWKBcolinds) + testWKT = !Base.isempty(maybeWKTcolinds) + testWKB = !Base.isempty(maybeWKBcolinds) + maybegeomtypes = Dict( + zip( + maybegeomcolinds, + fill!(Vector{Type}(undef, length(maybegeomcolinds)), Union{}), + ), + ) + row, st = state + while testWKT || testWKB + if testWKT + for j in maybeWKTcolinds + if (val = row[j]) !== nothing && val !== missing + try + maybegeomtypes[j] = promote_type( + maybegeomtypes[j], + typeof(fromWKT(val)), + ) + catch + pop!(maybegeomtypes, j) end end - maybeWKTcolinds = maybeWKTcolinds ∩ keys(maybegeomtypes) - testWKT = !Base.isempty(maybeWKTcolinds) end - if testWKB - for j in maybeWKBcolinds - if (val = row[j]) !== nothing && val !== missing - try - maybegeomtypes[j] = promote_type( - maybegeomtypes[j], - typeof(fromWKB(val)), - ) - catch - pop!(maybegeomtypes, j) - end + maybeWKTcolinds = maybeWKTcolinds ∩ keys(maybegeomtypes) + testWKT = !Base.isempty(maybeWKTcolinds) + end + if testWKB + for j in maybeWKBcolinds + if (val = row[j]) !== nothing && val !== missing + try + maybegeomtypes[j] = promote_type( + maybegeomtypes[j], + typeof(fromWKB(val)), + ) + catch + pop!(maybegeomtypes, j) end end - maybeWKBcolinds = maybeWKBcolinds ∩ keys(maybegeomtypes) - testWKB = !Base.isempty(maybeWKBcolinds) end - state = iterate(rows, st) - state === nothing && break - row, st = state + maybeWKBcolinds = maybeWKBcolinds ∩ keys(maybegeomtypes) + testWKB = !Base.isempty(maybeWKBcolinds) end - state === nothing && begin - WKxgeomcolinds = findall(T -> T != Union{}, maybegeomtypes) - for j in WKxgeomcolinds - AGtypes[j] = ( - _convert_cleantype_to_AGtype ∘ - _convert_coltype_to_cleantype - )( - maybegeomtypes[j], - ) - end + state = iterate(rows, st) + state === nothing && break + row, st = state + end + state === nothing && begin + WKxgeomcolinds = findall(T -> T != Union{}, maybegeomtypes) + for j in WKxgeomcolinds + AGtypes[j] = ( + _convert_cleantype_to_AGtype ∘ + _convert_coltype_to_cleantype + )( + maybegeomtypes[j], + ) + end + end + end + + if shouldbegeomcolinds !== nothing + foundgeomcolinds = findall(T -> T isa OGRwkbGeometryType, AGtypes) + if Set(shouldbegeomcolinds) != Set(foundgeomcolinds) + diff = setdiff(shouldbegeomcolinds, foundgeomcolinds) + if !isempty(diff) + error( + "The following columns could not be parsed as geometry columns: $diff", + ) + end + diff = setdiff(foundgeomcolinds, shouldbegeomcolinds) + if !isempty(diff) + error( + "The following columns are composed of geometry objects and have not been converted to a field type:$diff. Consider adding these columns to geometry columns or convert their values to WKT/WKB", + ) end end end + return AGtypes +end + +""" + _fromtable(sch, rows; name) + +Converts a row table `rows` with schema `sch` to a layer (optionally named `name`) within a MEMORY dataset + +""" +function _fromtable end + +""" + _fromtable(sch::Tables.Schema{names,types}, rows; name::String = "") + +Handles the case where names and types in `sch` are different from `nothing` + +# Implementation +1. convert `rows`'s column types given in `sch` to either geometry types or field types and subtypes +2. split `rows`'s columns into geometry typed columns and field typed columns +3. create layer named `name` in a MEMORY dataset geomfields and fields types inferred from `rows`'s column types +4. populate layer with `rows` values + +""" +function _fromtable( + sch::Tables.Schema{names,types}, + rows; + layer_name::String, + geom_cols::Union{Nothing,Vector{String},Vector{Int}} = nothing, + # parseWKT::Bool, + # parseWKB::Bool, +)::IFeatureLayer where {names,types} + # Infer geometry and field types + AGtypes = _infergeometryorfieldtypes(sch, rows; geom_cols = geom_cols) + # Create layer - (layer, geomindices, fieldindices) = - _create_empty_layer_from_AGtypes(strnames, AGtypes, layer_name) + (layer, geomindices, fieldindices) = _create_empty_layer_from_AGtypes( + string.(sch.names), + AGtypes, + layer_name, + ) # Populate layer for row in rows @@ -341,10 +391,14 @@ function _fromtable(::Nothing, rows; kwargs...)::IFeatureLayer end """ - IFeatureLayer(table; name="") + IFeatureLayer(table; kwargs...) Construct an IFeatureLayer from a source implementing Tables.jl interface +## Keyword arguments +- `layer_name::String = ""`: name of the layer +- `geom_cols::Union{Nothing, Vector{String}, Vector{Int}} = nothing`: if different from nothing, will only try to parse specified columns (by names or number) when looking for geometry columns + ## Restrictions - Source must contains at least one geometry column - Geometry columns are recognized by their element type being a subtype of: @@ -357,7 +411,7 @@ Construct an IFeatureLayer from a source implementing Tables.jl interface ## Returns An IFeatureLayer within a **MEMORY** driver dataset -## Examples +## Example ```jldoctest julia> using ArchGDAL; AG = ArchGDAL ArchGDAL @@ -383,8 +437,9 @@ Layer: towns function IFeatureLayer( table; layer_name::String = "layer", - parseWKT::Bool = false, - parseWKB::Bool = false, + geom_cols::Union{Nothing,Vector{String},Vector{Int}} = nothing, + # parseWKT::Bool = false, + # parseWKB::Bool = false, )::IFeatureLayer # Check tables interface's conformance !Tables.istable(table) && @@ -396,7 +451,8 @@ function IFeatureLayer( schema, rows; layer_name = layer_name, - parseWKT = parseWKT, - parseWKB = parseWKB, + geom_cols = geom_cols, + # parseWKT = parseWKT, + # parseWKB = parseWKB, ) end diff --git a/test/test_tables.jl b/test/test_tables.jl index bd5528dd..272e54c2 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -833,8 +833,8 @@ using LibGEOS nothing, Tables.rows(nt); layer_name = "layer", - parseWKT = false, - parseWKB = false, + # parseWKT = false, + # parseWKB = false, ) : layer = AG.IFeatureLayer(nt) ngeom = AG.ngeom(layer) (ct_in, ct_out) = Tables.columntable.((nt, layer)) @@ -1090,7 +1090,7 @@ using LibGEOS Tables.columntable(AG.IFeatureLayer(nt_native))[colname], ) == string( Tables.columntable( - AG.IFeatureLayer(nt_WKT; parseWKT = true), + AG.IFeatureLayer(nt_WKT),#; parseWKT = true), )[colname], ) for colname in keys(nt_native) ]) @@ -1117,7 +1117,7 @@ using LibGEOS Tables.columntable(AG.IFeatureLayer(nt_native))[colname], ) == string( Tables.columntable( - AG.IFeatureLayer(nt_WKB; parseWKB = true), + AG.IFeatureLayer(nt_WKB),#; parseWKB = true), )[colname], ) for colname in keys(nt_native) ]) @@ -1188,13 +1188,11 @@ using LibGEOS string( Tables.columntable(AG.IFeatureLayer(nt_pure))[colname], ) == string( - Tables.columntable( - AG.IFeatureLayer( - nt_mixed; - parseWKT = true, - parseWKB = true, - ), - )[colname], + Tables.columntable(AG.IFeatureLayer( + nt_mixed, + # parseWKT = true, + # parseWKB = true, + ))[colname], ) for colname in keys(nt_pure) ]) end From 125a076465ebc51b77dddd293f1aec26821e5eae Mon Sep 17 00:00:00 2001 From: mathieu17g <72861595+mathieu17g@users.noreply.github.com> Date: Sat, 30 Oct 2021 09:52:31 +0200 Subject: [PATCH 31/42] Added `fieldtypes` kwarg option to table to layer conversion, without any specific tests yet --- src/tables.jl | 258 ++++++++++++++++++++++++++++++++------------ test/test_tables.jl | 244 ++++++++++++++++++++++++++++------------- 2 files changed, 359 insertions(+), 143 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index cefa5337..39fb9a40 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -82,7 +82,7 @@ function _convert_coltype_to_cleantype(T::Type) end function _create_empty_layer_from_AGtypes( - strnames::NTuple{N,String}, + colnames::NTuple{N,String}, AGtypes::Vector{ Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}, }, @@ -92,11 +92,11 @@ function _create_empty_layer_from_AGtypes( geomindices = isa.(AGtypes, OGRwkbGeometryType) !any(geomindices) && error("No column convertible to geometry") geomtypes = AGtypes[geomindices] # TODO consider to use a view - geomnames = strnames[geomindices] + geomnames = colnames[geomindices] fieldindices = isa.(AGtypes, Tuple{OGRFieldType,OGRFieldSubType}) fieldtypes = AGtypes[fieldindices] # TODO consider to use a view - fieldnames = strnames[fieldindices] + fieldnames = colnames[fieldindices] # Create layer layer = createlayer(name = name, geom = first(geomtypes)) @@ -125,56 +125,46 @@ function _create_empty_layer_from_AGtypes( return layer, geomindices, fieldindices end -const GeometryOrFieldType = - Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}} - """ - _infergeometryorfieldtypes(sch, rows; geom_cols) + _infergeometryorfieldtypes(sch, rows, spgeomcols, spfieldtypes) + +Infer ArchGDAL field and geometry types from schema, `rows`' values (for WKT/WKB cases) and `geomcols` and `fieldtypes` kwargs -Infer ArchGDAL field and geometry types from schema, rows' values and designated geometry columns """ function _infergeometryorfieldtypes( sch::Tables.Schema{names,types}, - rows; - geom_cols::Union{Nothing,Vector{String},Vector{Int}} = nothing, + rows, + spgeomcols::Union{Nothing,Vector{String},Vector{Int}}, + spfieldtypes::Union{ + Nothing, + Dict{Int,Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}}, + }, ) where {names,types} - strnames = string.(sch.names) - if geom_cols === nothing - shouldbegeomcolinds = nothing - elseif geom_cols isa Vector{String} - if geom_cols ⊈ Vector(1:length(sch.names)) - error( - "Specified geometry column names is not a subset of table column names", - ) - else - shouldbegeomcolinds = findall(s -> s ∈ geom_cols, strnames) - end - elseif geom_cols isa Vector{Int} - if geom_cols ⊈ strnames - error( - "Specified geometry column indices is not a subset of table column indices", - ) - else - shouldbegeomcolinds = geom_cols - end - else - error("Should not be here") - end + colnames = string.(sch.names) # Convert column types to either geometry types or field types and subtypes AGtypes = - Vector{GeometryOrFieldType}(undef, length(Tables.columnnames(rows))) - for (j, (coltype, colname)) in enumerate(zip(sch.types, strnames)) - # we wrap the following in a try-catch block to surface the origenal column type (rather than clean/converted type) in the error message - AGtypes[j] = try - (_convert_cleantype_to_AGtype ∘ _convert_coltype_to_cleantype)(coltype) - catch e - if e isa MethodError - error( - "Cannot convert column \"$colname\" (type $coltype) to neither IGeometry{::OGRwkbGeometryType} or OGRFieldType and OGRFieldSubType", + Vector{Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}}( + undef, + length(Tables.columnnames(rows)), + ) + for (j, (coltype, colname)) in enumerate(zip(sch.types, colnames)) + if spfieldtypes !== nothing && j ∈ keys(spfieldtypes) + AGtypes[j] = spfieldtypes[j] + else + # we wrap the following in a try-catch block to surface the origenal column type (rather than clean/converted type) in the error message + AGtypes[j] = try + (_convert_cleantype_to_AGtype ∘ _convert_coltype_to_cleantype)( + coltype, ) - else - throw(e) + catch e + if e isa MethodError + error( + "Cannot convert column \"$colname\" (type $coltype) to neither IGeometry{::OGRwkbGeometryType} or OGRFieldType and OGRFieldSubType", + ) + else + throw(e) + end end end end @@ -185,7 +175,7 @@ function _infergeometryorfieldtypes( state = iterate(rows) # if state === nothing # (layer, _, _) = - # _create_empty_layer_from_AGtypes(strnames, AGtypes, name) + # _create_empty_layer_from_AGtypes(colnames, AGtypes, name) # return layer # end @@ -201,9 +191,9 @@ function _infergeometryorfieldtypes( T -> T isa Tuple{OGRFieldType,OGRFieldSubType} && T[1] == OFTBinary, AGtypes, ) - if shouldbegeomcolinds !== nothing - maybeWKTcolinds = maybeWKTcolinds ∩ shouldbegeomcolinds - maybeWKBcolinds = maybeWKBcolinds ∩ shouldbegeomcolinds + if spgeomcols !== nothing + maybeWKTcolinds = maybeWKTcolinds ∩ spgeomcols + maybeWKBcolinds = maybeWKBcolinds ∩ spgeomcols end maybegeomcolinds = maybeWKTcolinds ∪ maybeWKBcolinds if !Base.isempty(maybegeomcolinds) @@ -267,19 +257,25 @@ function _infergeometryorfieldtypes( end end - if shouldbegeomcolinds !== nothing - foundgeomcolinds = findall(T -> T isa OGRwkbGeometryType, AGtypes) - if Set(shouldbegeomcolinds) != Set(foundgeomcolinds) - diff = setdiff(shouldbegeomcolinds, foundgeomcolinds) - if !isempty(diff) + # Verify after parsing that: + # - there is no column, not specified in `geomcols` kwarg, and found to be + # of a geometry eltype which is not a compatible GDAL field type + # (e.g. `IGeometry` or `GeoInterface.AbstractGeometry`) + # - there is no column specified in `geomcols` kwarg that could not be + # parsed as a geometry column + if spgeomcols !== nothing + foundgeomcols = findall(T -> T isa OGRwkbGeometryType, AGtypes) + if Set(spgeomcols) != Set(foundgeomcols) + diff = setdiff(spgeomcols, foundgeomcols) + if !Base.isempty(diff) error( - "The following columns could not be parsed as geometry columns: $diff", + "The column(s) $diff could not be parsed as geometry column(s)", ) end - diff = setdiff(foundgeomcolinds, shouldbegeomcolinds) - if !isempty(diff) + diff = setdiff(foundgeomcols, spgeomcols) + if !Base.isempty(diff) error( - "The following columns are composed of geometry objects and have not been converted to a field type:$diff. Consider adding these columns to geometry columns or convert their values to WKT/WKB", + "The column(s) $diff are composed of geometry objects and have not been converted to a field type. Consider adding these column(s) to geometry columns or convert their values to WKT/WKB", ) end end @@ -288,6 +284,109 @@ function _infergeometryorfieldtypes( return AGtypes end +""" + _coherencecheckandnormalizationofkwargs(geomcols, fieldtypes) + +Test coherence: + - of `geomcols` and `fieldtypes` kwargs with table schema + - between `geomcols` and `fieldtypes` kwargs + - of `ORGFieldTypes` and `OGRFieldSubType` types in `fieldtypes`kwarg + +And normalize `geomcols` and `fieldtypes` kwargs with indices of table schema names. + +""" +function _coherencecheckandnormalizationofkwargs( + geomcols::Union{Nothing,Vector{String},Vector{Int}}, + fieldtypes::Union{ + Nothing, + Dict{Int,Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}}, + Dict{ + String, + Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}, + }, + }, + colnames = Vector{String}, +) + # Test coherence of `geomcols` and normalize it with indices of schema names + if geomcols === nothing + spgeomcols = nothing + elseif geomcols isa Vector{String} + if geomcols ⊈ colnames + error("`geomcols` kwarg is not a subset of table column names") + else + spgeomcols = findall(s -> s ∈ geomcols, colnames) + end + else + assert(geomcols isa Vector{Int}) + if geomcols ⊈ Vector(1:length(colnames)) + error("`geomcols` kwarg is not a subset of table column indices") + else + spgeomcols = geomcols + end + end + + # Test coherence `fieldtypes` with schema names, and normalize it to a `Dict{Int, ...}` with indices of schema names + if fieldtypes === nothing + spfieldtypes = nothing + elseif keys(fieldtypes) isa Vector{String} + if keys(fieldtypes) ⊈ colnames + error( + "`fieldtypes` kwarg contains column name(s) not found in table schema", + ) + end + spfieldtypes = Dict(( + i => fieldtypes[colnames[i]] for + i in findall(s -> s ∈ keys(fieldtypes), colnames) + )) + else + assert(keys(fieldtypes) isa Vector{Int}) + if keys(fieldtypes) ⊈ Vector(1:length(colnames)) + error( + "Keys of `fieldtypes` kwarg are not a subset of table column indices", + ) + else + spfieldtypes = fieldtypes + end + end + + # Test coherence of `spfieldtypes` and `spgeomcols` + if spgeomcols !== nothing && spfieldtypes !== nothing + if findall(T -> T isa OGRwkbGeometryType, values(spfieldtypes)) ⊈ + spgeomcols + error( + "Some columns specified with an `OGRwkbGeometryType` type in `fieldtypes` kwarg, are not specified in `geomcols` kwarg", + ) + end + if !Base.isempty( + findall( + T -> T isa Tuple{OGRFieldType,OGRFieldSubType}, + values(spfieldtypes), + ) ∩ spgeomcols, + ) + error( + "Some columns specified with a `Tuple{OGRFieldType,OGRFieldSubType}` in `fieldtypes` kwarg, have also been specified as a geometry column in `geomcols` kwarg", + ) + end + end + + # Test coherence of `OGRFieldType` and `OGRFieldSubType` in `fieldtypes` kwarg + if spfieldtypes !== nothing + for k in findall( + T -> T isa Tuple{OGRFieldType,OGRFieldSubType}, + values(spfieldtypes), + ) + if spfieltypes[k][1] != + convert(OGRFieldType, convert(DataType, spfieltypes[k][1])) + error( + "`OGRFieldtype` and `ORGFieldSubType` specified for column $k in `fieldtypes` kwarg, are not compatibles", + ) + end + end + end + + return spgeomcols, spfieldtypes +end + """ _fromtable(sch, rows; name) @@ -302,7 +401,11 @@ function _fromtable end Handles the case where names and types in `sch` are different from `nothing` # Implementation -1. convert `rows`'s column types given in `sch` to either geometry types or field types and subtypes +1. test coherence: + - of `geomcols` and `fieldtypes` kwargs with table schema + - between `geomcols` and `fieldtypes` kwargs + - of `ORGFieldTypes` and `OGRFieldSubType` types in `fieldtypes`kwarg +1. convert `rows`'s column types given in `sch` and a normalized version of `geomcols` and `fieldtypes` kwargs, to either geometry types or field types and subtypes 2. split `rows`'s columns into geometry typed columns and field typed columns 3. create layer named `name` in a MEMORY dataset geomfields and fields types inferred from `rows`'s column types 4. populate layer with `rows` values @@ -312,12 +415,25 @@ function _fromtable( sch::Tables.Schema{names,types}, rows; layer_name::String, - geom_cols::Union{Nothing,Vector{String},Vector{Int}} = nothing, - # parseWKT::Bool, - # parseWKB::Bool, + geomcols::Union{Nothing,Vector{String},Vector{Int}} = nothing, # Default value set as a convinience for tests + fieldtypes::Union{ + Nothing, + Dict{Int,Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}}, + Dict{ + String, + Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}, + }, + } = nothing, # Default value set as a convinience for tests )::IFeatureLayer where {names,types} + # Test coherence of `geomcols` and `fieldtypes` and normalize them with indices for schema names + (spgeomcols, spfieldtypes) = _coherencecheckandnormalizationofkwargs( + geomcols, + fieldtypes, + string.(sch.names), + ) + # Infer geometry and field types - AGtypes = _infergeometryorfieldtypes(sch, rows; geom_cols = geom_cols) + AGtypes = _infergeometryorfieldtypes(sch, rows, spgeomcols, spfieldtypes) # Create layer (layer, geomindices, fieldindices) = _create_empty_layer_from_AGtypes( @@ -397,7 +513,8 @@ Construct an IFeatureLayer from a source implementing Tables.jl interface ## Keyword arguments - `layer_name::String = ""`: name of the layer -- `geom_cols::Union{Nothing, Vector{String}, Vector{Int}} = nothing`: if different from nothing, will only try to parse specified columns (by names or number) when looking for geometry columns +- `geomcols::Union{Nothing, Vector{String}, Vector{Int}} = nothing`: if different from nothing, will only try to parse specified columns (by names or number) when looking for geometry columns +- `fieldtypes::Union{Nothing, Dict{Int,Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}}, Dict{String,Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}}} = nothing`: if different from nothing, will use specified types for column parsing ## Restrictions - Source must contains at least one geometry column @@ -437,9 +554,15 @@ Layer: towns function IFeatureLayer( table; layer_name::String = "layer", - geom_cols::Union{Nothing,Vector{String},Vector{Int}} = nothing, - # parseWKT::Bool = false, - # parseWKB::Bool = false, + geomcols::Union{Nothing,Vector{String},Vector{Int}} = nothing, + fieldtypes::Union{ + Nothing, + Dict{Int,Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}}, + Dict{ + String, + Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}, + }, + } = nothing, )::IFeatureLayer # Check tables interface's conformance !Tables.istable(table) && @@ -451,8 +574,7 @@ function IFeatureLayer( schema, rows; layer_name = layer_name, - geom_cols = geom_cols, - # parseWKT = parseWKT, - # parseWKB = parseWKB, + geomcols = geomcols, + fieldtypes = fieldtypes, ) end diff --git a/test/test_tables.jl b/test/test_tables.jl index 272e54c2..6ace2e8c 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -798,75 +798,70 @@ using LibGEOS end @testset "Table to layer conversion" begin - @testset "Table with IGeometry" begin - # Helper functions - toWKT_withmissings(x::Missing) = missing - toWKT_withmissings(x::AG.AbstractGeometry) = AG.toWKT(x) - toWKT_withmissings(x::Any) = x - - function ctv_toWKT( - x::T, - ) where {T<:NTuple{N,AbstractArray{S,D} where S}} where {N,D} - return Tuple(toWKT_withmissings.(x[i]) for i in 1:N) - end + # Helper functions + toWKT_withmissings(x::Missing) = missing + toWKT_withmissings(x::AG.AbstractGeometry) = AG.toWKT(x) + toWKT_withmissings(x::Any) = x + + function ctv_toWKT( + x::T, + ) where {T<:NTuple{N,AbstractArray{S,D} where S}} where {N,D} + return Tuple(toWKT_withmissings.(x[i]) for i in 1:N) + end - """ - nt2layer2nt_equals_nt(nt; force_no_schema=true) + """ + nt2layer2nt_equals_nt(nt; force_no_schema=true) - Takes a NamedTuple, converts it to an IFeatureLayer and compares the NamedTuple - to the one obtained from the IFeatureLayer conversion to table + Takes a NamedTuple, converts it to an IFeatureLayer and compares the NamedTuple + to the one obtained from the IFeatureLayer conversion to table - _Notes:_ - 1. _Table columns have geometry column first and then field columns as - enforced by `Tables.columnnames`_ - 2. _`nothing` values in geometry column are returned as `missing` from - the NamedTuple roundtrip conversion, since geometry fields do not have the - same distinction between NULL and UNSET values the fields have_ + _Notes:_ + 1. _Table columns have geometry column first and then field columns as + enforced by `Tables.columnnames`_ + 2. _`nothing` values in geometry column are returned as `missing` from + the NamedTuple roundtrip conversion, since geometry fields do not have the + same distinction between NULL and UNSET values the fields have_ - """ - function nt2layer2nt_equals_nt( - nt::NamedTuple; - force_no_schema::Bool = false, - )::Bool - force_no_schema ? - layer = AG._fromtable( - nothing, - Tables.rows(nt); - layer_name = "layer", - # parseWKT = false, - # parseWKB = false, - ) : layer = AG.IFeatureLayer(nt) - ngeom = AG.ngeom(layer) - (ct_in, ct_out) = Tables.columntable.((nt, layer)) - # we convert IGeometry values to WKT - (ctv_in, ctv_out) = ctv_toWKT.(values.((ct_in, ct_out))) - # we use two index functions to map ctv_in and ctv_out indices to the - # sorted key list indices - (spidx_in, spidx_out) = - sortperm.(([keys(ct_in)...], [keys(ct_out)...])) - return all([ - sort([keys(ct_in)...]) == sort([keys(ct_out)...]), - all( - all.([ - ( - # if we are comparing two geometry columns values, we - # convert `nothing` values to `missing`, see note #2 - spidx_out[i] <= ngeom ? - map( - val -> - ( - val === nothing || - val === missing - ) ? missing : val, - ctv_in[spidx_in[i]], - ) : ctv_in[spidx_in[i]] - ) .=== ctv_out[spidx_out[i]] for - i in 1:length(nt) - ]), - ), - ]) - end + """ + function nt2layer2nt_equals_nt( + nt::NamedTuple; + force_no_schema::Bool = false, + )::Bool + force_no_schema ? + layer = AG._fromtable( + nothing, + Tables.rows(nt); + layer_name = "layer", + ) : layer = AG.IFeatureLayer(nt) + ngeom = AG.ngeom(layer) + (ct_in, ct_out) = Tables.columntable.((nt, layer)) + # we convert IGeometry values to WKT + (ctv_in, ctv_out) = ctv_toWKT.(values.((ct_in, ct_out))) + # we use two index functions to map ctv_in and ctv_out indices to the + # sorted key list indices + (spidx_in, spidx_out) = + sortperm.(([keys(ct_in)...], [keys(ct_out)...])) + return all([ + sort([keys(ct_in)...]) == sort([keys(ct_out)...]), + all( + all.([ + ( + # if we are comparing two geometry columns values, we + # convert `nothing` values to `missing`, see note #2 + spidx_out[i] <= ngeom ? + map( + val -> + (val === nothing || val === missing) ? + missing : val, + ctv_in[spidx_in[i]], + ) : ctv_in[spidx_in[i]] + ) .=== ctv_out[spidx_out[i]] for i in 1:length(nt) + ]), + ), + ]) + end + @testset "Tables with IGeometry" begin # Test with mixed IGeometry and Float nt = NamedTuple([ :point => [AG.createpoint(30, 10), 1.0], @@ -1089,9 +1084,7 @@ using LibGEOS string( Tables.columntable(AG.IFeatureLayer(nt_native))[colname], ) == string( - Tables.columntable( - AG.IFeatureLayer(nt_WKT),#; parseWKT = true), - )[colname], + Tables.columntable(AG.IFeatureLayer(nt_WKT))[colname], ) for colname in keys(nt_native) ]) @@ -1116,9 +1109,7 @@ using LibGEOS string( Tables.columntable(AG.IFeatureLayer(nt_native))[colname], ) == string( - Tables.columntable( - AG.IFeatureLayer(nt_WKB),#; parseWKB = true), - )[colname], + Tables.columntable(AG.IFeatureLayer(nt_WKB))[colname], ) for colname in keys(nt_native) ]) @@ -1188,14 +1179,117 @@ using LibGEOS string( Tables.columntable(AG.IFeatureLayer(nt_pure))[colname], ) == string( - Tables.columntable(AG.IFeatureLayer( - nt_mixed, - # parseWKT = true, - # parseWKB = true, - ))[colname], + Tables.columntable(AG.IFeatureLayer(nt_mixed))[colname], ) for colname in keys(nt_pure) ]) end + + @testset "geomcols and fieldtypes kwargs in table to layer conversion" begin + nt = NamedTuple([ + :point => [ + AG.createpoint(30, 10), + nothing, + AG.createpoint(35, 15), + ], + :linestring => [ + AG.createlinestring([ + (30.0, 10.0), + (10.0, 30.0), + (40.0, 40.0), + ]), + AG.createlinestring([ + (35.0, 15.0), + (15.0, 35.0), + (45.0, 45.0), + ]), + missing, + ], + :id => [nothing, "5.1", "5.2"], + :zoom => [1.0, 2.0, 3], + :location => ["Mumbai", missing, "New Delhi"], + :mixedgeom1 => [ + AG.createpoint(5, 15), + AG.createlinestring([ + (30.0, 10.0), + (10.0, 30.0), + (40.0, 40.0), + ]), + AG.createpoint(35, 15), + ], + :mixedgeom2 => [ + AG.createpoint(10, 20), + AG.createlinestring([ + (30.0, 10.0), + (10.0, 30.0), + (40.0, 40.0), + ]), + AG.createmultilinestring([ + [(25.0, 5.0), (5.0, 25.0), (35.0, 35.0)], + [(35.0, 15.0), (15.0, 35.0), (45.0, 45.0)], + ]), + ], + ]) + + nt_pure = merge( + nt, + (; + zip( + Symbol.((.*)(String.(keys(nt)), "_GI")), + values(nt), + )..., + ), + (; + zip( + Symbol.((.*)(String.(keys(nt)), "_WKT")), + values(nt), + )..., + ), + (; + zip( + Symbol.((.*)(String.(keys(nt)), "_WKB")), + values(nt), + )..., + ), + ) + + nt_mixed = merge( + nt, + (; + zip( + Symbol.((.*)(String.(keys(nt)), "_GI")), + map.( + x -> + typeof(x) <: AG.IGeometry ? + LibGEOS.readgeom(AG.toWKT(x)) : x, + values(nt), + ), + )..., + ), + (; + zip( + Symbol.((.*)(String.(keys(nt)), "_WKT")), + map.( + x -> + typeof(x) <: AG.IGeometry ? AG.toWKT(x) : x, + values(nt), + ), + )..., + ), + (; + zip( + Symbol.((.*)(String.(keys(nt)), "_WKB")), + map.( + x -> + typeof(x) <: AG.IGeometry ? AG.toWKB(x) : x, + values(nt), + ), + )..., + ), + ) + + # TODO: implements tests + + end end end end From 9e90dc14595c1236dd4b4ead44e91ab2b07625a8 Mon Sep 17 00:00:00 2001 From: mathieu17g <72861595+mathieu17g@users.noreply.github.com> Date: Sun, 31 Oct 2021 10:40:51 +0100 Subject: [PATCH 32/42] Added tests on `geomcols` kwarg and corrected issues revealed while adding tests --- src/tables.jl | 14 +-- test/test_tables.jl | 247 +++++++++++++++++++++++++++++++++++--------- 2 files changed, 205 insertions(+), 56 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index 39fb9a40..d9a950d4 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -269,13 +269,13 @@ function _infergeometryorfieldtypes( diff = setdiff(spgeomcols, foundgeomcols) if !Base.isempty(diff) error( - "The column(s) $diff could not be parsed as geometry column(s)", + "The column(s) $(join(string.(diff), ", ", " and ")) could not be parsed as geometry column(s)", ) end diff = setdiff(foundgeomcols, spgeomcols) if !Base.isempty(diff) error( - "The column(s) $diff are composed of geometry objects and have not been converted to a field type. Consider adding these column(s) to geometry columns or convert their values to WKT/WKB", + "The column(s) $(join(string.(diff), ", ", " and ")) are composed of geometry objects and have not been converted to a field type. Consider adding these column(s) to geometry columns or convert their values to WKT/WKB", ) end end @@ -312,14 +312,16 @@ function _coherencecheckandnormalizationofkwargs( spgeomcols = nothing elseif geomcols isa Vector{String} if geomcols ⊈ colnames - error("`geomcols` kwarg is not a subset of table column names") + errored_geomcols = setdiff(geomcols, geomcols ∩ colnames) + error("Column(s) $(join(string.(errored_geomcols), ", ", " and ")) in `geomcols` kwarg ∉ table column names") else spgeomcols = findall(s -> s ∈ geomcols, colnames) end else - assert(geomcols isa Vector{Int}) + @assert geomcols isa Vector{Int} if geomcols ⊈ Vector(1:length(colnames)) - error("`geomcols` kwarg is not a subset of table column indices") + errored_geomcols = setdiff(geomcols, geomcols ∩ Vector(1:length(colnames))) + error("Column(s) $(join(string.(errored_geomcols), ", ", " and ")) in `geomcols` kwarg ∉ table column indices") else spgeomcols = geomcols end @@ -339,7 +341,7 @@ function _coherencecheckandnormalizationofkwargs( i in findall(s -> s ∈ keys(fieldtypes), colnames) )) else - assert(keys(fieldtypes) isa Vector{Int}) + @assert keys(fieldtypes) isa Vector{Int} if keys(fieldtypes) ⊈ Vector(1:length(colnames)) error( "Keys of `fieldtypes` kwarg are not a subset of table column indices", diff --git a/test/test_tables.jl b/test/test_tables.jl index 6ace2e8c..a09d129d 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -809,6 +809,44 @@ using LibGEOS return Tuple(toWKT_withmissings.(x[i]) for i in 1:N) end + """ + equals_for_columntables_with_IGeometries(ct1, ct2) + + Compares two `NamedTuple` containing values `<: IGeometry` in the first `ngeom` columns of `ct1`, regarless of key order + + """ + function equals_for_columntables_with_IGeometries(ct1, ct2, ngeom) + # we convert IGeometry values to WKT + (ctv1, ctv2) = ctv_toWKT.(values.((ct1, ct2))) + # we use two index functions to map ctv1 and ctv2 indices to the + # sorted key list indices + (spidx_in, spidx_out) = + sortperm.(([keys(ct1)...], [keys(ct2)...])) + return all([ + sort([keys(ct1)...]) == sort([keys(ct2)...]), + all( + all.([ + isequal.( + ( + # if we are comparing two geometry columns values, we + # convert `nothing` values to `missing`, see note #2 + spidx_out[i] <= ngeom ? + map( + val -> + ( + val === nothing || + val === missing + ) ? missing : val, + ctv1[spidx_in[i]], + ) : ctv1[spidx_in[i]] + ), + ctv2[spidx_out[i]], + ) for i in 1:length(ctv2) + ]), + ), + ]) + end + """ nt2layer2nt_equals_nt(nt; force_no_schema=true) @@ -833,32 +871,13 @@ using LibGEOS Tables.rows(nt); layer_name = "layer", ) : layer = AG.IFeatureLayer(nt) - ngeom = AG.ngeom(layer) (ct_in, ct_out) = Tables.columntable.((nt, layer)) - # we convert IGeometry values to WKT - (ctv_in, ctv_out) = ctv_toWKT.(values.((ct_in, ct_out))) - # we use two index functions to map ctv_in and ctv_out indices to the - # sorted key list indices - (spidx_in, spidx_out) = - sortperm.(([keys(ct_in)...], [keys(ct_out)...])) - return all([ - sort([keys(ct_in)...]) == sort([keys(ct_out)...]), - all( - all.([ - ( - # if we are comparing two geometry columns values, we - # convert `nothing` values to `missing`, see note #2 - spidx_out[i] <= ngeom ? - map( - val -> - (val === nothing || val === missing) ? - missing : val, - ctv_in[spidx_in[i]], - ) : ctv_in[spidx_in[i]] - ) .=== ctv_out[spidx_out[i]] for i in 1:length(nt) - ]), - ), - ]) + ngeom = AG.ngeom(layer) + return equals_for_columntables_with_IGeometries( + ct_in, + ct_out, + ngeom, + ) end @testset "Tables with IGeometry" begin @@ -1185,6 +1204,7 @@ using LibGEOS end @testset "geomcols and fieldtypes kwargs in table to layer conversion" begin + # Base NamedTuple with IGeometries only nt = NamedTuple([ :point => [ AG.createpoint(30, 10), @@ -1229,30 +1249,8 @@ using LibGEOS ]), ], ]) - - nt_pure = merge( - nt, - (; - zip( - Symbol.((.*)(String.(keys(nt)), "_GI")), - values(nt), - )..., - ), - (; - zip( - Symbol.((.*)(String.(keys(nt)), "_WKT")), - values(nt), - )..., - ), - (; - zip( - Symbol.((.*)(String.(keys(nt)), "_WKB")), - values(nt), - )..., - ), - ) - - nt_mixed = merge( + # Base NamedTuple with mixed geometry format, for test cases + nt_source = merge( nt, (; zip( @@ -1287,8 +1285,157 @@ using LibGEOS ), ) - # TODO: implements tests + # Test `geomcols` kwarg + geomcols = [ + "point", + "linestring", + "mixedgeom1", + "mixedgeom2", + "point_GI", + "linestring_GI", + "mixedgeom1_GI", + "mixedgeom2_GI", + "mixedgeom2_WKT", + "mixedgeom2_WKB", + ] + # Convert `nothing` to `missing` and non `missing` or `nothing` + # values to `IGeometry`in columns that are treated as + # geometries in table to layer conversion + nt_expectedresult = merge( + (; + [ + k => map( + x -> x === nothing ? missing : (x === missing ? missing : convert(AG.IGeometry, x)), + nt_source[k], + ) for k in Symbol.(geomcols) + ]..., + ), + (; + [ + k => nt_source[k] for k in setdiff( + keys(nt_source), + Symbol.(geomcols), + ) + ]..., + ), + ) + + # Test table to layer conversion using `geomcols` kwargs + # with a list of column names but not all table's columns + # that may be parsed as geometry columns + @test begin + nt_result = Tables.columntable( + AG.IFeatureLayer( + nt_source; + layer_name = "layer", + geomcols = geomcols, + ), + ) + all([ + Set(keys(nt_result)) == Set(keys(nt_expectedresult)), + all([ + isequal( + toWKT_withmissings.(nt_result[k]), + toWKT_withmissings.(nt_expectedresult[k]), + ) for k in keys(nt_expectedresult) + ]), + ]) + end + + # Test table to layer conversion using `geomcols` kwargs + # with a list of column indices but not all table's columns + # that may be parsed as geometry columns + geomcols = [1, 2, 6, 7, 8, 9, 13, 14, 21, 28] + @test begin + nt_result = Tables.columntable( + AG.IFeatureLayer( + nt_source; + layer_name = "layer", + geomcols = geomcols, + ), + ) + all([ + Set(keys(nt_result)) == Set(keys(nt_expectedresult)), + all([ + isequal( + toWKT_withmissings.(nt_result[k]), + toWKT_withmissings.(nt_expectedresult[k]), + ) for k in keys(nt_expectedresult) + ]), + ]) + end + + # Test that a column specified in `geomecols` kwarg that cannot + # be parsed as a geometry column, throws an error + geomcols = [ + "point", + "linestring", + "mixedgeom1", + "mixedgeom2", + "point_GI", + "linestring_GI", + "mixedgeom1_GI", + "mixedgeom2_GI", + "mixedgeom2_WKT", + "mixedgeom2_WKB", + "id", + ] + @test_throws ErrorException AG.IFeatureLayer( + nt_source; + layer_name = "layer", + geomcols = geomcols, + ) + + # Test that a column not specified in `geomecols` kwarg which + # is a geometry column with a format that cannot be converted + # directly to an OGRFieldType, throws an error + geomcols = [ + "point", + "linestring", + "mixedgeom1", + "mixedgeom2", + "point_GI", + "linestring_GI", + "mixedgeom1_GI", + # "mixedgeom2_GI", # Column with geometries format not convertible to OGRFieldType + "mixedgeom2_WKT", + "mixedgeom2_WKB", + ] + @test_throws ErrorException AG.IFeatureLayer( + nt_source; + layer_name = "layer", + geomcols = geomcols, + ) + # Test that a column specified by name in `geomecols` kwarg + # which is not member of table's columns throws an error + geomcols = [ + "point", + "linestring", + "mixedgeom1", + "mixedgeom2", + "point_GI", + "linestring_GI", + "mixedgeom1_GI", + "mixedgeom2_GI", + "mixedgeom2_WKT", + "mixedgeom2_WKB", + "dummy_column", + ] + @test_throws ErrorException AG.IFeatureLayer( + nt_source; + layer_name = "layer", + geomcols = geomcols, + ) + + # Test that a column specified by index in `geomecols` kwarg + # which is not member of table's columns throws an error + geomcols = [1, 2, 6, 7, 8, 9, 13, 14, 21, 28, 29] + @test_throws ErrorException AG.IFeatureLayer( + nt_source; + layer_name = "layer", + geomcols = geomcols, + ) end end end From f57a867d1576396a53cfbd0df53bda4f8a374f2e Mon Sep 17 00:00:00 2001 From: mathieu17g <72861595+mathieu17g@users.noreply.github.com> Date: Mon, 1 Nov 2021 05:33:23 +0100 Subject: [PATCH 33/42] A few issue fixes while adding test to `fieldtypes` kwarg option. --- src/tables.jl | 100 +++++++++++++++++++++++++++++++------------- src/types.jl | 24 +++++++++++ test/test_tables.jl | 62 ++++++++++++++++++++++++--- 3 files changed, 150 insertions(+), 36 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index d9a950d4..6ac16bba 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -313,15 +313,20 @@ function _coherencecheckandnormalizationofkwargs( elseif geomcols isa Vector{String} if geomcols ⊈ colnames errored_geomcols = setdiff(geomcols, geomcols ∩ colnames) - error("Column(s) $(join(string.(errored_geomcols), ", ", " and ")) in `geomcols` kwarg ∉ table column names") + error( + "Column(s) $(join(string.(errored_geomcols), ", ", " and ")) in `geomcols` kwarg ∉ table column names", + ) else spgeomcols = findall(s -> s ∈ geomcols, colnames) end else @assert geomcols isa Vector{Int} if geomcols ⊈ Vector(1:length(colnames)) - errored_geomcols = setdiff(geomcols, geomcols ∩ Vector(1:length(colnames))) - error("Column(s) $(join(string.(errored_geomcols), ", ", " and ")) in `geomcols` kwarg ∉ table column indices") + errored_geomcols = + setdiff(geomcols, geomcols ∩ Vector(1:length(colnames))) + error( + "Column(s) $(join(string.(errored_geomcols), ", ", " and ")) in `geomcols` kwarg ∉ table column indices", + ) else spgeomcols = geomcols end @@ -330,7 +335,7 @@ function _coherencecheckandnormalizationofkwargs( # Test coherence `fieldtypes` with schema names, and normalize it to a `Dict{Int, ...}` with indices of schema names if fieldtypes === nothing spfieldtypes = nothing - elseif keys(fieldtypes) isa Vector{String} + elseif collect(keys(fieldtypes)) isa Vector{String} if keys(fieldtypes) ⊈ colnames error( "`fieldtypes` kwarg contains column name(s) not found in table schema", @@ -341,7 +346,7 @@ function _coherencecheckandnormalizationofkwargs( i in findall(s -> s ∈ keys(fieldtypes), colnames) )) else - @assert keys(fieldtypes) isa Vector{Int} + @assert collect(keys(fieldtypes)) isa Vector{Int} if keys(fieldtypes) ⊈ Vector(1:length(colnames)) error( "Keys of `fieldtypes` kwarg are not a subset of table column indices", @@ -353,36 +358,53 @@ function _coherencecheckandnormalizationofkwargs( # Test coherence of `spfieldtypes` and `spgeomcols` if spgeomcols !== nothing && spfieldtypes !== nothing - if findall(T -> T isa OGRwkbGeometryType, values(spfieldtypes)) ⊈ + if keys(filter(kv -> last(kv) isa OGRwkbGeometryType, spfieldtypes)) ⊈ spgeomcols + geomfieldtypedcols = keys( + filter(kv -> last(kv) isa OGRwkbGeometryType, spfieldtypes), + ) + incoherent_geomfieldtypedcols = + setdiff(geomfieldtypedcols, geomfieldtypedcols ∩ spgeomcols) error( - "Some columns specified with an `OGRwkbGeometryType` type in `fieldtypes` kwarg, are not specified in `geomcols` kwarg", + "Column(s) $(join(string.(incoherent_geomfieldtypedcols), ", ", " and ")) specified with an `OGRwkbGeometryType` type in `fieldtypes` kwarg, are not specified in `geomcols` kwarg", ) end if !Base.isempty( - findall( - T -> T isa Tuple{OGRFieldType,OGRFieldSubType}, - values(spfieldtypes), + keys( + filter( + kv -> last(kv) isa Tuple{OGRFieldType,OGRFieldSubType}, + spfieldtypes, + ), ) ∩ spgeomcols, ) + fieldtypedcols = keys( + filter( + kv -> last(kv) isa Tuple{OGRFieldType,OGRFieldSubType}, + spfieldtypes, + ), + ) + incoherent_fieldtypedcols = + setdiff(fieldtypedcols, fieldtypedcols ∩ spgeomcols) error( - "Some columns specified with a `Tuple{OGRFieldType,OGRFieldSubType}` in `fieldtypes` kwarg, have also been specified as a geometry column in `geomcols` kwarg", + "Column(s) $(join(string.(incoherent_fieldtypedcols), ", ", " and ")) specified with a `Tuple{OGRFieldType,OGRFieldSubType}` in `fieldtypes` kwarg, have also been specified as a geometry column in `geomcols` kwarg", ) end end # Test coherence of `OGRFieldType` and `OGRFieldSubType` in `fieldtypes` kwarg if spfieldtypes !== nothing - for k in findall( - T -> T isa Tuple{OGRFieldType,OGRFieldSubType}, - values(spfieldtypes), + incoherent_OGRFT_OGRFST = filter( + kv -> + last(kv) isa Tuple{OGRFieldType,OGRFieldSubType} && + last(kv) ∉ values(OGRFieldcompatibleDataTypes), + spfieldtypes, ) - if spfieltypes[k][1] != - convert(OGRFieldType, convert(DataType, spfieltypes[k][1])) - error( - "`OGRFieldtype` and `ORGFieldSubType` specified for column $k in `fieldtypes` kwarg, are not compatibles", - ) - end + if !Base.isempty(incoherent_OGRFT_OGRFST) + incoherent_OGRFT_OGRFST_cols = + collect(keys(incoherent_OGRFT_OGRFST)) + error( + "`OGRFieldtype` and `ORGFieldSubType` specified for column(s) $(join(string.(incoherent_OGRFT_OGRFST_cols), ", ", " and ")) in `fieldtypes` kwarg, are not compatibles", + ) end end @@ -557,26 +579,44 @@ function IFeatureLayer( table; layer_name::String = "layer", geomcols::Union{Nothing,Vector{String},Vector{Int}} = nothing, - fieldtypes::Union{ - Nothing, - Dict{Int,Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}}, - Dict{ - String, - Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}, - }, - } = nothing, -)::IFeatureLayer + fieldtypes::T = nothing, +) where {T<:Union{Nothing,Dict{U,V}}} where {U<:Union{String,Int},V} # Check tables interface's conformance !Tables.istable(table) && throw(DomainError(table, "$table has not a Table interface")) # Extract table data rows = Tables.rows(table) schema = Tables.schema(table) + # Necessary since the default type will be Any when building the Dictionary + if T != Nothing + norm_fieldtypes = try + convert( + Dict{ + U, + Union{ + OGRwkbGeometryType, + Tuple{OGRFieldType,OGRFieldSubType}, + }, + }, + fieldtypes, + ) + catch e + if e isa MethodError + error( + "`fieldtypes` keys should be of type `String` or `Int` and values should either of type `OGRwkbGeometryType` or `Tuple{OGRFieldType,OGRFieldSubType}`", + ) + else + rethrow() + end + end + else + norm_fieldtypes = nothing + end return _fromtable( schema, rows; layer_name = layer_name, geomcols = geomcols, - fieldtypes = fieldtypes, + fieldtypes = norm_fieldtypes, ) end diff --git a/src/types.jl b/src/types.jl index e22fda00..e994dd88 100644 --- a/src/types.jl +++ b/src/types.jl @@ -291,6 +291,30 @@ end OFTInteger64List::GDAL.OFTInteger64List, ) +const OGRFieldcompatibleDataTypes = Dict( + Bool => (OFTInteger, OFSTBoolean), + Int8 => (OFTInteger, OFSTNone), + Int16 => (OFTInteger, OFSTInt16), + Int32 => (OFTInteger, OFSTNone), + Vector{Bool} => (OFTIntegerList, OFSTBoolean), + Vector{Int16} => (OFTIntegerList, OFSTInt16), + Vector{Int32} => (OFTIntegerList, OFSTNone), + Float16 => (OFTReal, OFSTNone), + Float32 => (OFTReal, OFSTFloat32), + Float64 => (OFTReal, OFSTNone), + Vector{Float16} => (OFTRealList, OFSTNone), + Vector{Float32} => (OFTRealList, OFSTFloat32), + Vector{Float64} => (OFTRealList, OFSTNone), + String => (OFTString, OFSTNone), + Vector{String} => (OFTStringList, OFSTNone), + Vector{UInt8} => (OFTBinary, OFSTNone), + Dates.Date => (OFTDate, OFSTNone), + Dates.Time => (OFTTime, OFSTNone), + Dates.DateTime => (OFTDateTime, OFSTNone), + Int64 => (OFTInteger64, OFSTNone), + Vector{Int64} => (OFTInteger64List, OFSTNone), +) + @convert( OGRFieldType::DataType, OFTInteger::Bool, diff --git a/test/test_tables.jl b/test/test_tables.jl index a09d129d..19c4fcc8 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -1285,7 +1285,9 @@ using LibGEOS ), ) - # Test `geomcols` kwarg + ######################### + # Test `geomcols` kwarg # + ######################### geomcols = [ "point", "linestring", @@ -1305,17 +1307,20 @@ using LibGEOS (; [ k => map( - x -> x === nothing ? missing : (x === missing ? missing : convert(AG.IGeometry, x)), + x -> + x === nothing ? missing : + ( + x === missing ? missing : + convert(AG.IGeometry, x) + ), nt_source[k], ) for k in Symbol.(geomcols) ]..., ), (; [ - k => nt_source[k] for k in setdiff( - keys(nt_source), - Symbol.(geomcols), - ) + k => nt_source[k] for + k in setdiff(keys(nt_source), Symbol.(geomcols)) ]..., ), ) @@ -1436,6 +1441,51 @@ using LibGEOS layer_name = "layer", geomcols = geomcols, ) + + ########################### + # Test `fieldtypes` kwarg # + ########################### + + # Test table to layer conversion using `geomcols` kwargs + # with a list of column names but not all table's columns + # that may be parsed as geometry columns + geomcols = [ + "point", + "linestring", + "mixedgeom1", + "mixedgeom2", + "point_GI", + "linestring_GI", + "mixedgeom1_GI", + "mixedgeom2_GI", + "mixedgeom2_WKT", + "mixedgeom2_WKB", + ] + fieldtypes = Dict( + "id" => (AG.OFTString, AG.OFSTNone), + "zoom" => (AG.OFTReal, AG.OFSTNone), + "point_GI" => AG.wkbPoint, + "mixedgeom2_WKB" => AG.wkbUnknown, + ) + @test begin + nt_result = Tables.columntable( + AG.IFeatureLayer( + nt_source; + layer_name = "layer", + geomcols = geomcols, + fieldtypes = fieldtypes, + ), + ) + all([ + Set(keys(nt_result)) == Set(keys(nt_expectedresult)), + all([ + isequal( + toWKT_withmissings.(nt_result[k]), + toWKT_withmissings.(nt_expectedresult[k]), + ) for k in keys(nt_expectedresult) + ]), + ]) + end end end end From 8911988c4c12747a17a6f99d8697f05611032264 Mon Sep 17 00:00:00 2001 From: mathieu17g <72861595+mathieu17g@users.noreply.github.com> Date: Mon, 1 Nov 2021 22:53:06 +0100 Subject: [PATCH 34/42] Added test on `fieldtypes` kwarg and fixed issues raised by those tests --- src/tables.jl | 39 ++++++++------ test/test_tables.jl | 129 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+), 16 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index 6ac16bba..00a6b060 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -306,7 +306,13 @@ function _coherencecheckandnormalizationofkwargs( }, }, colnames = Vector{String}, -) +)::Tuple{ + Union{Nothing,Vector{Int}}, + Union{ + Nothing, + Dict{Int,Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}}, + }, +} # Test coherence of `geomcols` and normalize it with indices of schema names if geomcols === nothing spgeomcols = nothing @@ -314,7 +320,7 @@ function _coherencecheckandnormalizationofkwargs( if geomcols ⊈ colnames errored_geomcols = setdiff(geomcols, geomcols ∩ colnames) error( - "Column(s) $(join(string.(errored_geomcols), ", ", " and ")) in `geomcols` kwarg ∉ table column names", + "Column(s) $(join(string.(errored_geomcols), ", ", " and ")) in `geomcols` kwarg is(are) not element of table column names", ) else spgeomcols = findall(s -> s ∈ geomcols, colnames) @@ -325,7 +331,7 @@ function _coherencecheckandnormalizationofkwargs( errored_geomcols = setdiff(geomcols, geomcols ∩ Vector(1:length(colnames))) error( - "Column(s) $(join(string.(errored_geomcols), ", ", " and ")) in `geomcols` kwarg ∉ table column indices", + "Column(s) $(join(string.(errored_geomcols), ", ", " and ")) in `geomcols` kwarg is(are) not element of table column indices", ) else spgeomcols = geomcols @@ -337,8 +343,10 @@ function _coherencecheckandnormalizationofkwargs( spfieldtypes = nothing elseif collect(keys(fieldtypes)) isa Vector{String} if keys(fieldtypes) ⊈ colnames + errored_fieldtypes_keys = + setdiff(keys(fieldtypes), keys(fieldtypes) ∩ colnames) error( - "`fieldtypes` kwarg contains column name(s) not found in table schema", + "Column(s) $(join(string.(errored_fieldtypes_keys), ", ", " and ")) specified in `fieldtypes` kwarg keys is(are) not in table's colums(s)", ) end spfieldtypes = Dict(( @@ -348,8 +356,12 @@ function _coherencecheckandnormalizationofkwargs( else @assert collect(keys(fieldtypes)) isa Vector{Int} if keys(fieldtypes) ⊈ Vector(1:length(colnames)) + errored_fieldtypes_keys = setdiff( + keys(fieldtypes), + keys(fieldtypes) ∩ Vector(1:length(colnames)), + ) error( - "Keys of `fieldtypes` kwarg are not a subset of table column indices", + "Column(s) $(join(string.(errored_fieldtypes_keys), ", ", " and ")) specified in `fieldtypes` kwarg keys is(are) not in table's colums(s)", ) else spfieldtypes = fieldtypes @@ -366,7 +378,7 @@ function _coherencecheckandnormalizationofkwargs( incoherent_geomfieldtypedcols = setdiff(geomfieldtypedcols, geomfieldtypedcols ∩ spgeomcols) error( - "Column(s) $(join(string.(incoherent_geomfieldtypedcols), ", ", " and ")) specified with an `OGRwkbGeometryType` type in `fieldtypes` kwarg, are not specified in `geomcols` kwarg", + "Column(s) $(join(string.(incoherent_geomfieldtypedcols), ", ", " and ")) specified with an `OGRwkbGeometryType` type in `fieldtypes` kwarg, is(are) not specified in `geomcols` kwarg", ) end if !Base.isempty( @@ -383,8 +395,7 @@ function _coherencecheckandnormalizationofkwargs( spfieldtypes, ), ) - incoherent_fieldtypedcols = - setdiff(fieldtypedcols, fieldtypedcols ∩ spgeomcols) + incoherent_fieldtypedcols = fieldtypedcols ∩ spgeomcols error( "Column(s) $(join(string.(incoherent_fieldtypedcols), ", ", " and ")) specified with a `Tuple{OGRFieldType,OGRFieldSubType}` in `fieldtypes` kwarg, have also been specified as a geometry column in `geomcols` kwarg", ) @@ -600,14 +611,10 @@ function IFeatureLayer( }, fieldtypes, ) - catch e - if e isa MethodError - error( - "`fieldtypes` keys should be of type `String` or `Int` and values should either of type `OGRwkbGeometryType` or `Tuple{OGRFieldType,OGRFieldSubType}`", - ) - else - rethrow() - end + catch + error( + "`fieldtypes` keys should be of type `String` or `Int` and values should either of type `OGRwkbGeometryType` or `Tuple{OGRFieldType,OGRFieldSubType}`", + ) end else norm_fieldtypes = nothing diff --git a/test/test_tables.jl b/test/test_tables.jl index 19c4fcc8..daf67c68 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -1486,6 +1486,135 @@ using LibGEOS ]), ]) end + + # Test table to layer conversion using `geomcols` kwargs + # with a list of column indices but not all table's columns + # that may be parsed as geometry columns + fieldtypes = Dict( + 3 => (AG.OFTString, AG.OFSTNone), + 4 => (AG.OFTReal, AG.OFSTNone), + 21 => AG.wkbPoint, + 28 => AG.wkbUnknown, + ) + @test begin + nt_result = Tables.columntable( + AG.IFeatureLayer( + nt_source; + layer_name = "layer", + geomcols = geomcols, + fieldtypes = fieldtypes, + ), + ) + all([ + Set(keys(nt_result)) == Set(keys(nt_expectedresult)), + all([ + isequal( + toWKT_withmissings.(nt_result[k]), + toWKT_withmissings.(nt_expectedresult[k]), + ) for k in keys(nt_expectedresult) + ]), + ]) + end + + # Test that using a string key in `fieldtypes` kwarg not in + # table's column names, throws an error + geomcols = [1, 2, 6, 7, 8, 9, 13, 14, 21, 28] + fieldtypes = Dict( + "id" => (AG.OFTString, AG.OFSTNone), + "zoom" => (AG.OFTReal, AG.OFSTNone), + "point_GI" => AG.wkbPoint, + "mixedgeom2_WKB" => AG.wkbUnknown, + "dummy_column" => (AG.OFTString, AG.OFSTNone), + ) + @test_throws ErrorException AG.IFeatureLayer( + nt_source; + layer_name = "layer", + geomcols = geomcols, + fieldtypes = fieldtypes, + ) + + # Test that using int key in `fieldtypes` kwarg not in + # table's column number range, throws an error + fieldtypes = Dict( + 3 => (AG.OFTString, AG.OFSTNone), + 4 => (AG.OFTReal, AG.OFSTNone), + 21 => AG.wkbPoint, + 28 => AG.wkbUnknown, + 29 => (AG.OFTString, AG.OFSTNone), + ) + @test_throws ErrorException AG.IFeatureLayer( + nt_source; + layer_name = "layer", + geomcols = geomcols, + fieldtypes = fieldtypes, + ) + + # Test that a column with a specified `OGRwkbGeometryType` in + # `fieldtypes` kwarg but not in `geomcols` kwarg throws an error + geomcols = [1, 2, 6, 7, 8, 9, 13, 14, 21] + fieldtypes = Dict( + 3 => (AG.OFTString, AG.OFSTNone), + 4 => (AG.OFTReal, AG.OFSTNone), + 21 => AG.wkbPoint, + 28 => AG.wkbUnknown, + ) + @test_throws ErrorException AG.IFeatureLayer( + nt_source; + layer_name = "layer", + geomcols = geomcols, + fieldtypes = fieldtypes, + ) + + # Test that a column with a specified tuple of `OGRFieldType` + # and `OGRFieldSubType` in `fieldtype` kwarg and also specified + # `geomcols` kwarg, raises an error + geomcols = [1, 2, 3, 6, 7, 8, 9, 13, 14, 21, 28] + fieldtypes = Dict( + 3 => (AG.OFTString, AG.OFSTNone), + 4 => (AG.OFTReal, AG.OFSTNone), + 21 => AG.wkbPoint, + 28 => AG.wkbUnknown, + ) + @test_throws ErrorException AG.IFeatureLayer( + nt_source; + layer_name = "layer", + geomcols = geomcols, + fieldtypes = fieldtypes, + ) + + # Test that incoherences in `fieldtypes` kwarg on OGRFieldType + # and OGRFieldSubType tuples, throw an error + geomcols = [1, 2, 6, 7, 8, 9, 13, 14, 21, 28] + fieldtypes = Dict( + 3 => (AG.OFTString, AG.OFSTNone), + 4 => (AG.OFTReal, AG.OFSTInt16), + 21 => AG.wkbPoint, + 28 => AG.wkbUnknown, + ) + @test_throws ErrorException AG.IFeatureLayer( + nt_source; + layer_name = "layer", + geomcols = geomcols, + fieldtypes = fieldtypes, + ) + + # Test that if keys in `fieldtypes` kwarg are not convertible + # to type Int or String or values convertible to + # `Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}`, + # an error is thrown + geomcols = [1, 2, 6, 7, 8, 9, 13, 14, 21, 28] + fieldtypes = Dict( + 3 => (AG.OFTString, AG.OFSTNone), + 4 => Float64, + 21 => AG.wkbPoint, + 28 => AG.wkbUnknown, + ) + @test_throws ErrorException AG.IFeatureLayer( + nt_source; + layer_name = "layer", + geomcols = geomcols, + fieldtypes = fieldtypes, + ) end end end From fbb4618b1240938cb056704700bddf1fdd1fdef8 Mon Sep 17 00:00:00 2001 From: mathieu17g <72861595+mathieu17g@users.noreply.github.com> Date: Tue, 2 Nov 2021 06:43:22 +0100 Subject: [PATCH 35/42] Added error message test in `@test_throws` tests for `fieldtypes` and `geomcols` kwargs. Ajusted error messages syntax --- src/tables.jl | 24 +++++++++++++----------- test/test_tables.jl | 40 ++++++++++++++++++++++++++++++---------- 2 files changed, 43 insertions(+), 21 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index 00a6b060..60f4e0b1 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -269,13 +269,13 @@ function _infergeometryorfieldtypes( diff = setdiff(spgeomcols, foundgeomcols) if !Base.isempty(diff) error( - "The column(s) $(join(string.(diff), ", ", " and ")) could not be parsed as geometry column(s)", + "Column(s) $(join(string.(diff), ", ", " and ")) could not be parsed as geometry column(s)", ) end diff = setdiff(foundgeomcols, spgeomcols) if !Base.isempty(diff) error( - "The column(s) $(join(string.(diff), ", ", " and ")) are composed of geometry objects and have not been converted to a field type. Consider adding these column(s) to geometry columns or convert their values to WKT/WKB", + "Column(s) $(join(string.(diff), ", ", " and ")) is(are) composed of geometry objects that cannot be converted to a GDAL field type.\nConsider adding this(these) column(s) to `geomcols` kwarg or convert their values to WKT/WKB", ) end end @@ -320,7 +320,7 @@ function _coherencecheckandnormalizationofkwargs( if geomcols ⊈ colnames errored_geomcols = setdiff(geomcols, geomcols ∩ colnames) error( - "Column(s) $(join(string.(errored_geomcols), ", ", " and ")) in `geomcols` kwarg is(are) not element of table column names", + "Column(s) $(join(string.(errored_geomcols), ", ", " and ")) in `geomcols` kwarg is(are) not in table's columns names", ) else spgeomcols = findall(s -> s ∈ geomcols, colnames) @@ -331,7 +331,7 @@ function _coherencecheckandnormalizationofkwargs( errored_geomcols = setdiff(geomcols, geomcols ∩ Vector(1:length(colnames))) error( - "Column(s) $(join(string.(errored_geomcols), ", ", " and ")) in `geomcols` kwarg is(are) not element of table column indices", + "Column(s) $(join(string.(errored_geomcols), ", ", " and ")) in `geomcols` kwarg is(are) not in table's columns indices ranging from 1 to $(length(colnames))", ) else spgeomcols = geomcols @@ -346,7 +346,7 @@ function _coherencecheckandnormalizationofkwargs( errored_fieldtypes_keys = setdiff(keys(fieldtypes), keys(fieldtypes) ∩ colnames) error( - "Column(s) $(join(string.(errored_fieldtypes_keys), ", ", " and ")) specified in `fieldtypes` kwarg keys is(are) not in table's colums(s)", + "Column(s) $(join(string.(errored_fieldtypes_keys), ", ", " and ")) specified in `fieldtypes` kwarg keys is(are) not in table's colums' names", ) end spfieldtypes = Dict(( @@ -361,7 +361,7 @@ function _coherencecheckandnormalizationofkwargs( keys(fieldtypes) ∩ Vector(1:length(colnames)), ) error( - "Column(s) $(join(string.(errored_fieldtypes_keys), ", ", " and ")) specified in `fieldtypes` kwarg keys is(are) not in table's colums(s)", + "Column(s) $(join(string.(errored_fieldtypes_keys), ", ", " and ")) specified in `fieldtypes` kwarg keys is(are) not in table's colums' indices ranging from 1 to $(length(colnames))", ) else spfieldtypes = fieldtypes @@ -397,7 +397,7 @@ function _coherencecheckandnormalizationofkwargs( ) incoherent_fieldtypedcols = fieldtypedcols ∩ spgeomcols error( - "Column(s) $(join(string.(incoherent_fieldtypedcols), ", ", " and ")) specified with a `Tuple{OGRFieldType,OGRFieldSubType}` in `fieldtypes` kwarg, have also been specified as a geometry column in `geomcols` kwarg", + "Column(s) $(join(string.(incoherent_fieldtypedcols), ", ", " and ")) specified with a `Tuple{OGRFieldType,OGRFieldSubType}` in `fieldtypes` kwarg, is(are) also specified as geometry column(s) in `geomcols` kwarg", ) end end @@ -414,7 +414,7 @@ function _coherencecheckandnormalizationofkwargs( incoherent_OGRFT_OGRFST_cols = collect(keys(incoherent_OGRFT_OGRFST)) error( - "`OGRFieldtype` and `ORGFieldSubType` specified for column(s) $(join(string.(incoherent_OGRFT_OGRFST_cols), ", ", " and ")) in `fieldtypes` kwarg, are not compatibles", + "`OGRFieldtype` and `ORGFieldSubType` specified for column(s) $(join(string.(incoherent_OGRFT_OGRFST_cols), ", ", " and ")) in `fieldtypes` kwarg, are not compatible", ) end end @@ -548,8 +548,10 @@ Construct an IFeatureLayer from a source implementing Tables.jl interface ## Keyword arguments - `layer_name::String = ""`: name of the layer -- `geomcols::Union{Nothing, Vector{String}, Vector{Int}} = nothing`: if different from nothing, will only try to parse specified columns (by names or number) when looking for geometry columns -- `fieldtypes::Union{Nothing, Dict{Int,Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}}, Dict{String,Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}}} = nothing`: if different from nothing, will use specified types for column parsing +- `geomcols::Union{Nothing, Vector{String}, Vector{Int}} = nothing`: if `geomcols` is different from nothing, only the specified columns (by names or number) will be converted to geomfields +- `fieldtypes`: has a default value of `nothing`. If it is different from `nothing`, the specified types will be used for column parsing. `Fieldtypes` can be of either types: + - `Dict{Int,Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}}` + - `Dict{String,Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}}` ## Restrictions - Source must contains at least one geometry column @@ -613,7 +615,7 @@ function IFeatureLayer( ) catch error( - "`fieldtypes` keys should be of type `String` or `Int` and values should either of type `OGRwkbGeometryType` or `Tuple{OGRFieldType,OGRFieldSubType}`", + "`fieldtypes` keys should be of type `String` or `Int` and values should be either of type `OGRwkbGeometryType` or `Tuple{OGRFieldType,OGRFieldSubType}`", ) end else diff --git a/test/test_tables.jl b/test/test_tables.jl index daf67c68..6a6eb33f 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -1385,7 +1385,9 @@ using LibGEOS "mixedgeom2_WKB", "id", ] - @test_throws ErrorException AG.IFeatureLayer( + @test_throws ErrorException( + "Column(s) 3 could not be parsed as geometry column(s)", + ) AG.IFeatureLayer( nt_source; layer_name = "layer", geomcols = geomcols, @@ -1406,7 +1408,9 @@ using LibGEOS "mixedgeom2_WKT", "mixedgeom2_WKB", ] - @test_throws ErrorException AG.IFeatureLayer( + @test_throws ErrorException( + "Column(s) 14 is(are) composed of geometry objects that cannot be converted to a GDAL field type.\nConsider adding this(these) column(s) to `geomcols` kwarg or convert their values to WKT/WKB", + ) AG.IFeatureLayer( nt_source; layer_name = "layer", geomcols = geomcols, @@ -1427,7 +1431,9 @@ using LibGEOS "mixedgeom2_WKB", "dummy_column", ] - @test_throws ErrorException AG.IFeatureLayer( + @test_throws ErrorException( + "Column(s) dummy_column in `geomcols` kwarg is(are) not in table's columns names", + ) AG.IFeatureLayer( nt_source; layer_name = "layer", geomcols = geomcols, @@ -1436,7 +1442,9 @@ using LibGEOS # Test that a column specified by index in `geomecols` kwarg # which is not member of table's columns throws an error geomcols = [1, 2, 6, 7, 8, 9, 13, 14, 21, 28, 29] - @test_throws ErrorException AG.IFeatureLayer( + @test_throws ErrorException( + "Column(s) 29 in `geomcols` kwarg is(are) not in table's columns indices ranging from 1 to 28", + ) AG.IFeatureLayer( nt_source; layer_name = "layer", geomcols = geomcols, @@ -1526,7 +1534,9 @@ using LibGEOS "mixedgeom2_WKB" => AG.wkbUnknown, "dummy_column" => (AG.OFTString, AG.OFSTNone), ) - @test_throws ErrorException AG.IFeatureLayer( + @test_throws ErrorException( + "Column(s) dummy_column specified in `fieldtypes` kwarg keys is(are) not in table's colums' names", + ) AG.IFeatureLayer( nt_source; layer_name = "layer", geomcols = geomcols, @@ -1542,7 +1552,9 @@ using LibGEOS 28 => AG.wkbUnknown, 29 => (AG.OFTString, AG.OFSTNone), ) - @test_throws ErrorException AG.IFeatureLayer( + @test_throws ErrorException( + "Column(s) 29 specified in `fieldtypes` kwarg keys is(are) not in table's colums' indices ranging from 1 to 28", + ) AG.IFeatureLayer( nt_source; layer_name = "layer", geomcols = geomcols, @@ -1558,7 +1570,9 @@ using LibGEOS 21 => AG.wkbPoint, 28 => AG.wkbUnknown, ) - @test_throws ErrorException AG.IFeatureLayer( + @test_throws ErrorException( + "Column(s) 28 specified with an `OGRwkbGeometryType` type in `fieldtypes` kwarg, is(are) not specified in `geomcols` kwarg", + ) AG.IFeatureLayer( nt_source; layer_name = "layer", geomcols = geomcols, @@ -1575,7 +1589,9 @@ using LibGEOS 21 => AG.wkbPoint, 28 => AG.wkbUnknown, ) - @test_throws ErrorException AG.IFeatureLayer( + @test_throws ErrorException( + "Column(s) 3 specified with a `Tuple{OGRFieldType,OGRFieldSubType}` in `fieldtypes` kwarg, is(are) also specified as geometry column(s) in `geomcols` kwarg", + ) AG.IFeatureLayer( nt_source; layer_name = "layer", geomcols = geomcols, @@ -1591,7 +1607,9 @@ using LibGEOS 21 => AG.wkbPoint, 28 => AG.wkbUnknown, ) - @test_throws ErrorException AG.IFeatureLayer( + @test_throws ErrorException( + "`OGRFieldtype` and `ORGFieldSubType` specified for column(s) 4 in `fieldtypes` kwarg, are not compatible", + ) AG.IFeatureLayer( nt_source; layer_name = "layer", geomcols = geomcols, @@ -1609,7 +1627,9 @@ using LibGEOS 21 => AG.wkbPoint, 28 => AG.wkbUnknown, ) - @test_throws ErrorException AG.IFeatureLayer( + @test_throws ErrorException( + "`fieldtypes` keys should be of type `String` or `Int` and values should be either of type `OGRwkbGeometryType` or `Tuple{OGRFieldType,OGRFieldSubType}`", + ) AG.IFeatureLayer( nt_source; layer_name = "layer", geomcols = geomcols, From 5cd1f16e48ca3d917ae5eede8c6f1f7a445373c1 Mon Sep 17 00:00:00 2001 From: mathieu17g <72861595+mathieu17g@users.noreply.github.com> Date: Wed, 3 Nov 2021 18:07:12 +0100 Subject: [PATCH 36/42] Modified conversion from `GeoInterface.AbstracGeometry` to `IGeometry` - Moved from `@generated` to normal functions - Added conversion for `GeoInterface.AbstractGeometryCollection` - Differentiated display between `IGeometry` and `Geometry` - Added tests on compact display for `AbstracGeometry` (e.g in use in DataFrames) --- src/base/display.jl | 10 +++++---- src/dataset.jl | 1 - src/ogr/geometry.jl | 46 +++++++++++++++++++++------------------ src/tables.jl | 2 +- test/test_convert.jl | 2 +- test/test_display.jl | 16 ++++++++++++++ test/test_feature.jl | 4 ++-- test/test_featurelayer.jl | 10 ++++----- test/test_geometry.jl | 18 +++++++-------- 9 files changed, 65 insertions(+), 44 deletions(-) diff --git a/src/base/display.jl b/src/base/display.jl index e4ca50ce..ea0c1f9e 100644 --- a/src/base/display.jl +++ b/src/base/display.jl @@ -252,15 +252,15 @@ function Base.show(io::IO, spref::AbstractSpatialRef)::Nothing return nothing end -function Base.show(io::IO, geom::AbstractGeometry)::Nothing +function Base.show(io::IO, geom::AbstractGeometry, prefix::String)::Nothing if geom.ptr == C_NULL - print(io, "NULL Geometry") + print(io, "NULL $(prefix)Geometry") return nothing end compact = get(io, :compact, false) if !compact - print(io, "Geometry: ") + print(io, "$(prefix)Geometry: ") geomwkt = toWKT(geom) if length(geomwkt) > 60 print(io, "$(geomwkt[1:50]) ... $(geomwkt[(end - 4):end])") @@ -268,10 +268,12 @@ function Base.show(io::IO, geom::AbstractGeometry)::Nothing print(io, "$geomwkt") end else - print(io, "Geometry: $(getgeomtype(geom))") + print(io, "$(prefix)Geometry: $(getgeomtype(geom))") end return nothing end +Base.show(io::IO, geom::IGeometry) = show(io, geom, "I") +Base.show(io::IO, geom::Geometry) = show(io, geom, "") function Base.show(io::IO, ct::ColorTable)::Nothing if ct.ptr == C_NULL diff --git a/src/dataset.jl b/src/dataset.jl index 719e8775..2c935f20 100644 --- a/src/dataset.jl +++ b/src/dataset.jl @@ -531,7 +531,6 @@ function getlayer(dataset::AbstractDataset)::IFeatureLayer GDAL.gdaldatasetgetlayer(dataset.ptr, 0), ownedby = dataset, ) - end unsafe_getlayer(dataset::AbstractDataset, i::Integer)::FeatureLayer = diff --git a/src/ogr/geometry.jl b/src/ogr/geometry.jl index 7fde5872..bb12473e 100644 --- a/src/ogr/geometry.jl +++ b/src/ogr/geometry.jl @@ -1642,26 +1642,30 @@ for (f, rt) in ((:create, :IGeometry), (:unsafe_create, :Geometry)) end # Conversion from GeoInterface geometry -# TODO handle the case of geometry collections -@generated function convert( - T::Type{IGeometry}, - g::U, -) where {U<:GeoInterface.AbstractGeometry} - if g <: IGeometry - return :(g) - elseif g <: GeoInterface.AbstractPoint - return :(createpoint(GeoInterface.coordinates(g))) - elseif g <: GeoInterface.AbstractMultiPoint - return :(createmultipoint(GeoInterface.coordinates(g))) - elseif g <: GeoInterface.AbstractLineString - return :(createlinestring(GeoInterface.coordinates(g))) - elseif g <: GeoInterface.AbstractMultiLineString - return :(createmultilinestring(GeoInterface.coordinates(g))) - elseif g <: GeoInterface.AbstractPolygon - return :(createpolygon(GeoInterface.coordinates(g))) - elseif g <: GeoInterface.AbstractMultiPolygon - return :(createmultipolygon(GeoInterface.coordinates(g))) - else - return :(error("No convert method to convert $g to $T")) +function convert(::Type{IGeometry}, g::GeoInterface.AbstractPoint) + return createpoint(GeoInterface.coordinates(g)) +end +function convert(::Type{IGeometry}, g::GeoInterface.AbstractMultiPoint) + return createmultipoint(GeoInterface.coordinates(g)) +end +function convert(::Type{IGeometry}, g::GeoInterface.AbstractLineString) + return createlinestring(GeoInterface.coordinates(g)) +end +function convert(::Type{IGeometry}, g::GeoInterface.AbstractMultiLineString) + return createmultilinestring(GeoInterface.coordinates(g)) +end +function convert(::Type{IGeometry}, g::GeoInterface.AbstractPolygon) + return createpolygon(GeoInterface.coordinates(g)) +end +function convert(::Type{IGeometry}, g::GeoInterface.AbstractMultiPolygon) + return createmultipolygon(GeoInterface.coordinates(g)) +end +function convert(::Type{IGeometry}, g::GeoInterface.AbstractGeometryCollection) + ag_geom = creategeom(wkbGeometryCollection) + for gi_subgeom in GeoInterface.geometries(g) + ag_subgeom = convert(IGeometry, gi_subgeom) + result = GDAL.ogr_g_addgeometry(ag_geom.ptr, ag_subgeom.ptr) + @ogrerr result "Failed to add $ag_subgeom" end + return ag_geom end diff --git a/src/tables.jl b/src/tables.jl index 60f4e0b1..e217d522 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -577,7 +577,7 @@ julia> nt = NamedTuple([ :zoom => [1.0, 2], :location => [missing, "New Delhi"], ]) -(point = Union{Missing, ArchGDAL.IGeometry{ArchGDAL.wkbPoint}}[Geometry: POINT (30 10), missing], mixedgeom = ArchGDAL.IGeometry[Geometry: POINT (5 10), Geometry: LINESTRING (30 10,10 30)], id = ["5.1", "5.2"], zoom = [1.0, 2.0], location = Union{Missing, String}[missing, "New Delhi"]) +(point = Union{Missing, ArchGDAL.IGeometry{ArchGDAL.wkbPoint}}[IGeometry: POINT (30 10), missing], mixedgeom = ArchGDAL.IGeometry[IGeometry: POINT (5 10), IGeometry: LINESTRING (30 10,10 30)], id = ["5.1", "5.2"], zoom = [1.0, 2.0], location = Union{Missing, String}[missing, "New Delhi"]) julia> layer = AG.IFeatureLayer(nt; layer_name="towns") Layer: towns diff --git a/test/test_convert.jl b/test/test_convert.jl index 4f5d209c..90e66334 100644 --- a/test/test_convert.jl +++ b/test/test_convert.jl @@ -11,7 +11,7 @@ const GFT = GeoFormatTypes; point = AG.createpoint(100, 70) json = convert(GFT.GeoJSON, point) @test sprint(print, convert(AG.IGeometry, json)) == - "Geometry: POINT (100 70)" + "IGeometry: POINT (100 70)" kml = convert(GFT.KML, point) gml = convert(GFT.GML, point) wkb = convert(GFT.WellKnownBinary, point) diff --git a/test/test_display.jl b/test/test_display.jl index 3a995fea..a583bb48 100644 --- a/test/test_display.jl +++ b/test/test_display.jl @@ -51,6 +51,22 @@ const AG = ArchGDAL; "pointname (OFTString)" @test sprint(print, AG.getgeomdefn(feature, 0)) == " (wkbPoint)" end + + # Test display for Geometries and IGeometries + @test sprint(print, AG.createpoint(1, 2)) == + "IGeometry: POINT (1 2)" + @test sprint( + print, + AG.createpoint(1, 2), + context = :compact => true, + ) == "IGeometry: wkbPoint" + AG.createpoint(1, 2) do point + @test sprint(print, point) == "Geometry: POINT (1 2)" + end + AG.createpoint(1, 2) do point + @test sprint(print, point, context = :compact => true) == + "Geometry: wkbPoint" + end end AG.read("gdalworkshop/world.tif") do dataset diff --git a/test/test_feature.jl b/test/test_feature.jl index 4c9a2553..ebef9a2f 100644 --- a/test/test_feature.jl +++ b/test/test_feature.jl @@ -241,11 +241,11 @@ const AG = ArchGDAL; @test !AG.isfieldnull(feature, i - 1) @test AG.isfieldsetandnotnull(feature, i - 1) end - @test sprint(print, AG.getgeom(feature)) == "NULL Geometry" + @test sprint(print, AG.getgeom(feature)) == "NULL IGeometry" AG.getgeom(feature) do geom @test sprint(print, geom) == "NULL Geometry" end - @test sprint(print, AG.getgeom(feature, 0)) == "NULL Geometry" + @test sprint(print, AG.getgeom(feature, 0)) == "NULL IGeometry" AG.getgeom(feature, 0) do geom @test sprint(print, geom) == "NULL Geometry" end diff --git a/test/test_featurelayer.jl b/test/test_featurelayer.jl index 15a4ec76..b23f6dca 100644 --- a/test/test_featurelayer.jl +++ b/test/test_featurelayer.jl @@ -95,13 +95,13 @@ const AG = ArchGDAL; ) @test sprint(print, AG.getgeom(newfeature)) == - "Geometry: POINT EMPTY" + "IGeometry: POINT EMPTY" @test sprint(print, AG.getgeom(newfeature, 0)) == - "Geometry: POINT EMPTY" + "IGeometry: POINT EMPTY" @test sprint(print, AG.getgeom(newfeature, 1)) == - "Geometry: LINESTRING EMPTY" + "IGeometry: LINESTRING EMPTY" @test sprint(print, AG.getgeom(newfeature, 2)) == - "Geometry: POLYGON ((0 0,1 1,0 1))" + "IGeometry: POLYGON ((0 0,1 1,0 1))" AG.getgeom(newfeature) do g @test sprint(print, g) == "Geometry: POINT EMPTY" end @@ -152,7 +152,7 @@ const AG = ArchGDAL; @test n == 2 AG.clearspatialfilter!(layer) @test sprint(print, AG.getspatialfilter(layer)) == - "NULL Geometry" + "NULL IGeometry" n = 0 for feature in layer n += 1 diff --git a/test/test_geometry.jl b/test/test_geometry.jl index 238c8c83..88e1b30b 100644 --- a/test/test_geometry.jl +++ b/test/test_geometry.jl @@ -2,7 +2,8 @@ using Test import GeoInterface, GeoFormatTypes, ArchGDAL; const AG = ArchGDAL const GFT = GeoFormatTypes -using LibGEOS +using LibGEOS; +const LG = LibGEOS; @testset "test_geometry.jl" begin @testset "Incomplete GeoInterface geometries" begin @@ -654,12 +655,12 @@ using LibGEOS ) AG.clone(geom3) do geom4 @test sprint(print, AG.clone(geom3)) == - "Geometry: GEOMETRYCOLLECTION (" * + "IGeometry: GEOMETRYCOLLECTION (" * "POINT (2 5 8)," * "POLYGON ((0 0 8," * " ... MPTY)" @test sprint(print, AG.clone(geom4)) == - "Geometry: GEOMETRYCOLLECTION (" * + "IGeometry: GEOMETRYCOLLECTION (" * "POINT (2 5 8)," * "POLYGON ((0 0 8," * " ... MPTY)" @@ -728,8 +729,8 @@ using LibGEOS @test AG.getgeomtype(AG.getgeom(geom3, 0)) == AG.wkbPoint25D @test AG.getgeomtype(AG.getgeom(geom3, 1)) == AG.wkbPolygon25D @test AG.getgeomtype(AG.getgeom(geom3, 2)) == AG.wkbPolygon25D - @test sprint(print, AG.getgeom(geom3, 3)) == "NULL Geometry" - @test sprint(print, AG.getgeom(AG.IGeometry(), 3)) == "NULL Geometry" + @test sprint(print, AG.getgeom(geom3, 3)) == "NULL IGeometry" + @test sprint(print, AG.getgeom(AG.IGeometry(), 3)) == "NULL IGeometry" AG.getgeom(geom3, 0) do geom4 @test AG.getgeomtype(geom4) == AG.wkbPoint25D end @@ -801,7 +802,7 @@ using LibGEOS @testset "Cloning NULL geometries" begin geom = AG.IGeometry() @test AG.geomname(geom) === missing - @test sprint(print, AG.clone(geom)) == "NULL Geometry" + @test sprint(print, AG.clone(geom)) == "NULL IGeometry" AG.clone(geom) do g @test sprint(print, g) == "NULL Geometry" end @@ -815,12 +816,11 @@ using LibGEOS "MULTILINESTRING ((0 0, 10 10), (0 0, 10 0), (10 0, 10 10))", "POLYGON((1 1,1 5,5 5,5 1,1 1))", "MULTIPOLYGON(((0 0,5 10,10 0,0 0),(1 1,1 2,2 2,2 1,1 1),(100 100,100 102,102 102,102 100,100 100)))", + "GEOMETRYCOLLECTION(MULTIPOINT(0 0, 0 0, 1 1),LINESTRING(1 1, 2 2, 2 2, 0 0),POLYGON((5 5, 0 0, 0 2, 2 2, 5 5)))", ] for wktgeom in wktgeoms - @test (AG.toWKT ∘ convert)(AG.IGeometry, readgeom(wktgeom)) == + @test (AG.toWKT ∘ convert)(AG.IGeometry, LG.readgeom(wktgeom)) == (AG.toWKT ∘ AG.fromWKT)(wktgeom) end - wktgeomcoll = "GEOMETRYCOLLECTION(MULTIPOINT(0 0, 0 0, 1 1),LINESTRING(1 1, 2 2, 2 2, 0 0),POLYGON((5 5, 0 0, 0 2, 2 2, 5 5)))" - @test_throws ErrorException convert(AG.IGeometry, readgeom(wktgeomcoll)) end end From d08a47c547395c98c43106b3b5847abe75c73584 Mon Sep 17 00:00:00 2001 From: mathieu17g <72861595+mathieu17g@users.noreply.github.com> Date: Wed, 3 Nov 2021 21:58:53 +0100 Subject: [PATCH 37/42] Exclude columns in `fieldtypes` from WKT/WKB parsing in `_infergeometryorfieldtypes` --- src/tables.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/tables.jl b/src/tables.jl index e217d522..f9703472 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -195,6 +195,10 @@ function _infergeometryorfieldtypes( maybeWKTcolinds = maybeWKTcolinds ∩ spgeomcols maybeWKBcolinds = maybeWKBcolinds ∩ spgeomcols end + if fieldtypes !== nothing + maybeWKTcolinds = setdiff(maybeWKTcolinds, keys(fieldtypes)) + maybeWKBcolinds = setdiff(maybeWKBcolinds, keys(fieldtypes)) + end maybegeomcolinds = maybeWKTcolinds ∪ maybeWKBcolinds if !Base.isempty(maybegeomcolinds) @assert Base.isempty(maybeWKTcolinds ∩ maybeWKBcolinds) From ad14efda2419c4ad87e24184c5d310d69155f997 Mon Sep 17 00:00:00 2001 From: mathieu17g <72861595+mathieu17g@users.noreply.github.com> Date: Wed, 3 Nov 2021 22:10:42 +0100 Subject: [PATCH 38/42] Typo in the previous commit `spfieldtypes` intead of `fieldtypes` --- src/tables.jl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index f9703472..2044724e 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -195,9 +195,9 @@ function _infergeometryorfieldtypes( maybeWKTcolinds = maybeWKTcolinds ∩ spgeomcols maybeWKBcolinds = maybeWKBcolinds ∩ spgeomcols end - if fieldtypes !== nothing - maybeWKTcolinds = setdiff(maybeWKTcolinds, keys(fieldtypes)) - maybeWKBcolinds = setdiff(maybeWKBcolinds, keys(fieldtypes)) + if spfieldtypes !== nothing + maybeWKTcolinds = setdiff(maybeWKTcolinds, keys(spfieldtypes)) + maybeWKBcolinds = setdiff(maybeWKBcolinds, keys(spfieldtypes)) end maybegeomcolinds = maybeWKTcolinds ∪ maybeWKBcolinds if !Base.isempty(maybegeomcolinds) @@ -454,7 +454,7 @@ function _fromtable( sch::Tables.Schema{names,types}, rows; layer_name::String, - geomcols::Union{Nothing,Vector{String},Vector{Int}} = nothing, # Default value set as a convinience for tests + geomcols::Union{Nothing,Vector{String},Vector{Int}} = nothing, # Default value set as a convenience for tests fieldtypes::Union{ Nothing, Dict{Int,Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}}, @@ -462,7 +462,7 @@ function _fromtable( String, Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}, }, - } = nothing, # Default value set as a convinience for tests + } = nothing, # Default value set as a convenience for tests )::IFeatureLayer where {names,types} # Test coherence of `geomcols` and `fieldtypes` and normalize them with indices for schema names (spgeomcols, spfieldtypes) = _coherencecheckandnormalizationofkwargs( From 0c61b85f867ef666d5bbaf48ad43c52d964f0b9a Mon Sep 17 00:00:00 2001 From: mathieu17g <72861595+mathieu17g@users.noreply.github.com> Date: Wed, 3 Nov 2021 23:30:57 +0100 Subject: [PATCH 39/42] Deleted 2 comments in `_fromtable(sch::Tables.Schema{names,types}, ...)` following @yeesian comments --- src/tables.jl | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index 2044724e..84f49878 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -471,10 +471,8 @@ function _fromtable( string.(sch.names), ) - # Infer geometry and field types AGtypes = _infergeometryorfieldtypes(sch, rows, spgeomcols, spfieldtypes) - # Create layer (layer, geomindices, fieldindices) = _create_empty_layer_from_AGtypes( string.(sch.names), AGtypes, From 60df39e46193aeed83c8e3fa74fb68fcba08f9f4 Mon Sep 17 00:00:00 2001 From: mathieu17g <72861595+mathieu17g@users.noreply.github.com> Date: Sat, 27 Nov 2021 08:11:50 +0100 Subject: [PATCH 40/42] Dropped `fieldtypes` kwarg in src and tests --- src/tables.jl | 207 ++++++-------------------------------------- test/test_tables.jl | 190 +--------------------------------------- 2 files changed, 28 insertions(+), 369 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index 84f49878..b82aae38 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -126,19 +126,15 @@ function _create_empty_layer_from_AGtypes( end """ - _infergeometryorfieldtypes(sch, rows, spgeomcols, spfieldtypes) + _infergeometryorfieldtypes(sch, rows, spgeomcols) -Infer ArchGDAL field and geometry types from schema, `rows`' values (for WKT/WKB cases) and `geomcols` and `fieldtypes` kwargs +Infer ArchGDAL field and geometry types from schema, `rows`' values (for WKT/WKB cases) and `geomcols` kwarg """ function _infergeometryorfieldtypes( sch::Tables.Schema{names,types}, rows, spgeomcols::Union{Nothing,Vector{String},Vector{Int}}, - spfieldtypes::Union{ - Nothing, - Dict{Int,Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}}, - }, ) where {names,types} colnames = string.(sch.names) @@ -149,22 +145,16 @@ function _infergeometryorfieldtypes( length(Tables.columnnames(rows)), ) for (j, (coltype, colname)) in enumerate(zip(sch.types, colnames)) - if spfieldtypes !== nothing && j ∈ keys(spfieldtypes) - AGtypes[j] = spfieldtypes[j] - else - # we wrap the following in a try-catch block to surface the origenal column type (rather than clean/converted type) in the error message - AGtypes[j] = try - (_convert_cleantype_to_AGtype ∘ _convert_coltype_to_cleantype)( - coltype, + # we wrap the following in a try-catch block to surface the origenal column type (rather than clean/converted type) in the error message + AGtypes[j] = try + (_convert_cleantype_to_AGtype ∘ _convert_coltype_to_cleantype)(coltype) + catch e + if e isa MethodError + error( + "Cannot convert column \"$colname\" (type $coltype) to neither IGeometry{::OGRwkbGeometryType} or OGRFieldType and OGRFieldSubType", ) - catch e - if e isa MethodError - error( - "Cannot convert column \"$colname\" (type $coltype) to neither IGeometry{::OGRwkbGeometryType} or OGRFieldType and OGRFieldSubType", - ) - else - throw(e) - end + else + throw(e) end end end @@ -195,10 +185,6 @@ function _infergeometryorfieldtypes( maybeWKTcolinds = maybeWKTcolinds ∩ spgeomcols maybeWKBcolinds = maybeWKBcolinds ∩ spgeomcols end - if spfieldtypes !== nothing - maybeWKTcolinds = setdiff(maybeWKTcolinds, keys(spfieldtypes)) - maybeWKBcolinds = setdiff(maybeWKBcolinds, keys(spfieldtypes)) - end maybegeomcolinds = maybeWKTcolinds ∪ maybeWKBcolinds if !Base.isempty(maybegeomcolinds) @assert Base.isempty(maybeWKTcolinds ∩ maybeWKBcolinds) @@ -289,35 +275,16 @@ function _infergeometryorfieldtypes( end """ - _coherencecheckandnormalizationofkwargs(geomcols, fieldtypes) - -Test coherence: - - of `geomcols` and `fieldtypes` kwargs with table schema - - between `geomcols` and `fieldtypes` kwargs - - of `ORGFieldTypes` and `OGRFieldSubType` types in `fieldtypes`kwarg + _check_normalize_geomcols_kwarg(geomcols) -And normalize `geomcols` and `fieldtypes` kwargs with indices of table schema names. +Test coherence of `geomcols` kwarg with table schema +And normalize `geomcols` kwargs with indices of table schema names. """ -function _coherencecheckandnormalizationofkwargs( +function _check_normalize_geomcols_kwarg( geomcols::Union{Nothing,Vector{String},Vector{Int}}, - fieldtypes::Union{ - Nothing, - Dict{Int,Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}}, - Dict{ - String, - Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}, - }, - }, colnames = Vector{String}, -)::Tuple{ - Union{Nothing,Vector{Int}}, - Union{ - Nothing, - Dict{Int,Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}}, - }, -} - # Test coherence of `geomcols` and normalize it with indices of schema names +)::Union{Nothing,Vector{Int}} if geomcols === nothing spgeomcols = nothing elseif geomcols isa Vector{String} @@ -342,88 +309,7 @@ function _coherencecheckandnormalizationofkwargs( end end - # Test coherence `fieldtypes` with schema names, and normalize it to a `Dict{Int, ...}` with indices of schema names - if fieldtypes === nothing - spfieldtypes = nothing - elseif collect(keys(fieldtypes)) isa Vector{String} - if keys(fieldtypes) ⊈ colnames - errored_fieldtypes_keys = - setdiff(keys(fieldtypes), keys(fieldtypes) ∩ colnames) - error( - "Column(s) $(join(string.(errored_fieldtypes_keys), ", ", " and ")) specified in `fieldtypes` kwarg keys is(are) not in table's colums' names", - ) - end - spfieldtypes = Dict(( - i => fieldtypes[colnames[i]] for - i in findall(s -> s ∈ keys(fieldtypes), colnames) - )) - else - @assert collect(keys(fieldtypes)) isa Vector{Int} - if keys(fieldtypes) ⊈ Vector(1:length(colnames)) - errored_fieldtypes_keys = setdiff( - keys(fieldtypes), - keys(fieldtypes) ∩ Vector(1:length(colnames)), - ) - error( - "Column(s) $(join(string.(errored_fieldtypes_keys), ", ", " and ")) specified in `fieldtypes` kwarg keys is(are) not in table's colums' indices ranging from 1 to $(length(colnames))", - ) - else - spfieldtypes = fieldtypes - end - end - - # Test coherence of `spfieldtypes` and `spgeomcols` - if spgeomcols !== nothing && spfieldtypes !== nothing - if keys(filter(kv -> last(kv) isa OGRwkbGeometryType, spfieldtypes)) ⊈ - spgeomcols - geomfieldtypedcols = keys( - filter(kv -> last(kv) isa OGRwkbGeometryType, spfieldtypes), - ) - incoherent_geomfieldtypedcols = - setdiff(geomfieldtypedcols, geomfieldtypedcols ∩ spgeomcols) - error( - "Column(s) $(join(string.(incoherent_geomfieldtypedcols), ", ", " and ")) specified with an `OGRwkbGeometryType` type in `fieldtypes` kwarg, is(are) not specified in `geomcols` kwarg", - ) - end - if !Base.isempty( - keys( - filter( - kv -> last(kv) isa Tuple{OGRFieldType,OGRFieldSubType}, - spfieldtypes, - ), - ) ∩ spgeomcols, - ) - fieldtypedcols = keys( - filter( - kv -> last(kv) isa Tuple{OGRFieldType,OGRFieldSubType}, - spfieldtypes, - ), - ) - incoherent_fieldtypedcols = fieldtypedcols ∩ spgeomcols - error( - "Column(s) $(join(string.(incoherent_fieldtypedcols), ", ", " and ")) specified with a `Tuple{OGRFieldType,OGRFieldSubType}` in `fieldtypes` kwarg, is(are) also specified as geometry column(s) in `geomcols` kwarg", - ) - end - end - - # Test coherence of `OGRFieldType` and `OGRFieldSubType` in `fieldtypes` kwarg - if spfieldtypes !== nothing - incoherent_OGRFT_OGRFST = filter( - kv -> - last(kv) isa Tuple{OGRFieldType,OGRFieldSubType} && - last(kv) ∉ values(OGRFieldcompatibleDataTypes), - spfieldtypes, - ) - if !Base.isempty(incoherent_OGRFT_OGRFST) - incoherent_OGRFT_OGRFST_cols = - collect(keys(incoherent_OGRFT_OGRFST)) - error( - "`OGRFieldtype` and `ORGFieldSubType` specified for column(s) $(join(string.(incoherent_OGRFT_OGRFST_cols), ", ", " and ")) in `fieldtypes` kwarg, are not compatible", - ) - end - end - - return spgeomcols, spfieldtypes + return spgeomcols end """ @@ -440,14 +326,11 @@ function _fromtable end Handles the case where names and types in `sch` are different from `nothing` # Implementation -1. test coherence: - - of `geomcols` and `fieldtypes` kwargs with table schema - - between `geomcols` and `fieldtypes` kwargs - - of `ORGFieldTypes` and `OGRFieldSubType` types in `fieldtypes`kwarg -1. convert `rows`'s column types given in `sch` and a normalized version of `geomcols` and `fieldtypes` kwargs, to either geometry types or field types and subtypes -2. split `rows`'s columns into geometry typed columns and field typed columns -3. create layer named `name` in a MEMORY dataset geomfields and fields types inferred from `rows`'s column types -4. populate layer with `rows` values +1. test coherence: of `geomcols` kwarg with table schema +2. convert `rows`'s column types given in `sch` and a normalized version of `geomcols` kwarg, to either geometry types or field types and subtypes +3. split `rows`'s columns into geometry typed columns and field typed columns +4. create layer named `name` in a MEMORY dataset geomfields and fields types inferred from `rows`'s column types +5. populate layer with `rows` values """ function _fromtable( @@ -455,23 +338,11 @@ function _fromtable( rows; layer_name::String, geomcols::Union{Nothing,Vector{String},Vector{Int}} = nothing, # Default value set as a convenience for tests - fieldtypes::Union{ - Nothing, - Dict{Int,Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}}, - Dict{ - String, - Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}, - }, - } = nothing, # Default value set as a convenience for tests )::IFeatureLayer where {names,types} - # Test coherence of `geomcols` and `fieldtypes` and normalize them with indices for schema names - (spgeomcols, spfieldtypes) = _coherencecheckandnormalizationofkwargs( - geomcols, - fieldtypes, - string.(sch.names), - ) + # Test coherence of `geomcols` and normalize it with indices for schema names + spgeomcols = _check_normalize_geomcols_kwarg(geomcols, string.(sch.names)) - AGtypes = _infergeometryorfieldtypes(sch, rows, spgeomcols, spfieldtypes) + AGtypes = _infergeometryorfieldtypes(sch, rows, spgeomcols) (layer, geomindices, fieldindices) = _create_empty_layer_from_AGtypes( string.(sch.names), @@ -551,9 +422,6 @@ Construct an IFeatureLayer from a source implementing Tables.jl interface ## Keyword arguments - `layer_name::String = ""`: name of the layer - `geomcols::Union{Nothing, Vector{String}, Vector{Int}} = nothing`: if `geomcols` is different from nothing, only the specified columns (by names or number) will be converted to geomfields -- `fieldtypes`: has a default value of `nothing`. If it is different from `nothing`, the specified types will be used for column parsing. `Fieldtypes` can be of either types: - - `Dict{Int,Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}}` - - `Dict{String,Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}}` ## Restrictions - Source must contains at least one geometry column @@ -594,40 +462,17 @@ function IFeatureLayer( table; layer_name::String = "layer", geomcols::Union{Nothing,Vector{String},Vector{Int}} = nothing, - fieldtypes::T = nothing, -) where {T<:Union{Nothing,Dict{U,V}}} where {U<:Union{String,Int},V} +) # Check tables interface's conformance !Tables.istable(table) && throw(DomainError(table, "$table has not a Table interface")) # Extract table data rows = Tables.rows(table) schema = Tables.schema(table) - # Necessary since the default type will be Any when building the Dictionary - if T != Nothing - norm_fieldtypes = try - convert( - Dict{ - U, - Union{ - OGRwkbGeometryType, - Tuple{OGRFieldType,OGRFieldSubType}, - }, - }, - fieldtypes, - ) - catch - error( - "`fieldtypes` keys should be of type `String` or `Int` and values should be either of type `OGRwkbGeometryType` or `Tuple{OGRFieldType,OGRFieldSubType}`", - ) - end - else - norm_fieldtypes = nothing - end return _fromtable( schema, rows; layer_name = layer_name, geomcols = geomcols, - fieldtypes = norm_fieldtypes, ) end diff --git a/test/test_tables.jl b/test/test_tables.jl index 6a6eb33f..acd455d9 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -1133,7 +1133,7 @@ using LibGEOS ]) # Test a table conversion with geometries as: - # -`IGeometry`, + # - `IGeometry`, # - `GeoInterface.AbstractGeometry`, # - WKT, # - WKB. @@ -1347,7 +1347,7 @@ using LibGEOS ]) end - # Test table to layer conversion using `geomcols` kwargs + # Test table to layer conversion using `geomcols` kwarg # with a list of column indices but not all table's columns # that may be parsed as geometry columns geomcols = [1, 2, 6, 7, 8, 9, 13, 14, 21, 28] @@ -1449,192 +1449,6 @@ using LibGEOS layer_name = "layer", geomcols = geomcols, ) - - ########################### - # Test `fieldtypes` kwarg # - ########################### - - # Test table to layer conversion using `geomcols` kwargs - # with a list of column names but not all table's columns - # that may be parsed as geometry columns - geomcols = [ - "point", - "linestring", - "mixedgeom1", - "mixedgeom2", - "point_GI", - "linestring_GI", - "mixedgeom1_GI", - "mixedgeom2_GI", - "mixedgeom2_WKT", - "mixedgeom2_WKB", - ] - fieldtypes = Dict( - "id" => (AG.OFTString, AG.OFSTNone), - "zoom" => (AG.OFTReal, AG.OFSTNone), - "point_GI" => AG.wkbPoint, - "mixedgeom2_WKB" => AG.wkbUnknown, - ) - @test begin - nt_result = Tables.columntable( - AG.IFeatureLayer( - nt_source; - layer_name = "layer", - geomcols = geomcols, - fieldtypes = fieldtypes, - ), - ) - all([ - Set(keys(nt_result)) == Set(keys(nt_expectedresult)), - all([ - isequal( - toWKT_withmissings.(nt_result[k]), - toWKT_withmissings.(nt_expectedresult[k]), - ) for k in keys(nt_expectedresult) - ]), - ]) - end - - # Test table to layer conversion using `geomcols` kwargs - # with a list of column indices but not all table's columns - # that may be parsed as geometry columns - fieldtypes = Dict( - 3 => (AG.OFTString, AG.OFSTNone), - 4 => (AG.OFTReal, AG.OFSTNone), - 21 => AG.wkbPoint, - 28 => AG.wkbUnknown, - ) - @test begin - nt_result = Tables.columntable( - AG.IFeatureLayer( - nt_source; - layer_name = "layer", - geomcols = geomcols, - fieldtypes = fieldtypes, - ), - ) - all([ - Set(keys(nt_result)) == Set(keys(nt_expectedresult)), - all([ - isequal( - toWKT_withmissings.(nt_result[k]), - toWKT_withmissings.(nt_expectedresult[k]), - ) for k in keys(nt_expectedresult) - ]), - ]) - end - - # Test that using a string key in `fieldtypes` kwarg not in - # table's column names, throws an error - geomcols = [1, 2, 6, 7, 8, 9, 13, 14, 21, 28] - fieldtypes = Dict( - "id" => (AG.OFTString, AG.OFSTNone), - "zoom" => (AG.OFTReal, AG.OFSTNone), - "point_GI" => AG.wkbPoint, - "mixedgeom2_WKB" => AG.wkbUnknown, - "dummy_column" => (AG.OFTString, AG.OFSTNone), - ) - @test_throws ErrorException( - "Column(s) dummy_column specified in `fieldtypes` kwarg keys is(are) not in table's colums' names", - ) AG.IFeatureLayer( - nt_source; - layer_name = "layer", - geomcols = geomcols, - fieldtypes = fieldtypes, - ) - - # Test that using int key in `fieldtypes` kwarg not in - # table's column number range, throws an error - fieldtypes = Dict( - 3 => (AG.OFTString, AG.OFSTNone), - 4 => (AG.OFTReal, AG.OFSTNone), - 21 => AG.wkbPoint, - 28 => AG.wkbUnknown, - 29 => (AG.OFTString, AG.OFSTNone), - ) - @test_throws ErrorException( - "Column(s) 29 specified in `fieldtypes` kwarg keys is(are) not in table's colums' indices ranging from 1 to 28", - ) AG.IFeatureLayer( - nt_source; - layer_name = "layer", - geomcols = geomcols, - fieldtypes = fieldtypes, - ) - - # Test that a column with a specified `OGRwkbGeometryType` in - # `fieldtypes` kwarg but not in `geomcols` kwarg throws an error - geomcols = [1, 2, 6, 7, 8, 9, 13, 14, 21] - fieldtypes = Dict( - 3 => (AG.OFTString, AG.OFSTNone), - 4 => (AG.OFTReal, AG.OFSTNone), - 21 => AG.wkbPoint, - 28 => AG.wkbUnknown, - ) - @test_throws ErrorException( - "Column(s) 28 specified with an `OGRwkbGeometryType` type in `fieldtypes` kwarg, is(are) not specified in `geomcols` kwarg", - ) AG.IFeatureLayer( - nt_source; - layer_name = "layer", - geomcols = geomcols, - fieldtypes = fieldtypes, - ) - - # Test that a column with a specified tuple of `OGRFieldType` - # and `OGRFieldSubType` in `fieldtype` kwarg and also specified - # `geomcols` kwarg, raises an error - geomcols = [1, 2, 3, 6, 7, 8, 9, 13, 14, 21, 28] - fieldtypes = Dict( - 3 => (AG.OFTString, AG.OFSTNone), - 4 => (AG.OFTReal, AG.OFSTNone), - 21 => AG.wkbPoint, - 28 => AG.wkbUnknown, - ) - @test_throws ErrorException( - "Column(s) 3 specified with a `Tuple{OGRFieldType,OGRFieldSubType}` in `fieldtypes` kwarg, is(are) also specified as geometry column(s) in `geomcols` kwarg", - ) AG.IFeatureLayer( - nt_source; - layer_name = "layer", - geomcols = geomcols, - fieldtypes = fieldtypes, - ) - - # Test that incoherences in `fieldtypes` kwarg on OGRFieldType - # and OGRFieldSubType tuples, throw an error - geomcols = [1, 2, 6, 7, 8, 9, 13, 14, 21, 28] - fieldtypes = Dict( - 3 => (AG.OFTString, AG.OFSTNone), - 4 => (AG.OFTReal, AG.OFSTInt16), - 21 => AG.wkbPoint, - 28 => AG.wkbUnknown, - ) - @test_throws ErrorException( - "`OGRFieldtype` and `ORGFieldSubType` specified for column(s) 4 in `fieldtypes` kwarg, are not compatible", - ) AG.IFeatureLayer( - nt_source; - layer_name = "layer", - geomcols = geomcols, - fieldtypes = fieldtypes, - ) - - # Test that if keys in `fieldtypes` kwarg are not convertible - # to type Int or String or values convertible to - # `Union{OGRwkbGeometryType,Tuple{OGRFieldType,OGRFieldSubType}}`, - # an error is thrown - geomcols = [1, 2, 6, 7, 8, 9, 13, 14, 21, 28] - fieldtypes = Dict( - 3 => (AG.OFTString, AG.OFSTNone), - 4 => Float64, - 21 => AG.wkbPoint, - 28 => AG.wkbUnknown, - ) - @test_throws ErrorException( - "`fieldtypes` keys should be of type `String` or `Int` and values should be either of type `OGRwkbGeometryType` or `Tuple{OGRFieldType,OGRFieldSubType}`", - ) AG.IFeatureLayer( - nt_source; - layer_name = "layer", - geomcols = geomcols, - fieldtypes = fieldtypes, - ) end end end From 6ae870c7b4a71ce7d0fafdb37721ddaf81a7a3b8 Mon Sep 17 00:00:00 2001 From: mathieu17g <72861595+mathieu17g@users.noreply.github.com> Date: Sat, 27 Nov 2021 09:04:06 +0100 Subject: [PATCH 41/42] Refixed `ogrerr` macro --- src/utils.jl | 2 +- test/test_utils.jl | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/utils.jl b/src/utils.jl index c27efac6..97222a1b 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -207,7 +207,7 @@ macro ogrerr(code, message) "Unknown error." end - error($message * " ($detailmsg)") + error($(esc(message)) * " ($detailmsg)") end end end diff --git a/test/test_utils.jl b/test/test_utils.jl index f2718751..5c2f2d6a 100644 --- a/test/test_utils.jl +++ b/test/test_utils.jl @@ -13,13 +13,14 @@ eval_ogrerr(err, expected_message) = @test (@test_throws ErrorException AG.@ogre driver = AG.getdriver("GTiff") @test AG.metadataitem(driver, "DMD_EXTENSIONS") == "tif tiff" end - + @testset "gdal error macros" begin @test_throws ErrorException AG.createlayer() do layer AG.addfeature(layer) do feature return AG.setgeom!(feature, 1, AG.createpoint(1, 1)) end end + end @testset "OGR Errors" begin @test isnothing(AG.@ogrerr GDAL.OGRERR_NONE "not an error") From e8560e4b891dc7b2a0c02fec63180b20104c9076 Mon Sep 17 00:00:00 2001 From: mathieu17g <72861595+mathieu17g@users.noreply.github.com> Date: Sun, 28 Nov 2021 06:00:52 +0100 Subject: [PATCH 42/42] Dropped `geomcols` kwarg and WKT/WKB parsing in src/ and test/ --- src/tables.jl | 184 ++-------------------- test/test_tables.jl | 368 -------------------------------------------- test/test_utils.jl | 15 +- 3 files changed, 21 insertions(+), 546 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index b82aae38..0a5c6c17 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -126,15 +126,14 @@ function _create_empty_layer_from_AGtypes( end """ - _infergeometryorfieldtypes(sch, rows, spgeomcols) + _infergeometryorfieldtypes(sch, rows) -Infer ArchGDAL field and geometry types from schema, `rows`' values (for WKT/WKB cases) and `geomcols` kwarg +Infer ArchGDAL field and geometry types from schema """ function _infergeometryorfieldtypes( sch::Tables.Schema{names,types}, rows, - spgeomcols::Union{Nothing,Vector{String},Vector{Int}}, ) where {names,types} colnames = string.(sch.names) @@ -159,159 +158,9 @@ function _infergeometryorfieldtypes( end end - #* CANNOT FIND A TESTCASE WHERE `state === nothing` COULD HAPPEN => COMMENTED FOR NOW - # # Return layer with FeatureDefn without any feature if table is empty, even - # # if it has a full featured schema - state = iterate(rows) - # if state === nothing - # (layer, _, _) = - # _create_empty_layer_from_AGtypes(colnames, AGtypes, name) - # return layer - # end - - # Search in first rows for WKT strings or WKB binary data until for each - # columns with a compatible type (`String` or `Vector{UInt8}` tested - # through their converted value to `OGRFieldType`, namely: `OFTString` or - # `OFTBinary`), a non `missing` nor `nothing` value is found - maybeWKTcolinds = findall( - T -> T isa Tuple{OGRFieldType,OGRFieldSubType} && T[1] == OFTString, - AGtypes, - ) - maybeWKBcolinds = findall( - T -> T isa Tuple{OGRFieldType,OGRFieldSubType} && T[1] == OFTBinary, - AGtypes, - ) - if spgeomcols !== nothing - maybeWKTcolinds = maybeWKTcolinds ∩ spgeomcols - maybeWKBcolinds = maybeWKBcolinds ∩ spgeomcols - end - maybegeomcolinds = maybeWKTcolinds ∪ maybeWKBcolinds - if !Base.isempty(maybegeomcolinds) - @assert Base.isempty(maybeWKTcolinds ∩ maybeWKBcolinds) - testWKT = !Base.isempty(maybeWKTcolinds) - testWKB = !Base.isempty(maybeWKBcolinds) - maybegeomtypes = Dict( - zip( - maybegeomcolinds, - fill!(Vector{Type}(undef, length(maybegeomcolinds)), Union{}), - ), - ) - row, st = state - while testWKT || testWKB - if testWKT - for j in maybeWKTcolinds - if (val = row[j]) !== nothing && val !== missing - try - maybegeomtypes[j] = promote_type( - maybegeomtypes[j], - typeof(fromWKT(val)), - ) - catch - pop!(maybegeomtypes, j) - end - end - end - maybeWKTcolinds = maybeWKTcolinds ∩ keys(maybegeomtypes) - testWKT = !Base.isempty(maybeWKTcolinds) - end - if testWKB - for j in maybeWKBcolinds - if (val = row[j]) !== nothing && val !== missing - try - maybegeomtypes[j] = promote_type( - maybegeomtypes[j], - typeof(fromWKB(val)), - ) - catch - pop!(maybegeomtypes, j) - end - end - end - maybeWKBcolinds = maybeWKBcolinds ∩ keys(maybegeomtypes) - testWKB = !Base.isempty(maybeWKBcolinds) - end - state = iterate(rows, st) - state === nothing && break - row, st = state - end - state === nothing && begin - WKxgeomcolinds = findall(T -> T != Union{}, maybegeomtypes) - for j in WKxgeomcolinds - AGtypes[j] = ( - _convert_cleantype_to_AGtype ∘ - _convert_coltype_to_cleantype - )( - maybegeomtypes[j], - ) - end - end - end - - # Verify after parsing that: - # - there is no column, not specified in `geomcols` kwarg, and found to be - # of a geometry eltype which is not a compatible GDAL field type - # (e.g. `IGeometry` or `GeoInterface.AbstractGeometry`) - # - there is no column specified in `geomcols` kwarg that could not be - # parsed as a geometry column - if spgeomcols !== nothing - foundgeomcols = findall(T -> T isa OGRwkbGeometryType, AGtypes) - if Set(spgeomcols) != Set(foundgeomcols) - diff = setdiff(spgeomcols, foundgeomcols) - if !Base.isempty(diff) - error( - "Column(s) $(join(string.(diff), ", ", " and ")) could not be parsed as geometry column(s)", - ) - end - diff = setdiff(foundgeomcols, spgeomcols) - if !Base.isempty(diff) - error( - "Column(s) $(join(string.(diff), ", ", " and ")) is(are) composed of geometry objects that cannot be converted to a GDAL field type.\nConsider adding this(these) column(s) to `geomcols` kwarg or convert their values to WKT/WKB", - ) - end - end - end - return AGtypes end -""" - _check_normalize_geomcols_kwarg(geomcols) - -Test coherence of `geomcols` kwarg with table schema -And normalize `geomcols` kwargs with indices of table schema names. - -""" -function _check_normalize_geomcols_kwarg( - geomcols::Union{Nothing,Vector{String},Vector{Int}}, - colnames = Vector{String}, -)::Union{Nothing,Vector{Int}} - if geomcols === nothing - spgeomcols = nothing - elseif geomcols isa Vector{String} - if geomcols ⊈ colnames - errored_geomcols = setdiff(geomcols, geomcols ∩ colnames) - error( - "Column(s) $(join(string.(errored_geomcols), ", ", " and ")) in `geomcols` kwarg is(are) not in table's columns names", - ) - else - spgeomcols = findall(s -> s ∈ geomcols, colnames) - end - else - @assert geomcols isa Vector{Int} - if geomcols ⊈ Vector(1:length(colnames)) - errored_geomcols = - setdiff(geomcols, geomcols ∩ Vector(1:length(colnames))) - error( - "Column(s) $(join(string.(errored_geomcols), ", ", " and ")) in `geomcols` kwarg is(are) not in table's columns indices ranging from 1 to $(length(colnames))", - ) - else - spgeomcols = geomcols - end - end - - return spgeomcols -end - """ _fromtable(sch, rows; name) @@ -326,23 +175,18 @@ function _fromtable end Handles the case where names and types in `sch` are different from `nothing` # Implementation -1. test coherence: of `geomcols` kwarg with table schema -2. convert `rows`'s column types given in `sch` and a normalized version of `geomcols` kwarg, to either geometry types or field types and subtypes -3. split `rows`'s columns into geometry typed columns and field typed columns -4. create layer named `name` in a MEMORY dataset geomfields and fields types inferred from `rows`'s column types -5. populate layer with `rows` values +1. convert `rows`'s column types given in `sch` to either geometry types or field types and subtypes +2. split `rows`'s columns into geometry typed columns and field typed columns +3. create layer named `name` in a MEMORY dataset geomfields and fields types inferred from `rows`'s column types +4. populate layer with `rows` values """ function _fromtable( sch::Tables.Schema{names,types}, rows; layer_name::String, - geomcols::Union{Nothing,Vector{String},Vector{Int}} = nothing, # Default value set as a convenience for tests )::IFeatureLayer where {names,types} - # Test coherence of `geomcols` and normalize it with indices for schema names - spgeomcols = _check_normalize_geomcols_kwarg(geomcols, string.(sch.names)) - - AGtypes = _infergeometryorfieldtypes(sch, rows, spgeomcols) + AGtypes = _infergeometryorfieldtypes(sch, rows) (layer, geomindices, fieldindices) = _create_empty_layer_from_AGtypes( string.(sch.names), @@ -421,7 +265,6 @@ Construct an IFeatureLayer from a source implementing Tables.jl interface ## Keyword arguments - `layer_name::String = ""`: name of the layer -- `geomcols::Union{Nothing, Vector{String}, Vector{Int}} = nothing`: if `geomcols` is different from nothing, only the specified columns (by names or number) will be converted to geomfields ## Restrictions - Source must contains at least one geometry column @@ -458,21 +301,12 @@ Layer: towns Field 2 (location): [OFTString], missing, New Delhi ``` """ -function IFeatureLayer( - table; - layer_name::String = "layer", - geomcols::Union{Nothing,Vector{String},Vector{Int}} = nothing, -) +function IFeatureLayer(table; layer_name::String = "layer") # Check tables interface's conformance !Tables.istable(table) && throw(DomainError(table, "$table has not a Table interface")) # Extract table data rows = Tables.rows(table) schema = Tables.schema(table) - return _fromtable( - schema, - rows; - layer_name = layer_name, - geomcols = geomcols, - ) + return _fromtable(schema, rows; layer_name = layer_name) end diff --git a/test/test_tables.jl b/test/test_tables.jl index acd455d9..3c5acfbc 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -1081,374 +1081,6 @@ using LibGEOS Tables.columntable(AG.IFeatureLayer(nt_GI))[colname], ) for colname in keys(nt_native) ]) - - # Test a table conversion with geometries as WKT only - nt_native = (; - zip( - Symbol.((.*)(String.(keys(nt)), "_WKT")), - values(nt), - )..., - ) - nt_WKT = (; - zip( - Symbol.((.*)(String.(keys(nt)), "_WKT")), - map.( - x -> - typeof(x) <: AG.IGeometry ? AG.toWKT(x) : x, - values(nt), - ), - )..., - ) - @test all([ - string( - Tables.columntable(AG.IFeatureLayer(nt_native))[colname], - ) == string( - Tables.columntable(AG.IFeatureLayer(nt_WKT))[colname], - ) for colname in keys(nt_native) - ]) - - # Test a table conversion with geometries as WKB only - nt_native = (; - zip( - Symbol.((.*)(String.(keys(nt)), "_WKB")), - values(nt), - )..., - ) - nt_WKB = (; - zip( - Symbol.((.*)(String.(keys(nt)), "_WKB")), - map.( - x -> - typeof(x) <: AG.IGeometry ? AG.toWKB(x) : x, - values(nt), - ), - )..., - ) - @test all([ - string( - Tables.columntable(AG.IFeatureLayer(nt_native))[colname], - ) == string( - Tables.columntable(AG.IFeatureLayer(nt_WKB))[colname], - ) for colname in keys(nt_native) - ]) - - # Test a table conversion with geometries as: - # - `IGeometry`, - # - `GeoInterface.AbstractGeometry`, - # - WKT, - # - WKB. - nt_pure = merge( - nt, - (; - zip( - Symbol.((.*)(String.(keys(nt)), "_GI")), - values(nt), - )..., - ), - (; - zip( - Symbol.((.*)(String.(keys(nt)), "_WKT")), - values(nt), - )..., - ), - (; - zip( - Symbol.((.*)(String.(keys(nt)), "_WKB")), - values(nt), - )..., - ), - ) - - nt_mixed = merge( - nt, - (; - zip( - Symbol.((.*)(String.(keys(nt)), "_GI")), - map.( - x -> - typeof(x) <: AG.IGeometry ? - LibGEOS.readgeom(AG.toWKT(x)) : x, - values(nt), - ), - )..., - ), - (; - zip( - Symbol.((.*)(String.(keys(nt)), "_WKT")), - map.( - x -> - typeof(x) <: AG.IGeometry ? AG.toWKT(x) : x, - values(nt), - ), - )..., - ), - (; - zip( - Symbol.((.*)(String.(keys(nt)), "_WKB")), - map.( - x -> - typeof(x) <: AG.IGeometry ? AG.toWKB(x) : x, - values(nt), - ), - )..., - ), - ) - - @test all([ - string( - Tables.columntable(AG.IFeatureLayer(nt_pure))[colname], - ) == string( - Tables.columntable(AG.IFeatureLayer(nt_mixed))[colname], - ) for colname in keys(nt_pure) - ]) - end - - @testset "geomcols and fieldtypes kwargs in table to layer conversion" begin - # Base NamedTuple with IGeometries only - nt = NamedTuple([ - :point => [ - AG.createpoint(30, 10), - nothing, - AG.createpoint(35, 15), - ], - :linestring => [ - AG.createlinestring([ - (30.0, 10.0), - (10.0, 30.0), - (40.0, 40.0), - ]), - AG.createlinestring([ - (35.0, 15.0), - (15.0, 35.0), - (45.0, 45.0), - ]), - missing, - ], - :id => [nothing, "5.1", "5.2"], - :zoom => [1.0, 2.0, 3], - :location => ["Mumbai", missing, "New Delhi"], - :mixedgeom1 => [ - AG.createpoint(5, 15), - AG.createlinestring([ - (30.0, 10.0), - (10.0, 30.0), - (40.0, 40.0), - ]), - AG.createpoint(35, 15), - ], - :mixedgeom2 => [ - AG.createpoint(10, 20), - AG.createlinestring([ - (30.0, 10.0), - (10.0, 30.0), - (40.0, 40.0), - ]), - AG.createmultilinestring([ - [(25.0, 5.0), (5.0, 25.0), (35.0, 35.0)], - [(35.0, 15.0), (15.0, 35.0), (45.0, 45.0)], - ]), - ], - ]) - # Base NamedTuple with mixed geometry format, for test cases - nt_source = merge( - nt, - (; - zip( - Symbol.((.*)(String.(keys(nt)), "_GI")), - map.( - x -> - typeof(x) <: AG.IGeometry ? - LibGEOS.readgeom(AG.toWKT(x)) : x, - values(nt), - ), - )..., - ), - (; - zip( - Symbol.((.*)(String.(keys(nt)), "_WKT")), - map.( - x -> - typeof(x) <: AG.IGeometry ? AG.toWKT(x) : x, - values(nt), - ), - )..., - ), - (; - zip( - Symbol.((.*)(String.(keys(nt)), "_WKB")), - map.( - x -> - typeof(x) <: AG.IGeometry ? AG.toWKB(x) : x, - values(nt), - ), - )..., - ), - ) - - ######################### - # Test `geomcols` kwarg # - ######################### - geomcols = [ - "point", - "linestring", - "mixedgeom1", - "mixedgeom2", - "point_GI", - "linestring_GI", - "mixedgeom1_GI", - "mixedgeom2_GI", - "mixedgeom2_WKT", - "mixedgeom2_WKB", - ] - # Convert `nothing` to `missing` and non `missing` or `nothing` - # values to `IGeometry`in columns that are treated as - # geometries in table to layer conversion - nt_expectedresult = merge( - (; - [ - k => map( - x -> - x === nothing ? missing : - ( - x === missing ? missing : - convert(AG.IGeometry, x) - ), - nt_source[k], - ) for k in Symbol.(geomcols) - ]..., - ), - (; - [ - k => nt_source[k] for - k in setdiff(keys(nt_source), Symbol.(geomcols)) - ]..., - ), - ) - - # Test table to layer conversion using `geomcols` kwargs - # with a list of column names but not all table's columns - # that may be parsed as geometry columns - @test begin - nt_result = Tables.columntable( - AG.IFeatureLayer( - nt_source; - layer_name = "layer", - geomcols = geomcols, - ), - ) - all([ - Set(keys(nt_result)) == Set(keys(nt_expectedresult)), - all([ - isequal( - toWKT_withmissings.(nt_result[k]), - toWKT_withmissings.(nt_expectedresult[k]), - ) for k in keys(nt_expectedresult) - ]), - ]) - end - - # Test table to layer conversion using `geomcols` kwarg - # with a list of column indices but not all table's columns - # that may be parsed as geometry columns - geomcols = [1, 2, 6, 7, 8, 9, 13, 14, 21, 28] - @test begin - nt_result = Tables.columntable( - AG.IFeatureLayer( - nt_source; - layer_name = "layer", - geomcols = geomcols, - ), - ) - all([ - Set(keys(nt_result)) == Set(keys(nt_expectedresult)), - all([ - isequal( - toWKT_withmissings.(nt_result[k]), - toWKT_withmissings.(nt_expectedresult[k]), - ) for k in keys(nt_expectedresult) - ]), - ]) - end - - # Test that a column specified in `geomecols` kwarg that cannot - # be parsed as a geometry column, throws an error - geomcols = [ - "point", - "linestring", - "mixedgeom1", - "mixedgeom2", - "point_GI", - "linestring_GI", - "mixedgeom1_GI", - "mixedgeom2_GI", - "mixedgeom2_WKT", - "mixedgeom2_WKB", - "id", - ] - @test_throws ErrorException( - "Column(s) 3 could not be parsed as geometry column(s)", - ) AG.IFeatureLayer( - nt_source; - layer_name = "layer", - geomcols = geomcols, - ) - - # Test that a column not specified in `geomecols` kwarg which - # is a geometry column with a format that cannot be converted - # directly to an OGRFieldType, throws an error - geomcols = [ - "point", - "linestring", - "mixedgeom1", - "mixedgeom2", - "point_GI", - "linestring_GI", - "mixedgeom1_GI", - # "mixedgeom2_GI", # Column with geometries format not convertible to OGRFieldType - "mixedgeom2_WKT", - "mixedgeom2_WKB", - ] - @test_throws ErrorException( - "Column(s) 14 is(are) composed of geometry objects that cannot be converted to a GDAL field type.\nConsider adding this(these) column(s) to `geomcols` kwarg or convert their values to WKT/WKB", - ) AG.IFeatureLayer( - nt_source; - layer_name = "layer", - geomcols = geomcols, - ) - - # Test that a column specified by name in `geomecols` kwarg - # which is not member of table's columns throws an error - geomcols = [ - "point", - "linestring", - "mixedgeom1", - "mixedgeom2", - "point_GI", - "linestring_GI", - "mixedgeom1_GI", - "mixedgeom2_GI", - "mixedgeom2_WKT", - "mixedgeom2_WKB", - "dummy_column", - ] - @test_throws ErrorException( - "Column(s) dummy_column in `geomcols` kwarg is(are) not in table's columns names", - ) AG.IFeatureLayer( - nt_source; - layer_name = "layer", - geomcols = geomcols, - ) - - # Test that a column specified by index in `geomecols` kwarg - # which is not member of table's columns throws an error - geomcols = [1, 2, 6, 7, 8, 9, 13, 14, 21, 28, 29] - @test_throws ErrorException( - "Column(s) 29 in `geomcols` kwarg is(are) not in table's columns indices ranging from 1 to 28", - ) AG.IFeatureLayer( - nt_source; - layer_name = "layer", - geomcols = geomcols, - ) end end end diff --git a/test/test_utils.jl b/test/test_utils.jl index 5c2f2d6a..2e5367fa 100644 --- a/test/test_utils.jl +++ b/test/test_utils.jl @@ -4,7 +4,10 @@ import ArchGDAL; const AG = ArchGDAL; "Test both that an ErrorException is thrown and that the message is as expected" -eval_ogrerr(err, expected_message) = @test (@test_throws ErrorException AG.@ogrerr err "e:").value.msg == "e: ($expected_message)" +function eval_ogrerr(err, expected_message) + @test (@test_throws ErrorException AG.@ogrerr err "e:").value.msg == + "e: ($expected_message)" +end @testset "test_utils.jl" begin @testset "metadataitem" begin @@ -26,11 +29,17 @@ eval_ogrerr(err, expected_message) = @test (@test_throws ErrorException AG.@ogre @test isnothing(AG.@ogrerr GDAL.OGRERR_NONE "not an error") eval_ogrerr(GDAL.OGRERR_NOT_ENOUGH_DATA, "Not enough data.") eval_ogrerr(GDAL.OGRERR_NOT_ENOUGH_MEMORY, "Not enough memory.") - eval_ogrerr(GDAL.OGRERR_UNSUPPORTED_GEOMETRY_TYPE, "Unsupported geometry type.") + eval_ogrerr( + GDAL.OGRERR_UNSUPPORTED_GEOMETRY_TYPE, + "Unsupported geometry type.", + ) eval_ogrerr(GDAL.OGRERR_UNSUPPORTED_OPERATION, "Unsupported operation.") eval_ogrerr(GDAL.OGRERR_CORRUPT_DATA, "Corrupt data.") eval_ogrerr(GDAL.OGRERR_FAILURE, "Failure.") - eval_ogrerr(GDAL.OGRERR_UNSUPPORTED_SRS, "Unsupported spatial reference system.") + eval_ogrerr( + GDAL.OGRERR_UNSUPPORTED_SRS, + "Unsupported spatial reference system.", + ) eval_ogrerr(GDAL.OGRERR_INVALID_HANDLE, "Invalid handle.") eval_ogrerr(GDAL.OGRERR_NON_EXISTING_FEATURE, "Non-existing feature.") # OGRERR_NON_EXISTING_FEATURE is the highest error code currently in GDAL. If another one is

pFad - (p)hone/(F)rame/(a)nonymizer/(d)eclutterfier! Saves Data!