Skip to content

Add support for geopackage #88

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
iandees merged 1 commit into from
Jan 30, 2026
Merged

Add support for geopackage #88

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,3 +60,5 @@ openaddr-process-one --skip-preview --layer addresses --layersource state exampl
```

Review https://github.com/openaddresses/openaddresses/blob/master/CONTRIBUTING.md for input json syntax.

Supported conform formats include `shapefile`, `geojson`, `csv`, `xml`, `gdb`, and `gpkg`.
28 changes: 25 additions & 3 deletions openaddr/conform.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -404,6 +404,28 @@ def find_source_path(data_source, source_paths):
return c
_L.warning("Source names file %s but could not find it", source_file_name)
return None
elif format_string == "gpkg":
candidates = []
for fn in source_paths:
basename, ext = os.path.splitext(fn)
if ext.lower() == ".gpkg":
candidates.append(fn)
if len(candidates) == 0:
_L.warning("No GPKG found in %s", source_paths)
return None
elif len(candidates) == 1:
_L.debug("Selected %s for source", candidates[0])
return candidates[0]
else:
if "file" not in conform:
_L.warning("Multiple GPKGs found, but source has no file attribute.")
return None
source_file_name = conform["file"]
for c in candidates:
if source_file_name == os.path.basename(c):
return c
_L.warning("Source names file %s but could not find it", source_file_name)
return None
elif format_string == "xml":
# Return file if it's specified, else return the first .gml file we find
if "file" in conform:
Expand All @@ -425,7 +447,7 @@ def find_source_path(data_source, source_paths):
return None

class ConvertToGeojsonTask(object):
known_types = ('.shp', '.json', '.csv', '.kml', '.gdb')
known_types = ('.shp', '.json', '.csv', '.kml', '.gdb', '.gpkg')

def convert(self, source_config, source_paths, workdir):
"Convert a list of source_paths and write results in workdir"
Expand Down Expand Up @@ -1151,7 +1173,7 @@ def extract_to_source_csv(source_config, source_path, extract_path):
format_string = source_config.data_source["conform"]['format']
protocol_string = source_config.data_source['protocol']

if format_string in ("shapefile", "xml", "gdb"):
if format_string in ("shapefile", "xml", "gdb", "gpkg"):
ogr_source_path = normalize_ogr_filename_case(source_path)
ogr_source_to_csv(source_config, ogr_source_path, extract_path)
elif format_string == "csv":
Expand Down Expand Up @@ -1194,7 +1216,7 @@ def conform_cli(source_config, source_path, dest_path):

format_string = source_config.data_source["conform"].get('format')

if not format_string in ["shapefile", "geojson", "csv", "xml", "gdb"]:
if not format_string in ["shapefile", "geojson", "csv", "xml", "gdb", "gpkg"]:
_L.warning("Skipping file with unknown conform: %s", source_path)
return 1

Expand Down
34 changes: 32 additions & 2 deletions openaddr/tests/conform.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1776,6 +1776,36 @@ def test_lake_man_gdb(self):
self.assertEqual(rows[5]['properties']['number'], '5115')
self.assertEqual(rows[5]['properties']['street'], 'OLD MILL RD')

def test_lake_man_gpkg(self):
with open(os.path.join(self.conforms_dir, "lake-man-gpkg.json")) as file:
source_config = SourceConfig(json.load(file), "addresses", "default")
source_path = os.path.join(self.conforms_dir, "lake-man.gpkg")
dest_path = os.path.join(self.testdir, 'lake-man-gpkg-conformed.csv')

rc = conform_cli(source_config, source_path, dest_path)
self.assertEqual(0, rc)

with open(dest_path) as fp:
rows = list(map(json.loads, list(fp)))

self.assertEqual('Point', rows[0]['geometry']['type'])
self.assertAlmostEqual(-122.2592497, rows[0]['geometry']['coordinates'][0], places=4)
self.assertAlmostEqual(37.8026126, rows[0]['geometry']['coordinates'][1], places=4)

self.assertEqual(6, len(rows))
self.assertEqual(rows[0]['properties']['number'], '5115')
self.assertEqual(rows[0]['properties']['street'], 'FRUITED PLAINS LN')
self.assertEqual(rows[1]['properties']['number'], '5121')
self.assertEqual(rows[1]['properties']['street'], 'FRUITED PLAINS LN')
self.assertEqual(rows[2]['properties']['number'], '5133')
self.assertEqual(rows[2]['properties']['street'], 'FRUITED PLAINS LN')
self.assertEqual(rows[3]['properties']['number'], '5126')
self.assertEqual(rows[3]['properties']['street'], 'FRUITED PLAINS LN')
self.assertEqual(rows[4]['properties']['number'], '5120')
self.assertEqual(rows[4]['properties']['street'], 'FRUITED PLAINS LN')
self.assertEqual(rows[5]['properties']['number'], '5115')
self.assertEqual(rows[5]['properties']['street'], 'OLD MILL RD')

def test_lake_man_split(self):
rc, dest_path = self._run_conform_on_source('lake-man-split', 'shp')
self.assertEqual(0, rc)
Expand Down Expand Up @@ -2309,8 +2339,8 @@ def test_srs(self):
self.assertEqual(r[0], u'n,s,{GEOM_FIELDNAME}'.format(**globals()))

r = r[1].split(',')
self.assertEquals(r[0], '3203')
self.assertEquals(r[1], 'SE WOODSTOCK BLVD')
self.assertEqual(r[0], '3203')
self.assertEqual(r[1], 'SE WOODSTOCK BLVD')

x,y = wkt_pt(r[2])
self.assertAlmostEqual(-122.630842186651, x, places=4)
Expand Down
101 changes: 101 additions & 0 deletions openaddr/tests/conforms/lake-man-gml.gfs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
<GMLFeatureClassList>
<GMLFeatureClass>
<Name>lake_man</Name>
<ElementPath>lake_man</ElementPath>
<GeometryName>geometryProperty</GeometryName>
<GeometryElementPath>geometryProperty</GeometryElementPath>
<!--POINT-->
<GeometryType>1</GeometryType>
<DatasetSpecificInfo>
<FeatureCount>6</FeatureCount>
<ExtentXMin>-122.25925</ExtentXMin>
<ExtentXMax>-122.25672</ExtentXMax>
<ExtentYMin>37.80071</ExtentYMin>
<ExtentYMax>37.80436</ExtentYMax>
</DatasetSpecificInfo>
<PropertyDefn>
<Name>MSTRID</Name>
<ElementPath>MSTRID</ElementPath>
<Type>Integer</Type>
</PropertyDefn>
<PropertyDefn>
<Name>CATEGORY</Name>
<ElementPath>CATEGORY</ElementPath>
<Type>String</Type>
<Width>7</Width>
</PropertyDefn>
<PropertyDefn>
<Name>ADDRESSID</Name>
<ElementPath>ADDRESSID</ElementPath>
<Type>Integer</Type>
</PropertyDefn>
<PropertyDefn>
<Name>BASEID</Name>
<ElementPath>BASEID</ElementPath>
<Type>Integer</Type>
</PropertyDefn>
<PropertyDefn>
<Name>REVDATE</Name>
<ElementPath>REVDATE</ElementPath>
<Type>String</Type>
<Width>10</Width>
</PropertyDefn>
<PropertyDefn>
<Name>FLOOR</Name>
<ElementPath>FLOOR</ElementPath>
<Type>Integer</Type>
</PropertyDefn>
<PropertyDefn>
<Name>NUMBER</Name>
<ElementPath>NUMBER</ElementPath>
<Type>Integer</Type>
</PropertyDefn>
<PropertyDefn>
<Name>ZIP</Name>
<ElementPath>ZIP</ElementPath>
<Type>Integer</Type>
</PropertyDefn>
<PropertyDefn>
<Name>ACTIVE</Name>
<ElementPath>ACTIVE</ElementPath>
<Type>String</Type>
<Width>3</Width>
</PropertyDefn>
<PropertyDefn>
<Name>MAILING</Name>
<ElementPath>MAILING</ElementPath>
<Type>String</Type>
<Width>3</Width>
</PropertyDefn>
<PropertyDefn>
<Name>FENAME</Name>
<ElementPath>FENAME</ElementPath>
<Type>String</Type>
<Width>14</Width>
</PropertyDefn>
<PropertyDefn>
<Name>FNAME</Name>
<ElementPath>FNAME</ElementPath>
<Type>String</Type>
<Width>14</Width>
</PropertyDefn>
<PropertyDefn>
<Name>FTYPE</Name>
<ElementPath>FTYPE</ElementPath>
<Type>String</Type>
<Width>2</Width>
</PropertyDefn>
<PropertyDefn>
<Name>STRNAME</Name>
<ElementPath>STRNAME</ElementPath>
<Type>String</Type>
<Width>17</Width>
</PropertyDefn>
<PropertyDefn>
<Name>id</Name>
<ElementPath>id</ElementPath>
<Type>String</Type>
<Width>9</Width>
</PropertyDefn>
</GMLFeatureClass>
</GMLFeatureClassList>
18 changes: 18 additions & 0 deletions openaddr/tests/conforms/lake-man-gpkg.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
{
"schema": 2,
"layers": {
"addresses": [{
"name": "default",
"data": "http://fake-web/lake-man.gpkg",
"cache": "http://fake-cache/lake-man.gpkg",
"protocol": "http",
"conform": {
"lon": "X",
"lat": "Y",
"number": "NUMBER",
"street": "STRNAME",
"format": "gpkg"
}
}]
}
}
Binary file added openaddr/tests/conforms/lake-man.gpkg
View file
Open in desktop
Binary file not shown.