From 6e73ea973b80dac6f30c09c476ea45b3f3927879 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20K=C3=A4fer?= Date: Fri, 2 Jun 2017 15:02:40 +0200 Subject: [PATCH 01/14] interpret missing field in vector tile values as null instead of false --- include/mapbox/vector_tile.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/mapbox/vector_tile.hpp b/include/mapbox/vector_tile.hpp index b418802..2225b6a 100644 --- a/include/mapbox/vector_tile.hpp +++ b/include/mapbox/vector_tile.hpp @@ -114,7 +114,7 @@ static mapbox::geometry::value parseValue(protozero::data_view const& value_view break; } } - return false; + return mapbox::geometry::null_value; } feature::feature(protozero::data_view const& feature_view, layer const& l) From 29e46f36f73ab7340d1123d1ef65b9bb527e2f3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20K=C3=A4fer?= Date: Fri, 2 Jun 2017 15:09:20 +0200 Subject: [PATCH 02/14] parse all fields in a value message rather than just the first one --- include/mapbox/vector_tile.hpp | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/include/mapbox/vector_tile.hpp b/include/mapbox/vector_tile.hpp index 2225b6a..c0b2cb8 100644 --- a/include/mapbox/vector_tile.hpp +++ b/include/mapbox/vector_tile.hpp @@ -91,30 +91,38 @@ class buffer { }; static mapbox::geometry::value parseValue(protozero::data_view const& value_view) { + mapbox::geometry::value value; protozero::pbf_reader value_reader(value_view); while (value_reader.next()) { switch (value_reader.tag()) { case ValueType::STRING: - return value_reader.get_string(); + value = value_reader.get_string(); + break; case ValueType::FLOAT: - return static_cast(value_reader.get_float()); + value = static_cast(value_reader.get_float()); + break; case ValueType::DOUBLE: - return value_reader.get_double(); + value = value_reader.get_double(); + break; case ValueType::INT: - return value_reader.get_int64(); + value = value_reader.get_int64(); + break; case ValueType::UINT: - return value_reader.get_uint64(); + value = value_reader.get_uint64(); + break; case ValueType::SINT: - return value_reader.get_sint64(); + value = value_reader.get_sint64(); + break; case ValueType::BOOL: - return value_reader.get_bool(); + value = value_reader.get_bool(); + break; default: value_reader.skip(); break; } } - return mapbox::geometry::null_value; + return value; } feature::feature(protozero::data_view const& feature_view, layer const& l) From b12e109281f8ecd55c54d15f297e7e3b73ae999d Mon Sep 17 00:00:00 2001 From: Blake Thompson Date: Tue, 12 Dec 2017 13:59:54 -0600 Subject: [PATCH 03/14] Prevented the creation of very massive vectors based on invalid command counts --- include/mapbox/vector_tile.hpp | 10 ++++++++++ test/test046.mvt | 3 +++ test/unit/vector_tile.test.cpp | 14 +++++++++++++- 3 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 test/test046.mvt diff --git a/include/mapbox/vector_tile.hpp b/include/mapbox/vector_tile.hpp index 1e9a3de..0d76df4 100644 --- a/include/mapbox/vector_tile.hpp +++ b/include/mapbox/vector_tile.hpp @@ -12,6 +12,7 @@ #include #include + template using optional = std::experimental::optional; @@ -239,6 +240,15 @@ GeometryCollectionType feature::getGeometries(float scale) const { std::uint32_t cmd_length = static_cast(*start_itr++); cmd = cmd_length & 0x7; length = len_reserve = cmd_length >> 3; + // Prevents the creation of vector tiles that would cause + // a denial of service from massive over allocation. Protection + // limit is based on the assumption of an int64_t point which is + // 16 bytes in size and wanting to have a maximum of 10 MB of memory + // used. + static const std::uint32_t MAX_LENGTH = (10 * 1024 * 1024) / 16; + if (len_reserve > MAX_LENGTH) { + len_reserve = MAX_LENGTH; + } } --length; diff --git a/test/test046.mvt b/test/test046.mvt new file mode 100644 index 0000000..55a11b1 --- /dev/null +++ b/test/test046.mvt @@ -0,0 +1,3 @@ +;x0 + +000000000000000"0000000(€000000"‰ûûíÿ000000000 \ No newline at end of file diff --git a/test/unit/vector_tile.test.cpp b/test/unit/vector_tile.test.cpp index 3e940ff..87d37d2 100644 --- a/test/unit/vector_tile.test.cpp +++ b/test/unit/vector_tile.test.cpp @@ -101,4 +101,16 @@ TEST_CASE( "Read Feature-single-polygon.mvt" ) { REQUIRE(stringify_geom(geom) == "25, 17"); }*/ - +TEST_CASE( "Prevent massive over allocation" ) { + std::string buffer = open_tile("test/test046.mvt"); + mapbox::vector_tile::buffer tile(buffer); + auto const layer_names = tile.layerNames(); + REQUIRE(layer_names.size() == 1); + REQUIRE(layer_names[0] == "0000000000"); + auto const layer = tile.getLayer("0000000000"); + REQUIRE(layer.featureCount() == 1); + REQUIRE(layer.getName() == "0000000000"); + auto const feature = mapbox::vector_tile::feature(layer.getFeature(0),layer); + mapbox::vector_tile::points_arrays_type geom = feature.getGeometries(1.0); + REQUIRE(geom.capacity() <= 655360); +} From a395f9ccbc9804e3d82be38de5c266a3ac1025ff Mon Sep 17 00:00:00 2001 From: Blake Thompson Date: Wed, 13 Dec 2017 09:21:27 -0600 Subject: [PATCH 04/14] Make limit constexpr --- include/mapbox/vector_tile.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/mapbox/vector_tile.hpp b/include/mapbox/vector_tile.hpp index 0d76df4..ef0b775 100644 --- a/include/mapbox/vector_tile.hpp +++ b/include/mapbox/vector_tile.hpp @@ -245,7 +245,7 @@ GeometryCollectionType feature::getGeometries(float scale) const { // limit is based on the assumption of an int64_t point which is // 16 bytes in size and wanting to have a maximum of 10 MB of memory // used. - static const std::uint32_t MAX_LENGTH = (10 * 1024 * 1024) / 16; + constexpr std::uint32_t MAX_LENGTH = (10 * 1024 * 1024) / 16; if (len_reserve > MAX_LENGTH) { len_reserve = MAX_LENGTH; } From 2b7d952e7b05aea2434fb3e0388d047285af7f08 Mon Sep 17 00:00:00 2001 From: Blake Thompson Date: Mon, 18 Dec 2017 15:22:24 -0600 Subject: [PATCH 05/14] Effectively force all close_path commands to be treated as if th they had a command count of 1 --- include/mapbox/vector_tile.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/include/mapbox/vector_tile.hpp b/include/mapbox/vector_tile.hpp index ef0b775..ba0c632 100644 --- a/include/mapbox/vector_tile.hpp +++ b/include/mapbox/vector_tile.hpp @@ -298,6 +298,7 @@ GeometryCollectionType feature::getGeometries(float scale) const { if (!paths.back().empty()) { paths.back().push_back(paths.back()[0]); } + length = 0; } else { throw std::runtime_error("unknown command"); } From cc2f74872270806b439612c75d789604a9a8a3ef Mon Sep 17 00:00:00 2001 From: Blake Thompson Date: Tue, 19 Dec 2017 10:32:33 -0600 Subject: [PATCH 06/14] Added shrink to fit for incorrect lengths even with 1mb to prevent repeated uses of this tactic to allocate and leave multiple vectors, shifted max reserve from 10mb to 1mb. --- include/mapbox/vector_tile.hpp | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/include/mapbox/vector_tile.hpp b/include/mapbox/vector_tile.hpp index ba0c632..364fd44 100644 --- a/include/mapbox/vector_tile.hpp +++ b/include/mapbox/vector_tile.hpp @@ -243,9 +243,9 @@ GeometryCollectionType feature::getGeometries(float scale) const { // Prevents the creation of vector tiles that would cause // a denial of service from massive over allocation. Protection // limit is based on the assumption of an int64_t point which is - // 16 bytes in size and wanting to have a maximum of 10 MB of memory + // 16 bytes in size and wanting to have a maximum of 1 MB of memory // used. - constexpr std::uint32_t MAX_LENGTH = (10 * 1024 * 1024) / 16; + constexpr std::uint32_t MAX_LENGTH = (1024 * 1024) / 16; if (len_reserve > MAX_LENGTH) { len_reserve = MAX_LENGTH; } @@ -270,6 +270,12 @@ GeometryCollectionType feature::getGeometries(float scale) const { } if (cmd == CommandType::MOVE_TO && !paths.back().empty()) { + if (paths.back().size() < paths.back().capacity()) { + // Assumign we had an invalid length before + // lets shrink to fit, just to make sure + // we don't have a large capcity unused vector + paths.back().shrink_to_fit(); + } paths.emplace_back(); if (!is_point) { first = true; @@ -303,6 +309,13 @@ GeometryCollectionType feature::getGeometries(float scale) const { throw std::runtime_error("unknown command"); } } + if (paths.size() < paths.capacity()) { + // Assumign we had an invalid length before + // lets shrink to fit, just to make sure + // we don't have a large capacity vector + // just wasting memory + paths.shrink_to_fit(); + } #if defined(DEBUG) for (auto const& p : paths) { assert(p.size() == p.capacity()); From 33ace751f48da1c79033bcaa4a163264bb6aa8fc Mon Sep 17 00:00:00 2001 From: Blake Thompson Date: Tue, 19 Dec 2017 12:09:05 -0600 Subject: [PATCH 07/14] Fix typo in comment --- include/mapbox/vector_tile.hpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/include/mapbox/vector_tile.hpp b/include/mapbox/vector_tile.hpp index 364fd44..55d0eab 100644 --- a/include/mapbox/vector_tile.hpp +++ b/include/mapbox/vector_tile.hpp @@ -271,9 +271,10 @@ GeometryCollectionType feature::getGeometries(float scale) const { if (cmd == CommandType::MOVE_TO && !paths.back().empty()) { if (paths.back().size() < paths.back().capacity()) { - // Assumign we had an invalid length before + // Assuming we had an invalid length before // lets shrink to fit, just to make sure - // we don't have a large capcity unused vector + // we don't have a large capacity vector + // just wasting memory paths.back().shrink_to_fit(); } paths.emplace_back(); @@ -310,7 +311,7 @@ GeometryCollectionType feature::getGeometries(float scale) const { } } if (paths.size() < paths.capacity()) { - // Assumign we had an invalid length before + // Assuming we had an invalid length before // lets shrink to fit, just to make sure // we don't have a large capacity vector // just wasting memory From 97cf4e723105e0140b646e6e0b9c54561b329067 Mon Sep 17 00:00:00 2001 From: Blake Thompson Date: Thu, 21 Dec 2017 10:34:44 -0600 Subject: [PATCH 08/14] Bump to 1.0.0 --- CHANGELOG.md | 5 +++++ include/mapbox/vector_tile/version.hpp | 2 +- package.json | 4 ++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5882d83..9839844 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## 1.0.0 + +- Allow nulls from property parsing +- Several bug fixes around geometry parsing to prevent over allocation of memory. + ## 1.0.0-alpha.1 - Added demo application showing usage of decoder diff --git a/include/mapbox/vector_tile/version.hpp b/include/mapbox/vector_tile/version.hpp index e85b1f4..dc1397c 100644 --- a/include/mapbox/vector_tile/version.hpp +++ b/include/mapbox/vector_tile/version.hpp @@ -13,4 +13,4 @@ #define VECTOR_TILE_VERSION_CODE (VECTOR_TILE_VERSION_MAJOR * 10000 + VECTOR_TILE_VERSION_MINOR * 100 + VECTOR_TILE_VERSION_PATCH) /// Version number as string -#define VECTOR_TILE_VERSION_STRING "1.0.0-rc7" +#define VECTOR_TILE_VERSION_STRING "1.0.0" diff --git a/package.json b/package.json index 3eca121..2370a12 100644 --- a/package.json +++ b/package.json @@ -1,10 +1,10 @@ { "name": "@mapbox/vector-tile", - "version": "1.0.0-alpha.1", + "version": "1.0.0", "description": "A C++ header only library for decoding and encoding Mapbox Vector Tiles", "main": "./include_dirs.js", "repository" : { "type" : "git", "url" : "git://github.com/mapbox/vector-tile.git" } -} \ No newline at end of file +} From 6e70d68ca947ed20c2859840093555be6b3db7a1 Mon Sep 17 00:00:00 2001 From: Blake Thompson Date: Thu, 21 Dec 2017 11:27:12 -0600 Subject: [PATCH 09/14] Fix failing unit test after bump --- test/unit/tags.test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/tags.test.cpp b/test/unit/tags.test.cpp index 2427afd..3bed2fe 100644 --- a/test/unit/tags.test.cpp +++ b/test/unit/tags.test.cpp @@ -4,7 +4,7 @@ #include TEST_CASE( "Version constant" ) { - CHECK(std::string(VECTOR_TILE_VERSION_STRING) == std::string("1.0.0-rc7")); + CHECK(std::string(VECTOR_TILE_VERSION_STRING) == std::string("1.0.0")); CHECK(VECTOR_TILE_VERSION_CODE == 10000); } From d3de5d24d05bef44ac902b68de178d1ad8d0c9d1 Mon Sep 17 00:00:00 2001 From: Blake Thompson Date: Thu, 21 Dec 2017 13:33:02 -0600 Subject: [PATCH 10/14] It is currently possible to have an overflow or underflow in our x or y position if multiple int32_t deltas push a value outside of the max range of an int32_t. This change bumps the x and y values to int64_t. It is still possible to have an overflow or underflow, however, this is much less likely and would require a large number of large deltas to reach this point. --- include/mapbox/vector_tile.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/mapbox/vector_tile.hpp b/include/mapbox/vector_tile.hpp index 9bb4d76..2308462 100644 --- a/include/mapbox/vector_tile.hpp +++ b/include/mapbox/vector_tile.hpp @@ -224,8 +224,8 @@ template GeometryCollectionType feature::getGeometries(float scale) const { std::uint8_t cmd = 1; std::uint32_t length = 0; - std::int32_t x = 0; - std::int32_t y = 0; + std::int64_t x = 0; + std::int64_t y = 0; GeometryCollectionType paths; From 8aa13db7bf5087a6e557734f5fe4ceb6e95dd572 Mon Sep 17 00:00:00 2001 From: Blake Thompson Date: Wed, 10 Jan 2018 10:47:27 -0600 Subject: [PATCH 11/14] Bump to v1.0.1 --- CHANGELOG.md | 4 ++++ include/mapbox/vector_tile/version.hpp | 4 ++-- test/unit/tags.test.cpp | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9839844..7682c2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 1.0.1 + +- Limit possibility rare situation where it might be possible to have an overflow or underflow of integers while decoding a vector tile's geometry + ## 1.0.0 - Allow nulls from property parsing diff --git a/include/mapbox/vector_tile/version.hpp b/include/mapbox/vector_tile/version.hpp index dc1397c..13b8755 100644 --- a/include/mapbox/vector_tile/version.hpp +++ b/include/mapbox/vector_tile/version.hpp @@ -7,10 +7,10 @@ #define VECTOR_TILE_VERSION_MINOR 0 /// The patch number -#define VECTOR_TILE_VERSION_PATCH 0 +#define VECTOR_TILE_VERSION_PATCH 1 /// The complete version number #define VECTOR_TILE_VERSION_CODE (VECTOR_TILE_VERSION_MAJOR * 10000 + VECTOR_TILE_VERSION_MINOR * 100 + VECTOR_TILE_VERSION_PATCH) /// Version number as string -#define VECTOR_TILE_VERSION_STRING "1.0.0" +#define VECTOR_TILE_VERSION_STRING "1.0.1" diff --git a/test/unit/tags.test.cpp b/test/unit/tags.test.cpp index 3bed2fe..928172c 100644 --- a/test/unit/tags.test.cpp +++ b/test/unit/tags.test.cpp @@ -4,8 +4,8 @@ #include TEST_CASE( "Version constant" ) { - CHECK(std::string(VECTOR_TILE_VERSION_STRING) == std::string("1.0.0")); - CHECK(VECTOR_TILE_VERSION_CODE == 10000); + CHECK(std::string(VECTOR_TILE_VERSION_STRING) == std::string("1.0.1")); + CHECK(VECTOR_TILE_VERSION_CODE == 10001); } TEST_CASE( "Protobuf Tag Constants" ) { From 7c0e2fc929ab9551aa82894839aed31014884bd7 Mon Sep 17 00:00:00 2001 From: Sudarsana Babu Nagineni Date: Thu, 19 Jul 2018 20:20:15 +0300 Subject: [PATCH 12/14] Fix compilation errors with libc++ on QNX 7 It seems that the QNX 7 compiler (i.e qcc based on GCC 5.4.0 with libc++ from LLVM) has a limited c++11 feature support and causing compilation errors with the inheriting constructors. This patch fixes the issues by providing the universal forwarding constructor. Also, resolves an ambiguous overload error with optional. --- include/mapbox/vector_tile.hpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/include/mapbox/vector_tile.hpp b/include/mapbox/vector_tile.hpp index 2308462..5ddae48 100644 --- a/include/mapbox/vector_tile.hpp +++ b/include/mapbox/vector_tile.hpp @@ -23,13 +23,15 @@ using point_type = mapbox::geometry::point; class points_array_type : public std::vector { public: using coordinate_type = point_type::coordinate_type; - using std::vector::vector; + template + points_array_type(Args&&... args) : std::vector(std::forward(args)...) {} }; class points_arrays_type : public std::vector { public: using coordinate_type = points_array_type::coordinate_type; - using std::vector::vector; + template + points_arrays_type(Args&&... args) : std::vector(std::forward(args)...) {} }; class layer; @@ -137,7 +139,7 @@ inline feature::feature(protozero::data_view const& feature_view, layer const& l while (feature_pbf.next()) { switch (feature_pbf.tag()) { case FeatureType::ID: - id = { feature_pbf.get_uint64() }; + id = optional{ feature_pbf.get_uint64() }; break; case FeatureType::TAGS: tags_iter = feature_pbf.get_packed_uint32(); From d46ae6110372eeecb0e1dc954aa6213785e188c5 Mon Sep 17 00:00:00 2001 From: Sudarsana Babu Nagineni Date: Fri, 20 Jul 2018 18:46:49 +0300 Subject: [PATCH 13/14] Bump to v1.0.2 --- CHANGELOG.md | 4 ++++ include/mapbox/vector_tile/version.hpp | 4 ++-- test/unit/tags.test.cpp | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7682c2b..7aa4594 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 1.0.2 + +- Add support for QNX 7 compiler. + ## 1.0.1 - Limit possibility rare situation where it might be possible to have an overflow or underflow of integers while decoding a vector tile's geometry diff --git a/include/mapbox/vector_tile/version.hpp b/include/mapbox/vector_tile/version.hpp index 13b8755..cc1974d 100644 --- a/include/mapbox/vector_tile/version.hpp +++ b/include/mapbox/vector_tile/version.hpp @@ -7,10 +7,10 @@ #define VECTOR_TILE_VERSION_MINOR 0 /// The patch number -#define VECTOR_TILE_VERSION_PATCH 1 +#define VECTOR_TILE_VERSION_PATCH 2 /// The complete version number #define VECTOR_TILE_VERSION_CODE (VECTOR_TILE_VERSION_MAJOR * 10000 + VECTOR_TILE_VERSION_MINOR * 100 + VECTOR_TILE_VERSION_PATCH) /// Version number as string -#define VECTOR_TILE_VERSION_STRING "1.0.1" +#define VECTOR_TILE_VERSION_STRING "1.0.2" diff --git a/test/unit/tags.test.cpp b/test/unit/tags.test.cpp index 928172c..1c0dcdd 100644 --- a/test/unit/tags.test.cpp +++ b/test/unit/tags.test.cpp @@ -4,8 +4,8 @@ #include TEST_CASE( "Version constant" ) { - CHECK(std::string(VECTOR_TILE_VERSION_STRING) == std::string("1.0.1")); - CHECK(VECTOR_TILE_VERSION_CODE == 10001); + CHECK(std::string(VECTOR_TILE_VERSION_STRING) == std::string("1.0.2")); + CHECK(VECTOR_TILE_VERSION_CODE == 10002); } TEST_CASE( "Protobuf Tag Constants" ) { From 00274af92f1b83efd03e3faa46557a25020ae244 Mon Sep 17 00:00:00 2001 From: John Firebaugh Date: Mon, 23 Jul 2018 11:08:20 -0700 Subject: [PATCH 14/14] Fix demo compile --- demo/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demo/Makefile b/demo/Makefile index c66450e..607d318 100644 --- a/demo/Makefile +++ b/demo/Makefile @@ -1,6 +1,6 @@ CC := $(CC) CXX := $(CXX) -CXXFLAGS := $(CXXFLAGS) -Iinclude -std=c++14 -Wall +CXXFLAGS := $(CXXFLAGS) -isystem../include -isystem ../mason_packages/.link/include/ -std=c++14 -Wall RELEASE_FLAGS := -O3 -DNDEBUG -fvisibility-inlines-hidden -fvisibility=hidden DEBUG_FLAGS := -g -O0 -DDEBUG -fno-inline-functions -fno-omit-frame-pointer