Skip to content

Commit 77c0a19

Browse files
marselestermarselester
committed
Update within() to make caching optional
1 parent 878a30d commit 77c0a19

8 files changed

Lines changed: 397 additions & 41 deletions

File tree

‎README.md‎

Lines changed: 115 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -87,14 +87,23 @@ if (result) |r| {
8787
}
8888
```
8989

90+
Use a `Cache` to skip decoding when different IPs resolve to the same record.
91+
92+
```zig
93+
var cache: maxminddb.Cache(maxminddb.geolite2.City) = .{};
94+
defer cache.deinit();
95+
96+
const city = try db.lookup(allocator, maxminddb.geolite2.City, ip, .{ .cache = &cache });
97+
```
98+
9099
Here are reference results on Apple M2 Pro (1M random IPv4 lookups against GeoLite2-City
91100
with `ipv4_index_first_n_bits = 16`):
92101

93-
| Benchmark | All fields | Filtered (city) |
94-
|--- |--- |--- |
95-
| `geolite2.City` | ~1,284,000 | ~1,348,000 |
96-
| `MyCity` | ~1,383,000 | — |
97-
| `any.Value` | ~1,254,000 | ~1,349,000 |
102+
| Type | Default | `.only` | `Cache` |
103+
|--- |--- |--- |--- |
104+
| `geolite2.City` | ~1,284,000 | ~1,348,000 | ~1,474,000 |
105+
| `MyCity` | ~1,383,000 | | |
106+
| `any.Value` | ~1,254,000 | ~1,349,000 | |
98107

99108
<details>
100109

@@ -140,6 +149,30 @@ Lookups Per Second (avg):1315870.3443053183
140149

141150
<details>
142151

152+
<summary>geolite2.City with Cache</summary>
153+
154+
```sh
155+
$ for i in $(seq 1 10); do
156+
zig build benchmark_lookup_cache -Doptimize=ReleaseFast -- GeoLite2-City.mmdb 1000000 \
157+
2>&1 | grep 'Lookups Per Second'
158+
done
159+
160+
Lookups Per Second (avg):1493822.3908664712
161+
Lookups Per Second (avg):1503051.0049070602
162+
Lookups Per Second (avg):1499514.437731375
163+
Lookups Per Second (avg):1491749.9700251492
164+
Lookups Per Second (avg):1449924.9391983037
165+
Lookups Per Second (avg):1396100.6211600688
166+
Lookups Per Second (avg):1465750.9875955326
167+
Lookups Per Second (avg):1515611.9396877384
168+
Lookups Per Second (avg):1485235.6423035355
169+
Lookups Per Second (avg):1439334.222943596
170+
```
171+
172+
</details>
173+
174+
<details>
175+
143176
<summary>MyCity</summary>
144177

145178
```sh
@@ -203,3 +236,80 @@ Lookups Per Second (avg):1315986.2950186788
203236
```
204237

205238
</details>
239+
240+
Use `within()` to iterate over all networks in the database.
241+
A `Cache` avoids re-decoding networks that share the same record.
242+
243+
```zig
244+
var cache: maxminddb.Cache(maxminddb.any.Value) = .{};
245+
defer cache.deinit();
246+
247+
var it = try db.within(
248+
allocator,
249+
maxminddb.any.Value,
250+
maxminddb.Network.all_ipv6,
251+
.{ .cache = &cache },
252+
);
253+
defer it.deinit();
254+
255+
while (try it.next()) |item| {
256+
std.debug.print("{f} {f}\n", .{item.network, item.value});
257+
}
258+
```
259+
260+
Without a cache each result owns its memory and must be freed with `item.deinit()`.
261+
262+
Here are reference results on Apple M2 Pro (full GeoLite2-City scan using `any.Value`):
263+
264+
| Mode | Records/sec |
265+
|--- |--- |
266+
| Default | ~1,235,000 |
267+
| `Cache` | ~2,900,000 |
268+
269+
<details>
270+
271+
<summary>no cache (any.Value)</summary>
272+
273+
```sh
274+
$ for i in $(seq 1 10); do
275+
zig build benchmark_within -Doptimize=ReleaseFast -- GeoLite2-City.mmdb \
276+
2>&1 | grep 'Records Per Second'
277+
done
278+
279+
Records Per Second: 1216758.945145436
280+
Records Per Second: 1238440.9772222256
281+
Records Per Second: 1234710.6362391203
282+
Records Per Second: 1229527.4688849829
283+
Records Per Second: 1243478.3908140333
284+
Records Per Second: 1226863.3718734735
285+
Records Per Second: 1240073.3248202254
286+
Records Per Second: 1247541.1528026997
287+
Records Per Second: 1230510.441029532
288+
Records Per Second: 1246311.587919839
289+
```
290+
291+
</details>
292+
293+
<details>
294+
295+
<summary>cache (any.Value)</summary>
296+
297+
```sh
298+
$ for i in $(seq 1 10); do
299+
zig build benchmark_within_cache -Doptimize=ReleaseFast -- GeoLite2-City.mmdb \
300+
2>&1 | grep 'Records Per Second'
301+
done
302+
303+
Records Per Second: 2847560.3756875996
304+
Records Per Second: 2925388.867798729
305+
Records Per Second: 2919203.9046571665
306+
Records Per Second: 2814410.555872645
307+
Records Per Second: 2933972.04386147
308+
Records Per Second: 2900700.06160036
309+
Records Per Second: 2922279.338699886
310+
Records Per Second: 2862525.847598088
311+
Records Per Second: 2916760.542913819
312+
Records Per Second: 2908245.98918392
313+
```
314+
315+
</details>

‎benchmarks/lookup_cache.zig‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
const std = @import("std");
2+
const maxminddb = @import("maxminddb");
3+
4+
const default_db_path: []const u8 = "GeoLite2-City.mmdb";
5+
const default_num_lookups: u64 = 1_000_000;
6+
const max_mmdb_fields = 32;
7+
8+
pub fn main() !void {
9+
const allocator = std.heap.smp_allocator;
10+
11+
const args = try std.process.argsAlloc(allocator);
12+
defer std.process.argsFree(allocator, args);
13+
14+
var db_path: []const u8 = default_db_path;
15+
var num_lookups = default_num_lookups;
16+
var fields: ?[]const []const u8 = null;
17+
if (args.len > 1) db_path = args[1];
18+
if (args.len > 2) num_lookups = try std.fmt.parseUnsigned(u64, args[2], 10);
19+
if (args.len > 3) {
20+
var items: [max_mmdb_fields][]const u8 = undefined;
21+
22+
var it = std.mem.splitScalar(u8, args[3], ',');
23+
var i: usize = 0;
24+
while (it.next()) |part| : (i += 1) {
25+
items[i] = part;
26+
}
27+
28+
fields = items[0..i];
29+
}
30+
31+
std.debug.print("Benchmarking with:\n", .{});
32+
std.debug.print(" Database: {s}\n", .{db_path});
33+
std.debug.print(" Lookups: {d}\n", .{num_lookups});
34+
std.debug.print("Opening database...\n", .{});
35+
36+
var open_timer = try std.time.Timer.start();
37+
var db = try maxminddb.Reader.mmap(allocator, db_path, .{ .ipv4_index_first_n_bits = 16 });
38+
defer db.close();
39+
const open_time_ms = @as(f64, @floatFromInt(open_timer.read())) /
40+
@as(f64, @floatFromInt(std.time.ns_per_ms));
41+
std.debug.print("Database opened successfully in {d} ms. Type: {s}\n", .{
42+
open_time_ms,
43+
db.metadata.database_type,
44+
});
45+
46+
var cache: maxminddb.Cache(maxminddb.geolite2.City) = .{};
47+
defer cache.deinit();
48+
49+
var arena = std.heap.ArenaAllocator.init(allocator);
50+
defer arena.deinit();
51+
const arena_allocator = arena.allocator();
52+
53+
std.debug.print("Starting benchmark...\n", .{});
54+
var timer = try std.time.Timer.start();
55+
var not_found_count: u64 = 0;
56+
var lookup_errors: u64 = 0;
57+
var ip_bytes: [4]u8 = undefined;
58+
59+
for (0..num_lookups) |_| {
60+
std.crypto.random.bytes(&ip_bytes);
61+
const ip = std.net.Address.initIp4(ip_bytes, 0);
62+
63+
const result = db.lookup(
64+
arena_allocator,
65+
maxminddb.geolite2.City,
66+
ip,
67+
.{ .only = fields, .cache = &cache },
68+
) catch |err| {
69+
std.debug.print("! Lookup error for IP {any}: {any}\n", .{ ip, err });
70+
lookup_errors += 1;
71+
continue;
72+
};
73+
if (result == null) {
74+
not_found_count += 1;
75+
continue;
76+
}
77+
78+
_ = arena.reset(.retain_capacity);
79+
}
80+
81+
const elapsed_ns = timer.read();
82+
const elapsed_s = @as(f64, @floatFromInt(elapsed_ns)) /
83+
@as(f64, @floatFromInt(std.time.ns_per_s));
84+
const lookups_per_second = if (elapsed_s > 0)
85+
@as(f64, @floatFromInt(num_lookups)) / elapsed_s
86+
else
87+
0.0;
88+
const successful_lookups = num_lookups - not_found_count - lookup_errors;
89+
90+
std.debug.print("\n--- Benchmark Finished ---\n", .{});
91+
std.debug.print("Total Lookups Attempted: {d}\n", .{num_lookups});
92+
std.debug.print("Successful Lookups: {d}\n", .{successful_lookups});
93+
std.debug.print("IPs Not Found: {d}\n", .{not_found_count});
94+
std.debug.print("Lookup Errors: {d}\n", .{lookup_errors});
95+
std.debug.print("Elapsed Time: {d} s\n", .{elapsed_s});
96+
std.debug.print("Lookups Per Second (avg):{d}\n", .{lookups_per_second});
97+
}

‎benchmarks/within.zig‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
const std = @import("std");
2+
const maxminddb = @import("maxminddb");
3+
4+
const default_db_path: []const u8 = "GeoLite2-City.mmdb";
5+
6+
pub fn main() !void {
7+
const allocator = std.heap.smp_allocator;
8+
9+
const args = try std.process.argsAlloc(allocator);
10+
defer std.process.argsFree(allocator, args);
11+
12+
var db_path: []const u8 = default_db_path;
13+
if (args.len > 1) db_path = args[1];
14+
15+
std.debug.print("Benchmarking with:\n", .{});
16+
std.debug.print(" Database: {s}\n", .{db_path});
17+
std.debug.print("Opening database...\n", .{});
18+
19+
var open_timer = try std.time.Timer.start();
20+
var db = try maxminddb.Reader.mmap(allocator, db_path, .{});
21+
defer db.close();
22+
const open_time_ms = @as(f64, @floatFromInt(open_timer.read())) /
23+
@as(f64, @floatFromInt(std.time.ns_per_ms));
24+
std.debug.print("Database opened successfully in {d} ms. Type: {s}\n", .{
25+
open_time_ms,
26+
db.metadata.database_type,
27+
});
28+
29+
const network = if (db.metadata.ip_version == 4)
30+
maxminddb.Network.all_ipv4
31+
else
32+
maxminddb.Network.all_ipv6;
33+
34+
std.debug.print("Starting benchmark...\n", .{});
35+
var timer = try std.time.Timer.start();
36+
37+
var it = try db.within(allocator, maxminddb.any.Value, network, .{});
38+
defer it.deinit();
39+
40+
var n: usize = 0;
41+
while (try it.next()) |item| {
42+
n += 1;
43+
item.deinit();
44+
}
45+
46+
const elapsed_ns = timer.read();
47+
const elapsed_s = @as(f64, @floatFromInt(elapsed_ns)) /
48+
@as(f64, @floatFromInt(std.time.ns_per_s));
49+
50+
const records_per_second = if (elapsed_s > 0)
51+
@as(f64, @floatFromInt(n)) / elapsed_s
52+
else
53+
0.0;
54+
55+
std.debug.print("\n--- Benchmark Finished ---\n", .{});
56+
std.debug.print("Records: {d}\n", .{n});
57+
std.debug.print("Elapsed Time: {d} s\n", .{elapsed_s});
58+
std.debug.print("Records Per Second: {d}\n", .{records_per_second});
59+
}

‎benchmarks/within_cache.zig‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
const std = @import("std");
2+
const maxminddb = @import("maxminddb");
3+
4+
const default_db_path: []const u8 = "GeoLite2-City.mmdb";
5+
6+
pub fn main() !void {
7+
const allocator = std.heap.smp_allocator;
8+
9+
const args = try std.process.argsAlloc(allocator);
10+
defer std.process.argsFree(allocator, args);
11+
12+
var db_path: []const u8 = default_db_path;
13+
if (args.len > 1) db_path = args[1];
14+
15+
std.debug.print("Benchmarking with:\n", .{});
16+
std.debug.print(" Database: {s}\n", .{db_path});
17+
std.debug.print("Opening database...\n", .{});
18+
19+
var open_timer = try std.time.Timer.start();
20+
var db = try maxminddb.Reader.mmap(allocator, db_path, .{});
21+
defer db.close();
22+
const open_time_ms = @as(f64, @floatFromInt(open_timer.read())) /
23+
@as(f64, @floatFromInt(std.time.ns_per_ms));
24+
std.debug.print("Database opened successfully in {d} ms. Type: {s}\n", .{
25+
open_time_ms,
26+
db.metadata.database_type,
27+
});
28+
29+
const network = if (db.metadata.ip_version == 4)
30+
maxminddb.Network.all_ipv4
31+
else
32+
maxminddb.Network.all_ipv6;
33+
34+
var cache: maxminddb.Cache(maxminddb.any.Value) = .{};
35+
defer cache.deinit();
36+
37+
std.debug.print("Starting benchmark...\n", .{});
38+
var timer = try std.time.Timer.start();
39+
40+
var it = try db.within(
41+
allocator,
42+
maxminddb.any.Value,
43+
network,
44+
.{ .cache = &cache },
45+
);
46+
defer it.deinit();
47+
48+
var n: usize = 0;
49+
while (try it.next()) |_| {
50+
n += 1;
51+
}
52+
53+
const elapsed_ns = timer.read();
54+
const elapsed_s = @as(f64, @floatFromInt(elapsed_ns)) /
55+
@as(f64, @floatFromInt(std.time.ns_per_s));
56+
57+
const records_per_second = if (elapsed_s > 0)
58+
@as(f64, @floatFromInt(n)) / elapsed_s
59+
else
60+
0.0;
61+
62+
std.debug.print("\n--- Benchmark Finished ---\n", .{});
63+
std.debug.print("Records: {d}\n", .{n});
64+
std.debug.print("Elapsed Time: {d} s\n", .{elapsed_s});
65+
std.debug.print("Records Per Second: {d}\n", .{records_per_second});
66+
}

‎build.zig‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,11 @@ pub fn build(b: *std.Build) void {
3030
.{ .file = "examples/within.zig", .name = "example_within" },
3131
.{ .file = "examples/inspect.zig", .name = "example_inspect" },
3232
.{ .file = "benchmarks/lookup.zig", .name = "benchmark_lookup" },
33+
.{ .file = "benchmarks/lookup_cache.zig", .name = "benchmark_lookup_cache" },
3334
.{ .file = "benchmarks/mycity.zig", .name = "benchmark_mycity" },
3435
.{ .file = "benchmarks/inspect.zig", .name = "benchmark_inspect" },
36+
.{ .file = "benchmarks/within.zig", .name = "benchmark_within" },
37+
.{ .file = "benchmarks/within_cache.zig", .name = "benchmark_within_cache" },
3538
};
3639

3740
{

‎examples/within.zig‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,10 @@ pub fn main() !void {
1919
var it = try db.within(allocator, maxminddb.geolite2.City, network, .{});
2020
defer it.deinit();
2121

22-
// The iterator owns the values; each next() call invalidates the previous item.
2322
var n: usize = 0;
2423
while (try it.next()) |item| {
24+
defer item.deinit();
25+
2526
const continent = item.value.continent.code;
2627
const country = item.value.country.iso_code;
2728
var city: []const u8 = "";

0 commit comments

Comments
 (0)