From 7b6fc9216cc20b86b1779547dc5b0205f9f13e32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20BR=C3=89ZOT?= Date: Wed, 9 Aug 2023 15:16:33 +0200 Subject: [PATCH 01/16] feat: use Findex v5.0 --- scripts/get_native_libraries.py | 4 +- .../java/com/cosmian/jna/findex/Findex.java | 70 +++++++++++++------ .../jna/findex/ffi/FindexNativeWrapper.java | 3 +- .../cosmian/jna/findex/ffi/UpsertResults.java | 52 ++++++++++++++ .../java/com/cosmian/findex/TestRedis.java | 15 ++++ .../java/com/cosmian/findex/TestSqlite.java | 14 ++++ 6 files changed, 135 insertions(+), 23 deletions(-) create mode 100644 src/main/java/com/cosmian/jna/findex/ffi/UpsertResults.java diff --git a/scripts/get_native_libraries.py b/scripts/get_native_libraries.py index c0ad6b8..e0b84ad 100644 --- a/scripts/get_native_libraries.py +++ b/scripts/get_native_libraries.py @@ -62,6 +62,6 @@ def download_native_libraries(version: str) -> bool: if __name__ == '__main__': - ret = download_native_libraries('v2.0.1') + ret = download_native_libraries('v2.2.0') if ret is False and getenv('GITHUB_ACTIONS'): - download_native_libraries('last_build/release/v2.0.1') + download_native_libraries('last_build/feature/findex_5_0_0') diff --git a/src/main/java/com/cosmian/jna/findex/Findex.java b/src/main/java/com/cosmian/jna/findex/Findex.java index 97fceca..9b39eb3 100644 --- a/src/main/java/com/cosmian/jna/findex/Findex.java +++ b/src/main/java/com/cosmian/jna/findex/Findex.java @@ -9,6 +9,7 @@ import com.cosmian.jna.findex.ffi.Progress; import com.cosmian.jna.findex.ffi.ProgressResults; import com.cosmian.jna.findex.ffi.SearchResults; +import com.cosmian.jna.findex.ffi.UpsertResults; import com.cosmian.jna.findex.serde.Leb128Reader; import com.cosmian.jna.findex.structs.IndexedValue; import com.cosmian.jna.findex.structs.Keyword; @@ -18,13 +19,12 @@ public final class Findex extends FindexBase { - public static void upsert( - byte[] key, - byte[] label, - Map> additions, - Map> deletions, - int entryTableNumber, - Database db) + public static UpsertResults upsert(byte[] key, + byte[] label, + Map> additions, + Map> deletions, + int entryTableNumber, + Database db) throws CloudproofException { try ( @@ -34,8 +34,17 @@ public static void upsert( labelPointer.write(0, label, 0, label.length); long start = System.currentTimeMillis(); - // Indexes creation + insertion/update - unwrap(INSTANCE.h_upsert( + + // + // Prepare outputs + // + // start with an arbitration buffer allocation size of 131072 (around 4096 + // indexedValues) + byte[] newKeywordsBuffer = new byte[1024]; + IntByReference newKeywordsBufferSize = new IntByReference(newKeywordsBuffer.length); + + int ffiCode = INSTANCE.h_upsert( + newKeywordsBuffer, newKeywordsBufferSize, keyPointer, key.length, labelPointer, label.length, indexedValuesToJson(additions), @@ -43,25 +52,46 @@ public static void upsert( entryTableNumber, db.fetchEntryCallback(), db.upsertEntryCallback(), - db.upsertChainCallback()), start); + db.upsertChainCallback()); + + FindexCallbackException.rethrowOnErrorCode(ffiCode, start, System.currentTimeMillis()); + + if (ffiCode != 0) { + // Retry with correct allocated size + newKeywordsBuffer = new byte[newKeywordsBufferSize.getValue()]; + long startRetry = System.currentTimeMillis(); + unwrap(INSTANCE.h_upsert( + newKeywordsBuffer, newKeywordsBufferSize, + keyPointer, key.length, + labelPointer, label.length, + indexedValuesToJson(additions), + indexedValuesToJson(deletions), + entryTableNumber, + db.fetchEntryCallback(), + db.upsertEntryCallback(), + db.upsertChainCallback()), startRetry); + } + + byte[] newKeywordsBytes = Arrays.copyOfRange(newKeywordsBuffer, 0, newKeywordsBufferSize.getValue()); + + return new Leb128Reader(newKeywordsBytes).readObject(UpsertResults.class); } } - public static void upsert( - byte[] key, - byte[] label, - Map> additions, - Map> deletions, - Database db) + public static UpsertResults upsert(byte[] key, + byte[] label, + Map> additions, + Map> deletions, + Database db) throws CloudproofException { // Make entryTableNumber equals to 1 by default - upsert(key, label, additions, deletions, 1, db); + return upsert(key, label, additions, deletions, 1, db); } - public static void upsert(IndexRequest request) + public static UpsertResults upsert(IndexRequest request) throws CloudproofException { - upsert(request.key, request.label, request.additions, request.deletions, request.entryTableNumber, - request.database); + return upsert(request.key, request.label, request.additions, request.deletions, request.entryTableNumber, + request.database); } public static SearchResults search(SearchRequest request) diff --git a/src/main/java/com/cosmian/jna/findex/ffi/FindexNativeWrapper.java b/src/main/java/com/cosmian/jna/findex/ffi/FindexNativeWrapper.java index c7281c3..8184933 100644 --- a/src/main/java/com/cosmian/jna/findex/ffi/FindexNativeWrapper.java +++ b/src/main/java/com/cosmian/jna/findex/ffi/FindexNativeWrapper.java @@ -78,7 +78,8 @@ boolean apply(Pointer output, throws CloudproofException; } - int h_upsert( + int h_upsert(byte[] newKeywordsBufferPtr, + IntByReference newKeywordsBufferLen, Pointer masterKeyPtr, int masterKeyLen, Pointer labelPtr, diff --git a/src/main/java/com/cosmian/jna/findex/ffi/UpsertResults.java b/src/main/java/com/cosmian/jna/findex/ffi/UpsertResults.java new file mode 100644 index 0000000..1a36c65 --- /dev/null +++ b/src/main/java/com/cosmian/jna/findex/ffi/UpsertResults.java @@ -0,0 +1,52 @@ +package com.cosmian.jna.findex.ffi; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.HashSet; +import java.util.Set; +import com.cosmian.jna.findex.serde.Leb128Serializable; +import com.cosmian.jna.findex.structs.Keyword; +import com.cosmian.utils.CloudproofException; +import com.cosmian.utils.Leb128; + +public class UpsertResults implements Leb128Serializable { + + private Set results; + + public UpsertResults() { + this.results = new HashSet<>(); + } + + public Set getResults() { + return results; + } + + public boolean isEmpty() { + return results.isEmpty(); + } + + public int numberOfKeywords() { + return results.size(); + } + + @Override + public void readObject(InputStream is) throws CloudproofException { + try { + + int setLen = (int) Leb128.readU64(is); + for (int i = 0; i < setLen; i++) { + Keyword keyword = new Keyword(Leb128.readByteArray(is)); + results.add(keyword); + } + } catch (IOException e) { + throw new CloudproofException("failed deserializing the upsert results: " + e.getMessage(), e); + } + + } + + @Override + public void writeObject(OutputStream os) throws CloudproofException { + throw new CloudproofException("Upsert results are not serializable"); + } +} diff --git a/src/test/java/com/cosmian/findex/TestRedis.java b/src/test/java/com/cosmian/findex/TestRedis.java index ec42c4d..3c28090 100644 --- a/src/test/java/com/cosmian/findex/TestRedis.java +++ b/src/test/java/com/cosmian/findex/TestRedis.java @@ -19,6 +19,7 @@ import com.cosmian.jna.findex.structs.IndexedValue; import com.cosmian.jna.findex.ffi.SearchResults; import com.cosmian.jna.findex.structs.Keyword; +import com.cosmian.jna.findex.structs.Location; import com.cosmian.jna.findex.structs.NextKeyword; import com.cosmian.utils.CloudproofException; @@ -83,6 +84,20 @@ public void testUpsertAndSearchRedis() throws Exception { System.out .println("After insertion: chain_table size: " + db.getAllKeys(Redis.CHAIN_TABLE_INDEX).size()); + // + // Upsert a new keyword + // + HashMap> newIndexedKeyword = new HashMap<>(); + Set expectdeKeywords = new HashSet<>(); + expectdeKeywords.add(new Keyword("test")); + newIndexedKeyword.put(new IndexedValue(new Location("ici")), expectdeKeywords); + // It is returned the first time it is added. + Set newKeywords = Findex.upsert(new Findex.IndexRequest(key, label, db).add(newIndexedKeyword)).getResults(); + assertEquals(expectdeKeywords, newKeywords, "new keyword is not returned"); + // It is *not* returned the second time it is added. + newKeywords = Findex.upsert(new Findex.IndexRequest(key, label, db).add(newIndexedKeyword)).getResults(); + assert(newKeywords.isEmpty()); + // // Search // diff --git a/src/test/java/com/cosmian/findex/TestSqlite.java b/src/test/java/com/cosmian/findex/TestSqlite.java index eec528a..1e1c92c 100644 --- a/src/test/java/com/cosmian/findex/TestSqlite.java +++ b/src/test/java/com/cosmian/findex/TestSqlite.java @@ -137,6 +137,20 @@ public void testUpsertAndSearchSqlite() throws Exception { System.out .println("After insertion: chain_table size: " + db.getAllKeyValueItems("chain_table").size()); + // + // Upsert a new keyword + // + HashMap> newIndexedKeyword = new HashMap<>(); + Set expectdeKeywords = new HashSet<>(); + expectdeKeywords.add(new Keyword("test")); + newIndexedKeyword.put(new IndexedValue(new Location(new Long(1))), expectdeKeywords); + // It is returned the first time it is added. + Set newKeywords = Findex.upsert(new Findex.IndexRequest(key, label, db).add(newIndexedKeyword)).getResults(); + assertEquals(expectdeKeywords, newKeywords, "new keyword is not returned"); + // It is *not* returned the second time it is added. + newKeywords = Findex.upsert(new Findex.IndexRequest(key, label, db).add(newIndexedKeyword)).getResults(); + assert(newKeywords.isEmpty()); + // // Search // From 5d207bd80c8312eb2425316464ca4472cc546f3a Mon Sep 17 00:00:00 2001 From: Manuthor Date: Wed, 9 Aug 2023 18:04:32 +0200 Subject: [PATCH 02/16] fix(ci): cloudproof_js job --- .github/workflows/maven.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 133c926..40e76af 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -162,6 +162,7 @@ jobs: branch: develop target: wasm32-unknown-unknown kms-version: 4.3.3 + findex-cloud-version: 0.1.0 copy_fresh_build: false copy_regression_files: | cp ./cloudproof_java/non_regression_vector.json tests/data/cover_crypt/non_regression/java_non_regression_vector.json From f4de3aac878b30378a766304af5f69fb162dfd18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20BR=C3=89ZOT?= Date: Wed, 9 Aug 2023 18:18:23 +0200 Subject: [PATCH 03/16] fix: allocate enough memory needed for upsert result --- .../java/com/cosmian/jna/findex/Findex.java | 47 +++++++------------ .../java/com/cosmian/findex/TestRedis.java | 4 +- .../java/com/cosmian/findex/TestSqlite.java | 5 +- 3 files changed, 23 insertions(+), 33 deletions(-) diff --git a/src/main/java/com/cosmian/jna/findex/Findex.java b/src/main/java/com/cosmian/jna/findex/Findex.java index 9b39eb3..f8fa18c 100644 --- a/src/main/java/com/cosmian/jna/findex/Findex.java +++ b/src/main/java/com/cosmian/jna/findex/Findex.java @@ -4,6 +4,7 @@ import java.util.Arrays; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import com.cosmian.jna.findex.ffi.FindexUserCallbacks.SearchProgress; import com.cosmian.jna.findex.ffi.Progress; @@ -38,39 +39,23 @@ public static UpsertResults upsert(byte[] key, // // Prepare outputs // - // start with an arbitration buffer allocation size of 131072 (around 4096 - // indexedValues) - byte[] newKeywordsBuffer = new byte[1024]; + // Allocate the amount of memory needed to store all upserted keywords. + Set upsertedKeywords = additions.values().stream().flatMap(set -> set.stream()).collect(Collectors.toSet()); + upsertedKeywords.addAll(deletions.values().stream().flatMap(set -> set.stream()).collect(Collectors.toSet())); + Integer memoryToAllocate = upsertedKeywords.stream().map(keyword -> keyword.getBytes().length + 8).reduce(Integer::sum).get(); + byte[] newKeywordsBuffer = new byte[memoryToAllocate]; IntByReference newKeywordsBufferSize = new IntByReference(newKeywordsBuffer.length); - int ffiCode = INSTANCE.h_upsert( - newKeywordsBuffer, newKeywordsBufferSize, - keyPointer, key.length, - labelPointer, label.length, - indexedValuesToJson(additions), - indexedValuesToJson(deletions), - entryTableNumber, - db.fetchEntryCallback(), - db.upsertEntryCallback(), - db.upsertChainCallback()); - - FindexCallbackException.rethrowOnErrorCode(ffiCode, start, System.currentTimeMillis()); - - if (ffiCode != 0) { - // Retry with correct allocated size - newKeywordsBuffer = new byte[newKeywordsBufferSize.getValue()]; - long startRetry = System.currentTimeMillis(); - unwrap(INSTANCE.h_upsert( - newKeywordsBuffer, newKeywordsBufferSize, - keyPointer, key.length, - labelPointer, label.length, - indexedValuesToJson(additions), - indexedValuesToJson(deletions), - entryTableNumber, - db.fetchEntryCallback(), - db.upsertEntryCallback(), - db.upsertChainCallback()), startRetry); - } + unwrap(INSTANCE.h_upsert(newKeywordsBuffer, newKeywordsBufferSize, + keyPointer, key.length, + labelPointer, label.length, + indexedValuesToJson(additions), + indexedValuesToJson(deletions), + entryTableNumber, + db.fetchEntryCallback(), + db.upsertEntryCallback(), + db.upsertChainCallback()), + start); byte[] newKeywordsBytes = Arrays.copyOfRange(newKeywordsBuffer, 0, newKeywordsBufferSize.getValue()); diff --git a/src/test/java/com/cosmian/findex/TestRedis.java b/src/test/java/com/cosmian/findex/TestRedis.java index 3c28090..b6a59b1 100644 --- a/src/test/java/com/cosmian/findex/TestRedis.java +++ b/src/test/java/com/cosmian/findex/TestRedis.java @@ -18,6 +18,7 @@ import com.cosmian.jna.findex.ffi.ProgressResults; import com.cosmian.jna.findex.structs.IndexedValue; import com.cosmian.jna.findex.ffi.SearchResults; +import com.cosmian.jna.findex.ffi.UpsertResults; import com.cosmian.jna.findex.structs.Keyword; import com.cosmian.jna.findex.structs.Location; import com.cosmian.jna.findex.structs.NextKeyword; @@ -78,7 +79,8 @@ public void testUpsertAndSearchRedis() throws Exception { // Upsert // Map> indexedValuesAndWords = IndexUtils.index(testFindexDataset); - Findex.upsert(new Findex.IndexRequest(key, label, db).add(indexedValuesAndWords)); + UpsertResults res = Findex.upsert(new Findex.IndexRequest(key, label, db).add(indexedValuesAndWords)); + assertEquals(583, res.getResults().size(), "wrong number of new upserted keywords"); System.out .println("After insertion: entry_table size: " + db.getAllKeys(Redis.ENTRY_TABLE_INDEX).size()); System.out diff --git a/src/test/java/com/cosmian/findex/TestSqlite.java b/src/test/java/com/cosmian/findex/TestSqlite.java index 1e1c92c..a94344a 100644 --- a/src/test/java/com/cosmian/findex/TestSqlite.java +++ b/src/test/java/com/cosmian/findex/TestSqlite.java @@ -15,6 +15,7 @@ import com.cosmian.TestUtils; import com.cosmian.jna.findex.Findex; import com.cosmian.jna.findex.ffi.SearchResults; +import com.cosmian.jna.findex.ffi.UpsertResults; import com.cosmian.jna.findex.structs.IndexedValue; import com.cosmian.jna.findex.structs.Keyword; import com.cosmian.jna.findex.structs.Location; @@ -131,7 +132,9 @@ public void testUpsertAndSearchSqlite() throws Exception { // Upsert // Map> indexedValuesAndWords = IndexUtils.index(testFindexDataset); - Findex.upsert(new Findex.IndexRequest(key, label, db).add(indexedValuesAndWords)); + UpsertResults res = Findex.upsert(new Findex.IndexRequest(key, label, db).add(indexedValuesAndWords)); + assertEquals(583, res.getResults().size(), "wrong number of new upserted keywords"); + System.out.println("Upserted " + res.getResults().size() + " new keywords."); System.out .println("After insertion: entry_table size: " + db.getAllKeyValueItems("entry_table").size()); System.out From e2c871c07d6614c0b17308488198eaac6ef37d0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20BR=C3=89ZOT?= Date: Fri, 11 Aug 2023 10:00:04 +0200 Subject: [PATCH 04/16] fix: follow Rust modifications --- .../java/com/cosmian/jna/findex/Findex.java | 32 ++++++++++++------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/cosmian/jna/findex/Findex.java b/src/main/java/com/cosmian/jna/findex/Findex.java index f8fa18c..ea69247 100644 --- a/src/main/java/com/cosmian/jna/findex/Findex.java +++ b/src/main/java/com/cosmian/jna/findex/Findex.java @@ -4,7 +4,6 @@ import java.util.Arrays; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; import com.cosmian.jna.findex.ffi.FindexUserCallbacks.SearchProgress; import com.cosmian.jna.findex.ffi.Progress; @@ -34,19 +33,14 @@ public static UpsertResults upsert(byte[] key, keyPointer.write(0, key, 0, key.length); labelPointer.write(0, label, 0, label.length); - long start = System.currentTimeMillis(); - // // Prepare outputs // // Allocate the amount of memory needed to store all upserted keywords. - Set upsertedKeywords = additions.values().stream().flatMap(set -> set.stream()).collect(Collectors.toSet()); - upsertedKeywords.addAll(deletions.values().stream().flatMap(set -> set.stream()).collect(Collectors.toSet())); - Integer memoryToAllocate = upsertedKeywords.stream().map(keyword -> keyword.getBytes().length + 8).reduce(Integer::sum).get(); - byte[] newKeywordsBuffer = new byte[memoryToAllocate]; + byte[] newKeywordsBuffer = new byte[0]; IntByReference newKeywordsBufferSize = new IntByReference(newKeywordsBuffer.length); - - unwrap(INSTANCE.h_upsert(newKeywordsBuffer, newKeywordsBufferSize, + long start = System.currentTimeMillis(); + int ffiCode = INSTANCE.h_upsert(newKeywordsBuffer, newKeywordsBufferSize, keyPointer, key.length, labelPointer, label.length, indexedValuesToJson(additions), @@ -54,8 +48,24 @@ public static UpsertResults upsert(byte[] key, entryTableNumber, db.fetchEntryCallback(), db.upsertEntryCallback(), - db.upsertChainCallback()), - start); + db.upsertChainCallback()); + FindexCallbackException.rethrowOnErrorCode(ffiCode, start, System.currentTimeMillis()); + + if (ffiCode != 0) { + // Retry with correct allocated size + newKeywordsBuffer = new byte[newKeywordsBufferSize.getValue()]; + long startRetry = System.currentTimeMillis(); + unwrap(INSTANCE.h_upsert(newKeywordsBuffer, newKeywordsBufferSize, + keyPointer, key.length, + labelPointer, label.length, + indexedValuesToJson(additions), + indexedValuesToJson(deletions), + entryTableNumber, + db.fetchEntryCallback(), + db.upsertEntryCallback(), + db.upsertChainCallback()) + , startRetry); + } byte[] newKeywordsBytes = Arrays.copyOfRange(newKeywordsBuffer, 0, newKeywordsBufferSize.getValue()); From 7d908663388a9aaa15f47abd8b7301f7ba48d348 Mon Sep 17 00:00:00 2001 From: Manuthor Date: Fri, 11 Aug 2023 10:43:14 +0200 Subject: [PATCH 05/16] ci: update kms container --- .github/workflows/maven.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 40e76af..63af3e0 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -49,12 +49,7 @@ jobs: build_in_docker: services: kms: - image: cosmian/kms:4.3.3 - env: - COSMIAN_SERVER_URL: http://localhost:9998 - KMS_PUBLIC_PATH: /tmp - KMS_PRIVATE_PATH: /tmp - KMS_SHARED_PATH: /tmp + image: ghcr.io/cosmian/kms:4.3.3 ports: - 9998:9998 findex_cloud: From 7714fbebd2c6dab101dcfe5f18939b23f671ec27 Mon Sep 17 00:00:00 2001 From: Manuthor Date: Fri, 11 Aug 2023 10:45:50 +0200 Subject: [PATCH 06/16] ci: update kms container --- .github/workflows/maven.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 63af3e0..a83099c 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -49,7 +49,7 @@ jobs: build_in_docker: services: kms: - image: ghcr.io/cosmian/kms:4.3.3 + image: ghcr.io/cosmian/kms:4.4.3 ports: - 9998:9998 findex_cloud: From d5b703b94fc434fe966e171451f0bab84648067a Mon Sep 17 00:00:00 2001 From: Manuthor Date: Fri, 11 Aug 2023 11:02:52 +0200 Subject: [PATCH 07/16] ci: display ldd version --- .github/workflows/maven.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index a83099c..b4d1771 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -92,6 +92,11 @@ jobs: - run: yum -y install java-1.8.0-openjdk maven python3 python3-pip - run: python3 scripts/get_native_libraries.py + - name: Display ldd version + run: | + ldd --version + ldd src/main/resources/linux-x86-64/libcloudproof.so + - name: Build with Maven run: mvn compile env: From fe022104afee591d7c80c1464c69d654475bdb4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20BR=C3=89ZOT?= Date: Fri, 11 Aug 2023 11:42:11 +0200 Subject: [PATCH 08/16] fix: Findex cloud --- docker-compose.yml | 5 ++ .../java/com/cosmian/jna/findex/Findex.java | 1 - .../com/cosmian/jna/findex/FindexCloud.java | 59 +++++++++----- .../jna/findex/ffi/FindexNativeWrapper.java | 4 +- .../com/cosmian/findex/TestFindexCloud.java | 79 ++++++++++--------- 5 files changed, 88 insertions(+), 60 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1cb0dfb..7e0a683 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,3 +28,8 @@ services: image: redis:latest ports: - 6379:6379 + + findex_cloud: + image: ghcr.io/cosmian/findex_cloud:0.1.0 + ports: + - 8080:8080 diff --git a/src/main/java/com/cosmian/jna/findex/Findex.java b/src/main/java/com/cosmian/jna/findex/Findex.java index ea69247..df796aa 100644 --- a/src/main/java/com/cosmian/jna/findex/Findex.java +++ b/src/main/java/com/cosmian/jna/findex/Findex.java @@ -68,7 +68,6 @@ public static UpsertResults upsert(byte[] key, } byte[] newKeywordsBytes = Arrays.copyOfRange(newKeywordsBuffer, 0, newKeywordsBufferSize.getValue()); - return new Leb128Reader(newKeywordsBytes).readObject(UpsertResults.class); } } diff --git a/src/main/java/com/cosmian/jna/findex/FindexCloud.java b/src/main/java/com/cosmian/jna/findex/FindexCloud.java index b0882cc..a8955c5 100644 --- a/src/main/java/com/cosmian/jna/findex/FindexCloud.java +++ b/src/main/java/com/cosmian/jna/findex/FindexCloud.java @@ -6,6 +6,7 @@ import java.util.Set; import com.cosmian.jna.findex.ffi.SearchResults; +import com.cosmian.jna.findex.ffi.UpsertResults; import com.cosmian.jna.findex.serde.Leb128Reader; import com.cosmian.jna.findex.structs.IndexedValue; import com.cosmian.jna.findex.structs.Keyword; @@ -51,12 +52,11 @@ public static String generateNewToken( } } - public static void upsert( - String token, - byte[] label, - Map> additions, - Map> deletions, - String baseUrl) + public static UpsertResults upsert(String token, + byte[] label, + Map> additions, + Map> deletions, + String baseUrl) throws CloudproofException { try ( @@ -64,27 +64,46 @@ public static void upsert( labelPointer.write(0, label, 0, label.length); // Indexes creation + insertion/update - unwrap(INSTANCE.h_upsert_cloud( - token, - labelPointer, label.length, - indexedValuesToJson(additions), - indexedValuesToJson(deletions), - baseUrl)); + byte[] newKeywordsBuffer = new byte[0]; + IntByReference newKeywordsBufferSize = new IntByReference(newKeywordsBuffer.length); + long start = System.currentTimeMillis(); + int ffiCode = INSTANCE.h_upsert_cloud(newKeywordsBuffer, newKeywordsBufferSize, + token, + labelPointer, label.length, + indexedValuesToJson(additions), + indexedValuesToJson(deletions), + baseUrl); + FindexCallbackException.rethrowOnErrorCode(ffiCode, start, System.currentTimeMillis()); + + if (ffiCode != 0) { + // Retry with correct allocated size + newKeywordsBuffer = new byte[newKeywordsBufferSize.getValue()]; + long startRetry = System.currentTimeMillis(); + unwrap(INSTANCE.h_upsert_cloud(newKeywordsBuffer, newKeywordsBufferSize, + token, + labelPointer, label.length, + indexedValuesToJson(additions), + indexedValuesToJson(deletions), + baseUrl), + startRetry); + } + + byte[] newKeywordsBytes = Arrays.copyOfRange(newKeywordsBuffer, 0, newKeywordsBufferSize.getValue()); + return new Leb128Reader(newKeywordsBytes).readObject(UpsertResults.class); } } - public static void upsert(IndexRequest request) + public static UpsertResults upsert(IndexRequest request) throws CloudproofException { - upsert(request.token, request.label, request.additions, request.deletions, request.baseUrl); + return upsert(request.token, request.label, request.additions, request.deletions, request.baseUrl); } - public static void upsert( - String token, - byte[] label, - Map> additions, - Map> deletions) + public static UpsertResults upsert(String token, + byte[] label, + Map> additions, + Map> deletions) throws CloudproofException { - upsert(token, label, additions, deletions, null); + return upsert(token, label, additions, deletions, null); } public static SearchResults search(SearchRequest request) diff --git a/src/main/java/com/cosmian/jna/findex/ffi/FindexNativeWrapper.java b/src/main/java/com/cosmian/jna/findex/ffi/FindexNativeWrapper.java index 8184933..1759369 100644 --- a/src/main/java/com/cosmian/jna/findex/ffi/FindexNativeWrapper.java +++ b/src/main/java/com/cosmian/jna/findex/ffi/FindexNativeWrapper.java @@ -125,7 +125,9 @@ int h_search_cloud(byte[] dbUidsPtr, String keywords, String baseUrl); - int h_upsert_cloud(String token, + int h_upsert_cloud(byte[] newKeywordsBufferPtr, + IntByReference newKeywordsBufferLen, + String token, Pointer labelPtr, int labelLen, String additions, diff --git a/src/test/java/com/cosmian/findex/TestFindexCloud.java b/src/test/java/com/cosmian/findex/TestFindexCloud.java index 329c4b4..47f9243 100644 --- a/src/test/java/com/cosmian/findex/TestFindexCloud.java +++ b/src/test/java/com/cosmian/findex/TestFindexCloud.java @@ -9,58 +9,61 @@ import com.cosmian.jna.findex.FindexCloud; import com.cosmian.jna.findex.ffi.SearchResults; +import com.cosmian.jna.findex.ffi.UpsertResults; import com.cosmian.jna.findex.structs.Location; import com.cosmian.utils.RestClient; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; -public class TestFindexCloud { - @Test - public void testFindexCloud() throws Exception { - ObjectMapper mapper = new ObjectMapper(); - mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - String baseUrl = System.getenv("COSMIAN_FINDEX_CLOUD_BASE_URL"); - if (baseUrl == null) { - System.out.println("No COSMIAN_FINDEX_CLOUD_BASE_URL: ignoring"); - return; - } +/*public class TestFindexCloud {*/ + /*@Test*/ + /*public void testFindexCloud() throws Exception {*/ + /*ObjectMapper mapper = new ObjectMapper();*/ + /*mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);*/ + /*String baseUrl = System.getenv("COSMIAN_FINDEX_CLOUD_BASE_URL");*/ + /*if (baseUrl == null) {*/ + /*System.out.println("No COSMIAN_FINDEX_CLOUD_BASE_URL: ignoring");*/ + /*return;*/ + /*}*/ - String label = "Hello World!"; + /*String label = "Hello World!";*/ - RestClient client = new RestClient(baseUrl, Optional.empty()); - String response = client.json_post("/indexes", "{ \"name\": \"Test\" }"); + /*RestClient client = new RestClient(baseUrl, Optional.empty());*/ + /*String response = client.json_post("/indexes", "{ \"name\": \"Test\" }");*/ - Index index = mapper.readValue(response, Index.class); + /*Index index = mapper.readValue(response, Index.class);*/ - String token = FindexCloud.generateNewToken(index.publicId, index.fetchEntriesKey, index.fetchChainsKey, index.upsertEntriesKey, index.insertChainsKey); + /*String token = FindexCloud.generateNewToken(index.publicId, index.fetchEntriesKey, index.fetchChainsKey, index.upsertEntriesKey, index.insertChainsKey);*/ - FindexCloud.IndexRequest indexRequest = new FindexCloud.IndexRequest(token, label) - .add(new Location(1337), new String[] { "John", "Doe" }) - .add(new Location(42), new String[] { "Jane", "Doe" }); + /*FindexCloud.IndexRequest indexRequest = new FindexCloud.IndexRequest(token, label)*/ + /*.add(new Location(1337), new String[] { "John", "Doe" })*/ + /*.add(new Location(42), new String[] { "Jane", "Doe" });*/ - FindexCloud.upsert(indexRequest); + /*UpsertResults res = FindexCloud.upsert(indexRequest);*/ + /*assertEquals(0, res.getResults().size(), "wrong number of new keywords returned");*/ + /*assertEquals(1, res.getResults().size(), "wrong number of new keywords returned");*/ - FindexCloud.SearchRequest searchRequest = new FindexCloud.SearchRequest(token, label) - .keywords(new String[] { "Doe" }); + /*FindexCloud.SearchRequest searchRequest = new FindexCloud.SearchRequest(token, label)*/ + /*.keywords(new String[] { "Doe" });*/ - SearchResults searchResults = FindexCloud.search(searchRequest); + /*SearchResults searchResults = FindexCloud.search(searchRequest);*/ - assertEquals(new HashSet<>(Arrays.asList(new Long(1337), new Long(42))), searchResults.getNumbers()); - } + /*assertEquals(new HashSet<>(Arrays.asList(new Long(1337), new Long(42))), searchResults.getNumbers());*/ + /*}*/ - public static class Index { - @JsonProperty(value = "public_id") - String publicId; - @JsonProperty(value = "fetch_entries_key") - byte[] fetchEntriesKey; - @JsonProperty(value = "fetch_chains_key") - byte[] fetchChainsKey; - @JsonProperty(value = "upsert_entries_key") - byte[] upsertEntriesKey; - @JsonProperty(value = "insert_chains_key") - byte[] insertChainsKey; + /*public static class Index {*/ + /*@JsonProperty(value = "public_id")*/ + /*String publicId;*/ + /*@JsonProperty(value = "fetch_entries_key")*/ + /*byte[] fetchEntriesKey;*/ + /*@JsonProperty(value = "fetch_chains_key")*/ + /*byte[] fetchChainsKey;*/ + /*@JsonProperty(value = "upsert_entries_key")*/ + /*byte[] upsertEntriesKey;*/ + /*@JsonProperty(value = "insert_chains_key")*/ + /*byte[] insertChainsKey;*/ - Index() {} - } -} + /*Index() {}*/ + /*}*/ +/*}*/ From c4ab323222758eca688d85b3afa678d482d4c1e3 Mon Sep 17 00:00:00 2001 From: Manuthor Date: Fri, 11 Aug 2023 11:51:01 +0200 Subject: [PATCH 09/16] ci: update kms container --- .github/workflows/maven.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index b4d1771..2e8b849 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -161,7 +161,7 @@ jobs: with: branch: develop target: wasm32-unknown-unknown - kms-version: 4.3.3 + kms-version: 4.4.3 findex-cloud-version: 0.1.0 copy_fresh_build: false copy_regression_files: | @@ -174,7 +174,7 @@ jobs: with: branch: develop target: x86_64-unknown-linux-gnu - kms-version: 4.3.3 + kms-version: 4.4.3 copy_fresh_build: false copy_regression_files: | cp ./cloudproof_java/non_regression_vector.json tests/data/cover_crypt/non_regression/java_non_regression_vector.json From 763d439330ef647373f19a00c426af59c4fe2512 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20BR=C3=89ZOT?= Date: Fri, 11 Aug 2023 12:34:41 +0200 Subject: [PATCH 10/16] fix: uncomment Findex cloud tests --- .../com/cosmian/findex/TestFindexCloud.java | 79 +++++++++---------- 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/src/test/java/com/cosmian/findex/TestFindexCloud.java b/src/test/java/com/cosmian/findex/TestFindexCloud.java index 47f9243..d8d1d9d 100644 --- a/src/test/java/com/cosmian/findex/TestFindexCloud.java +++ b/src/test/java/com/cosmian/findex/TestFindexCloud.java @@ -16,54 +16,53 @@ import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; -/*public class TestFindexCloud {*/ - /*@Test*/ - /*public void testFindexCloud() throws Exception {*/ - /*ObjectMapper mapper = new ObjectMapper();*/ - /*mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);*/ - /*String baseUrl = System.getenv("COSMIAN_FINDEX_CLOUD_BASE_URL");*/ - /*if (baseUrl == null) {*/ - /*System.out.println("No COSMIAN_FINDEX_CLOUD_BASE_URL: ignoring");*/ - /*return;*/ - /*}*/ +public class TestFindexCloud { + @Test + public void testFindexCloud() throws Exception { + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + String baseUrl = System.getenv("COSMIAN_FINDEX_CLOUD_BASE_URL"); + if (baseUrl == null) { + System.out.println("No COSMIAN_FINDEX_CLOUD_BASE_URL: ignoring"); + return; + } - /*String label = "Hello World!";*/ + String label = "Hello World!"; - /*RestClient client = new RestClient(baseUrl, Optional.empty());*/ - /*String response = client.json_post("/indexes", "{ \"name\": \"Test\" }");*/ + RestClient client = new RestClient(baseUrl, Optional.empty()); + String response = client.json_post("/indexes", "{ \"name\": \"Test\" }"); - /*Index index = mapper.readValue(response, Index.class);*/ + Index index = mapper.readValue(response, Index.class); - /*String token = FindexCloud.generateNewToken(index.publicId, index.fetchEntriesKey, index.fetchChainsKey, index.upsertEntriesKey, index.insertChainsKey);*/ + String token = FindexCloud.generateNewToken(index.publicId, index.fetchEntriesKey, index.fetchChainsKey, index.upsertEntriesKey, index.insertChainsKey); - /*FindexCloud.IndexRequest indexRequest = new FindexCloud.IndexRequest(token, label)*/ - /*.add(new Location(1337), new String[] { "John", "Doe" })*/ - /*.add(new Location(42), new String[] { "Jane", "Doe" });*/ + FindexCloud.IndexRequest indexRequest = new FindexCloud.IndexRequest(token, label) + .add(new Location(1337), new String[] { "John", "Doe" }) + .add(new Location(42), new String[] { "Jane", "Doe" }); - /*UpsertResults res = FindexCloud.upsert(indexRequest);*/ - /*assertEquals(0, res.getResults().size(), "wrong number of new keywords returned");*/ - /*assertEquals(1, res.getResults().size(), "wrong number of new keywords returned");*/ + UpsertResults res = FindexCloud.upsert(indexRequest); + assertEquals(3, res.getResults().size(), "wrong number of new keywords returned"); - /*FindexCloud.SearchRequest searchRequest = new FindexCloud.SearchRequest(token, label)*/ - /*.keywords(new String[] { "Doe" });*/ + FindexCloud.SearchRequest searchRequest = new FindexCloud.SearchRequest(token, label) + .keywords(new String[] { "Doe" }); - /*SearchResults searchResults = FindexCloud.search(searchRequest);*/ + SearchResults searchResults = FindexCloud.search(searchRequest); - /*assertEquals(new HashSet<>(Arrays.asList(new Long(1337), new Long(42))), searchResults.getNumbers());*/ - /*}*/ + assertEquals(new HashSet<>(Arrays.asList(new Long(1337), new Long(42))), searchResults.getNumbers()); + } - /*public static class Index {*/ - /*@JsonProperty(value = "public_id")*/ - /*String publicId;*/ - /*@JsonProperty(value = "fetch_entries_key")*/ - /*byte[] fetchEntriesKey;*/ - /*@JsonProperty(value = "fetch_chains_key")*/ - /*byte[] fetchChainsKey;*/ - /*@JsonProperty(value = "upsert_entries_key")*/ - /*byte[] upsertEntriesKey;*/ - /*@JsonProperty(value = "insert_chains_key")*/ - /*byte[] insertChainsKey;*/ + public static class Index { + @JsonProperty(value = "public_id") + String publicId; + @JsonProperty(value = "fetch_entries_key") + byte[] fetchEntriesKey; + @JsonProperty(value = "fetch_chains_key") + byte[] fetchChainsKey; + @JsonProperty(value = "upsert_entries_key") + byte[] upsertEntriesKey; + @JsonProperty(value = "insert_chains_key") + byte[] insertChainsKey; - /*Index() {}*/ - /*}*/ -/*}*/ + Index() {} + } +} From 3785e73f1ec08dd231d5d520f04d92ac615b0bb3 Mon Sep 17 00:00:00 2001 From: Manuthor Date: Fri, 11 Aug 2023 13:38:50 +0200 Subject: [PATCH 11/16] test: update kms version in docker-compose.yml --- docker-compose.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7e0a683..0e1e2a3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,12 +13,7 @@ services: - PGDATA=/tmp/postgres2 kms: container_name: kms - image: cosmian/kms:4.3.3 - environment: - - KMS_HOSTNAME=0.0.0.0 - - KMS_PUBLIC_PATH=/tmp - - KMS_PRIVATE_PATH=/tmp - - KMS_SHARED_PATH=/tmp + image: ghcr.io/cosmian/kms:4.4.3 ports: - 9998:9998 depends_on: From a63cfa671ddf87e8934962638e57fb7ad84494f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20BR=C3=89ZOT?= Date: Fri, 11 Aug 2023 14:03:20 +0200 Subject: [PATCH 12/16] Update src/main/java/com/cosmian/jna/findex/Findex.java Co-authored-by: Manuthor <32013169+Manuthor@users.noreply.github.com> --- src/main/java/com/cosmian/jna/findex/Findex.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/cosmian/jna/findex/Findex.java b/src/main/java/com/cosmian/jna/findex/Findex.java index df796aa..3afa11e 100644 --- a/src/main/java/com/cosmian/jna/findex/Findex.java +++ b/src/main/java/com/cosmian/jna/findex/Findex.java @@ -51,7 +51,7 @@ public static UpsertResults upsert(byte[] key, db.upsertChainCallback()); FindexCallbackException.rethrowOnErrorCode(ffiCode, start, System.currentTimeMillis()); - if (ffiCode != 0) { + if (ffiCode == 1) { // Retry with correct allocated size newKeywordsBuffer = new byte[newKeywordsBufferSize.getValue()]; long startRetry = System.currentTimeMillis(); From d2ac9d70f1dd24d328e64dfbe0b93000799a2beb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20BR=C3=89ZOT?= Date: Fri, 11 Aug 2023 14:14:31 +0200 Subject: [PATCH 13/16] fix: review --- .../java/com/cosmian/jna/findex/Findex.java | 34 +++++++++---------- .../com/cosmian/jna/findex/FindexCloud.java | 16 +++++---- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/src/main/java/com/cosmian/jna/findex/Findex.java b/src/main/java/com/cosmian/jna/findex/Findex.java index 3afa11e..98f3ee7 100644 --- a/src/main/java/com/cosmian/jna/findex/Findex.java +++ b/src/main/java/com/cosmian/jna/findex/Findex.java @@ -33,28 +33,27 @@ public static UpsertResults upsert(byte[] key, keyPointer.write(0, key, 0, key.length); labelPointer.write(0, label, 0, label.length); - // - // Prepare outputs - // - // Allocate the amount of memory needed to store all upserted keywords. + // Do not allocate memory. The Rust FFI function will directly + // return after setting newKeywordsBufferSize to an upper bound on + // the amount of memory to allocate. byte[] newKeywordsBuffer = new byte[0]; - IntByReference newKeywordsBufferSize = new IntByReference(newKeywordsBuffer.length); + IntByReference newKeywordsBufferSize = new IntByReference(); + long start = System.currentTimeMillis(); int ffiCode = INSTANCE.h_upsert(newKeywordsBuffer, newKeywordsBufferSize, - keyPointer, key.length, - labelPointer, label.length, - indexedValuesToJson(additions), - indexedValuesToJson(deletions), - entryTableNumber, - db.fetchEntryCallback(), - db.upsertEntryCallback(), - db.upsertChainCallback()); + keyPointer, key.length, + labelPointer, label.length, + indexedValuesToJson(additions), + indexedValuesToJson(deletions), + entryTableNumber, + db.fetchEntryCallback(), + db.upsertEntryCallback(), + db.upsertChainCallback()); + FindexCallbackException.rethrowOnErrorCode(ffiCode, start, System.currentTimeMillis()); if (ffiCode == 1) { - // Retry with correct allocated size newKeywordsBuffer = new byte[newKeywordsBufferSize.getValue()]; - long startRetry = System.currentTimeMillis(); unwrap(INSTANCE.h_upsert(newKeywordsBuffer, newKeywordsBufferSize, keyPointer, key.length, labelPointer, label.length, @@ -63,8 +62,9 @@ public static UpsertResults upsert(byte[] key, entryTableNumber, db.fetchEntryCallback(), db.upsertEntryCallback(), - db.upsertChainCallback()) - , startRetry); + db.upsertChainCallback())); + } else if (ffiCode != 0) { + unwrap(ffiCode); } byte[] newKeywordsBytes = Arrays.copyOfRange(newKeywordsBuffer, 0, newKeywordsBufferSize.getValue()); diff --git a/src/main/java/com/cosmian/jna/findex/FindexCloud.java b/src/main/java/com/cosmian/jna/findex/FindexCloud.java index a8955c5..2d0967e 100644 --- a/src/main/java/com/cosmian/jna/findex/FindexCloud.java +++ b/src/main/java/com/cosmian/jna/findex/FindexCloud.java @@ -63,9 +63,12 @@ public static UpsertResults upsert(String token, final Memory labelPointer = new Memory(label.length)) { labelPointer.write(0, label, 0, label.length); - // Indexes creation + insertion/update + // Do not allocate memory. The Rust FFI function will directly + // return after setting newKeywordsBufferSize to an upper bound on + // the amount of memory to allocate. byte[] newKeywordsBuffer = new byte[0]; - IntByReference newKeywordsBufferSize = new IntByReference(newKeywordsBuffer.length); + IntByReference newKeywordsBufferSize = new IntByReference(); + long start = System.currentTimeMillis(); int ffiCode = INSTANCE.h_upsert_cloud(newKeywordsBuffer, newKeywordsBufferSize, token, @@ -75,17 +78,16 @@ public static UpsertResults upsert(String token, baseUrl); FindexCallbackException.rethrowOnErrorCode(ffiCode, start, System.currentTimeMillis()); - if (ffiCode != 0) { - // Retry with correct allocated size + if (ffiCode == 1) { newKeywordsBuffer = new byte[newKeywordsBufferSize.getValue()]; - long startRetry = System.currentTimeMillis(); unwrap(INSTANCE.h_upsert_cloud(newKeywordsBuffer, newKeywordsBufferSize, token, labelPointer, label.length, indexedValuesToJson(additions), indexedValuesToJson(deletions), - baseUrl), - startRetry); + baseUrl)); + } else if (ffiCode != 0) { + unwrap(ffiCode); } byte[] newKeywordsBytes = Arrays.copyOfRange(newKeywordsBuffer, 0, newKeywordsBufferSize.getValue()); From 36f1c810d5f493fc3f1dc82c0aaa074a811b665a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20BR=C3=89ZOT?= Date: Thu, 10 Aug 2023 16:34:48 +0200 Subject: [PATCH 14/16] feat: allocate FFI output from Rust --- .../java/com/cosmian/jna/findex/Findex.java | 51 +++++++------------ .../com/cosmian/jna/findex/FindexCloud.java | 41 ++++++--------- .../jna/findex/ffi/FindexNativeWrapper.java | 4 +- 3 files changed, 35 insertions(+), 61 deletions(-) diff --git a/src/main/java/com/cosmian/jna/findex/Findex.java b/src/main/java/com/cosmian/jna/findex/Findex.java index 98f3ee7..716af92 100644 --- a/src/main/java/com/cosmian/jna/findex/Findex.java +++ b/src/main/java/com/cosmian/jna/findex/Findex.java @@ -15,6 +15,8 @@ import com.cosmian.jna.findex.structs.Keyword; import com.cosmian.utils.CloudproofException; import com.sun.jna.Memory; +import com.sun.jna.Native; +import com.sun.jna.Pointer; import com.sun.jna.ptr.IntByReference; public final class Findex extends FindexBase { @@ -33,41 +35,24 @@ public static UpsertResults upsert(byte[] key, keyPointer.write(0, key, 0, key.length); labelPointer.write(0, label, 0, label.length); - // Do not allocate memory. The Rust FFI function will directly - // return after setting newKeywordsBufferSize to an upper bound on - // the amount of memory to allocate. - byte[] newKeywordsBuffer = new byte[0]; - IntByReference newKeywordsBufferSize = new IntByReference(); + // Allocate the amount of memory needed to store a pointer. + Memory newKeywordsBuffer = new Memory(8); + IntByReference newKeywordsBufferSize = new IntByReference(0); long start = System.currentTimeMillis(); - int ffiCode = INSTANCE.h_upsert(newKeywordsBuffer, newKeywordsBufferSize, - keyPointer, key.length, - labelPointer, label.length, - indexedValuesToJson(additions), - indexedValuesToJson(deletions), - entryTableNumber, - db.fetchEntryCallback(), - db.upsertEntryCallback(), - db.upsertChainCallback()); - - FindexCallbackException.rethrowOnErrorCode(ffiCode, start, System.currentTimeMillis()); - - if (ffiCode == 1) { - newKeywordsBuffer = new byte[newKeywordsBufferSize.getValue()]; - unwrap(INSTANCE.h_upsert(newKeywordsBuffer, newKeywordsBufferSize, - keyPointer, key.length, - labelPointer, label.length, - indexedValuesToJson(additions), - indexedValuesToJson(deletions), - entryTableNumber, - db.fetchEntryCallback(), - db.upsertEntryCallback(), - db.upsertChainCallback())); - } else if (ffiCode != 0) { - unwrap(ffiCode); - } - - byte[] newKeywordsBytes = Arrays.copyOfRange(newKeywordsBuffer, 0, newKeywordsBufferSize.getValue()); + unwrap(INSTANCE.h_upsert(newKeywordsBuffer, newKeywordsBufferSize, + keyPointer, key.length, + labelPointer, label.length, + indexedValuesToJson(additions), + indexedValuesToJson(deletions), + entryTableNumber, + db.fetchEntryCallback(), + db.upsertEntryCallback(), + db.upsertChainCallback()), + start); + + byte[] newKeywordsBytes = newKeywordsBuffer.getPointer(0).getByteArray(0, newKeywordsBufferSize.getValue()); + Native.free(Pointer.nativeValue(newKeywordsBuffer.getPointer(0))); return new Leb128Reader(newKeywordsBytes).readObject(UpsertResults.class); } } diff --git a/src/main/java/com/cosmian/jna/findex/FindexCloud.java b/src/main/java/com/cosmian/jna/findex/FindexCloud.java index 2d0967e..1d62a72 100644 --- a/src/main/java/com/cosmian/jna/findex/FindexCloud.java +++ b/src/main/java/com/cosmian/jna/findex/FindexCloud.java @@ -12,6 +12,8 @@ import com.cosmian.jna.findex.structs.Keyword; import com.cosmian.utils.CloudproofException; import com.sun.jna.Memory; +import com.sun.jna.Pointer; +import com.sun.jna.Native; import com.sun.jna.ptr.IntByReference; public final class FindexCloud extends FindexBase { @@ -63,34 +65,21 @@ public static UpsertResults upsert(String token, final Memory labelPointer = new Memory(label.length)) { labelPointer.write(0, label, 0, label.length); - // Do not allocate memory. The Rust FFI function will directly - // return after setting newKeywordsBufferSize to an upper bound on - // the amount of memory to allocate. - byte[] newKeywordsBuffer = new byte[0]; - IntByReference newKeywordsBufferSize = new IntByReference(); + // Allocate the amount of memory needed to store a pointer. + Memory newKeywordsBuffer = new Memory(8); + IntByReference newKeywordsBufferSize = new IntByReference(0); long start = System.currentTimeMillis(); - int ffiCode = INSTANCE.h_upsert_cloud(newKeywordsBuffer, newKeywordsBufferSize, - token, - labelPointer, label.length, - indexedValuesToJson(additions), - indexedValuesToJson(deletions), - baseUrl); - FindexCallbackException.rethrowOnErrorCode(ffiCode, start, System.currentTimeMillis()); - - if (ffiCode == 1) { - newKeywordsBuffer = new byte[newKeywordsBufferSize.getValue()]; - unwrap(INSTANCE.h_upsert_cloud(newKeywordsBuffer, newKeywordsBufferSize, - token, - labelPointer, label.length, - indexedValuesToJson(additions), - indexedValuesToJson(deletions), - baseUrl)); - } else if (ffiCode != 0) { - unwrap(ffiCode); - } - - byte[] newKeywordsBytes = Arrays.copyOfRange(newKeywordsBuffer, 0, newKeywordsBufferSize.getValue()); + unwrap(INSTANCE.h_upsert_cloud(newKeywordsBuffer, newKeywordsBufferSize, + token, + labelPointer, label.length, + indexedValuesToJson(additions), + indexedValuesToJson(deletions), + baseUrl), + start); + + byte[] newKeywordsBytes = newKeywordsBuffer.getPointer(0).getByteArray(0, newKeywordsBufferSize.getValue()); + Native.free(Pointer.nativeValue(newKeywordsBuffer.getPointer(0))); return new Leb128Reader(newKeywordsBytes).readObject(UpsertResults.class); } } diff --git a/src/main/java/com/cosmian/jna/findex/ffi/FindexNativeWrapper.java b/src/main/java/com/cosmian/jna/findex/ffi/FindexNativeWrapper.java index 1759369..72ea0d3 100644 --- a/src/main/java/com/cosmian/jna/findex/ffi/FindexNativeWrapper.java +++ b/src/main/java/com/cosmian/jna/findex/ffi/FindexNativeWrapper.java @@ -78,7 +78,7 @@ boolean apply(Pointer output, throws CloudproofException; } - int h_upsert(byte[] newKeywordsBufferPtr, + int h_upsert(Pointer newKeywordsBufferPtr, IntByReference newKeywordsBufferLen, Pointer masterKeyPtr, int masterKeyLen, @@ -125,7 +125,7 @@ int h_search_cloud(byte[] dbUidsPtr, String keywords, String baseUrl); - int h_upsert_cloud(byte[] newKeywordsBufferPtr, + int h_upsert_cloud(Pointer newKeywordsBufferPtr, IntByReference newKeywordsBufferLen, String token, Pointer labelPtr, From d0603d20e969c8331df930c29d514f2d0b4ba5fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20BR=C3=89ZOT?= Date: Thu, 10 Aug 2023 16:47:51 +0200 Subject: [PATCH 15/16] fix: pull the correct branch --- scripts/get_native_libraries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/get_native_libraries.py b/scripts/get_native_libraries.py index e0b84ad..0041dcd 100644 --- a/scripts/get_native_libraries.py +++ b/scripts/get_native_libraries.py @@ -64,4 +64,4 @@ def download_native_libraries(version: str) -> bool: if __name__ == '__main__': ret = download_native_libraries('v2.2.0') if ret is False and getenv('GITHUB_ACTIONS'): - download_native_libraries('last_build/feature/findex_5_0_0') + download_native_libraries('last_build/feature/ffi_allocate_from_rust') From 46b3d56cf0724a3a075ad4ca9fe727b6951f2b03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20BR=C3=89ZOT?= Date: Fri, 11 Aug 2023 16:32:52 +0200 Subject: [PATCH 16/16] feat: exptend modification to all Findex FFI --- .../java/com/cosmian/jna/findex/Findex.java | 50 ++++--------- .../com/cosmian/jna/findex/FindexCloud.java | 74 +++++++------------ .../jna/findex/ffi/FindexNativeWrapper.java | 6 +- 3 files changed, 43 insertions(+), 87 deletions(-) diff --git a/src/main/java/com/cosmian/jna/findex/Findex.java b/src/main/java/com/cosmian/jna/findex/Findex.java index 716af92..9182b6d 100644 --- a/src/main/java/com/cosmian/jna/findex/Findex.java +++ b/src/main/java/com/cosmian/jna/findex/Findex.java @@ -104,13 +104,6 @@ public static SearchResults search(byte[] key, Database db, SearchProgress progressCallback) throws CloudproofException { - // - // Prepare outputs - // - // start with an arbitration buffer allocation size of 131072 (around 4096 - // indexedValues) - byte[] indexedValuesBuffer = new byte[131072]; - IntByReference indexedValuesBufferSize = new IntByReference(indexedValuesBuffer.length); // Findex master keys if (key == null) { @@ -128,37 +121,22 @@ public static SearchResults search(byte[] key, String wordsJson = keywordsToJson(keyWords); - // Indexes creation + insertion/update - long start = System.currentTimeMillis(); - int ffiCode = INSTANCE.h_search( - indexedValuesBuffer, indexedValuesBufferSize, - keyPointer, key.length, - labelPointer, label.length, - wordsJson, - entryTableNumber, - wrappedProgress, - db.fetchEntryCallback(), - db.fetchChainCallback()); - - FindexCallbackException.rethrowOnErrorCode(ffiCode, start, System.currentTimeMillis()); - - if (ffiCode != 0) { - // Retry with correct allocated size - indexedValuesBuffer = new byte[indexedValuesBufferSize.getValue()]; - long startRetry = System.currentTimeMillis(); - unwrap(INSTANCE.h_search( - indexedValuesBuffer, indexedValuesBufferSize, - keyPointer, key.length, - labelPointer, label.length, - wordsJson, - entryTableNumber, - wrappedProgress, - db.fetchEntryCallback(), - db.fetchChainCallback()), startRetry); - } + Memory indexedValuesBuffer = new Memory(8); + IntByReference indexedValuesBufferSize = new IntByReference(0); - byte[] indexedValuesBytes = Arrays.copyOfRange(indexedValuesBuffer, 0, indexedValuesBufferSize.getValue()); + long start = System.currentTimeMillis(); + unwrap(INSTANCE.h_search(indexedValuesBuffer, indexedValuesBufferSize, + keyPointer, key.length, + labelPointer, label.length, + wordsJson, + entryTableNumber, + wrappedProgress, + db.fetchEntryCallback(), + db.fetchChainCallback()), + start); + byte[] indexedValuesBytes = indexedValuesBuffer.getPointer(0).getByteArray(0, indexedValuesBufferSize.getValue()); + Native.free(Pointer.nativeValue(indexedValuesBuffer.getPointer(0))); return new Leb128Reader(indexedValuesBytes).readObject(SearchResults.class); } } diff --git a/src/main/java/com/cosmian/jna/findex/FindexCloud.java b/src/main/java/com/cosmian/jna/findex/FindexCloud.java index 1d62a72..5eabf97 100644 --- a/src/main/java/com/cosmian/jna/findex/FindexCloud.java +++ b/src/main/java/com/cosmian/jna/findex/FindexCloud.java @@ -1,7 +1,6 @@ package com.cosmian.jna.findex; import java.nio.charset.StandardCharsets; -import java.util.Arrays; import java.util.Map; import java.util.Set; @@ -36,20 +35,20 @@ public static String generateNewToken( upsertEntriesSeedPointer.write(0, upsertEntriesSeed, 0, upsertEntriesSeed.length); insertChainsSeedPointer.write(0, insertChainsSeed, 0, insertChainsSeed.length); - byte[] tokenBuffer = new byte[200]; - IntByReference tokenBufferSize = new IntByReference(tokenBuffer.length); - - unwrap(INSTANCE.h_generate_new_token( - tokenBuffer, - tokenBufferSize, - indexId, - fetchEntriesSeedPointer, fetchEntriesSeed.length, - fetchChainsSeedPointer, fetchChainsSeed.length, - upsertEntriesSeedPointer, upsertEntriesSeed.length, - insertChainsSeedPointer, insertChainsSeed.length)); - - byte[] tokenBytes = Arrays.copyOfRange(tokenBuffer, 0, tokenBufferSize.getValue()); - + // Allocate the amount of memory needed to store a pointer. + Memory tokenBuffer = new Memory(8); + IntByReference tokenBufferSize = new IntByReference(0); + + unwrap(INSTANCE.h_generate_new_token(tokenBuffer, + tokenBufferSize, + indexId, + fetchEntriesSeedPointer, fetchEntriesSeed.length, + fetchChainsSeedPointer, fetchChainsSeed.length, + upsertEntriesSeedPointer, upsertEntriesSeed.length, + insertChainsSeedPointer, insertChainsSeed.length)); + + byte[] tokenBytes = tokenBuffer.getPointer(0).getByteArray(0, tokenBufferSize.getValue()); + Native.free(Pointer.nativeValue(tokenBuffer.getPointer(0))); return new String(tokenBytes, StandardCharsets.UTF_8); } } @@ -69,14 +68,12 @@ public static UpsertResults upsert(String token, Memory newKeywordsBuffer = new Memory(8); IntByReference newKeywordsBufferSize = new IntByReference(0); - long start = System.currentTimeMillis(); unwrap(INSTANCE.h_upsert_cloud(newKeywordsBuffer, newKeywordsBufferSize, token, labelPointer, label.length, indexedValuesToJson(additions), indexedValuesToJson(deletions), - baseUrl), - start); + baseUrl)); byte[] newKeywordsBytes = newKeywordsBuffer.getPointer(0).getByteArray(0, newKeywordsBufferSize.getValue()); Native.free(Pointer.nativeValue(newKeywordsBuffer.getPointer(0))); @@ -114,45 +111,26 @@ public static SearchResults search(String token, Set keyWords, String baseUrl) throws CloudproofException { - // - // Prepare outputs - // - // start with an arbitration buffer allocation size of 131072 (around 4096 - // indexedValues) - byte[] indexedValuesBuffer = new byte[131072]; - IntByReference indexedValuesBufferSize = new IntByReference(indexedValuesBuffer.length); if (token == null) { throw new CloudproofException("Token cannot be null"); } + try (final Memory labelPointer = new Memory(label.length)) { labelPointer.write(0, label, 0, label.length); - String wordsJson = keywordsToJson(keyWords); - // Indexes creation + insertion/update - int ffiCode = INSTANCE.h_search_cloud( - indexedValuesBuffer, indexedValuesBufferSize, - token, - labelPointer, label.length, - wordsJson, - baseUrl); - if (ffiCode != 0) { - // Retry with correct allocated size - indexedValuesBuffer = new byte[indexedValuesBufferSize.getValue()]; - ffiCode = INSTANCE.h_search_cloud( - indexedValuesBuffer, indexedValuesBufferSize, - token, - labelPointer, label.length, - wordsJson, - baseUrl); - if (ffiCode != 0) { - throw new CloudproofException(get_last_error(4095)); - } - } - - byte[] indexedValuesBytes = Arrays.copyOfRange(indexedValuesBuffer, 0, indexedValuesBufferSize.getValue()); + Memory indexedValuesBuffer = new Memory(8); + IntByReference indexedValuesBufferSize = new IntByReference(0); + + unwrap(INSTANCE.h_search_cloud(indexedValuesBuffer, indexedValuesBufferSize, + token, + labelPointer, label.length, + wordsJson, + baseUrl)); + byte[] indexedValuesBytes = indexedValuesBuffer.getPointer(0).getByteArray(0, indexedValuesBufferSize.getValue()); + Native.free(Pointer.nativeValue(indexedValuesBuffer.getPointer(0))); return new Leb128Reader(indexedValuesBytes).readObject(SearchResults.class); } } diff --git a/src/main/java/com/cosmian/jna/findex/ffi/FindexNativeWrapper.java b/src/main/java/com/cosmian/jna/findex/ffi/FindexNativeWrapper.java index 72ea0d3..123059d 100644 --- a/src/main/java/com/cosmian/jna/findex/ffi/FindexNativeWrapper.java +++ b/src/main/java/com/cosmian/jna/findex/ffi/FindexNativeWrapper.java @@ -105,7 +105,7 @@ int h_compact(Pointer oldMasterKeyPtr, UpdateLinesCallback updateLines, ListRemovedLocationsCallback listRemovedLocations); - int h_search(byte[] searchresultsPtr, + int h_search(Pointer searchresultsPtr, IntByReference searchresultsLen, Pointer masterKeyPtr, int masterKeyLength, @@ -117,7 +117,7 @@ int h_search(byte[] searchresultsPtr, FetchEntryCallback fetchEntry, FetchChainCallback fetchChain); - int h_search_cloud(byte[] dbUidsPtr, + int h_search_cloud(Pointer dbUidsPtr, IntByReference dbUidsLen, String token, Pointer labelPtr, @@ -135,7 +135,7 @@ int h_upsert_cloud(Pointer newKeywordsBufferPtr, String baseUrl); - int h_generate_new_token(byte[] tokenPtr, + int h_generate_new_token(Pointer tokenPtr, IntByReference tokenLen, String indexIdPtr, Pointer fetchEntriesSeedPtr,