diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 133c926..2e8b849 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -49,12 +49,7 @@ jobs: build_in_docker: services: kms: - image: cosmian/kms:4.3.3 - env: - COSMIAN_SERVER_URL: http://localhost:9998 - KMS_PUBLIC_PATH: /tmp - KMS_PRIVATE_PATH: /tmp - KMS_SHARED_PATH: /tmp + image: ghcr.io/cosmian/kms:4.4.3 ports: - 9998:9998 findex_cloud: @@ -97,6 +92,11 @@ jobs: - run: yum -y install java-1.8.0-openjdk maven python3 python3-pip - run: python3 scripts/get_native_libraries.py + - name: Display ldd version + run: | + ldd --version + ldd src/main/resources/linux-x86-64/libcloudproof.so + - name: Build with Maven run: mvn compile env: @@ -161,7 +161,8 @@ jobs: with: branch: develop target: wasm32-unknown-unknown - kms-version: 4.3.3 + kms-version: 4.4.3 + findex-cloud-version: 0.1.0 copy_fresh_build: false copy_regression_files: | cp ./cloudproof_java/non_regression_vector.json tests/data/cover_crypt/non_regression/java_non_regression_vector.json @@ -173,7 +174,7 @@ jobs: with: branch: develop target: x86_64-unknown-linux-gnu - kms-version: 4.3.3 + kms-version: 4.4.3 copy_fresh_build: false copy_regression_files: | cp ./cloudproof_java/non_regression_vector.json tests/data/cover_crypt/non_regression/java_non_regression_vector.json diff --git a/docker-compose.yml b/docker-compose.yml index 1cb0dfb..0e1e2a3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,12 +13,7 @@ services: - PGDATA=/tmp/postgres2 kms: container_name: kms - image: cosmian/kms:4.3.3 - environment: - - KMS_HOSTNAME=0.0.0.0 - - KMS_PUBLIC_PATH=/tmp - - KMS_PRIVATE_PATH=/tmp - - KMS_SHARED_PATH=/tmp + image: ghcr.io/cosmian/kms:4.4.3 ports: - 9998:9998 depends_on: @@ -28,3 +23,8 @@ services: image: redis:latest ports: - 6379:6379 + + findex_cloud: + image: ghcr.io/cosmian/findex_cloud:0.1.0 + ports: + - 8080:8080 diff --git a/scripts/get_native_libraries.py b/scripts/get_native_libraries.py index c0ad6b8..0041dcd 100644 --- a/scripts/get_native_libraries.py +++ b/scripts/get_native_libraries.py @@ -62,6 +62,6 @@ def download_native_libraries(version: str) -> bool: if __name__ == '__main__': - ret = download_native_libraries('v2.0.1') + ret = download_native_libraries('v2.2.0') if ret is False and getenv('GITHUB_ACTIONS'): - download_native_libraries('last_build/release/v2.0.1') + download_native_libraries('last_build/feature/ffi_allocate_from_rust') diff --git a/src/main/java/com/cosmian/jna/findex/Findex.java b/src/main/java/com/cosmian/jna/findex/Findex.java index 97fceca..9182b6d 100644 --- a/src/main/java/com/cosmian/jna/findex/Findex.java +++ b/src/main/java/com/cosmian/jna/findex/Findex.java @@ -9,22 +9,24 @@ import com.cosmian.jna.findex.ffi.Progress; import com.cosmian.jna.findex.ffi.ProgressResults; import com.cosmian.jna.findex.ffi.SearchResults; +import com.cosmian.jna.findex.ffi.UpsertResults; import com.cosmian.jna.findex.serde.Leb128Reader; import com.cosmian.jna.findex.structs.IndexedValue; import com.cosmian.jna.findex.structs.Keyword; import com.cosmian.utils.CloudproofException; import com.sun.jna.Memory; +import com.sun.jna.Native; +import com.sun.jna.Pointer; import com.sun.jna.ptr.IntByReference; public final class Findex extends FindexBase { - public static void upsert( - byte[] key, - byte[] label, - Map> additions, - Map> deletions, - int entryTableNumber, - Database db) + public static UpsertResults upsert(byte[] key, + byte[] label, + Map> additions, + Map> deletions, + int entryTableNumber, + Database db) throws CloudproofException { try ( @@ -33,35 +35,42 @@ public static void upsert( keyPointer.write(0, key, 0, key.length); labelPointer.write(0, label, 0, label.length); + // Allocate the amount of memory needed to store a pointer. + Memory newKeywordsBuffer = new Memory(8); + IntByReference newKeywordsBufferSize = new IntByReference(0); + long start = System.currentTimeMillis(); - // Indexes creation + insertion/update - unwrap(INSTANCE.h_upsert( - keyPointer, key.length, - labelPointer, label.length, - indexedValuesToJson(additions), - indexedValuesToJson(deletions), - entryTableNumber, - db.fetchEntryCallback(), - db.upsertEntryCallback(), - db.upsertChainCallback()), start); + unwrap(INSTANCE.h_upsert(newKeywordsBuffer, newKeywordsBufferSize, + keyPointer, key.length, + labelPointer, label.length, + indexedValuesToJson(additions), + indexedValuesToJson(deletions), + entryTableNumber, + db.fetchEntryCallback(), + db.upsertEntryCallback(), + db.upsertChainCallback()), + start); + + byte[] newKeywordsBytes = newKeywordsBuffer.getPointer(0).getByteArray(0, newKeywordsBufferSize.getValue()); + Native.free(Pointer.nativeValue(newKeywordsBuffer.getPointer(0))); + return new Leb128Reader(newKeywordsBytes).readObject(UpsertResults.class); } } - public static void upsert( - byte[] key, - byte[] label, - Map> additions, - Map> deletions, - Database db) + public static UpsertResults upsert(byte[] key, + byte[] label, + Map> additions, + Map> deletions, + Database db) throws CloudproofException { // Make entryTableNumber equals to 1 by default - upsert(key, label, additions, deletions, 1, db); + return upsert(key, label, additions, deletions, 1, db); } - public static void upsert(IndexRequest request) + public static UpsertResults upsert(IndexRequest request) throws CloudproofException { - upsert(request.key, request.label, request.additions, request.deletions, request.entryTableNumber, - request.database); + return upsert(request.key, request.label, request.additions, request.deletions, request.entryTableNumber, + request.database); } public static SearchResults search(SearchRequest request) @@ -95,13 +104,6 @@ public static SearchResults search(byte[] key, Database db, SearchProgress progressCallback) throws CloudproofException { - // - // Prepare outputs - // - // start with an arbitration buffer allocation size of 131072 (around 4096 - // indexedValues) - byte[] indexedValuesBuffer = new byte[131072]; - IntByReference indexedValuesBufferSize = new IntByReference(indexedValuesBuffer.length); // Findex master keys if (key == null) { @@ -119,37 +121,22 @@ public static SearchResults search(byte[] key, String wordsJson = keywordsToJson(keyWords); - // Indexes creation + insertion/update - long start = System.currentTimeMillis(); - int ffiCode = INSTANCE.h_search( - indexedValuesBuffer, indexedValuesBufferSize, - keyPointer, key.length, - labelPointer, label.length, - wordsJson, - entryTableNumber, - wrappedProgress, - db.fetchEntryCallback(), - db.fetchChainCallback()); - - FindexCallbackException.rethrowOnErrorCode(ffiCode, start, System.currentTimeMillis()); - - if (ffiCode != 0) { - // Retry with correct allocated size - indexedValuesBuffer = new byte[indexedValuesBufferSize.getValue()]; - long startRetry = System.currentTimeMillis(); - unwrap(INSTANCE.h_search( - indexedValuesBuffer, indexedValuesBufferSize, - keyPointer, key.length, - labelPointer, label.length, - wordsJson, - entryTableNumber, - wrappedProgress, - db.fetchEntryCallback(), - db.fetchChainCallback()), startRetry); - } - - byte[] indexedValuesBytes = Arrays.copyOfRange(indexedValuesBuffer, 0, indexedValuesBufferSize.getValue()); + Memory indexedValuesBuffer = new Memory(8); + IntByReference indexedValuesBufferSize = new IntByReference(0); + long start = System.currentTimeMillis(); + unwrap(INSTANCE.h_search(indexedValuesBuffer, indexedValuesBufferSize, + keyPointer, key.length, + labelPointer, label.length, + wordsJson, + entryTableNumber, + wrappedProgress, + db.fetchEntryCallback(), + db.fetchChainCallback()), + start); + + byte[] indexedValuesBytes = indexedValuesBuffer.getPointer(0).getByteArray(0, indexedValuesBufferSize.getValue()); + Native.free(Pointer.nativeValue(indexedValuesBuffer.getPointer(0))); return new Leb128Reader(indexedValuesBytes).readObject(SearchResults.class); } } diff --git a/src/main/java/com/cosmian/jna/findex/FindexCloud.java b/src/main/java/com/cosmian/jna/findex/FindexCloud.java index b0882cc..5eabf97 100644 --- a/src/main/java/com/cosmian/jna/findex/FindexCloud.java +++ b/src/main/java/com/cosmian/jna/findex/FindexCloud.java @@ -1,16 +1,18 @@ package com.cosmian.jna.findex; import java.nio.charset.StandardCharsets; -import java.util.Arrays; import java.util.Map; import java.util.Set; import com.cosmian.jna.findex.ffi.SearchResults; +import com.cosmian.jna.findex.ffi.UpsertResults; import com.cosmian.jna.findex.serde.Leb128Reader; import com.cosmian.jna.findex.structs.IndexedValue; import com.cosmian.jna.findex.structs.Keyword; import com.cosmian.utils.CloudproofException; import com.sun.jna.Memory; +import com.sun.jna.Pointer; +import com.sun.jna.Native; import com.sun.jna.ptr.IntByReference; public final class FindexCloud extends FindexBase { @@ -33,58 +35,63 @@ public static String generateNewToken( upsertEntriesSeedPointer.write(0, upsertEntriesSeed, 0, upsertEntriesSeed.length); insertChainsSeedPointer.write(0, insertChainsSeed, 0, insertChainsSeed.length); - byte[] tokenBuffer = new byte[200]; - IntByReference tokenBufferSize = new IntByReference(tokenBuffer.length); + // Allocate the amount of memory needed to store a pointer. + Memory tokenBuffer = new Memory(8); + IntByReference tokenBufferSize = new IntByReference(0); - unwrap(INSTANCE.h_generate_new_token( - tokenBuffer, - tokenBufferSize, - indexId, - fetchEntriesSeedPointer, fetchEntriesSeed.length, - fetchChainsSeedPointer, fetchChainsSeed.length, - upsertEntriesSeedPointer, upsertEntriesSeed.length, - insertChainsSeedPointer, insertChainsSeed.length)); - - byte[] tokenBytes = Arrays.copyOfRange(tokenBuffer, 0, tokenBufferSize.getValue()); + unwrap(INSTANCE.h_generate_new_token(tokenBuffer, + tokenBufferSize, + indexId, + fetchEntriesSeedPointer, fetchEntriesSeed.length, + fetchChainsSeedPointer, fetchChainsSeed.length, + upsertEntriesSeedPointer, upsertEntriesSeed.length, + insertChainsSeedPointer, insertChainsSeed.length)); + byte[] tokenBytes = tokenBuffer.getPointer(0).getByteArray(0, tokenBufferSize.getValue()); + Native.free(Pointer.nativeValue(tokenBuffer.getPointer(0))); return new String(tokenBytes, StandardCharsets.UTF_8); } } - public static void upsert( - String token, - byte[] label, - Map> additions, - Map> deletions, - String baseUrl) + public static UpsertResults upsert(String token, + byte[] label, + Map> additions, + Map> deletions, + String baseUrl) throws CloudproofException { try ( final Memory labelPointer = new Memory(label.length)) { labelPointer.write(0, label, 0, label.length); - // Indexes creation + insertion/update - unwrap(INSTANCE.h_upsert_cloud( - token, - labelPointer, label.length, - indexedValuesToJson(additions), - indexedValuesToJson(deletions), - baseUrl)); + // Allocate the amount of memory needed to store a pointer. + Memory newKeywordsBuffer = new Memory(8); + IntByReference newKeywordsBufferSize = new IntByReference(0); + + unwrap(INSTANCE.h_upsert_cloud(newKeywordsBuffer, newKeywordsBufferSize, + token, + labelPointer, label.length, + indexedValuesToJson(additions), + indexedValuesToJson(deletions), + baseUrl)); + + byte[] newKeywordsBytes = newKeywordsBuffer.getPointer(0).getByteArray(0, newKeywordsBufferSize.getValue()); + Native.free(Pointer.nativeValue(newKeywordsBuffer.getPointer(0))); + return new Leb128Reader(newKeywordsBytes).readObject(UpsertResults.class); } } - public static void upsert(IndexRequest request) + public static UpsertResults upsert(IndexRequest request) throws CloudproofException { - upsert(request.token, request.label, request.additions, request.deletions, request.baseUrl); + return upsert(request.token, request.label, request.additions, request.deletions, request.baseUrl); } - public static void upsert( - String token, - byte[] label, - Map> additions, - Map> deletions) + public static UpsertResults upsert(String token, + byte[] label, + Map> additions, + Map> deletions) throws CloudproofException { - upsert(token, label, additions, deletions, null); + return upsert(token, label, additions, deletions, null); } public static SearchResults search(SearchRequest request) @@ -104,45 +111,26 @@ public static SearchResults search(String token, Set keyWords, String baseUrl) throws CloudproofException { - // - // Prepare outputs - // - // start with an arbitration buffer allocation size of 131072 (around 4096 - // indexedValues) - byte[] indexedValuesBuffer = new byte[131072]; - IntByReference indexedValuesBufferSize = new IntByReference(indexedValuesBuffer.length); if (token == null) { throw new CloudproofException("Token cannot be null"); } + try (final Memory labelPointer = new Memory(label.length)) { labelPointer.write(0, label, 0, label.length); - String wordsJson = keywordsToJson(keyWords); - // Indexes creation + insertion/update - int ffiCode = INSTANCE.h_search_cloud( - indexedValuesBuffer, indexedValuesBufferSize, - token, - labelPointer, label.length, - wordsJson, - baseUrl); - if (ffiCode != 0) { - // Retry with correct allocated size - indexedValuesBuffer = new byte[indexedValuesBufferSize.getValue()]; - ffiCode = INSTANCE.h_search_cloud( - indexedValuesBuffer, indexedValuesBufferSize, - token, - labelPointer, label.length, - wordsJson, - baseUrl); - if (ffiCode != 0) { - throw new CloudproofException(get_last_error(4095)); - } - } - - byte[] indexedValuesBytes = Arrays.copyOfRange(indexedValuesBuffer, 0, indexedValuesBufferSize.getValue()); + Memory indexedValuesBuffer = new Memory(8); + IntByReference indexedValuesBufferSize = new IntByReference(0); + + unwrap(INSTANCE.h_search_cloud(indexedValuesBuffer, indexedValuesBufferSize, + token, + labelPointer, label.length, + wordsJson, + baseUrl)); + byte[] indexedValuesBytes = indexedValuesBuffer.getPointer(0).getByteArray(0, indexedValuesBufferSize.getValue()); + Native.free(Pointer.nativeValue(indexedValuesBuffer.getPointer(0))); return new Leb128Reader(indexedValuesBytes).readObject(SearchResults.class); } } diff --git a/src/main/java/com/cosmian/jna/findex/ffi/FindexNativeWrapper.java b/src/main/java/com/cosmian/jna/findex/ffi/FindexNativeWrapper.java index c7281c3..123059d 100644 --- a/src/main/java/com/cosmian/jna/findex/ffi/FindexNativeWrapper.java +++ b/src/main/java/com/cosmian/jna/findex/ffi/FindexNativeWrapper.java @@ -78,7 +78,8 @@ boolean apply(Pointer output, throws CloudproofException; } - int h_upsert( + int h_upsert(Pointer newKeywordsBufferPtr, + IntByReference newKeywordsBufferLen, Pointer masterKeyPtr, int masterKeyLen, Pointer labelPtr, @@ -104,7 +105,7 @@ int h_compact(Pointer oldMasterKeyPtr, UpdateLinesCallback updateLines, ListRemovedLocationsCallback listRemovedLocations); - int h_search(byte[] searchresultsPtr, + int h_search(Pointer searchresultsPtr, IntByReference searchresultsLen, Pointer masterKeyPtr, int masterKeyLength, @@ -116,7 +117,7 @@ int h_search(byte[] searchresultsPtr, FetchEntryCallback fetchEntry, FetchChainCallback fetchChain); - int h_search_cloud(byte[] dbUidsPtr, + int h_search_cloud(Pointer dbUidsPtr, IntByReference dbUidsLen, String token, Pointer labelPtr, @@ -124,7 +125,9 @@ int h_search_cloud(byte[] dbUidsPtr, String keywords, String baseUrl); - int h_upsert_cloud(String token, + int h_upsert_cloud(Pointer newKeywordsBufferPtr, + IntByReference newKeywordsBufferLen, + String token, Pointer labelPtr, int labelLen, String additions, @@ -132,7 +135,7 @@ int h_upsert_cloud(String token, String baseUrl); - int h_generate_new_token(byte[] tokenPtr, + int h_generate_new_token(Pointer tokenPtr, IntByReference tokenLen, String indexIdPtr, Pointer fetchEntriesSeedPtr, diff --git a/src/main/java/com/cosmian/jna/findex/ffi/UpsertResults.java b/src/main/java/com/cosmian/jna/findex/ffi/UpsertResults.java new file mode 100644 index 0000000..1a36c65 --- /dev/null +++ b/src/main/java/com/cosmian/jna/findex/ffi/UpsertResults.java @@ -0,0 +1,52 @@ +package com.cosmian.jna.findex.ffi; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.HashSet; +import java.util.Set; +import com.cosmian.jna.findex.serde.Leb128Serializable; +import com.cosmian.jna.findex.structs.Keyword; +import com.cosmian.utils.CloudproofException; +import com.cosmian.utils.Leb128; + +public class UpsertResults implements Leb128Serializable { + + private Set results; + + public UpsertResults() { + this.results = new HashSet<>(); + } + + public Set getResults() { + return results; + } + + public boolean isEmpty() { + return results.isEmpty(); + } + + public int numberOfKeywords() { + return results.size(); + } + + @Override + public void readObject(InputStream is) throws CloudproofException { + try { + + int setLen = (int) Leb128.readU64(is); + for (int i = 0; i < setLen; i++) { + Keyword keyword = new Keyword(Leb128.readByteArray(is)); + results.add(keyword); + } + } catch (IOException e) { + throw new CloudproofException("failed deserializing the upsert results: " + e.getMessage(), e); + } + + } + + @Override + public void writeObject(OutputStream os) throws CloudproofException { + throw new CloudproofException("Upsert results are not serializable"); + } +} diff --git a/src/test/java/com/cosmian/findex/TestFindexCloud.java b/src/test/java/com/cosmian/findex/TestFindexCloud.java index 329c4b4..d8d1d9d 100644 --- a/src/test/java/com/cosmian/findex/TestFindexCloud.java +++ b/src/test/java/com/cosmian/findex/TestFindexCloud.java @@ -9,6 +9,7 @@ import com.cosmian.jna.findex.FindexCloud; import com.cosmian.jna.findex.ffi.SearchResults; +import com.cosmian.jna.findex.ffi.UpsertResults; import com.cosmian.jna.findex.structs.Location; import com.cosmian.utils.RestClient; import com.fasterxml.jackson.annotation.JsonProperty; @@ -39,7 +40,8 @@ public void testFindexCloud() throws Exception { .add(new Location(1337), new String[] { "John", "Doe" }) .add(new Location(42), new String[] { "Jane", "Doe" }); - FindexCloud.upsert(indexRequest); + UpsertResults res = FindexCloud.upsert(indexRequest); + assertEquals(3, res.getResults().size(), "wrong number of new keywords returned"); FindexCloud.SearchRequest searchRequest = new FindexCloud.SearchRequest(token, label) .keywords(new String[] { "Doe" }); diff --git a/src/test/java/com/cosmian/findex/TestRedis.java b/src/test/java/com/cosmian/findex/TestRedis.java index ec42c4d..b6a59b1 100644 --- a/src/test/java/com/cosmian/findex/TestRedis.java +++ b/src/test/java/com/cosmian/findex/TestRedis.java @@ -18,7 +18,9 @@ import com.cosmian.jna.findex.ffi.ProgressResults; import com.cosmian.jna.findex.structs.IndexedValue; import com.cosmian.jna.findex.ffi.SearchResults; +import com.cosmian.jna.findex.ffi.UpsertResults; import com.cosmian.jna.findex.structs.Keyword; +import com.cosmian.jna.findex.structs.Location; import com.cosmian.jna.findex.structs.NextKeyword; import com.cosmian.utils.CloudproofException; @@ -77,12 +79,27 @@ public void testUpsertAndSearchRedis() throws Exception { // Upsert // Map> indexedValuesAndWords = IndexUtils.index(testFindexDataset); - Findex.upsert(new Findex.IndexRequest(key, label, db).add(indexedValuesAndWords)); + UpsertResults res = Findex.upsert(new Findex.IndexRequest(key, label, db).add(indexedValuesAndWords)); + assertEquals(583, res.getResults().size(), "wrong number of new upserted keywords"); System.out .println("After insertion: entry_table size: " + db.getAllKeys(Redis.ENTRY_TABLE_INDEX).size()); System.out .println("After insertion: chain_table size: " + db.getAllKeys(Redis.CHAIN_TABLE_INDEX).size()); + // + // Upsert a new keyword + // + HashMap> newIndexedKeyword = new HashMap<>(); + Set expectdeKeywords = new HashSet<>(); + expectdeKeywords.add(new Keyword("test")); + newIndexedKeyword.put(new IndexedValue(new Location("ici")), expectdeKeywords); + // It is returned the first time it is added. + Set newKeywords = Findex.upsert(new Findex.IndexRequest(key, label, db).add(newIndexedKeyword)).getResults(); + assertEquals(expectdeKeywords, newKeywords, "new keyword is not returned"); + // It is *not* returned the second time it is added. + newKeywords = Findex.upsert(new Findex.IndexRequest(key, label, db).add(newIndexedKeyword)).getResults(); + assert(newKeywords.isEmpty()); + // // Search // diff --git a/src/test/java/com/cosmian/findex/TestSqlite.java b/src/test/java/com/cosmian/findex/TestSqlite.java index eec528a..a94344a 100644 --- a/src/test/java/com/cosmian/findex/TestSqlite.java +++ b/src/test/java/com/cosmian/findex/TestSqlite.java @@ -15,6 +15,7 @@ import com.cosmian.TestUtils; import com.cosmian.jna.findex.Findex; import com.cosmian.jna.findex.ffi.SearchResults; +import com.cosmian.jna.findex.ffi.UpsertResults; import com.cosmian.jna.findex.structs.IndexedValue; import com.cosmian.jna.findex.structs.Keyword; import com.cosmian.jna.findex.structs.Location; @@ -131,12 +132,28 @@ public void testUpsertAndSearchSqlite() throws Exception { // Upsert // Map> indexedValuesAndWords = IndexUtils.index(testFindexDataset); - Findex.upsert(new Findex.IndexRequest(key, label, db).add(indexedValuesAndWords)); + UpsertResults res = Findex.upsert(new Findex.IndexRequest(key, label, db).add(indexedValuesAndWords)); + assertEquals(583, res.getResults().size(), "wrong number of new upserted keywords"); + System.out.println("Upserted " + res.getResults().size() + " new keywords."); System.out .println("After insertion: entry_table size: " + db.getAllKeyValueItems("entry_table").size()); System.out .println("After insertion: chain_table size: " + db.getAllKeyValueItems("chain_table").size()); + // + // Upsert a new keyword + // + HashMap> newIndexedKeyword = new HashMap<>(); + Set expectdeKeywords = new HashSet<>(); + expectdeKeywords.add(new Keyword("test")); + newIndexedKeyword.put(new IndexedValue(new Location(new Long(1))), expectdeKeywords); + // It is returned the first time it is added. + Set newKeywords = Findex.upsert(new Findex.IndexRequest(key, label, db).add(newIndexedKeyword)).getResults(); + assertEquals(expectdeKeywords, newKeywords, "new keyword is not returned"); + // It is *not* returned the second time it is added. + newKeywords = Findex.upsert(new Findex.IndexRequest(key, label, db).add(newIndexedKeyword)).getResults(); + assert(newKeywords.isEmpty()); + // // Search //