Skip to content

WIP: Demonstrate that pre-1504 keys are restored for large video files #1124

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
bertm wants to merge 5 commits into from
Closed

WIP: Demonstrate that pre-1504 keys are restored for large video files #1124

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
9fe9511
♻️ Add getter for insert context
Bombe Feb 13, 2026
ddcddf5
🐛 Exhaust input streams completely when hashing
Bombe Feb 13, 2026
d6232cb
Revert "Remove duplicated MIME type"
bertm Feb 14, 2026
adbb147
Revert "Add a bunch of more new MIME types"
bertm Feb 14, 2026
73ed2b7
WIP: Demonstrate that pre-1504 keys are restored for large video files
bertm Feb 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 0 additions & 20 deletions src/freenet/client/DefaultMIMETypes.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -755,26 +755,6 @@ public synchronized static short byName(String s) {
addMIMEType((short)623, "image/avif", "avif");
addMIMEType((short)624, "image/heic", "heic");
addMIMEType((short)625, "image/heif", "heif");
addMIMEType((short)626, "image/bpg", "bpg");
addMIMEType((short)627, "image/jxl", "jxl");
addMIMEType((short)628, "image/emf", "emf");
addMIMEType((short)629, "image/vnd-ms.dds", "dds");
addMIMEType((short)630, "image/x-exr", "exr");
addMIMEType((short)631, "video/webm", "webm");
addMIMEType((short)632, "video/3gpp", "3gp");
addMIMEType((short)633, "video/3gpp2", "3g2");
addMIMEType((short)634, "audio/x-ape", "ape");
addMIMEType((short)635, "application/gzip", "gz");
addMIMEType((short)636, "application/x-xz", "xz");
addMIMEType((short)637, "application/zstd", "zst");
addMIMEType((short)638, "application/vnd.ms-cab-compressed", "cab");
addMIMEType((short)639, "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "docx");
addMIMEType((short)640, "application/vnd.openxmlformats-officedocument.presentationml.presentation", "pptx");
addMIMEType((short)641, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "xlsx");
addMIMEType((short)642, "application/msix", "msix");
addMIMEType((short)643, "application/vnd.android.package-archive", "apk");
addMIMEType((short)644, "application/vnd.rn-realmedia", "rm");
addMIMEType((short)645, "application/vnd.rn-realmedia-vbr", "rmvb");
}

/** Guess a MIME type from a filename.
Expand Down
8 changes: 6 additions & 2 deletions src/freenet/client/async/InsertCompressor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ public InsertCompressor(SingleFileInserter inserter, RandomAccessBucket origData
this.minSize = minSize;
this.bucketFactory = bf;
this.persistent = persistent;
this.compressorDescriptor = inserter.ctx.compressorDescriptor;
this.compressorDescriptor = inserter.getInsertContext().compressorDescriptor;
this.generateHashes = generateHashes;
this.pre1254 = pre1254;
this.config = config;
Expand Down Expand Up @@ -148,7 +148,11 @@ public boolean run(ClientContext context) {
amountOfDataToCheckCompressionRatio, minimumCompressionPercentage);
} catch (CompressionOutputSizeException | CompressionRatioException e) {
if(hasher != null) {
is.skip(Long.MAX_VALUE);
int eof;
do {
is.skip(Long.MAX_VALUE);
eof = is.read();
} while (eof != -1);
hashes = hasher.getResults();
first = false;
}
Expand Down
6 changes: 5 additions & 1 deletion src/freenet/client/async/SingleFileInserter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ public void shouldUpdate() {

final BaseClientPutter parent;
InsertBlock block;
final InsertContext ctx;
private final InsertContext ctx;
final boolean metadata;
final PutCompletionCallback cb;
final ARCHIVE_TYPE archiveType;
Expand Down Expand Up @@ -1008,6 +1008,10 @@ public BaseClientPutter getParent() {
return parent;
}

InsertContext getInsertContext() {
return ctx;
}

@Override
public void cancel(ClientContext context) {
if(logMINOR) Logger.minor(this, "Cancel "+this);
Expand Down
231 changes: 231 additions & 0 deletions test/freenet/client/InsertCHKTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,231 @@
package freenet.client;

import static freenet.node.RequestStarter.MAXIMUM_PRIORITY_CLASS;
import static org.junit.Assert.assertEquals;
import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Mockito.RETURNS_MOCKS;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;

import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.util.Random;
import java.util.function.Consumer;

import freenet.client.async.ClientContext;
import freenet.config.Config;
import freenet.config.Dimension;
import freenet.config.IntOption;
import freenet.config.LongOption;
import freenet.config.SubConfig;
import freenet.crypt.RandomSource;
import freenet.crypt.Yarrow;
import freenet.keys.FreenetURI;
import freenet.node.NodeClientCore;
import freenet.support.Executor;
import freenet.support.FileLoggerHook;
import freenet.support.Logger;
import freenet.support.LoggerHook;
import freenet.support.MemoryLimitedJobRunner;
import freenet.support.PooledExecutor;
import freenet.support.TrivialTicker;
import freenet.support.WaitableExecutor;
import freenet.support.api.RandomAccessBucket;
import freenet.support.compress.RealCompressor;
import freenet.support.io.FileBucket;
import freenet.support.io.FilenameGenerator;
import freenet.support.io.TempBucketFactory;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;

public class InsertCHKTest {

private static final int MAX_RAM_BUCKET_SIZE = 1024 * 1024;
private static final int MAX_RAM_BUCKET_TOTAL_SIZE = 10 * MAX_RAM_BUCKET_SIZE;
private static final boolean REALTIME = true;
private static final Config CONFIG = config(
subConfig("node",
longOption("amountOfDataToCheckCompressionRatio", "8MiB"),
intOption("minimumCompressionPercentage", "10")
));

@Rule
public final TemporaryFolder temporaryFolder = new TemporaryFolder();

private final WaitableExecutor executor = new WaitableExecutor(new PooledExecutor());
private final TrivialTicker ticker = new TrivialTicker(executor);
private final RealCompressor realCompressor = new RealCompressor();

private static FileLoggerHook loggerHook;

private NodeClientCore core;

@Test
public void insertVideoShouldHaveCorrectUri() throws InsertException, IOException {
System.out.println("Downloading video");
File video = temporaryFolder.newFile("LJXYCF-vlc-ffmpeg-librairies-kyber-2026.av1.webm");
URL downloadUrl = new URL("https://mirrors.dotsrc.org/fosdem/2026/k4601/LJXYCF-vlc-ffmpeg-librairies-kyber-2026.av1.webm");
Files.copy(downloadUrl.openStream(), video.toPath(), StandardCopyOption.REPLACE_EXISTING);

System.out.println("Computing CHK for video");
FreenetURI uri = getInsertCHK(video, "video/webm");

assertEquals("CHK@fpmrlTBPfcMOyUK2mwh40GAyvPXy8PwLln07PEW4j8g,9X0pllMCjRj3UMDpQVud-11rxrk0CA-ISHJTKkMq0yc,AAMC--8/LJXYCF-vlc-ffmpeg-librairies-kyber-2026.av1.webm",
uri.toString());
}

@Before
public void setUp() throws Exception {
core = mock(NodeClientCore.class);
doReturn(executor).when(core).getExecutor();
doReturn(ticker).when(core).getTicker();

RandomSource random = new Yarrow(temporaryFolder.newFile("prng.seed"));
doReturn(random).when(core).getRandom();

TempBucketFactory tempBucketFactory = createTempBucketFactory(executor, random, temporaryFolder.newFolder("temp"));
doReturn(tempBucketFactory).when(core).getTempBucketFactory();

ClientContext clientContext = createClientContext(core, realCompressor);
doReturn(clientContext).when(core).getClientContext();

realCompressor.setClientContext(core.getClientContext());
}

@After
public void tearDown() {
executor.waitForIdle();
ticker.shutdown();
realCompressor.shutdown();
}

@BeforeClass
public static void startLogging() throws LoggerHook.InvalidThresholdException {
loggerHook = Logger.setupStdoutLogging(Logger.LogLevel.ERROR, null);
}

@AfterClass
public static void stopLogging() {
Logger.globalRemoveHook(loggerHook);
Logger.destroyChainIfEmpty();
loggerHook.close();
}

private static TempBucketFactory createTempBucketFactory(Executor executor, Random random, File root) throws IOException {
FilenameGenerator tempFilenameGenerator = new FilenameGenerator(random, false, root, "bucket-");
return new TempBucketFactory(
executor,
tempFilenameGenerator,
MAX_RAM_BUCKET_SIZE,
MAX_RAM_BUCKET_TOTAL_SIZE,
random,
false,
0,
null
);
}

private static ClientContext createClientContext(NodeClientCore core, RealCompressor compressor) {
MemoryLimitedJobRunner memoryLimitedJobRunner = new MemoryLimitedJobRunner(Long.MAX_VALUE, 1, core.getExecutor(), 1);
ClientContext clientContext = spy(new ClientContext(
0,
core.getClientLayerPersister(),
core.getExecutor(),
null,
core.getPersistentTempBucketFactory(),
core.getTempBucketFactory(),
null,
core.getHealingQueue(),
core.getUskManager(),
core.getRandom(),
core.getRandom(),
core.getTicker(),
memoryLimitedJobRunner,
core.getTempFilenameGenerator(),
core.getPersistentFilenameGenerator(),
null,
null,
null,
null,
compressor,
core.getStoreChecker(),
null,
null,
core.getLinkFilterExceptionProvider(),
null,
null,
CONFIG
));
doAnswer(RETURNS_MOCKS).when(clientContext).getChkInsertScheduler(anyBoolean());
return clientContext;
}

private HighLevelSimpleClient createHighLevelSimpleClient() {
return new HighLevelSimpleClientImpl(
core,
core.getTempBucketFactory(),
core.getRandom(),
MAXIMUM_PRIORITY_CLASS,
false,
REALTIME
);
}

private FreenetURI getInsertCHK(File file, String mimeType) throws InsertException {
HighLevelSimpleClient hlsl = createHighLevelSimpleClient();
RandomAccessBucket bucket = new FileBucket(file, true, false, false, false);
try {
InsertContext context = hlsl.getInsertContext(true);
context.getCHKOnly = true;
ClientMetadata clientMetadata = new ClientMetadata(mimeType);
InsertBlock insertBlock = new InsertBlock(bucket, clientMetadata, FreenetURI.EMPTY_CHK_URI);
return hlsl.insert(insertBlock, file.getName(), MAXIMUM_PRIORITY_CLASS, context);
} finally {
bucket.free();
}
}

@SafeVarargs
private static Config config(Consumer<Config>... options) {
Config self = new Config();
for (Consumer<Config> option : options) {
option.accept(self);
}
return self;
}

@SafeVarargs
private static Consumer<Config> subConfig(String name, Consumer<SubConfig>... options) {
return config -> {
SubConfig self = config.createSubConfig(name);
for (Consumer<SubConfig> option : options) {
option.accept(self);
}
};
}

private static Consumer<SubConfig> intOption(String name, String value) {
return config -> {
IntOption option = new IntOption(config, name, value, -1, false, false, null, null, null, Dimension.NOT);
config.register(option);
};
}

private static Consumer<SubConfig> longOption(String name, String value) {
return config -> {
LongOption option = new LongOption(config, name, value, -1, false, false, null, null, null, false);
config.register(option);
};
}

}
73 changes: 73 additions & 0 deletions test/freenet/client/async/InsertCompressorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
package freenet.client.async;

import freenet.config.Config;
import freenet.config.SubConfig;
import freenet.crypt.HashResult;
import freenet.crypt.HashType;
import freenet.support.api.IntCallback;
import freenet.support.api.LongCallback;
import freenet.support.io.ArrayBucket;
import freenet.support.io.ArrayBucketFactory;
import java.util.Arrays;
import java.util.List;
import org.junit.Test;
import org.mockito.ArgumentCaptor;

import static java.util.Arrays.asList;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyString;
import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;

public class InsertCompressorTest {

@Test
public void onCompressedOnSingleFileInserterIsCalledWhenCompressionIsDone() throws Exception {
// set up the compressor
SingleFileInserter inserter = mock(SingleFileInserter.class, RETURNS_DEEP_STUBS);
InsertCompressor insertCompressor = createInsertCompressor(inserter);

// set up the client context
ClientContext clientContext = mock(ClientContext.class, RETURNS_DEEP_STUBS);
insertCompressor.tryCompress(clientContext);

// extract hashes from compression results
ArgumentCaptor<Runnable> runnableCaptor = ArgumentCaptor.forClass(Runnable.class);
verify(clientContext.getMainExecutor()).execute(runnableCaptor.capture(), anyString());
runnableCaptor.getValue().run();
ArgumentCaptor<CompressionOutput> compressionOutputCaptor = ArgumentCaptor.forClass(CompressionOutput.class);
verify(inserter).onCompressed(compressionOutputCaptor.capture(), any());
List<HashResult> hashes = asList(compressionOutputCaptor.getValue().hashes);

// verify hashes
assertThat(getHashForType(hashes, HashType.MD5), equalTo("74c0684aa38f8aab87ab57a2b72afcb0"));
assertThat(getHashForType(hashes, HashType.SHA1), equalTo("74ca11d7683a360d708070a3c949801fd00863d8"));
assertThat(getHashForType(hashes, HashType.SHA256), equalTo("3064d7f5bb3fcd7c4885d71deeba2bca3e494176ca16fae902d8a5f2353f1023"));
assertThat(getHashForType(hashes, HashType.SHA384), equalTo("ae57d8c8285df87e0735960a42421ba8cdf66ed6caba967be2c8d8954e951bac6a66edf2dd0429d25f6c075eb0885504"));
assertThat(getHashForType(hashes, HashType.SHA512), equalTo("75b64fa03dd9ab8a83654cb0ed4da7c4cc090384a2cec66029d9266a5fb5b83880d6299211dc53c7f6b8228d3d7666945592f1745c8333c7a13c4f6ce715bf89"));
}

private InsertCompressor createInsertCompressor(SingleFileInserter singleFileInserter) {
Config config = new Config();
SubConfig nodeConfig = config.createSubConfig("node");
nodeConfig.register("amountOfDataToCheckCompressionRatio", 32768L, 0, false, false, "", "", mock(LongCallback.class), true);
nodeConfig.register("minimumCompressionPercentage", 101, 0, false, false, "", "", mock(IntCallback.class), false);
ArrayBucket data = new ArrayBucket(create100KBytesOfZeroFollowedByAOne());
return new InsertCompressor(singleFileInserter, data, 0, new ArrayBucketFactory(), false, 31, false, config);
}

private String getHashForType(List<HashResult> hashes, HashType hashType) {
return hashes.stream().filter(hashResult -> hashResult.type == hashType).findFirst().get().hashAsHex();
}

private byte[] create100KBytesOfZeroFollowedByAOne() {
byte[] data = new byte[100001];
Arrays.fill(data, (byte) 0);
data[100000] = 1;
return data;
}

}