apache · tteofili · Jun 19, 2026
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/MorphFSTLoader.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/MorphFSTLoader.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.morph;
+
+import java.io.BufferedInputStream;
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.InputStreamDataInput;
+import org.apache.lucene.store.MMapDirectory;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.fst.DirectBufferFSTStore;
+import org.apache.lucene.util.fst.FST;
+import org.apache.lucene.util.fst.FSTReader;
+import org.apache.lucene.util.fst.OffHeapFSTStore;
+import org.apache.lucene.util.fst.PositiveIntOutputs;
+
+/**
+ * Loads morphological dictionary {@link FST}s with storage appropriate to the byte source.
+ *
+ * <ul>
+ *   <li>{@link #loadFromPath(Path)} — zero-copy mmap via {@link OffHeapFSTStore}
+ *   <li>{@link #loadFromStream(InputStream)} — one copy into a direct buffer for classpath/JAR
+ *       streams
+ *   <li>{@link #loadFromUrl(URL)} — mmap for {@code file:} URLs, otherwise stream copy
+ *   <li>{@link #loadCompiled(FST.FSTMetadata, FSTReader)} — relocates in-memory compiler output
+ *       off-heap
+ * </ul>
+ */
+public final class MorphFSTLoader {
+
+  private static final PositiveIntOutputs OUTPUTS = PositiveIntOutputs.getSingleton();
+
+  private MorphFSTLoader() {}
+
+  /**
+   * An FST plus an optional resource that must remain open for mmap-backed FSTs. Callers loading
+   * from a {@link Path} must retain this holder for the lifetime of the returned {@link FST}.
+   */
+  public static final class LoadedFST implements Closeable {
+    private final FST<Long> fst;
+    private final Closeable resource;
+
+    LoadedFST(FST<Long> fst, Closeable resource) {
+      this.fst = fst;
+      this.resource = resource;
+    }
+
+    public FST<Long> fst() {
+      return fst;
+    }
+
+    /** Returns the mmap resource, or {@code null} when the FST does not require one. */
+    public Closeable resource() {
+      return resource;
+    }
+
+    @Override
+    public void close() throws IOException {
+      if (resource != null) {
+        resource.close();
+      }
+    }
+  }
+
+  /**
+   * Load an FST from a file using mmap ({@link OffHeapFSTStore}). The returned {@link LoadedFST}
+   * must be kept alive while the FST is in use.
+   */
+  public static LoadedFST loadFromPath(Path fstFile) throws IOException {
+    Directory dir = new MMapDirectory(fstFile.getParent());
+    IndexInput in = dir.openInput(fstFile.getFileName().toString(), IOContext.READONCE);
+    FST.FSTMetadata<Long> metadata = FST.readMetadata(in, OUTPUTS);
+    OffHeapFSTStore store = new OffHeapFSTStore(in, in.getFilePointer(), metadata);
+    FST<Long> fst = FST.fromFSTReader(metadata, store);
+    Closeable resource = () -> IOUtils.close(in, dir);
+    return new LoadedFST(fst, resource);
+  }
+
+  /**
+   * Load an FST from a stream by copying bytes into a direct buffer ({@link
+   * DirectBufferFSTStore}).
+   */
+  public static FST<Long> loadFromStream(InputStream in) throws IOException {
+    DataInput dataIn = new InputStreamDataInput(new BufferedInputStream(in));
+    FST.FSTMetadata<Long> metadata = FST.readMetadata(dataIn, OUTPUTS);
+    return FST.fromFSTReader(
+        metadata, new DirectBufferFSTStore(dataIn, metadata.getNumBytes()));
+  }
+
+  /**
+   * Load an FST from a URL. {@code file:} URLs are mmap'd; other schemes are read as streams.
+   */
+  public static LoadedFST loadFromUrl(URL fstUrl) throws IOException {
+    if ("file".equalsIgnoreCase(fstUrl.getProtocol())) {
+      try {
+        URI uri = fstUrl.toURI();
+        return loadFromPath(Paths.get(uri));
+      } catch (URISyntaxException e) {
+        throw new IOException("Bad file URL: " + fstUrl, e);
+      }
+    }
+    try (InputStream is = new BufferedInputStream(fstUrl.openStream())) {
+      return new LoadedFST(loadFromStream(is), null);
+    }
+  }
+
+  /** Relocate a freshly compiled on-heap FST into a direct buffer. */
+  public static FST<Long> loadCompiled(
+      FST.FSTMetadata<Long> metadata, FSTReader compilerReader) throws IOException {
+    return FST.fromFSTReader(
+        metadata, new DirectBufferFSTStore(compilerReader, metadata.getNumBytes()));
+  }
+}
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/morph/TestMorphFSTLoader.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/morph/TestMorphFSTLoader.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.morph;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.file.Path;
+import org.apache.lucene.store.OutputStreamDataOutput;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.util.IntsRefBuilder;
+import org.apache.lucene.util.fst.FST;
+import org.apache.lucene.util.fst.FSTCompiler;
+import org.apache.lucene.util.fst.IntsRefFSTEnum;
+import org.apache.lucene.util.fst.PositiveIntOutputs;
+
+public class TestMorphFSTLoader extends LuceneTestCase {
+
+  private static final long MAX_SHALLOW_FST_RAM = 10_000L;
+
+  public void testLoadFromPathMmapsWithoutHeapCopy() throws Exception {
+    Path fstFile = writeSampleFstToPath();
+    try (MorphFSTLoader.LoadedFST loaded = MorphFSTLoader.loadFromPath(fstFile)) {
+      assertNotNull(loaded.resource());
+      assertTrue(loaded.fst().ramBytesUsed() < MAX_SHALLOW_FST_RAM);
+      assertEnumEquals(buildSampleFst(), loaded.fst());
+    }
+  }
+
+  public void testLoadFromStreamCopiesOffHeap() throws Exception {
+    FST<Long> expected = buildSampleFst();
+    ByteArrayOutputStream bos = new ByteArrayOutputStream();
+    expected.save(new OutputStreamDataOutput(bos), new OutputStreamDataOutput(bos));
+    FST<Long> loaded =
+        MorphFSTLoader.loadFromStream(new java.io.ByteArrayInputStream(bos.toByteArray()));
+    assertTrue(loaded.ramBytesUsed() < MAX_SHALLOW_FST_RAM);
+    assertEnumEquals(expected, loaded);
+  }
+
+  public void testLoadCompiledCopiesOffHeap() throws Exception {
+    PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton();
+    FSTCompiler<Long> compiler =
+        new FSTCompiler.Builder<>(FST.INPUT_TYPE.BYTE1, outputs).build();
+    IntsRefBuilder scratch = new IntsRefBuilder();
+    scratch.append(42);
+    compiler.add(scratch.get(), 7L);
+    FST.FSTMetadata<Long> metadata = compiler.compile();
+    FST<Long> loaded = MorphFSTLoader.loadCompiled(metadata, compiler.getFSTReader());
+    assertTrue(loaded.ramBytesUsed() < MAX_SHALLOW_FST_RAM);
+  }
+
+  private static Path writeSampleFstToPath() throws IOException {
+    FST<Long> fst = buildSampleFst();
+    Path path = createTempFile("morph-fst", ".dat");
+    fst.save(path);
+    return path;
+  }
+
+  private static FST<Long> buildSampleFst() throws IOException {
+    PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton();
+    FSTCompiler<Long> compiler =
+        new FSTCompiler.Builder<>(FST.INPUT_TYPE.BYTE1, outputs).build();
+    IntsRefBuilder scratch = new IntsRefBuilder();
+    for (int i = 0; i < 64; i++) {
+      scratch.clear();
+      scratch.append(i);
+      compiler.add(scratch.get(), (long) i);
+    }
+    return FST.fromFSTReader(compiler.compile(), compiler.getFSTReader());
+  }
+
+  private static void assertEnumEquals(FST<Long> expected, FST<Long> actual) throws IOException {
+    IntsRefFSTEnum<Long> expectedEnum = new IntsRefFSTEnum<>(expected);
+    IntsRefFSTEnum<Long> actualEnum = new IntsRefFSTEnum<>(actual);
+    IntsRefFSTEnum.InputOutput<Long> e;
+    while ((e = expectedEnum.next()) != null) {
+      IntsRefFSTEnum.InputOutput<Long> a = actualEnum.next();
+      assertNotNull(a);
+      assertEquals(e.input, a.input);
+      assertEquals(e.output, a.output);
+    }
+    assertNull(actualEnum.next());
+  }
+}
diff --git a/...ne/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoDictionary.java b/...ne/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoDictionary.java
@@ -16,21 +16,16 @@
  */
 package org.apache.lucene.analysis.ja.dict;
 
-import static org.apache.lucene.util.fst.FST.readMetadata;
-
 import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import org.apache.lucene.analysis.morph.BinaryDictionary;
-import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.InputStreamDataInput;
+import org.apache.lucene.analysis.morph.MorphFSTLoader;
 import org.apache.lucene.util.IOSupplier;
 import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.fst.FST;
-import org.apache.lucene.util.fst.PositiveIntOutputs;
 
 /**
  * Binary dictionary implementation for a known-word dictionary model: Words are encoded into an FST
@@ -42,6 +37,9 @@ public final class TokenInfoDictionary extends BinaryDictionary<TokenInfoMorphDa
 
   private final TokenInfoFST fst;
   private final TokenInfoMorphData morphAtts;
+  /** Keeps mmap resources alive for {@link MorphFSTLoader#loadFromPath}; otherwise null. */
+  @SuppressWarnings("unused")
+  private final MorphFSTLoader.LoadedFST fstHolder;
 
   /**
    * Create a {@link TokenInfoDictionary} from an external resource path.
@@ -54,11 +52,17 @@ public final class TokenInfoDictionary extends BinaryDictionary<TokenInfoMorphDa
    */
   public TokenInfoDictionary(Path targetMapFile, Path posDictFile, Path dictFile, Path fstFile)
       throws IOException {
-    this(
+    super(
         () -> Files.newInputStream(targetMapFile),
-        () -> Files.newInputStream(posDictFile),
         () -> Files.newInputStream(dictFile),
-        () -> Files.newInputStream(fstFile));
+        DictionaryConstants.TARGETMAP_HEADER,
+        DictionaryConstants.DICT_HEADER,
+        DictionaryConstants.VERSION);
+    this.morphAtts =
+        new TokenInfoMorphData(buffer, () -> Files.newInputStream(posDictFile));
+    MorphFSTLoader.LoadedFST loaded = MorphFSTLoader.loadFromPath(fstFile);
+    this.fstHolder = loaded;
+    this.fst = new TokenInfoFST(loaded.fst(), true);
   }
 
   /**
@@ -73,11 +77,16 @@ public TokenInfoDictionary(Path targetMapFile, Path posDictFile, Path dictFile,
    */
   public TokenInfoDictionary(URL targetMapUrl, URL posDictUrl, URL dictUrl, URL fstUrl)
       throws IOException {
-    this(
+    super(
         () -> targetMapUrl.openStream(),
-        () -> posDictUrl.openStream(),
         () -> dictUrl.openStream(),
-        () -> fstUrl.openStream());
+        DictionaryConstants.TARGETMAP_HEADER,
+        DictionaryConstants.DICT_HEADER,
+        DictionaryConstants.VERSION);
+    this.morphAtts = new TokenInfoMorphData(buffer, () -> posDictUrl.openStream());
+    MorphFSTLoader.LoadedFST loaded = MorphFSTLoader.loadFromUrl(fstUrl);
+    this.fstHolder = loaded.resource() != null ? loaded : null;
+    this.fst = new TokenInfoFST(loaded.fst(), true);
   }
 
   private TokenInfoDictionary() throws IOException {
@@ -101,14 +110,11 @@ private TokenInfoDictionary(
         DictionaryConstants.DICT_HEADER,
         DictionaryConstants.VERSION);
     this.morphAtts = new TokenInfoMorphData(buffer, posResource);
-
-    FST<Long> fst;
     try (InputStream is = new BufferedInputStream(fstResource.get())) {
-      DataInput in = new InputStreamDataInput(is);
-      fst = new FST<>(readMetadata(in, PositiveIntOutputs.getSingleton()), in);
+      // TODO: some way to configure?
+      this.fst = new TokenInfoFST(MorphFSTLoader.loadFromStream(is), true);
     }
-    // TODO: some way to configure?
-    this.fst = new TokenInfoFST(fst, true);
+    this.fstHolder = null;
   }
 
   static InputStream getClassResource(String suffix) throws IOException {

diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java
@@ -23,6 +23,7 @@
 import java.util.List;
 import java.util.regex.Pattern;
 import org.apache.lucene.analysis.morph.Dictionary;
+import org.apache.lucene.analysis.morph.MorphFSTLoader;
 import org.apache.lucene.analysis.util.CSVUtil;
 import org.apache.lucene.util.IntsRefBuilder;
 import org.apache.lucene.util.fst.FST;
@@ -141,9 +142,10 @@ private UserDictionary(List<String[]> featureEntries) throws IOException {
       segmentations.add(wordIdAndLength);
       ord++;
     }
+    FST.FSTMetadata<Long> metadata = fstCompiler.compile();
     this.fst =
         new TokenInfoFST(
-            FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader()), false);
+            MorphFSTLoader.loadCompiled(metadata, fstCompiler.getFSTReader()), false);
     this.morphAtts = new UserMorphData(data.toArray(String[]::new));
     this.segmentations = segmentations.toArray(int[][]::new);
   }

diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizerFactory.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizerFactory.java
@@ -59,6 +59,10 @@
  *       {@link DecompoundMode}
  *   <li>outputUnknownUnigrams: If true outputs unigrams for unknown words.
  *   <li>discardPunctuation: true if punctuation tokens should be dropped from the output.
+ *   <li>cacheHangulRootArcs: if true caches Hangul syllable root arcs in user dictionary FST for
+ *       faster tokenization at the cost of additional heap. Default is true. The system dictionary
+ *       honors system property {@link
+ *       org.apache.lucene.analysis.ko.dict.TokenInfoDictionary#CACHE_HANGUL_ROOT_ARCS_PROPERTY}.
  * </ul>
  *
  * @lucene.experimental
@@ -75,6 +79,7 @@ public class KoreanTokenizerFactory extends TokenizerFactory implements Resource
   private static final String DECOMPOUND_MODE = "decompoundMode";
   private static final String OUTPUT_UNKNOWN_UNIGRAMS = "outputUnknownUnigrams";
   private static final String DISCARD_PUNCTUATION = "discardPunctuation";
+  private static final String CACHE_HANGUL_ROOT_ARCS = "cacheHangulRootArcs";
 
   private final String userDictionaryPath;
   private final String userDictionaryEncoding;
@@ -83,6 +88,7 @@ public class KoreanTokenizerFactory extends TokenizerFactory implements Resource
   private final KoreanTokenizer.DecompoundMode mode;
   private final boolean outputUnknownUnigrams;
   private final boolean discardPunctuation;
+  private final boolean cacheHangulRootArcs;
 
   /** Creates a new KoreanTokenizerFactory */
   public KoreanTokenizerFactory(Map<String, String> args) {
@@ -95,6 +101,7 @@ public KoreanTokenizerFactory(Map<String, String> args) {
                 .toUpperCase(Locale.ROOT));
     outputUnknownUnigrams = getBoolean(args, OUTPUT_UNKNOWN_UNIGRAMS, false);
     discardPunctuation = getBoolean(args, DISCARD_PUNCTUATION, true);
+    cacheHangulRootArcs = getBoolean(args, CACHE_HANGUL_ROOT_ARCS, true);
 
     if (!args.isEmpty()) {
       throw new IllegalArgumentException("Unknown parameters: " + args);
@@ -120,7 +127,7 @@ public void inform(ResourceLoader loader) throws IOException {
                 .onMalformedInput(CodingErrorAction.REPORT)
                 .onUnmappableCharacter(CodingErrorAction.REPORT);
         Reader reader = new InputStreamReader(stream, decoder);
-        userDictionary = UserDictionary.open(reader);
+        userDictionary = UserDictionary.open(reader, cacheHangulRootArcs);
       }
     } else {
       userDictionary = null;