From 6c3a985b49b00e755f5df9b65c48088ba7d5c874 Mon Sep 17 00:00:00 2001 From: tchivs Date: Wed, 10 Jun 2026 16:43:36 +0800 Subject: [PATCH 1/4] [common] Support catalog context without Hadoop configuration --- .../apache/paimon/catalog/CatalogContext.java | 34 +++++-- .../paimon/catalog/CatalogFactoryTest.java | 89 +++++++++++++++++++ 2 files changed, 115 insertions(+), 8 deletions(-) diff --git a/paimon-common/src/main/java/org/apache/paimon/catalog/CatalogContext.java b/paimon-common/src/main/java/org/apache/paimon/catalog/CatalogContext.java index 9eb8a5538920..e43513ca85a5 100644 --- a/paimon-common/src/main/java/org/apache/paimon/catalog/CatalogContext.java +++ b/paimon-common/src/main/java/org/apache/paimon/catalog/CatalogContext.java @@ -45,19 +45,22 @@ public class CatalogContext implements Serializable { private static final long serialVersionUID = 1L; private final Options options; - private final SerializableConfiguration hadoopConf; + @Nullable private final SerializableConfiguration hadoopConf; @Nullable private final FileIOLoader preferIOLoader; @Nullable private final FileIOLoader fallbackIOLoader; private CatalogContext( Options options, @Nullable Configuration hadoopConf, + boolean loadHadoopConf, @Nullable FileIOLoader preferIOLoader, @Nullable FileIOLoader fallbackIOLoader) { this.options = checkNotNull(options); this.hadoopConf = - new SerializableConfiguration( - hadoopConf == null ? getHadoopConfiguration(options) : hadoopConf); + hadoopConf == null && !loadHadoopConf + ? null + : new SerializableConfiguration( + hadoopConf == null ? getHadoopConfiguration(options) : hadoopConf); this.preferIOLoader = preferIOLoader; this.fallbackIOLoader = fallbackIOLoader; } @@ -69,20 +72,20 @@ public static CatalogContext create(Path warehouse) { } public static CatalogContext create(Options options) { - return new CatalogContext(options, null, null, null); + return new CatalogContext(options, null, true, null, null); } public static CatalogContext create(Options options, Configuration hadoopConf) { - return new CatalogContext(options, hadoopConf, null, null); + return new CatalogContext(options, hadoopConf, true, null, null); } public static CatalogContext create(Options options, FileIOLoader fallbackIOLoader) { - return new CatalogContext(options, null, null, fallbackIOLoader); + return new CatalogContext(options, null, true, null, fallbackIOLoader); } public static CatalogContext create( Options options, FileIOLoader preferIOLoader, FileIOLoader fallbackIOLoader) { - return new CatalogContext(options, null, preferIOLoader, fallbackIOLoader); + return new CatalogContext(options, null, true, preferIOLoader, fallbackIOLoader); } public static CatalogContext create( @@ -90,7 +93,18 @@ public static CatalogContext create( Configuration hadoopConf, FileIOLoader preferIOLoader, FileIOLoader fallbackIOLoader) { - return new CatalogContext(options, hadoopConf, preferIOLoader, fallbackIOLoader); + return new CatalogContext(options, hadoopConf, true, preferIOLoader, fallbackIOLoader); + } + + /** + * Create a catalog context without initializing Hadoop configuration. + * + *

This should be used only by engines that provide their own {@link FileIOLoader} and do not + * need Paimon's Hadoop-based FileIO path. + */ + public static CatalogContext createWithoutHadoop( + Options options, FileIOLoader preferIOLoader, FileIOLoader fallbackIOLoader) { + return new CatalogContext(options, null, false, preferIOLoader, fallbackIOLoader); } public Options options() { @@ -99,6 +113,10 @@ public Options options() { /** Return hadoop {@link Configuration}. */ public Configuration hadoopConf() { + if (hadoopConf == null) { + throw new IllegalStateException( + "Hadoop configuration is not available for this CatalogContext."); + } return hadoopConf.get(); } diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogFactoryTest.java b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogFactoryTest.java index 7f93b8d61c55..13a52388d471 100644 --- a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogFactoryTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogFactoryTest.java @@ -24,13 +24,19 @@ import org.apache.paimon.table.CatalogTableType; import org.apache.paimon.utils.HadoopUtilsITCase.TestFileIOLoader; import org.apache.paimon.utils.InstantiationUtil; +import org.apache.paimon.utils.TraceableFileIO; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import java.io.File; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.Arrays; import static org.apache.paimon.options.CatalogOptions.TABLE_TYPE; import static org.apache.paimon.options.CatalogOptions.WAREHOUSE; @@ -87,6 +93,40 @@ public void testContextDefaultHadoopConf(@TempDir java.nio.file.Path path) { assertThat(conf.get("dfs.replication")).isEqualTo(replication); } + @Test + public void testCreateCatalogWithoutHadoop(@TempDir java.nio.file.Path path) { + Path root = new Path(path.toUri().toString()); + Options options = new Options(); + options.set(WAREHOUSE, new Path(root, "warehouse").toString()); + + CatalogContext context = + CatalogContext.createWithoutHadoop( + options, new TraceableFileIO.Loader(), null); + + assertThat(CatalogFactory.createCatalog(context).listDatabases()).isEmpty(); + assertThatThrownBy(context::hadoopConf) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Hadoop configuration is not available"); + } + + @Test + public void testCreateCatalogWithoutHadoopClasses(@TempDir java.nio.file.Path path) + throws Exception { + try (URLClassLoader classLoader = new NoHadoopClassLoader(testClasspathWithoutHadoop())) { + Class runner = + Class.forName( + NoHadoopCatalogContextRunner.class.getName(), true, classLoader); + runner.getMethod("run", String.class, ClassLoader.class) + .invoke(null, new Path(path.toUri().toString(), "warehouse").toString(), classLoader); + } catch (InvocationTargetException e) { + Throwable cause = e.getCause(); + if (cause instanceof Exception) { + throw (Exception) cause; + } + throw (Error) cause; + } + } + @Test public void testContextSerializable() throws IOException, ClassNotFoundException { Configuration conf = new Configuration(false); @@ -97,4 +137,53 @@ public void testContextSerializable() throws IOException, ClassNotFoundException context = InstantiationUtil.clone(context); assertThat(context.hadoopConf().get("my_key")).isEqualTo(conf.get("my_key")); } + + private static URL[] testClasspathWithoutHadoop() { + return Arrays.stream(System.getProperty("java.class.path").split(File.pathSeparator)) + .filter(path -> !path.contains("/hadoop-")) + .filter(path -> !path.contains("/htrace-core")) + .filter(path -> !path.contains("/woodstox-core")) + .filter(path -> !path.contains("/stax2-api")) + .map( + path -> { + try { + return new File(path).toURI().toURL(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + .toArray(URL[]::new); + } + + private static class NoHadoopClassLoader extends URLClassLoader { + + private NoHadoopClassLoader(URL[] urls) { + super(urls, ClassLoader.getSystemClassLoader().getParent()); + } + + @Override + protected Class loadClass(String name, boolean resolve) throws ClassNotFoundException { + if (name.startsWith("org.apache.hadoop.")) { + throw new ClassNotFoundException(name); + } + return super.loadClass(name, resolve); + } + } + + public static class NoHadoopCatalogContextRunner { + + public static void run(String warehouse, ClassLoader classLoader) throws Exception { + assertThatThrownBy(() -> classLoader.loadClass("org.apache.hadoop.conf.Configuration")) + .isInstanceOf(ClassNotFoundException.class); + + Options options = new Options(); + options.set("warehouse", warehouse); + CatalogContext context = + CatalogContext.createWithoutHadoop( + options, new TraceableFileIO.Loader(), null); + Catalog catalog = CatalogFactory.createCatalog(context, classLoader); + + assertThat(catalog.listDatabases()).isEmpty(); + } + } } From a3e6b566cc3c6f46b0b58320c83cdbecca55307b Mon Sep 17 00:00:00 2001 From: tchivs Date: Thu, 11 Jun 2026 11:35:27 +0800 Subject: [PATCH 2/4] [core] Fix catalog factory test checkstyle --- .../test/java/org/apache/paimon/catalog/CatalogFactoryTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogFactoryTest.java b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogFactoryTest.java index 13a52388d471..5d8dbc2ec4be 100644 --- a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogFactoryTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogFactoryTest.java @@ -170,6 +170,7 @@ protected Class loadClass(String name, boolean resolve) throws ClassNotFoundE } } + /** Runner loaded by {@link NoHadoopClassLoader} to verify no-Hadoop catalog creation. */ public static class NoHadoopCatalogContextRunner { public static void run(String warehouse, ClassLoader classLoader) throws Exception { From 80be9dede825b008f3ef99aaa4d2eecdbeaeffd6 Mon Sep 17 00:00:00 2001 From: tchivs Date: Thu, 11 Jun 2026 11:40:11 +0800 Subject: [PATCH 3/4] [core] Format catalog factory test --- .../apache/paimon/catalog/CatalogFactoryTest.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogFactoryTest.java b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogFactoryTest.java index 5d8dbc2ec4be..0cbf3f0f24a9 100644 --- a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogFactoryTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogFactoryTest.java @@ -100,8 +100,7 @@ public void testCreateCatalogWithoutHadoop(@TempDir java.nio.file.Path path) { options.set(WAREHOUSE, new Path(root, "warehouse").toString()); CatalogContext context = - CatalogContext.createWithoutHadoop( - options, new TraceableFileIO.Loader(), null); + CatalogContext.createWithoutHadoop(options, new TraceableFileIO.Loader(), null); assertThat(CatalogFactory.createCatalog(context).listDatabases()).isEmpty(); assertThatThrownBy(context::hadoopConf) @@ -114,10 +113,12 @@ public void testCreateCatalogWithoutHadoopClasses(@TempDir java.nio.file.Path pa throws Exception { try (URLClassLoader classLoader = new NoHadoopClassLoader(testClasspathWithoutHadoop())) { Class runner = - Class.forName( - NoHadoopCatalogContextRunner.class.getName(), true, classLoader); + Class.forName(NoHadoopCatalogContextRunner.class.getName(), true, classLoader); runner.getMethod("run", String.class, ClassLoader.class) - .invoke(null, new Path(path.toUri().toString(), "warehouse").toString(), classLoader); + .invoke( + null, + new Path(path.toUri().toString(), "warehouse").toString(), + classLoader); } catch (InvocationTargetException e) { Throwable cause = e.getCause(); if (cause instanceof Exception) { @@ -180,8 +181,7 @@ public static void run(String warehouse, ClassLoader classLoader) throws Excepti Options options = new Options(); options.set("warehouse", warehouse); CatalogContext context = - CatalogContext.createWithoutHadoop( - options, new TraceableFileIO.Loader(), null); + CatalogContext.createWithoutHadoop(options, new TraceableFileIO.Loader(), null); Catalog catalog = CatalogFactory.createCatalog(context, classLoader); assertThat(catalog.listDatabases()).isEmpty(); From 8924d69d826204182e1c74bbc9e5bd27dc4ec41c Mon Sep 17 00:00:00 2001 From: tchivs Date: Mon, 15 Jun 2026 10:30:18 +0800 Subject: [PATCH 4/4] [common] Load Hadoop config defensively --- .../apache/paimon/catalog/CatalogContext.java | 47 ++++++++++--------- .../paimon/catalog/CatalogFactoryTest.java | 21 ++------- 2 files changed, 30 insertions(+), 38 deletions(-) diff --git a/paimon-common/src/main/java/org/apache/paimon/catalog/CatalogContext.java b/paimon-common/src/main/java/org/apache/paimon/catalog/CatalogContext.java index e43513ca85a5..d53fe9a5b54f 100644 --- a/paimon-common/src/main/java/org/apache/paimon/catalog/CatalogContext.java +++ b/paimon-common/src/main/java/org/apache/paimon/catalog/CatalogContext.java @@ -52,15 +52,10 @@ public class CatalogContext implements Serializable { private CatalogContext( Options options, @Nullable Configuration hadoopConf, - boolean loadHadoopConf, @Nullable FileIOLoader preferIOLoader, @Nullable FileIOLoader fallbackIOLoader) { this.options = checkNotNull(options); - this.hadoopConf = - hadoopConf == null && !loadHadoopConf - ? null - : new SerializableConfiguration( - hadoopConf == null ? getHadoopConfiguration(options) : hadoopConf); + this.hadoopConf = loadHadoopConfiguration(options, hadoopConf); this.preferIOLoader = preferIOLoader; this.fallbackIOLoader = fallbackIOLoader; } @@ -72,20 +67,20 @@ public static CatalogContext create(Path warehouse) { } public static CatalogContext create(Options options) { - return new CatalogContext(options, null, true, null, null); + return new CatalogContext(options, null, null, null); } public static CatalogContext create(Options options, Configuration hadoopConf) { - return new CatalogContext(options, hadoopConf, true, null, null); + return new CatalogContext(options, hadoopConf, null, null); } public static CatalogContext create(Options options, FileIOLoader fallbackIOLoader) { - return new CatalogContext(options, null, true, null, fallbackIOLoader); + return new CatalogContext(options, null, null, fallbackIOLoader); } public static CatalogContext create( Options options, FileIOLoader preferIOLoader, FileIOLoader fallbackIOLoader) { - return new CatalogContext(options, null, true, preferIOLoader, fallbackIOLoader); + return new CatalogContext(options, null, preferIOLoader, fallbackIOLoader); } public static CatalogContext create( @@ -93,18 +88,7 @@ public static CatalogContext create( Configuration hadoopConf, FileIOLoader preferIOLoader, FileIOLoader fallbackIOLoader) { - return new CatalogContext(options, hadoopConf, true, preferIOLoader, fallbackIOLoader); - } - - /** - * Create a catalog context without initializing Hadoop configuration. - * - *

This should be used only by engines that provide their own {@link FileIOLoader} and do not - * need Paimon's Hadoop-based FileIO path. - */ - public static CatalogContext createWithoutHadoop( - Options options, FileIOLoader preferIOLoader, FileIOLoader fallbackIOLoader) { - return new CatalogContext(options, null, false, preferIOLoader, fallbackIOLoader); + return new CatalogContext(options, hadoopConf, preferIOLoader, fallbackIOLoader); } public Options options() { @@ -120,6 +104,25 @@ public Configuration hadoopConf() { return hadoopConf.get(); } + @Nullable + private static SerializableConfiguration loadHadoopConfiguration( + Options options, @Nullable Configuration hadoopConf) { + try { + return new SerializableConfiguration( + hadoopConf == null ? getHadoopConfiguration(options) : hadoopConf); + } catch (NoClassDefFoundError e) { + if (isHadoopClassNotFound(e)) { + return null; + } + throw e; + } + } + + private static boolean isHadoopClassNotFound(NoClassDefFoundError e) { + String message = e.getMessage(); + return message != null && message.startsWith("org/apache/hadoop/"); + } + @Nullable public FileIOLoader preferIO() { return preferIOLoader; diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogFactoryTest.java b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogFactoryTest.java index 0cbf3f0f24a9..03d18e9ceb26 100644 --- a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogFactoryTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogFactoryTest.java @@ -33,6 +33,7 @@ import java.io.File; import java.io.IOException; +import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.net.URL; import java.net.URLClassLoader; @@ -93,21 +94,6 @@ public void testContextDefaultHadoopConf(@TempDir java.nio.file.Path path) { assertThat(conf.get("dfs.replication")).isEqualTo(replication); } - @Test - public void testCreateCatalogWithoutHadoop(@TempDir java.nio.file.Path path) { - Path root = new Path(path.toUri().toString()); - Options options = new Options(); - options.set(WAREHOUSE, new Path(root, "warehouse").toString()); - - CatalogContext context = - CatalogContext.createWithoutHadoop(options, new TraceableFileIO.Loader(), null); - - assertThat(CatalogFactory.createCatalog(context).listDatabases()).isEmpty(); - assertThatThrownBy(context::hadoopConf) - .isInstanceOf(IllegalStateException.class) - .hasMessageContaining("Hadoop configuration is not available"); - } - @Test public void testCreateCatalogWithoutHadoopClasses(@TempDir java.nio.file.Path path) throws Exception { @@ -181,10 +167,13 @@ public static void run(String warehouse, ClassLoader classLoader) throws Excepti Options options = new Options(); options.set("warehouse", warehouse); CatalogContext context = - CatalogContext.createWithoutHadoop(options, new TraceableFileIO.Loader(), null); + CatalogContext.create(options, new TraceableFileIO.Loader(), null); Catalog catalog = CatalogFactory.createCatalog(context, classLoader); assertThat(catalog.listDatabases()).isEmpty(); + Field hadoopConfField = CatalogContext.class.getDeclaredField("hadoopConf"); + hadoopConfField.setAccessible(true); + assertThat(hadoopConfField.get(context)).isNull(); } } }