Merge remote-tracking branch 'origin/main' into fix-win-tests

Author: m-dzianishchyts

RELEASE-NOTES.md

@@ -1,3 +1,8 @@
+## 1.0-a7
+
+* #67 JJAVA_STARTUP_SCRIPT can't start up spark
+* #102 Eval startup snippets explicitly
+
 ## 1.0-a6
 
 * #68 CompilationException doesn't provide any information about the underlying problem

jjava-distro/src/main/java/org/dflib/jjava/distro/JJava.java

@@ -55,7 +55,6 @@ public static void main(String[] args) throws Exception {
                 .version((String) pomProps.getOrDefault("version", ""))
 
                 .extensionsEnabled(Env.extensionsEnabled())
-                .startupSnippets(Env.startupSnippets())
                 .compilerOpts(Env.compilerOpts())
                 .timeout(Env.timeout())
 
@@ -79,6 +78,9 @@ public static void main(String[] args) throws Exception {
         // process custom locations: expand JShell classpath, install extensions from those places (if enabled)
         kernel.addToClasspath(Env.extraClasspath());
 
+        // run user defined startup snippets explicitly after the default startup
+        Env.startupSnippets().forEach(kernel::evalRaw);
+
         // connect to Jupyter
         kernel.becomeHandlerForConnection(connection);
         connection.connect();

jjava-distro/src/main/java/org/dflib/jjava/distro/Opts.java

@@ -5,7 +5,7 @@
 
 class Opts {
 
-    // TODO: see 7205d03766c8 and c91ade8fec0cdc. A similar split method in MagicParser got altered in a few subtle
+    // TODO: see 7205d03766c8 and c91ade8fec0cdc. A similar split method in MagicsResolver got altered in a few subtle
     //  ways... Do those changes alloy here, and should we have a single splitter?
     //  7205d03766c8
     //  - if (current.length() > 0 && inQuotes) {

jjava-distro/src/test/java/org/dflib/jjava/distro/KernelStartupIT.java

@@ -3,9 +3,11 @@
 import org.junit.jupiter.api.Test;
 import org.testcontainers.containers.Container;
 
+import java.util.Map;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.matchesPattern;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 class KernelStartupIT extends ContainerizedKernelCase {
@@ -19,4 +21,17 @@ void startUp() throws Exception {
         assertThat(snippetResult.getStdout(), not(containsString("|")));
         assertThat(snippetResult.getStdout(), containsString("1001.0"));
     }
+
+    @Test
+    void startUp_scriptRequiresClasspath() throws Exception {
+        Map<String, String> env = Map.of(
+                Env.JJAVA_CLASSPATH, TEST_CLASSPATH,
+                Env.JJAVA_STARTUP_SCRIPT, "var obj = new org.dflib.jjava.Dummy()"
+        );
+        String snippet = "\"hash = \" + obj.hashCode()";
+        Container.ExecResult snippetResult = executeInKernel(snippet, env);
+
+        assertThat(snippetResult.getStdout(), not(containsString("|")));
+        assertThat(snippetResult.getStdout(), matchesPattern("(?s).*hash = -?\\d+.*"));
+    }
 }

jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/BaseKernel.java

@@ -11,7 +11,7 @@
 import org.dflib.jjava.jupyter.kernel.display.common.Url;
 import org.dflib.jjava.jupyter.kernel.history.HistoryEntry;
 import org.dflib.jjava.jupyter.kernel.history.HistoryManager;
-import org.dflib.jjava.jupyter.kernel.magic.MagicParser;
+import org.dflib.jjava.jupyter.kernel.magic.MagicsResolver;
 import org.dflib.jjava.jupyter.kernel.magic.MagicsRegistry;
 import org.dflib.jjava.jupyter.kernel.util.PathsHandler;
 import org.dflib.jjava.jupyter.kernel.util.StringStyler;
@@ -85,7 +85,7 @@ public abstract class BaseKernel {
     protected final JupyterIO io;
     protected final CommManager commManager;
     protected final Renderer renderer;
-    protected final MagicParser magicParser;
+    protected final MagicsResolver magicsResolver;
     protected final MagicsRegistry magicsRegistry;
     protected final Map<String, Extension> extensions;
     protected final boolean extensionsEnabled;
@@ -111,7 +111,7 @@ protected BaseKernel(
             JupyterIO io,
             CommManager commManager,
             Renderer renderer,
-            MagicParser magicParser,
+            MagicsResolver magicsResolver,
             MagicsRegistry magicsRegistry,
             boolean extensionsEnabled,
             StringStyler errorStyler) {
@@ -127,7 +127,7 @@ protected BaseKernel(
         this.io = Objects.requireNonNull(io);
         this.commManager = Objects.requireNonNull(commManager);
         this.renderer = Objects.requireNonNull(renderer);
-        this.magicParser = magicParser;
+        this.magicsResolver = magicsResolver;
         this.magicsRegistry = magicsRegistry;
         this.extensionsEnabled = extensionsEnabled;
         this.extensions = new ConcurrentHashMap<>();
@@ -156,8 +156,8 @@ public MagicsRegistry getMagicsRegistry() {
         return magicsRegistry;
     }
 
-    public MagicParser getMagicParser() {
-        return magicParser;
+    public MagicsResolver getMagicsResolver() {
+        return magicsResolver;
     }
 
     public JupyterIO getIO() {

jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/BaseKernelBuilder.java

@@ -5,7 +5,7 @@
 import org.dflib.jjava.jupyter.kernel.history.HistoryManager;
 import org.dflib.jjava.jupyter.kernel.magic.CellMagic;
 import org.dflib.jjava.jupyter.kernel.magic.LineMagic;
-import org.dflib.jjava.jupyter.kernel.magic.MagicParser;
+import org.dflib.jjava.jupyter.kernel.magic.MagicsResolver;
 import org.dflib.jjava.jupyter.kernel.magic.MagicTranspiler;
 import org.dflib.jjava.jupyter.kernel.magic.MagicsRegistry;
 import org.dflib.jjava.jupyter.kernel.util.StringStyler;
@@ -24,7 +24,7 @@ public abstract class BaseKernelBuilder<
     protected String name;
     protected String version;
     protected Charset jupyterIOEncoding;
-    protected MagicParser magicParser;
+    protected MagicsResolver magicsResolver;
     protected MagicTranspiler magicTranspiler;
     protected HistoryManager historyManager;
     protected Boolean extensionsEnabled;
@@ -64,8 +64,8 @@ public B cellMagic(String name, CellMagic<?, ?> magic) {
         return (B) this;
     }
 
-    public B magicParser(MagicParser magicParser) {
-        this.magicParser = magicParser;
+    public B magicsResolver(MagicsResolver magicsResolver) {
+        this.magicsResolver = magicsResolver;
         return (B) this;
     }
 

jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/magic/CellMagicArgs.java

@@ -8,7 +8,7 @@
 import java.util.stream.Collectors;
 
 /**
- * A converter (aka "transpiler") of the generic magic syntax into kernel-specific syntax (such as Java).
+ * A converter (aka "transpiler") of generic syntax of a single magic into kernel-specific syntax (such as Java).
  */
 public class MagicTranspiler {
 
@@ -21,29 +21,29 @@ public class MagicTranspiler {
 
     public String transpileCell(ParsedCellMagic magic) {
         return String.format(CELL_CALL_TEMPLATE,
-                argWithEscapingToJava(magic.magicCall.name),
-                magic.magicCall.args.stream()
+                argWithEscapingToJava(magic.name),
+                magic.args.stream()
                         .map(this::argWithEscapingToJava)
                         .collect(Collectors.joining(",")),
-                argWithEscapingToJava(magic.magicCall.body)
+                argWithEscapingToJava(magic.cellBodyAfterMagic)
         );
     }
 
     public String transpileLine(ParsedLineMagic magic) {
         boolean inString = false;
-        Matcher m = UNESCAPED_QUOTE.matcher(magic.linePrefix);
+        Matcher m = UNESCAPED_QUOTE.matcher(magic.magicLinePrefix);
         while (m.find()) {
             inString = !inString;
         }
 
         // If in a string literal, don't apply the magic, just use the original
         if (inString) {
-            return magic.raw;
+            return magic.unparsedMagic;
         }
 
         return String.format(LINE_CALL_TEMPLATE,
-                argWithEscapingToJava(magic.magicCall.name),
-                magic.magicCall.args.stream()
+                argWithEscapingToJava(magic.name),
+                magic.args.stream()
                         .map(this::argWithEscapingToJava)
                         .collect(Collectors.joining(","))
         );

jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/magic/LineMagicArgs.java

@@ -2,14 +2,79 @@
 
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Optional;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 /**
  * Locates line and cell magic syntax in the cell code and replaces it with code native to the underlying kernel.
+ * Code replacement happens via {@link MagicTranspiler}.
  */
-public class MagicParser {
+public class MagicsResolver {
+
+    private final Pattern lineMagicPattern;
+    private final Pattern cellMagicPattern;
+    private final MagicTranspiler magicTranspiler;
+
+    public MagicsResolver(String lineMagicStart, String cellMagicStart, MagicTranspiler magicTranspiler) {
+        this.lineMagicPattern = Pattern.compile(lineMagicStart + "(?<args>\\w.*?)$", Pattern.MULTILINE);
+        this.cellMagicPattern = Pattern.compile("^(?<argsLine>" + cellMagicStart + "(?<args>\\w.*?))\\R(?<body>(?sU).+?)$");
+        this.magicTranspiler = magicTranspiler;
+    }
+
+    /**
+     * Replaces cell and line magics in the source with native kernel code.
+     */
+    public String resolve(String cellSource) {
+        ParsedCellMagic parsedCell = parseCellMagic(cellSource);
+        return parsedCell != null
+                ? magicTranspiler.transpileCell(parsedCell)
+                : resolveLineMagics(cellSource);
+    }
+
+    String resolveLineMagics(String cellSource) {
+
+        StringBuffer out = new StringBuffer();
+        Matcher m = lineMagicPattern.matcher(cellSource);
+
+        while (m.find()) {
+            ParsedLineMagic parsed = parseLineMagic(cellSource, m);
+            String transformed = magicTranspiler.transpileLine(parsed);
+            m.appendReplacement(out, Matcher.quoteReplacement(transformed));
+        }
+
+        m.appendTail(out);
+        return out.toString();
+    }
+
+    ParsedCellMagic parseCellMagic(String cellSource) {
+        Matcher m = cellMagicPattern.matcher(cellSource);
+
+        if (!m.matches()) {
+            return null;
+        }
+
+        List<String> split = split(m.group("args"));
+        String bodyAfterMagic = m.group("body");
+
+        return new ParsedCellMagic(
+                split.get(0),
+                split.subList(1, split.size()),
+                bodyAfterMagic);
+    }
+
+    private ParsedLineMagic parseLineMagic(String cellSource, Matcher matchedLine) {
+        List<String> split = split(matchedLine.group("args"));
+
+        String rawLinePrefix = cellSource.substring(0, matchedLine.start());
+        String linePrefix = rawLinePrefix.substring(rawLinePrefix.lastIndexOf('\n') + 1);
+
+        return new ParsedLineMagic(
+                split.get(0),
+                split.subList(1, split.size()),
+                linePrefix,
+                matchedLine.group()
+        );
+    }
 
     static List<String> split(String args) {
         args = args.trim();
@@ -64,69 +129,4 @@ static List<String> split(String args) {
 
         return split;
     }
-
-    private final Pattern lineMagicPattern;
-    private final Pattern cellMagicPattern;
-    private final MagicTranspiler transpiler;
-
-    public MagicParser(String lineMagicStart, String cellMagicStart, MagicTranspiler transpiler) {
-        this.lineMagicPattern = Pattern.compile(lineMagicStart + "(?<args>\\w.*?)$", Pattern.MULTILINE);
-        this.cellMagicPattern = Pattern.compile("^(?<argsLine>" + cellMagicStart + "(?<args>\\w.*?))\\R(?<body>(?sU).+?)$");
-        this.transpiler = transpiler;
-    }
-
-    /**
-     * Replaces cell and line magics in the source with native kernel code.
-     */
-    public String resolveMagics(String cellSource) {
-        return transpileCellMagic(cellSource).orElse(transpileLineMagics(cellSource));
-    }
-
-    private Optional<String> transpileCellMagic(String cellSource) {
-        ParsedCellMagic parsedCell = parseCellMagic(cellSource);
-        return Optional.ofNullable(parsedCell).map(transpiler::transpileCell);
-    }
-
-    String transpileLineMagics(String cellSource) {
-
-        StringBuffer out = new StringBuffer();
-        Matcher m = lineMagicPattern.matcher(cellSource);
-
-        while (m.find()) {
-            ParsedLineMagic parsed = parseLineMagic(cellSource, m);
-            String transformed = transpiler.transpileLine(parsed);
-            m.appendReplacement(out, Matcher.quoteReplacement(transformed));
-        }
-
-        m.appendTail(out);
-        return out.toString();
-    }
-
-    ParsedCellMagic parseCellMagic(String cellSource) {
-        Matcher m = this.cellMagicPattern.matcher(cellSource);
-
-        if (!m.matches()) {
-            return null;
-        }
-
-        String rawArgsLine = m.group("argsLine");
-        String rawArgs = m.group("args");
-        String body = m.group("body");
-        List<String> split = split(rawArgs);
-
-        CellMagicArgs args = new CellMagicArgs(split.get(0), split.subList(1, split.size()), body);
-        return new ParsedCellMagic(args, rawArgsLine, cellSource);
-    }
-
-    private ParsedLineMagic parseLineMagic(String cellSource, Matcher matchedLine) {
-        String raw = matchedLine.group();
-        String rawArgs = matchedLine.group("args");
-        List<String> split = split(rawArgs);
-
-        LineMagicArgs args = new LineMagicArgs(split.get(0), split.subList(1, split.size()));
-
-        String rawLinePrefix = cellSource.substring(0, matchedLine.start());
-        String linePrefix = rawLinePrefix.substring(rawLinePrefix.lastIndexOf('\n') + 1);
-        return new ParsedLineMagic(args, raw, cellSource, linePrefix);
-    }
 }