diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cb16f88..42b6f60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,8 @@ jobs: runs-on: ubuntu-latest env: LBUG_VERSION: main + LBUG_SOURCE_DIR: lbug-src + LBUG_BUILD_FROM_SOURCE: 1 CARGO_HOME: ${{ github.workspace }}/.cargo RUSTC_WRAPPER: sccache SCCACHE_GHA_ENABLED: "true" diff --git a/.gitignore b/.gitignore index eb5a316..55a23b1 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ target +.cache diff --git a/Cargo.lock b/Cargo.lock index d898bf8..20c63f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -524,7 +524,7 @@ dependencies = [ [[package]] name = "lbug" -version = "0.11.1" +version = "0.16.0" dependencies = [ "anyhow", "arrow", diff --git a/Cargo.toml b/Cargo.toml index 2bbc505..e0c1c1e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lbug" -version = "0.11.1" +version = "0.16.0" description = "An in-process property graph database management system built for query speed and scalability" keywords = ["database", "graph", "ffi"] readme = "lbug-src/README.md" @@ -15,6 +15,7 @@ edition = "2021" # Only include files required to build to keep the crate size small include = [ "build.rs", + "/scripts", "/src", "/include", # If more files are added to lbug-src, they should also be added to the rerun-if-changed diff --git a/build.rs b/build.rs index caca16d..d7774ea 100644 --- a/build.rs +++ b/build.rs @@ -1,6 +1,8 @@ use std::env; use std::path::{Path, PathBuf}; +const PREBUILT_LIB_DIR: &str = ".cache/lbug-prebuilt/lib"; + fn link_mode() -> &'static str { if env::var("LBUG_SHARED").is_ok() { "dylib" @@ -13,7 +15,7 @@ fn get_target() -> String { env::var("PROFILE").unwrap() } -fn link_libraries() { +fn link_libraries(link_bundled_deps: bool, prebuilt_lbug_root: Option<&Path>) { // This also needs to be set by any crates using it if they want to use extensions if !cfg!(windows) && link_mode() == "static" { println!("cargo:rustc-link-arg=-rdynamic"); @@ -38,6 +40,13 @@ fn link_libraries() { println!("cargo:rustc-link-lib=dylib=stdc++"); } + if !link_bundled_deps { + if let Some(lbug_root) = prebuilt_lbug_root { + link_prebuilt_libraries(lbug_root); + } + return; + } + for lib in [ "utf8proc", "antlr4_cypher", @@ -66,12 +75,125 @@ fn link_libraries() { } } +fn link_prebuilt_libraries(lbug_root: &Path) { + let build_dir = lbug_root.join("build").join("release"); + for (dir, lib) in [ + ("antlr4_cypher", "antlr4_cypher"), + ("antlr4_runtime", "antlr4_runtime"), + ("brotli", "brotlidec"), + ("brotli", "brotlicommon"), + ("utf8proc", "utf8proc"), + ("re2", "re2"), + ("fastpfor", "fastpfor"), + ("parquet", "parquet"), + ("snappy", "snappy"), + ("thrift", "thrift"), + ("yyjson", "yyjson"), + ("zstd", "zstd"), + ("miniz", "miniz"), + ("mbedtls", "mbedtls"), + ("lz4", "lz4"), + ("roaring_bitmap", "roaring_bitmap"), + ("simsimd", "simsimd"), + ] { + let lib_dir = build_dir.join("third_party").join(dir); + let lib_path = lib_dir.join(format!("lib{lib}.a")); + if lib_path.exists() { + println!("cargo:rustc-link-search=native={}", lib_dir.display()); + println!("cargo:rustc-link-lib=static={lib}"); + } + } +} + +fn manifest_dir() -> PathBuf { + PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap()) +} + +fn static_lbug_file_name() -> &'static str { + if cfg!(windows) { + "lbug.lib" + } else { + "liblbug.a" + } +} + +fn try_download_prebuilt_lbug(manifest_dir: &Path) -> bool { + if link_mode() != "static" { + return false; + } + if env::var("LBUG_BUILD_FROM_SOURCE").is_ok() || env::var("LBUG_RUST_BUILD_FROM_SOURCE").is_ok() + { + println!("cargo:warning=Skipping prebuilt liblbug because source build was requested"); + return false; + } + + let lib_dir = manifest_dir.join(PREBUILT_LIB_DIR); + let lib_path = lib_dir.join(static_lbug_file_name()); + if lib_path.exists() { + return true; + } + + let script = manifest_dir.join("scripts").join("download_lbug.sh"); + if !script.exists() { + return false; + } + + println!("cargo:warning=Downloading prebuilt liblbug static archive..."); + let status = std::process::Command::new("sh") + .arg(&script) + .current_dir(manifest_dir) + .status(); + + match status { + Ok(status) if status.success() && lib_path.exists() => true, + Ok(status) => { + println!( + "cargo:warning=Prebuilt liblbug download failed with status {status}; building from source" + ); + false + } + Err(error) => { + println!( + "cargo:warning=Could not run prebuilt liblbug downloader ({error}); building from source" + ); + false + } + } +} + +fn use_prebuilt_lbug(manifest_dir: &Path) -> Option<(Vec, PathBuf)> { + if !try_download_prebuilt_lbug(manifest_dir) { + return None; + } + + let lib_dir = manifest_dir.join(PREBUILT_LIB_DIR); + let lbug_root = get_lbug_root(); + println!("cargo:rustc-link-search=native={}", lib_dir.display()); + println!("cargo:rerun-if-changed={}", lib_dir.display()); + println!( + "cargo:warning=Using prebuilt liblbug from {}", + lib_dir.join(static_lbug_file_name()).display() + ); + Some((vec![lib_dir, lbug_root.join("src/include")], lbug_root)) +} + fn get_lbug_root() -> PathBuf { - let manifest_dir_str = std::env::var("CARGO_MANIFEST_DIR").unwrap(); - let manifest_dir = Path::new(&manifest_dir_str); - let root = manifest_dir.join("lbug-src"); - if root.is_symlink() || root.is_dir() { - return root; + let manifest_dir = manifest_dir(); + if let Ok(lbug_source_dir) = std::env::var("LBUG_SOURCE_DIR") { + let root = PathBuf::from(lbug_source_dir); + if root.is_symlink() || root.is_dir() { + return root; + } + } + + let sibling_root = manifest_dir.join("../ladybug"); + if sibling_root.is_symlink() || sibling_root.is_dir() { + return sibling_root; + } + + let bundled_root = manifest_dir.join("lbug-src"); + if bundled_root.is_symlink() || bundled_root.is_dir() { + return bundled_root; } if cfg!(windows) { return manifest_dir.join("../.."); @@ -99,7 +221,7 @@ fn get_lbug_root() -> PathBuf { .args(["-sL", &url]) .arg("-o") .arg("ladybug.tar.gz") - .current_dir(manifest_dir) + .current_dir(&manifest_dir) .output() .expect("Failed to download ladybug source"); @@ -116,7 +238,7 @@ fn get_lbug_root() -> PathBuf { "-C", "lbug-src", ]) - .current_dir(manifest_dir) + .current_dir(&manifest_dir) .status() .expect("Failed to extract ladybug source"); @@ -212,7 +334,6 @@ fn build_ffi( if link_mode() == "static" { build.define("LBUG_STATIC_DEFINE", None); } - build.includes(include_paths); println!("cargo:rerun-if-env-changed=LBUG_SHARED"); @@ -243,9 +364,11 @@ fn main() { return; } + let manifest_dir = manifest_dir(); let mut bundled = false; - let mut include_paths = - vec![Path::new(&std::env::var("CARGO_MANIFEST_DIR").unwrap()).join("include")]; + let mut link_bundled_deps = false; + let mut prebuilt_lbug_root = None; + let mut include_paths = vec![manifest_dir.join("include")]; if let (Ok(lbug_lib_dir), Ok(lbug_include)) = (env::var("LBUG_LIBRARY_DIR"), env::var("LBUG_INCLUDE_DIR")) @@ -253,12 +376,16 @@ fn main() { println!("cargo:rustc-link-search=native={lbug_lib_dir}"); println!("cargo:rustc-link-arg=-Wl,-rpath,{lbug_lib_dir}"); include_paths.push(Path::new(&lbug_include).to_path_buf()); + } else if let Some((prebuilt_include_paths, lbug_root)) = use_prebuilt_lbug(&manifest_dir) { + include_paths.extend(prebuilt_include_paths); + prebuilt_lbug_root = Some(lbug_root); } else { include_paths.extend(build_bundled_cmake()); bundled = true; + link_bundled_deps = true; } if link_mode() == "static" { - link_libraries(); + link_libraries(link_bundled_deps, prebuilt_lbug_root.as_deref()); } build_ffi( "src/ffi.rs", @@ -278,6 +405,6 @@ fn main() { ); } if link_mode() == "dylib" { - link_libraries(); + link_libraries(link_bundled_deps, prebuilt_lbug_root.as_deref()); } } diff --git a/examples/Cargo.lock b/examples/Cargo.lock index f585470..f27121c 100644 --- a/examples/Cargo.lock +++ b/examples/Cargo.lock @@ -502,7 +502,7 @@ dependencies = [ [[package]] name = "lbug" -version = "0.11.1" +version = "0.16.0" dependencies = [ "arrow", "cmake", diff --git a/examples/src/main.rs b/examples/src/main.rs index 5b1caf0..eef9392 100644 --- a/examples/src/main.rs +++ b/examples/src/main.rs @@ -9,6 +9,9 @@ fn main() -> Result<(), Error> { )?; let connection = Connection::new(&db)?; + // Load the full-text search extension. + connection.query("install fts; load fts;")?; + // Create schema. connection.query("CREATE NODE TABLE Person(name STRING, age INT64, PRIMARY KEY(name));")?; // Create nodes. diff --git a/include/lbug_rs.h b/include/lbug_rs.h index 065a145..cff4b6c 100644 --- a/include/lbug_rs.h +++ b/include/lbug_rs.h @@ -18,6 +18,7 @@ #include "storage/storage_version_info.h" #else #include +#include "common/enums/statement_type.h" #endif namespace lbug_rs { diff --git a/scripts/download-liblbug.sh b/scripts/download-liblbug.sh new file mode 100755 index 0000000..b108c64 --- /dev/null +++ b/scripts/download-liblbug.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +# Download prebuilt liblbug archives from GitHub releases or workflow artifacts. +set -euo pipefail + +LIB_KIND="${LBUG_LIB_KIND:-shared}" +LINUX_VARIANT="${LBUG_LINUX_VARIANT:-compat}" +REPOSITORY="${LBUG_GITHUB_REPOSITORY:-LadybugDB/ladybug}" +RUN_ID="${LBUG_PRECOMPILED_RUN_ID:-}" +VERSION_OVERRIDE="${LBUG_VERSION:-}" + +if [ "$LIB_KIND" != "shared" ] && [ "$LIB_KIND" != "static" ]; then + echo "Unsupported LBUG_LIB_KIND: $LIB_KIND (expected 'shared' or 'static')" >&2 + exit 1 +fi + +if [ "$LINUX_VARIANT" != "compat" ] && [ "$LINUX_VARIANT" != "perf" ]; then + echo "Unsupported LBUG_LINUX_VARIANT: $LINUX_VARIANT (expected 'compat' or 'perf')" >&2 + exit 1 +fi + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +TARGET_DIR="${LBUG_TARGET_DIR:-$PROJECT_DIR/lib}" + +OS="$(uname -s)" +ARCH="$(uname -m)" + +case "$OS" in + Darwin) + if [ "$ARCH" = "x86_64" ]; then + ARCHIVE_ARCH="x86_64" + elif [ "$ARCH" = "arm64" ]; then + ARCHIVE_ARCH="arm64" + else + echo "Unsupported macOS architecture: $ARCH" >&2 + exit 1 + fi + if [ "$LIB_KIND" = "static" ]; then + ARCHIVE="liblbug-static-osx-${ARCHIVE_ARCH}.tar.gz" + ARTIFACT_NAME="liblbug-static-osx-${ARCHIVE_ARCH}" + LIB_NAME="liblbug.a" + else + ARCHIVE="liblbug-osx-${ARCHIVE_ARCH}.tar.gz" + ARTIFACT_NAME="liblbug-osx-${ARCHIVE_ARCH}" + LIB_NAME="liblbug.dylib" + fi + ;; + Linux) + if [ "$ARCH" = "x86_64" ]; then + ARCHIVE_ARCH="x86_64" + elif [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then + ARCHIVE_ARCH="aarch64" + else + echo "Unsupported Linux architecture: $ARCH" >&2 + exit 1 + fi + if [ "$LIB_KIND" = "static" ]; then + ARCHIVE="liblbug-static-linux-${ARCHIVE_ARCH}-${LINUX_VARIANT}.tar.gz" + ARTIFACT_NAME="liblbug-static-linux-${ARCHIVE_ARCH}-${LINUX_VARIANT}" + LIB_NAME="liblbug.a" + else + ARCHIVE="liblbug-linux-${ARCHIVE_ARCH}.tar.gz" + ARTIFACT_NAME="liblbug-linux-${ARCHIVE_ARCH}" + LIB_NAME="liblbug.so" + fi + ;; + MINGW*|MSYS*|CYGWIN*) + if [ "$ARCH" = "x86_64" ] || [ "$ARCH" = "AMD64" ]; then + ARCHIVE_ARCH="x86_64" + else + echo "Unsupported Windows architecture: $ARCH" >&2 + exit 1 + fi + if [ "$LIB_KIND" = "static" ]; then + ARCHIVE="liblbug-static-windows-${ARCHIVE_ARCH}.zip" + ARTIFACT_NAME="liblbug-static-windows-${ARCHIVE_ARCH}" + LIB_NAME="lbug.lib" + else + ARCHIVE="liblbug-windows-${ARCHIVE_ARCH}.zip" + ARTIFACT_NAME="liblbug-windows-${ARCHIVE_ARCH}" + LIB_NAME="lbug_shared.dll" + fi + ;; + *) + echo "Unsupported OS: $OS" >&2 + exit 1 + ;; +esac + +if [ -f "$TARGET_DIR/$LIB_NAME" ]; then + echo "liblbug already exists in $TARGET_DIR" + exit 0 +fi + +mkdir -p "$TARGET_DIR" +TMPDIR="$(mktemp -d)" +trap 'rm -rf "$TMPDIR"' EXIT + +fetch_release_archive() { + local version + if [ -n "$VERSION_OVERRIDE" ]; then + version="$VERSION_OVERRIDE" + else + version="$(curl -sS "https://api.github.com/repos/${REPOSITORY}/releases/latest" | grep -o '"tag_name": "v\([^"]*\)"' | cut -d'"' -f4 | cut -c2-)" + fi + local download_url="https://github.com/${REPOSITORY}/releases/download/v${version}/${ARCHIVE}" + curl -fSL "$download_url" -o "$TMPDIR/$ARCHIVE" + echo "release:v${version}" +} + +fetch_run_artifact() { + if ! command -v gh >/dev/null 2>&1; then + echo "gh CLI is required when LBUG_PRECOMPILED_RUN_ID is set" >&2 + exit 1 + fi + gh run download "$RUN_ID" --repo "$REPOSITORY" --name "$ARTIFACT_NAME" --dir "$TMPDIR/artifact" >/dev/null + local extracted_archive + extracted_archive="$(find "$TMPDIR/artifact" -type f -name "$ARCHIVE" | head -n1)" + if [ -z "$extracted_archive" ]; then + echo "Artifact ${ARTIFACT_NAME} does not contain ${ARCHIVE}" >&2 + exit 1 + fi + mv "$extracted_archive" "$TMPDIR/$ARCHIVE" + echo "run:${RUN_ID}/${ARTIFACT_NAME}" +} + +if [ -n "$RUN_ID" ]; then + SOURCE_DESC="$(fetch_run_artifact)" +else + SOURCE_DESC="$(fetch_release_archive)" +fi + +if [ "${ARCHIVE##*.}" = "zip" ]; then + unzip -o "$TMPDIR/$ARCHIVE" -d "$TARGET_DIR" +else + tar xzf "$TMPDIR/$ARCHIVE" -C "$TARGET_DIR" +fi + +echo "Installed ${ARCHIVE} from ${SOURCE_DESC} to $TARGET_DIR" diff --git a/scripts/download_lbug.sh b/scripts/download_lbug.sh new file mode 100755 index 0000000..b8be8fb --- /dev/null +++ b/scripts/download_lbug.sh @@ -0,0 +1,62 @@ +#!/bin/sh +# Download prebuilt static liblbug into the Rust crate cache. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +ENV_FILE="${1:-$PROJECT_DIR/.cache/lbug-prebuilt.env}" +CACHE_LIB_DIR="${LBUG_TARGET_DIR:-$PROJECT_DIR/.cache/lbug-prebuilt/lib}" +LIB_KIND="${LBUG_LIB_KIND:-static}" +UPSTREAM_SCRIPT="$SCRIPT_DIR/download-liblbug.sh" +UPSTREAM_URL="https://raw.githubusercontent.com/LadybugDB/ladybug/refs/heads/main/scripts/download-liblbug.sh" + +if [ ! -f "$UPSTREAM_SCRIPT" ]; then + echo "Fetching $UPSTREAM_URL ..." + curl -fsSL "$UPSTREAM_URL" -o "$UPSTREAM_SCRIPT" + chmod +x "$UPSTREAM_SCRIPT" +fi + +LBUG_TARGET_DIR="$CACHE_LIB_DIR" LBUG_LIB_KIND="$LIB_KIND" bash "$UPSTREAM_SCRIPT" + +OS="$(uname -s)" +if [ "$LIB_KIND" = "shared" ]; then + case "$OS" in + Darwin) + LIB_PATH="$CACHE_LIB_DIR/liblbug.dylib" + ;; + Linux) + LIB_PATH="$CACHE_LIB_DIR/liblbug.so" + ;; + MINGW*|MSYS*|CYGWIN*) + LIB_PATH="$CACHE_LIB_DIR/lbug_shared.dll" + ;; + *) + echo "Unsupported OS: $OS" >&2 + exit 1 + ;; + esac +else + case "$OS" in + MINGW*|MSYS*|CYGWIN*) + LIB_PATH="$CACHE_LIB_DIR/lbug.lib" + ;; + *) + LIB_PATH="$CACHE_LIB_DIR/liblbug.a" + ;; + esac +fi + +if [ ! -f "$LIB_PATH" ]; then + echo "Expected precompiled library not found at $LIB_PATH" >&2 + exit 1 +fi + +mkdir -p "$(dirname "$ENV_FILE")" +cat > "$ENV_FILE" <