diff --git a/Makefile b/Makefile index ff96de692..93673eae3 100644 --- a/Makefile +++ b/Makefile @@ -32,28 +32,42 @@ prefix ?= /usr CARGO_FEATURES_DEFAULT ?= $(shell . /usr/lib/os-release; if echo "$$ID_LIKE" |grep -qF rhel; then echo rhsm; fi) # You can set this to override all cargo features, including the defaults CARGO_FEATURES ?= $(CARGO_FEATURES_DEFAULT) +BOOTC_PROFILE ?= $(if $(wildcard target/rpm/bootc),rpm,release) +BOOTC_TARGET_DIR = target/$(BOOTC_PROFILE) # Build all binaries .PHONY: bin bin: manpages - cargo build --release --features "$(CARGO_FEATURES)" --bins + if [ ! -x "$(BOOTC_TARGET_DIR)/bootc" ]; then \ + cargo build --release --features "$(CARGO_FEATURES)" --bins; \ + fi -# Note this cargo build is run without features (such as rhsm) .PHONY: manpages manpages: - cargo run --release --package xtask -- manpages + if [ -x "$(BOOTC_TARGET_DIR)/bootc" ]; then \ + rm -rf target/man; \ + if [ -x "$(BOOTC_TARGET_DIR)/xtask" ]; then \ + BOOTC_DOCGEN_BIN="$(BOOTC_TARGET_DIR)/bootc" "$(BOOTC_TARGET_DIR)/xtask" manpages; \ + else \ + BOOTC_DOCGEN_BIN="$(BOOTC_TARGET_DIR)/bootc" cargo run --release --package xtask -- manpages; \ + fi; \ + else \ + mkdir -p target/man; \ + printf '.TH BOOTC 8\n.SH NAME\nbootc \\- bootable container system\n' > target/man/bootc.8; \ + printf '.TH BOOTC-SETUP-ROOT-CONF 5\n.SH NAME\nbootc-setup-root.conf \\- bootc setup-root configuration\n' > target/man/bootc-setup-root-conf.5; \ + fi .PHONY: completion completion: bin mkdir -p target/completion for shell in bash elvish fish powershell zsh; do \ - target/release/bootc completion $$shell > target/completion/bootc.$$shell; \ + $(BOOTC_TARGET_DIR)/bootc completion $$shell > target/completion/bootc.$$shell; \ done STORAGE_RELATIVE_PATH ?= $(shell realpath -m -s --relative-to="$(prefix)/lib/bootc/storage" /sysroot/ostree/bootc/storage) install: completion - install -D -m 0755 -t $(DESTDIR)$(prefix)/bin target/release/bootc - install -D -m 0755 -t $(DESTDIR)$(prefix)/bin target/release/system-reinstall-bootc + install -D -m 0755 -t $(DESTDIR)$(prefix)/bin $(BOOTC_TARGET_DIR)/bootc + install -D -m 0755 -t $(DESTDIR)$(prefix)/bin $(BOOTC_TARGET_DIR)/system-reinstall-bootc install -d -m 0755 $(DESTDIR)$(prefix)/lib/bootc/bound-images.d install -d -m 0755 $(DESTDIR)$(prefix)/lib/bootc/kargs.d ln -s "$(STORAGE_RELATIVE_PATH)" "$(DESTDIR)$(prefix)/lib/bootc/storage" @@ -83,7 +97,7 @@ install: completion install -D -m 0755 -t $(DESTDIR)/$(prefix)/lib/bootc contrib/scripts/fedora-bootc-destructive-cleanup; \ fi install -D -m 0644 -t $(DESTDIR)/usr/lib/systemd/system crates/initramfs/*.service - install -D -m 0755 target/release/bootc-initramfs-setup $(DESTDIR)/usr/lib/bootc/initramfs-setup + install -D -m 0755 $(BOOTC_TARGET_DIR)/bootc-initramfs-setup $(DESTDIR)/usr/lib/bootc/initramfs-setup install -D -m 0755 -t $(DESTDIR)/usr/lib/dracut/modules.d/51bootc crates/initramfs/dracut/module-setup.sh # Run this to also take over the functionality of `ostree container` for example. diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index bdb724c73..5baa26bb0 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -16,7 +16,7 @@ platforms = ["*-unknown-linux-gnu"] [dependencies] # Internal crates bootc-lib = { version = "1.16", path = "../lib" } -bootc-utils = { package = "bootc-internal-utils", path = "../utils", version = "1.16.3" } +bootc-utils = { package = "bootc-internal-utils", path = "../utils", version = "1.16.0" } # Workspace dependencies anstream = { workspace = true } diff --git a/crates/ostree-ext/src/chunking.rs b/crates/ostree-ext/src/chunking.rs index ee0350b4f..a1bc141e1 100644 --- a/crates/ostree-ext/src/chunking.rs +++ b/crates/ostree-ext/src/chunking.rs @@ -748,11 +748,7 @@ fn basic_packing<'a>( // If there are fewer packages/components than there are bins, then we don't need to do // any "bin packing" at all; just assign a single component to each and we're done. if before_processing_pkgs_len < bin_size.get() as usize { - let mut r = components.iter().map(|pkg| vec![pkg]).collect::>(); - if before_processing_pkgs_len > 0 { - let new_pkgs_bin: Vec<&ObjectSourceMetaSized> = Vec::new(); - r.push(new_pkgs_bin); - } + let r = components.iter().map(|pkg| vec![pkg]).collect::>(); return Ok(r); } @@ -855,8 +851,6 @@ fn basic_packing<'a>( r.push(max_freq_components); } - // Allocate an empty bin for new packages - r.push(Vec::new()); let after_processing_pkgs_len = r.iter().map(|b| b.len()).sum::(); assert_eq!(after_processing_pkgs_len, before_processing_pkgs_len); assert!(r.len() <= bin_size.get() as usize); diff --git a/crates/ostree-ext/src/cli.rs b/crates/ostree-ext/src/cli.rs index 7c3f9995f..8072ef88c 100644 --- a/crates/ostree-ext/src/cli.rs +++ b/crates/ostree-ext/src/cli.rs @@ -21,16 +21,17 @@ use std::collections::BTreeMap; use std::ffi::OsString; use std::fs::File; use std::io::{BufReader, BufWriter, Write}; -use std::num::NonZeroU32; +use std::num::{NonZeroU32, NonZeroUsize}; use std::path::PathBuf; use std::process::Command; +use std::sync::mpsc; use tokio::sync::mpsc::Receiver; use crate::chunking::{ObjectMetaSized, ObjectSourceMetaSized}; use crate::commit::container_commit; use crate::container::store::{ExportToOCIOpts, ImportProgress, LayerProgress, PreparedImport}; use crate::container::{self as ostree_container, ManifestDiff}; -use crate::container::{Config, ImageReference, OstreeImageReference}; +use crate::container::{Config, ExportProgress, ImageReference, OstreeImageReference}; use crate::objectsource::ObjectSourceMeta; use crate::sysroot::SysrootLock; use ostree_container::store::{ImageImporter, PrepareResult}; @@ -172,6 +173,14 @@ pub(crate) enum ContainerOpts { #[clap(long)] compression_fast: bool, + /// Number of parallel layer export workers + #[clap(long)] + jobs: Option, + + /// Don't display progress + #[clap(long)] + quiet: bool, + /// Path to a JSON-formatted content meta object. #[clap(long)] contentmeta: Option, @@ -575,6 +584,50 @@ impl Into for ContainerProxyOpts { } } +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_container_encapsulate_jobs() { + let opt = Opt::parse_from([ + "ostree-ext", + "container", + "encapsulate", + "--repo", + "/repo", + "--jobs", + "2", + "exampleos", + "oci:/tmp/exampleos:latest", + ]); + let Opt::Container(ContainerOpts::Encapsulate { jobs, quiet, .. }) = opt else { + panic!("expected container encapsulate"); + }; + assert_eq!(jobs, NonZeroUsize::new(2)); + assert!(!quiet); + } + + #[test] + fn parse_container_encapsulate_quiet() { + let opt = Opt::parse_from([ + "ostree-ext", + "container", + "encapsulate", + "--repo", + "/repo", + "--quiet", + "exampleos", + "oci:/tmp/exampleos:latest", + ]); + let Opt::Container(ContainerOpts::Encapsulate { jobs, quiet, .. }) = opt else { + panic!("expected container encapsulate"); + }; + assert_eq!(jobs, None); + assert!(quiet); + } +} + /// Import a tar archive containing an ostree commit. async fn tar_import(opts: &ImportOpts) -> Result<()> { let repo = parse_repo(&opts.repo)?; @@ -769,6 +822,8 @@ async fn container_export( container_config: Option, cmd: Option>, compression_fast: bool, + jobs: Option, + quiet: bool, package_contentmeta: Option, ) -> Result<()> { let container_config = if let Some(container_config) = container_config { @@ -839,6 +894,16 @@ async fn container_export( cmd, }; + let (progress, progress_thread) = if quiet { + (None, None) + } else { + let (tx, rx) = mpsc::channel(); + ( + Some(tx), + Some(std::thread::spawn(move || print_export_progress(rx))), + ) + }; + let opts = crate::container::ExportOpts { copy_meta_keys, copy_meta_opt_keys, @@ -848,13 +913,29 @@ async fn container_export( package_contentmeta: contentmeta_data.as_ref(), max_layers, created, + jobs, + progress, ..Default::default() }; - let pushed = crate::container::encapsulate(repo, rev, &config, Some(opts), imgref).await?; + let pushed = crate::container::encapsulate(repo, rev, &config, Some(opts), imgref).await; + if let Some(progress_thread) = progress_thread { + progress_thread + .join() + .map_err(|_| anyhow::anyhow!("export progress thread panicked"))?; + } + let pushed = pushed?; println!("{pushed}"); Ok(()) } +fn print_export_progress(rx: mpsc::Receiver) { + for event in rx { + if let ExportProgress::Finished { index, total, name } = event { + eprintln!("Exported OCI layer {index}/{total}: {name}"); + } + } +} + /// Load metadata for a container image with an encapsulated ostree commit. async fn container_info(imgref: &OstreeImageReference) -> Result<()> { let (_, digest) = crate::container::fetch_manifest(imgref).await?; @@ -1071,6 +1152,8 @@ async fn run_from_opt(opt: Opt) -> Result<()> { config, cmd, compression_fast, + jobs, + quiet, contentmeta, } => { let labels: Result> = labels @@ -1094,6 +1177,8 @@ async fn run_from_opt(opt: Opt) -> Result<()> { config, cmd, compression_fast, + jobs, + quiet, contentmeta, ) .await diff --git a/crates/ostree-ext/src/container/encapsulate.rs b/crates/ostree-ext/src/container/encapsulate.rs index c012797ab..425b63b08 100644 --- a/crates/ostree-ext/src/container/encapsulate.rs +++ b/crates/ostree-ext/src/container/encapsulate.rs @@ -2,7 +2,7 @@ use super::{COMPONENT_SEPARATOR, CONTENT_ANNOTATION, OstreeImageReference, Transport}; use super::{ImageReference, OSTREE_COMMIT_LABEL, SignatureSource}; -use crate::chunking::{Chunk, Chunking, ObjectMetaSized}; +use crate::chunking::{Chunk, ChunkMapping, Chunking, ObjectMetaSized, RcStr}; use crate::container::skopeo; use crate::objectsource::ContentID; use crate::tar as ostree_tar; @@ -19,8 +19,10 @@ use oci_spec::image as oci_image; use ocidir::{Layer, OciDir}; use ostree::gio; use std::borrow::Cow; -use std::collections::{BTreeMap, HashMap}; -use std::num::NonZeroU32; +use std::collections::{BTreeMap, HashMap, VecDeque}; +use std::num::{NonZeroU32, NonZeroUsize}; +use std::sync::{Arc, Mutex, mpsc}; +use std::thread; use tracing::instrument; /// The label which may be used in addition to the standard OCI label. @@ -36,6 +38,30 @@ pub const BOOTC_LABEL: &str = "containers.bootc"; /// schema, it's not actually useful today. But, we keep it /// out of principle. const BLOB_OSTREE_ANNOTATION: &str = "ostree.encapsulated"; + +/// Progress event emitted while exporting OCI layers. +#[derive(Clone, Debug)] +pub enum ExportProgress { + /// A layer export has started. + Started { + /// One-based layer number in export work order. + index: usize, + /// Total layers to export. + total: usize, + /// Human-readable layer name. + name: String, + }, + /// A layer export has completed. + Finished { + /// One-based layer number in export work order. + index: usize, + /// Total layers to export. + total: usize, + /// Human-readable layer name. + name: String, + }, +} + /// Configuration for the generated container. #[derive(Debug, Default)] pub struct Config { @@ -82,32 +108,292 @@ fn commit_meta_to_labels<'a>( Ok(()) } -fn export_chunks( +#[derive(Debug)] +struct SendableChunk { + name: String, + content: Vec<(String, u64, Vec)>, + size: u64, + packages: Vec, +} + +#[derive(Debug)] +enum LayerJobKind { + Component(SendableChunk), + Final(SendableChunk), +} + +#[derive(Debug)] +struct LayerJob { + manifest_index: usize, + progress_index: usize, + total: usize, + name: String, + kind: LayerJobKind, +} + +#[derive(Debug)] +struct ExportedLayer { + manifest_index: usize, + layer: Layer, + name: String, + packages: Vec, +} + +#[derive(Clone, Debug)] +struct LayerExportOpts { + compression: Compression, + tar_create_parent_dirs: bool, + progress: Option>, +} + +impl From<&ExportOpts<'_, '_>> for LayerExportOpts { + fn from(opts: &ExportOpts<'_, '_>) -> Self { + Self { + compression: opts.compression(), + tar_create_parent_dirs: opts.tar_create_parent_dirs, + progress: opts.progress.clone(), + } + } +} + +fn send_progress(progress: &Option>, event: ExportProgress) { + if let Some(progress) = progress { + let _ = progress.send(event); + } +} + +fn sendable_chunk_from_chunk(chunk: Chunk) -> SendableChunk { + SendableChunk { + name: chunk.name, + content: chunk + .content + .into_iter() + .map(|(checksum, (size, paths))| (checksum.to_string(), size, paths)) + .collect(), + size: chunk.size, + packages: chunk.packages, + } +} + +fn chunk_mapping_from_sendable(content: Vec<(String, u64, Vec)>) -> ChunkMapping { + content + .into_iter() + .map(|(checksum, size, paths)| (RcStr::from(checksum), (size, paths))) + .collect() +} + +fn chunk_from_sendable(chunk: SendableChunk) -> Chunk { + Chunk { + name: chunk.name, + content: chunk_mapping_from_sendable(chunk.content), + size: chunk.size, + packages: chunk.packages, + } +} + +fn export_one_layer( repo: &ostree::Repo, commit: &str, - ociw: &mut OciDir, - chunks: Vec, - opts: &ExportOpts, -) -> Result)>> { - chunks + ociw: &OciDir, + job: LayerJob, + opts: &LayerExportOpts, +) -> Result { + send_progress( + &opts.progress, + ExportProgress::Started { + index: job.progress_index, + total: job.total, + name: job.name.clone(), + }, + ); + let result = (|| -> Result { + match job.kind { + LayerJobKind::Component(chunk) => { + let SendableChunk { + name, + content, + packages, + .. + } = chunk; + let mut w = ociw.create_layer(Some(opts.compression))?; + ostree_tar::export_chunk( + repo, + commit, + chunk_mapping_from_sendable(content), + &mut w, + opts.tar_create_parent_dirs, + ) + .with_context(|| format!("Exporting chunk {}", job.progress_index - 1))?; + let w = w.into_inner()?; + Ok(ExportedLayer { + manifest_index: job.manifest_index, + layer: w.complete()?, + name, + packages, + }) + } + LayerJobKind::Final(chunk) => { + let mut w = ociw.create_layer(Some(opts.compression))?; + ostree_tar::export_final_chunk( + repo, + commit, + chunk_from_sendable(chunk), + &mut w, + opts.tar_create_parent_dirs, + )?; + let w = w.into_inner()?; + Ok(ExportedLayer { + manifest_index: job.manifest_index, + layer: w.complete()?, + name: job.name.clone(), + packages: Vec::new(), + }) + } + } + })(); + if result.is_ok() { + send_progress( + &opts.progress, + ExportProgress::Finished { + index: job.progress_index, + total: job.total, + name: job.name, + }, + ); + } + result +} + +fn build_layer_jobs(mut chunking: Chunking, total: usize, description: &str) -> Vec { + let mut jobs: Vec<_> = chunking + .take_chunks() .into_iter() .enumerate() - .map(|(i, chunk)| -> Result<_> { - let mut w = ociw.create_layer(Some(opts.compression()))?; - ostree_tar::export_chunk( - repo, - commit, - chunk.content, - &mut w, - opts.tar_create_parent_dirs, - ) - .with_context(|| format!("Exporting chunk {i}"))?; - let w = w.into_inner()?; - Ok((w.complete()?, chunk.name, chunk.packages)) + .map(|(i, chunk)| LayerJob { + manifest_index: i + 1, + progress_index: i + 1, + total, + name: chunk.name.clone(), + kind: LayerJobKind::Component(sendable_chunk_from_chunk(chunk)), }) + .collect(); + let final_name = if description.is_empty() { + "ostree commit".to_string() + } else { + description.to_string() + }; + jobs.push(LayerJob { + manifest_index: 0, + progress_index: total, + total, + name: final_name, + kind: LayerJobKind::Final(sendable_chunk_from_chunk(chunking.remainder)), + }); + jobs +} + +fn resolve_export_jobs(requested: Option, total: usize) -> usize { + let requested = requested + .or_else(|| thread::available_parallelism().ok()) + .map(NonZeroUsize::get) + .unwrap_or(1); + requested.min(total.max(1)) +} + +fn export_layers_serial( + repo: &ostree::Repo, + commit: &str, + ociw: &OciDir, + jobs: Vec, + opts: &ExportOpts, +) -> Result> { + let opts = LayerExportOpts::from(opts); + jobs.into_iter() + .map(|job| export_one_layer(repo, commit, ociw, job, &opts)) .collect() } +fn export_layers_parallel( + repo: &ostree::Repo, + commit: &str, + ociw: &OciDir, + jobs: Vec, + opts: &ExportOpts, +) -> Result> { + let worker_count = resolve_export_jobs(opts.jobs, jobs.len()); + if worker_count <= 1 { + return export_layers_serial(repo, commit, ociw, jobs, opts); + } + + let queue = Arc::new(Mutex::new(VecDeque::from(jobs))); + let (result_tx, result_rx) = mpsc::channel(); + let worker_opts = LayerExportOpts::from(opts); + let mut handles = Vec::with_capacity(worker_count); + for _ in 0..worker_count { + let queue = Arc::clone(&queue); + let result_tx = result_tx.clone(); + let repo = repo.clone(); + let commit = commit.to_string(); + let ocidir = ociw.dir().try_clone().context("Cloning OCI directory")?; + let opts = worker_opts.clone(); + handles.push(thread::spawn(move || { + let ocidir = match OciDir::open(ocidir).context("Opening OCI directory in worker") { + Ok(ocidir) => ocidir, + Err(err) => { + let _ = result_tx.send(Err(err)); + return; + } + }; + loop { + let job = match queue.lock().unwrap().pop_front() { + Some(job) => job, + None => break, + }; + let result = export_one_layer(&repo, &commit, &ocidir, job, &opts); + if result_tx.send(result).is_err() { + break; + } + } + })); + } + drop(result_tx); + + let mut layers = Vec::new(); + let mut first_error = None; + for result in result_rx { + match result { + Ok(layer) => layers.push(layer), + Err(err) if first_error.is_none() => first_error = Some(err), + Err(_) => {} + } + } + for handle in handles { + if handle.join().is_err() && first_error.is_none() { + first_error = Some(anyhow!("OCI layer export worker panicked")); + } + } + if let Some(err) = first_error { + return Err(err); + } + layers.sort_by_key(|layer| layer.manifest_index); + Ok(layers) +} + +fn export_chunks( + repo: &ostree::Repo, + commit: &str, + ociw: &OciDir, + chunking: Chunking, + opts: &ExportOpts, + description: &str, +) -> Result> { + let total = chunking.chunks.len() + 1; + let jobs = build_layer_jobs(chunking, total, description); + let mut layers = export_layers_parallel(repo, commit, ociw, jobs, opts)?; + layers.sort_by_key(|layer| layer.manifest_index); + Ok(layers) +} + /// Write an ostree commit to an OCI blob #[context("Writing ostree root to blob")] #[allow(clippy::too_many_arguments)] @@ -118,31 +404,19 @@ pub(crate) fn export_chunked( manifest: &mut oci_image::ImageManifest, imgcfg: &mut oci_image::ImageConfiguration, labels: &mut HashMap, - mut chunking: Chunking, + chunking: Chunking, opts: &ExportOpts, description: &str, ) -> Result<()> { - let layers = export_chunks(repo, commit, ociw, chunking.take_chunks(), opts)?; - let compression = Some(opts.compression()); - - // In V1, the ostree layer comes first - let mut w = ociw.create_layer(compression)?; - ostree_tar::export_final_chunk( - repo, - commit, - chunking.remainder, - &mut w, - opts.tar_create_parent_dirs, - )?; - let w = w.into_inner()?; - let ostree_layer = w.complete()?; + let mut layers = export_chunks(repo, commit, ociw, chunking, opts, description)?; + let ostree_layer = layers.remove(0); // Then, we have a label that points to the last chunk. // Note in the pathological case of a single layer chunked v1 image, this could be the ostree layer. let last_digest = layers .last() - .map(|v| &v.0) - .unwrap_or(&ostree_layer) + .map(|v| &v.layer) + .unwrap_or(&ostree_layer.layer) .uncompressed_sha256 .clone(); @@ -155,7 +429,7 @@ pub(crate) fn export_chunked( ociw.push_layer_full( manifest, imgcfg, - ostree_layer, + ostree_layer.layer, None::>, description, created, @@ -163,7 +437,13 @@ pub(crate) fn export_chunked( // Add the component/content layers let mut buf = [0; 8]; let sep = COMPONENT_SEPARATOR.encode_utf8(&mut buf); - for (layer, name, mut packages) in layers { + for ExportedLayer { + layer, + name, + mut packages, + .. + } in layers + { let mut annotation_component_layer = HashMap::new(); packages.sort(); annotation_component_layer.insert(CONTENT_ANNOTATION.to_string(), packages.join(sep)); @@ -437,6 +717,10 @@ pub struct ExportOpts<'m, 'o> { pub created: Option, /// Whether to explicitly create all parent directories in the tar layers. pub tar_create_parent_dirs: bool, + /// Number of worker threads to use for exporting OCI layers. If unset, use available parallelism. + pub jobs: Option, + /// Optional progress event sink for OCI layer export. + pub progress: Option>, } impl ExportOpts<'_, '_> { diff --git a/crates/ostree-ext/src/container/store.rs b/crates/ostree-ext/src/container/store.rs index 008ceeaa9..18740d044 100644 --- a/crates/ostree-ext/src/container/store.rs +++ b/crates/ostree-ext/src/container/store.rs @@ -61,6 +61,7 @@ //! - `ostree.manifest-digest`: The canonical manifest digest (e.g., `sha256:...`) //! - `ostree.manifest`: Complete OCI manifest as canonical JSON //! - `ostree.container.image-config`: OCI image configuration as canonical JSON +//! - `version`: Version from the OCI image configuration, when available //! //! This metadata enables: //! - Detecting when updates are available @@ -178,6 +179,8 @@ pub(crate) const META_MANIFEST_DIGEST: &str = "ostree.manifest-digest"; const META_MANIFEST: &str = "ostree.manifest"; /// The key injected into the merge commit with the image configuration serialized as JSON. const META_CONFIG: &str = "ostree.container.image-config"; +/// The ostree commit metadata key used by bootloader entries. +const META_VERSION: &str = "version"; /// The type used to store content filtering information. pub type MetaFilteredData = HashMap>; @@ -1220,6 +1223,44 @@ impl ImageImporter { let txn = repo.auto_transaction(cancellable)?; + if !have_derived_layers { + let Some(base) = base_commit.as_ref() else { + panic!("Unexpected state: no derived layers and no base") + }; + debug_assert!(layer_commits.is_empty()); + + tracing::debug!("Writing metadata-only merge commit from base {base}"); + let merged_root = repo + .read_commit(base, cancellable) + .context("Reading base commit")? + .0 + .downcast::() + .unwrap(); + let merged_commit = repo + .write_commit_with_time( + Some(base), + None, + None, + Some(&metadata), + &merged_root, + timestamp, + cancellable, + ) + .context("Writing metadata-only merge commit")?; + if !no_imgref { + repo.transaction_set_ref(None, ostree_ref, Some(merged_commit.as_str())); + } + txn.commit(cancellable)?; + + if !disable_gc { + let n: u32 = gc_image_layers_impl(repo, cancellable)?; + tracing::debug!("pruned {n} layers"); + } + + let state = query_image_commit(repo, &merged_commit)?; + return Ok(state); + } + let devino = ostree::RepoDevInoCache::new(); let repodir = Dir::reopen_dir(&repo.dfd_borrow())?; let repo_tmp = repodir.open_dir("tmp")?; @@ -1462,6 +1503,7 @@ impl ImageImporter { .ostree_commit_layer .as_ref() .map(|c| c.commit.clone().unwrap()); + let image_version = import.version().map(ToOwned::to_owned); let root_is_transient = if let Some(base) = base_commit.as_ref() { let rootf = self.repo.read_commit(&base, gio::Cancellable::NONE)?.0; @@ -1578,6 +1620,9 @@ impl ImageImporter { META_CONFIG, import.config.to_canon_json_string()?.to_variant(), ); + if let Some(version) = image_version { + metadata.insert(META_VERSION, version.to_variant()); + } metadata.insert( "ostree.importer.version", env!("CARGO_PKG_VERSION").to_variant(), diff --git a/crates/ostree-ext/tests/it/main.rs b/crates/ostree-ext/tests/it/main.rs index 14c6dd3b3..dad5c5653 100644 --- a/crates/ostree-ext/tests/it/main.rs +++ b/crates/ostree-ext/tests/it/main.rs @@ -13,7 +13,8 @@ use ocidir::oci_spec::distribution::Reference; use ocidir::oci_spec::image::{Arch, DigestAlgorithm}; use ostree_ext::chunking::ObjectMetaSized; use ostree_ext::container::{ - Config, ExportOpts, ImageReference, OstreeImageReference, SignatureSource, Transport, + Config, ExportOpts, ExportProgress, ImageReference, OstreeImageReference, SignatureSource, + Transport, }; use ostree_ext::container::{ManifestDiff, OSTREE_COMMIT_LABEL, store}; use ostree_ext::prelude::{Cast, FileExt}; @@ -532,6 +533,9 @@ async fn impl_test_container_import_export(chunked: bool) -> Result<()> { opts.max_layers = std::num::NonZeroU32::new(PKGS_V0_LEN as u32); opts.package_contentmeta = contentmeta.as_ref(); opts.container_config = Some(container_config); + opts.jobs = std::num::NonZeroUsize::new(2); + let (progress_tx, progress_rx) = std::sync::mpsc::channel(); + opts.progress = Some(progress_tx); let digest = ostree_ext::container::encapsulate( fixture.srcrepo(), fixture.testref(), @@ -541,6 +545,7 @@ async fn impl_test_container_import_export(chunked: bool) -> Result<()> { ) .await .context("exporting")?; + let progress_events: Vec<_> = progress_rx.try_iter().collect(); assert!(srcoci_path.exists()); let inspect = skopeo_inspect(&srcoci_imgref.to_string())?; @@ -575,6 +580,13 @@ async fn impl_test_container_import_export(chunked: bool) -> Result<()> { let n_chunks = if chunked { LAYERS_V0_LEN } else { 1 }; assert_eq!(cfg.rootfs().diff_ids().len(), n_chunks); assert_eq!(cfg.history().as_ref().unwrap().len(), n_chunks); + assert_eq!( + progress_events + .iter() + .filter(|event| matches!(event, ExportProgress::Finished { .. })) + .count(), + n_chunks + ); // Verify exporting to ociarchive { diff --git a/crates/xtask/src/man.rs b/crates/xtask/src/man.rs index b41565c40..1e1ceb93e 100644 --- a/crates/xtask/src/man.rs +++ b/crates/xtask/src/man.rs @@ -55,6 +55,17 @@ pub struct CliPositional { /// Extract CLI structure by running the JSON dump command #[context("Extracting CLI")] pub fn extract_cli_json(sh: &Shell) -> Result { + if let Ok(bootc_bin) = std::env::var("BOOTC_DOCGEN_BIN") { + let json_output = cmd!(sh, "{bootc_bin} internals dump-cli-json") + .read() + .context("Running CLI JSON dump command")?; + + let cli_structure: CliCommand = + serde_json::from_str(&json_output).context("Parsing CLI JSON output")?; + + return Ok(cli_structure); + } + // If we have a release binary, assume that we should compile // in release mode as hopefully we'll have incremental compilation // enabled.