Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
124 commits
Select commit Hold shift + click to select a range
b0a3b4f
feat: Added draft for pithos 0.7.0
St4NNi Feb 25, 2025
4963580
feat: Added next draft for pithos 0.7.0 specification
St4NNi Feb 25, 2025
030b975
chore: First 1.0.0 draft
St4NNi Jul 22, 2025
24a4070
feat: Modified spec; Implemented structs including serialization and
das-Abroxas Jul 22, 2025
4f5862f
chore: Cleanup
das-Abroxas Jul 22, 2025
f75747e
chore: Cleanup pithos crate
das-Abroxas Jul 22, 2025
4dabdeb
chore: Add x25519-dalek as workspace dependency
das-Abroxas Jul 22, 2025
bb5b2b3
chore: Add pkcs8 and rand dependencies to pithos_lib
das-Abroxas Jul 22, 2025
16cc5cd
feat: Implement X25519 Key Management Module
das-Abroxas Jul 22, 2025
0984463
chore: Cleanup
das-Abroxas Jul 22, 2025
5f9f947
chore: Create dummy key pairs for reproducible testing
das-Abroxas Jul 25, 2025
8e170bc
refactor: Move structs with de-/serialization into own directory
das-Abroxas Jul 25, 2025
5cffeb8
feat: Implement hash helper
das-Abroxas Jul 25, 2025
97bd051
feat: extend x25519 helper with function derive shared key
das-Abroxas Jul 25, 2025
b453a9c
feat: Implement helper for Directory creation/handling
das-Abroxas Jul 25, 2025
a0020c3
feat: Implement helper for Zstd compression/decompression
das-Abroxas Jul 25, 2025
0ec5ce7
feat: Implement helper for ChaChaPoly1305 encryption/decryption
das-Abroxas Jul 25, 2025
44c122e
feat: Implement rudimentary Pithos file writer
das-Abroxas Jul 25, 2025
cb74898
feat: Add unfinished Pithos file reader
das-Abroxas Jul 25, 2025
415b613
feat: Implement ProcessingFlags functions to set bits accordingly
das-Abroxas Jul 29, 2025
69f0503
fix: Incorrect use of write_varint in de-/serialization
das-Abroxas Jul 29, 2025
97e2681
test: Add tests for de-/serialization
das-Abroxas Jul 29, 2025
95a023c
refactor: Simplify payload creation in decrypt_chunk function
das-Abroxas Jul 29, 2025
46d5643
test: Add test for chunk encryption/decryption
das-Abroxas Jul 29, 2025
51cc310
fix: Increment file_index at the correct place
das-Abroxas Jul 29, 2025
9d798e5
refactor: Replace imported serialization
das-Abroxas Jul 29, 2025
a9cbaba
refactor: Return decrypted recipient data entries
das-Abroxas Jul 29, 2025
26d1edb
chore: Add encrypted and compression_level fields to FileWithMetadata
das-Abroxas Jul 29, 2025
14acfd1
chore: Update rust edition
das-Abroxas Jul 29, 2025
7d21a2a
chore: Cleanup dependencies
das-Abroxas Jul 29, 2025
fcd6394
chore: Move tests into tests directory
das-Abroxas Jul 29, 2025
8589002
refactor: Remove bytes dependency
das-Abroxas Jul 30, 2025
e1d8845
feat: Extend directory with modification functions
das-Abroxas Jul 30, 2025
b6b5bc6
feat: Add Zstd magic number and compression level mapping
das-Abroxas Jul 30, 2025
f875215
feat: Add validation to FileHeader and BlockHeader deserialization
das-Abroxas Jul 30, 2025
992e743
chore: Improve error handling
das-Abroxas Jul 30, 2025
229ec78
feat: Add FileEntry constructor and convenience function to add block…
das-Abroxas Jul 30, 2025
9a68199
feat: Add ProcessingFlags constructors
das-Abroxas Jul 30, 2025
0588b10
refactor: Extract PithosWriter functionality in individual functions
das-Abroxas Jul 30, 2025
ea9f51a
refactor: Adapt PithosReader to changes
das-Abroxas Jul 30, 2025
5022705
chore: Cargo clippy fixes and fmt
das-Abroxas Jul 30, 2025
1b22d9f
chore: Implement util functions
das-Abroxas Aug 4, 2025
369a075
chore: Add walkdir dependency
das-Abroxas Aug 5, 2025
4199a7d
feat: Implement directory ingestion for PithosWriter
das-Abroxas Aug 5, 2025
bea2185
feat: PithosReader support to recreate ingested directory structure
das-Abroxas Aug 5, 2025
30a93ed
feat: PithosReader support to read specific byte range of file in index
das-Abroxas Aug 5, 2025
85d1b14
chore: Cleanup, cargo clippy fixes and fmt
das-Abroxas Aug 5, 2025
706f52b
tests: Adapt to changes
das-Abroxas Aug 5, 2025
fc3e2d7
chore: Cleanup, cargo clippy fixes and fmt
das-Abroxas Aug 5, 2025
be989d7
feat: Add core RO-Crate data structures and types
das-Abroxas Aug 11, 2025
6a6f654
feat: implement RO-Crate entity abstractions
das-Abroxas Aug 11, 2025
4137b88
feat: add builder pattern for programmatic RO-Crate creation
das-Abroxas Aug 11, 2025
eb5b6c6
feat: implement RO-Crate metadata and JSON-LD context handling
das-Abroxas Aug 11, 2025
7e63946
feat: add error and validation error types for RO-Crate operations
das-Abroxas Aug 11, 2025
8c97a17
feat: implement RO-Crate validation framework
das-Abroxas Aug 11, 2025
996887e
feat: add RO-Crate readers for directory and ZIP formats
das-Abroxas Aug 11, 2025
2f52c71
feat: add RO-Crate writers for directory and ZIP formats
das-Abroxas Aug 11, 2025
91d39b8
feat: provide top-level convenience functions for reading, writing, a…
das-Abroxas Aug 11, 2025
6d4c081
feat: add utility modules for serialization and time formatting
das-Abroxas Aug 11, 2025
bbade44
feat: Extend directory helper methods
das-Abroxas Aug 11, 2025
af1c9d4
feat: Add RO-Crate helper functions
das-Abroxas Aug 11, 2025
70d4541
feat: Implement TryFrom<&DataEntity> for FileType
das-Abroxas Aug 11, 2025
3bf2552
feat: Add additional FileEntry constructor and helper function
das-Abroxas Aug 11, 2025
81f03a7
feat: Implement TryFrom<&mut FileEntry> for Reference
das-Abroxas Aug 11, 2025
03dbeb6
refactor: Adapt PithosWriter to extended input
das-Abroxas Aug 11, 2025
c9ada1f
feat: Extend PithosWriter with RO-Crate processing
das-Abroxas Aug 11, 2025
f06ec48
chore: Add additional constructor for PithosReaderSimple
das-Abroxas Aug 11, 2025
a24f602
test: Added test for ChaChaPoly1305 helper
das-Abroxas Aug 11, 2025
5f20336
test: Added test for struct marshalling
das-Abroxas Aug 11, 2025
d520fc3
test: Added test for X25519 helper functions
das-Abroxas Aug 11, 2025
c0a9ac4
test: Added unfinished tests for PithosReader and PithosWriter
das-Abroxas Aug 11, 2025
0d5f510
test: Added test util functions
das-Abroxas Aug 11, 2025
cc473ac
test: Added tempfile dev-dependency
das-Abroxas Aug 11, 2025
2cb803a
chore: Cargo clippy fixes and fmt
das-Abroxas Aug 11, 2025
6157ea4
chore: Move thiserror to workspace dependencies
das-Abroxas Aug 13, 2025
4043ab7
chore: Cleanup Pithos CLI dependencies
das-Abroxas Aug 13, 2025
f1fbb28
feat: Integrate updated keypair creation in CLI
das-Abroxas Aug 13, 2025
7d32bf0
refactor: Limit compression probe input to 4096 bytes
das-Abroxas Aug 13, 2025
0a2d263
feat: Implement Crypt4GH conversion support
das-Abroxas Sep 2, 2025
579ee91
test: Add rudimentary test for Pithos to Crypt4GH conversion
das-Abroxas Sep 2, 2025
06bb45e
chore: Use workspace dependency for thiserror
das-Abroxas Sep 2, 2025
b421752
chore: Add Other(String) option to SerializationError
das-Abroxas Sep 2, 2025
190f1b4
refactor: Improve EncryptionSection handling by replacing Vectors wit…
das-Abroxas Sep 2, 2025
35ff875
feat: Add initial append support
das-Abroxas Sep 3, 2025
ee56ad3
feat: Implement Pithos file creation in CLI
das-Abroxas Sep 3, 2025
cb21915
feat: Implement file extraction from Pithos in CLI
das-Abroxas Sep 3, 2025
e956cc4
chore: Clean up, dependency updates
das-Abroxas Sep 3, 2025
b7d0704
fix: CLI argument parsing error
das-Abroxas Sep 4, 2025
92ebb0d
fix: Incorrect compression probe ration calculation
das-Abroxas Sep 6, 2025
168025c
refactor: Replace block index vector with content-addressable IndexMap
das-Abroxas Sep 6, 2025
6d40fc4
fix: Correctly set ProcessingFlags compression level
das-Abroxas Sep 6, 2025
73761c9
feat: Implement rudimentary remote block reading
das-Abroxas Sep 6, 2025
bf0743b
feat: Make FastCDC values configurable
das-Abroxas Sep 6, 2025
a8c97b5
refactor: Always encrypt recipient sections before directory write
das-Abroxas Sep 6, 2025
6625341
feat: Add option to set FastCDC values in Create Command
das-Abroxas Sep 6, 2025
a0b467c
chore: Add dummy data to tests and move keys in subdirectory
das-Abroxas Sep 6, 2025
cd4650e
refactor: Improve ro-crate-metadata parsing
das-Abroxas Sep 6, 2025
373d100
chore: Add util function to load test keys
das-Abroxas Sep 6, 2025
bb67667
chore: Add util function to write dummy Pithos file to temp dir
das-Abroxas Sep 6, 2025
d60fca9
test: Adapt tests to changes
das-Abroxas Sep 6, 2025
f9dae36
chore: Cleanup and dependency updates
das-Abroxas Sep 6, 2025
f6aa7c4
chore: Cargo clippy fixes and fmt
das-Abroxas Sep 6, 2025
d4cea72
chore: Cargo clippy fixes and fmt
das-Abroxas Sep 6, 2025
9ac19ff
refactor: Consolidate error types and handling in pithos_lib
das-Abroxas Sep 7, 2025
b6f45e5
chore: Adapt cli pithos_lib error changes
das-Abroxas Sep 7, 2025
ca09647
feat: Add tracing support to pithos_lib
das-Abroxas Sep 7, 2025
0a12e1c
feat: Improve Pithos CLI logging
das-Abroxas Sep 7, 2025
7c98468
chore: Add missing level argument to tracing::instrument
das-Abroxas Sep 7, 2025
eb29074
feat: Add default relations to directory creation
das-Abroxas Sep 7, 2025
8b7fc84
fix: Correctly evaluate output target also for directories
das-Abroxas Sep 7, 2025
7902858
feat: Impl Display for FileEntry
das-Abroxas Sep 7, 2025
fbdc082
feat: Improve CLI
das-Abroxas Sep 7, 2025
70a7c58
chore: Update version to 0.7.0
das-Abroxas Sep 7, 2025
29b000b
refactor: Extend data read to multiple paths
das-Abroxas Sep 9, 2025
e0ab963
feat: App Crypt4GH export to CLI
das-Abroxas Sep 9, 2025
96336de
feat: Implement Pithos file append in CLI
das-Abroxas Sep 9, 2025
5004c23
feat: Extend keypair output in CLI
das-Abroxas Sep 9, 2025
3091bda
refactor: Additionally print file id with list command
das-Abroxas Sep 9, 2025
3d1bfed
feat: Implement add readers in CLI
das-Abroxas Sep 9, 2025
d5ae760
chore: Cleanup
das-Abroxas Sep 9, 2025
eb59bfa
chore: Dependency update
das-Abroxas Sep 9, 2025
566518b
refactor: Adapt and cleanup specification draft
das-Abroxas Sep 9, 2025
c7b9729
doc: Update README.md
das-Abroxas Sep 9, 2025
328ac21
Merge remote-tracking branch 'origin/main' into feat/0.7_rework
das-Abroxas Sep 9, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2,022 changes: 1,101 additions & 921 deletions Cargo.lock

Large diffs are not rendered by default.

19 changes: 15 additions & 4 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,18 @@ resolver = "2"

[workspace.package]
version = "0.6.0"
authors = ["Sebastian Beyvers <sb@pus.de>"]
edition = "2021"
repository = "https://github.com/ArunaStorage/aruna-file"
license = "MIT OR Apache-2.0"
authors = ["Jannis Schlegel <das_Abroxas@proton.me>", "Sebastian Beyvers <sb@pus.de>"]
edition = "2024"
repository = "https://github.com/arunaengine/pithos"
license = "MIT OR Apache-2.0"

[workspace.dependencies]
thiserror = "2.0.16"
tracing = "0.1.41"
tracing-subscriber = { version = "0.3.20", features = [
"env-filter",
"time",
"fmt",
"registry",
] }
x25519-dalek = { version = "2.0.1", features = ["static_secrets"] }
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@

## Description

Pithos (a large ancient greek storage container) is a packaging file format for arbitrary data that enhances the use of Object Storage for (research) data management. This is done by combining multiple existing file standards with new enhancements. A format specification can be found [[here]](./spec/SPECIFICATION.md).
Pithos (a large ancient greek storage container) is a packaging file format for arbitrary data that enhances the use of Object Storage for (research) data management. This is done by combining multiple existing file standards with new enhancements. A format specification can be found [[here]](./spec/PITHOS_1.0.0_draft.md).

## Features

Expand Down
34 changes: 9 additions & 25 deletions crates/pithos/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,33 +1,17 @@
[package]
name = "pithos"
description = "Client for the Pithos object storage file format"
version = "0.6.2"
description = "CLI for the Pithos container file format"
version = "0.7.0"
authors.workspace = true
edition.workspace = true
repository.workspace = true
license.workspace = true

[dependencies]
anyhow = "1.0.98"
async-channel = "2.3.1"
async-trait = "0.1.88"
base64 = "0.22.1"
borsh = { version = "1.5.7", features = ["std", "derive"] }
bytes = "1.10.1"
chacha20poly1305 = "0.10.1"
clap = { version = "4.5.38", features = ["derive"] }
crypto_kx = { version = "0.2.1", features = ["serde"] }
dotenvy = "0.15.7"
futures = "0.3.31"
futures-util = "0.3.31"
openssl = "0.10.72"
pithos_lib = { path = "../pithos_lib", version = "0.6.2" }
rand = "0.9.1"
serde_json = "1.0.140"
tokio = { version = "1.45.0", features = ["full"] }
tokio-stream = "0.1.17"
tokio-util = "0.7.15"
tracing = "0.1.41"
tracing-subscriber = { version = "0.3.19", features = ["env-filter", "time"] }
x25519-dalek = "2.0.1"
indicatif = "0.17.11"
clap = { version = "4.5.21", features = ["derive"] }
pithos_lib = { path = "../pithos_lib" }
thiserror = { workspace = true }
tracing = "0.1.40"
tracing-subscriber = {version = "0.3.18", features = ["env-filter", "time"]}
x25519-dalek = {workspace = true}
#indicatif = "0.17.9"
50 changes: 12 additions & 38 deletions crates/pithos/src/io/utils.rs
Original file line number Diff line number Diff line change
@@ -1,51 +1,25 @@
use anyhow::Result;
use openssl::pkey::{Id, PKey};
use pithos_lib::error::PithosError;
use std::fs::File;
use std::io::Read;
use std::path::PathBuf;
use x25519_dalek::{PublicKey, StaticSecret};

//TODO: Try to load openSSL pem and if fails try to load Crypt4GH pem
pub fn load_key_from_pem(filepath: &PathBuf, private: bool) -> Result<[u8; 32]> {
pub fn load_private_key_from_pem(filepath: &PathBuf) -> Result<StaticSecret, PithosError> {
// Open file handle and read file bytes
let mut file = File::open(filepath)?;
let mut file_content = vec![0u8; file.metadata()?.len() as usize];
file.read_exact(&mut file_content)?;

// Parse into key
let mut key_bytes: [u8; 32] = [0; 32];
if private {
key_bytes.copy_from_slice(
PKey::private_key_from_pem(&file_content)?
.raw_private_key()?
.as_slice(),
);
} else {
key_bytes.copy_from_slice(
PKey::public_key_from_pem(&file_content)?
.raw_public_key()?
.as_slice(),
);
}

Ok(key_bytes)
Ok(pithos_lib::helpers::x25519_keys::private_key_from_pem_bytes(&file_content)?)
}

pub fn _load_key_from_string(key_bytes: &[u8], private: bool) -> Result<[u8; 32]> {
// Parse into key
let mut x25519_key_bytes: [u8; 32] = [0; 32];
if private {
x25519_key_bytes.copy_from_slice(
PKey::private_key_from_raw_bytes(key_bytes, Id::X25519)?
.raw_private_key()?
.as_slice(),
);
} else {
x25519_key_bytes.copy_from_slice(
PKey::public_key_from_raw_bytes(key_bytes, Id::X25519)?
.raw_public_key()?
.as_slice(),
);
}
pub fn load_public_key_from_pem(filepath: &PathBuf) -> Result<PublicKey, PithosError> {
// Open file handle and read file bytes
let mut file = File::open(filepath)?;
let mut file_content = vec![0u8; file.metadata()?.len() as usize];
file.read_exact(&mut file_content)?;

Ok(x25519_key_bytes)
Ok(pithos_lib::helpers::x25519_keys::public_key_from_pem_bytes(
&file_content,
)?)
}
Loading
Loading