Commit 52c7d86
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
| |||
1 | 1 | | |
2 | 2 | | |
3 | 3 | | |
| 4 | + | |
Submodule dlio_benchmark updated 66 files
- .github/workflows/ci.yml+82-2
- README.md+124-3
- dlio_benchmark/__init__.py+6
- dlio_benchmark/checkpointing/checkpointing_factory.py+6-3
- dlio_benchmark/checkpointing/pytorch_checkpointing.py+134-21
- dlio_benchmark/checkpointing/pytorch_obj_store_checkpointing.py+302
- dlio_benchmark/checkpointing/pytorch_s3_checkpointing.py-106
- dlio_benchmark/checkpointing/simple_streaming_checkpointing.py+56
- dlio_benchmark/checkpointing/tf_checkpointing.py+23-10
- dlio_benchmark/common/enumerations.py+5-5
- dlio_benchmark/data_generator/csv_generator.py+37-28
- dlio_benchmark/data_generator/data_generator.py+108-1
- dlio_benchmark/data_generator/generator_factory.py+5-16
- dlio_benchmark/data_generator/hdf5_generator.py+42-26
- dlio_benchmark/data_generator/indexed_binary_generator.py+93-35
- dlio_benchmark/data_generator/jpeg_generator.py+19-20
- dlio_benchmark/data_generator/npy_generator.py+13-14
- dlio_benchmark/data_generator/npy_generator_s3.py-57
- dlio_benchmark/data_generator/npz_generator.py+22-14
- dlio_benchmark/data_generator/npz_generator_s3.py-59
- dlio_benchmark/data_generator/parquet_generator.py+223-202
- dlio_benchmark/data_generator/png_generator.py+19-20
- dlio_benchmark/data_generator/synthetic_generator.py+17-8
- dlio_benchmark/data_generator/tf_generator.py+49-22
- dlio_benchmark/framework/tf_framework.py+38-2
- dlio_benchmark/main.py+22-6
- dlio_benchmark/profiler/profiler_factory.py+6-5
- dlio_benchmark/reader/_s3_iterable_mixin.py+342
- dlio_benchmark/reader/csv_reader_s3_iterable.py+86
- dlio_benchmark/reader/hdf5_reader_s3_iterable.py+85
- dlio_benchmark/reader/image_reader_s3_iterable.py+103
- dlio_benchmark/reader/npy_reader_s3.py+1-1
- dlio_benchmark/reader/npy_reader_s3_iterable.py+105
- dlio_benchmark/reader/npz_reader_s3.py+1-1
- dlio_benchmark/reader/npz_reader_s3_iterable.py+74
- dlio_benchmark/reader/parquet_reader.py+131-265
- dlio_benchmark/reader/parquet_reader_s3_iterable.py+476
- dlio_benchmark/reader/reader_factory.py+45-4
- dlio_benchmark/reader/tfrecord_reader_s3_iterable.py+132
- dlio_benchmark/storage/aistore_storage.py+273
- dlio_benchmark/storage/file_storage.py+9-2
- dlio_benchmark/storage/obj_store_lib.py+490
- dlio_benchmark/storage/s3_torch_storage.py-183
- dlio_benchmark/storage/storage_factory.py+19-4
- dlio_benchmark/storage/storage_handler.py+7
- dlio_benchmark/utils/config.py+252-116
- dlio_benchmark/utils/utility.py+102-3
- docs/AIStore_Analysis.md+349
- docs/DLIO-Object-Storage_Analysis.md+107
- docs/DLIO_IO_Issues-Executive_Summary_2026-03-28.md+159
- docs/DLIO_IO_Issues-Proposal_2026-03-28.md+1.1k
- docs/data_generator_analysis.md+286
- docs/source/config.rst+69-1
- docs/source/contribute.rst+1-1
- docs/source/examples.rst+79
- docs/source/install.rst+9-1
- pyproject.toml+1-1
- requirements-test.txt+2
- requirements.txt+1
- setup.py+10-2
- tests/conftest.py+37
- tests/dlio_aistore_benchmark_test.py+394
- tests/dlio_benchmark_test.py+16-6
- tests/dlio_s3_benchmark_test.py+56-26
- tests/test_data_generator_improvements.py+678
- tests/test_s3dlio_object_store.py+400
0 commit comments