Refactor: add main to run all at once (with temporary files)

.gitignore

@@ -4,6 +4,7 @@ __pycache__
 test/tmp/
 
 *.log
+*.copc.laz
 
 # Library deployment
 *.egg-info

CHANGELOG.md

@@ -1,12 +1,13 @@
 # dev
-use pdal>=2.9 and deactivate the Dockerfile.pdal from CI
+- use pdal>=2.9 and deactivate the Dockerfile.pdal from CI
+- [BREAKING CHANGE] Add a [main.py](las_digital_models/main.py) to run the whole pipeline at once (buffer, DTM, DSM, DHM). Use temporary folders for intermediate values. Refactor config files.
 
 # v2.1.1
-fix sur le déploiement de l'image Docker
+fix CI to deploy docker image
 
 # v2.1.0
 Custom PDAL: in the docker image, compile custom PDAL (waiting for PDAL 2.9)
-fix run_extract_z_virtual_lines_from_raster: output geometries are only LineString (no more MultiLineString) 
+fix run_extract_z_virtual_lines_from_raster: output geometries are only LineString (no more MultiLineString)
 
 # v2.0.0
 Rename produit_derive_lidar to las_digital_models

README.md

@@ -124,7 +124,7 @@ the `configs` folder.
 > * `tile_geometry.tile_width` must contain the tile size in meters
 > * `tile_geometry.tile_coord_scale` must contain the `coord_x` and `coord_y` scale so that `coord_{x or y} * tile_geometry.tile_coord_scale` are the coordinates of the top-left corner in meters
 
-### Whole pipeline
+### Whole pipeline on a folder
 
 To run the whole pipeline (DSM + DTM + DHM) on all the LAS files in a folder, use `run.sh`.
 
@@ -147,6 +147,24 @@ It will generate:
   * ${OUTPUT_DIR}/DSM
   * ${OUTPUT_DIR}/DHM
 
+### Whole pipeline on a single file
+
+To run the whole pipeline (buffer + DTM + DSM + DHM) on a single file:
+
+```bash
+python -m las_digital_models.main \
+  io.input_dir=INPUT_DIR \
+  io.input_filename=INPUT_FILENAME \
+  io.output_dir=OUTPUT_DIR \
+  tile_geometry.pixel_size=${PIXEL_SIZE}
+  buffer.size=10
+```
+Any of DTM, DSM or DHM computation can be deactivated using `tasks.dtm=false`, `tasks.dsm=false`
+or `tasks.dhm=false`.
+
+Any other parameter in the `./configs` tree can be overriden in the command (see the doc of
+[hydra](https://hydra.cc/) for more details on usage)
+
 ### Buffer
 
 To add a buffer to a point cloud using `ign-pdal-tools`:
@@ -155,8 +173,7 @@ To add a buffer to a point cloud using `ign-pdal-tools`:
 python -m las_digital_models.filter_one_tile \
   io.input_dir=INPUT_DIR \
   io.input_filename=INPUT_FILENAME \
-  io.output_dir=OUTPUT_DIR \
-  buffer.size=10
+  io.output_dir=OUTPUT_DIR
 ```
 
 Any other parameter in the `./configs` tree can be overriden in the command (see the doc of
@@ -172,8 +189,8 @@ python -m las_digital_models.ip_one_tile \
     io.input_filename={} \
     io.output_dir=${DXM_DIR} \
     tile_geometry.pixel_size=${PIXEL_SIZE} \
-    filter.dimension="Classification" \
-    filter.keep_values=[2,66]
+    interpolation.custom.filter.dimension="Classification" \
+    interpolation.custom.filter.keep_values=[2,66]
 ```
 
 `filter.keep_values` must be a list inside `[]`, separated by `,` without spaces.

configs/buffer/default.yaml

@@ -1 +1,3 @@
-size: 100  # bufer size in meters
+size: 100  # buffer size in meters
+
+output_subdir: null  # subdirectory of oi.output_dir in which to save files with buffer, for debug only

configs/buffer/test.yaml

@@ -1 +1,3 @@
-size: 10
+size: 10
+
+output_subdir: null  # subdirectory of oi.output_dir in which to save files with buffer, for debug only

configs/config.yaml

@@ -6,15 +6,19 @@
 # learn more here: https://hydra.cc/docs/next/tutorials/basic/running_your_app/working_directory
 work_dir: ${hydra:runtime.cwd}
 
+tasks:
+  dtm: true
+  dsm: true
+  dhm: true
+
 # specify here default training configuration
 defaults:
   - _self_  # for hydra legacy reasons
-
   - buffer: default.yaml
-  - filter: dtm.yaml
+  - interpolation: default.yaml
+  - dhm: default.yaml
   - io: default.yaml
   - tile_geometry: default.yaml  # describes input features and classes
-  - dhm: default.yaml
   - extract_stat: default.yaml
 
   # disable hydra logging

configs/dhm/default.yaml

@@ -1,2 +1,9 @@
-input_dsm_dir: /path/to/dsm/dir
-input_dtm_dir: /path/to/dtm/dir
+output_subfolder: "DHM"
+
+# To be used only when using dhm_one_tile directly (standalone mode) to run
+# DHM from any input DTM and DSM
+# DSM and DTM are associated to the tile by their names, which should be like:
+#  f"{input filename without extension}_{pixel size with unit}"
+input_dtm_dir: /path/to/input/dtm/folder
+input_dsm_dir: /path/to/input/dsm/folder
+

configs/dhm/test.yaml

@@ -1,2 +1,9 @@
+output_subfolder: "DHM"
+
+# To be used only when using dhm_one_tile directly (standalone mode) to run
+# DHM from any input DTM and DSM
+# DSM and DTM are associated to the tile by their names, which should be like:
+#  f"{input filename without extension}_{pixel size with unit}"
+input_dtm_dir: ./test/data/DTM
 input_dsm_dir: ./test/data/DSM
-input_dtm_dir: ./test/data/DTM
+

configs/interpolation/default.yaml

@@ -0,0 +1,19 @@
+dsm:
+  filter:
+    dimension: Classification
+    keep_values: [2, 3, 4, 5, 6, 9, 17]   # Classes to keep when running DSM
+  output_subfolder: "DSM"
+
+dtm:
+  filter:
+    dimension: Classification
+    keep_values: [2, 9, 66]   # Classes to keep when running DTM
+  output_subfolder: "DTM"
+
+
+# To be used only when using ip_one_tile directly (standalone mode) to run
+# interpolation on any kind of filtered data
+custom:
+  filter:
+    dimension: Classification
+    keep_values: [2, 9, 66]   # Classes to keep when running ip_one_tile

configs/interpolation/test.yaml

@@ -0,0 +1,19 @@
+dsm:
+  filter:
+    dimension: Classification
+    keep_values: [2, 3, 4, 5, 6, 9, 17]   # Classes to keep when running DSM
+  output_subfolder: "DSM"
+
+dtm:
+  filter:
+    dimension: Classification
+    keep_values: [2, 9, 66]   # Classes to keep when running DTM
+  output_subfolder: "DTM"
+
+
+# To be used only when using ip_one_tile directly (standalone mode) to run
+# interpolation on any kind of filtered data
+custom:
+  filter:
+    dimension: Classification
+    keep_values: [2, 66]   # ground + virtual points

configs/io/default.yaml

@@ -6,13 +6,14 @@ input_filename: filename.las
 # Shapefile used to burn no-data value (inside its polygons)
 no_data_mask_shapefile: null  # /path/to/mask.shp
 
-# Force input/output files extension to .las or .laz. If None or not set, use original extension
-# Used for output for filter
-# Used for input + output in buffer
-# Use for input in interpolation
-forced_intermediate_ext: null  # can be "las", "laz" or null. (intermediate results extension)
-
 # Spatial reference to use to override the one from input las.
 spatial_reference: EPSG:2154
 
-output_dir: /path/to/output/folder  # Directory folder for saving the outputs
+output_dir: /path/to/output/folder  # Directory folder for saving the outputs
+
+# To be used only when using dhm_one_tile directly (standalone mode) to run
+# DHM from any input DTM and DSM
+# DSM and DTM are associated to the tile by their names, which should be like:
+#  f"{input filename without extension}_{pixel size with unit}"
+input_dtm_dir: /path/to/input/dtm/folder
+input_dsm_dir: /path/to/input/dsm/folder

configs/io/test.yaml

@@ -6,4 +6,11 @@ forced_intermediate_ext: null
 
 spatial_reference: EPSG:2154
 
-output_dir: ./test/tmp/hydra_ip
+output_dir: ./test/tmp/hydra_ip
+
+# To be used only when using dhm_one_tile directly (standalone mode) to run
+# DHM from any input DTM and DSM
+# DSM and DTM are associated to the tile by their names, which should be like:
+#  f"{input filename without extension}_{pixel size with unit}"
+input_dsm_dir: ./test/data/DSM
+input_dtm_dir: ./test/data/DTM

configs/test.yaml

@@ -3,14 +3,19 @@
 # Special config used for tests (test_run_script.py)
 work_dir: ${hydra:runtime.cwd}
 
+tasks:
+  dtm: true
+  dsm: true
+  dhm: true
+
 defaults:
   - _self_  # for hydra legacy reasons
-
   - buffer: test.yaml
-  - filter: default_test.yaml
+  - interpolation: test.yaml
+  - dhm: test.yaml
   - io: test.yaml
   - tile_geometry: test.yaml  # describes input features and classes
-  - dhm: test.yaml
+  - extract_stat: default.yaml
 
   # disable hydra logging
   - override hydra/hydra_logging: disabled

environment.yml

@@ -6,11 +6,9 @@ dependencies:
   - python==3.10.*
   - pip
   - pytest
-  - cgal
   - gdal
   - laspy
   - numpy
-  - scipy
   - fiona
   - pyproj
   - pdal>=2.9.0
@@ -31,5 +29,4 @@ dependencies:
   - pip:
       - ign-pdal-tools
       - rasterstats
-      # - rasterio # moved here since pdal>=2.9.0 force modern libs to be used and make dependancy graph non resolved
-
+      - rasterio # moved here since pdal>=2.9.0 force modern libs to be used and make dependancy graph non resolved

las_digital_models/dhm_one_tile.py

@@ -12,26 +12,41 @@
 log = commons.get_logger(__name__)
 
 
-@hydra.main(config_path="../configs/", config_name="config.yaml", version_base="1.2")
-def run_dhm_on_tile(config: DictConfig):
-    os.makedirs(config.io.output_dir, exist_ok=True)
-    tilename, _ = os.path.splitext(config.io.input_filename)
+def run_dhm_on_tile(
+    input_las_filename: str,
+    input_dtm_dir: str,
+    input_dsm_dir: str,
+    output_dir: str,
+    pixel_size: float,
+    no_data_value: float,
+):
+    os.makedirs(output_dir, exist_ok=True)
+    tilename, _ = os.path.splitext(input_las_filename)
 
     # for export
-    _size = commons.give_name_resolution_raster(config.tile_geometry.pixel_size)
+    _size = commons.give_name_resolution_raster(pixel_size)
     geotiff_filename = f"{tilename}{_size}.tif"
-    geotiff_dsm = os.path.join(config.dhm.input_dsm_dir, geotiff_filename)
-    geotiff_dtm = os.path.join(config.dhm.input_dtm_dir, geotiff_filename)
-    geotiff_output = os.path.join(config.io.output_dir, geotiff_filename)
+    geotiff_dsm = os.path.join(input_dsm_dir, geotiff_filename)
+    geotiff_dtm = os.path.join(input_dtm_dir, geotiff_filename)
+    geotiff_output = os.path.join(output_dir, geotiff_filename)
     # process
-    calculate_dhm(geotiff_dsm, geotiff_dtm, geotiff_output, no_data_value=config.tile_geometry.no_data_value)
+    calculate_dhm(geotiff_dsm, geotiff_dtm, geotiff_output, no_data_value=no_data_value)
 
     return
 
 
-def main():
+@hydra.main(config_path="../configs/", config_name="config.yaml", version_base="1.2")
+def main(config: DictConfig):
     logging.basicConfig(level=logging.INFO)
-    run_dhm_on_tile()
+
+    run_dhm_on_tile(
+        input_las_filename=config.io.input_filename,
+        input_dtm_dir=config.dhm.input_dtm_dir,
+        input_dsm_dir=config.dhm.input_dsm_dir,
+        output_dir=config.io.output_dir,
+        pixel_size=config.tile_geometry.pixel_size,
+        no_data_value=config.tile_geometry.no_data_value,
+    )
 
 
 if __name__ == "__main__":