dev -> main pour version 3.0.0

.github/workflows/cicd_deploy.yml

@@ -29,12 +29,9 @@ jobs:
       uses: docker/build-push-action@v6
       with:
         context: .
-        file: ./Dockerfile.pdal
+        file: ./Dockerfile
         load: true
         tags: ${{ env.TEST_TAG }}
-        build-args: |
-          GITHUB_REPOSITORY=alavenant/PDAL
-          GITHUB_SHA=master_28_05_25
 
     # run the test on the docker image
     - name: Run tests in docker image
@@ -107,11 +104,11 @@ jobs:
 
       - name: Run tests with pytest
         shell: micromamba-shell {0}
-        run: python -m pytest ./test -s --log-cli-level DEBUG -m "not pdal_custom"
+        run: python -m pytest ./test -s --log-cli-level DEBUG
 
       - name: Build pip package
         shell: micromamba-shell {0}
         run: make build
 
       - name: pypi-publish
-        uses: pypa/gh-action-pypi-publish@v1.12.4
+        uses: pypa/gh-action-pypi-publish@v1.13.0

.github/workflows/cicd_full.yml

@@ -34,12 +34,9 @@ jobs:
       uses: docker/build-push-action@v6
       with:
         context: .
-        file: ./Dockerfile.pdal
+        file: ./Dockerfile
         load: true
         tags: ${{ env.TEST_TAG }}
-        build-args: |
-          GITHUB_REPOSITORY=alavenant/PDAL
-          GITHUB_SHA=master_28_05_25
 
     # run the test on the docker image
     - name: Run tests in docker image
@@ -77,6 +74,6 @@ jobs:
         shell: micromamba-shell {0}
         run: >
             python -m pytest -s
-            --log-cli-level=DEBUG -m "not pdal_custom"
+            --log-cli-level=DEBUG
             --log-format="%(asctime)s %(levelname)s %(message)s"
             --log-date-format="%Y-%m-%d %H:%M:%S"

.github/workflows/cicd_light.yml

@@ -32,7 +32,7 @@ jobs:
         shell: micromamba-shell {0}
         run: >
           python -m pytest -s
-          --log-cli-level=DEBUG -m "not pdal_custom"
+          --log-cli-level=DEBUG
           --log-format="%(asctime)s %(levelname)s %(message)s"
           --log-date-format="%Y-%m-%d %H:%M:%S"
 

.gitignore

@@ -4,6 +4,7 @@ __pycache__
 test/tmp/
 
 *.log
+*.copc.laz
 
 # Library deployment
 *.egg-info

CHANGELOG.md

@@ -1,9 +1,15 @@
+# dev
+
+# v3.0.0
+- use pdal>=2.9 and deactivate the Dockerfile.pdal from CI
+- [BREAKING CHANGE] Add a [main.py](las_digital_models/main.py) to run the whole pipeline at once (buffer, DTM, DSM, DHM). Use temporary folders for intermediate values. Refactor config files.
+
 # v2.1.1
-fix sur le déploiement de l'image Docker
+fix CI to deploy docker image
 
 # v2.1.0
 Custom PDAL: in the docker image, compile custom PDAL (waiting for PDAL 2.9)
-fix run_extract_z_virtual_lines_from_raster: output geometries are only LineString (no more MultiLineString) 
+fix run_extract_z_virtual_lines_from_raster: output geometries are only LineString (no more MultiLineString)
 
 # v2.0.0
 Rename produit_derive_lidar to las_digital_models

Dockerfile.pdal

@@ -1,3 +1,11 @@
+## Note :
+## This Dockerfile allows us to build a PDAL image with the custom fork and branch
+## It's used when it's necessary to use a specific version of PDAL 
+## typicaly when some features are not available in the conda-forge channel
+## the PDAL version is specified by variables GITHUB_SERVER_URL and GITHUB_REPOSITORY
+## and the branch by GITHUB_SHA
+##
+
 # code from https://github.com/PDAL/PDAL/blob/master/scripts/docker/ubuntu/Dockerfile
 FROM condaforge/mambaforge:latest AS mamba_pdal
 

Makefile

@@ -54,21 +54,20 @@ testing:
 REGISTRY=ghcr.io
 NAMESPACE=ignf
 IMAGE_NAME=las-digital-models
+CUSTOM_PDAL_SHA=master_28_05_25
+CUSTOM_PDAL_REPOSITORY=alavenant/PDAL
 VERSION=`python -m las_digital_models.version`
 FULL_IMAGE_NAME=${REGISTRY}/${NAMESPACE}/${IMAGE_NAME}:${VERSION}
 
 docker-build:
 	docker build -t ${IMAGE_NAME}:${VERSION} -f Dockerfile .
 
-docker-build-pdal: clean
-	docker build --build-arg GITHUB_REPOSITORY=alavenant/PDAL --build-arg GITHUB_SHA=master_28_05_25 -t ${IMAGE_NAME}:${VERSION} -f Dockerfile.pdal .
+docker-build-custum-pdal: clean
+	docker build --build-arg GITHUB_REPOSITORY=${CUSTOM_PDAL_REPOSITORY} --build-arg GITHUB_SHA=${CUSTOM_PDAL_SHA} -t ${IMAGE_NAME}:${VERSION} -f Dockerfile.pdal .
 
 docker-test-pdal-version: clean
 	docker run --rm  -t ${IMAGE_NAME}:${VERSION} pdal --version
 
-docker-test-pdal-custom: clean
-	docker run --rm  -t ${IMAGE_NAME}:${VERSION} python -m pytest -s -m "pdal_custom"
-
 docker-test:
 	docker run --rm ${IMAGE_NAME}:${VERSION} python -m pytest -s -m "not functional_test"
 

README.md

@@ -124,7 +124,7 @@ the `configs` folder.
 > * `tile_geometry.tile_width` must contain the tile size in meters
 > * `tile_geometry.tile_coord_scale` must contain the `coord_x` and `coord_y` scale so that `coord_{x or y} * tile_geometry.tile_coord_scale` are the coordinates of the top-left corner in meters
 
-### Whole pipeline
+### Whole pipeline on a folder
 
 To run the whole pipeline (DSM + DTM + DHM) on all the LAS files in a folder, use `run.sh`.
 
@@ -147,6 +147,24 @@ It will generate:
   * ${OUTPUT_DIR}/DSM
   * ${OUTPUT_DIR}/DHM
 
+### Whole pipeline on a single file
+
+To run the whole pipeline (buffer + DTM + DSM + DHM) on a single file:
+
+```bash
+python -m las_digital_models.main \
+  io.input_dir=INPUT_DIR \
+  io.input_filename=INPUT_FILENAME \
+  io.output_dir=OUTPUT_DIR \
+  tile_geometry.pixel_size=${PIXEL_SIZE}
+  buffer.size=10
+```
+Any of DTM, DSM or DHM computation can be deactivated using `tasks.dtm=false`, `tasks.dsm=false`
+or `tasks.dhm=false`.
+
+Any other parameter in the `./configs` tree can be overriden in the command (see the doc of
+[hydra](https://hydra.cc/) for more details on usage)
+
 ### Buffer
 
 To add a buffer to a point cloud using `ign-pdal-tools`:
@@ -155,8 +173,7 @@ To add a buffer to a point cloud using `ign-pdal-tools`:
 python -m las_digital_models.filter_one_tile \
   io.input_dir=INPUT_DIR \
   io.input_filename=INPUT_FILENAME \
-  io.output_dir=OUTPUT_DIR \
-  buffer.size=10
+  io.output_dir=OUTPUT_DIR
 ```
 
 Any other parameter in the `./configs` tree can be overriden in the command (see the doc of
@@ -172,8 +189,8 @@ python -m las_digital_models.ip_one_tile \
     io.input_filename={} \
     io.output_dir=${DXM_DIR} \
     tile_geometry.pixel_size=${PIXEL_SIZE} \
-    filter.dimension="Classification" \
-    filter.keep_values=[2,66]
+    interpolation.custom.filter.dimension="Classification" \
+    interpolation.custom.filter.keep_values=[2,66]
 ```
 
 `filter.keep_values` must be a list inside `[]`, separated by `,` without spaces.

configs/buffer/default.yaml

@@ -1 +1,3 @@
-size: 100  # bufer size in meters
+size: 100  # buffer size in meters
+
+output_subdir: null  # subdirectory of oi.output_dir in which to save files with buffer, for debug only

configs/buffer/test.yaml

@@ -1 +1,3 @@
-size: 10
+size: 10
+
+output_subdir: null  # subdirectory of oi.output_dir in which to save files with buffer, for debug only

configs/config.yaml

@@ -6,15 +6,19 @@
 # learn more here: https://hydra.cc/docs/next/tutorials/basic/running_your_app/working_directory
 work_dir: ${hydra:runtime.cwd}
 
+tasks:
+  dtm: true
+  dsm: true
+  dhm: true
+
 # specify here default training configuration
 defaults:
   - _self_  # for hydra legacy reasons
-
   - buffer: default.yaml
-  - filter: dtm.yaml
+  - interpolation: default.yaml
+  - dhm: default.yaml
   - io: default.yaml
   - tile_geometry: default.yaml  # describes input features and classes
-  - dhm: default.yaml
   - extract_stat: default.yaml
 
   # disable hydra logging

configs/dhm/default.yaml

@@ -1,2 +1,9 @@
-input_dsm_dir: /path/to/dsm/dir
-input_dtm_dir: /path/to/dtm/dir
+output_subfolder: "DHM"
+
+# To be used only when using dhm_one_tile directly (standalone mode) to run
+# DHM from any input DTM and DSM
+# DSM and DTM are associated to the tile by their names, which should be like:
+#  f"{input filename without extension}_{pixel size with unit}"
+input_dtm_dir: /path/to/input/dtm/folder
+input_dsm_dir: /path/to/input/dsm/folder
+

configs/dhm/test.yaml

@@ -1,2 +1,9 @@
+output_subfolder: "DHM"
+
+# To be used only when using dhm_one_tile directly (standalone mode) to run
+# DHM from any input DTM and DSM
+# DSM and DTM are associated to the tile by their names, which should be like:
+#  f"{input filename without extension}_{pixel size with unit}"
+input_dtm_dir: ./test/data/DTM
 input_dsm_dir: ./test/data/DSM
-input_dtm_dir: ./test/data/DTM
+

configs/interpolation/default.yaml

@@ -0,0 +1,19 @@
+dsm:
+  filter:
+    dimension: Classification
+    keep_values: [2, 3, 4, 5, 6, 9, 17]   # Classes to keep when running DSM
+  output_subfolder: "DSM"
+
+dtm:
+  filter:
+    dimension: Classification
+    keep_values: [2, 9, 66]   # Classes to keep when running DTM
+  output_subfolder: "DTM"
+
+
+# To be used only when using ip_one_tile directly (standalone mode) to run
+# interpolation on any kind of filtered data
+custom:
+  filter:
+    dimension: Classification
+    keep_values: [2, 9, 66]   # Classes to keep when running ip_one_tile

configs/interpolation/test.yaml

@@ -0,0 +1,19 @@
+dsm:
+  filter:
+    dimension: Classification
+    keep_values: [2, 3, 4, 5, 6, 9, 17]   # Classes to keep when running DSM
+  output_subfolder: "DSM"
+
+dtm:
+  filter:
+    dimension: Classification
+    keep_values: [2, 9, 66]   # Classes to keep when running DTM
+  output_subfolder: "DTM"
+
+
+# To be used only when using ip_one_tile directly (standalone mode) to run
+# interpolation on any kind of filtered data
+custom:
+  filter:
+    dimension: Classification
+    keep_values: [2, 66]   # ground + virtual points

configs/io/default.yaml

@@ -6,13 +6,14 @@ input_filename: filename.las
 # Shapefile used to burn no-data value (inside its polygons)
 no_data_mask_shapefile: null  # /path/to/mask.shp
 
-# Force input/output files extension to .las or .laz. If None or not set, use original extension
-# Used for output for filter
-# Used for input + output in buffer
-# Use for input in interpolation
-forced_intermediate_ext: null  # can be "las", "laz" or null. (intermediate results extension)
-
 # Spatial reference to use to override the one from input las.
 spatial_reference: EPSG:2154
 
-output_dir: /path/to/output/folder  # Directory folder for saving the outputs
+output_dir: /path/to/output/folder  # Directory folder for saving the outputs
+
+# To be used only when using dhm_one_tile directly (standalone mode) to run
+# DHM from any input DTM and DSM
+# DSM and DTM are associated to the tile by their names, which should be like:
+#  f"{input filename without extension}_{pixel size with unit}"
+input_dtm_dir: /path/to/input/dtm/folder
+input_dsm_dir: /path/to/input/dsm/folder

configs/io/test.yaml

@@ -6,4 +6,11 @@ forced_intermediate_ext: null
 
 spatial_reference: EPSG:2154
 
-output_dir: ./test/tmp/hydra_ip
+output_dir: ./test/tmp/hydra_ip
+
+# To be used only when using dhm_one_tile directly (standalone mode) to run
+# DHM from any input DTM and DSM
+# DSM and DTM are associated to the tile by their names, which should be like:
+#  f"{input filename without extension}_{pixel size with unit}"
+input_dsm_dir: ./test/data/DSM
+input_dtm_dir: ./test/data/DTM

configs/test.yaml

@@ -3,14 +3,19 @@
 # Special config used for tests (test_run_script.py)
 work_dir: ${hydra:runtime.cwd}
 
+tasks:
+  dtm: true
+  dsm: true
+  dhm: true
+
 defaults:
   - _self_  # for hydra legacy reasons
-
   - buffer: test.yaml
-  - filter: default_test.yaml
+  - interpolation: test.yaml
+  - dhm: test.yaml
   - io: test.yaml
   - tile_geometry: test.yaml  # describes input features and classes
-  - dhm: test.yaml
+  - extract_stat: default.yaml
 
   # disable hydra logging
   - override hydra/hydra_logging: disabled

environment.yml

@@ -6,16 +6,13 @@ dependencies:
   - python==3.10.*
   - pip
   - pytest
-  - cgal
   - gdal
   - laspy
   - numpy
-  - scipy
   - fiona
-  - rasterio
   - pyproj
-  - pdal>=2.6
-  - python-pdal>=3.2.1
+  - pdal>=2.9.0
+  - python-pdal>=3.4.0
   - geopandas
   - parallel
     # --------- hydra configs --------- #
@@ -32,4 +29,4 @@ dependencies:
   - pip:
       - ign-pdal-tools
       - rasterstats
-
+      - rasterio # moved here since pdal>=2.9.0 force modern libs to be used and make dependancy graph non resolved