Import Monitoring

.github/workflows/pytest.yml

@@ -8,23 +8,23 @@ on:
       - main
       - hotfixes
       - develop
-      - improve-module-part-indivuduals
+      - feat/import-monitoring
+      - reorganize-import-install
   pull_request:
     paths:
       - "backend/**"
     branches:
       - main
       - hotfixes
       - develop
-      - improve-module-part-indivuduals
 
 jobs:
   build:
     strategy:
       fail-fast: false
       matrix:
-        geonature_ref: ["develop"] # Mettre version compatible
-    uses: pnx-si/geonature/.github/workflows/gn-module-pytest.yml@develop
+        geonature_ref: ["master"] # Mettre version compatible
+    uses: pnx-si/geonature/.github/workflows/gn-module-pytest.yml@master
     with:
       geonature_ref: ${{ matrix.geonature_ref }}
       upload_coverage: true

README.md

@@ -256,5 +256,7 @@ Les sites et groupes de sites peuvent être associés à plusieurs protocoles (s
 
 * [Suivi d'individus (CMR)](docs/individuals.md)
 * [Gestion de la synthèse](docs/synthese.md)
+* [Import de fichiers CSV](docs/import.md)
 * [Documentation technique](docs/documentation_technique.md)
 * [Liste des commandes](docs/commandes.md)
+

backend/gn_module_monitoring/command/cmd.py

@@ -1,26 +1,30 @@
 import click
 
-from pathlib import Path
 from flask.cli import with_appcontext
-from sqlalchemy.sql import text, select
+from sqlalchemy.sql import select
+from sqlalchemy import exists
 
 from geonature.utils.env import DB
 from geonature.core.gn_synthese.models import TSources
 from geonature.core.gn_synthese.utils.process import import_from_table
 from geonature.core.gn_commons.models import TModules
+from geonature.core.imports.models import Destination
 
-from gn_module_monitoring.config.repositories import get_config
-from gn_module_monitoring.config.utils import monitoring_module_config_path
 from gn_module_monitoring.monitoring.models import TMonitoringModules
 from gn_module_monitoring.modules.repositories import get_simple_module
-
+from gn_module_monitoring.config.repositories import get_config
+from gn_module_monitoring.config.utils import monitoring_module_config_path
+from gn_module_monitoring.command.nomenclature import add_nomenclature
+from gn_module_monitoring.command.permissions import process_available_permissions
+from gn_module_monitoring.command.sql import process_sql_files
 from gn_module_monitoring.command.utils import (
-    process_available_permissions,
-    remove_monitoring_module,
-    add_nomenclature,
     available_modules,
     installed_modules,
-    process_sql_files,
+    is_module_configured,
+    process_module_import,
+    process_update_module_import,
+    remove_monitoring_module,
+    validate_json_file_protocol,
 )
 
 
@@ -57,7 +61,6 @@ def cmd_install_monitoring_module(module_code):
                 où se situe les fichiers de configuration du module
         - module_code (str): code du module (par defaut la dernière partie de module_config_dir_path )
     """
-
     # module_config_dir_path = Path(module_config_dir_path)
     # module_code = module_code or module_config_dir_path.name
 
@@ -81,30 +84,29 @@ def cmd_install_monitoring_module(module_code):
             )
         return
 
-    click.secho(f"Installation du sous-module monitoring {module_code}")
+    success, errors = validate_json_file_protocol(module_code)
+    if not success:
+        click.echo("Erreurs détectées dans les fichiers de configuration:")
+        for error in errors:
+            click.echo(f"- {error}")
+        click.echo("Installation annulée")
+        return
 
     module_monitoring = get_simple_module("module_code", "MONITORINGS")
 
-    try:
-        module = get_simple_module("module_code", module_code)
-        # test si le module existe
-        if module:
-            click.secho(f"Le module {module_code} existe déjà", fg="red")
-            return
-    except Exception:
-        pass
-
-    # process Synthese
-    process_sql_files(dir=None, module_code=module_code, depth=1)
-    # process Exports
-    process_sql_files(dir=None, module_code=module_code, depth=None, allowed_files=None)
-
     config = get_config(module_code, force=True)
 
     if not config:
         click.secho(f"config directory for module {module_code} does not exist", fg="red")
         return None
 
+    click.secho(f"Installation du sous-module monitoring {module_code}")
+
+    # process Synthese
+    process_sql_files(dir=None, module_code=module_code, depth=1)
+    # process Exports
+    process_sql_files(dir=None, module_code=module_code, depth=None, allowed_files=None)
+
     module_desc = config["module"].get("module_desc")
     module_label = config["module"].get("module_label")
     synthese_object = (
@@ -130,10 +132,13 @@ def cmd_install_monitoring_module(module_code):
     }
 
     click.secho("ajout du module {} en base".format(module_code))
-    module = TMonitoringModules()
-    module.from_dict(module_data)
-    DB.session.add(module)
-    DB.session.commit()
+    if not DB.session.scalar(
+        exists(TMonitoringModules).where(TMonitoringModules.module_code == module_code).select()
+    ):
+        module = TMonitoringModules()
+        module.from_dict(module_data)
+        DB.session.add(module)
+        DB.session.commit()
 
     # Ajouter les permissions disponibles
     process_available_permissions(module_code, session=DB.session)
@@ -159,11 +164,67 @@ def cmd_install_monitoring_module(module_code):
     DB.session.add(source)
     DB.session.commit()
 
-    # TODO ++++ create specific tables
     click.secho(f"Sous-module monitoring '{module_code}' installé", fg="green")
     return
 
 
+@click.command("process_import")
+@click.argument("module_code", type=str, required=True)
+@with_appcontext
+def cmd_add_update_import_on_protocole(module_code):
+    module_code_installed = [module["module_code"] for module in installed_modules()]
+    if not module_code in module_code_installed:
+        raise KeyError(
+            f"Le module {module_code} n'est pas installé. Pour pouvoir importer dans ce protocole, vous devez d'abord l'installer !"
+        )
+
+    config = get_config(module_code, force=True)
+
+    if not is_module_configured(module_code):
+        return
+
+    module_monitoring = get_simple_module("module_code", "MONITORINGS")
+    module_data = {
+        "module_picto": "fa-puzzle-piece",
+        **config["module"],
+        "module_code": module_code,
+        "module_path": "{}/module/{}".format(module_monitoring.module_path, module_code),
+        "active_frontend": False,
+        "active_backend": False,
+        "type": "monitoring_module",
+    }
+
+    try:
+        module = get_simple_module("module_code", module_code)
+        destination_exists = DB.session.scalar(
+            exists(Destination).where(Destination.code == module_code).select()
+        )
+        # Vérifier si le module existe
+        if module and destination_exists:
+            # Effectuer une mise à jour
+            try:
+                click.secho(f"Mise à jour du module {module_code}")
+                state = process_update_module_import(config, module_code)
+                if state is None:
+                    click.secho(f"Le module {module_code} est déjà à jour", fg="yellow")
+                    return
+                if state:
+                    click.secho(f"Module {module_code} mis à jour", fg="green")
+                else:
+                    click.secho(
+                        f"La mise à jour du module {module_code} a était annulée", fg="red"
+                    )
+            except Exception:
+                return
+            return
+    except Exception:
+        pass
+
+    # Ajouter les destinations disponibles
+    process_module_import(module_data)
+    DB.session.commit()
+
+
 @click.command("update_module_available_permissions")
 @click.argument("module_code", required=False, default="")
 @with_appcontext
@@ -239,4 +300,5 @@ def synchronize_synthese(module_code, offset):
     cmd_add_module_nomenclature_cli,
     cmd_process_sql,
     synchronize_synthese,
+    cmd_add_update_import_on_protocole,
 ]

backend/gn_module_monitoring/command/imports/constant.py

@@ -0,0 +1,76 @@
+TYPE_WIDGET = {
+    "select": "varchar",
+    "checkbox": "varchar[]",
+    "radio": "varchar",
+    "html": "text",
+    "bool_checkbox": "boolean",
+    "number": "integer",
+    "multiselect": "varchar[]",
+    "observers": "integer[]",
+    "media": "varchar",
+    "medias": "varchar[]",
+    "date": "date",
+    "nomenclature": "integer",
+    "datalist": "integer",
+    "text": "varchar",
+    "textarea": "text",
+    "integer": "integer",
+    "jsonb": "jsonb",
+    "time": "varchar",
+    "taxonomy": "integer",
+    "site": "integer",
+    "individuals": "integer",
+}
+
+FORBIDDEN_SQL_INSTRUCTION = [
+    "INSERT ",
+    "DELETE ",
+    "UPDATE ",
+    "EXECUTE ",
+    "TRUNCATE ",
+    "ALTER ",
+    "GRANT ",
+    "COPY ",
+    "PERFORM ",
+    "CASCADE",
+]
+
+PERMISSION_LABEL = {
+    "MONITORINGS_MODULES": {"label": "modules", "actions": ["R", "U", "E"]},
+    "MONITORINGS_GRP_SITES": {"label": "groupes de sites", "actions": ["C", "R", "U", "D"]},
+    "MONITORINGS_SITES": {"label": "sites", "actions": ["C", "R", "U", "D"]},
+    "MONITORINGS_VISITES": {"label": "visites", "actions": ["C", "R", "U", "D"]},
+    "MONITORINGS_INDIVIDUALS": {"label": "individus", "actions": ["C", "R", "U", "D"]},
+    "MONITORINGS_MARKINGS": {"label": "marquages", "actions": ["C", "R", "U", "D"]},
+}
+
+ACTION_LABEL = {
+    "C": "Créer des",
+    "R": "Voir les",
+    "U": "Modifier les",
+    "D": "Supprimer des",
+    "E": "Exporter les",
+}
+
+TABLE_NAME_SUBMODULE = {
+    "sites_group": "t_sites_groups",
+    "site": "t_base_sites",
+    "visit": "t_base_visits",
+    "observation": "t_observations",
+    "observation_detail": "t_observations_details",
+}
+
+UUID_FIELD_NAME = {
+    "sites_group": "uuid_sites_group",
+    "site": "uuid_base_site",
+    "visit": "uuid_base_visit",
+    "observation": "uuid_observation",
+    "observation_detail": "uuid_observation_detail",
+}
+
+TOOLTIPS = {
+    "id_base_site_origin": "Identifiant alphanumérique permettant de faire le lien entre les sites et leurs visites si aucun UUID est fourni",
+    "id_base_visit_origin": "Identifiant alphanumérique permettant de faire le lien entre les visites et leurs observations si aucun UUID est fourni",
+}
+
+ENTITIES_NOT_AVAILABLE = ["sites_group", "observation_detail"]

backend/gn_module_monitoring/command/imports/destination.py

@@ -0,0 +1,61 @@
+from gn_module_monitoring.command.imports.utils import destination_name
+from sqlalchemy import select, exists
+import sqlalchemy as sa
+
+
+from geonature.utils.env import db
+from geonature.core.gn_commons.models import TModules
+from geonature.core.imports.models import (
+    Destination,
+)
+
+
+def upsert_bib_destination(module_data: dict) -> Destination:
+    """
+    Ajoute ou met à jour une destination dans `bib_destinations`.
+
+    Parameters
+    ----------
+    module_data : dict
+        Données de la table gn_commons.t_modules du module à importer.
+
+    Returns
+    -------
+    Destination
+        L'objet Destination inséré ou mis à jour (SQLAlchemy model)
+    """
+    dest_exists = db.session.execute(
+        exists().where(Destination.code == module_data["module_code"]).select()
+    ).scalar()
+
+    # If the destination already exists, update it
+    if dest_exists:
+        existing_destination = db.session.execute(
+            select(Destination).filter_by(code=module_data["module_code"])
+        ).scalar_one()
+
+        data = {
+            "label": module_data["module_label"],
+            "table_name": f"t_imports_{module_data['module_code'].lower()}",
+            "module_code": module_data["module_code"],
+        }
+        for key, value in data.items():
+            setattr(existing_destination, key, value)
+        db.session.flush()
+        return existing_destination
+
+    # else, create a new destination
+    module_monitoring_code = db.session.execute(
+        select(TModules).filter_by(module_code=module_data["module_code"])
+    ).scalar_one()
+    destination_data = {
+        "id_module": module_monitoring_code.id_module,
+        "code": module_data["module_code"],
+        "label": destination_name(module_data["module_label"]),
+        "table_name": f"t_imports_{module_data['module_code'].lower()}",
+        "active": True,
+    }
+    destination = Destination(**destination_data)
+    db.session.add(destination)
+    db.session.flush()
+    return destination