From 9ac1cddabd96a6a4db33f789406d5554b684b0b3 Mon Sep 17 00:00:00 2001 From: Andrea Minetti Date: Thu, 15 Jan 2026 11:39:36 +0100 Subject: [PATCH 1/3] Bug fixing on maps --- .../ch/wsl/box/client/geo/MapControlStandaloneExpanded.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/main/scala/ch/wsl/box/client/geo/MapControlStandaloneExpanded.scala b/client/src/main/scala/ch/wsl/box/client/geo/MapControlStandaloneExpanded.scala index 1385f0ca..b60ce550 100644 --- a/client/src/main/scala/ch/wsl/box/client/geo/MapControlStandaloneExpanded.scala +++ b/client/src/main/scala/ch/wsl/box/client/geo/MapControlStandaloneExpanded.scala @@ -102,7 +102,7 @@ class MapControlStandaloneExpanded(params:MapControlsParams, title:String, selec if (geometry.nonEmpty) controlButton(Icons.move, SharedLabels.map.move, Control.MOVE,nested) else frag(), if (geometry.size > 1) controlButton(Icons.trash, SharedLabels.map.delete, Control.DELETE,nested) else frag(), if (geometry.size == 1) { - val (el,tt) = WidgetUtils.addTooltip(Some(SharedLabels.map.delete))(button(ClientConf.style.mapButton)( + val (el,tt) = WidgetUtils.addTooltip(Some(Labels(SharedLabels.map.delete)))(button(ClientConf.style.mapButton)( onclick :+= { (e: Event) => if(dom.window.confirm(SharedLabels.form.removeMap)) { sourceMap(_.clear()) From a329eb768039e314a0bf8a0e83041e02da25aa3e Mon Sep 17 00:00:00 2001 From: Andrea Minetti Date: Thu, 15 Jan 2026 15:25:13 +0100 Subject: [PATCH 2/3] XLS Import draft --- .../wsl/box/client/services/HttpClient.scala | 2 + .../ch/wsl/box/client/services/Labels.scala | 1 + .../ch/wsl/box/client/services/REST.scala | 2 + .../client/services/impl/HttpClientImpl.scala | 8 + .../box/client/services/impl/RestImpl.scala | 7 + .../box/client/views/EntityTableView.scala | 22 ++ .../wsl/box/client/mocks/HttpClientMock.scala | 3 + .../ch/wsl/box/client/mocks/RestMock.scala | 2 + project/Settings.scala | 2 + .../scala/ch/wsl/box/rest/io/xls/XLS.scala | 30 ++- .../ch/wsl/box/rest/io/xls/XLSImport.scala | 222 ++++++++++++++++++ .../ch/wsl/box/rest/routes/Exporters.scala | 8 +- .../scala/ch/wsl/box/rest/routes/Form.scala | 2 +- .../scala/ch/wsl/box/rest/routes/Table.scala | 5 +- .../wsl/box/model/shared/SharedLabels.scala | 2 + 15 files changed, 310 insertions(+), 8 deletions(-) create mode 100644 server/src/main/scala/ch/wsl/box/rest/io/xls/XLSImport.scala diff --git a/client/src/main/scala/ch/wsl/box/client/services/HttpClient.scala b/client/src/main/scala/ch/wsl/box/client/services/HttpClient.scala index c8fc111c..0a131f19 100755 --- a/client/src/main/scala/ch/wsl/box/client/services/HttpClient.scala +++ b/client/src/main/scala/ch/wsl/box/client/services/HttpClient.scala @@ -8,6 +8,7 @@ import org.scalajs.dom.{File, FormData, XMLHttpRequest} import scribe.Logging import scala.concurrent.{ExecutionContext, Future, Promise} +import scala.scalajs.js /** * Created by andre on 4/26/2017. @@ -25,6 +26,7 @@ trait HttpClient{ def maybeGet[T](url: String)(implicit decoder: io.circe.Decoder[T], ex:ExecutionContext): Future[Option[T]] def delete[T](url: String)(implicit decoder: io.circe.Decoder[T], ex:ExecutionContext): Future[T] def sendFile[T](url: String, file: File)(implicit decoder: io.circe.Decoder[T], ex:ExecutionContext):Future[T] + def sendRaw[T](url: String, data: js.Any)(implicit decoder: io.circe.Decoder[T], ex:ExecutionContext):Future[T] def setHandleAuthFailure(f:() => Unit) } diff --git a/client/src/main/scala/ch/wsl/box/client/services/Labels.scala b/client/src/main/scala/ch/wsl/box/client/services/Labels.scala index ec43eb73..b96549d5 100644 --- a/client/src/main/scala/ch/wsl/box/client/services/Labels.scala +++ b/client/src/main/scala/ch/wsl/box/client/services/Labels.scala @@ -144,6 +144,7 @@ object Labels { def confirmRevert = get(SharedLabels.entity.confirmRevert) def csv = get(SharedLabels.entity.csv) def xls = get(SharedLabels.entity.xls) + def importxls = get(SharedLabels.entity.importxls) def shp = get(SharedLabels.entity.shp) def geoPackage = get(SharedLabels.entity.geopackage) } diff --git a/client/src/main/scala/ch/wsl/box/client/services/REST.scala b/client/src/main/scala/ch/wsl/box/client/services/REST.scala index 114c59d8..edc9965e 100755 --- a/client/src/main/scala/ch/wsl/box/client/services/REST.scala +++ b/client/src/main/scala/ch/wsl/box/client/services/REST.scala @@ -54,6 +54,8 @@ trait REST{ //files def sendFile(file:File, id:JSONID, entity:String)(implicit ec:ExecutionContext): Future[Int] + def importXLS(kind:String, lang:String, entity:String,data:File)(implicit ec:ExecutionContext):Future[Int] + //other utilsString def login(request:LoginRequest)(implicit ec:ExecutionContext):Future[UserInfo] def authenticate(code:String,provider_id:String)(implicit ec:ExecutionContext):Future[UserInfo] diff --git a/client/src/main/scala/ch/wsl/box/client/services/impl/HttpClientImpl.scala b/client/src/main/scala/ch/wsl/box/client/services/impl/HttpClientImpl.scala index a8302cfa..e6f57605 100644 --- a/client/src/main/scala/ch/wsl/box/client/services/impl/HttpClientImpl.scala +++ b/client/src/main/scala/ch/wsl/box/client/services/impl/HttpClientImpl.scala @@ -9,6 +9,7 @@ import org.scalajs.dom.{File, FormData, XMLHttpRequest} import scribe.Logging import scala.concurrent.{ExecutionContext, ExecutionContextExecutor, Future, Promise} +import scala.scalajs.js class HttpClientImpl extends HttpClient with Logging { @@ -183,6 +184,13 @@ class HttpClientImpl extends HttpClient with Logging { }.map(handle404) + override def sendRaw[T](url: String, data: js.Any)(implicit decoder: Decoder[T], ex: ExecutionContext): Future[T] = { + httpCallWithNoticeInterceptor[T]("POST", url, file = true) { xhr => + xhr.send(data) + } + + }.map(handle404) + private var handleAuthFailure: () => Unit = () => {} override def setHandleAuthFailure(f: () => Unit): Unit = { handleAuthFailure = f diff --git a/client/src/main/scala/ch/wsl/box/client/services/impl/RestImpl.scala b/client/src/main/scala/ch/wsl/box/client/services/impl/RestImpl.scala index ed9f9c3b..15832707 100644 --- a/client/src/main/scala/ch/wsl/box/client/services/impl/RestImpl.scala +++ b/client/src/main/scala/ch/wsl/box/client/services/impl/RestImpl.scala @@ -111,6 +111,13 @@ class RestImpl(httpClient:HttpClient) extends REST with Logging { //files def sendFile(file:File, id:JSONID, entity:String)(implicit ec:ExecutionContext): Future[Int] = httpClient.sendFile[Int](Routes.apiV1(s"/file/$entity/${id.asString}"),file) + override def importXLS(kind: String, lang: String, entity: String, data: File)(implicit ec:ExecutionContext): Future[Int] = { + for{ + bytea <- data.arrayBuffer().toFuture + count <- httpClient.sendRaw[Int](Routes.apiV1(s"/$kind/$lang/$entity/xlsx/import"),bytea) + } yield count + } + //other utilsString def login(request:LoginRequest)(implicit ec:ExecutionContext) = httpClient.post[LoginRequest,UserInfo](Routes.apiV1("/login"),request) def authenticate(code:String,provider_id:String)(implicit ec:ExecutionContext) = httpClient.get[UserInfo](Routes.apiV1(s"/sso/$provider_id?code=$code")) diff --git a/client/src/main/scala/ch/wsl/box/client/views/EntityTableView.scala b/client/src/main/scala/ch/wsl/box/client/views/EntityTableView.scala index 56a32b98..8013ceb7 100755 --- a/client/src/main/scala/ch/wsl/box/client/views/EntityTableView.scala +++ b/client/src/main/scala/ch/wsl/box/client/views/EntityTableView.scala @@ -556,6 +556,27 @@ case class EntityTablePresenter(model:ModelProperty[EntityTableModel], onSelect: e.preventDefault() } + val importXLS = (e:Event) => { + + val kind = EntityKind(model.subProp(_.kind).get).entityOrForm + val modelName = model.subProp(_.name).get + val lang = services.clientSession.lang() + + import scalatags.JsDom.all._ + val el = input(`type` := "file", accept := ".xlsx").render + el.onchange = (ev: Event) => { + ev.preventDefault() + el.files.headOption match { + case Some(file) => services.rest.importXLS(kind,lang,modelName,file).map{ i => + Notification.add(Labels(s"Imported $i rows")) + } + case None => Notification.add(Labels("No files found")) + } + } + el.click() + e.preventDefault() + } + private def download(format:String) = { val kind = EntityKind(model.subProp(_.kind).get).entityOrForm @@ -1124,6 +1145,7 @@ case class EntityTableView(model:ModelProperty[EntityTableModel], presenter:Enti tableContent(metadata), button(`type` := "button", onclick :+= presenter.downloadCSV, ClientConf.style.boxButton, Labels.entity.csv), button(`type` := "button", onclick :+= presenter.downloadXLS, ClientConf.style.boxButton, Labels.entity.xls), + button(`type` := "button", onclick :+= presenter.importXLS, ClientConf.style.boxButton, Labels.entity.importxls), if (presenter.hasGeometry()) { Seq( //button(`type` := "button", onclick :+= presenter.downloadSHP, ClientConf.style.boxButton, Labels.entity.shp), diff --git a/client/src/test/scala/ch/wsl/box/client/mocks/HttpClientMock.scala b/client/src/test/scala/ch/wsl/box/client/mocks/HttpClientMock.scala index f2bc865e..d52a21ce 100644 --- a/client/src/test/scala/ch/wsl/box/client/mocks/HttpClientMock.scala +++ b/client/src/test/scala/ch/wsl/box/client/mocks/HttpClientMock.scala @@ -5,6 +5,7 @@ import io.circe.{Decoder, Encoder} import org.scalajs.dom.File import scala.concurrent.{ExecutionContext, Future} +import scala.scalajs.js class HttpClientMock extends HttpClient { override def post[D, R](url: String, obj: D)(implicit decoder: Decoder[R], encoder: Encoder[D], ec:ExecutionContext): Future[R] = throw new Exception("post not implemented") @@ -21,5 +22,7 @@ class HttpClientMock extends HttpClient { override def sendFile[T](url: String, file: File)(implicit decoder: Decoder[T], ec:ExecutionContext): Future[T] = throw new Exception("sendFile not implemented") + override def sendRaw[T](url: String, data: js.Any)(implicit decoder: Decoder[T], ex: ExecutionContext): Future[T] = ??? + override def setHandleAuthFailure(f: () => Unit): Unit = {} } diff --git a/client/src/test/scala/ch/wsl/box/client/mocks/RestMock.scala b/client/src/test/scala/ch/wsl/box/client/mocks/RestMock.scala index 5f54b982..fa5b5baf 100644 --- a/client/src/test/scala/ch/wsl/box/client/mocks/RestMock.scala +++ b/client/src/test/scala/ch/wsl/box/client/mocks/RestMock.scala @@ -140,6 +140,8 @@ class RestMock(values:Values) extends REST with Logging { ??? } + override def importXLS(kind: String, lang: String, entity: String, data: File)(implicit ec:ExecutionContext): Future[Int] = ??? + val userInfo = UserInfo("t","t",None,Seq(),Json.Null) override def login(request: LoginRequest)(implicit ec:ExecutionContext): Future[UserInfo] = Future.successful{ userInfo diff --git a/project/Settings.scala b/project/Settings.scala index 9b04cb3a..83e93235 100755 --- a/project/Settings.scala +++ b/project/Settings.scala @@ -152,6 +152,8 @@ object Settings { "com.github.spullara.mustache.java" % "compiler" % "0.9.6", "com.fasterxml.jackson.core" % "jackson-databind" % "2.10.3", "com.norbitltd" %% "spoiwo" % "2.2.1", + "org.apache.poi" % "poi" % "5.2.5", + "org.apache.poi" % "poi-ooxml" % "5.2.5", "io.github.cquiroz" %% "scala-java-time" % "2.0.0", "com.nrinaudo" %% "kantan.csv" % versions.kantan, "org.wvlet.airframe" %%% "airframe" % versions.airframe, diff --git a/server/src/main/scala/ch/wsl/box/rest/io/xls/XLS.scala b/server/src/main/scala/ch/wsl/box/rest/io/xls/XLS.scala index 425b18a2..470cc7ef 100644 --- a/server/src/main/scala/ch/wsl/box/rest/io/xls/XLS.scala +++ b/server/src/main/scala/ch/wsl/box/rest/io/xls/XLS.scala @@ -1,13 +1,18 @@ package ch.wsl.box.rest.io.xls import java.io.ByteArrayOutputStream - -import akka.http.scaladsl.model.{HttpEntity, HttpResponse, MediaTypes} +import akka.http.scaladsl.model.{ContentType, HttpEntity, HttpResponse, MediaTypes, StatusCodes} import akka.http.scaladsl.model.headers.{ContentDispositionTypes, `Content-Disposition`} -import akka.http.scaladsl.server.Directives.{complete, get, parameters, path, respondWithHeader} +import akka.http.scaladsl.server.Directives._ import akka.http.scaladsl.server.Route +import ch.wsl.box.jdbc.UserDatabase +import ch.wsl.box.model.shared.{JSONMetadata, XLSTable} +import ch.wsl.box.rest.logic.{JSONTableActions, TableActions} +import io.circe.Json +import slick.dbio.DBIO -import ch.wsl.box.model.shared.{XLSTable} +import scala.concurrent.ExecutionContext +import scala.util.{Failure, Success} object XLS { @@ -22,4 +27,21 @@ object XLS { } } } + + def importXls(metadata:JSONMetadata,actions:TableActions[Json],db:UserDatabase)(implicit ec:ExecutionContext):Route = post{ + entity(as[Array[Byte]]) { data => + XLSImport.xlsToJson(data, metadata) match { + case Failure(exception) => complete(StatusCodes.BadRequest, exception.toString) + case Success(value) => { + complete { + db.run { + DBIO.sequence { + value.map(row => actions.insert(row)) + } + }.map(x => HttpResponse(entity = HttpEntity(MediaTypes.`application/json`,x.length.toString))) + } + } + } + } + } } diff --git a/server/src/main/scala/ch/wsl/box/rest/io/xls/XLSImport.scala b/server/src/main/scala/ch/wsl/box/rest/io/xls/XLSImport.scala new file mode 100644 index 00000000..b8c9d24c --- /dev/null +++ b/server/src/main/scala/ch/wsl/box/rest/io/xls/XLSImport.scala @@ -0,0 +1,222 @@ +package ch.wsl.box.rest.io.xls + +import ch.wsl.box.model.shared.{JSONField, JSONFieldTypes, JSONMetadata} +import io.circe.Json +import io.circe.syntax.EncoderOps +import org.apache.poi.ss.usermodel._ +import org.apache.poi.ss.usermodel.DateUtil +import org.apache.poi.openxml4j.exceptions.InvalidFormatException + +import java.io.{ByteArrayInputStream, File} +import java.sql.{Connection, DriverManager, PreparedStatement, Timestamp, Date => SqlDate} +import java.time.{Instant, LocalDate, LocalDateTime, ZoneId} +import java.time.format.DateTimeFormatter +import scala.util.{Try, Using} + +object XLSImport { + + + + def xlsToJson(xls:Array[Byte],metadata:JSONMetadata): Try[Seq[Json]] = Try { + + Using.resource(new ByteArrayInputStream(xls)) { xls => + + Using.resource(WorkbookFactory.create(xls)) { workbook => + val sheet = workbook.getSheetAt(0) + + + val headerRow = Option(sheet.getRow(0)) + .getOrElse(throw new IllegalArgumentException(s"Header row not found at index 0")) + + val headerIndex = buildHeaderIndex(headerRow) + + // Map each ColumnSpec -> excel column index + val excelIndexes: Seq[(Int, JSONField)] = metadata.fields.flatMap { c => + headerIndex.get(c.title).map(i => i -> c) + } + + + val rows = 1 to sheet.getLastRowNum + rows.flatMap { rowIndex => + val row = sheet.getRow(rowIndex) + if (row != null && !isRowEmpty(row)) Some { + bindRow( row, excelIndexes) + } else None + } + } + } + } + + + // ---- Excel helpers ---- + + private def buildHeaderIndex(headerRow: Row): Map[String, Int] = { + val fmt = new DataFormatter() + val m = scala.collection.mutable.Map.empty[String, Int] + val lastCell = headerRow.getLastCellNum.toInt + var c = 0 + while (c < lastCell) { + val cell = headerRow.getCell(c, Row.MissingCellPolicy.RETURN_BLANK_AS_NULL) + if (cell != null) { + val v = fmt.formatCellValue(cell).trim + if (v.nonEmpty) m += (v -> c) + } + c += 1 + } + m.toMap + } + + private def isRowEmpty(row: Row): Boolean = { + val lastCell = row.getLastCellNum.toInt + var c = 0 + while (c < lastCell) { + val cell = row.getCell(c, Row.MissingCellPolicy.RETURN_BLANK_AS_NULL) + if (cell != null && cell.getCellType != CellType.BLANK) { + val fmt = new DataFormatter() + if (fmt.formatCellValue(cell).trim.nonEmpty) return false + } + c += 1 + } + true + } + + // ---- Binding / parsing ---- + + private def bindRow( + row: Row, + excelIndexes: Seq[(Int,JSONField)] + ) = { + + val cells = excelIndexes.map{ case (i,field) => + val cell = row.getCell(i, Row.MissingCellPolicy.RETURN_BLANK_AS_NULL) + val valueOpt = parseCell(cell, field) + (valueOpt, field.nullable) match { + case (None, false) => + throw new IllegalArgumentException( + s"Non-nullable column '${field.title}' is empty at Excel row ${row.getRowNum + 1}" + ) + case (None,true) => field.name -> Json.Null + case (Some(v),_) => field.name -> v + } + } + + Json.fromFields(cells) + + } + + private def parseCell(cell: Cell, tpe: JSONField): Option[Json] = { + if (cell == null) return None + + import ch.wsl.box.rest.utils.JSONSupport._ + + val fmt = new DataFormatter() + tpe.`type` match { + case JSONFieldTypes.STRING => + val s = fmt.formatCellValue(cell).trim + if (s.isEmpty) None else Some(Json.fromString(s)) + + case JSONFieldTypes.INTEGER => + val n = parseNumber(cell, fmt).map(_.longValue) + n.map { x => + Json.fromLong(x) + } + + case JSONFieldTypes.NUMBER => + parseNumber(cell, fmt).flatMap(n => Json.fromDouble(n)) + + case JSONFieldTypes.BOOLEAN => + val s = fmt.formatCellValue(cell).trim.toLowerCase + if (s.isEmpty) None + else { + val b = + s match { + case "true" | "t" | "yes" | "y" | "1" => true + case "false" | "f" | "no" | "n" | "0" => false + case _ => + throw new IllegalArgumentException(s"Invalid boolean: '$s'") + } + Some(Json.fromBoolean(b)) + } + + case JSONFieldTypes.DATE => + parseAsDate(cell, fmt, ZoneId.systemDefault()).map(d => d.asJson) + + case JSONFieldTypes.DATETIME | JSONFieldTypes.DATETIMETZ => + parseAsInstant(cell, fmt, ZoneId.systemDefault()).map(ts => ts.asJson) + } + } + + private def parseNumber(cell: Cell, fmt: DataFormatter): Option[java.lang.Double] = { + cell.getCellType match { + case CellType.NUMERIC => + Some(java.lang.Double.valueOf(cell.getNumericCellValue)) + case CellType.STRING => + val s = cell.getStringCellValue.trim + if (s.isEmpty) None else Some(java.lang.Double.valueOf(s.replace(",", "."))) + case CellType.FORMULA => + cell.getCachedFormulaResultType match { + case CellType.NUMERIC => Some(java.lang.Double.valueOf(cell.getNumericCellValue)) + case CellType.STRING => + val s = fmt.formatCellValue(cell).trim + if (s.isEmpty) None else Some(java.lang.Double.valueOf(s.replace(",", "."))) + case _ => None + } + case CellType.BLANK => None + case _ => + val s = fmt.formatCellValue(cell).trim + if (s.isEmpty) None else Some(java.lang.Double.valueOf(s.replace(",", "."))) + } + } + + + private def parseAsInstant(cell: Cell, fmt: DataFormatter, zoneId: ZoneId): Option[LocalDateTime] = { + if (cell.getCellType == CellType.NUMERIC && DateUtil.isCellDateFormatted(cell)) { + val dt = cell.getDateCellValue.toInstant.atZone(zoneId).toLocalDateTime + Some(dt) + } else { + val s = fmt.formatCellValue(cell).trim + if (s.isEmpty) None + else { + // Try a few common timestamp patterns; extend as needed. + val patterns = Seq( + "yyyy-MM-dd HH:mm:ss", + "yyyy-MM-dd'T'HH:mm:ss", + "yyyy-MM-dd HH:mm", + "yyyy-MM-dd" + ) + val parsed = patterns.view.flatMap { p => + val f = DateTimeFormatter.ofPattern(p) + Try { + if (p == "yyyy-MM-dd") { + LocalDate.parse(s, f).atStartOfDay(zoneId).toLocalDateTime + } else { + java.time.LocalDateTime.parse(s, f) + } + }.toOption + }.headOption + + parsed.orElse { + // Last resort: try Instant.parse (ISO-8601) + Try(LocalDateTime.parse(s)).toOption + } + } + } + } + + private def parseAsDate(cell: Cell, fmt: DataFormatter, zoneId: ZoneId): Option[LocalDate] = { + if (cell.getCellType == CellType.NUMERIC && DateUtil.isCellDateFormatted(cell)) { + val inst = cell.getDateCellValue.toInstant + Some(inst.atZone(zoneId).toLocalDate) + } else { + val s = fmt.formatCellValue(cell).trim + if (s.isEmpty) None + else { + val patterns = Seq("yyyy-MM-dd", "dd.MM.yyyy", "MM/dd/yyyy") + patterns.view.flatMap { p => + val f = DateTimeFormatter.ofPattern(p) + Try(LocalDate.parse(s, f)).toOption + }.headOption + } + } + } +} diff --git a/server/src/main/scala/ch/wsl/box/rest/routes/Exporters.scala b/server/src/main/scala/ch/wsl/box/rest/routes/Exporters.scala index ff307122..72428d8b 100644 --- a/server/src/main/scala/ch/wsl/box/rest/routes/Exporters.scala +++ b/server/src/main/scala/ch/wsl/box/rest/routes/Exporters.scala @@ -2,7 +2,7 @@ package ch.wsl.box.rest.routes import akka.http.scaladsl.model.{HttpEntity, HttpResponse, MediaTypes} import akka.http.scaladsl.model.headers.{ContentDispositionTypes, `Content-Disposition`} -import akka.http.scaladsl.server.Directives.{_symbol2NR, complete, get, onSuccess, parameters, path, respondWithHeader} +import akka.http.scaladsl.server.Directives._ import akka.http.scaladsl.server.Route import akka.stream.Materializer import ch.wsl.box.model.shared.{CSVTable, JSONFieldLookupData, JSONFieldLookupExtractor, JSONFieldLookupRemote, JSONMetadata, JSONQuery, XLSTable} @@ -38,6 +38,7 @@ trait Exporters { val name:String val metadataFactory: MetadataFactory def tabularMetadata(fields:Option[Seq[String]] = None): DBIO[JSONMetadata] + def actions:FormActions def mergeWithForeignKeys(extractFk: Boolean,data:Seq[Json],fk: Map[String,Seq[Json]],metadata:JSONMetadata):Seq[Json] = { if(extractFk) { @@ -61,7 +62,10 @@ trait Exporters { } } - def xls(implicit session:BoxSession, db:FullDatabase, mat:Materializer, ec:ExecutionContext, services:Services) = path("xlsx") { + def xls(implicit session:BoxSession, db:FullDatabase, mat:Materializer, ec:ExecutionContext, services:Services) = pathPrefix("xlsx") { + path("import") { + XLS.importXls(actions.metadata,actions.jsonAction,db.db) + } ~ get { parameters('q,'fk.?) { case (q,fk) => val extractFk = fk.forall(_ == "resolve_fk") diff --git a/server/src/main/scala/ch/wsl/box/rest/routes/Form.scala b/server/src/main/scala/ch/wsl/box/rest/routes/Form.scala index 3a8718b0..fe61a4fb 100755 --- a/server/src/main/scala/ch/wsl/box/rest/routes/Form.scala +++ b/server/src/main/scala/ch/wsl/box/rest/routes/Form.scala @@ -58,7 +58,7 @@ case class Form( implicit val boxDb = FullDatabase(db,services.connection.adminDB) def metadata: JSONMetadata = Await.result(boxDb.adminDb.run(metadataFactory.of(name,lang,session.user)),10.seconds) - private def actions:FormActions = FormActions(metadata,registry,metadataFactory) + def actions:FormActions = FormActions(metadata,registry,metadataFactory) private def _tabMetadata(fields:Option[Seq[String]] = None,m:JSONMetadata): Seq[JSONField] = { fields match { diff --git a/server/src/main/scala/ch/wsl/box/rest/routes/Table.scala b/server/src/main/scala/ch/wsl/box/rest/routes/Table.scala index d896f959..40504101 100755 --- a/server/src/main/scala/ch/wsl/box/rest/routes/Table.scala +++ b/server/src/main/scala/ch/wsl/box/rest/routes/Table.scala @@ -86,7 +86,10 @@ case class Table[T <: ch.wsl.box.jdbc.PostgresProfile.api.Table[M] with UpdateTa Await.result(fut,20.seconds) } - def xls:Route = path("xlsx") { + def xls:Route = pathPrefix("xlsx") { + path("import") { + XLS.importXls(jsonMetadata,jsonActions,db) + } ~ get { parameters('q) { q => val query = parse(q).right.get.as[JSONQuery].right.get diff --git a/shared/src/main/scala/ch/wsl/box/model/shared/SharedLabels.scala b/shared/src/main/scala/ch/wsl/box/model/shared/SharedLabels.scala index ae3b6618..d802218d 100644 --- a/shared/src/main/scala/ch/wsl/box/model/shared/SharedLabels.scala +++ b/shared/src/main/scala/ch/wsl/box/model/shared/SharedLabels.scala @@ -226,6 +226,7 @@ object SharedLabels extends LabelsCollection { def confirmRevert = "table.confirmRevert" def csv = "table.csv" def xls = "table.xls" + def importxls = "table.importxls" def shp = "table.shp" def geopackage = "table.geopackage" def all = Seq( @@ -240,6 +241,7 @@ object SharedLabels extends LabelsCollection { confirmRevert, csv, xls, + importxls, shp, geopackage ) From 23a1a7aa84f12d3f0972855469fceb964f82c845 Mon Sep 17 00:00:00 2001 From: Andrea Minetti Date: Wed, 21 Jan 2026 14:17:35 +0100 Subject: [PATCH 3/3] Import with lookups --- .../box/client/views/EntityTableView.scala | 2 + .../scala/ch/wsl/box/rest/io/xls/XLS.scala | 24 +-- .../ch/wsl/box/rest/io/xls/XLSImport.scala | 142 +++++++++++------- 3 files changed, 105 insertions(+), 63 deletions(-) diff --git a/client/src/main/scala/ch/wsl/box/client/views/EntityTableView.scala b/client/src/main/scala/ch/wsl/box/client/views/EntityTableView.scala index 8013ceb7..e9f32ea7 100755 --- a/client/src/main/scala/ch/wsl/box/client/views/EntityTableView.scala +++ b/client/src/main/scala/ch/wsl/box/client/views/EntityTableView.scala @@ -569,6 +569,8 @@ case class EntityTablePresenter(model:ModelProperty[EntityTableModel], onSelect: el.files.headOption match { case Some(file) => services.rest.importXLS(kind,lang,modelName,file).map{ i => Notification.add(Labels(s"Imported $i rows")) + println("reload rows AAAAA") + reloadRows(model.get.pages) } case None => Notification.add(Labels("No files found")) } diff --git a/server/src/main/scala/ch/wsl/box/rest/io/xls/XLS.scala b/server/src/main/scala/ch/wsl/box/rest/io/xls/XLS.scala index 470cc7ef..3695bc4a 100644 --- a/server/src/main/scala/ch/wsl/box/rest/io/xls/XLS.scala +++ b/server/src/main/scala/ch/wsl/box/rest/io/xls/XLS.scala @@ -5,9 +5,11 @@ import akka.http.scaladsl.model.{ContentType, HttpEntity, HttpResponse, MediaTyp import akka.http.scaladsl.model.headers.{ContentDispositionTypes, `Content-Disposition`} import akka.http.scaladsl.server.Directives._ import akka.http.scaladsl.server.Route +import akka.http.scaladsl.server.directives.AuthenticationDirective import ch.wsl.box.jdbc.UserDatabase import ch.wsl.box.model.shared.{JSONMetadata, XLSTable} import ch.wsl.box.rest.logic.{JSONTableActions, TableActions} +import ch.wsl.box.services.Services import io.circe.Json import slick.dbio.DBIO @@ -28,20 +30,20 @@ object XLS { } } - def importXls(metadata:JSONMetadata,actions:TableActions[Json],db:UserDatabase)(implicit ec:ExecutionContext):Route = post{ + def importXls(metadata:JSONMetadata,actions:TableActions[Json],db:UserDatabase)(implicit ec:ExecutionContext, services:Services):Route = post{ entity(as[Array[Byte]]) { data => - XLSImport.xlsToJson(data, metadata) match { - case Failure(exception) => complete(StatusCodes.BadRequest, exception.toString) - case Success(value) => { - complete { - db.run { - DBIO.sequence { - value.map(row => actions.insert(row)) - } - }.map(x => HttpResponse(entity = HttpEntity(MediaTypes.`application/json`,x.length.toString))) + + val insertData = for { + data <- new XLSImport(db).xlsToJson(data, metadata) + insert <- db.run { + DBIO.sequence { + data.map(row => actions.insert(row)) } } - } + } yield HttpResponse(entity = HttpEntity(MediaTypes.`application/json`, insert.length.toString)) + + completeOrRecoverWith(insertData) { ex => complete(StatusCodes.BadRequest, ex.toString) } + } } } diff --git a/server/src/main/scala/ch/wsl/box/rest/io/xls/XLSImport.scala b/server/src/main/scala/ch/wsl/box/rest/io/xls/XLSImport.scala index b8c9d24c..03690722 100644 --- a/server/src/main/scala/ch/wsl/box/rest/io/xls/XLSImport.scala +++ b/server/src/main/scala/ch/wsl/box/rest/io/xls/XLSImport.scala @@ -1,23 +1,30 @@ package ch.wsl.box.rest.io.xls -import ch.wsl.box.model.shared.{JSONField, JSONFieldTypes, JSONMetadata} +import ch.wsl.box.jdbc.UserDatabase +import ch.wsl.box.model.shared.JSONQueryFilter.WHERE +import ch.wsl.box.model.shared.{JSONField, JSONFieldLookup, JSONFieldLookupData, JSONFieldLookupExtractor, JSONFieldLookupRemote, JSONFieldTypes, JSONMetadata, JSONQuery} +import ch.wsl.box.rest.runtime.Registry +import ch.wsl.box.services.Services +import ch.wsl.box.shared.utils.JSONUtils.EnhancedJson import io.circe.Json import io.circe.syntax.EncoderOps import org.apache.poi.ss.usermodel._ import org.apache.poi.ss.usermodel.DateUtil import org.apache.poi.openxml4j.exceptions.InvalidFormatException +import org.apache.poi.ss.util.CellRangeAddressList import java.io.{ByteArrayInputStream, File} import java.sql.{Connection, DriverManager, PreparedStatement, Timestamp, Date => SqlDate} import java.time.{Instant, LocalDate, LocalDateTime, ZoneId} import java.time.format.DateTimeFormatter -import scala.util.{Try, Using} +import scala.concurrent.{ExecutionContext, Future} +import scala.util.{Failure, Success, Try, Using} -object XLSImport { +class XLSImport(db:UserDatabase)(implicit ex:ExecutionContext, services:Services) { + new CellRangeAddressList(0,5,0,0); - - def xlsToJson(xls:Array[Byte],metadata:JSONMetadata): Try[Seq[Json]] = Try { + def xlsToJson(xls:Array[Byte],metadata:JSONMetadata): Future[Seq[Json]] = Try { Using.resource(new ByteArrayInputStream(xls)) { xls => @@ -37,14 +44,20 @@ object XLSImport { val rows = 1 to sheet.getLastRowNum - rows.flatMap { rowIndex => - val row = sheet.getRow(rowIndex) - if (row != null && !isRowEmpty(row)) Some { - bindRow( row, excelIndexes) - } else None + Future.sequence { + rows.flatMap { rowIndex => + val row = sheet.getRow(rowIndex) + if (row != null && !isRowEmpty(row)) Some { + bindRow(row, excelIndexes) + } else None + } } + } } + } match { + case Failure(exception) => Future.failed(exception) + case Success(value) => value } @@ -89,60 +102,85 @@ object XLSImport { val cells = excelIndexes.map{ case (i,field) => val cell = row.getCell(i, Row.MissingCellPolicy.RETURN_BLANK_AS_NULL) - val valueOpt = parseCell(cell, field) - (valueOpt, field.nullable) match { - case (None, false) => - throw new IllegalArgumentException( - s"Non-nullable column '${field.title}' is empty at Excel row ${row.getRowNum + 1}" - ) - case (None,true) => field.name -> Json.Null - case (Some(v),_) => field.name -> v + parseCell(cell, field).map { valueOpt => + (valueOpt, field.nullable) match { + case (None, false) => + throw new IllegalArgumentException( + s"Non-nullable column '${field.title}' is empty at Excel row ${row.getRowNum + 1}" + ) + case (None, true) => field.name -> Json.Null + case (Some(v), _) => field.name -> v + } } } - - Json.fromFields(cells) + Future.sequence(cells).map(Json.fromFields) } - private def parseCell(cell: Cell, tpe: JSONField): Option[Json] = { - if (cell == null) return None + private def parseCell(cell: Cell, tpe: JSONField): Future[Option[Json]] = { + if (cell == null) return Future.successful(None) import ch.wsl.box.rest.utils.JSONSupport._ - val fmt = new DataFormatter() - tpe.`type` match { - case JSONFieldTypes.STRING => - val s = fmt.formatCellValue(cell).trim - if (s.isEmpty) None else Some(Json.fromString(s)) + val maybeLookup = tpe.lookup match { + case Some(lookup) => parseLookup(cell,lookup) + case None => Future.successful(None) + } - case JSONFieldTypes.INTEGER => - val n = parseNumber(cell, fmt).map(_.longValue) - n.map { x => - Json.fromLong(x) - } + def _cell() = { + val fmt = new DataFormatter() + tpe.`type` match { + case JSONFieldTypes.STRING => + val s = fmt.formatCellValue(cell).trim + if (s.isEmpty) None else Some(Json.fromString(s)) + + case JSONFieldTypes.INTEGER => + val n = parseNumber(cell, fmt).map(_.longValue) + n.map { x => + Json.fromLong(x) + } + + case JSONFieldTypes.NUMBER => + parseNumber(cell, fmt).flatMap(n => Json.fromDouble(n)) + + case JSONFieldTypes.BOOLEAN => + val s = fmt.formatCellValue(cell).trim.toLowerCase + if (s.isEmpty) None + else { + val b = + s match { + case "true" | "t" | "yes" | "y" | "1" => true + case "false" | "f" | "no" | "n" | "0" => false + case _ => + throw new IllegalArgumentException(s"Invalid boolean: '$s'") + } + Some(Json.fromBoolean(b)) + } + + case JSONFieldTypes.DATE => + parseAsDate(cell, fmt, ZoneId.systemDefault()).map(d => d.asJson) + + case JSONFieldTypes.DATETIME | JSONFieldTypes.DATETIMETZ => + parseAsInstant(cell, fmt, ZoneId.systemDefault()).map(ts => ts.asJson) + } + } - case JSONFieldTypes.NUMBER => - parseNumber(cell, fmt).flatMap(n => Json.fromDouble(n)) - - case JSONFieldTypes.BOOLEAN => - val s = fmt.formatCellValue(cell).trim.toLowerCase - if (s.isEmpty) None - else { - val b = - s match { - case "true" | "t" | "yes" | "y" | "1" => true - case "false" | "f" | "no" | "n" | "0" => false - case _ => - throw new IllegalArgumentException(s"Invalid boolean: '$s'") - } - Some(Json.fromBoolean(b)) - } + maybeLookup.map { _.orElse(_cell()) } - case JSONFieldTypes.DATE => - parseAsDate(cell, fmt, ZoneId.systemDefault()).map(d => d.asJson) + } - case JSONFieldTypes.DATETIME | JSONFieldTypes.DATETIMETZ => - parseAsInstant(cell, fmt, ZoneId.systemDefault()).map(ts => ts.asJson) + private def parseLookup(cell: Cell,lookup: JSONFieldLookup):Future[Option[Json]] = { + val fmt = new DataFormatter() + val s = fmt.formatCellValue(cell).trim + lookup match { + case JSONFieldLookupRemote(lookupEntity, map, lookupQuery) => { + val filters = map.foreign.labelColumns.toSeq.map{ c => WHERE.eq(c,s) } + db.run { + Registry().actions(lookupEntity).findSimple(JSONQuery.filterWith(filters: _*)).map(_.headOption.flatMap(js => map.foreign.keyColumns.headOption.map(js.js(_)))) + } + } + case JSONFieldLookupExtractor(extractor) => ??? + case JSONFieldLookupData(data) => Future.successful(data.find(_.value == s).map(_.id)) } }