From 6f62d17a602a5d7f688c18c3a8808249f02d26ff Mon Sep 17 00:00:00 2001 From: Dmitriy Benyuk Date: Fri, 11 Apr 2025 10:58:10 +0300 Subject: [PATCH] Add select source Folder feature --- ...Integration.Providers.ExcelProvider.csproj | 2 +- src/ExcelProvider.cs | 210 +++++++++++++----- 2 files changed, 155 insertions(+), 57 deletions(-) diff --git a/src/Dynamicweb.DataIntegration.Providers.ExcelProvider.csproj b/src/Dynamicweb.DataIntegration.Providers.ExcelProvider.csproj index 3127496..6bc5f15 100644 --- a/src/Dynamicweb.DataIntegration.Providers.ExcelProvider.csproj +++ b/src/Dynamicweb.DataIntegration.Providers.ExcelProvider.csproj @@ -1,6 +1,6 @@  - 10.6.3 + 10.6.4 1.0.0 Excel Provider Excel Provider diff --git a/src/ExcelProvider.cs b/src/ExcelProvider.cs index c9648b6..8ed8fb9 100644 --- a/src/ExcelProvider.cs +++ b/src/ExcelProvider.cs @@ -11,6 +11,7 @@ using System.Data; using System.Globalization; using System.IO; +using System.Linq; using System.Xml; using System.Xml.Linq; @@ -20,9 +21,12 @@ namespace Dynamicweb.DataIntegration.Providers.ExcelProvider public class ExcelProvider : BaseProvider, ISource, IDestination, IParameterOptions { private const string ExcelExtension = ".xlsx"; - //path should point to a folder - if it doesn't, write will fail. + private const string ExcelFilesSearchPattern = "*.xls*"; - [AddInParameter("Source file"), AddInParameterEditor(typeof(FileManagerEditor), "folder=/Files/;required"), AddInParameterGroup("Source")] + [AddInParameter("Source folder"), AddInParameterEditor(typeof(FolderSelectEditor), "folder=/Files/;"), AddInParameterGroup("Source")] + public string SourceFolder { get; set; } + + [AddInParameter("Source file"), AddInParameterEditor(typeof(FileManagerEditor), "folder=/Files/;Tooltip=Selecting a source file will override source folder selection"), AddInParameterGroup("Source")] public string SourceFile { get; set; } [AddInParameter("Destination file"), AddInParameterEditor(typeof(TextParameterEditor), $"append={ExcelExtension};required"), AddInParameterGroup("Destination")] @@ -56,38 +60,61 @@ public override Schema GetOriginalDestinationSchema() public override bool SchemaIsEditable => true; + private bool IsFolderUsed => string.IsNullOrEmpty(SourceFile); + public override Schema GetOriginalSourceSchema() { - Schema result = new Schema(); + Schema result = new Schema(); - var sourceFilePath = GetSourceFilePath(); - if (File.Exists(sourceFilePath)) + if (!IsFolderUsed) { - try + var sourceFilePath = GetSourceFilePath(SourceFile); + if (File.Exists(sourceFilePath)) { - if (SourceFile.EndsWith(".xls", StringComparison.OrdinalIgnoreCase) || - SourceFile.EndsWith(".xlsx", StringComparison.OrdinalIgnoreCase) || - SourceFile.EndsWith(".xlsm", StringComparison.OrdinalIgnoreCase)) + try { - Dictionary excelReaders = new Dictionary + if (SourceFile.EndsWith(".xls", StringComparison.OrdinalIgnoreCase) || + SourceFile.EndsWith(".xlsx", StringComparison.OrdinalIgnoreCase) || + SourceFile.EndsWith(".xlsm", StringComparison.OrdinalIgnoreCase)) + { + Dictionary excelReaders = new Dictionary { { sourceFilePath, new ExcelReader(sourceFilePath) } }; - GetSchemaForTableFromFile(result, excelReaders); + GetSchemaForTableFromFile(result, excelReaders); + } + else + { + Logger?.Error("File is not an Excel file"); + } } - else + catch (Exception ex) { - Logger?.Error("File is not an Excel file"); + Logger?.Error(string.Format("GetOriginalSourceSchema error reading file: {0} message: {1} stack: {2}", sourceFilePath, ex.Message, ex.StackTrace)); } } - catch (Exception ex) + else { - Logger?.Error(string.Format("GetOriginalSourceSchema error reading file: {0} message: {1} stack: {2}", sourceFilePath, ex.Message, ex.StackTrace)); + Logger?.Error($"Source file {sourceFilePath} does not exist"); } } else { - Logger?.Error($"Source file {sourceFilePath} does not exist"); + foreach (var sourceFilePath in GetSourceFolderFiles()) + { + try + { + Dictionary excelReaders = new Dictionary + { + { sourceFilePath, new ExcelReader(sourceFilePath) } + }; + GetSchemaForTableFromFile(result, excelReaders, true); + } + catch (Exception ex) + { + Logger?.Error(string.Format("GetOriginalSourceSchema error reading file: {0} message: {1} stack: {2}", sourceFilePath, ex.Message, ex.StackTrace)); + } + } } return result; @@ -103,28 +130,41 @@ public override string WorkingDirectory set { workingDirectory = value.Replace("\\", "/"); } } - private string GetSourceFilePath() + private string GetSourceFilePath(string filePath) { string srcFilePath = string.Empty; - if (!string.IsNullOrEmpty(SourceFile)) + if (!string.IsNullOrEmpty(filePath)) { - if (SourceFile.StartsWith("..")) + if (filePath.StartsWith("..")) { - srcFilePath = WorkingDirectory.CombinePaths(SourceFile.TrimStart(new char[] { '.' })).Replace("\\", "/"); + srcFilePath = WorkingDirectory.CombinePaths(filePath.TrimStart(new char[] { '.' })).Replace("\\", "/"); } else { - srcFilePath = SystemInformation.MapPath(FilePathHelper.GetRelativePath(SourceFile, "/Files")); + srcFilePath = SystemInformation.MapPath(FilePathHelper.GetRelativePath(filePath, "/Files")); } } return srcFilePath; } + private string SourceFolderPath => SystemInformation.MapPath(FilePathHelper.GetRelativePath(SourceFolder, "/Files")); + + private IEnumerable GetSourceFolderFiles() + { + var folderPath = SourceFolderPath; + if (Directory.Exists(folderPath)) + { + return Directory.EnumerateFiles(folderPath, ExcelFilesSearchPattern, SearchOption.TopDirectoryOnly); + } + return Enumerable.Empty(); + } + public override void UpdateSourceSettings(ISource source) { ExcelProvider newProvider = (ExcelProvider)source; SourceFile = newProvider.SourceFile; + SourceFolder = newProvider.SourceFolder; } public override string Serialize() @@ -135,6 +175,7 @@ public override string Serialize() document.Add(root); root.Add(CreateParameterNode(GetType(), "Source file", SourceFile)); + root.Add(CreateParameterNode(GetType(), "Source folder", SourceFolder)); root.Add(CreateParameterNode(GetType(), "Destination file", DestinationFile)); root.Add(CreateParameterNode(GetType(), "Destination folder", DestinationFolder)); @@ -144,6 +185,7 @@ public override string Serialize() void ISource.SaveAsXml(XmlTextWriter xmlTextWriter) { xmlTextWriter.WriteElementString("SourcePath", SourceFile); + xmlTextWriter.WriteElementString("SourceFolder", SourceFolder); (this as ISource).GetSchema().SaveAsXml(xmlTextWriter); } @@ -156,28 +198,45 @@ void IDestination.SaveAsXml(XmlTextWriter xmlTextWriter) public new ISourceReader GetReader(Mapping mapping) { - if (SourceFile.EndsWith(".xlsx", StringComparison.OrdinalIgnoreCase) || - SourceFile.EndsWith(".xls", StringComparison.OrdinalIgnoreCase) || - SourceFile.EndsWith(".xlsm", StringComparison.OrdinalIgnoreCase)) + string filePath; + if (!IsFolderUsed) { - if (!string.IsNullOrEmpty(WorkingDirectory)) - { - var sourceFilePath = GetSourceFilePath(); - if (!File.Exists(sourceFilePath)) - throw new Exception($"Source file {SourceFile} does not exist - Working Directory {WorkingDirectory}"); + filePath = SourceFile; - return new ExcelSourceReader(sourceFilePath, mapping, this); - } - else + if (filePath.EndsWith(".xlsx", StringComparison.OrdinalIgnoreCase) || + filePath.EndsWith(".xls", StringComparison.OrdinalIgnoreCase) || + filePath.EndsWith(".xlsm", StringComparison.OrdinalIgnoreCase)) { - if (!File.Exists(SourceFile)) - throw new Exception($"Source file {SourceFile} does not exist - Working Directory {WorkingDirectory}"); + if (!string.IsNullOrEmpty(WorkingDirectory)) + { + var sourceFilePath = GetSourceFilePath(filePath); + if (!File.Exists(sourceFilePath)) + throw new Exception($"Source file {SourceFile} does not exist - Working Directory {WorkingDirectory}"); - return new ExcelSourceReader(SourceFile, mapping, this); + return new ExcelSourceReader(sourceFilePath, mapping, this); + } + else + { + if (!File.Exists(filePath)) + throw new Exception($"Source file {filePath} does not exist - Working Directory {WorkingDirectory}"); + + return new ExcelSourceReader(filePath, mapping, this); + } } + else + throw new Exception("The file is not a Excel file"); } else - throw new Exception("The file is not a Excel file"); + { + string folderPath = SourceFolderPath; + var fileName = mapping.SourceTable.SqlSchema; + filePath = Directory.EnumerateFiles(folderPath, ExcelFilesSearchPattern, SearchOption.TopDirectoryOnly).FirstOrDefault(f => f.EndsWith(fileName, StringComparison.OrdinalIgnoreCase)); + if (!File.Exists(filePath)) + { + throw new Exception($"Source file {fileName} does not exist in the Directory {folderPath}"); + } + return new ExcelSourceReader(filePath, mapping, this); + } } public override void Close() @@ -264,13 +323,17 @@ private CultureInfo GetCultureInfo(string culture) return CultureInfo.CurrentCulture; } - private void GetSchemaForTableFromFile(Schema schema, Dictionary excelReaders) + private void GetSchemaForTableFromFile(Schema schema, Dictionary excelReaders, bool isFolderUsed = false) { foreach (var reader in excelReaders) { foreach (DataTable dt in reader.Value.ExcelSet.Tables) { Table excelTable = schema.AddTable(dt.TableName); + if (isFolderUsed) + { + excelTable.SqlSchema = Path.GetFileName(reader.Key); + } try { int columnCount; @@ -339,6 +402,12 @@ public ExcelProvider(XmlNode xmlNode) SourceFile = node.FirstChild.Value; } break; + case "SourceFolder": + if (node.HasChildNodes) + { + SourceFolder = node.FirstChild.Value; + } + break; case "DestinationFile": if (node.HasChildNodes) { @@ -373,7 +442,7 @@ public override void OverwriteSourceSchemaToOriginal() } public override void OverwriteDestinationSchemaToOriginal() - { + { } public override string ValidateDestinationSettings() @@ -388,40 +457,69 @@ public override string ValidateDestinationSettings() public override string ValidateSourceSettings() { - ExcelPackage.LicenseContext = LicenseContext.Commercial; - if (SourceFile.EndsWith(".xlsx", StringComparison.OrdinalIgnoreCase) || - SourceFile.EndsWith(".xls", StringComparison.OrdinalIgnoreCase) || - SourceFile.EndsWith(".xlsm", StringComparison.OrdinalIgnoreCase)) + if (string.IsNullOrEmpty(SourceFile) && string.IsNullOrEmpty(SourceFolder)) { - string filename = GetSourceFilePath(); - if (!File.Exists(filename)) + return "No Source file neither folder are selected"; + } + if (IsFolderUsed) + { + string srcFolderPath = SourceFolderPath; + + if (!Directory.Exists(srcFolderPath)) { - return $"Excel file \"{SourceFile}\" does not exist. WorkingDirectory - {WorkingDirectory}"; + return "Source folder \"" + SourceFolder + "\" does not exist"; } + else + { + var files = GetSourceFolderFiles(); - try + if (files.Count() == 0) + { + return "There are no Excel files with the extensions: [*.xlsx, *.xls, *.xlsm] in the source folder "; + } + } + } + else + { + ExcelPackage.LicenseContext = LicenseContext.Commercial; + if (SourceFile.EndsWith(".xlsx", StringComparison.OrdinalIgnoreCase) || + SourceFile.EndsWith(".xls", StringComparison.OrdinalIgnoreCase) || + SourceFile.EndsWith(".xlsm", StringComparison.OrdinalIgnoreCase)) { - using (var package = new ExcelPackage(new FileInfo(filename))) + string filename = GetSourceFilePath(SourceFile); + if (!File.Exists(filename)) { - foreach (var worksheet in package.Workbook.Worksheets) - { - string sheetName = worksheet.Name; + return $"Excel file \"{SourceFile}\" does not exist. WorkingDirectory - {WorkingDirectory}"; + } - if (sheetName.Contains(' ')) + try + { + using (var package = new ExcelPackage(new FileInfo(filename))) + { + foreach (var worksheet in package.Workbook.Worksheets) { - return $"{sheetName} contains whitespaces"; + string sheetName = worksheet.Name; + + if (sheetName.Contains(' ')) + { + return $"{sheetName} contains whitespaces"; + } } } } + catch (Exception ex) + { + return $"Could not open source file: {filename} message: {ex.Message} stack: {ex.StackTrace}"; + } } - catch (Exception ex) + else { - return $"Could not open source file: {filename} message: {ex.Message} stack: {ex.StackTrace}"; + return "The file is not an Excel file"; } } - else + if (!string.IsNullOrEmpty(SourceFile) && !string.IsNullOrEmpty(SourceFolder)) { - return "The file is not an Excel file"; + return "Warning: In your Excel Provider source, you selected both a source file and a source folder. The source folder selection will be ignored, and only the source file will be used."; } return null; }