Skip to content

Add select source Folder feature #33

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
DWDBE merged 1 commit into from
Apr 16, 2025
Merged

Add select source Folder feature #33

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/Dynamicweb.DataIntegration.Providers.ExcelProvider.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<Version>10.6.3</Version>
<Version>10.6.4</Version>
<AssemblyVersion>1.0.0</AssemblyVersion>
<Title>Excel Provider</Title>
<Description>Excel Provider</Description>
Expand Down
210 changes: 154 additions & 56 deletions src/ExcelProvider.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
using System.Data;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Xml;
using System.Xml.Linq;

Expand All @@ -20,9 +21,12 @@ namespace Dynamicweb.DataIntegration.Providers.ExcelProvider
public class ExcelProvider : BaseProvider, ISource, IDestination, IParameterOptions
{
private const string ExcelExtension = ".xlsx";
//path should point to a folder - if it doesn't, write will fail.
private const string ExcelFilesSearchPattern = "*.xls*";

[AddInParameter("Source file"), AddInParameterEditor(typeof(FileManagerEditor), "folder=/Files/;required"), AddInParameterGroup("Source")]
[AddInParameter("Source folder"), AddInParameterEditor(typeof(FolderSelectEditor), "folder=/Files/;"), AddInParameterGroup("Source")]
public string SourceFolder { get; set; }

[AddInParameter("Source file"), AddInParameterEditor(typeof(FileManagerEditor), "folder=/Files/;Tooltip=Selecting a source file will override source folder selection"), AddInParameterGroup("Source")]
public string SourceFile { get; set; }

[AddInParameter("Destination file"), AddInParameterEditor(typeof(TextParameterEditor), $"append={ExcelExtension};required"), AddInParameterGroup("Destination")]
Expand Down Expand Up @@ -56,38 +60,61 @@ public override Schema GetOriginalDestinationSchema()

public override bool SchemaIsEditable => true;

private bool IsFolderUsed => string.IsNullOrEmpty(SourceFile);

public override Schema GetOriginalSourceSchema()
{
Schema result = new Schema();
Schema result = new Schema();

var sourceFilePath = GetSourceFilePath();
if (File.Exists(sourceFilePath))
if (!IsFolderUsed)
{
try
var sourceFilePath = GetSourceFilePath(SourceFile);
if (File.Exists(sourceFilePath))
{
if (SourceFile.EndsWith(".xls", StringComparison.OrdinalIgnoreCase) ||
SourceFile.EndsWith(".xlsx", StringComparison.OrdinalIgnoreCase) ||
SourceFile.EndsWith(".xlsm", StringComparison.OrdinalIgnoreCase))
try
{
Dictionary<string, ExcelReader> excelReaders = new Dictionary<string, ExcelReader>
if (SourceFile.EndsWith(".xls", StringComparison.OrdinalIgnoreCase) ||
SourceFile.EndsWith(".xlsx", StringComparison.OrdinalIgnoreCase) ||
SourceFile.EndsWith(".xlsm", StringComparison.OrdinalIgnoreCase))
{
Dictionary<string, ExcelReader> excelReaders = new Dictionary<string, ExcelReader>
{
{ sourceFilePath, new ExcelReader(sourceFilePath) }
};
GetSchemaForTableFromFile(result, excelReaders);
GetSchemaForTableFromFile(result, excelReaders);
}
else
{
Logger?.Error("File is not an Excel file");
}
}
else
catch (Exception ex)
{
Logger?.Error("File is not an Excel file");
Logger?.Error(string.Format("GetOriginalSourceSchema error reading file: {0} message: {1} stack: {2}", sourceFilePath, ex.Message, ex.StackTrace));
}
}
catch (Exception ex)
else
{
Logger?.Error(string.Format("GetOriginalSourceSchema error reading file: {0} message: {1} stack: {2}", sourceFilePath, ex.Message, ex.StackTrace));
Logger?.Error($"Source file {sourceFilePath} does not exist");
}
}
else
{
Logger?.Error($"Source file {sourceFilePath} does not exist");
foreach (var sourceFilePath in GetSourceFolderFiles())
{
try
{
Dictionary<string, ExcelReader> excelReaders = new Dictionary<string, ExcelReader>
{
{ sourceFilePath, new ExcelReader(sourceFilePath) }
};
GetSchemaForTableFromFile(result, excelReaders, true);
}
catch (Exception ex)
{
Logger?.Error(string.Format("GetOriginalSourceSchema error reading file: {0} message: {1} stack: {2}", sourceFilePath, ex.Message, ex.StackTrace));
}
}
}

return result;
Expand All @@ -103,28 +130,41 @@ public override string WorkingDirectory
set { workingDirectory = value.Replace("\\", "/"); }
}

private string GetSourceFilePath()
private string GetSourceFilePath(string filePath)
{
string srcFilePath = string.Empty;

if (!string.IsNullOrEmpty(SourceFile))
if (!string.IsNullOrEmpty(filePath))
{
if (SourceFile.StartsWith(".."))
if (filePath.StartsWith(".."))
{
srcFilePath = WorkingDirectory.CombinePaths(SourceFile.TrimStart(new char[] { '.' })).Replace("\\", "/");
srcFilePath = WorkingDirectory.CombinePaths(filePath.TrimStart(new char[] { '.' })).Replace("\\", "/");
}
else
{
srcFilePath = SystemInformation.MapPath(FilePathHelper.GetRelativePath(SourceFile, "/Files"));
srcFilePath = SystemInformation.MapPath(FilePathHelper.GetRelativePath(filePath, "/Files"));
}
}
return srcFilePath;
}

private string SourceFolderPath => SystemInformation.MapPath(FilePathHelper.GetRelativePath(SourceFolder, "/Files"));

private IEnumerable<string> GetSourceFolderFiles()
{
var folderPath = SourceFolderPath;
if (Directory.Exists(folderPath))
{
return Directory.EnumerateFiles(folderPath, ExcelFilesSearchPattern, SearchOption.TopDirectoryOnly);
}
return Enumerable.Empty<string>();
}

public override void UpdateSourceSettings(ISource source)
{
ExcelProvider newProvider = (ExcelProvider)source;
SourceFile = newProvider.SourceFile;
SourceFolder = newProvider.SourceFolder;
}

public override string Serialize()
Expand All @@ -135,6 +175,7 @@ public override string Serialize()
document.Add(root);

root.Add(CreateParameterNode(GetType(), "Source file", SourceFile));
root.Add(CreateParameterNode(GetType(), "Source folder", SourceFolder));
root.Add(CreateParameterNode(GetType(), "Destination file", DestinationFile));
root.Add(CreateParameterNode(GetType(), "Destination folder", DestinationFolder));

Expand All @@ -144,6 +185,7 @@ public override string Serialize()
void ISource.SaveAsXml(XmlTextWriter xmlTextWriter)
{
xmlTextWriter.WriteElementString("SourcePath", SourceFile);
xmlTextWriter.WriteElementString("SourceFolder", SourceFolder);
(this as ISource).GetSchema().SaveAsXml(xmlTextWriter);
}

Expand All @@ -156,28 +198,45 @@ void IDestination.SaveAsXml(XmlTextWriter xmlTextWriter)

public new ISourceReader GetReader(Mapping mapping)
{
if (SourceFile.EndsWith(".xlsx", StringComparison.OrdinalIgnoreCase) ||
SourceFile.EndsWith(".xls", StringComparison.OrdinalIgnoreCase) ||
SourceFile.EndsWith(".xlsm", StringComparison.OrdinalIgnoreCase))
string filePath;
if (!IsFolderUsed)
{
if (!string.IsNullOrEmpty(WorkingDirectory))
{
var sourceFilePath = GetSourceFilePath();
if (!File.Exists(sourceFilePath))
throw new Exception($"Source file {SourceFile} does not exist - Working Directory {WorkingDirectory}");
filePath = SourceFile;

return new ExcelSourceReader(sourceFilePath, mapping, this);
}
else
if (filePath.EndsWith(".xlsx", StringComparison.OrdinalIgnoreCase) ||
filePath.EndsWith(".xls", StringComparison.OrdinalIgnoreCase) ||
filePath.EndsWith(".xlsm", StringComparison.OrdinalIgnoreCase))
{
if (!File.Exists(SourceFile))
throw new Exception($"Source file {SourceFile} does not exist - Working Directory {WorkingDirectory}");
if (!string.IsNullOrEmpty(WorkingDirectory))
{
var sourceFilePath = GetSourceFilePath(filePath);
if (!File.Exists(sourceFilePath))
throw new Exception($"Source file {SourceFile} does not exist - Working Directory {WorkingDirectory}");

return new ExcelSourceReader(SourceFile, mapping, this);
return new ExcelSourceReader(sourceFilePath, mapping, this);
}
else
{
if (!File.Exists(filePath))
throw new Exception($"Source file {filePath} does not exist - Working Directory {WorkingDirectory}");

return new ExcelSourceReader(filePath, mapping, this);
}
}
else
throw new Exception("The file is not a Excel file");
}
else
throw new Exception("The file is not a Excel file");
{
string folderPath = SourceFolderPath;
var fileName = mapping.SourceTable.SqlSchema;
filePath = Directory.EnumerateFiles(folderPath, ExcelFilesSearchPattern, SearchOption.TopDirectoryOnly).FirstOrDefault(f => f.EndsWith(fileName, StringComparison.OrdinalIgnoreCase));
if (!File.Exists(filePath))
{
throw new Exception($"Source file {fileName} does not exist in the Directory {folderPath}");
}
return new ExcelSourceReader(filePath, mapping, this);
}
}

public override void Close()
Expand Down Expand Up @@ -264,13 +323,17 @@ private CultureInfo GetCultureInfo(string culture)
return CultureInfo.CurrentCulture;
}

private void GetSchemaForTableFromFile(Schema schema, Dictionary<string, ExcelReader> excelReaders)
private void GetSchemaForTableFromFile(Schema schema, Dictionary<string, ExcelReader> excelReaders, bool isFolderUsed = false)
{
foreach (var reader in excelReaders)
{
foreach (DataTable dt in reader.Value.ExcelSet.Tables)
{
Table excelTable = schema.AddTable(dt.TableName);
if (isFolderUsed)
{
excelTable.SqlSchema = Path.GetFileName(reader.Key);
}
try
{
int columnCount;
Expand Down Expand Up @@ -339,6 +402,12 @@ public ExcelProvider(XmlNode xmlNode)
SourceFile = node.FirstChild.Value;
}
break;
case "SourceFolder":
if (node.HasChildNodes)
{
SourceFolder = node.FirstChild.Value;
}
break;
case "DestinationFile":
if (node.HasChildNodes)
{
Expand Down Expand Up @@ -373,7 +442,7 @@ public override void OverwriteSourceSchemaToOriginal()
}

public override void OverwriteDestinationSchemaToOriginal()
{
{
}

public override string ValidateDestinationSettings()
Expand All @@ -388,40 +457,69 @@ public override string ValidateDestinationSettings()

public override string ValidateSourceSettings()
{
ExcelPackage.LicenseContext = LicenseContext.Commercial;
if (SourceFile.EndsWith(".xlsx", StringComparison.OrdinalIgnoreCase) ||
SourceFile.EndsWith(".xls", StringComparison.OrdinalIgnoreCase) ||
SourceFile.EndsWith(".xlsm", StringComparison.OrdinalIgnoreCase))
if (string.IsNullOrEmpty(SourceFile) && string.IsNullOrEmpty(SourceFolder))
{
string filename = GetSourceFilePath();
if (!File.Exists(filename))
return "No Source file neither folder are selected";
}
if (IsFolderUsed)
{
string srcFolderPath = SourceFolderPath;

if (!Directory.Exists(srcFolderPath))
{
return $"Excel file \"{SourceFile}\" does not exist. WorkingDirectory - {WorkingDirectory}";
return "Source folder \"" + SourceFolder + "\" does not exist";
}
else
{
var files = GetSourceFolderFiles();

try
if (files.Count() == 0)
{
return "There are no Excel files with the extensions: [*.xlsx, *.xls, *.xlsm] in the source folder ";
}
}
}
else
{
ExcelPackage.LicenseContext = LicenseContext.Commercial;
if (SourceFile.EndsWith(".xlsx", StringComparison.OrdinalIgnoreCase) ||
SourceFile.EndsWith(".xls", StringComparison.OrdinalIgnoreCase) ||
SourceFile.EndsWith(".xlsm", StringComparison.OrdinalIgnoreCase))
{
using (var package = new ExcelPackage(new FileInfo(filename)))
string filename = GetSourceFilePath(SourceFile);
if (!File.Exists(filename))
{
foreach (var worksheet in package.Workbook.Worksheets)
{
string sheetName = worksheet.Name;
return $"Excel file \"{SourceFile}\" does not exist. WorkingDirectory - {WorkingDirectory}";
}

if (sheetName.Contains(' '))
try
{
using (var package = new ExcelPackage(new FileInfo(filename)))
{
foreach (var worksheet in package.Workbook.Worksheets)
{
return $"{sheetName} contains whitespaces";
string sheetName = worksheet.Name;

if (sheetName.Contains(' '))
{
return $"{sheetName} contains whitespaces";
}
}
}
}
catch (Exception ex)
{
return $"Could not open source file: {filename} message: {ex.Message} stack: {ex.StackTrace}";
}
}
catch (Exception ex)
else
{
return $"Could not open source file: {filename} message: {ex.Message} stack: {ex.StackTrace}";
return "The file is not an Excel file";
}
}
else
if (!string.IsNullOrEmpty(SourceFile) && !string.IsNullOrEmpty(SourceFolder))
{
return "The file is not an Excel file";
return "Warning: In your Excel Provider source, you selected both a source file and a source folder. The source folder selection will be ignored, and only the source file will be used.";
}
return null;
}
Expand Down