Skip to content

added sqlstatements to mask/delete/truncate tables #49

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
elcoda wants to merge 1 commit into
base: master
Choose a base branch
from
Open

added sqlstatements to mask/delete/truncate tables #49

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
278 changes: 147 additions & 131 deletions src/DataMasker.Runner/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,165 +13,181 @@

namespace DataMasker.Runner
{
internal class Program
{
private static readonly Dictionary<ProgressType, ProgressbarUpdate> _progressBars = new Dictionary<ProgressType, ProgressbarUpdate>();

private static Options cliOptions;

private static void Main(
string[] args)
internal class Program
{
private static readonly Dictionary<ProgressType, ProgressbarUpdate> _progressBars = new Dictionary<ProgressType, ProgressbarUpdate>();

Parser.Default.ParseArguments<Options>(args)
.WithParsed(
options =>
{
cliOptions = options;
try
{
RuntimeArgumentHandle();
}
catch (Exception ex)
{
WriteLine(ex.Message);
}
});
}
private static Options cliOptions;

private static void InitProgressBars()
{
if (cliOptions.NoOutput)
{
return;
}

_progressBars.Add(
ProgressType.Overall,
new ProgressbarUpdate { ProgressBar = new ProgressBar(PbStyle.SingleLine, 0), LastMessage = "Overall Progress" });

_progressBars.Add(
ProgressType.Updating,
new ProgressbarUpdate { ProgressBar = new ProgressBar(PbStyle.SingleLine, 0), LastMessage = "Update Progress" });
private static void Main(
string[] args)
{

_progressBars.Add(
ProgressType.Masking,
new ProgressbarUpdate { ProgressBar = new ProgressBar(PbStyle.SingleLine, 0), LastMessage = "Masking Progress" });
}
Parser.Default.ParseArguments<Options>(args)
.WithParsed(
options =>
{
cliOptions = options;
try
{
RuntimeArgumentHandle();
}
catch (Exception ex)
{
WriteLine(ex.Message);
}
});
}

private static void InitProgressBars()
{
if (cliOptions.NoOutput)
{
return;
}

_progressBars.Add(
ProgressType.Overall,
new ProgressbarUpdate { ProgressBar = new ProgressBar(PbStyle.SingleLine, 0), LastMessage = "Overall Progress" });

_progressBars.Add(
ProgressType.Updating,
new ProgressbarUpdate { ProgressBar = new ProgressBar(PbStyle.SingleLine, 0), LastMessage = "Update Progress" });

_progressBars.Add(
ProgressType.Masking,
new ProgressbarUpdate { ProgressBar = new ProgressBar(PbStyle.SingleLine, 0), LastMessage = "Masking Progress" });
}

private static void UpdateProgress(
ProgressType progressType,
int current,
int? max = null,
string message = null)
{
if (cliOptions.NoOutput)
{
return;
}

private static void UpdateProgress(
ProgressType progressType,
int current,
int? max = null,
string message = null)
{
if (cliOptions.NoOutput)
{
return;
}
max = max ??
_progressBars[progressType]
.ProgressBar.Max;

max = max ??
_progressBars[progressType]
.ProgressBar.Max;
.ProgressBar.Max = max.Value;

_progressBars[progressType]
.ProgressBar.Max = max.Value;
message = message ??
_progressBars[progressType]
.LastMessage;

message = message ??
_progressBars[progressType]
.LastMessage;
_progressBars[progressType]
.ProgressBar.Refresh(current, message);
}
private static void RuntimeArgumentHandle()
{
if (cliOptions.PrintOptions)
{
WriteLine();
WriteLine(JsonConvert.SerializeObject(cliOptions, Formatting.Indented));
WriteLine();
return;
}

InitProgressBars();
Config config = Config.Load(cliOptions.ConfigFile);
if (cliOptions.DryRun != null)
{
config.DataSource.DryRun = cliOptions.DryRun.Value;
}

if (!string.IsNullOrEmpty(cliOptions.Locale))
{
config.DataGeneration.Locale = cliOptions.Locale;
}

if (cliOptions.UpdateBatchSize != null)
{
config.DataSource.UpdateBatchSize = cliOptions.UpdateBatchSize;
}

Execute(config);
}

private static void WriteLine(
string message = null)
{
if (!cliOptions.NoOutput)
{
Console.WriteLine(message);
}
}

private static void Execute(
Config config)
{
WriteLine("Masking Data");
UpdateProgress(ProgressType.Overall, 0, config.Tables.Count, "Overall Progress");

_progressBars[progressType]
.ProgressBar.Refresh(current, message);
}
private static void RuntimeArgumentHandle()
{
if (cliOptions.PrintOptions)
{
WriteLine();
WriteLine(JsonConvert.SerializeObject(cliOptions, Formatting.Indented));
WriteLine();
return;
}

InitProgressBars();
Config config = Config.Load(cliOptions.ConfigFile);
if (cliOptions.DryRun != null)
var dataProviders = new List<IDataProvider>
{
config.DataSource.DryRun = cliOptions.DryRun.Value;
}
new BogusDataProvider(config.DataGeneration),
new SqlDataProvider(new System.Data.SqlClient.SqlConnection(config.DataSource.GetConnectionString()))
};

if (!string.IsNullOrEmpty(cliOptions.Locale))
{
config.DataGeneration.Locale = cliOptions.Locale;
}
//create a data masker
IDataMasker dataMasker = new DataMasker(dataProviders);

if (cliOptions.UpdateBatchSize != null)
{
config.DataSource.UpdateBatchSize = cliOptions.UpdateBatchSize;
}
//grab our dataSource from the config, note: you could just ignore the config.DataSource.Type
//and initialize your own instance
IDataSource dataSource = DataSourceProvider.Provide(config.DataSource.Type, config.DataSource);

Execute(config);
}
for (int i = 0; i < config.Tables.Count; i++)
{
TableConfig tableConfig = config.Tables[i];

private static void WriteLine(
string message = null)
{
if (!cliOptions.NoOutput)
{
Console.WriteLine(message);
}
}

private static void Execute(
Config config)
{
WriteLine("Masking Data");
UpdateProgress(ProgressType.Overall, 0, config.Tables.Count, "Overall Progress");
var rowCount = dataSource.GetCount(tableConfig);
UpdateProgress(ProgressType.Masking, 0, (int)rowCount, "Masking Progress");
UpdateProgress(ProgressType.Updating, 0, (int)rowCount, "Update Progress");

var dataProviders = new List<IDataProvider>
{
new BogusDataProvider(config.DataGeneration),
new SqlDataProvider(new System.Data.SqlClient.SqlConnection(config.DataSource.GetConnectionString()))
};
IEnumerable<IDictionary<string, object>> rows = dataSource.GetData(tableConfig);

//create a data masker
IDataMasker dataMasker = new DataMasker(dataProviders);
int rowIndex = 0;

//grab our dataSource from the config, note: you could just ignore the config.DataSource.Type
//and initialize your own instance
IDataSource dataSource = DataSourceProvider.Provide(config.DataSource.Type, config.DataSource);
var maskedRows = rows.Select(row =>
{
rowIndex++;

for (int i = 0; i < config.Tables.Count; i++)
{
TableConfig tableConfig = config.Tables[i];
//update per row, or see below,
//dataSource.UpdateRow(row, tableConfig);
UpdateProgress(ProgressType.Masking, rowIndex);

return dataMasker.Mask(row, tableConfig);
});

var rowCount = dataSource.GetCount(tableConfig);
UpdateProgress(ProgressType.Masking, 0, (int)rowCount, "Masking Progress");
UpdateProgress(ProgressType.Updating, 0, (int)rowCount, "Update Progress");
//update all rows
dataSource.UpdateRows(maskedRows, rowCount, tableConfig, totalUpdated => UpdateProgress(ProgressType.Updating, totalUpdated));
UpdateProgress(ProgressType.Overall, i + 1);
}

IEnumerable<IDictionary<string, object>> rows = dataSource.GetData(tableConfig);
ISqlStatementDataSource sqlStatementDataSource = SqlStatementDataProvier.Provide(config.DataSource.Type, config.DataSource);

int rowIndex = 0;
for (int i = 0; i < config.SqlStatements.Count; i++)
{
SqlStatementConfig sqlStatementConfig = config.SqlStatements[i];

var maskedRows = rows.Select(row =>
{
rowIndex++;
//IDataMasker dataMasker = new DataMasker(dataProviders);

//update per row, or see below,
//dataSource.UpdateRow(row, tableConfig);
UpdateProgress(ProgressType.Masking, rowIndex);
var rowCount = config.SqlStatements.Count();
UpdateProgress(ProgressType.Masking, 0, (int)rowCount, "Masking Progress");
UpdateProgress(ProgressType.Updating, 0, (int)rowCount, "Update Progress");
sqlStatementDataSource.Execute(sqlStatementConfig);

return dataMasker.Mask(row, tableConfig);
});

//update all rows
dataSource.UpdateRows(maskedRows, rowCount, tableConfig, totalUpdated => UpdateProgress(ProgressType.Updating, totalUpdated));
UpdateProgress(ProgressType.Overall, i + 1);
}
}

WriteLine("Done");
WriteLine("Done");
}
}
}
}
18 changes: 11 additions & 7 deletions src/DataMasker/DataMasker.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using System.Linq;
using System.Text;
using Bogus.DataSets;
using DataMasker.DataSources;
using DataMasker.Interfaces;
using DataMasker.Models;

Expand Down Expand Up @@ -38,13 +39,16 @@ public DataMasker(
_dataProviders = dataProviders;
}

/// <summary>
/// Masks the specified object with new data
/// </summary>
/// <param name="obj">The object to mask</param>
/// <param name="tableConfig">The table configuration.</param>
/// <returns></returns>
public IDictionary<string, object> Mask(



/// <summary>
/// Masks the specified object with new data
/// </summary>
/// <param name="obj">The object to mask</param>
/// <param name="tableConfig">The table configuration.</param>
/// <returns></returns>
public IDictionary<string, object> Mask(
IDictionary<string, object> obj,
TableConfig tableConfig)
{
Expand Down
2 changes: 2 additions & 0 deletions src/DataMasker/DataSourceProvider.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,5 +25,7 @@ public static IDataSource Provide(

throw new ArgumentOutOfRangeException(nameof(dataSourceType), dataSourceType, null);
}


}
}
Loading