From b27b4309b4a970978f4b1649efbbdb1bb9e8786b Mon Sep 17 00:00:00 2001 From: Aravind Ramachandran Date: Mon, 2 Oct 2017 12:01:27 -0700 Subject: [PATCH] add uploader --- cosmosdb-uploader/.gitignore | 4 + cosmosdb-uploader/App.config | 34 ++ cosmosdb-uploader/CokeTrans.json | 14 + cosmosdb-uploader/DocumentDBBenchmark.csproj | 93 ++++++ cosmosdb-uploader/DocumentDBBenchmark.sln | 22 ++ cosmosdb-uploader/KeyValue.json | 4 + cosmosdb-uploader/Player.json | 26 ++ cosmosdb-uploader/Program.cs | 328 +++++++++++++++++++ cosmosdb-uploader/packages.config | 5 + 9 files changed, 530 insertions(+) create mode 100644 cosmosdb-uploader/.gitignore create mode 100644 cosmosdb-uploader/App.config create mode 100644 cosmosdb-uploader/CokeTrans.json create mode 100644 cosmosdb-uploader/DocumentDBBenchmark.csproj create mode 100644 cosmosdb-uploader/DocumentDBBenchmark.sln create mode 100644 cosmosdb-uploader/KeyValue.json create mode 100644 cosmosdb-uploader/Player.json create mode 100644 cosmosdb-uploader/Program.cs create mode 100644 cosmosdb-uploader/packages.config diff --git a/cosmosdb-uploader/.gitignore b/cosmosdb-uploader/.gitignore new file mode 100644 index 0000000..ff57bb8 --- /dev/null +++ b/cosmosdb-uploader/.gitignore @@ -0,0 +1,4 @@ +bin/ +obj/ +packages/ +ElasticCollectionsDemo.v12.suo \ No newline at end of file diff --git a/cosmosdb-uploader/App.config b/cosmosdb-uploader/App.config new file mode 100644 index 0000000..7e85273 --- /dev/null +++ b/cosmosdb-uploader/App.config @@ -0,0 +1,34 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/cosmosdb-uploader/CokeTrans.json b/cosmosdb-uploader/CokeTrans.json new file mode 100644 index 0000000..e82cc87 --- /dev/null +++ b/cosmosdb-uploader/CokeTrans.json @@ -0,0 +1,14 @@ +{ + "shardkey": "fffa", + "id": "G02-D06-a067ff", + "activityId": "fffa", + "version": 1, + "submitDt": "2017-01-01", + "transDt": "2017-01-01", + "finDt": "2017-01-01", + "key": "A", + "pcTotal": 100, + "pcAdj": 200, + "ucTotal": 110, + "ucAdj": 210 +} \ No newline at end of file diff --git a/cosmosdb-uploader/DocumentDBBenchmark.csproj b/cosmosdb-uploader/DocumentDBBenchmark.csproj new file mode 100644 index 0000000..7371455 --- /dev/null +++ b/cosmosdb-uploader/DocumentDBBenchmark.csproj @@ -0,0 +1,93 @@ + + + + + Debug + AnyCPU + {0AEFFA67-EC42-4BEB-998B-7F0B9379436D} + Exe + Properties + DocumentDBBenchmark + DocumentDBBenchmark + v4.5.1 + 512 + + + + + + x64 + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + packages\Microsoft.Azure.DocumentDB.1.17.0\lib\net45\Microsoft.Azure.Documents.Client.dll + True + + + packages\Newtonsoft.Json.6.0.8\lib\net45\Newtonsoft.Json.dll + True + + + + + + + + + + + + + + + + + + Designer + + + Always + + + Always + + + + Always + + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + \ No newline at end of file diff --git a/cosmosdb-uploader/DocumentDBBenchmark.sln b/cosmosdb-uploader/DocumentDBBenchmark.sln new file mode 100644 index 0000000..a09f413 --- /dev/null +++ b/cosmosdb-uploader/DocumentDBBenchmark.sln @@ -0,0 +1,22 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 2013 +VisualStudioVersion = 12.0.40629.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DocumentDBBenchmark", "DocumentDBBenchmark.csproj", "{0AEFFA67-EC42-4BEB-998B-7F0B9379436D}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {0AEFFA67-EC42-4BEB-998B-7F0B9379436D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0AEFFA67-EC42-4BEB-998B-7F0B9379436D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0AEFFA67-EC42-4BEB-998B-7F0B9379436D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0AEFFA67-EC42-4BEB-998B-7F0B9379436D}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/cosmosdb-uploader/KeyValue.json b/cosmosdb-uploader/KeyValue.json new file mode 100644 index 0000000..8ca1816 --- /dev/null +++ b/cosmosdb-uploader/KeyValue.json @@ -0,0 +1,4 @@ +{ + "id":"G02-D06-a067ff", + "playerId":"a067ff" +} \ No newline at end of file diff --git a/cosmosdb-uploader/Player.json b/cosmosdb-uploader/Player.json new file mode 100644 index 0000000..21633d4 --- /dev/null +++ b/cosmosdb-uploader/Player.json @@ -0,0 +1,26 @@ +{ + "id": "G02-D06-a067ff", + "playerId": "a067ff", + "hashedId": "bb0091", + "countryCode": "hk", + "nickname": "DannyBoy", + "nicknameLower": "dannyboy", + "score": 0, + "secondaryScore": 18, + "indexScore": 0.18, + "level": 1, + "lastSaveUnixTime": 1446591499, + "lastLoadUnixTime": 1446590552, + "disconnectedUnixTime": 1446591499, + "facebookId": "FB_1010006092353214", + "gameCenterId": "GC_G:1939511430", + "chatMessages": [ + { + "playerId": "67879d8e", + "name": "lee", + "message": "hi", + "time": 760455049, + "notificationType": "None" + } + ] +} \ No newline at end of file diff --git a/cosmosdb-uploader/Program.cs b/cosmosdb-uploader/Program.cs new file mode 100644 index 0000000..9049a43 --- /dev/null +++ b/cosmosdb-uploader/Program.cs @@ -0,0 +1,328 @@ +namespace DocumentDBBenchmark +{ + using System; + using System.Collections.Generic; + using System.Collections.Concurrent; + using System.Configuration; + using System.Diagnostics; + using System.Net; + using System.IO; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Azure.Documents; + using Microsoft.Azure.Documents.Client; + using Newtonsoft.Json; + + /// + /// This sample demonstrates how to achieve high performance writes using DocumentDB. + /// + public sealed class Program + { + private static readonly string DatabaseName = ConfigurationManager.AppSettings["DatabaseName"]; + private static readonly string DataCollectionName = ConfigurationManager.AppSettings["CollectionName"]; + private static readonly int CollectionThroughput = int.Parse(ConfigurationManager.AppSettings["CollectionThroughput"]); + + private static readonly ConnectionPolicy ConnectionPolicy = new ConnectionPolicy + { + RequestTimeout = new TimeSpan(1, 0, 0), + MaxConnectionLimit = 1000, + RetryOptions = new RetryOptions + { + MaxRetryAttemptsOnThrottledRequests = 10, + MaxRetryWaitTimeInSeconds = 60 + } + }; + + private static readonly string InstanceId = Dns.GetHostEntry("LocalHost").HostName + Process.GetCurrentProcess().Id; + private const int MinThreadPoolSize = 100; + + private int pendingTaskCount; + private long documentsInserted; + private ConcurrentDictionary requestUnitsConsumed = new ConcurrentDictionary(); + private DocumentClient client; + + /// + /// Initializes a new instance of the class. + /// + /// The DocumentDB client instance. + private Program(DocumentClient client) + { + this.client = client; + } + + /// + /// Main method for the sample. + /// + /// command line arguments. + public static void Main(string[] args) + { + ThreadPool.SetMinThreads(MinThreadPoolSize, MinThreadPoolSize); + + string endpoint = ConfigurationManager.AppSettings["EndPointUrl"]; + string authKey = ConfigurationManager.AppSettings["AuthorizationKey"]; + + Console.WriteLine("Summary:"); + Console.WriteLine("--------------------------------------------------------------------- "); + Console.WriteLine("Endpoint: {0}", endpoint); + Console.WriteLine("Collection : {0}.{1} at {2} request units per second", DatabaseName, DataCollectionName, ConfigurationManager.AppSettings["CollectionThroughput"]); + Console.WriteLine("Document Template*: {0}", ConfigurationManager.AppSettings["DocumentTemplateFile"]); + Console.WriteLine("Degree of parallelism*: {0}", ConfigurationManager.AppSettings["DegreeOfParallelism"]); + Console.WriteLine("--------------------------------------------------------------------- "); + Console.WriteLine(); + + Console.WriteLine("DocumentDBBenchmark starting..."); + + try + { + using (var client = new DocumentClient( + new Uri(endpoint), + authKey, + ConnectionPolicy)) + { + var program = new Program(client); + program.RunAsync().Wait(); + Console.WriteLine("DocumentDBBenchmark completed successfully."); + } + } + +#if !DEBUG + catch (Exception e) + { + // If the Exception is a DocumentClientException, the "StatusCode" value might help identity + // the source of the problem. + Console.WriteLine("Samples failed with exception:{0}", e); + } +#endif + + finally + { + Console.WriteLine("Press any key to exit..."); + Console.ReadLine(); + } + } + + /// + /// Run samples for Order By queries. + /// + /// a Task object. + private async Task RunAsync() + { + + DocumentCollection dataCollection = GetCollectionIfExists(DatabaseName, DataCollectionName); + int currentCollectionThroughput = 0; + + if (bool.Parse(ConfigurationManager.AppSettings["ShouldCleanupOnStart"]) || dataCollection == null) + { + Database database = GetDatabaseIfExists(DatabaseName); + if (database != null) + { + await client.DeleteDatabaseAsync(database.SelfLink); + } + + Console.WriteLine("Creating database {0}", DatabaseName); + database = await client.CreateDatabaseAsync(new Database { Id = DatabaseName }); + + Console.WriteLine("Creating collection {0} with {1} RU/s", DataCollectionName, CollectionThroughput); + dataCollection = await this.CreatePartitionedCollectionAsync(DatabaseName, DataCollectionName); + + currentCollectionThroughput = CollectionThroughput; + } + else + { + OfferV2 offer = (OfferV2)client.CreateOfferQuery().Where(o => o.ResourceLink == dataCollection.SelfLink).AsEnumerable().FirstOrDefault(); + currentCollectionThroughput = offer.Content.OfferThroughput; + + Console.WriteLine("Found collection {0} with {1} RU/s", DataCollectionName, currentCollectionThroughput); + } + + int taskCount; + int degreeOfParallelism = int.Parse(ConfigurationManager.AppSettings["DegreeOfParallelism"]); + + if (degreeOfParallelism == -1) + { + // set TaskCount = 10 for each 10k RUs + taskCount = currentCollectionThroughput / 1000; + if (taskCount >= 250) + { + taskCount = 250; + } + } + else + { + taskCount = degreeOfParallelism; + } + + Console.WriteLine("Starting Inserts with {0} tasks", taskCount); + string sampleDocument = File.ReadAllText(ConfigurationManager.AppSettings["DocumentTemplateFile"]); + + pendingTaskCount = taskCount; + var tasks = new List(); + tasks.Add(this.LogOutputStats()); + + long numberOfDocumentsToInsert = long.Parse(ConfigurationManager.AppSettings["NumberOfDocumentsToInsert"]) / taskCount; + for (var i = 0; i < taskCount; i++) + { + tasks.Add(this.InsertDocument(i, client, dataCollection, sampleDocument, numberOfDocumentsToInsert)); + } + + await Task.WhenAll(tasks); + + if (bool.Parse(ConfigurationManager.AppSettings["ShouldCleanupOnFinish"])) + { + Console.WriteLine("Deleting Database {0}", DatabaseName); + await client.DeleteDatabaseAsync(UriFactory.CreateDatabaseUri(DatabaseName)); + } + } + + private async Task InsertDocument(int taskId, DocumentClient client, DocumentCollection collection, string sampleJson, long numberOfDocumentsToInsert) + { + requestUnitsConsumed[taskId] = 0; + string partitionKeyProperty = collection.PartitionKey.Paths[0].Replace("/", ""); + Dictionary newDictionary = JsonConvert.DeserializeObject>(sampleJson); + + for (var i = 0; i < numberOfDocumentsToInsert; i++) + { + newDictionary["id"] = Guid.NewGuid().ToString(); + newDictionary[partitionKeyProperty] = Guid.NewGuid().ToString(); + + List dateKeys = newDictionary.Keys.Where(k => k.EndsWith("Dt")).ToList(); + foreach (string propertyName in dateKeys) + { + newDictionary[propertyName] = DateTime.UtcNow; + } + + try + { + ResourceResponse response = await client.CreateDocumentAsync( + UriFactory.CreateDocumentCollectionUri(DatabaseName, DataCollectionName), + newDictionary, + new RequestOptions() { }); + + string partition = response.SessionToken.Split(':')[0]; + requestUnitsConsumed[taskId] += response.RequestCharge; + Interlocked.Increment(ref this.documentsInserted); + } + catch (Exception e) + { + if (e is DocumentClientException) + { + DocumentClientException de = (DocumentClientException)e; + if (de.StatusCode != HttpStatusCode.Forbidden) + { + Trace.TraceError("Failed to write {0}. Exception was {1}", JsonConvert.SerializeObject(newDictionary), e); + } + else + { + Interlocked.Increment(ref this.documentsInserted); + } + } + } + } + + Interlocked.Decrement(ref this.pendingTaskCount); + } + + private async Task LogOutputStats() + { + long lastCount = 0; + double lastRequestUnits = 0; + double lastSeconds = 0; + double requestUnits = 0; + double ruPerSecond = 0; + double ruPerMonth = 0; + + Stopwatch watch = new Stopwatch(); + watch.Start(); + + while (this.pendingTaskCount > 0) + { + await Task.Delay(TimeSpan.FromSeconds(1)); + double seconds = watch.Elapsed.TotalSeconds; + + requestUnits = 0; + foreach (int taskId in requestUnitsConsumed.Keys) + { + requestUnits += requestUnitsConsumed[taskId]; + } + + long currentCount = this.documentsInserted; + ruPerSecond = (requestUnits / seconds); + ruPerMonth = ruPerSecond * 86400 * 30; + + Console.WriteLine("Inserted {0} docs @ {1} writes/s, {2} RU/s ({3}B max monthly 1KB reads)", + currentCount, + Math.Round(this.documentsInserted / seconds), + Math.Round(ruPerSecond), + Math.Round(ruPerMonth / (1000 * 1000 * 1000))); + + lastCount = documentsInserted; + lastSeconds = seconds; + lastRequestUnits = requestUnits; + } + + double totalSeconds = watch.Elapsed.TotalSeconds; + ruPerSecond = (requestUnits / totalSeconds); + ruPerMonth = ruPerSecond * 86400 * 30; + + Console.WriteLine(); + Console.WriteLine("Summary:"); + Console.WriteLine("--------------------------------------------------------------------- "); + Console.WriteLine("Inserted {0} docs @ {1} writes/s, {2} RU/s ({3}B max monthly 1KB reads)", + lastCount, + Math.Round(this.documentsInserted / watch.Elapsed.TotalSeconds), + Math.Round(ruPerSecond), + Math.Round(ruPerMonth / (1000 * 1000 * 1000))); + Console.WriteLine("--------------------------------------------------------------------- "); + } + + /// + /// Create a partitioned collection. + /// + /// The created collection. + private async Task CreatePartitionedCollectionAsync(string databaseName, string collectionName) + { + DocumentCollection existingCollection = GetCollectionIfExists(databaseName, collectionName); + + DocumentCollection collection = new DocumentCollection(); + collection.Id = collectionName; + collection.PartitionKey.Paths.Add(ConfigurationManager.AppSettings["CollectionPartitionKey"]); + + // Show user cost of running this test + double estimatedCostPerMonth = 0.06 * CollectionThroughput; + double estimatedCostPerHour = estimatedCostPerMonth / (24 * 30); + Console.WriteLine("The collection will cost an estimated ${0} per hour (${1} per month)", Math.Round(estimatedCostPerHour, 2), Math.Round(estimatedCostPerMonth, 2)); + Console.WriteLine("Press enter to continue ..."); + Console.ReadLine(); + + return await client.CreateDocumentCollectionAsync( + UriFactory.CreateDatabaseUri(databaseName), + collection, + new RequestOptions { OfferThroughput = CollectionThroughput }); + } + + /// + /// Get the database if it exists, null if it doesn't + /// + /// The requested database + private Database GetDatabaseIfExists(string databaseName) + { + return client.CreateDatabaseQuery().Where(d => d.Id == databaseName).AsEnumerable().FirstOrDefault(); + } + + /// + /// Get the collection if it exists, null if it doesn't + /// + /// The requested collection + private DocumentCollection GetCollectionIfExists(string databaseName, string collectionName) + { + if (GetDatabaseIfExists(databaseName) == null) + { + return null; + } + + return client.CreateDocumentCollectionQuery(UriFactory.CreateDatabaseUri(databaseName)) + .Where(c => c.Id == collectionName).AsEnumerable().FirstOrDefault(); + } + } +} diff --git a/cosmosdb-uploader/packages.config b/cosmosdb-uploader/packages.config new file mode 100644 index 0000000..554cc00 --- /dev/null +++ b/cosmosdb-uploader/packages.config @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file