CDM/objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Tests/Persistence/ModelJson/DataPartitionTests.cs at 2bc3eedd7b0bb937b58f6ea6abfb7a2b6d26c6f0 · microsoft/CDM · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.

namespace Microsoft.CommonDataModel.ObjectModel.Tests.Persistence.ModelJson
{
    using System.IO;

    using Microsoft.CommonDataModel.ObjectModel.Cdm;
    using Microsoft.VisualStudio.TestTools.UnitTesting;
    using modelJsonPersistence = Microsoft.CommonDataModel.ObjectModel.Persistence.ModelJson;

    using Newtonsoft.Json;
    using System.Threading.Tasks;
    using Newtonsoft.Json.Linq;
    using Microsoft.CommonDataModel.ObjectModel.Persistence.ModelJson.types;
    using Microsoft.CommonDataModel.ObjectModel.Utilities;
    using Microsoft.CommonDataModel.ObjectModel.Persistence.ModelJson;

    [TestClass]
    public class DataPartitionTest : ModelJsonTestsBase
    {
        /// <summary>
        /// The path between TestDataPath and TestName.
        /// </summary>
        private readonly string testsSubpath = Path.Combine("Persistence", "ModelJson", "DataPartition");

        /// <summary>
        /// Testing whether DataPartition Location is consistently populated when:
        /// 1. Manifest is read directly.
        /// 2. Manifest is obtained by converting a model.json.
        /// </summary>
        [TestMethod]
        public async Task TestModelJsonDataPartitionLocationConsistency()
        {
            var cdmCorpus = TestHelper.GetLocalCorpus(testsSubpath, "TestModelJsonDataPartitionLocationConsistency");
            var manifestRead = await cdmCorpus.FetchObjectAsync<CdmManifestDefinition>("default.manifest.cdm.json", cdmCorpus.Storage.FetchRootFolder("local"));
            Assert.AreEqual("EpisodeOfCare/partition-data.csv", manifestRead.Entities[0].DataPartitions[0].Location);

            var convertedToModelJson = await modelJsonPersistence.ManifestPersistence.ToData(manifestRead, null, null);
            string location = (convertedToModelJson.Entities[0]["partitions"][0]["location"] as JValue).Value<string>();
            // Model Json uses absolute adapter path.
            Assert.IsTrue(location.Contains(Path.Combine("TestData", "Persistence", "ModelJson", "DataPartition", "TestModelJsonDataPartitionLocationConsistency", "Input", "EpisodeOfCare", "partition-data.csv")));

            var cdmCorpus2 = TestHelper.GetLocalCorpus(testsSubpath, "TestModelJsonDataPartitionLocationConsistency");
            var manifestAfterConvertion = await modelJsonPersistence.ManifestPersistence.FromObject(cdmCorpus2.Ctx, convertedToModelJson, cdmCorpus2.Storage.FetchRootFolder("local"));
            Assert.AreEqual("EpisodeOfCare/partition-data.csv", manifestAfterConvertion.Entities[0].DataPartitions[0].Location);
        }

        /// <summary>
        /// Tests that the trait is.partition.format.CSV is merged with the fileFormatSettings property during load.
        /// </summary>
        [TestMethod]
        public async Task TestLoadingCsvPartitionTraits()
        {
            var cdmCorpus = TestHelper.GetLocalCorpus(testsSubpath, nameof(TestLoadingCsvPartitionTraits));
            var manifest = await cdmCorpus.FetchObjectAsync<CdmManifestDefinition>("model.json");

            var dataPartition = manifest.Entities[0].DataPartitions[0];

            // Ensure that the fileFormatSettings and the is.partition.format.CSV trait got merged into one trait.
            Assert.AreEqual(1, dataPartition.ExhibitsTraits.Count);

            var csvTrait = dataPartition.ExhibitsTraits[0] as CdmTraitReference;

            Assert.AreEqual("true", csvTrait.Arguments[0].Value);
            Assert.AreEqual("CsvStyle.QuoteAlways", csvTrait.Arguments[1].Value);
            Assert.AreEqual(",", csvTrait.Arguments[2].Value);
            Assert.AreEqual("QuoteStyle.Csv", csvTrait.Arguments[3].Value);
            Assert.AreEqual("UTF-8", csvTrait.Arguments[4].Value);
            Assert.AreEqual("\n", csvTrait.Arguments[5].Value);
        }

        /// <summary>
        /// Tests that the trait is.partition.format.CSV is merged with the fileFormatSettings property during load.
        /// Given that the trait is does not have arguments present on fileFormatSettings.
        /// </summary>
        [TestMethod]
        public async Task TestLoadingCsvPartitionTraitsFromFileFormatSettings()
        {
            var cdmCorpus = TestHelper.GetLocalCorpus(testsSubpath, nameof(TestLoadingCsvPartitionTraitsFromFileFormatSettings));
            var manifest = await cdmCorpus.FetchObjectAsync<CdmManifestDefinition>("model.json");

            var dataPartition = manifest.Entities[0].DataPartitions[0];

            // Ensure that the fileFormatSettings and the is.partition.format.CSV trait got merged into one trait.
            Assert.AreEqual(1, dataPartition.ExhibitsTraits.Count);

            var csvTrait = dataPartition.ExhibitsTraits[0] as CdmTraitReference;

            Assert.AreEqual("\n", csvTrait.Arguments[0].Value);
            Assert.AreEqual("false", csvTrait.Arguments[1].Value);
            Assert.AreEqual("CsvStyle.QuoteAfterDelimiter", csvTrait.Arguments[2].Value);
            Assert.AreEqual(";", csvTrait.Arguments[3].Value);
            Assert.AreEqual("QuoteStyle.None", csvTrait.Arguments[4].Value);
            Assert.AreEqual("ASCII", csvTrait.Arguments[5].Value);
        }

        /// <summary>
        /// Tests that the trait is.partition.format.CSV is saved when contains arguments not supported by fileFormatSettings.
        /// </summary>
        [TestMethod]
        public async Task TestLoadingAndSavingCsvPartitionTraits()
        {
            var cdmCorpus = TestHelper.GetLocalCorpus(testsSubpath, nameof(TestLoadingCsvPartitionTraitsFromFileFormatSettings));
            var manifest = await cdmCorpus.FetchObjectAsync<CdmManifestDefinition>("model.json");

            // If the data partition trait is.partition.format.CSV being saved has arguments that are not supported by fileFormatSettings
            // the trait should also be persisted.
            var manifestData = await ManifestPersistence.ToData(manifest, new ResolveOptions(manifest.InDocument), new CopyOptions());
            var localEntity = manifestData.Entities[0].ToObject<LocalEntity>();
            Assert.AreEqual(1, localEntity.Partitions[0].Traits.Count);

            // Remove the argument that is not supported by fileFormatSettings and check if the trait is removed after that.
            var csvTrait = manifest.Entities[0].DataPartitions[0].ExhibitsTraits[0] as CdmTraitReference;
            csvTrait.Arguments.Remove(csvTrait.Arguments.Item("newline"));

            manifestData = await ManifestPersistence.ToData(manifest, new ResolveOptions(manifest.InDocument), new CopyOptions());
            localEntity = manifestData.Entities[0].ToObject<LocalEntity>();
            Assert.IsNull(localEntity.Partitions[0].Traits);
        }
    }
}