Support snapshot reference (#232)

* wip

* support snapshot reference

* add test

* update lint rule

* update

Author: zhiyuanliang-ms

package-lock.json

@@ -66,7 +66,7 @@
     "playwright": "^1.55.0"
   },
   "dependencies": {
-    "@azure/app-configuration": "^1.9.0",
+    "@azure/app-configuration": "^1.10.0",
     "@azure/core-rest-pipeline": "^1.6.0",
     "@azure/identity": "^4.2.1",
     "@azure/keyvault-secrets": "^4.7.0",

package.json

@@ -11,6 +11,9 @@ import {
     featureFlagPrefix,
     isFeatureFlag,
     isSecretReference,
+    isSnapshotReference,
+    parseSnapshotReference,
+    SnapshotReferenceValue,
     GetSnapshotOptions,
     ListConfigurationSettingsForSnapshotOptions,
     GetSnapshotResponse,
@@ -57,7 +60,7 @@ import { AIConfigurationTracingOptions } from "./requestTracing/aiConfigurationT
 import { KeyFilter, LabelFilter, SettingWatcher, SettingSelector, PagedSettingsWatcher, WatchedSetting } from "./types.js";
 import { ConfigurationClientManager } from "./configurationClientManager.js";
 import { getFixedBackoffDuration, getExponentialBackoffDuration } from "./common/backoffUtils.js";
-import { InvalidOperationError, ArgumentError, isFailoverableError, isInputError } from "./common/errors.js";
+import { InvalidOperationError, ArgumentError, isFailoverableError, isInputError, SnapshotReferenceError } from "./common/errors.js";
 import { ErrorMessages } from "./common/errorMessages.js";
 
 const MIN_DELAY_FOR_UNHANDLED_FAILURE = 5_000; // 5 seconds
@@ -82,6 +85,7 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
     #featureFlagTracing: FeatureFlagTracingOptions | undefined;
     #fmVersion: string | undefined;
     #aiConfigurationTracing: AIConfigurationTracingOptions | undefined;
+    #useSnapshotReference: boolean = false;
 
     // Refresh
     #refreshInProgress: boolean = false;
@@ -213,7 +217,8 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
             isFailoverRequest: this.#isFailoverRequest,
             featureFlagTracing: this.#featureFlagTracing,
             fmVersion: this.#fmVersion,
-            aiConfigurationTracing: this.#aiConfigurationTracing
+            aiConfigurationTracing: this.#aiConfigurationTracing,
+            useSnapshotReference: this.#useSnapshotReference
         };
     }
 
@@ -504,17 +509,29 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
                 selector.pageWatchers = pageWatchers;
                 settings = items;
             } else { // snapshot selector
-                const snapshot = await this.#getSnapshot(selector.snapshotName);
-                if (snapshot === undefined) {
-                    throw new InvalidOperationError(`Could not find snapshot with name ${selector.snapshotName}.`);
-                }
-                if (snapshot.compositionType != KnownSnapshotComposition.Key) {
-                    throw new InvalidOperationError(`Composition type for the selected snapshot with name ${selector.snapshotName} must be 'key'.`);
-                }
-                settings = await this.#listConfigurationSettingsForSnapshot(selector.snapshotName);
+                settings = await this.#loadConfigurationSettingsFromSnapshot(selector.snapshotName);
             }
 
             for (const setting of settings) {
+                if (isSnapshotReference(setting) && !loadFeatureFlag) {
+                    this.#useSnapshotReference = true;
+
+                    const snapshotRef: ConfigurationSetting<SnapshotReferenceValue> = parseSnapshotReference(setting);
+                    const snapshotName = snapshotRef.value.snapshotName;
+                    if (!snapshotName) {
+                        throw new SnapshotReferenceError(`Invalid format for Snapshot reference setting '${setting.key}'.`);
+                    }
+                    const settingsFromSnapshot = await this.#loadConfigurationSettingsFromSnapshot(snapshotName);
+
+                    for (const snapshotSetting of settingsFromSnapshot) {
+                        if (!isFeatureFlag(snapshotSetting)) {
+                            // Feature flags inside snapshot are ignored. This is consistent the behavior that key value selectors ignore feature flags.
+                            loadedSettings.set(snapshotSetting.key, snapshotSetting);
+                        }
+                    }
+                    continue;
+                }
+
                 if (loadFeatureFlag === isFeatureFlag(setting)) {
                     loadedSettings.set(setting.key, setting);
                 }
@@ -575,6 +592,18 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
         }
     }
 
+    async #loadConfigurationSettingsFromSnapshot(snapshotName: string): Promise<ConfigurationSetting[]> {
+        const snapshot = await this.#getSnapshot(snapshotName);
+        if (snapshot === undefined) {
+            return []; // treat non-existing snapshot as empty
+        }
+        if (snapshot.compositionType != KnownSnapshotComposition.Key) {
+            throw new InvalidOperationError(`Composition type for the selected snapshot with name ${snapshotName} must be 'key'.`);
+        }
+        const settings: ConfigurationSetting[] = await this.#listConfigurationSettingsForSnapshot(snapshotName);
+        return settings;
+    }
+
     /**
      * Clears all existing key-values in the local configuration except feature flags.
      */

src/appConfigurationImpl.ts

@@ -33,6 +33,13 @@ export class KeyVaultReferenceError extends Error {
     }
 }
 
+export class SnapshotReferenceError extends Error {
+    constructor(message: string, options?: ErrorOptions) {
+        super(message, options);
+        this.name = "SnapshotReferenceError";
+    }
+}
+
 export function isFailoverableError(error: any): boolean {
     if (!isRestError(error)) {
         return false;

src/common/errors.ts

@@ -51,6 +51,7 @@ export const REPLICA_COUNT_KEY = "ReplicaCount";
 export const KEY_VAULT_CONFIGURED_TAG = "UsesKeyVault";
 export const KEY_VAULT_REFRESH_CONFIGURED_TAG = "RefreshesKeyVault";
 export const FAILOVER_REQUEST_TAG = "Failover";
+export const SNAPSHOT_REFERENCE_TAG = "SnapshotRef";
 
 // Compact feature tags
 export const FEATURES_KEY = "Features";

src/requestTracing/constants.ts

@@ -41,7 +41,8 @@ import {
     FM_VERSION_KEY,
     DELIMITER,
     AI_CONFIGURATION_TAG,
-    AI_CHAT_COMPLETION_CONFIGURATION_TAG
+    AI_CHAT_COMPLETION_CONFIGURATION_TAG,
+    SNAPSHOT_REFERENCE_TAG
 } from "./constants.js";
 
 export interface RequestTracingOptions {
@@ -53,6 +54,7 @@ export interface RequestTracingOptions {
     featureFlagTracing: FeatureFlagTracingOptions | undefined;
     fmVersion: string | undefined;
     aiConfigurationTracing: AIConfigurationTracingOptions | undefined;
+    useSnapshotReference: boolean;
 }
 
 // Utils
@@ -195,6 +197,9 @@ function createFeaturesString(requestTracingOptions: RequestTracingOptions): str
     if (requestTracingOptions.aiConfigurationTracing?.usesAIChatCompletionConfiguration) {
         tags.push(AI_CHAT_COMPLETION_CONFIGURATION_TAG);
     }
+    if (requestTracingOptions.useSnapshotReference) {
+        tags.push(SNAPSHOT_REFERENCE_TAG);
+    }
     return tags.join(DELIMITER);
 }
 

src/requestTracing/utils.ts

@@ -415,8 +415,14 @@ describe("feature flags", function () {
 
     it("should load feature flags from snapshot", async () => {
         const snapshotName = "Test";
-        mockAppConfigurationClientGetSnapshot(snapshotName, {compositionType: "key"});
-        mockAppConfigurationClientListConfigurationSettingsForSnapshot(snapshotName, [[createMockedFeatureFlag("TestFeature", { enabled: true })]]);
+        const snapshotResponses = new Map([
+            [snapshotName, { compositionType: "key" }]
+        ]);
+        const snapshotKVs = new Map([
+            [snapshotName, [[createMockedFeatureFlag("TestFeature", { enabled: true })]]]
+        ]);
+        mockAppConfigurationClientGetSnapshot(snapshotResponses);
+        mockAppConfigurationClientListConfigurationSettingsForSnapshot(snapshotKVs);
         const connectionString = createMockedConnectionString();
         const settings = await load(connectionString, {
             featureFlagOptions: {

test/featureFlag.test.ts

@@ -581,18 +581,22 @@ describe("load", function () {
 
     it("should load key values from snapshot", async () => {
         const snapshotName = "Test";
-        mockAppConfigurationClientGetSnapshot(snapshotName, {compositionType: "key"});
-        mockAppConfigurationClientListConfigurationSettingsForSnapshot(snapshotName, [[{key: "TestKey", value: "TestValue"}].map(createMockedKeyValue)]);
+        const snapshotResponses = new Map([
+            [snapshotName, { compositionType: "key" }]
+        ]);
+        const snapshotKVs = new Map([
+            [snapshotName, [[{key: "TestKey", value: "TestValue"}].map(createMockedKeyValue)]]]
+        );
+        mockAppConfigurationClientGetSnapshot(snapshotResponses);
+        mockAppConfigurationClientListConfigurationSettingsForSnapshot(snapshotKVs);
         const connectionString = createMockedConnectionString();
         const settings = await load(connectionString, {
             selectors: [{
                 snapshotName: snapshotName
             }]
         });
         expect(settings).not.undefined;
-        expect(settings).not.undefined;
         expect(settings.get("TestKey")).eq("TestValue");
-        restoreMocks();
     });
 });
 /* eslint-enable @typescript-eslint/no-unused-expressions */

test/load.test.ts

@@ -0,0 +1,104 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+/* eslint-disable @typescript-eslint/no-unused-expressions */
+import * as chai from "chai";
+import chaiAsPromised from "chai-as-promised";
+chai.use(chaiAsPromised);
+const expect = chai.expect;
+import { load } from "../src/index.js";
+import {
+    mockAppConfigurationClientListConfigurationSettings,
+    mockAppConfigurationClientGetSnapshot,
+    mockAppConfigurationClientListConfigurationSettingsForSnapshot,
+    restoreMocks,
+    createMockedConnectionString,
+    createMockedKeyValue,
+    createMockedSnapshotReference,
+    createMockedFeatureFlag,
+    sleepInMs
+} from "./utils/testHelper.js";
+import * as uuid from "uuid";
+
+const mockedKVs = [{
+    key: "TestKey1",
+    value: "Value1",
+}, {
+    key: "TestKey2",
+    value: "Value2",
+}
+].map(createMockedKeyValue);
+
+mockedKVs.push(createMockedSnapshotReference("TestSnapshotRef", "TestSnapshot1"));
+
+// TestSnapshot1
+const snapshot1 = [{
+    key: "TestKey1",
+    value: "Value1 in snapshot1",
+}
+].map(createMockedKeyValue);
+const testFeatureFlag = createMockedFeatureFlag("TestFeatureFlag");
+snapshot1.push(testFeatureFlag);
+
+// TestSnapshot2
+const snapshot2 = [{
+    key: "TestKey1",
+    value: "Value1 in snapshot2",
+}
+].map(createMockedKeyValue);
+
+describe("snapshot reference", function () {
+
+    beforeEach(() => {
+        const snapshotResponses = new Map([
+            ["TestSnapshot1", { compositionType: "key" }],
+            ["TestSnapshot2", { compositionType: "key" }]]
+        );
+        const snapshotKVs = new Map([
+            ["TestSnapshot1", [snapshot1]],
+            ["TestSnapshot2", [snapshot2]]]
+        );
+        mockAppConfigurationClientGetSnapshot(snapshotResponses);
+        mockAppConfigurationClientListConfigurationSettingsForSnapshot(snapshotKVs);
+        mockAppConfigurationClientListConfigurationSettings([mockedKVs]);
+    });
+
+    afterEach(() => {
+        restoreMocks();
+    });
+
+    it("should resolve snapshot reference", async () => {
+        const connectionString = createMockedConnectionString();
+        const settings = await load(connectionString);
+        expect(settings.get("TestKey1")).eq("Value1 in snapshot1");
+
+        // it should ignore feature flags in snapshot
+        expect(settings.get(testFeatureFlag.key)).to.be.undefined;
+        expect(settings.get("feature_management")).to.be.undefined;
+
+        // it should not load the snapshot reference key
+        expect(settings.get("TestSnapshotRef")).to.be.undefined;
+    });
+
+    it("should refresh when snapshot reference changes", async () => {
+        const connectionString = createMockedConnectionString();
+        const settings = await load(connectionString, {
+            refreshOptions: {
+                enabled: true,
+                refreshIntervalInMs: 2000
+            }
+        });
+        expect(settings.get("TestKey1")).eq("Value1 in snapshot1");
+
+        const setting = mockedKVs.find(kv => kv.key === "TestSnapshotRef");
+        setting!.value = "{\"snapshot_name\":\"TestSnapshot2\"}";
+        setting!.etag = uuid.v4();
+
+        await sleepInMs(2 * 1000 + 1);
+
+        await settings.refresh();
+
+        expect(settings.get("TestKey1")).eq("Value1 in snapshot2");
+    });
+
+});