From dbc7eb238a850bc6759882f62a4067bf5789f4ac Mon Sep 17 00:00:00 2001 From: Foxocube Date: Thu, 17 Aug 2023 16:10:47 +0100 Subject: [PATCH 1/2] Update Statiq.Web.Azure.csproj --- src/Statiq.Web.Azure/Statiq.Web.Azure.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Statiq.Web.Azure/Statiq.Web.Azure.csproj b/src/Statiq.Web.Azure/Statiq.Web.Azure.csproj index ac486ab8c..9efcfd1ee 100644 --- a/src/Statiq.Web.Azure/Statiq.Web.Azure.csproj +++ b/src/Statiq.Web.Azure/Statiq.Web.Azure.csproj @@ -4,7 +4,7 @@ Statiq Static StaticContent StaticSite Blog BlogEngine Azure AppService - + @@ -21,4 +21,4 @@ - \ No newline at end of file + From c78ec6ee6c80c28bacdd829da5bddc39dff28e86 Mon Sep 17 00:00:00 2001 From: Foxocube Date: Thu, 17 Aug 2023 16:11:17 +0100 Subject: [PATCH 2/2] Update DeploySearchIndex.cs --- src/Statiq.Web.Azure/DeploySearchIndex.cs | 80 +++++++++++------------ 1 file changed, 38 insertions(+), 42 deletions(-) diff --git a/src/Statiq.Web.Azure/DeploySearchIndex.cs b/src/Statiq.Web.Azure/DeploySearchIndex.cs index bde24d209..cbf607a1f 100644 --- a/src/Statiq.Web.Azure/DeploySearchIndex.cs +++ b/src/Statiq.Web.Azure/DeploySearchIndex.cs @@ -1,16 +1,15 @@ -using System; +using System; using System.Collections.Generic; -using System.IO; using System.Linq; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Text; using System.Threading.Tasks; -using Microsoft.Azure.Search; -using Microsoft.Azure.Search.Models; +using Azure; using Microsoft.Extensions.Logging; using Polly; using Statiq.Common; +using Azure.Search.Documents; +using Azure.Search.Documents.Indexes; +using Azure.Search.Documents.Indexes.Models; +using Azure.Search.Documents.Models; namespace Statiq.Web.Azure { @@ -41,7 +40,7 @@ public DeploySearchIndex( Config searchServiceName, Config indexName, Config apiKey, - Config> fields) + Config> fields) : base( new Dictionary { @@ -59,70 +58,67 @@ protected override async Task> ExecuteConfigAsync(IDocume string searchServiceName = values.GetString(SearchServiceName) ?? throw new ExecutionException("Invalid search service name"); string indexName = values.GetString(IndexName) ?? throw new ExecutionException("Invalid search index name"); string apiKey = values.GetString(ApiKey) ?? throw new ExecutionException("Invalid search API key"); - IList fields = values.GetList(Fields)?.ToList() ?? throw new ExecutionException("Invalid search fields"); + IList fields = values.GetList(Fields)?.ToList() ?? throw new ExecutionException("Invalid search fields"); - SearchServiceClient client = new SearchServiceClient(searchServiceName, new SearchCredentials(apiKey)); - - // Delete the index if it currently exists (recreating is the easiest way to update it) - CorsOptions corsOptions = null; - if (await client.Indexes.ExistsAsync(indexName, null, context.CancellationToken)) + if (!Uri.TryCreate(searchServiceName, UriKind.Absolute, out Uri searchServiceUri)) { - // Get the CORS options because we'll need to recreate those - Microsoft.Azure.Search.Models.Index existingIndex = await client.Indexes.GetAsync(indexName, null, context.CancellationToken); - corsOptions = existingIndex.CorsOptions; - - // Delete the existing index - context.LogDebug($"Deleting existing search index {indexName}"); - await client.Indexes.DeleteAsync(indexName, null, null, context.CancellationToken); + throw new ExecutionException("Invalid search service name"); } + SearchIndexClient searchIndexClient = new SearchIndexClient(searchServiceUri, new AzureKeyCredential(apiKey)); + // Create the index - Microsoft.Azure.Search.Models.Index index = new Microsoft.Azure.Search.Models.Index + SearchIndex index = new SearchIndex(indexName) { - Name = indexName, Fields = fields, - CorsOptions = corsOptions }; context.LogDebug($"Creating search index {indexName}"); - await client.Indexes.CreateAsync(index, null, context.CancellationToken); + await searchIndexClient.CreateOrUpdateIndexAsync(index, true, true, context.CancellationToken); // Upload the documents to the search index in batches context.LogDebug($"Uploading {context.Inputs.Length} documents to search index {indexName}..."); - ISearchIndexClient indexClient = client.Indexes.GetClient(indexName); + SearchClient indexClient = searchIndexClient.GetSearchClient(indexName); int start = 0; do { // Create the dynamic search documents and batch - IndexAction[] indexActions = context.Inputs + IndexDocumentsAction[] indexActions = context.Inputs .Skip(start) .Take(BatchSize) .Select(doc => { - Microsoft.Azure.Search.Models.Document searchDocument = new Microsoft.Azure.Search.Models.Document(); - foreach (Field field in fields) + SearchDocument searchDocument = new SearchDocument(); + foreach (SearchField field in fields) { if (doc.ContainsKey(field.Name)) { searchDocument[field.Name] = doc.Get(field.Name); } } - return IndexAction.Upload(searchDocument); + + return IndexDocumentsAction.Upload(searchDocument); }) .ToArray(); - IndexBatch indexBatch = IndexBatch.New(indexActions); + IndexDocumentsBatch indexBatch = IndexDocumentsBatch.Create(indexActions); // Upload the batch with exponential retry for failures await Policy - .Handle() - .WaitAndRetryAsync( - 5, - attempt => - { - context.LogWarning($"Failure while uploading batch {(start / BatchSize) + 1}, retry number {attempt}"); - return TimeSpan.FromSeconds(Math.Pow(2, attempt)); - }, - (ex, _) => indexBatch = ((IndexBatchException)ex).FindFailedActionsToRetry(indexBatch, fields.Single(x => x.IsKey == true).Name)) - .ExecuteAsync(async ct => await indexClient.Documents.IndexAsync(indexBatch, null, ct), context.CancellationToken); + .Handle() + .OrResult(r => r.Results.Any(result => !result.Succeeded)) + .WaitAndRetryAsync( + 5, + attempt => + { + context.LogWarning($"Failure while uploading batch {(start / BatchSize) + 1}, retry number {attempt}"); + return TimeSpan.FromSeconds(Math.Pow(2, attempt)); + }, + (ex, _) => + { + IEnumerable failedResults = ex.Result.Results.Where(r => !r.Succeeded).Select(result => result.Key); + IEnumerable> failedActions = indexActions.Where(action => action.Document.Keys.Any(key => failedResults.Contains(key))); + indexBatch = IndexDocumentsBatch.Create(failedActions.ToArray()); + }) + .ExecuteAsync(async ct => await indexClient.IndexDocumentsAsync(indexBatch, null, ct), context.CancellationToken); context.LogDebug($"Uploaded {start + indexActions.Length} documents to search index {indexName}"); start += 1000; @@ -132,4 +128,4 @@ await Policy return context.Inputs; } } -} \ No newline at end of file +}