Move HighlightQuery to HighlightField

More complete integration test for highlighting
Closes #1789

Author: russcam

src/Nest/Search/Search/Highlighting/Highlight.cs

@@ -45,9 +45,6 @@ public interface IHighlight
 
 		[JsonProperty("boundary_chars")]
 		string BoundaryChars { get; set; }
-
-		[JsonProperty("highlight_query")]
-		QueryContainer HighlightQuery { get; set; }
 	}
 
 	public class Highlight : IHighlight
@@ -64,7 +61,6 @@ public class Highlight : IHighlight
 		public Dictionary<Field, IHighlightField> Fields { get; set; }
 		public bool? RequireFieldMatch { get; set; }
 		public string BoundaryChars { get; set; }
-		public QueryContainer HighlightQuery { get; set; }
 	}
 
 	public class HighlightDescriptor<T> : DescriptorBase<HighlightDescriptor<T> ,IHighlight>, IHighlight
@@ -83,7 +79,6 @@ public class HighlightDescriptor<T> : DescriptorBase<HighlightDescriptor<T> ,IHi
 		Dictionary<Field, IHighlightField> IHighlight.Fields { get; set; }
 		bool? IHighlight.RequireFieldMatch { get; set; }
 		string IHighlight.BoundaryChars { get; set; }
-		QueryContainer IHighlight.HighlightQuery { get; set; }
 
 		public HighlightDescriptor<T> Fields(params Func<HighlightFieldDescriptor<T>, IHighlightField>[] fieldHighlighters) =>
 			Assign(a => a.Fields = fieldHighlighters?
@@ -120,8 +115,5 @@ public HighlightDescriptor<T> Fields(params Func<HighlightFieldDescriptor<T>, IH
 		public HighlightDescriptor<T> BoundaryCharacters(string boundaryCharacters) => Assign(a => a.BoundaryChars = boundaryCharacters);
 
 		public HighlightDescriptor<T> BoundaryMaxSize(int boundaryMaxSize) => Assign(a => a.BoundaryMaxSize = boundaryMaxSize);
-
-		public HighlightDescriptor<T> HighlightQuery(Func<QueryContainerDescriptor<T>, QueryContainer> querySelector) =>
-			Assign(a => a.HighlightQuery = querySelector?.InvokeQuery(new QueryContainerDescriptor<T>()));
 	}
 }

src/Nest/Search/Search/Highlighting/HighlightField.cs

@@ -55,6 +55,9 @@ public interface IHighlightField
 
 		[JsonProperty("matched_fields")]
 		Fields MatchedFields { get; set; }
+
+		[JsonProperty("highlight_query")]
+		QueryContainer HighlightQuery { get; set; }
 	}
 
 	public class HighlightField : IHighlightField
@@ -75,6 +78,7 @@ public class HighlightField : IHighlightField
 		public HighlighterType? Type { get; set; }
 		public bool? ForceSource { get; set; }
 		public Fields MatchedFields { get; set; }
+		public QueryContainer HighlightQuery { get; set; }
 	}
 
 	public class HighlightFieldDescriptor<T> : DescriptorBase<HighlightFieldDescriptor<T>,IHighlightField>, IHighlightField
@@ -112,6 +116,8 @@ public class HighlightFieldDescriptor<T> : DescriptorBase<HighlightFieldDescript
 
 		Fields IHighlightField.MatchedFields { get; set; }
 
+		QueryContainer IHighlightField.HighlightQuery { get; set; }
+
 		public HighlightFieldDescriptor<T> Field(Field field) => Assign(a => a.Field = field);
 		public HighlightFieldDescriptor<T> Field(Expression<Func<T, object>> objectPath) => Assign(a => a.Field = objectPath);
 
@@ -151,5 +157,8 @@ public class HighlightFieldDescriptor<T> : DescriptorBase<HighlightFieldDescript
 
 		public HighlightFieldDescriptor<T> MatchedFields(Func<FieldsDescriptor<T>, IPromise<Fields>> fields) =>
 			Assign(a => a.MatchedFields = fields?.Invoke(new FieldsDescriptor<T>())?.Value);
+
+		public HighlightFieldDescriptor<T> HighlightQuery(Func<QueryContainerDescriptor<T>, QueryContainer> querySelector) =>
+			Assign(a => a.HighlightQuery = querySelector?.InvokeQuery(new QueryContainerDescriptor<T>()));
 	}
 }

src/Nest/Search/Search/Highlighting/HighlighterType.cs

@@ -4,13 +4,34 @@
 
 namespace Nest
 {
+	/// <summary>
+	/// Type of highlighter
+	/// </summary>
 	[JsonConverter(typeof(StringEnumConverter))]
 	public enum HighlighterType
 	{
+		/// <summary>
+		/// Plain Highlighter.
+		/// The default choice of highlighter is of type plain and uses the Lucene highlighter. 
+		/// It tries hard to reflect the query matching logic in terms of understanding word 
+		/// importance and any word positioning criteria in phrase queries.
+		/// </summary>
 		[EnumMember(Value = "plain")]
 		Plain,
+
+		/// <summary>
+		/// Postings Highlighter.
+		/// If index_options is set to offsets in the mapping the postings highlighter 
+		/// will be used instead of the plain highlighter
+		/// </summary>
 		[EnumMember(Value = "postings")]
 		Postings,
+
+		/// <summary>
+		/// Fast Vector Highlighter.
+		/// If term_vector information is provided by setting term_vector to with_positions_offsets 
+		/// in the mapping then the fast vector highlighter will be used instead of the plain highlighter
+		/// </summary>
 		[EnumMember(Value = "fvh")]
 		Fvh
 	}

src/Tests/Framework/Integration/Bootstrappers/Seeder.cs

@@ -132,13 +132,21 @@ private void CreateProjectIndex()
 		public static TypeMappingDescriptor<Project> MapProject(TypeMappingDescriptor<Project> m) => m
 			.Properties(props => props
 				.String(s => s
-					.Name(p => p.Name).NotAnalyzed()
+					.Name(p => p.Name)
+					.NotAnalyzed()
 					.Fields(fs => fs
+						.String(ss => ss.Name("standard").Analyzer("standard"))
 						.Completion(cm => cm.Name("suggest"))
 					)
 				)
 				.Date(d => d.Name(p => p.StartedOn))
-				.String(d => d.Name(p => p.State).NotAnalyzed())
+				.String(d => d
+					.Name(p => p.State)
+					.NotAnalyzed()
+					.Fields(fs => fs
+						.String(st => st.Name("offsets").IndexOptions(IndexOptions.Offsets))
+					)
+				)
 				.Nested<Tag>(mo => mo
 					.Name(p => p.Tags)
 					.Properties(TagProperties)

src/Tests/Search/Request/HighlightingUsageTests.cs

@@ -1,8 +1,13 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
+using Bogus;
 using Nest;
 using Tests.Framework.Integration;
 using Tests.Framework.MockData;
+using FluentAssertions;
+using Newtonsoft.Json.Linq;
+using Xunit;
 
 namespace Tests.Search.Request
 {
@@ -12,77 +17,207 @@ public HighlightingUsageTests(ReadOnlyCluster cluster, EndpointUsage usage) : ba
 
 		protected override object ExpectJson => new
 		{
+			query = new
+			{
+				match = new JObject
+				{
+					{ "name.standard", new JObject
+						{
+							{ "query", "Upton Sons Shield Rice Rowe Roberts" }
+						}
+					}
+				}
+			},
 			highlight = new
 			{
 				pre_tags = new[] { "<tag1>" },
 				post_tags = new[] { "</tag1>" },
-				fields = new
+				fields = new JObject
 				{
-					name = new
-					{
-						type = "plain",
-						force_source = true,
-						fragment_size = 150,
-						number_of_fragments = 3,
-						no_match_size = 150
+					{ "name.standard", new JObject
+						{
+							{ "type", "plain" },
+							{ "force_source", true},
+							{ "fragment_size", 150 },
+							{ "number_of_fragments", 3},
+							{ "no_match_size", 150 }
+						}
+					},
+					{ "leadDeveloper.firstName", new JObject
+						{
+							{ "type", "fvh" },
+							{ "pre_tags", new JArray { "<name>" } },
+							{ "post_tags", new JArray { "</name>" } },
+							{ "highlight_query", new JObject
+								{
+									{ "match", new JObject
+										{
+											{ "leadDeveloper.firstName", new JObject
+												{
+													{ "query", "Kurt Edgardo Naomi Dariana Justice Felton" }
+												}
+											}
+										}
+									}
+								}
+							}
+						}
 					},
-					leadDeveloper = new { type = "fvh" },
-					tags = new { type = "postings" }
+					{ "state.offsets", new JObject
+						{
+							{ "type", "postings" },
+							{ "pre_tags", new JArray { "<state>" } },
+							{ "post_tags", new JArray { "</state>" } },
+							{ "highlight_query", new JObject
+								{
+									{ "terms", new JObject
+										{
+											{ "state.offsets", new JArray { "stable" , "bellyup" } }
+										}
+									}
+								}
+							}
+						}
+					}
 				}
 			}
 		};
 
 		protected override Func<SearchDescriptor<Project>, ISearchRequest> Fluent => s => s
+			.Query(q => q
+				.Match(m => m
+					.Field(f => f.Name.Suffix("standard"))
+					.Query("Upton Sons Shield Rice Rowe Roberts")
+				)
+			)
 			.Highlight(h => h
 				.PreTags("<tag1>")
 				.PostTags("</tag1>")
 				.Fields(
 					fs => fs
-						.Field(p => p.Name)
+						.Field(p => p.Name.Suffix("standard"))
 						.Type(HighlighterType.Plain)
 						.ForceSource()
 						.FragmentSize(150)
 						.NumberOfFragments(3)
 						.NoMatchSize(150),
 					fs => fs
-						.Field(p => p.LeadDeveloper)
-						.Type(HighlighterType.Fvh),
+						.Field(p => p.LeadDeveloper.FirstName)
+						.Type(HighlighterType.Fvh)
+						.PreTags("<name>")
+						.PostTags("</name>")
+						.HighlightQuery(q => q
+							.Match(m => m
+								.Field(p => p.LeadDeveloper.FirstName)
+								.Query("Kurt Edgardo Naomi Dariana Justice Felton")
+							)
+						),
 					fs => fs
-						.Field(p => p.Tags)
+						.Field(p => p.State.Suffix("offsets"))
 						.Type(HighlighterType.Postings)
+						.PreTags("<state>")
+						.PostTags("</state>")
+						.HighlightQuery(q => q
+							.Terms(t => t
+								.Field(f => f.State.Suffix("offsets"))
+								.Terms(
+									StateOfBeing.Stable.ToString().ToLowerInvariant(), 
+									StateOfBeing.BellyUp.ToString().ToLowerInvariant()
+								)
+							)
+						)
 				)
 			);
 
 		protected override SearchRequest<Project> Initializer =>
 			new SearchRequest<Project>
 			{
+				Query = new MatchQuery
+				{
+					Query = "Upton Sons Shield Rice Rowe Roberts",
+					Field = "name.standard"
+				},
 				Highlight = new Highlight
 				{
 					PreTags = new[] { "<tag1>" },
 					PostTags = new[] { "</tag1>" },
 					Fields = new Dictionary<Field, IHighlightField>
 					{
-							{ "name", new HighlightField
-								{
-									Type = HighlighterType.Plain,
-									ForceSource = true,
-									FragmentSize = 150,
-									NumberOfFragments = 3,
-									NoMatchSize = 150
-								}
-							},
-							{ "leadDeveloper", new HighlightField
+						{ "name.standard", new HighlightField
+							{
+								Type = HighlighterType.Plain,
+								ForceSource = true,
+								FragmentSize = 150,
+								NumberOfFragments = 3,
+								NoMatchSize = 150
+							}
+						},
+						{ "leadDeveloper.firstName", new HighlightField
+							{
+								Type = HighlighterType.Fvh,
+								PreTags = new[] { "<name>"},
+								PostTags = new[] { "</name>"},
+								HighlightQuery = new MatchQuery
 								{
-									Type = HighlighterType.Fvh,
+									Field = "leadDeveloper.firstName",
+									Query = "Kurt Edgardo Naomi Dariana Justice Felton"
 								}
-							},
-							{ "tags", new HighlightField
+							}
+						},
+						{ "state.offsets", new HighlightField
+							{
+								Type = HighlighterType.Postings,
+								PreTags = new[] { "<state>"},
+								PostTags = new[] { "</state>"},
+								HighlightQuery = new TermsQuery
 								{
-									Type = HighlighterType.Postings
+									Field = "state.offsets",
+									Terms = new [] { "stable", "bellyup" }
 								}
 							}
+						}
 					}
 				}
 			};
+
+		protected override void ExpectResponse(ISearchResponse<Project> response)
+		{
+			response.IsValid.Should().BeTrue();
+
+			foreach (var highlightsByDocumentId in response.Highlights)
+			{
+				foreach (var highlightHit in highlightsByDocumentId.Value)
+				{
+					if (highlightHit.Key == "name.standard")
+					{
+						foreach (var highlight in highlightHit.Value.Highlights)
+						{
+							highlight.Should().Contain("<tag1>");
+							highlight.Should().Contain("</tag1>");
+						}
+					}
+					else if (highlightHit.Key == "leadDeveloper.firstName")
+					{
+						foreach (var highlight in highlightHit.Value.Highlights)
+						{
+							highlight.Should().Contain("<name>");
+							highlight.Should().Contain("</name>");
+						}
+					}
+					else if (highlightHit.Key == "state.offsets")
+					{
+						foreach (var highlight in highlightHit.Value.Highlights)
+						{
+							highlight.Should().Contain("<state>");
+							highlight.Should().Contain("</state>");
+						}
+					}
+					else
+					{
+						Assert.True(false, $"highlights contains unexpected key {highlightHit.Key}");
+					}
+				}
+			}
+		}
 	}
 }