sliced scroll support (#2436)

Author: Mpdreamz

src/Nest/Nest.csproj

@@ -1175,6 +1175,7 @@
     <Compile Include="Search\Scroll\ClearScroll\ElasticClient-ClearScroll.cs" />
     <Compile Include="Search\Scroll\Scroll\ElasticClient-Scroll.cs" />
     <Compile Include="Search\Scroll\Scroll\ScrollRequest.cs" />
+    <Compile Include="Search\Scroll\Scroll\SlicedScroll.cs" />
     <Compile Include="Search\SearchShards\ElasticClient-SearchShards.cs" />
     <Compile Include="Search\SearchShards\SearchShardsRequest.cs" />
     <Compile Include="Search\SearchShards\SearchShardsResponse.cs" />

src/Nest/Search/Scroll/Scroll/ScrollRequest.cs

@@ -11,8 +11,8 @@ public partial interface IScrollRequest : ICovariantSearchRequest
 		[JsonProperty("scroll_id")]
 		string ScrollId { get; set; }
 	}
-	
-	public partial class ScrollRequest 
+
+	public partial class ScrollRequest
 	{
 		private Type _clrType { get; set; }
 		Type ICovariantSearchRequest.ClrType => this._clrType;
@@ -48,7 +48,7 @@ public partial class ScrollDescriptor<T> where T : class
 
 		public ScrollDescriptor<T> ScrollId(string scrollId) => Assign(a => a.ScrollId = scrollId);
 
-		public ScrollDescriptor<T> ConcreteTypeSelector(Func<dynamic, Hit<dynamic>, Type> typeSelector) => 
+		public ScrollDescriptor<T> ConcreteTypeSelector(Func<dynamic, Hit<dynamic>, Type> typeSelector) =>
 			Assign(a => a.TypeSelector = typeSelector);
 
 		public ScrollDescriptor<T> CovariantTypes(Types covariantTypes) => Assign(a=> this._covariantTypes = covariantTypes);

src/Nest/Search/Scroll/Scroll/SlicedScroll.cs

@@ -0,0 +1,43 @@
+using System;
+using System.Linq.Expressions;
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	[JsonConverter(typeof(ReadAsTypeJsonConverter<SlicedScroll>))]
+	public interface ISlicedScroll
+	{
+		[JsonProperty("id")]
+		int? Id { get; set; }
+		[JsonProperty("max")]
+		int? Max { get; set; }
+		[JsonProperty("field")]
+		Field Field { get; set; }
+	}
+
+	public class SlicedScroll : ISlicedScroll
+	{
+		public int? Id { get; set; }
+		public int? Max { get; set; }
+		public Field Field { get; set; }
+
+	}
+
+
+	public class SlicedScrollDescriptor<T> : DescriptorBase<SlicedScrollDescriptor<T>, ISlicedScroll>, ISlicedScroll
+		where T : class
+	{
+		int? ISlicedScroll.Id { get; set; }
+		int? ISlicedScroll.Max { get; set; }
+		Field ISlicedScroll.Field { get; set; }
+
+		public SlicedScrollDescriptor<T> Id(int id) => Assign(a => a.Id = id);
+
+		public SlicedScrollDescriptor<T> Max(int max) => Assign(a => a.Max = max);
+
+		public SlicedScrollDescriptor<T> Field(Field field) => Assign(a => a.Field = field);
+
+		public SlicedScrollDescriptor<T> Field(Expression<Func<T, object>> objectPath) => Assign(a => a.Field = objectPath);
+
+	}
+}

src/Nest/Search/Search/SearchRequest.cs

@@ -72,6 +72,9 @@ public partial interface ISearchRequest : ICovariantSearchRequest
 		[JsonProperty("aggs")]
 		AggregationDictionary Aggregations { get; set; }
 
+		[JsonProperty("slice")]
+		ISlicedScroll Slice { get; set; }
+
 		[JsonProperty("query")]
 		QueryContainer Query { get; set; }
 
@@ -114,6 +117,7 @@ public partial class SearchRequest
 		public IList<object> SearchAfter { get; set; }
 		public IDictionary<IndexName, double> IndicesBoost { get; set; }
 		public QueryContainer PostFilter { get; set; }
+		public ISlicedScroll Slice { get; set; }
 		public QueryContainer Query { get; set; }
 		public IList<IRescore> Rescore { get; set; }
 		public ISuggestContainer Suggest { get; set; }
@@ -157,6 +161,7 @@ public partial class SearchRequest<T>
 		public IList<object> SearchAfter { get; set; }
 		public IDictionary<IndexName, double> IndicesBoost { get; set; }
 		public QueryContainer PostFilter { get; set; }
+		public ISlicedScroll Slice { get; set; }
 		public QueryContainer Query { get; set; }
 		public IList<IRescore> Rescore { get; set; }
 		public ISuggestContainer Suggest { get; set; }
@@ -212,6 +217,7 @@ public partial class SearchDescriptor<T> where T : class
 		ISuggestContainer ISearchRequest.Suggest { get; set; }
 		IHighlight ISearchRequest.Highlight { get; set; }
 		IList<IRescore> ISearchRequest.Rescore { get; set; }
+		ISlicedScroll ISearchRequest.Slice { get; set; }
 		QueryContainer ISearchRequest.Query { get; set; }
 		QueryContainer ISearchRequest.PostFilter { get; set; }
 		Fields ISearchRequest.StoredFields { get; set; }
@@ -398,6 +404,12 @@ public SearchDescriptor<T> Suggest(Func<SuggestContainerDescriptor<T>, IPromise<
 		public SearchDescriptor<T> Query(Func<QueryContainerDescriptor<T>, QueryContainer> query) =>
 			Assign(a => a.Query = query?.Invoke(new QueryContainerDescriptor<T>()));
 
+		/// <summary>
+		/// For scroll queries that return a lot of documents it is possible to split the scroll in multiple slices which can be consumed independently
+		/// </summary>
+		public SearchDescriptor<T> Slice(Func<SlicedScrollDescriptor<T>, ISlicedScroll> query) =>
+			Assign(a => a.Slice = query?.Invoke(new SlicedScrollDescriptor<T>()));
+
 		/// <summary>
 		/// Shortcut to default to a match all query
 		/// </summary>

src/Tests/Search/Request/SlicedScrollSearchUsageTests.cs

@@ -0,0 +1,30 @@
+using System;
+using Nest;
+using Tests.Framework.Integration;
+using Tests.Framework.MockData;
+using static Nest.Infer;
+
+namespace Tests.Search.Request
+{
+	public class SlicedScrollSearchUsageTests : SearchUsageTestBase
+	{
+
+		public SlicedScrollSearchUsageTests(ReadOnlyCluster cluster, EndpointUsage usage) : base(cluster, usage) { }
+
+		protected override string UrlPath => "/project/project/_search?scroll=1m";
+
+		protected override object ExpectJson =>
+			new { slice = new { id = 0, max = 5 } }; 
+
+		protected override SearchRequest<Project> Initializer =>
+			new SearchRequest<Project>()
+			{
+				Scroll = "1m",
+				Slice = new SlicedScroll {  Id = 0, Max = 5 }
+			};
+
+		protected override Func<SearchDescriptor<Project>, ISearchRequest> Fluent => s => s
+			.Scroll("1m")
+			.Slice(ss=>ss.Id(0).Max(5));
+	}
+}

src/Tests/Search/Scroll/Scroll/SlicedScrollApiTests.cs

@@ -0,0 +1,62 @@
+using System;
+using System.Threading;
+using Elasticsearch.Net;
+using Nest;
+using Tests.Framework;
+using Tests.Framework.Integration;
+using Tests.Framework.MockData;
+using Xunit;
+
+namespace Tests.Search.Scroll.Scroll
+{
+	public class SlicedScrollApiTests : ApiIntegrationTestBase<ReadOnlyCluster, ISearchResponse<Project>, IScrollRequest, ScrollDescriptor<Project>, ScrollRequest>
+	{
+		public SlicedScrollApiTests(ReadOnlyCluster cluster, EndpointUsage usage) : base(cluster, usage) { }
+
+		private string _scrollId = "default-for-unit-tests";
+
+		protected override LazyResponses ClientUsage() => Calls(
+			fluent: (c, f) => c.Scroll("1m", _scrollId, f),
+			fluentAsync: (c, f) => c.ScrollAsync("1m", _scrollId, f),
+			request: (c, r) => c.Scroll<Project>(r),
+			requestAsync: (c, r) => c.ScrollAsync<Project>(r)
+		);
+
+		protected override object ExpectJson => new
+		{
+			scroll = "1m",
+			scroll_id = _scrollId
+		};
+
+		protected override int ExpectStatusCode => 200;
+		protected override bool ExpectIsValid => true;
+		protected override HttpMethod HttpMethod => HttpMethod.POST;
+		protected override string UrlPath => $"/_search/scroll";
+		protected override bool SupportsDeserialization => false;
+
+		protected override Func<ScrollDescriptor<Project>, IScrollRequest> Fluent => s => s.Scroll("1m").ScrollId(_scrollId);
+
+		protected override ScrollRequest Initializer => new ScrollRequest(_scrollId, "1m");
+
+		protected int _slice = 0;
+		protected override void OnBeforeCall(IElasticClient client)
+		{
+			var maxSlices = 2; // number of shards we use by default for test indices
+			var currentSlice = Interlocked.Increment(ref this._slice) % maxSlices;
+			var scrollTimeout = TimeSpan.FromMinutes(1);
+			var response = client.Search<Project>(s => s
+				.Scroll(scrollTimeout)
+				.Slice(ss=>ss.Max(maxSlices).Id(currentSlice))
+				.Sort(ss=>ss.Field("_doc", SortOrder.Ascending))
+			);
+			if (!response.IsValid)
+				throw new Exception("Scroll setup failed");
+			_scrollId = response.ScrollId ?? _scrollId;
+		}
+
+		protected override void OnAfterCall(IElasticClient client)
+		{
+			client.ClearScroll(cs => cs.ScrollId(_scrollId));
+		}
+	}
+}

src/Tests/Tests.csproj

@@ -654,7 +654,9 @@
     <Compile Include="Search\Percolator\UnregisterPercolator\UnregisterPercolatorUrlTests.cs" />
     <Compile Include="Reproduce\GithubIssue2052.cs" />
     <Compile Include="Reproduce\GithubIssue2173.cs" />
+    <Compile Include="Search\Request\SlicedScrollSearchUsageTests.cs" />
     <Compile Include="Search\Request\SearchAfterUsageTests.cs" />
+    <Compile Include="Search\Scroll\Scroll\SlicedScrollApiTests.cs" />
     <Compile Include="Search\Search\Rescoring\RescoreUsageTests.cs" />
     <Compile Include="Search\Search\InvalidSearchApiTests.cs" />
     <Compile Include="Search\Suggesters\SuggestUrlTests.cs" />

src/Tests/tests.yaml

@@ -10,3 +10,4 @@ force_reseed: true
 # do not spawn nodes as part of the test setup if we find a node is already running
 # this is opt in during development in CI we never want to see our tests running against an already running node
 test_against_already_running_elasticsearch: true
+