Merge pull request #17 from M-Files/filedownloadstream-blocksize_exceedsmaximum

FileDownloadStream.Read handles large block size

Author: CraigHawker

MFilesAPI.Extensions.Tests/Files/Downloading/FileDownloadStream/FileDownloadStream.cs

@@ -403,8 +403,8 @@ public void ReadOpensDownloadSessionIfNotOpen()
 					// Mock a download session to return.
 					var downloadSessionMock = new Mock<FileDownloadSession>();
 					downloadSessionMock.SetupGet(m => m.DownloadID).Returns(1);
-					downloadSessionMock.SetupGet(m => m.FileSize).Returns(1000);
-					downloadSessionMock.SetupGet(m => m.FileSize32).Returns(1000);
+					downloadSessionMock.SetupGet(m => m.FileSize).Returns(3);
+					downloadSessionMock.SetupGet(m => m.FileSize32).Returns(3);
 					return downloadSessionMock.Object;
 				})
 				.Verifiable();
@@ -441,7 +441,7 @@ public void ReadOpensDownloadSessionIfNotOpen()
 
 			// Attempt to read.
 			byte[] data = new byte[4096];
-			stream.Read(data, 0, 4096);
+			Assert.AreEqual(3, stream.Read(data, 0, 4096));
 
 			// Ensure that the download session is not empty.
 			Assert.IsNotNull(stream.DownloadSession);
@@ -451,6 +451,94 @@ public void ReadOpensDownloadSessionIfNotOpen()
 			vaultObjectFileOperationsMock.Verify();
 		}
 
+		[TestMethod]
+		public void ReadWithLargeBlockSizeRetrievesMultipleBlocks()
+		{
+
+			// Set up a file to download.
+			var objectFileMock = new Mock<ObjectFile>();
+			objectFileMock.SetupGet(m => m.ID).Returns(12345);
+			objectFileMock.SetupGet(m => m.Version).Returns(1);
+
+			// Set up the vault object file operations mock.
+			var vaultObjectFileOperationsMock = new Mock<VaultObjectFileOperations>();
+
+			// We don't really care what we download, as long as we can check it happened.
+			vaultObjectFileOperationsMock
+				.Setup(m => m.DownloadFileInBlocks_BeginEx
+				(
+					Moq.It.IsAny<int>(),
+					Moq.It.IsAny<int>(),
+					Moq.It.IsAny<MFFileFormat>()
+				))
+				.Returns((int receivedFileId, int receivedFileVersion, MFFileFormat receivedFileFormat) =>
+				{
+					// Mock a download session to return.
+					var downloadSessionMock = new Mock<FileDownloadSession>();
+					downloadSessionMock.SetupGet(m => m.DownloadID).Returns(1);
+					downloadSessionMock.SetupGet(m => m.FileSize).Returns(8 * 1024 * 1024 + 4);
+					downloadSessionMock.SetupGet(m => m.FileSize32).Returns(8 * 1024 * 1024 + 4);
+					return downloadSessionMock.Object;
+				})
+				.Verifiable();
+
+			// When DownloadFileInBlocks_ReadBlock is called (reading a block of content), return something.
+			vaultObjectFileOperationsMock
+				.Setup(m => m.DownloadFileInBlocks_ReadBlock
+				(
+					Moq.It.IsAny<int>(),
+					Moq.It.IsAny<int>(),
+					Moq.It.IsAny<long>()
+				))
+				.Returns((int downloadSession, int blockSize, long offset) =>
+				{
+					if (offset < 8 * 1024 * 1024)
+						return new byte[4 * 1024 * 1024];
+					return new byte[4];
+				})
+				.Verifiable();
+
+			// Set up the mock vault.
+			var vaultMock = new Mock<Vault>();
+			vaultMock
+				.SetupGet(m => m.ObjectFileOperations)
+				.Returns(vaultObjectFileOperationsMock.Object);
+
+			// Create the stream.
+			var stream = new FileDownloadStreamProxy(objectFileMock.Object, vaultMock.Object);
+
+			// Read a large set of data (just over 8MB)
+			// to cause the method to go back to the server for
+			// multiple blocks.
+			byte[] data = new byte[8 * 1024 * 1024 + 4];
+			int ret = stream.Read(data, 0, data.Length);
+
+			// Make sure that we got the data we expected.
+			Assert.AreEqual(ret, data.Length);
+
+			// Ensure we got hit as expected.
+			vaultMock.Verify();
+			vaultObjectFileOperationsMock
+				.Verify(
+					// Did it get block one (0 -> 4MB)?
+					(m) => m.DownloadFileInBlocks_ReadBlock(1, 4 * 1024 * 1024, 0),
+					Times.Once
+				);
+			vaultObjectFileOperationsMock
+				.Verify(
+					// Did it get block two (4MB -> 8MB)?
+					(m) => m.DownloadFileInBlocks_ReadBlock(1, 4 * 1024 * 1024, 4 * 1024 * 1024),
+					Times.Once
+				);
+			vaultObjectFileOperationsMock
+				.Verify(
+					// Did it get block three (8MB -> end)?
+					(m) => m.DownloadFileInBlocks_ReadBlock(1, 4, 8 * 1024 * 1024),
+					Times.Once
+				);
+
+		}
+
 		[TestMethod]
 		public void ReadIteratesThroughData()
 		{
@@ -628,8 +716,8 @@ private class FileDownloadStreamProxy
 				set => base.DownloadSession = value;
 			}
 
-			private long length = 0;
-			public override long Length => this.length;
+			private long? length;
+			public override long Length => this.length ?? base.Length;
 
 			/// <inheritdoc />
 			public override void SetLength(long value)

MFilesAPI.Extensions.Tests/MFilesAPI.Extensions.Tests.csproj

@@ -1,17 +1,16 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net45</TargetFramework>
-
+    <TargetFramework>net462</TargetFramework>
+	  <IsTestProject>true</IsTestProject>
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.2.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.9.0" />
     <PackageReference Include="Moq" Version="4.13.1" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.0.0" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.0.0" />
     <PackageReference Include="coverlet.collector" Version="1.0.1" />
+	  <PackageReference Include="MSTest" Version="3.3.1" />
   </ItemGroup>
 
   <ItemGroup>
@@ -20,12 +19,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\MFilesAPI.Extensions\MFilesAPI.Extensions.csproj" />
-  </ItemGroup>
-  
-  <ItemGroup>
-    <Reference Include="Interop.MFilesAPI">
-      <HintPath>..\lib\Interop.MFilesAPI.dll</HintPath>
-    </Reference>
+    <PackageReference Include="Interop.MFilesAPI" Version="21.11.3" />
   </ItemGroup>
 
 </Project>

MFilesAPI.Extensions.Tests/packages.config

@@ -5,13 +5,13 @@
 using System.Text;
 
 namespace MFilesAPI.Extensions
-{
+{
 	/// <summary>
 	/// Allows access to files from within the M-Files vault in an implementation of <see cref="Stream"/>.
 	/// </summary>
 	public class FileDownloadStream
 		: Stream
-	{
+	{
 		/// <summary>
 		/// The vault to download from.
 		/// </summary>
@@ -35,13 +35,21 @@ public class FileDownloadStream
 		/// <summary>
 		/// The format of the file to download.
 		/// </summary>
-		public MFFileFormat FileFormat { get; protected set; }
-
+		public MFFileFormat FileFormat { get; protected set; }
+
 		/// <summary>
 		/// The open file download session.
 		/// </summary>
 		public FileDownloadSession DownloadSession { get; protected set; }
 
+		/// <summary>
+		/// The M-Files API limits download block size.
+		/// If a block larger than this is requested
+		/// via <see cref="Read(byte[], int, int)"/>
+		/// then data will be read from M-Files in blocks of this size.
+		/// </summary>
+		public const int MaximumBlockSize = 4 * 1024 * 1024;
+
 		/// <summary>
 		/// Creates a <see cref="FileDownloadStream"/> but does not open the download session.
 		/// </summary>
@@ -168,27 +176,47 @@ public override int Read(byte[] buffer, int offset, int count)
 			if (this.Position >= this.DownloadSession.FileSize)
 				return 0;
 
-			// Read the block.
-			var blockData = this
-				.Vault
-				.ObjectFileOperations
-				.DownloadFileInBlocks_ReadBlock
-				(
-					this.DownloadSession.DownloadID,
-					count,
-					this.position
-				);
+			// Loop through and get max 4MB blocks.
+			int dataRead = 0;
+			bool atEnd = false;
+			while (dataRead < count && !atEnd)
+			{
+				// Work out how much to get.
+				var blockSize = count;
+				if(blockSize > MaximumBlockSize)
+					blockSize = MaximumBlockSize;
+				if (dataRead + blockSize > this.Length)
+					blockSize = (int)(this.Length - dataRead);
+				if (blockSize <= 0)
+					break;
+
+				// Read the block.
+				var blockData = this
+					.Vault
+					.ObjectFileOperations
+					.DownloadFileInBlocks_ReadBlock
+					(
+						this.DownloadSession.DownloadID,
+						blockSize,
+						this.position
+					);
+
+				// Check the buffer is big enough.
+				if (dataRead + blockData.Length > buffer.Length)
+					throw new ArgumentException($"The buffer size ({buffer.Length}) is not big enough to hold the amount of data requested ({dataRead + blockData.Length}).", nameof(buffer));
 
-			// Check the buffer is big enough.
-			if (blockData.Length > buffer.Length)
-				throw new ArgumentException($"The buffer size ({buffer.Length}) is not big enough to hold the amount of data requested ({count}).", nameof(buffer));
+				// Copy the data into the supplied buffer.
+				Array.Copy(blockData, 0, buffer, dataRead + offset, blockData.Length);
 
-			// Copy the data into the supplied buffer.
-			Array.Copy(blockData, buffer, blockData.Length);
+				// Update our position with the number of bytes read.
+				this.position += blockData.Length;
+				dataRead += blockData.Length;
+				atEnd = blockData.Length < blockSize;
 
-			// Return the number of bytes read.
-			this.position += blockData.Length;
-			return blockData.Length;
+			}
+
+
+			return dataRead;
 		}
 
 		/// <inheritdoc />
@@ -207,7 +235,7 @@ public override void Write(byte[] buffer, int offset, int count)
 		public override bool CanWrite => false;
 
 		/// <inheritdoc />
-		public override long Length => this.DownloadSession?.FileSize ?? this.FileSize;
+		public override long Length => (long)(this.DownloadSession?.FileSize ?? this.FileSize);
 
 		private long position = 0;
 

MFilesAPI.Extensions/Files/Downloading/FileDownloadStream.cs

@@ -1,6 +1,6 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
-    <TargetFrameworks>net45;netstandard2.0;net472</TargetFrameworks>
+    <TargetFrameworks>netstandard2.0</TargetFrameworks>
     <Copyright>M-Files Corporation 2020 onwards</Copyright>
     <GeneratePackageOnBuild>False</GeneratePackageOnBuild>
     <Version>1.0.6</Version>