C# 如何使用HttpWebRequest实现文件并行
我正在尝试制作一个像IDM这样的程序,可以同时下载部分文件。C# 如何使用HttpWebRequest实现文件并行,c#,task-parallel-library,C#,Task Parallel Library,我正在尝试制作一个像IDM这样的程序,可以同时下载部分文件。 我用来实现这一点的工具是C#.Net4.5中的TPL 但是我在使用任务使操作并行时遇到了一个问题。 sequence功能运行良好,正在正确下载文件。 使用任务的并行函数一直在工作,直到发生奇怪的事情: 我用Factory.StartNew()创建了4个任务,在每个任务中都给出了开始位置和结束位置,任务将下载这些文件,然后以字节[]返回,一切都进行得很顺利,任务工作正常,但在某个时刻,执行冻结,仅此而已,程序停止,没有其他事情发生。 并
我用来实现这一点的工具是C#.Net4.5中的TPL
但是我在使用
任务
使操作并行时遇到了一个问题。sequence功能运行良好,正在正确下载文件。
使用任务的并行函数一直在工作,直到发生奇怪的事情:
我用
Factory.StartNew()
创建了4个任务,在每个任务中都给出了开始位置和结束位置,任务将下载这些文件,然后以字节[]返回,一切都进行得很顺利,任务工作正常,但在某个时刻,执行冻结,仅此而已,程序停止,没有其他事情发生。并行功能的实现:
static void DownloadPartsParallel()
{
string uriPath = "http://mschnlnine.vo.llnwd.net/d1/pdc08/PPTX/BB01.pptx";
Uri uri = new Uri(uriPath);
long l = GetFileSize(uri);
Console.WriteLine("Size={0}", l);
int granularity = 4;
byte[][] arr = new byte[granularity][];
Task<byte[]>[] tasks = new Task<byte[]>[granularity];
tasks[0] = Task<byte[]>.Factory.StartNew(() => DownloadPartOfFile(uri, 0, l / granularity));
tasks[1] = Task<byte[]>.Factory.StartNew(() => DownloadPartOfFile(uri, l / granularity + 1, l / granularity + l / granularity));
tasks[2] = Task<byte[]>.Factory.StartNew(() => DownloadPartOfFile(uri, l / granularity + l / granularity + 1, l / granularity + l / granularity + l / granularity));
tasks[3] = Task<byte[]>.Factory.StartNew(() => DownloadPartOfFile(uri, l / granularity + l / granularity + l / granularity + 1, l));//(l / granularity) + (l / granularity) + (l / granularity) + (l / granularity)
arr[0] = tasks[0].Result;
arr[1] = tasks[1].Result;
arr[2] = tasks[2].Result;
arr[3] = tasks[3].Result;
Stream localStream;
localStream = File.Create("E:\\a\\" + Path.GetFileName(uri.LocalPath));
for (int i = 0; i < granularity; i++)
{
if (i == granularity - 1)
{
for (int j = 0; j < arr[i].Length - 1; j++)
{
localStream.WriteByte(arr[i][j]);
}
}
else
for (int j = 0; j < arr[i].Length; j++)
{
localStream.WriteByte(arr[i][j]);
}
}
}
static void DownloadPartsParallel()
{
字符串路径=”http://mschnlnine.vo.llnwd.net/d1/pdc08/PPTX/BB01.pptx";
Uri=新Uri(uriPath);
long l=GetFileSize(uri);
WriteLine(“Size={0}”,l);
int粒度=4;
字节[][]arr=新字节[粒度][];
任务[]任务=新任务[粒度];
tasks[0]=Task.Factory.StartNew(()=>downloadpartofile(uri,0,l/粒度));
tasks[1]=Task.Factory.StartNew(()=>downloadpartofile(uri,l/粒度+1,l/粒度+l/粒度));
tasks[2]=Task.Factory.StartNew(()=>downloadpartofile(uri,l/粒度+l/粒度+1,l/粒度+l/粒度+l/粒度));
tasks[3]=Task.Factory.StartNew(()=>downloadpartofile(uri,l/粒度+l/粒度+l/粒度+1,l));/(l/粒度)+(l/粒度)+(l/粒度)+(l/粒度)+(l/粒度)
arr[0]=任务[0]。结果;
arr[1]=任务[1]。结果;
arr[2]=任务[2]。结果;
arr[3]=任务[3]。结果;
流本地流;
localStream=File.Create(“E:\\a\\”+Path.GetFileName(uri.LocalPath));
for(int i=0;i
DownloadPartOfFile函数实现:
public static byte[] DownloadPartOfFile(Uri fileUrl, long from, long to)
{
int bytesProcessed = 0;
BinaryReader reader = null;
WebResponse response = null;
byte[] bytes = new byte[(to - from) + 1];
try
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(fileUrl);
request.AddRange(from, to);
request.ReadWriteTimeout = int.MaxValue;
request.Timeout = int.MaxValue;
if (request != null)
{
response = request.GetResponse();
if (response != null)
{
reader = new BinaryReader(response.GetResponseStream());
int bytesRead;
do
{
byte[] buffer = new byte[1024];
bytesRead = reader.Read(buffer, 0, buffer.Length);
if (bytesRead == 0)
{
break;
}
Array.Resize<byte>(ref buffer, bytesRead);
buffer.CopyTo(bytes, bytesProcessed);
bytesProcessed += bytesRead;
Console.WriteLine(Thread.CurrentThread.ManagedThreadId + ",Downloading" + bytesProcessed);
} while (bytesRead > 0);
}
}
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
finally
{
if (response != null) response.Close();
if (reader != null) reader.Close();
}
return bytes;
}
publicstaticbyte[]downloadpartofile(urifileurl,long-from,long-to)
{
int字节数=0;
BinaryReader=null;
WebResponse=null;
字节[]字节=新字节[(到-从)+1];
尝试
{
HttpWebRequest请求=(HttpWebRequest)WebRequest.Create(fileUrl);
request.AddRange(从、到);
request.ReadWriteTimeout=int.MaxValue;
request.Timeout=int.MaxValue;
if(请求!=null)
{
response=request.GetResponse();
if(响应!=null)
{
reader=newbinaryreader(response.GetResponseStream());
int字节读取;
做
{
字节[]缓冲区=新字节[1024];
bytesRead=reader.Read(buffer,0,buffer.Length);
如果(字节读==0)
{
打破
}
Array.Resize(ref buffer,bytesRead);
CopyTo(字节,字节处理);
字节处理+=字节读取;
Console.WriteLine(Thread.CurrentThread.ManagedThreadId+”,下载“+字节处理);
}而(字节读取>0);
}
}
}
捕获(例外e)
{
控制台写入线(e.Message);
}
最后
{
if(response!=null)response.Close();
if(reader!=null)reader.Close();
}
返回字节;
}
我试图通过将int.MaxValue设置为读取超时、写入读取超时和超时来解决此问题,这就是程序冻结的原因。如果我不这样做,则在函数DownloadPartsParallel中会出现超时异常所以,有没有一个解决方案或任何其他建议可能会有所帮助,谢谢。好的,下面是我将如何做您正在尝试的事情。这基本上是相同的想法,只是实施方式不同
public static void DownloadFileInPiecesAndSave()
{
//test
var uri = new Uri("http://www.w3.org/");
var bytes = DownloadInPieces(uri, 4);
File.WriteAllBytes(@"c:\temp\RangeDownloadSample.html", bytes);
}
/// <summary>
/// Donwload a file via HTTP in multiple pieces using a Range request.
/// </summary>
public static byte[] DownloadInPieces(Uri uri, uint numberOfPieces)
{
//I'm just fudging this for expository purposes. In reality you would probably want to do a HEAD request to get total file size.
ulong totalFileSize = 1003;
var pieceSize = totalFileSize / numberOfPieces;
List<Task<byte[]>> tasks = new List<Task<byte[]>>();
for (uint i = 0; i < numberOfPieces; i++)
{
var start = i * pieceSize;
var end = start + (i == numberOfPieces - 1 ? pieceSize + totalFileSize % numberOfPieces : pieceSize);
tasks.Add(DownloadFilePiece(uri, start, end));
}
Task.WaitAll(tasks.ToArray());
//This is probably not the single most efficient way to combine byte arrays, but it is succinct...
return tasks.SelectMany(t => t.Result).ToArray();
}
private static async Task<byte[]> DownloadFilePiece(Uri uri, ulong rangeStart, ulong rangeEnd)
{
try
{
var request = (HttpWebRequest)WebRequest.Create(uri);
request.AddRange((long)rangeStart, (long)rangeEnd);
request.Proxy = WebProxy.GetDefaultProxy();
using (var response = await request.GetResponseAsync())
using (var responseStream = response.GetResponseStream())
using (var memoryStream = new MemoryStream((int)(rangeEnd - rangeStart)))
{
await responseStream.CopyToAsync(memoryStream);
return memoryStream.ToArray();
}
}
catch (WebException wex)
{
//Do lots of error handling here, lots of things can go wrong
//In particular watch for 416 Requested Range Not Satisfiable
return null;
}
catch (Exception ex)
{
//handle the unexpected here...
return null;
}
}
publicstaticvoid下载fileinpiecesandsave()
{
//试验
var uri=新的uri(“http://www.w3.org/");
var字节=下载的输入文件(uri,4);
writealBytes(@“c:\temp\RangeDownloadSample.html”,字节);
}
///
///使用范围请求通过HTTP将文件分为多个部分加载。
///
公共静态字节[]下载输入片段(Uri、uint numberOfPieces)
{
//我只是出于解释的目的编造了这个。实际上,您可能需要执行HEAD请求以获得总文件大小。
ulong totalFileSize=1003;
var pieceSize=totalFileSize/numberOfPieces;
列表任务=新列表();
对于(uint i=0;it.Result).ToArray();
}
私有静态异步任务下载文件(Uri、ulong rangeStart、ulong rangeEnd)
{
尝试
{
var request=(HttpWebRequest)WebRequest.Create(uri);
请求。添加范围((长)范围
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
namespace Console_21737681
{
class Program
{
const int MAX_PARALLEL = 4; // max parallel downloads
const int CHUNK_SIZE = 2048; // size of a single chunk
// a chunk of downloaded data
class Chunk
{
public long Start { get; set; }
public int Length { get; set; }
public byte[] Data { get; set; }
};
// throttle downloads
SemaphoreSlim _throttleSemaphore = new SemaphoreSlim(MAX_PARALLEL);
// get a chunk
async Task<Chunk> GetChunk(HttpClient client, long start, int length, string url)
{
await _throttleSemaphore.WaitAsync();
try
{
using (var request = new HttpRequestMessage(HttpMethod.Get, url))
{
request.Headers.Range = new System.Net.Http.Headers.RangeHeaderValue(start, start + length - 1);
using (var response = await client.SendAsync(request))
{
var data = await response.Content.ReadAsByteArrayAsync();
return new Chunk { Start = start, Length = length/*, Data = data*/ };
}
}
}
finally
{
_throttleSemaphore.Release();
}
}
// download the URL in parallel by chunks
async Task<Chunk[]> DownloadAsync(string url)
{
using (var client = new HttpClient())
{
var request = new HttpRequestMessage(HttpMethod.Head, url);
var response = await client.SendAsync(request);
var contentLength = response.Content.Headers.ContentLength;
if (!contentLength.HasValue)
throw new InvalidOperationException("ContentLength");
var numOfChunks = (int)((contentLength.Value + CHUNK_SIZE - 1) / CHUNK_SIZE);
var tasks = Enumerable.Range(0, numOfChunks).Select(i =>
{
// start a new chunk
long start = i * CHUNK_SIZE;
var length = (int)Math.Min(CHUNK_SIZE, contentLength.Value - start);
return GetChunk(client, start, length, url);
}).ToList();
await Task.WhenAll(tasks);
// the order of chunks is random
return tasks.Select(task => task.Result).ToArray();
}
}
static void Main(string[] args)
{
var program = new Program();
var chunks = program.DownloadAsync("http://flaglane.com/download/australian-flag/australian-flag-large.png").Result;
Console.WriteLine("Chunks: " + chunks.Count());
Console.ReadLine();
}
}
}