Vb.net 如何限制并发异步webrequests

我经常需要发出大量的webrequests，而不会使网络过载

目前，我通过并行运行同步请求来实现这一点，利用ThreadPool.SetMinThreads和MaxDegreeOfParallelism精确指定并发运行的请求数

现在这一切都很好，但感觉不对

我真的很想使用异步方法，但我无法确定如何限制并发请求的数量

我的并行方法的一个简化示例（为简洁起见，使用webclient和无错误处理）：

这是一个阻塞函数，这是我所需要的。 现在，我已经在常规for循环中测试了webclient.downloadStringAsync方法，它几乎会一次触发所有请求，从而使网络过载

我想做的是首先发出X个请求，然后在每个响应返回时发出新的请求

我相当肯定任务是一条路要走，我肯定我已经阅读了一些非常好的c#实现，但我的c#经验有限，我很难将c#lambadas翻译成vb.net

我也仅限于vs2010和.net4，因此.net4.5 async await的细节不适合我

---

非常感谢任何帮助

您可以在VB.NET中使用Wintellect Powerthreading库的类异步执行此操作，您可以从NuGet获得该类

这为您提供了Await的一些功能，但在VS2010中可以使用.Net 2.0到4.0，同时为您提供了4.5异步功能的升级路径

缺点是WebClient异步方法需要基于AsyncEnumerator使用的任务的EAP到APM垫片，因此代码要复杂得多

控制并发请求数量的最简单方法是启动X个异步操作，然后每次完成一个异步操作时启动另一个异步操作

示例代码：

Imports System.Collections.Generic
Imports System.Runtime.CompilerServices
Imports System.Threading.Tasks
Imports System.Net
Imports Wintellect.Threading.AsyncProgModel

Module TaskExtension
    REM http://msdn.microsoft.com/en-us/library/hh873178.aspx
    <Extension()>
    Public Function AsApm(Of T1)(ByVal task As Task(Of T1), callback As AsyncCallback, state As Object) As IAsyncResult
        If (task Is Nothing) Then
            Throw New ArgumentNullException("task")
        End If
        Dim tcs = New TaskCompletionSource(Of T1)(state)
        task.ContinueWith(Sub(t As Task(Of T1))
                              If (t.IsFaulted) Then
                                  tcs.TrySetException(t.Exception.InnerExceptions)
                              ElseIf t.IsCanceled Then
                                  tcs.TrySetCanceled()
                              Else : tcs.TrySetResult(t.Result)
                              End If
                              If (Not callback Is Nothing) Then
                                  callback(tcs.Task)
                              End If
                          End Sub, TaskScheduler.Default)
        Return tcs.Task
    End Function
End Module

Module ApmAsyncDownload
    Public Function DownloadStringAsync(url As Uri) As Task(Of String)
        Dim tcs As New TaskCompletionSource(Of String)
        Dim wc As New WebClient()
        AddHandler wc.DownloadStringCompleted, Sub(s As Object, e As System.Net.DownloadStringCompletedEventArgs)
                                                   If (Not (e.Error Is Nothing)) Then
                                                       tcs.TrySetException(e.Error)
                                                   ElseIf e.Cancelled Then
                                                       tcs.TrySetCanceled()
                                                   Else : tcs.TrySetResult(e.Result)
                                                   End If
                                               End Sub
        wc.DownloadStringAsync(url)
        Return tcs.Task
    End Function
    Public Function BeginDownloadString(url As Uri, callback As AsyncCallback, state As Object) As IAsyncResult
        Return DownloadStringAsync(url).AsApm(callback, state)
    End Function
    Public Function EndDownloadString(asyncResult As IAsyncResult) As String
        Dim castToTask As Task(Of String) = asyncResult
        Return castToTask.Result
    End Function
End Module

Public Class AsyncIterators
    Private Shared Iterator Function SearchUrl(ae As AsyncEnumerator(Of Boolean), keyword As String, uri As Uri) As IEnumerator(Of Int32)
        ae.Result = False
        ApmAsyncDownload.BeginDownloadString(uri, ae.End(0, AddressOf ApmAsyncDownload.EndDownloadString), Nothing)
        Yield 1
        If (ae.IsCanceled()) Then
            Return
        End If
        Try
            Dim page As String = ApmAsyncDownload.EndDownloadString(ae.DequeueAsyncResult)
            ae.Result = page.Contains(keyword)
        Catch ex As AggregateException
        End Try
    End Function
    Public Shared Iterator Function SearchIterator(ae As AsyncEnumerator(Of List(Of String)), keyword As String, urls As List(Of Uri)) As IEnumerator(Of Int32)
        ae.Result = New List(Of String)
        'Control how many searches are started asynchonously
        Dim startSearches = Math.Min(3, urls.Count)
        Dim enumerator = urls.GetEnumerator
        Dim toBeCompleted = urls.Count
        Do Until (toBeCompleted <= 0)
            While (startSearches > 0)
                If enumerator.MoveNext Then
                    Dim subAe = New AsyncEnumerator(Of Boolean)()
                    subAe.SyncContext = Nothing
                    subAe.BeginExecute(SearchUrl(subAe, keyword, enumerator.Current), ae.End(0, Function(ar As IAsyncResult) As AsyncEnumerator.EndObjectXxx
                                                                                                    subAe.EndExecute(ar)
                                                                                                End Function), enumerator.Current)
                End If
                startSearches = startSearches - 1
            End While
            'Wait for first async search to complete
            Yield 1
            toBeCompleted = toBeCompleted - 1
            If (ae.IsCanceled()) Then
                Exit Do
            End If
            'Get result of the search and add to results
            Dim result = ae.DequeueAsyncResult()
            Dim completedAe = AsyncEnumerator(Of Boolean).FromAsyncResult(result)
            If (completedAe.EndExecute(result)) Then
                Dim uri As Uri = result.AsyncState
                ae.Result.Add(uri.OriginalString)
            End If
            'Start 1 more search
            startSearches = startSearches + 1
        Loop
    End Function
End Class

Module Module1
    Sub Main()
        Dim searchAe = New AsyncEnumerator(Of List(Of String))()
        searchAe.SyncContext = Nothing
        Dim urlStrings = New List(Of String) From {"http://www.google.com", "http://www.yahoo.com", "http://www.dogpile.com"}
        Dim uris = urlStrings.Select(Function(urlString As String) As Uri
                                         Return New Uri(urlString)
                                     End Function).ToList()
        For Each Str As String In searchAe.EndExecute(searchAe.BeginExecute(AsyncIterators.SearchIterator(searchAe, "search", uris), Nothing, Nothing))
            Console.WriteLine(Str)
        Next
        Console.ReadKey()
    End Sub
End Module

导入System.Collections.Generic
导入System.Runtime.CompilerServices
导入System.Threading.Tasks
导入系统.Net
导入Wintellect.Threading.AsyncProgModel
模块任务扩展
雷姆http://msdn.microsoft.com/en-us/library/hh873178.aspx
作为IAsyncResult的公共函数AsApm（属于T1）（ByVal任务作为任务（属于T1），回调作为异步回调，状态作为对象）
如果（任务不算什么）那么
抛出新的ArgumentNullException（“任务”）
如果结束
Dim tcs=新任务完成源（T1的）（状态）
任务。继续（子任务（t为任务（共T1））
如果（t.IsFaulted），则
tcs.TrySetException（t.Exception.InnerException）
如果他被取消了
tcs.TrySetCanceled（）
其他：tcs.TrySetResult（t.Result）
如果结束
如果（不是回调什么都不是）那么
回调（tcs.Task）
如果结束
结束子任务，TaskScheduler.Default）
返回tcs.Task
端函数
端模块
模块ApmAsyncDownload
公共函数下载StringAsync（url作为Uri）作为任务（字符串的）
将tcs设置为新TaskCompletionSource（字符串的）
Dim wc作为新的WebClient（）
AddHandler wc.DownloadStringCompleted，Sub（s作为对象，e作为System.Net.DownloadStringCompletedEventArgs）
如果（不是（即错误为零），则
tcs.TrySetException（例如错误）
艾尔塞夫取消了
tcs.TrySetCanceled（）
其他：tcs.TrySetResult（e.Result）
如果结束
端接头
wc.DownloadStringAsync（url）
返回tcs.Task
端函数
公共函数BeginDownloadString（url作为Uri，回调作为AsyncCallback，状态作为Object）作为IAsyncResult
返回DownloadStringAsync（url）.AsApm（回调，状态）
端函数
公共函数EndDownloadString（asyncResult作为IAsyncResult）作为字符串
Dim castToTask As Task（字符串的）=asyncResult
返回castToTask.Result
端函数
端模块
公共类异步迭代器
私有共享迭代器函数SearchUrl（ae作为异步枚举器（布尔型），关键字作为字符串，uri作为uri）作为IEnumerator（Int32型）
ae.结果=错误
ApmAsyncDownload.BeginDownloadString（uri，ae.End（0，ApmAsyncDownload.EndDownloadString的地址），无任何内容）
收益率1
如果（ae.IsCanceled（）），则
返回
如果结束
尝试
Dim页面为String=ApmAsyncDownload.EndDownloadString（ae.DequeueAsyncResult）
ae.Result=page.Contains（关键字）
捕获ex作为聚合异常
结束尝试
端函数
公共共享迭代器函数SearchIterator（ae作为异步枚举器（属于列表（属于字符串）），关键字作为字符串，URL作为列表（属于Uri））作为IEnumerator（属于Int32）
ae.Result=新列表（字符串）
'控制异步启动的搜索数量
Dim startSearchs=Math.Min（3，url.Count）
Dim枚举器=URL.GetEnumerator
Dim toBeCompleted=url.Count
直到（完成0）为止
如果enumerator.MoveNext，则
Dim subAe=新的异步枚举数（布尔值）（）
subAe.SyncContext=无
subAe.beginecute（SearchUrl（subAe，关键字，enumerator.Current），ae.End（0，函数（ar作为IAsyncResult））作为AsyncEnumerator.EndObjectXxx
subAe.EndExecute（ar）
Imports System.Collections.Generic
Imports System.Runtime.CompilerServices
Imports System.Threading.Tasks
Imports System.Net
Imports Wintellect.Threading.AsyncProgModel

Module TaskExtension
    REM http://msdn.microsoft.com/en-us/library/hh873178.aspx
    <Extension()>
    Public Function AsApm(Of T1)(ByVal task As Task(Of T1), callback As AsyncCallback, state As Object) As IAsyncResult
        If (task Is Nothing) Then
            Throw New ArgumentNullException("task")
        End If
        Dim tcs = New TaskCompletionSource(Of T1)(state)
        task.ContinueWith(Sub(t As Task(Of T1))
                              If (t.IsFaulted) Then
                                  tcs.TrySetException(t.Exception.InnerExceptions)
                              ElseIf t.IsCanceled Then
                                  tcs.TrySetCanceled()
                              Else : tcs.TrySetResult(t.Result)
                              End If
                              If (Not callback Is Nothing) Then
                                  callback(tcs.Task)
                              End If
                          End Sub, TaskScheduler.Default)
        Return tcs.Task
    End Function
End Module

Module ApmAsyncDownload
    Public Function DownloadStringAsync(url As Uri) As Task(Of String)
        Dim tcs As New TaskCompletionSource(Of String)
        Dim wc As New WebClient()
        AddHandler wc.DownloadStringCompleted, Sub(s As Object, e As System.Net.DownloadStringCompletedEventArgs)
                                                   If (Not (e.Error Is Nothing)) Then
                                                       tcs.TrySetException(e.Error)
                                                   ElseIf e.Cancelled Then
                                                       tcs.TrySetCanceled()
                                                   Else : tcs.TrySetResult(e.Result)
                                                   End If
                                               End Sub
        wc.DownloadStringAsync(url)
        Return tcs.Task
    End Function
    Public Function BeginDownloadString(url As Uri, callback As AsyncCallback, state As Object) As IAsyncResult
        Return DownloadStringAsync(url).AsApm(callback, state)
    End Function
    Public Function EndDownloadString(asyncResult As IAsyncResult) As String
        Dim castToTask As Task(Of String) = asyncResult
        Return castToTask.Result
    End Function
End Module

Public Class AsyncIterators
    Private Shared Iterator Function SearchUrl(ae As AsyncEnumerator(Of Boolean), keyword As String, uri As Uri) As IEnumerator(Of Int32)
        ae.Result = False
        ApmAsyncDownload.BeginDownloadString(uri, ae.End(0, AddressOf ApmAsyncDownload.EndDownloadString), Nothing)
        Yield 1
        If (ae.IsCanceled()) Then
            Return
        End If
        Try
            Dim page As String = ApmAsyncDownload.EndDownloadString(ae.DequeueAsyncResult)
            ae.Result = page.Contains(keyword)
        Catch ex As AggregateException
        End Try
    End Function
    Public Shared Iterator Function SearchIterator(ae As AsyncEnumerator(Of List(Of String)), keyword As String, urls As List(Of Uri)) As IEnumerator(Of Int32)
        ae.Result = New List(Of String)
        'Control how many searches are started asynchonously
        Dim startSearches = Math.Min(3, urls.Count)
        Dim enumerator = urls.GetEnumerator
        Dim toBeCompleted = urls.Count
        Do Until (toBeCompleted <= 0)
            While (startSearches > 0)
                If enumerator.MoveNext Then
                    Dim subAe = New AsyncEnumerator(Of Boolean)()
                    subAe.SyncContext = Nothing
                    subAe.BeginExecute(SearchUrl(subAe, keyword, enumerator.Current), ae.End(0, Function(ar As IAsyncResult) As AsyncEnumerator.EndObjectXxx
                                                                                                    subAe.EndExecute(ar)
                                                                                                End Function), enumerator.Current)
                End If
                startSearches = startSearches - 1
            End While
            'Wait for first async search to complete
            Yield 1
            toBeCompleted = toBeCompleted - 1
            If (ae.IsCanceled()) Then
                Exit Do
            End If
            'Get result of the search and add to results
            Dim result = ae.DequeueAsyncResult()
            Dim completedAe = AsyncEnumerator(Of Boolean).FromAsyncResult(result)
            If (completedAe.EndExecute(result)) Then
                Dim uri As Uri = result.AsyncState
                ae.Result.Add(uri.OriginalString)
            End If
            'Start 1 more search
            startSearches = startSearches + 1
        Loop
    End Function
End Class

Module Module1
    Sub Main()
        Dim searchAe = New AsyncEnumerator(Of List(Of String))()
        searchAe.SyncContext = Nothing
        Dim urlStrings = New List(Of String) From {"http://www.google.com", "http://www.yahoo.com", "http://www.dogpile.com"}
        Dim uris = urlStrings.Select(Function(urlString As String) As Uri
                                         Return New Uri(urlString)
                                     End Function).ToList()
        For Each Str As String In searchAe.EndExecute(searchAe.BeginExecute(AsyncIterators.SearchIterator(searchAe, "search", uris), Nothing, Nothing))
            Console.WriteLine(Str)
        Next
        Console.ReadKey()
    End Sub
End Module

    Dim google As String = "http://www.google.com/#&q="

    Dim qsites As New Concurrent.ConcurrentQueue(Of String)
    For Each k In {"foo", "bar", "john", "jack", "stackoverflow", "basic", "ship", "car", "42"}
        qsites.Enqueue(google & k)
    Next

    Dim cde As New System.Threading.CountdownEvent(qsites.Count)

    Dim strings As New Concurrent.ConcurrentBag(Of String)
    Dim completedhandler = Sub(wco As Object, ev As Net.DownloadStringCompletedEventArgs)
                               Dim wc = DirectCast(wco, Net.WebClient)
                               Debug.Print("got one!")
                               strings.Add(ev.Result)
                               cde.Signal()
                               Dim s As String = String.Empty
                               If qsites.TryDequeue(s) Then
                                   Debug.Print("downloading from {0}", s)
                                   wc.DownloadStringAsync(New Uri(s))
                               End If
                           End Sub

    Dim numthreads As Integer = 4

    System.Threading.Tasks.Task.Factory.StartNew(Sub()
                                                     For i = 1 To numthreads
                                                         Dim s As String = String.Empty
                                                         If qsites.TryDequeue(s) Then
                                                             Dim wc As New Net.WebClient
                                                             wc.Proxy = Nothing
                                                             AddHandler wc.DownloadStringCompleted, completedhandler
                                                             Debug.Print("downloading from {0}", s)
                                                             wc.DownloadStringAsync(New Uri(s))
                                                         End If
                                                     Next
                                                 End Sub)

    cde.Wait()