Java HTTP请求多线程：为什么不执行所有请求？

你好。请帮助我使用java中的线程。下面代码的精髓是从20个站点获得答案，每个站点在一个单独的线程中

问题在于，出于某种原因，并非所有流都被处理。嗅探器显示了10-14个查询，虽然应该是全部20个。 注意这个问题：为什么不去所有的请求

import java.net.*;
import java.io.*;
import java.util.*;
import java.util.concurrent.*;


public class ThreadsTest implements Callable<String> {

    private String url;

    public ThreadsTest(String url) {
        this.url = url;
    }

    public String call() throws Exception {


        Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress("127.0.0.1", 7777));

        URLConnection connection = new URL("http://" + url).openConnection(proxy);
        HttpURLConnection stopRedirect = (HttpURLConnection) connection;
        stopRedirect.setInstanceFollowRedirects(false);
        connection.setRequestProperty("User-Agent", "Mozilla/5.0 (compatible; MSIE 5.01; Windows NT 5.0)");
        connection.setRequestProperty("Referer", "http://www.google.co.uk");
        connection.setConnectTimeout(30000);
        connection.setReadTimeout(30000);

        BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()));

        String inputLine;
        String content = "";

        while ((inputLine = in.readLine()) != null) content+= inputLine;
        in.close();

        return content;
    }



    public static void main(String[] args)  throws Exception {


        String[] sites = {"courseassignment.com", "errocks.com", "villsoft.com", "miraclegarciniacambogiarx.net", "www.mcqplay.com", "www.ajnad-sham.com", "bstore.in", "rajestateagency.in", "royalboutique.in", "www.masiada.com",
                        "awebstreet.com", "adbfc.com", "starhairforum.com",  "akflix.com",  "planetskool.com",  "sceteducation.org",  "www.smarts3i.com",  "www.my.aybhost.com",  "www.svmconstructions.in",  "www.nssj.co.in"
        };

        System.out.println(sites.length);


        ExecutorService service = Executors.newCachedThreadPool();
        List<Future<String>> futures = new ArrayList<Future<String>>();

        for (int i = 0; i < sites.length; i++) {
            //запуск нового потока с параметром из массива
            try{

                Future<String> future = service.submit(new ThreadsTest(sites[i]));
                futures.add(future);

            }
            catch (Exception e) {  }
        }   


        for (Future<String> future : futures) {

            try {

                future.get();

                //System.out.println("get result from Future: " + future.get());

            }
             catch (InterruptedException | ExecutionException e) {
                //e.printStackTrace();
            }

        }

        service.shutdown();
        while (!service.isTerminated()) {

        }

    }
}

---

您需要配置ExecutorService的最大线程数。

问题 如果取消注释e.printStackTrace，您应该会看到某个URL返回403状态码，这会导致调用方法出现异常

Exception in thread "main" java.util.concurrent.ExecutionException: java.io.IOException: Server returned HTTP response code: 403 for URL: http://www.my.aybhost.com

当客户机接收到403状态响应头时，则没有要读取的响应主体，因此当尝试从连接的输入流创建读取器时，它会引发异常。您可能希望以不同的方式处理非200响应代码

解决方案

---

使用wireshark，我还验证了所有20个http请求都是从我的机器发出的。因此，如果您仍然没有看到20个请求被发送出去，那么它可能与代码无关；您的防火墙、代理、java安全或其他网络设置可能存在问题。

此时！服务是邪恶的。永远不要这样做。这是有原因的。重新发明轮子？为什么不直接使用f.ex.？此外，我刚刚注意到：catch Exception e{}。你确定没有抛出异常而你只是在吞咽它？您似乎到处都在吞咽所有异常…当使用newCachedThreadPool时，默认配置是无限的。我使用了您的代码，但不是所有请求都再次处理10-14个请求。可能是操作系统或Java的问题？我使用Windows XP 64位和JDK 7u60 x64.Hmmm。。。您应该看到20个请求，但只有17个请求有html正文可供读取。它可能是阻止出站HTTP通信的本地防火墙或代理设置。您如何验证已处理哪些请求？您正在使用哪个网络嗅探器？我建议使用Wireshark嗅出HTTP请求。您可能还需要执行System.out.printlnurl+returned+connection.getResponseCode+响应代码；就在call方法的return语句之前，打印出正在处理的URL。

public String url;

public ThreadsTest(String url) {
    this.url = url;
}

public String call() throws Exception {
    HttpURLConnection connection = (HttpURLConnection) new URL("http://"
            + url).openConnection();
    connection.setInstanceFollowRedirects(false);
    connection.setRequestProperty("User-Agent",
            "Mozilla/5.0 (compatible; MSIE 5.01; Windows NT 5.0)");
    connection.setRequestProperty("Referer", "http://www.google.co.uk");
    connection.setConnectTimeout(30000);
    connection.setReadTimeout(30000);

    String content = "";
    if (connection.getResponseCode() == 200) {
        BufferedReader in = new BufferedReader(new InputStreamReader(
                connection.getInputStream()));
        String inputLine;

        while ((inputLine = in.readLine()) != null)
            content += inputLine;
        in.close();
        System.out.println(url + " returned "
                + connection.getResponseCode() + " response code");
    } else {
        System.err.println(url + " returned "
                + connection.getResponseCode() + " response code");
    }

    return content;
}

public static void main(String[] args) throws Exception {

    String[] sites = { "courseassignment.com", "errocks.com",
            "villsoft.com", "miraclegarciniacambogiarx.net",
            "www.mcqplay.com", "www.ajnad-sham.com", "bstore.in",
            "rajestateagency.in", "royalboutique.in", "www.masiada.com",
            "awebstreet.com", "adbfc.com", "starhairforum.com",
            "akflix.com", "planetskool.com", "sceteducation.org",
            "www.smarts3i.com", "www.my.aybhost.com",
            "www.svmconstructions.in", "www.nssj.co.in" };

    System.out.println(sites.length);

    ExecutorService service = Executors.newCachedThreadPool();
    List<Future<String>> futures = new ArrayList<Future<String>>();

    for (int i = 0; i < sites.length; i++) {

        Future<String> future = service.submit(new ThreadsTest(sites[i]));
        futures.add(future);

    }

    for (Future<String> future : futures) {
        try {
            future.get();
        } catch (InterruptedException | ExecutionException e) {
            e.printStackTrace();
        }
    }

    service.shutdown();
    if (!service.awaitTermination(10, TimeUnit.SECONDS)) {
        // Timed Out waiting to finish, so force a shutdown
        service.shutdownNow();
    }
}

awebstreet.com returned 301 response code
starhairforum.com returned 200 response code
www.nssj.co.in returned 200 response code
royalboutique.in returned 200 response code
planetskool.com returned 200 response code
www.svmconstructions.in returned 200 response code
sceteducation.org returned 200 response code
bstore.in returned 200 response code
rajestateagency.in returned 200 response code
villsoft.com returned 200 response code
www.mcqplay.com returned 200 response code
www.my.aybhost.com returned 403 response code
errocks.com returned 200 response code
adbfc.com returned 200 response code
miraclegarciniacambogiarx.net returned 200 response code
akflix.com returned 200 response code
www.ajnad-sham.com returned 200 response code
www.smarts3i.com returned 200 response code
www.masiada.com returned 301 response code
courseassignment.com returned 200 response code