Java 基于Hadoop的JMH基准测试

我已经为我的MapReduce工作编写了一个JMH基准测试。如果我在本地模式下运行我的应用程序，它会工作，但是当我在hadoop集群上使用Thread脚本运行它时，我会得到以下错误：

[cloudera@quickstart Desktop]$ ./launch_mapreduce.sh 
# JMH 1.10 (released 5 days ago)
# VM invoker: /usr/java/jdk1.7.0_67-cloudera/jre/bin/java
# VM options: -Dproc_jar -Xmx1000m -Xms825955249 -Xmx825955249 -Djava.net.preferIPv4Stack=true -Dhadoop.log.dir=/usr/lib/hadoop-yarn/logs -Dyarn.log.dir=/usr/lib/hadoop-yarn/logs -Dhadoop.log.file=yarn.log -Dyarn.log.file=yarn.log -Dyarn.home.dir=/usr/lib/hadoop-yarn -Dhadoop.home.dir=/usr/lib/hadoop-yarn -Dhadoop.root.logger=INFO,console -Dyarn.root.logger=INFO,console -Djava.library.path=/usr/lib/hadoop/lib/native
# Warmup: 5 iterations, 1 s each
# Measurement: 5 iterations, 1 s each
# Timeout: 10 min per iteration
# Threads: 1 thread, will synchronize iterations
# Benchmark mode: Throughput, ops/time
# Benchmark: mgm.tp.bigdata.ma_mapreduce.MapReduceBenchmark.test

# Run progress: 0.00% complete, ETA 00:00:10
# Fork: 1 of 1
Error: Could not find or load main class org.openjdk.jmh.runner.ForkedMain
<forked VM failed with exit code 1>
<stdout last='20 lines'>
</stdout>
<stderr last='20 lines'>
Error: Could not find or load main class org.openjdk.jmh.runner.ForkedMain
</stderr>

# Run complete. Total time: 00:00:00

Benchmark  Mode  Cnt  Score   Error  Units

我的基准选项包括：

public static void main(String[] args) throws Exception {
    Options opt = new OptionsBuilder()
            .include(MapReduceBenchmark.class.getSimpleName())
            .warmupIterations(5)
            .measurementIterations(5)
            .forks(1)
            .build();

    new Runner(opt).run();
}

---

在我看来，jhm没有在hadoop集群上运行，因为在集群的每个节点上，基准测试都希望启动自己的jvm。如果不起作用，节点将为并行化进行通信。首先，我测量程序执行的时间并重复这个过程，最后计算容错性

import java.util.ArrayList;
import java.util.List;
import java.util.Properties;

public class MapReduceBenchmarkLauncher {

    private static List<Long> times = new ArrayList<Long>();

    public static void main(String[] args) throws Exception {
        Properties pro = new Properties();
        pro.load(MapReduceBenchmarkLauncher.class.getResourceAsStream("/config.properties"));
        int benchRounds = Integer.parseInt(pro.getProperty("benchmark.rounds"));

        for(int i=0;i<benchRounds;i++) {
            JobMain jm = new JobMain();// app being tested

            long start = System.nanoTime();
            jm.run();
            long end = System.nanoTime();

            times.add(end-start);
        }
        writeTimekeepingToFile(times, "mapreduce_benchmark");
    }

    public static void writeTimekeepingToFile(List<Long> times, String benchfile) throws Exception {
        // arithmetic mean
        double am = 0;
        for(int i=0;i<times.size();i++) {
            am = am + times.get(i);
        }
        am = am / times.size();
        // varinaz calculation
        double v = 0;
        for(int i=0;i<times.size();i++) {
            v = v + (times.get(i)-am)*(times.get(i)-am); // no math lib, cause long
        }
        v = v / times.size();
        // calculating standard deviation
        double s = Math.sqrt(v);
        // output
        BufferedWriter br = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(benchfile), "utf-8"));
        for(int i=0;i<times.size();i++) {
            br.write("round "+(i+1)+": "+times.get(i)+" ns\n");
        }
        br.write("varianz: v="+v+"\n");
        br.write("standard deviation: t=("+am+" \u00B1 "+s+") ns" );
        br.close();
    }
}

import java.util.ArrayList；
导入java.util.List；
导入java.util.Properties；
公共类MapReduceBenchmarkLauncher{
私有静态列表时间=new ArrayList（）；
公共静态void main（字符串[]args）引发异常{
Properties pro=新属性（）；
load（MapReduceBenchmarkLauncher.class.getResourceAsStream（“/config.properties”）；
int benchRounds=Integer.parseInt（pro.getProperty（“benchmark.rounds”）；
对于（int i=0；i我认为jhm没有在hadoop集群上运行，因为在集群的每个节点上，基准都希望启动一个自己的jvm。这不起作用，节点为并行化进行通信。首先，我测量程序的执行时间，并重复此操作，最后计算容错性

import java.util.ArrayList;
import java.util.List;
import java.util.Properties;

public class MapReduceBenchmarkLauncher {

    private static List<Long> times = new ArrayList<Long>();

    public static void main(String[] args) throws Exception {
        Properties pro = new Properties();
        pro.load(MapReduceBenchmarkLauncher.class.getResourceAsStream("/config.properties"));
        int benchRounds = Integer.parseInt(pro.getProperty("benchmark.rounds"));

        for(int i=0;i<benchRounds;i++) {
            JobMain jm = new JobMain();// app being tested

            long start = System.nanoTime();
            jm.run();
            long end = System.nanoTime();

            times.add(end-start);
        }
        writeTimekeepingToFile(times, "mapreduce_benchmark");
    }

    public static void writeTimekeepingToFile(List<Long> times, String benchfile) throws Exception {
        // arithmetic mean
        double am = 0;
        for(int i=0;i<times.size();i++) {
            am = am + times.get(i);
        }
        am = am / times.size();
        // varinaz calculation
        double v = 0;
        for(int i=0;i<times.size();i++) {
            v = v + (times.get(i)-am)*(times.get(i)-am); // no math lib, cause long
        }
        v = v / times.size();
        // calculating standard deviation
        double s = Math.sqrt(v);
        // output
        BufferedWriter br = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(benchfile), "utf-8"));
        for(int i=0;i<times.size();i++) {
            br.write("round "+(i+1)+": "+times.get(i)+" ns\n");
        }
        br.write("varianz: v="+v+"\n");
        br.write("standard deviation: t=("+am+" \u00B1 "+s+") ns" );
        br.close();
    }
}

import java.util.ArrayList；
导入java.util.List；
导入java.util.Properties；
公共类MapReduceBenchmarkLauncher{
私有静态列表时间=new ArrayList（）；
公共静态void main（字符串[]args）引发异常{
Properties pro=新属性（）；
load（MapReduceBenchmarkLauncher.class.getResourceAsStream（“/config.properties”）；
int benchRounds=Integer.parseInt（pro.getProperty（“benchmark.rounds”）；
对于（int i=0；i