Hbase批量定期装载HFiles和小压实关系

我有一个场景，在这个场景中，我们必须以拨号方式定期将HFiles加载到HBase表中

每个区域每次运行的文件大小可能在50到150 MB之间。这些负荷可能是每天12次，在某些情况下每15分钟一次

在进行测试时，我发现即使在区域中立即有3个以上的文件，也不会触发轻微压缩。这可能会导致问题出现，因为有很多文件为同一行键保存行

我已经看到压缩线程在10000秒（大约2小时45分钟）后被唤醒，开始压缩并将压缩任务放入队列

是否有任何配置可以告知在批量加载（completebulkload）写入3个或更多hFiles后立即触发小型压缩，而不管HFile的大小

Hbase版本：Hbase 1.1.2.2.6.5.4-1

---

在查看API时，我发现可以在hbase表级别异步调用次要或主要压缩

如果批量加载更频繁地推送冗余数据，并且使用恒定区域拆分大小策略，则可以使用HBase管理API根据避免拆分的需要调用压缩

下面是一个在Java中执行此操作的示例代码：

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Table;

public class Compaction {

    private String tableName;
    private String compactionType;
    private Configuration conf;
    private Connection hbaseConnection;
    private String confFile = "/usr/hdp/current/hbase-client/conf/hbase-site.xml";
    private Admin admin;
    private Table table;
    private int sleepTime = 1 ;

    public Compaction(String tableName, String compactionType) {
        this.tableName = tableName;
        this.compactionType = compactionType;

    }

    private void initHBaseConnection() {
        this.conf = HBaseConfiguration.create();
        try {
            conf.addResource(new FileInputStream(new File(confFile )));
            hbaseConnection = ConnectionFactory.createConnection(conf);
            table = hbaseConnection.getTable(TableName.valueOf(tableName));
            admin = hbaseConnection.getAdmin();

        } catch (IOException e) {
            e.printStackTrace();
        }

    }

    public boolean perfom() throws InterruptedException {


          System.out.println("Performing action: Compact table " + tableName + ", compact type =" + compactionType);
          try {
            if (compactionType.equalsIgnoreCase("major")) {
              admin.majorCompact(table.getName());
            } else {
              admin.compact(table.getName());
            }
          } catch (Exception ex) {
            System.err.println("Compaction failed, might be caused by other chaos: " + ex.getMessage());
            return false;
          }
          if (sleepTime  > 0) {
            Thread.sleep(sleepTime);
          }
        return true;
    }
    public static void main(String[] args) throws InterruptedException {
        String tableName = args[0];
        String compactionType = args[1];
        Compaction compaction = new Compaction(tableName, compactionType);
        compaction.initHBaseConnection();
        compaction.perfom();
    }

} 

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Table;

public class Compaction {

    private String tableName;
    private String compactionType;
    private Configuration conf;
    private Connection hbaseConnection;
    private String confFile = "/usr/hdp/current/hbase-client/conf/hbase-site.xml";
    private Admin admin;
    private Table table;
    private int sleepTime = 1 ;

    public Compaction(String tableName, String compactionType) {
        this.tableName = tableName;
        this.compactionType = compactionType;

    }

    private void initHBaseConnection() {
        this.conf = HBaseConfiguration.create();
        try {
            conf.addResource(new FileInputStream(new File(confFile )));
            hbaseConnection = ConnectionFactory.createConnection(conf);
            table = hbaseConnection.getTable(TableName.valueOf(tableName));
            admin = hbaseConnection.getAdmin();

        } catch (IOException e) {
            e.printStackTrace();
        }

    }

    public boolean perfom() throws InterruptedException {


          System.out.println("Performing action: Compact table " + tableName + ", compact type =" + compactionType);
          try {
            if (compactionType.equalsIgnoreCase("major")) {
              admin.majorCompact(table.getName());
            } else {
              admin.compact(table.getName());
            }
          } catch (Exception ex) {
            System.err.println("Compaction failed, might be caused by other chaos: " + ex.getMessage());
            return false;
          }
          if (sleepTime  > 0) {
            Thread.sleep(sleepTime);
          }
        return true;
    }
    public static void main(String[] args) throws InterruptedException {
        String tableName = args[0];
        String compactionType = args[1];
        Compaction compaction = new Compaction(tableName, compactionType);
        compaction.initHBaseConnection();
        compaction.perfom();
    }

}