目前有6台机器,每台机器配置40核 160G内存 9T固态硬盘,然后使用CDH6.3.1部署了kudu1.10.0集群,其中一台为master,6台也同时作为Tablet Server。
然后做了如下优化:
memory_limit_hard_bytes 40G
block_cache_capacity_mb 10G
maintenance_manager_num_threads 8
然后创建了一张test表,四个字段(bigint,decimal,string,string),哈希分区成4个分区。
然后程序如下,往这张表插入1000条数据耗时1秒(有时候需要2-3秒),这也太慢了?各位看看如何优化,或者问题出在哪里?不甚感激!
public class KuduUtils {
public static KuduClient getKuduClient() {
return new KuduClient.KuduClientBuilder(AppParams.kudu_url()).defaultSocketReadTimeoutMs(600000).build();
}
public static void main(String[] args) throws Exception {
KuduClient kuduClient = new KuduClient.KuduClientBuilder("bigdata-0:7051").build();
int bufferSpace = 10000;
KuduTable kuduTable = kuduClient.openTable("impala::test.test");
KuduSession kuduSession = kuduClient.newSession();
kuduSession.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
kuduSession.setMutationBufferSpace(bufferSpace);
long l = System.currentTimeMillis();
long count = 0;
//准备数据
for (int i = 1; i <= 1000; i++) {
Upsert insert = kuduTable.newUpsert();
PartialRow insertRow = insert.getRow();
//设置字段的内容
insertRow.addLong("id", i);
insertRow.addDecimal("ff", BigDecimal.valueOf(i));
insertRow.addString("aa", "gfsdfdseeedddddddgfdgfsdfdseeedddddddgfdgfsdfdseeedddddddgfdgfsdfdsee");
insertRow.addString("cc", "gfsdfdseeedddddddgfdgfsdfdseeedddddddgfdgfsdfdseeedddddddgfdgfsdfdsee");
kuduSession.apply(insert);
count++;
if (count > bufferSpace / 2) {
count = 0;
kuduSession.flush();
}
}
kuduSession.close();
kuduClient.close();
System.out.println(System.currentTimeMillis() - l);
}
}