diff --git a/hbase098/pom.xml b/hbase098/pom.xml index 5d72a0f9..6402ad06 100644 --- a/hbase098/pom.xml +++ b/hbase098/pom.xml @@ -1,48 +1,44 @@ 4.0.0 com.yahoo.ycsb binding-parent 0.13.0-SNAPSHOT ../binding-parent/ hbase098-binding HBase 0.98.x DB Binding - - false - - org.apache.hbase hbase-client ${hbase098.version} com.yahoo.ycsb core ${project.version} provided diff --git a/hbase098/src/main/java/com/yahoo/ycsb/db/HBaseClient.java b/hbase098/src/main/java/com/yahoo/ycsb/db/HBaseClient.java index f2bcfede..0d62baad 100644 --- a/hbase098/src/main/java/com/yahoo/ycsb/db/HBaseClient.java +++ b/hbase098/src/main/java/com/yahoo/ycsb/db/HBaseClient.java @@ -1,597 +1,481 @@ /** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * + * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. + *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import com.yahoo.ycsb.ByteArrayByteIterator; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.measurements.Measurements; - -import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.client.HConnectionManager; -import org.apache.hadoop.hbase.client.HConnection; -import org.apache.hadoop.hbase.client.HTableInterface; -import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.filter.PageFilter; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.security.UserGroupInformation; import java.io.IOException; -import java.util.ConcurrentModificationException; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Properties; -import java.util.Random; -import java.util.Set; -import java.util.Vector; +import java.util.*; import java.util.concurrent.atomic.AtomicInteger; import static com.yahoo.ycsb.workloads.CoreWorkload.TABLENAME_PROPERTY; import static com.yahoo.ycsb.workloads.CoreWorkload.TABLENAME_PROPERTY_DEFAULT; /** - * HBase client for YCSB framework + * HBase client for YCSB framework. */ -public class HBaseClient extends com.yahoo.ycsb.DB -{ - // BFC: Change to fix broken build (with HBase 0.20.6) - //private static final Configuration config = HBaseConfiguration.create(); - private static final Configuration config = HBaseConfiguration.create(); //new HBaseConfiguration(); - private static final AtomicInteger THREAD_COUNT = new AtomicInteger(0); - - public boolean _debug=false; - - public String _table=""; - private static HConnection _hConn=null; - public HTableInterface _hTable=null; - public String _columnFamily=""; - public byte _columnFamilyBytes[]; - public boolean _clientSideBuffering = false; - public long _writeBufferSize = 1024 * 1024 * 12; - /** Whether or not a page filter should be used to limit scan length. */ - public boolean _usePageFilter = true; - - public static final int HttpError=-2; - - public static final Object tableLock = new Object(); - - /** - * Initialize any state for this DB. - * Called once per DB instance; there is one DB instance per client thread. - */ - public void init() throws DBException - { - if ( (getProperties().getProperty("debug")!=null) && - (getProperties().getProperty("debug").compareTo("true")==0) ) - { - _debug=true; - } - - if (getProperties().containsKey("clientbuffering")) - { - _clientSideBuffering = Boolean.parseBoolean(getProperties().getProperty("clientbuffering")); - } - if (getProperties().containsKey("writebuffersize")) - { - _writeBufferSize = Long.parseLong(getProperties().getProperty("writebuffersize")); - } - if ("false".equals(getProperties().getProperty("hbase.usepagefilter", "true"))) { - _usePageFilter = false; - } - if ("kerberos".equalsIgnoreCase(config.get("hbase.security.authentication"))) { - config.set("hadoop.security.authentication", "Kerberos"); - UserGroupInformation.setConfiguration(config); - } - if ( (getProperties().getProperty("principal")!=null) && (getProperties().getProperty("keytab")!=null) ){ - try { - UserGroupInformation.loginUserFromKeytab(getProperties().getProperty("principal"), getProperties().getProperty("keytab")); - } catch (IOException e) { - System.err.println("Keytab file is not readable or not found"); - throw new DBException(e); - } - } - try { - THREAD_COUNT.getAndIncrement(); - synchronized(THREAD_COUNT) { - if (_hConn == null){ - _hConn = HConnectionManager.createConnection(config); - } - } - } catch (IOException e) { - System.err.println("Connection to HBase was not successful"); - throw new DBException(e); - } - _columnFamily = getProperties().getProperty("columnfamily"); - if (_columnFamily == null) - { - System.err.println("Error, must specify a columnfamily for HBase table"); - throw new DBException("No columnfamily specified"); - } - _columnFamilyBytes = Bytes.toBytes(_columnFamily); - - // Terminate right now if table does not exist, since the client - // will not propagate this error upstream once the workload - // starts. - String table = getProperties().getProperty(TABLENAME_PROPERTY, TABLENAME_PROPERTY_DEFAULT); - try - { - HTableInterface ht = _hConn.getTable(table); - ht.getTableDescriptor(); - } - catch (IOException e) - { - throw new DBException(e); - } +public class HBaseClient extends com.yahoo.ycsb.DB { + private static final Configuration CONFIG = HBaseConfiguration.create(); + private static final AtomicInteger THREAD_COUNT = new AtomicInteger(0); + + private boolean debug = false; + + private String tableName = ""; + private static HConnection hConn = null; + private HTableInterface hTable = null; + private String columnFamily = ""; + private byte[] columnFamilyBytes; + private boolean clientSideBuffering = false; + private long writeBufferSize = 1024 * 1024 * 12; + /** + * Whether or not a page filter should be used to limit scan length. + */ + private boolean usePageFilter = true; + + private static final Object TABLE_LOCK = new Object(); + + /** + * Initialize any state for this DB. + * Called once per DB instance; there is one DB instance per client thread. + */ + public void init() throws DBException { + if ((getProperties().getProperty("debug") != null) && + (getProperties().getProperty("debug").compareTo("true") == 0)) { + debug = true; } - /** - * Cleanup any state for this DB. - * Called once per DB instance; there is one DB instance per client thread. - */ - public void cleanup() throws DBException - { - // Get the measurements instance as this is the only client that should - // count clean up time like an update since autoflush is off. - Measurements _measurements = Measurements.getMeasurements(); - try { - long st=System.nanoTime(); - if (_hTable != null) { - _hTable.flushCommits(); - } - synchronized(THREAD_COUNT) { - int threadCount = THREAD_COUNT.decrementAndGet(); - if (threadCount <= 0 && _hConn != null) { - _hConn.close(); - } - } - long en=System.nanoTime(); - _measurements.measure("UPDATE", (int)((en-st)/1000)); - } catch (IOException e) { - throw new DBException(e); - } + if (getProperties().containsKey("clientbuffering")) { + clientSideBuffering = Boolean.parseBoolean(getProperties().getProperty("clientbuffering")); + } + if (getProperties().containsKey("writebuffersize")) { + writeBufferSize = Long.parseLong(getProperties().getProperty("writebuffersize")); + } + if ("false".equals(getProperties().getProperty("hbase.usepagefilter", "true"))) { + usePageFilter = false; + } + if ("kerberos".equalsIgnoreCase(CONFIG.get("hbase.security.authentication"))) { + CONFIG.set("hadoop.security.authentication", "Kerberos"); + UserGroupInformation.setConfiguration(CONFIG); } + if ((getProperties().getProperty("principal") != null) && (getProperties().getProperty("keytab") != null)) { + try { + UserGroupInformation.loginUserFromKeytab(getProperties().getProperty("principal"), + getProperties().getProperty("keytab")); + } catch (IOException e) { + System.err.println("Keytab file is not readable or not found"); + throw new DBException(e); + } + } + try { + THREAD_COUNT.getAndIncrement(); + synchronized (THREAD_COUNT) { + if (hConn == null) { + hConn = HConnectionManager.createConnection(CONFIG); + } + } + } catch (IOException e) { + System.err.println("Connection to HBase was not successful"); + throw new DBException(e); + } + columnFamily = getProperties().getProperty("columnfamily"); + if (columnFamily == null) { + System.err.println("Error, must specify a columnfamily for HBase tableName"); + throw new DBException("No columnfamily specified"); + } + columnFamilyBytes = Bytes.toBytes(columnFamily); + + // Terminate right now if tableName does not exist, since the client + // will not propagate this error upstream once the workload + // starts. + String table = getProperties().getProperty(TABLENAME_PROPERTY, TABLENAME_PROPERTY_DEFAULT); + try { + HTableInterface ht = hConn.getTable(table); + ht.getTableDescriptor(); + } catch (IOException e) { + throw new DBException(e); + } + } - public void getHTable(String table) throws IOException - { - synchronized (tableLock) { - _hTable = _hConn.getTable(table); - //2 suggestions from http://ryantwopointoh.blogspot.com/2009/01/performance-of-hbase-importing.html - _hTable.setAutoFlush(!_clientSideBuffering, true); - _hTable.setWriteBufferSize(_writeBufferSize); - //return hTable; - } + /** + * Cleanup any state for this DB. + * Called once per DB instance; there is one DB instance per client thread. + */ + public void cleanup() throws DBException { + // Get the measurements instance as this is the only client that should + // count clean up time like an update since autoflush is off. + Measurements measurements = Measurements.getMeasurements(); + try { + long st = System.nanoTime(); + if (hTable != null) { + hTable.flushCommits(); + } + synchronized (THREAD_COUNT) { + int threadCount = THREAD_COUNT.decrementAndGet(); + if (threadCount <= 0 && hConn != null) { + hConn.close(); + } + } + long en = System.nanoTime(); + measurements.measure("UPDATE", (int) ((en - st) / 1000)); + } catch (IOException e) { + throw new DBException(e); + } + } + private void getHTable(String table) throws IOException { + synchronized (TABLE_LOCK) { + hTable = hConn.getTable(table); + //2 suggestions from http://ryantwopointoh.blogspot.com/2009/01/performance-of-hbase-importing.html + hTable.setAutoFlush(!clientSideBuffering, true); + hTable.setWriteBufferSize(writeBufferSize); + //return hTable; } - /** - * Read a record from the database. Each field/value pair from the result will be stored in a HashMap. - * - * @param table The name of the table - * @param key The record key of the record to read. - * @param fields The list of fields to read, or null for all of them - * @param result A HashMap of field/value pairs for the result - * @return Zero on success, a non-zero error code on error - */ - public Status read(String table, String key, Set fields, HashMap result) - { - //if this is a "new" table, init HTable object. Else, use existing one - if (!_table.equals(table)) { - _hTable = null; - try - { - getHTable(table); - _table = table; - } - catch (IOException e) - { - System.err.println("Error accessing HBase table: "+e); - return Status.ERROR; - } - } + } - Result r = null; - try - { - if (_debug) { - System.out.println("Doing read from HBase columnfamily "+_columnFamily); - System.out.println("Doing read for key: "+key); - } - Get g = new Get(Bytes.toBytes(key)); - if (fields == null) { - g.addFamily(_columnFamilyBytes); - } else { - for (String field : fields) { - g.addColumn(_columnFamilyBytes, Bytes.toBytes(field)); - } - } - r = _hTable.get(g); - } - catch (IOException e) - { - System.err.println("Error doing get: "+e); - return Status.ERROR; - } - catch (ConcurrentModificationException e) - { - //do nothing for now...need to understand HBase concurrency model better - return Status.ERROR; - } + /** + * Read a record from the database. Each field/value pair from the result will be stored in a HashMap. + * + * @param table The name of the tableName + * @param key The record key of the record to read. + * @param fields The list of fields to read, or null for all of them + * @param result A HashMap of field/value pairs for the result + * @return Zero on success, a non-zero error code on error + */ + public Status read(String table, String key, Set fields, HashMap result) { + //if this is a "new" tableName, init HTable object. Else, use existing one + if (!this.tableName.equals(table)) { + hTable = null; + try { + getHTable(table); + this.tableName = table; + } catch (IOException e) { + System.err.println("Error accessing HBase tableName: " + e); + return Status.ERROR; + } + } - for (KeyValue kv : r.raw()) { - result.put( - Bytes.toString(kv.getQualifier()), - new ByteArrayByteIterator(kv.getValue())); - if (_debug) { - System.out.println("Result for field: "+Bytes.toString(kv.getQualifier())+ - " is: "+Bytes.toString(kv.getValue())); + Result r; + try { + if (debug) { + System.out.println("Doing read from HBase columnfamily " + columnFamily); + System.out.println("Doing read for key: " + key); + } + Get g = new Get(Bytes.toBytes(key)); + if (fields == null) { + g.addFamily(columnFamilyBytes); + } else { + for (String field : fields) { + g.addColumn(columnFamilyBytes, Bytes.toBytes(field)); + } + } + r = hTable.get(g); + } catch (IOException e) { + System.err.println("Error doing get: " + e); + return Status.ERROR; + } catch (ConcurrentModificationException e) { + //do nothing for now...need to understand HBase concurrency model better + return Status.ERROR; } - } + for (KeyValue kv : r.raw()) { + result.put( + Bytes.toString(kv.getQualifier()), + new ByteArrayByteIterator(kv.getValue())); + if (debug) { + System.out.println("Result for field: " + Bytes.toString(kv.getQualifier()) + + " is: " + Bytes.toString(kv.getValue())); + } + + } return Status.OK; + } + + /** + * Perform a range scan for a set of records in the database. Each field/value pair from the result will be stored + * in a HashMap. + * + * @param table The name of the tableName + * @param startkey The record key of the first record to read. + * @param recordcount The number of records to read + * @param fields The list of fields to read, or null for all of them + * @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record + * @return Zero on success, a non-zero error code on error + */ + public Status scan(String table, String startkey, int recordcount, Set fields, + Vector> result) { + //if this is a "new" tableName, init HTable object. Else, use existing one + if (!this.tableName.equals(table)) { + hTable = null; + try { + getHTable(table); + this.tableName = table; + } catch (IOException e) { + System.err.println("Error accessing HBase tableName: " + e); + return Status.ERROR; + } } - /** - * Perform a range scan for a set of records in the database. Each field/value pair from the result will be stored in a HashMap. - * - * @param table The name of the table - * @param startkey The record key of the first record to read. - * @param recordcount The number of records to read - * @param fields The list of fields to read, or null for all of them - * @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record - * @return Zero on success, a non-zero error code on error - */ - public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) - { - //if this is a "new" table, init HTable object. Else, use existing one - if (!_table.equals(table)) { - _hTable = null; - try - { - getHTable(table); - _table = table; - } - catch (IOException e) - { - System.err.println("Error accessing HBase table: "+e); - return Status.ERROR; - } - } + Scan s = new Scan(Bytes.toBytes(startkey)); + //HBase has no record limit. Here, assume recordcount is small enough to bring back in one call. + //We get back recordcount records + s.setCaching(recordcount); + if (this.usePageFilter) { + s.setFilter(new PageFilter(recordcount)); + } - Scan s = new Scan(Bytes.toBytes(startkey)); - //HBase has no record limit. Here, assume recordcount is small enough to bring back in one call. - //We get back recordcount records - s.setCaching(recordcount); - if (this._usePageFilter) { - s.setFilter(new PageFilter(recordcount)); - } + //add specified fields or else all fields + if (fields == null) { + s.addFamily(columnFamilyBytes); + } else { + for (String field : fields) { + s.addColumn(columnFamilyBytes, Bytes.toBytes(field)); + } + } - //add specified fields or else all fields - if (fields == null) - { - s.addFamily(_columnFamilyBytes); - } - else - { - for (String field : fields) - { - s.addColumn(_columnFamilyBytes,Bytes.toBytes(field)); - } + //get results + try (ResultScanner scanner = hTable.getScanner(s)) { + int numResults = 0; + for (Result rr = scanner.next(); rr != null; rr = scanner.next()) { + //get row key + String key = Bytes.toString(rr.getRow()); + if (debug) { + System.out.println("Got scan result for key: " + key); } - //get results - ResultScanner scanner = null; - try { - scanner = _hTable.getScanner(s); - int numResults = 0; - for (Result rr = scanner.next(); rr != null; rr = scanner.next()) - { - //get row key - String key = Bytes.toString(rr.getRow()); - if (_debug) - { - System.out.println("Got scan result for key: "+key); - } - - HashMap rowResult = new HashMap(); - - for (KeyValue kv : rr.raw()) { - rowResult.put( - Bytes.toString(kv.getQualifier()), - new ByteArrayByteIterator(kv.getValue())); - } - //add rowResult to result vector - result.add(rowResult); - numResults++; - - // PageFilter does not guarantee that the number of results is <= pageSize, so this - // break is required. - if (numResults >= recordcount) //if hit recordcount, bail out - { - break; - } - } //done with row + HashMap rowResult = new HashMap<>(); + for (KeyValue kv : rr.raw()) { + rowResult.put( + Bytes.toString(kv.getQualifier()), + new ByteArrayByteIterator(kv.getValue())); } + //add rowResult to result vector + result.add(rowResult); + numResults++; - catch (IOException e) { - if (_debug) - { - System.out.println("Error in getting/parsing scan result: "+e); - } - return Status.ERROR; - } - - finally { - scanner.close(); + // PageFilter does not guarantee that the number of results is <= pageSize, so this + // break is required. + //if hit recordcount, bail out + if (numResults >= recordcount) { + break; } + } //done with row - return Status.OK; + } catch (IOException e) { + if (debug) { + System.out.println("Error in getting/parsing scan result: " + e); + } + return Status.ERROR; } - /** - * Update a record in the database. Any field/value pairs in the specified values HashMap will be written into the record with the specified - * record key, overwriting any existing values with the same field name. - * - * @param table The name of the table - * @param key The record key of the record to write - * @param values A HashMap of field/value pairs to update in the record - * @return Zero on success, a non-zero error code on error - */ - public Status update(String table, String key, HashMap values) - { - //if this is a "new" table, init HTable object. Else, use existing one - if (!_table.equals(table)) { - _hTable = null; - try - { - getHTable(table); - _table = table; - } - catch (IOException e) - { - System.err.println("Error accessing HBase table: "+e); - return Status.ERROR; - } - } + return Status.OK; + } + /** + * Update a record in the database. Any field/value pairs in the specified values HashMap will be written into the + * record with the specified record key, overwriting any existing values with the same field name. + * + * @param table The name of the tableName + * @param key The record key of the record to write + * @param values A HashMap of field/value pairs to update in the record + * @return Zero on success, a non-zero error code on error + */ + public Status update(String table, String key, HashMap values) { + //if this is a "new" tableName, init HTable object. Else, use existing one + if (!this.tableName.equals(table)) { + hTable = null; + try { + getHTable(table); + this.tableName = table; + } catch (IOException e) { + System.err.println("Error accessing HBase tableName: " + e); + return Status.ERROR; + } + } - if (_debug) { - System.out.println("Setting up put for key: "+key); - } - Put p = new Put(Bytes.toBytes(key)); - for (Map.Entry entry : values.entrySet()) - { - byte[] value = entry.getValue().toArray(); - if (_debug) { - System.out.println("Adding field/value " + entry.getKey() + "/"+ - Bytes.toStringBinary(value) + " to put request"); - } - p.add(_columnFamilyBytes,Bytes.toBytes(entry.getKey()), value); - } - try - { - _hTable.put(p); - } - catch (IOException e) - { - if (_debug) { - System.err.println("Error doing put: "+e); - } - return Status.ERROR; - } - catch (ConcurrentModificationException e) - { - //do nothing for now...hope this is rare - return Status.ERROR; - } + if (debug) { + System.out.println("Setting up put for key: " + key); + } + Put p = new Put(Bytes.toBytes(key)); + for (Map.Entry entry : values.entrySet()) { + byte[] value = entry.getValue().toArray(); + if (debug) { + System.out.println("Adding field/value " + entry.getKey() + "/" + + Bytes.toStringBinary(value) + " to put request"); + } + p.add(columnFamilyBytes, Bytes.toBytes(entry.getKey()), value); + } - return Status.OK; + try { + hTable.put(p); + } catch (IOException e) { + if (debug) { + System.err.println("Error doing put: " + e); + } + return Status.ERROR; + } catch (ConcurrentModificationException e) { + //do nothing for now...hope this is rare + return Status.ERROR; } - /** - * Insert a record in the database. Any field/value pairs in the specified values HashMap will be written into the record with the specified - * record key. - * - * @param table The name of the table - * @param key The record key of the record to insert. - * @param values A HashMap of field/value pairs to insert in the record - * @return Zero on success, a non-zero error code on error - */ - public Status insert(String table, String key, HashMap values) - { - return update(table,key,values); + return Status.OK; + } + + /** + * Insert a record in the database. Any field/value pairs in the specified values HashMap will be written into the + * record with the specified record key. + * + * @param table The name of the tableName + * @param key The record key of the record to insert. + * @param values A HashMap of field/value pairs to insert in the record + * @return Zero on success, a non-zero error code on error + */ + public Status insert(String table, String key, HashMap values) { + return update(table, key, values); + } + + /** + * Delete a record from the database. + * + * @param table The name of the tableName + * @param key The record key of the record to delete. + * @return Zero on success, a non-zero error code on error + */ + public Status delete(String table, String key) { + //if this is a "new" tableName, init HTable object. Else, use existing one + if (!this.tableName.equals(table)) { + hTable = null; + try { + getHTable(table); + this.tableName = table; + } catch (IOException e) { + System.err.println("Error accessing HBase tableName: " + e); + return Status.ERROR; + } } - /** - * Delete a record from the database. - * - * @param table The name of the table - * @param key The record key of the record to delete. - * @return Zero on success, a non-zero error code on error - */ - public Status delete(String table, String key) - { - //if this is a "new" table, init HTable object. Else, use existing one - if (!_table.equals(table)) { - _hTable = null; - try - { - getHTable(table); - _table = table; - } - catch (IOException e) - { - System.err.println("Error accessing HBase table: "+e); - return Status.ERROR; - } - } + if (debug) { + System.out.println("Doing delete for key: " + key); + } - if (_debug) { - System.out.println("Doing delete for key: "+key); - } + Delete d = new Delete(Bytes.toBytes(key)); + try { + hTable.delete(d); + } catch (IOException e) { + if (debug) { + System.err.println("Error doing delete: " + e); + } + return Status.ERROR; + } - Delete d = new Delete(Bytes.toBytes(key)); - try - { - _hTable.delete(d); - } - catch (IOException e) - { - if (_debug) { - System.err.println("Error doing delete: "+e); - } - return Status.ERROR; - } + return Status.OK; + } - return Status.OK; + public static void main(String[] args) { + if (args.length != 3) { + System.out.println("Please specify a threadcount, columnfamily and operation count"); + System.exit(0); } - public static void main(String[] args) - { - if (args.length!=3) - { - System.out.println("Please specify a threadcount, columnfamily and operation count"); - System.exit(0); - } + final int keyspace = 10000; //120000000; - final int keyspace=10000; //120000000; - - final int threadcount=Integer.parseInt(args[0]); - - final String columnfamily=args[1]; - - - final int opcount=Integer.parseInt(args[2])/threadcount; - - Vector allthreads=new Vector(); - - for (int i=0; i result=new HashMap(); - - long accum=0; - - for (int i=0; i s = new HashSet(); - s.add("field1"); - s.add("field2"); - - rescode=cli.read("table1", key, s, result); - //rescode=cli.delete("table1",key); - rescode=cli.read("table1", key, s, result); - */ - HashSet scanFields = new HashSet(); - scanFields.add("field1"); - scanFields.add("field3"); - Vector> scanResults = new Vector>(); - result = cli.scan("table1","user2",20,null,scanResults); - - long en=System.currentTimeMillis(); - - accum+=(en-st); - - if (!result.equals(Status.OK)) - { - System.out.println("Error "+result+" for "+key); - } - - if (i%1==0) - { - System.out.println(i+" operations, average latency: "+(((double)accum)/((double)i))); - } - } - - //System.out.println("Average latency: "+(((double)accum)/((double)opcount))); - //System.out.println("Average get latency: "+(((double)cli.TotalGetTime)/((double)cli.TotalGetOps))); - } - catch (Exception e) - { - e.printStackTrace(); - } - } - }; - allthreads.add(t); - } + final int threadcount = Integer.parseInt(args[0]); - long st=System.currentTimeMillis(); - for (Thread t: allthreads) - { - t.start(); - } + final String columnfamily = args[1]; - for (Thread t: allthreads) - { - try - { - t.join(); - } - catch (InterruptedException e) - { + + final int opcount = Integer.parseInt(args[2]) / threadcount; + + Vector allthreads = new Vector<>(); + + for (int i = 0; i < threadcount; i++) { + Thread t = new Thread() { + public void run() { + try { + Random random = new Random(); + + HBaseClient cli = new HBaseClient(); + + Properties props = new Properties(); + props.setProperty("columnfamily", columnfamily); + props.setProperty("debug", "true"); + cli.setProperties(props); + + cli.init(); + + long accum = 0; + + for (int i = 0; i < opcount; i++) { + int keynum = random.nextInt(keyspace); + String key = "user" + keynum; + long st = System.currentTimeMillis(); + Status result; + Vector> scanResults = new Vector<>(); + result = cli.scan("table1", "user2", 20, null, scanResults); + + long en = System.currentTimeMillis(); + + accum += (en - st); + + if (!result.equals(Status.OK)) { + System.out.println("Error " + result + " for " + key); + } + + if (i % 10 == 0) { + System.out.println(i + " operations, average latency: " + (((double) accum) / ((double) i))); + } } + } catch (Exception e) { + e.printStackTrace(); + } } - long en=System.currentTimeMillis(); - - System.out.println("Throughput: "+((1000.0)*(((double)(opcount*threadcount))/((double)(en-st))))+" ops/sec"); + }; + allthreads.add(t); + } + long st = System.currentTimeMillis(); + for (Thread t : allthreads) { + t.start(); } -} -/* For customized vim control - * set autoindent - * set si - * set shiftwidth=4 -*/ + for (Thread t : allthreads) { + try { + t.join(); + } catch (InterruptedException ignored) { + //ignored + } + } + long en = System.currentTimeMillis(); + System.out.println("Throughput: " + ((1000.0) * (((double) (opcount * threadcount)) / ((double) (en - st)))) + + " ops/sec"); + } +} diff --git a/hbase098/src/main/java/com/yahoo/ycsb/db/package-info.java b/hbase098/src/main/java/com/yahoo/ycsb/db/package-info.java new file mode 100644 index 00000000..a0c1469f --- /dev/null +++ b/hbase098/src/main/java/com/yahoo/ycsb/db/package-info.java @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2017, Yahoo!, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. See accompanying LICENSE file. + */ + +/** + * The YCSB binding for HBase + * 0.98.X. The classes in this package are also used for the HBase 0.94.X + * but with different dependencies via the hbase094 binding. + */ +package com.yahoo.ycsb.db;