diff --git a/bin/ycsb.sh b/bin/ycsb.sh old mode 100644 new mode 100755 index 697ec7dd..5669b7ff --- a/bin/ycsb.sh +++ b/bin/ycsb.sh @@ -1,125 +1,121 @@ #! /usr/bin/env bash # Set the YCSB specific environment. Adds all the required libraries to the class path. # Copyright (c) 2010 Yahoo! Inc. All rights reserved. # * # * Licensed under the Apache License, Version 2.0 (the "License"); you # * may not use this file except in compliance with the License. You # * may obtain a copy of the License at # * # * http://www.apache.org/licenses/LICENSE-2.0 # * # * Unless required by applicable law or agreed to in writing, software # * distributed under the License is distributed on an "AS IS" BASIS, # * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # * implied. See the License for the specific language governing # * permissions and limitations under the License. See accompanying # * LICENSE file. # # The Java implementation to use. This is required. #export JAVA_HOME= # Any JVM options to pass. #export YCSB_OPTS="-Djava.compiler=NONE" # YCSB client heap size. #export YCSB_HEAP_SIZE=500 this=`dirname "$0"` this=`cd "$this"; pwd` while [ -h "$this" ]; do ls=`ls -ld "$this"` link=`expr "$ls" : '.*-> \(.*\)$'` if expr "$link" : '.*/.*' > /dev/null; then this="$link" else this=`dirname "$this"`/"$link" fi done bin=`dirname "$this"` script=`basename "$this"` bin=`cd "$bin"; pwd` this="$bin/$script" # the root of the Hadoop installation export YCSB_HOME=`dirname "$this"` echo "YCSB_HOME $YCSB_HOME" cygwin=false case "`uname`" in CYGWIN*) cygwin=true;; esac # if no args specified, show usage if [ $# = 0 ]; then echo "Usage: ycsb CLASSNAME" echo "where CLASSNAME is the name of the class to run" echo "The jar file for the class must be in bin, build, lib, or db/*/lib." exit 1 fi # get arguments COMMAND=$1 shift JAVA="" if [ "$JAVA_HOME" != "" ]; then JAVA=$JAVA_HOME/bin/java else echo "JAVA_HOME must be set." exit 1 fi JAVA_HEAP_MAX=-Xmx500m # check envvars which might override default args if [ "$YCSB_HEAP_SIZE" != "" ]; then JAVA_HEAP_MAX="-Xmx""$YCSB_HEAP_SIZE""m" fi # Set the classpath. if [ "$CLASSPATH" != "" ]; then CLASSPATH=${CLASSPATH}:$JAVA_HOME/lib/tools.jar else CLASSPATH=$JAVA_HOME/lib/tools.jar fi # so that filenames w/ spaces are handled correctly in loops below IFS= -for f in $YCSB_HOME/build/*.jar; do - CLASSPATH=${CLASSPATH}:$f -done - for f in $YCSB_HOME/lib/*.jar; do CLASSPATH=${CLASSPATH}:$f done for f in $YCSB_HOME/db/*; do if [ -d $f ]; then for j in $f/lib/*.jar; do CLASSPATH=${CLASSPATH}:$j done fi done -#echo "CLASSPATH=$CLASSPATH" +echo "CLASSPATH=$CLASSPATH" # restore ordinary behavior unset IFS CLASS=$COMMAND # cygwin path translation if $cygwin; then CLASSPATH=`cygpath -p -w "$CLASSPATH"` YCSB_HOME=`cygpath -w "$YCSB_HOME"` fi #echo "Executing command $CLASS with options $JAVA_HEAP_MAX $YCSB_OPTS $CLASS $@" exec "$JAVA" $JAVA_HEAP_MAX $YCSB_OPTS -classpath "$CLASSPATH" $CLASS "$@" diff --git a/db/cassandra-0.5/lib/README b/db/cassandra-0.5/lib/README deleted file mode 100644 index 935c9e9f..00000000 --- a/db/cassandra-0.5/lib/README +++ /dev/null @@ -1 +0,0 @@ -This directory should contain jars for building and running Cassandra 0.5 diff --git a/db/cassandra-0.5/src/com/yahoo/ycsb/db/CassandraClient5.java b/db/cassandra-0.5/src/com/yahoo/ycsb/db/CassandraClient5.java deleted file mode 100644 index 19f9f528..00000000 --- a/db/cassandra-0.5/src/com/yahoo/ycsb/db/CassandraClient5.java +++ /dev/null @@ -1,502 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -/* - * Cassandra client specific to version 0.5 of Cassandra. - */ - -package com.yahoo.ycsb.db; - -import com.yahoo.ycsb.*; -import java.util.List; -import java.util.Set; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Vector; -import java.util.Random; -import java.util.Properties; - -import org.apache.thrift.transport.TTransport; -import org.apache.thrift.transport.TSocket; -import org.apache.thrift.protocol.TProtocol; -import org.apache.thrift.protocol.TBinaryProtocol; -import org.apache.cassandra.service.*; - - -// XXXX if we do replication, fix the consistency levels -/** - * Cassandra 0.5 client for YCSB framework - */ -public class CassandraClient5 extends DB -{ - static Random random=new Random(); - public static final int Ok=0; - public static final int Error=-1; - - public int ConnectionRetries; - public int OperationRetries; - - public static final String CONNECTION_RETRY_PROPERTY="cassandra.connectionretries"; - public static final String CONNECTION_RETRY_PROPERTY_DEFAULT="300"; - - public static final String OPERATION_RETRY_PROPERTY="cassandra.operationretries"; - public static final String OPERATION_RETRY_PROPERTY_DEFAULT="300"; - - - TTransport tr; - Cassandra.Client client; - - boolean _debug=false; - - /** - * Initialize any state for this DB. - * Called once per DB instance; there is one DB instance per client thread. - */ - public void init() throws DBException - { - String hosts=getProperties().getProperty("hosts"); - if (hosts==null) - { - throw new DBException("Required property \"hosts\" missing for CassandraClient5"); - } - - ConnectionRetries=Integer.parseInt(getProperties().getProperty(CONNECTION_RETRY_PROPERTY,CONNECTION_RETRY_PROPERTY_DEFAULT)); - OperationRetries=Integer.parseInt(getProperties().getProperty(OPERATION_RETRY_PROPERTY,OPERATION_RETRY_PROPERTY_DEFAULT)); - - _debug=Boolean.parseBoolean(getProperties().getProperty("debug","false")); - - String[] allhosts=hosts.split(","); - String myhost=allhosts[random.nextInt(allhosts.length)]; - //System.out.println("My host: ["+myhost+"]"); - //System.exit(0); - - Exception connectexception=null; - - for (int retry=0; retry fields, HashMap result) - { - Exception errorexception=null; - - for (int i=0; i fieldlist=new Vector(); - for (String s : fields) - { - fieldlist.add(s.getBytes("UTF-8")); - } - predicate = new SlicePredicate(fieldlist,null); - } - ColumnParent parent = new ColumnParent("data", null); - List results = client.get_slice(table, key, parent, predicate, ConsistencyLevel.ONE); - - if (_debug) - { - System.out.print("READ: "); - } - - for (ColumnOrSuperColumn oneresult : results) - { - Column column=oneresult.column; - result.put(new String(column.name),new ByteArrayByteIterator(column.value)); - - if (_debug) - { - System.out.print("("+new String(column.name)+"="+new String(column.value)+")"); - } - } - - if (_debug) - { - System.out.println(""); - } - - return Ok; - } - catch (Exception e) - { - errorexception=e; - } - - try - { - Thread.sleep(500); - } - catch (InterruptedException e) - { - } - } - errorexception.printStackTrace(); - errorexception.printStackTrace(System.out); - return Error; - - } - - /** - * Perform a range scan for a set of records in the database. Each field/value pair from the result will be stored in a HashMap. - * - * @param table The name of the table - * @param startkey The record key of the first record to read. - * @param recordcount The number of records to read - * @param fields The list of fields to read, or null for all of them - * @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record - * @return Zero on success, a non-zero error code on error - */ - public int scan(String table, String startkey, int recordcount, Set fields, Vector> result) - { - Exception errorexception=null; - - for (int i=0; i fieldlist=new Vector(); - for (String s : fields) - { - fieldlist.add(s.getBytes("UTF-8")); - } - predicate = new SlicePredicate(fieldlist,null); - } - ColumnParent parent = new ColumnParent("data", null); - - List results = client.get_range_slice(table,parent,predicate,startkey,"",recordcount,ConsistencyLevel.ONE); - - if (_debug) - { - System.out.println("SCAN:"); - } - - for (KeySlice oneresult : results) - { - HashMap tuple = new HashMap(); - - for (ColumnOrSuperColumn onecol : oneresult.columns) - { - Column column=onecol.column; - tuple.put(new String(column.name),new ByteArrayByteIterator(column.value)); - - if (_debug) - { - System.out.print("("+new String(column.name)+"="+new String(column.value)+")"); - } - } - - result.add(tuple); - if (_debug) - { - System.out.println(); - } - } - - return Ok; - } - catch (Exception e) - { - errorexception=e; - } - try - { - Thread.sleep(500); - } - catch (InterruptedException e) - { - } - } - errorexception.printStackTrace(); - errorexception.printStackTrace(System.out); - return Error; - } - - /** - * Update a record in the database. Any field/value pairs in the specified values HashMap will be written into the record with the specified - * record key, overwriting any existing values with the same field name. - * - * @param table The name of the table - * @param key The record key of the record to write. - * @param values A HashMap of field/value pairs to update in the record - * @return Zero on success, a non-zero error code on error - */ - public int update(String table, String key, HashMap values) - { - return insert(table,key,values); - } - - /** - * Insert a record in the database. Any field/value pairs in the specified values HashMap will be written into the record with the specified - * record key. - * - * @param table The name of the table - * @param key The record key of the record to insert. - * @param values A HashMap of field/value pairs to insert in the record - * @return Zero on success, a non-zero error code on error - */ - public int insert(String table, String key, HashMap values) - { - Exception errorexception=null; - - for (int i=0; i> batch_mutation=new HashMap>(); - Vector v=new Vector(); - batch_mutation.put("data",v); - - try - { - for (String field : values.keySet()) - { - String val=values.get(field).toString(); - Column col=new Column(field.getBytes("UTF-8"),val.getBytes("UTF-8"),timestamp); - - ColumnOrSuperColumn c=new ColumnOrSuperColumn(); - c.setColumn(col); - c.unsetSuper_column(); - v.add(c); - } - - - client.batch_insert(table, - key, - batch_mutation, - ConsistencyLevel.ONE); - - if (_debug) - { - System.out.println("INSERT"); - } - - return Ok; - } - catch (Exception e) - { - errorexception=e; - } - try - { - Thread.sleep(500); - } - catch (InterruptedException e) - { - } - } - - errorexception.printStackTrace(); - errorexception.printStackTrace(System.out); - return Error; - } - - /** - * Delete a record from the database. - * - * @param table The name of the table - * @param key The record key of the record to delete. - * @return Zero on success, a non-zero error code on error - */ - public int delete(String table, String key) - { - Exception errorexception=null; - - for (int i=0; i vals=new HashMap(); - vals.put("age",new StringByteIterator("57")); - vals.put("middlename",new StringByteIterator("bradley")); - vals.put("favoritecolor",new StringByteIterator("blue")); - int res=cli.insert("usertable","BrianFrankCooper",vals); - System.out.println("Result of insert: "+res); - - HashMap result=new HashMap(); - HashSet fields=new HashSet(); - fields.add("middlename"); - fields.add("age"); - fields.add("favoritecolor"); - res=cli.read("usertable", "BrianFrankCooper", null, result); - System.out.println("Result of read: "+res); - for (String s: result.keySet()) - { - System.out.println("["+s+"]=["+result.get(s)+"]"); - } - - res=cli.delete("usertable","BrianFrankCooper"); - System.out.println("Result of delete: "+res); - } - - - /* - public static void main(String[] args) - throws TException, InvalidRequestException, UnavailableException, UnsupportedEncodingException, NotFoundException - { - - - - String key_user_id = "1"; - - - - - client.insert("Keyspace1", - key_user_id, - new ColumnPath("Standard1", null, "age".getBytes("UTF-8")), - "24".getBytes("UTF-8"), - timestamp, - ConsistencyLevel.ONE); - - - // read single column - ColumnPath path = new ColumnPath("Standard1", null, "name".getBytes("UTF-8")); - - System.out.println(client.get("Keyspace1", key_user_id, path, ConsistencyLevel.ONE)); - - - // read entire row - SlicePredicate predicate = new SlicePredicate(null, new SliceRange(new byte[0], new byte[0], false, 10)); - - ColumnParent parent = new ColumnParent("Standard1", null); - - List results = client.get_slice("Keyspace1", key_user_id, parent, predicate, ConsistencyLevel.ONE); - - for (ColumnOrSuperColumn result : results) - { - - Column column = result.column; - - System.out.println(new String(column.name, "UTF-8") + " -> " + new String(column.value, "UTF-8")); - - } - - - - - } - */ -} diff --git a/db/cassandra-0.6/lib/README b/db/cassandra-0.6/lib/README deleted file mode 100644 index b41667f1..00000000 --- a/db/cassandra-0.6/lib/README +++ /dev/null @@ -1 +0,0 @@ -This directory should contain jars for building and running Cassandra 0.6 diff --git a/db/cassandra-0.6/src/com/yahoo/ycsb/db/CassandraClient6.java b/db/cassandra-0.6/src/com/yahoo/ycsb/db/CassandraClient6.java deleted file mode 100644 index 6d4a5725..00000000 --- a/db/cassandra-0.6/src/com/yahoo/ycsb/db/CassandraClient6.java +++ /dev/null @@ -1,515 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package com.yahoo.ycsb.db; - -import com.yahoo.ycsb.*; -import java.util.List; -import java.util.Set; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Vector; -import java.util.Random; -import java.util.Properties; - -import org.apache.thrift.transport.TTransport; -import org.apache.thrift.transport.TSocket; -import org.apache.thrift.protocol.TProtocol; -import org.apache.thrift.protocol.TBinaryProtocol; -import org.apache.cassandra.service.*; -import org.apache.cassandra.thrift.*; - - -//XXXX if we do replication, fix the consistency levels -/** - * Cassandra 0.6 client for YCSB framework - */ -public class CassandraClient6 extends DB -{ - static Random random=new Random(); - public static final int Ok=0; - public static final int Error=-1; - - public int ConnectionRetries; - public int OperationRetries; - - public static final String CONNECTION_RETRY_PROPERTY="cassandra.connectionretries"; - public static final String CONNECTION_RETRY_PROPERTY_DEFAULT="300"; - - public static final String OPERATION_RETRY_PROPERTY="cassandra.operationretries"; - public static final String OPERATION_RETRY_PROPERTY_DEFAULT="300"; - - - TTransport tr; - Cassandra.Client client; - - boolean _debug=false; - - /** - * Initialize any state for this DB. - * Called once per DB instance; there is one DB instance per client thread. - */ - public void init() throws DBException - { - String hosts=getProperties().getProperty("hosts"); - if (hosts==null) - { - throw new DBException("Required property \"hosts\" missing for CassandraClient"); - } - - ConnectionRetries=Integer.parseInt(getProperties().getProperty(CONNECTION_RETRY_PROPERTY,CONNECTION_RETRY_PROPERTY_DEFAULT)); - OperationRetries=Integer.parseInt(getProperties().getProperty(OPERATION_RETRY_PROPERTY,OPERATION_RETRY_PROPERTY_DEFAULT)); - - _debug=Boolean.parseBoolean(getProperties().getProperty("debug","false")); - - String[] allhosts=hosts.split(","); - String myhost=allhosts[random.nextInt(allhosts.length)]; - //System.out.println("My host: ["+myhost+"]"); - //System.exit(0); - - Exception connectexception=null; - - for (int retry=0; retry fields, HashMap result) - { - Exception errorexception=null; - - for (int i=0; i fieldlist=new Vector(); - for (String s : fields) - { - fieldlist.add(s.getBytes("UTF-8")); - } - - predicate = new SlicePredicate(); - predicate.setColumn_names(fieldlist); - } - - ColumnParent parent = new ColumnParent("data"); - List results = client.get_slice(table, key, parent, predicate, ConsistencyLevel.ONE); - - if (_debug) - { - System.out.print("READ: "); - } - - for (ColumnOrSuperColumn oneresult : results) - { - Column column=oneresult.column; - result.put(new String(column.name),new ByteArrayByteIterator(column.value)); - - if (_debug) - { - System.out.print("("+new String(column.name)+"="+new String(column.value)+")"); - } - } - - if (_debug) - { - System.out.println(""); - } - - return Ok; - } - catch (Exception e) - { - errorexception=e; - } - - try - { - Thread.sleep(500); - } - catch (InterruptedException e) - { - } - } - errorexception.printStackTrace(); - errorexception.printStackTrace(System.out); - return Error; - - } - - /** - * Perform a range scan for a set of records in the database. Each field/value pair from the result will be stored in a HashMap. - * - * @param table The name of the table - * @param startkey The record key of the first record to read. - * @param recordcount The number of records to read - * @param fields The list of fields to read, or null for all of them - * @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record - * @return Zero on success, a non-zero error code on error - */ - public int scan(String table, String startkey, int recordcount, Set fields, Vector> result) - { - Exception errorexception=null; - - for (int i=0; i fieldlist=new Vector(); - for (String s : fields) - { - fieldlist.add(s.getBytes("UTF-8")); - } - predicate = new SlicePredicate(); - predicate.setColumn_names(fieldlist); - } - ColumnParent parent = new ColumnParent("data"); - - List results = client.get_range_slice(table,parent,predicate,startkey,"",recordcount,ConsistencyLevel.ONE); - - if (_debug) - { - System.out.println("SCAN:"); - } - - for (KeySlice oneresult : results) - { - HashMap tuple = new HashMap(); - - for (ColumnOrSuperColumn onecol : oneresult.columns) - { - Column column=onecol.column; - tuple.put(new String(column.name),new ByteArrayByteIterator(column.value)); - - if (_debug) - { - System.out.print("("+new String(column.name)+"="+new String(column.value)+")"); - } - } - - result.add(tuple); - if (_debug) - { - System.out.println(); - } - } - - return Ok; - } - catch (Exception e) - { - errorexception=e; - } - try - { - Thread.sleep(500); - } - catch (InterruptedException e) - { - } - } - errorexception.printStackTrace(); - errorexception.printStackTrace(System.out); - return Error; - } - - /** - * Update a record in the database. Any field/value pairs in the specified values HashMap will be written into the record with the specified - * record key, overwriting any existing values with the same field name. - * - * @param table The name of the table - * @param key The record key of the record to write. - * @param values A HashMap of field/value pairs to update in the record - * @return Zero on success, a non-zero error code on error - */ - public int update(String table, String key, HashMap values) - { - return insert(table,key,values); - } - - /** - * Insert a record in the database. Any field/value pairs in the specified values HashMap will be written into the record with the specified - * record key. - * - * @param table The name of the table - * @param key The record key of the record to insert. - * @param values A HashMap of field/value pairs to insert in the record - * @return Zero on success, a non-zero error code on error - */ - public int insert(String table, String key, HashMap values) - { - Exception errorexception=null; - - for (int i=0; i> batch_mutation=new HashMap>(); - Vector v=new Vector(); - batch_mutation.put("data",v); - - try - { - for (String field : values.keySet()) - { - ByteIterator val=values.get(field); - Column col=new Column(field.getBytes("UTF-8"),val.toArray(),timestamp); - - ColumnOrSuperColumn c=new ColumnOrSuperColumn(); - c.setColumn(col); - c.unsetSuper_column(); - v.add(c); - } - - - client.batch_insert(table, - key, - batch_mutation, - ConsistencyLevel.ONE); - - if (_debug) - { - System.out.println("INSERT"); - } - - return Ok; - } - catch (Exception e) - { - errorexception=e; - } - try - { - Thread.sleep(500); - } - catch (InterruptedException e) - { - } - } - - errorexception.printStackTrace(); - errorexception.printStackTrace(System.out); - return Error; - } - - /** - * Delete a record from the database. - * - * @param table The name of the table - * @param key The record key of the record to delete. - * @return Zero on success, a non-zero error code on error - */ - public int delete(String table, String key) - { - Exception errorexception=null; - - for (int i=0; i vals=new HashMap(); - vals.put("age",new StringByteIterator("57")); - vals.put("middlename",new StringByteIterator("bradley")); - vals.put("favoritecolor",new StringByteIterator("blue")); - int res=cli.insert("usertable","BrianFrankCooper",vals); - System.out.println("Result of insert: "+res); - - HashMap result=new HashMap(); - HashSet fields=new HashSet(); - fields.add("middlename"); - fields.add("age"); - fields.add("favoritecolor"); - res=cli.read("usertable", "BrianFrankCooper", null, result); - System.out.println("Result of read: "+res); - for (String s: result.keySet()) - { - System.out.println("["+s+"]=["+result.get(s)+"]"); - } - - res=cli.delete("usertable","BrianFrankCooper"); - System.out.println("Result of delete: "+res); - } - - - /* - public static void main(String[] args) - throws TException, InvalidRequestException, UnavailableException, UnsupportedEncodingException, NotFoundException - { - - - - String key_user_id = "1"; - - - - - client.insert("Keyspace1", - key_user_id, - new ColumnPath("Standard1", null, "age".getBytes("UTF-8")), - "24".getBytes("UTF-8"), - timestamp, - ConsistencyLevel.ONE); - - - // read single column - ColumnPath path = new ColumnPath("Standard1", null, "name".getBytes("UTF-8")); - - System.out.println(client.get("Keyspace1", key_user_id, path, ConsistencyLevel.ONE)); - - - // read entire row - SlicePredicate predicate = new SlicePredicate(null, new SliceRange(new byte[0], new byte[0], false, 10)); - - ColumnParent parent = new ColumnParent("Standard1", null); - - List results = client.get_slice("Keyspace1", key_user_id, parent, predicate, ConsistencyLevel.ONE); - - for (ColumnOrSuperColumn result : results) - { - - Column column = result.column; - - System.out.println(new String(column.name, "UTF-8") + " -> " + new String(column.value, "UTF-8")); - - } - - - - - } - */ -} diff --git a/db/cassandra-0.7/lib/README b/db/cassandra-0.7/lib/README deleted file mode 100644 index 1f752f6b..00000000 --- a/db/cassandra-0.7/lib/README +++ /dev/null @@ -1 +0,0 @@ -This directory should contain jars for building and running Cassandra 0.7 diff --git a/db/cassandra-0.7/src/com/yahoo/ycsb/db/CassandraClient7.java b/db/cassandra-0.7/src/com/yahoo/ycsb/db/CassandraClient7.java deleted file mode 100644 index c5a39692..00000000 --- a/db/cassandra-0.7/src/com/yahoo/ycsb/db/CassandraClient7.java +++ /dev/null @@ -1,615 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package com.yahoo.ycsb.db; - -import com.yahoo.ycsb.*; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Vector; -import java.util.Random; -import java.util.Properties; -import java.nio.ByteBuffer; - -import org.apache.thrift.transport.TTransport; -import org.apache.thrift.transport.TFramedTransport; -import org.apache.thrift.transport.TSocket; -import org.apache.thrift.protocol.TProtocol; -import org.apache.thrift.protocol.TBinaryProtocol; -import org.apache.cassandra.thrift.*; - - -//XXXX if we do replication, fix the consistency levels -/** - * Cassandra 0.7 client for YCSB framework - */ -public class CassandraClient7 extends DB -{ - static Random random = new Random(); - public static final int Ok = 0; - public static final int Error = -1; - public static final ByteBuffer emptyByteBuffer = ByteBuffer.wrap(new byte[0]); - - public int ConnectionRetries; - public int OperationRetries; - public String column_family; - - public static final String CONNECTION_RETRY_PROPERTY = "cassandra.connectionretries"; - public static final String CONNECTION_RETRY_PROPERTY_DEFAULT = "300"; - - public static final String OPERATION_RETRY_PROPERTY = "cassandra.operationretries"; - public static final String OPERATION_RETRY_PROPERTY_DEFAULT = "300"; - - public static final String USERNAME_PROPERTY = "cassandra.username"; - public static final String PASSWORD_PROPERTY = "cassandra.password"; - - public static final String COLUMN_FAMILY_PROPERTY = "cassandra.columnfamily"; - public static final String COLUMN_FAMILY_PROPERTY_DEFAULT = "data"; - - TTransport tr; - Cassandra.Client client; - - boolean _debug = false; - - String _table = ""; - Exception errorexception = null; - - List mutations = new ArrayList(); - Map> mutationMap = new HashMap>(); - Map>> record = new HashMap>>(); - - ColumnParent parent; - - /** - * Initialize any state for this DB. Called once per DB instance; there is one - * DB instance per client thread. - */ - public void init() throws DBException - { - String hosts = getProperties().getProperty("hosts"); - if (hosts == null) - { - throw new DBException("Required property \"hosts\" missing for CassandraClient"); - } - - column_family = getProperties().getProperty(COLUMN_FAMILY_PROPERTY, COLUMN_FAMILY_PROPERTY_DEFAULT); - parent = new ColumnParent(column_family); - - ConnectionRetries = Integer.parseInt(getProperties().getProperty(CONNECTION_RETRY_PROPERTY, - CONNECTION_RETRY_PROPERTY_DEFAULT)); - OperationRetries = Integer.parseInt(getProperties().getProperty(OPERATION_RETRY_PROPERTY, - OPERATION_RETRY_PROPERTY_DEFAULT)); - - String username = getProperties().getProperty(USERNAME_PROPERTY); - String password = getProperties().getProperty(PASSWORD_PROPERTY); - - _debug = Boolean.parseBoolean(getProperties().getProperty("debug", "false")); - - String[] allhosts = hosts.split(","); - String myhost = allhosts[random.nextInt(allhosts.length)]; - - Exception connectexception = null; - - for (int retry = 0; retry < ConnectionRetries; retry++) - { - tr = new TFramedTransport(new TSocket(myhost, 9160)); - TProtocol proto = new TBinaryProtocol(tr); - client = new Cassandra.Client(proto); - try - { - tr.open(); - connectexception = null; - break; - } catch (Exception e) - { - connectexception = e; - } - try - { - Thread.sleep(1000); - } catch (InterruptedException e) - { - } - } - if (connectexception != null) - { - System.err.println("Unable to connect to " + myhost + " after " + ConnectionRetries - + " tries"); - System.out.println("Unable to connect to " + myhost + " after " + ConnectionRetries - + " tries"); - throw new DBException(connectexception); - } - - if (username != null && password != null) - { - Map cred = new HashMap(); - cred.put("username", username); - cred.put("password", password); - AuthenticationRequest req = new AuthenticationRequest(cred); - try - { - client.login(req); - } - catch (Exception e) - { - throw new DBException(e); - } - } - } - - /** - * Cleanup any state for this DB. Called once per DB instance; there is one DB - * instance per client thread. - */ - public void cleanup() throws DBException - { - tr.close(); - } - - /** - * Read a record from the database. Each field/value pair from the result will - * be stored in a HashMap. - * - * @param table - * The name of the table - * @param key - * The record key of the record to read. - * @param fields - * The list of fields to read, or null for all of them - * @param result - * A HashMap of field/value pairs for the result - * @return Zero on success, a non-zero error code on error - */ - public int read(String table, String key, Set fields, HashMap result) - { - if (!_table.equals(table)) { - try - { - client.set_keyspace(table); - _table = table; - } - catch (Exception e) - { - e.printStackTrace(); - e.printStackTrace(System.out); - return Error; - } - } - - for (int i = 0; i < OperationRetries; i++) - { - - try - { - SlicePredicate predicate; - if (fields == null) - { - predicate = new SlicePredicate().setSlice_range(new SliceRange(emptyByteBuffer, emptyByteBuffer, false, 1000000)); - - } else { - ArrayList fieldlist = new ArrayList(fields.size()); - for (String s : fields) - { - fieldlist.add(ByteBuffer.wrap(s.getBytes("UTF-8"))); - } - - predicate = new SlicePredicate().setColumn_names(fieldlist); - } - - List results = client.get_slice(ByteBuffer.wrap(key.getBytes("UTF-8")), parent, predicate, ConsistencyLevel.ONE); - - if (_debug) - { - System.out.print("Reading key: " + key); - } - - Column column; - String name; - ByteIterator value; - for (ColumnOrSuperColumn oneresult : results) - { - - column = oneresult.column; - name = new String(column.name.array(), column.name.position()+column.name.arrayOffset(), column.name.remaining()); - value = new ByteArrayByteIterator(column.value.array(), column.value.position()+column.value.arrayOffset(), column.value.remaining()); - - result.put(name,value); - - if (_debug) - { - System.out.print("(" + name + "=" + value + ")"); - } - } - - if (_debug) - { - System.out.println(); - } - - return Ok; - } catch (Exception e) - { - errorexception = e; - } - - try - { - Thread.sleep(500); - } catch (InterruptedException e) - { - } - } - errorexception.printStackTrace(); - errorexception.printStackTrace(System.out); - return Error; - - } - - /** - * Perform a range scan for a set of records in the database. Each field/value - * pair from the result will be stored in a HashMap. - * - * @param table - * The name of the table - * @param startkey - * The record key of the first record to read. - * @param recordcount - * The number of records to read - * @param fields - * The list of fields to read, or null for all of them - * @param result - * A Vector of HashMaps, where each HashMap is a set field/value - * pairs for one record - * @return Zero on success, a non-zero error code on error - */ - public int scan(String table, String startkey, int recordcount, Set fields, - Vector> result) - { - if (!_table.equals(table)) { - try - { - client.set_keyspace(table); - _table = table; - } - catch (Exception e) - { - e.printStackTrace(); - e.printStackTrace(System.out); - return Error; - } - } - - for (int i = 0; i < OperationRetries; i++) - { - - try - { - SlicePredicate predicate; - if (fields == null) - { - predicate = new SlicePredicate().setSlice_range(new SliceRange(emptyByteBuffer, emptyByteBuffer, false, 1000000)); - - } else { - ArrayList fieldlist = new ArrayList(fields.size()); - for (String s : fields) - { - fieldlist.add(ByteBuffer.wrap(s.getBytes("UTF-8"))); - } - - predicate = new SlicePredicate().setColumn_names(fieldlist); - } - - KeyRange kr = new KeyRange().setStart_key(startkey.getBytes("UTF-8")).setEnd_key(new byte[] {}).setCount(recordcount); - - List results = client.get_range_slices(parent, predicate, kr, ConsistencyLevel.ONE); - - if (_debug) - { - System.out.println("Scanning startkey: " + startkey); - } - - HashMap tuple; - for (KeySlice oneresult : results) - { - tuple = new HashMap(); - - Column column; - String name; - ByteIterator value; - for (ColumnOrSuperColumn onecol : oneresult.columns) - { - column = onecol.column; - name = new String(column.name.array(), column.name.position()+column.name.arrayOffset(), column.name.remaining()); - value = new ByteArrayByteIterator(column.value.array(), column.value.position()+column.value.arrayOffset(), column.value.remaining()); - - tuple.put(name, value); - - if (_debug) - { - System.out.print("(" + name + "=" + value + ")"); - } - } - - result.add(tuple); - if (_debug) - { - System.out.println(); - } - } - - return Ok; - } catch (Exception e) - { - errorexception = e; - } - try - { - Thread.sleep(500); - } catch (InterruptedException e) - { - } - } - errorexception.printStackTrace(); - errorexception.printStackTrace(System.out); - return Error; - } - - /** - * Update a record in the database. Any field/value pairs in the specified - * values HashMap will be written into the record with the specified record - * key, overwriting any existing values with the same field name. - * - * @param table - * The name of the table - * @param key - * The record key of the record to write. - * @param values - * A HashMap of field/value pairs to update in the record - * @return Zero on success, a non-zero error code on error - */ - public int update(String table, String key, HashMap values) - { - return insert(table, key, values); - } - - /** - * Insert a record in the database. Any field/value pairs in the specified - * values HashMap will be written into the record with the specified record - * key. - * - * @param table - * The name of the table - * @param key - * The record key of the record to insert. - * @param values - * A HashMap of field/value pairs to insert in the record - * @return Zero on success, a non-zero error code on error - */ - public int insert(String table, String key, HashMap values) - { - if (!_table.equals(table)) { - try - { - client.set_keyspace(table); - _table = table; - } - catch (Exception e) - { - e.printStackTrace(); - e.printStackTrace(System.out); - return Error; - } - } - - for (int i = 0; i < OperationRetries; i++) - { - if (_debug) - { - System.out.println("Inserting key: " + key); - } - - try - { - ByteBuffer wrappedKey = ByteBuffer.wrap(key.getBytes("UTF-8")); - - ColumnOrSuperColumn column; - for (Map.Entry entry : values.entrySet()) - { - column = new ColumnOrSuperColumn(); - column.setColumn( new Column( ByteBuffer.wrap(entry.getKey().getBytes("UTF-8")), - ByteBuffer.wrap(entry.getValue().toArray()), - System.currentTimeMillis()) ); - - mutations.add(new Mutation().setColumn_or_supercolumn(column)); - } - - mutationMap.put(column_family, mutations); - record.put(wrappedKey, mutationMap); - - client.batch_mutate(record, ConsistencyLevel.ONE); - - mutations.clear(); - mutationMap.clear(); - record.clear(); - - return Ok; - } catch (Exception e) - { - errorexception = e; - } - try - { - Thread.sleep(500); - } catch (InterruptedException e) - { - } - } - - errorexception.printStackTrace(); - errorexception.printStackTrace(System.out); - return Error; - } - - /** - * Delete a record from the database. - * - * @param table - * The name of the table - * @param key - * The record key of the record to delete. - * @return Zero on success, a non-zero error code on error - */ - public int delete(String table, String key) - { - if (!_table.equals(table)) { - try - { - client.set_keyspace(table); - _table = table; - } - catch (Exception e) - { - e.printStackTrace(); - e.printStackTrace(System.out); - return Error; - } - } - - for (int i = 0; i < OperationRetries; i++) - { - try - { - client.remove(ByteBuffer.wrap(key.getBytes("UTF-8")), - new ColumnPath(column_family), - System.currentTimeMillis(), - ConsistencyLevel.ONE); - - if (_debug) - { - System.out.println("Delete key: " + key); - } - - return Ok; - } catch (Exception e) - { - errorexception = e; - } - try - { - Thread.sleep(500); - } catch (InterruptedException e) - { - } - } - errorexception.printStackTrace(); - errorexception.printStackTrace(System.out); - return Error; - } - - public static void main(String[] args) - { - CassandraClient7 cli = new CassandraClient7(); - - Properties props = new Properties(); - - props.setProperty("hosts", args[0]); - cli.setProperties(props); - - try - { - cli.init(); - } catch (Exception e) - { - e.printStackTrace(); - System.exit(0); - } - - HashMap vals = new HashMap(); - vals.put("age", new StringByteIterator("57")); - vals.put("middlename", new StringByteIterator("bradley")); - vals.put("favoritecolor", new StringByteIterator("blue")); - int res = cli.insert("usertable", "BrianFrankCooper", vals); - System.out.println("Result of insert: " + res); - - HashMap result = new HashMap(); - HashSet fields = new HashSet(); - fields.add("middlename"); - fields.add("age"); - fields.add("favoritecolor"); - res = cli.read("usertable", "BrianFrankCooper", null, result); - System.out.println("Result of read: " + res); - for (String s : result.keySet()) - { - System.out.println("[" + s + "]=[" + result.get(s) + "]"); - } - - res = cli.delete("usertable", "BrianFrankCooper"); - System.out.println("Result of delete: " + res); - } - - /* - * public static void main(String[] args) throws TException, - * InvalidRequestException, UnavailableException, - * UnsupportedEncodingException, NotFoundException { - * - * - * - * String key_user_id = "1"; - * - * - * - * - * client.insert("Keyspace1", key_user_id, new ColumnPath("Standard1", null, - * "age".getBytes("UTF-8")), "24".getBytes("UTF-8"), timestamp, - * ConsistencyLevel.ONE); - * - * - * // read single column ColumnPath path = new ColumnPath("Standard1", null, - * "name".getBytes("UTF-8")); - * - * System.out.println(client.get("Keyspace1", key_user_id, path, - * ConsistencyLevel.ONE)); - * - * - * // read entire row SlicePredicate predicate = new SlicePredicate(null, new - * SliceRange(new byte[0], new byte[0], false, 10)); - * - * ColumnParent parent = new ColumnParent("Standard1", null); - * - * List results = client.get_slice("Keyspace1", - * key_user_id, parent, predicate, ConsistencyLevel.ONE); - * - * for (ColumnOrSuperColumn result : results) { - * - * Column column = result.column; - * - * System.out.println(new String(column.name, "UTF-8") + " -> " + new - * String(column.value, "UTF-8")); - * - * } - * - * - * - * - * } - */ -} diff --git a/db/cassandra-0.8/lib/README b/db/cassandra-0.8/lib/README deleted file mode 100644 index c02f9a57..00000000 --- a/db/cassandra-0.8/lib/README +++ /dev/null @@ -1 +0,0 @@ -This directory should contain jars for building and running Cassandra 0.8 diff --git a/db/cassandra-0.8/src/com/yahoo/ycsb/db/CassandraClient8.java b/db/cassandra-0.8/src/com/yahoo/ycsb/db/CassandraClient8.java deleted file mode 100644 index ec8abf32..00000000 --- a/db/cassandra-0.8/src/com/yahoo/ycsb/db/CassandraClient8.java +++ /dev/null @@ -1,619 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package com.yahoo.ycsb.db; - -import com.yahoo.ycsb.*; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Vector; -import java.util.Random; -import java.util.Properties; -import java.nio.ByteBuffer; - -import org.apache.thrift.transport.TTransport; -import org.apache.thrift.transport.TFramedTransport; -import org.apache.thrift.transport.TSocket; -import org.apache.thrift.protocol.TProtocol; -import org.apache.thrift.protocol.TBinaryProtocol; -import org.apache.cassandra.thrift.*; - - -//XXXX if we do replication, fix the consistency levels -/** - * Cassandra 0.8 client for YCSB framework - */ -public class CassandraClient8 extends DB -{ - static Random random = new Random(); - public static final int Ok = 0; - public static final int Error = -1; - public static final ByteBuffer emptyByteBuffer = ByteBuffer.wrap(new byte[0]); - - public int ConnectionRetries; - public int OperationRetries; - public String column_family; - - public static final String CONNECTION_RETRY_PROPERTY = "cassandra.connectionretries"; - public static final String CONNECTION_RETRY_PROPERTY_DEFAULT = "300"; - - public static final String OPERATION_RETRY_PROPERTY = "cassandra.operationretries"; - public static final String OPERATION_RETRY_PROPERTY_DEFAULT = "300"; - - public static final String USERNAME_PROPERTY = "cassandra.username"; - public static final String PASSWORD_PROPERTY = "cassandra.password"; - - public static final String COLUMN_FAMILY_PROPERTY = "cassandra.columnfamily"; - public static final String COLUMN_FAMILY_PROPERTY_DEFAULT = "data"; - - TTransport tr; - Cassandra.Client client; - - boolean _debug = false; - - String _table = ""; - Exception errorexception = null; - - List mutations = new ArrayList(); - Map> mutationMap = new HashMap>(); - Map>> record = new HashMap>>(); - - ColumnParent parent; - - /** - * Initialize any state for this DB. Called once per DB instance; there is one - * DB instance per client thread. - */ - public void init() throws DBException - { - String hosts = getProperties().getProperty("hosts"); - if (hosts == null) - { - throw new DBException("Required property \"hosts\" missing for CassandraClient"); - } - - column_family = getProperties().getProperty(COLUMN_FAMILY_PROPERTY, COLUMN_FAMILY_PROPERTY_DEFAULT); - parent = new ColumnParent(column_family); - - ConnectionRetries = Integer.parseInt(getProperties().getProperty(CONNECTION_RETRY_PROPERTY, - CONNECTION_RETRY_PROPERTY_DEFAULT)); - OperationRetries = Integer.parseInt(getProperties().getProperty(OPERATION_RETRY_PROPERTY, - OPERATION_RETRY_PROPERTY_DEFAULT)); - - String username = getProperties().getProperty(USERNAME_PROPERTY); - String password = getProperties().getProperty(PASSWORD_PROPERTY); - - _debug = Boolean.parseBoolean(getProperties().getProperty("debug", "false")); - - String[] allhosts = hosts.split(","); - String myhost = allhosts[random.nextInt(allhosts.length)]; - - Exception connectexception = null; - - for (int retry = 0; retry < ConnectionRetries; retry++) - { - tr = new TFramedTransport(new TSocket(myhost, 9160)); - TProtocol proto = new TBinaryProtocol(tr); - client = new Cassandra.Client(proto); - try - { - tr.open(); - connectexception = null; - break; - } catch (Exception e) - { - connectexception = e; - } - try - { - Thread.sleep(1000); - } catch (InterruptedException e) - { - } - } - if (connectexception != null) - { - System.err.println("Unable to connect to " + myhost + " after " + ConnectionRetries - + " tries"); - System.out.println("Unable to connect to " + myhost + " after " + ConnectionRetries - + " tries"); - throw new DBException(connectexception); - } - - if (username != null && password != null) - { - Map cred = new HashMap(); - cred.put("username", username); - cred.put("password", password); - AuthenticationRequest req = new AuthenticationRequest(cred); - try - { - client.login(req); - } - catch (Exception e) - { - throw new DBException(e); - } - } - } - - /** - * Cleanup any state for this DB. Called once per DB instance; there is one DB - * instance per client thread. - */ - public void cleanup() throws DBException - { - tr.close(); - } - - /** - * Read a record from the database. Each field/value pair from the result will - * be stored in a HashMap. - * - * @param table - * The name of the table - * @param key - * The record key of the record to read. - * @param fields - * The list of fields to read, or null for all of them - * @param result - * A HashMap of field/value pairs for the result - * @return Zero on success, a non-zero error code on error - */ - public int read(String table, String key, Set fields, HashMap result) - { - if (!_table.equals(table)) { - try - { - client.set_keyspace(table); - _table = table; - } - catch (Exception e) - { - e.printStackTrace(); - e.printStackTrace(System.out); - return Error; - } - } - - for (int i = 0; i < OperationRetries; i++) - { - - try - { - SlicePredicate predicate; - if (fields == null) - { - predicate = new SlicePredicate().setSlice_range(new SliceRange(emptyByteBuffer, emptyByteBuffer, false, 1000000)); - - } else { - ArrayList fieldlist = new ArrayList(fields.size()); - for (String s : fields) - { - fieldlist.add(ByteBuffer.wrap(s.getBytes("UTF-8"))); - } - - predicate = new SlicePredicate().setColumn_names(fieldlist); - } - - List results = client.get_slice(ByteBuffer.wrap(key.getBytes("UTF-8")), parent, predicate, ConsistencyLevel.ONE); - - if (_debug) - { - System.out.print("Reading key: " + key); - } - - Column column; - String name; - ByteIterator value; - for (ColumnOrSuperColumn oneresult : results) - { - - column = oneresult.column; - name = new String(column.name.array(), column.name.position()+column.name.arrayOffset(), column.name.remaining()); - value = new ByteArrayByteIterator(column.value.array(), column.value.position()+column.value.arrayOffset(), column.value.remaining()); - - result.put(name,value); - - if (_debug) - { - System.out.print("(" + name + "=" + value + ")"); - } - } - - if (_debug) - { - System.out.println(); - } - - return Ok; - } catch (Exception e) - { - errorexception = e; - } - - try - { - Thread.sleep(500); - } catch (InterruptedException e) - { - } - } - errorexception.printStackTrace(); - errorexception.printStackTrace(System.out); - return Error; - - } - - /** - * Perform a range scan for a set of records in the database. Each field/value - * pair from the result will be stored in a HashMap. - * - * @param table - * The name of the table - * @param startkey - * The record key of the first record to read. - * @param recordcount - * The number of records to read - * @param fields - * The list of fields to read, or null for all of them - * @param result - * A Vector of HashMaps, where each HashMap is a set field/value - * pairs for one record - * @return Zero on success, a non-zero error code on error - */ - public int scan(String table, String startkey, int recordcount, Set fields, - Vector> result) - { - if (!_table.equals(table)) { - try - { - client.set_keyspace(table); - _table = table; - } - catch (Exception e) - { - e.printStackTrace(); - e.printStackTrace(System.out); - return Error; - } - } - - for (int i = 0; i < OperationRetries; i++) - { - - try - { - SlicePredicate predicate; - if (fields == null) - { - predicate = new SlicePredicate().setSlice_range(new SliceRange(emptyByteBuffer, emptyByteBuffer, false, 1000000)); - - } else { - ArrayList fieldlist = new ArrayList(fields.size()); - for (String s : fields) - { - fieldlist.add(ByteBuffer.wrap(s.getBytes("UTF-8"))); - } - - predicate = new SlicePredicate().setColumn_names(fieldlist); - } - - KeyRange kr = new KeyRange().setStart_key(startkey.getBytes("UTF-8")).setEnd_key(new byte[] {}).setCount(recordcount); - - List results = client.get_range_slices(parent, predicate, kr, ConsistencyLevel.ONE); - - if (_debug) - { - System.out.println("Scanning startkey: " + startkey); - } - - HashMap tuple; - for (KeySlice oneresult : results) - { - tuple = new HashMap(); - - Column column; - String name; - ByteIterator value; - for (ColumnOrSuperColumn onecol : oneresult.columns) - { - column = onecol.column; - name = new String(column.name.array(), column.name.position()+column.name.arrayOffset(), column.name.remaining()); - value = new ByteArrayByteIterator(column.value.array(), column.value.position()+column.value.arrayOffset(), column.value.remaining()); - - tuple.put(name, value); - - if (_debug) - { - System.out.print("(" + name + "=" + value + ")"); - } - } - - result.add(tuple); - if (_debug) - { - System.out.println(); - } - } - - return Ok; - } catch (Exception e) - { - errorexception = e; - } - try - { - Thread.sleep(500); - } catch (InterruptedException e) - { - } - } - errorexception.printStackTrace(); - errorexception.printStackTrace(System.out); - return Error; - } - - /** - * Update a record in the database. Any field/value pairs in the specified - * values HashMap will be written into the record with the specified record - * key, overwriting any existing values with the same field name. - * - * @param table - * The name of the table - * @param key - * The record key of the record to write. - * @param values - * A HashMap of field/value pairs to update in the record - * @return Zero on success, a non-zero error code on error - */ - public int update(String table, String key, HashMap values) - { - return insert(table, key, values); - } - - /** - * Insert a record in the database. Any field/value pairs in the specified - * values HashMap will be written into the record with the specified record - * key. - * - * @param table - * The name of the table - * @param key - * The record key of the record to insert. - * @param values - * A HashMap of field/value pairs to insert in the record - * @return Zero on success, a non-zero error code on error - */ - public int insert(String table, String key, HashMap values) - { - if (!_table.equals(table)) { - try - { - client.set_keyspace(table); - _table = table; - } - catch (Exception e) - { - e.printStackTrace(); - e.printStackTrace(System.out); - return Error; - } - } - - for (int i = 0; i < OperationRetries; i++) - { - if (_debug) - { - System.out.println("Inserting key: " + key); - } - - try - { - ByteBuffer wrappedKey = ByteBuffer.wrap(key.getBytes("UTF-8")); - - Column col; - ColumnOrSuperColumn column; - for (Map.Entry entry : values.entrySet()) - { - col = new Column(); - col.setName(ByteBuffer.wrap(entry.getKey().getBytes("UTF-8"))); - col.setValue(ByteBuffer.wrap(entry.getValue().toArray())); - col.setTimestamp(System.currentTimeMillis()); - - column = new ColumnOrSuperColumn(); - column.setColumn(col); - - mutations.add(new Mutation().setColumn_or_supercolumn(column)); - } - - mutationMap.put(column_family, mutations); - record.put(wrappedKey, mutationMap); - - client.batch_mutate(record, ConsistencyLevel.ONE); - - mutations.clear(); - mutationMap.clear(); - record.clear(); - - return Ok; - } catch (Exception e) - { - errorexception = e; - } - try - { - Thread.sleep(500); - } catch (InterruptedException e) - { - } - } - - errorexception.printStackTrace(); - errorexception.printStackTrace(System.out); - return Error; - } - - /** - * Delete a record from the database. - * - * @param table - * The name of the table - * @param key - * The record key of the record to delete. - * @return Zero on success, a non-zero error code on error - */ - public int delete(String table, String key) - { - if (!_table.equals(table)) { - try - { - client.set_keyspace(table); - _table = table; - } - catch (Exception e) - { - e.printStackTrace(); - e.printStackTrace(System.out); - return Error; - } - } - - for (int i = 0; i < OperationRetries; i++) - { - try - { - client.remove(ByteBuffer.wrap(key.getBytes("UTF-8")), - new ColumnPath(column_family), - System.currentTimeMillis(), - ConsistencyLevel.ONE); - - if (_debug) - { - System.out.println("Delete key: " + key); - } - - return Ok; - } catch (Exception e) - { - errorexception = e; - } - try - { - Thread.sleep(500); - } catch (InterruptedException e) - { - } - } - errorexception.printStackTrace(); - errorexception.printStackTrace(System.out); - return Error; - } - - public static void main(String[] args) - { - CassandraClient8 cli = new CassandraClient8(); - - Properties props = new Properties(); - - props.setProperty("hosts", args[0]); - cli.setProperties(props); - - try - { - cli.init(); - } catch (Exception e) - { - e.printStackTrace(); - System.exit(0); - } - - HashMap vals = new HashMap(); - vals.put("age", new StringByteIterator("57")); - vals.put("middlename", new StringByteIterator("bradley")); - vals.put("favoritecolor", new StringByteIterator("blue")); - int res = cli.insert("usertable", "BrianFrankCooper", vals); - System.out.println("Result of insert: " + res); - - HashMap result = new HashMap(); - HashSet fields = new HashSet(); - fields.add("middlename"); - fields.add("age"); - fields.add("favoritecolor"); - res = cli.read("usertable", "BrianFrankCooper", null, result); - System.out.println("Result of read: " + res); - for (String s : result.keySet()) - { - System.out.println("[" + s + "]=[" + result.get(s) + "]"); - } - - res = cli.delete("usertable", "BrianFrankCooper"); - System.out.println("Result of delete: " + res); - } - - /* - * public static void main(String[] args) throws TException, - * InvalidRequestException, UnavailableException, - * UnsupportedEncodingException, NotFoundException { - * - * - * - * String key_user_id = "1"; - * - * - * - * - * client.insert("Keyspace1", key_user_id, new ColumnPath("Standard1", null, - * "age".getBytes("UTF-8")), "24".getBytes("UTF-8"), timestamp, - * ConsistencyLevel.ONE); - * - * - * // read single column ColumnPath path = new ColumnPath("Standard1", null, - * "name".getBytes("UTF-8")); - * - * System.out.println(client.get("Keyspace1", key_user_id, path, - * ConsistencyLevel.ONE)); - * - * - * // read entire row SlicePredicate predicate = new SlicePredicate(null, new - * SliceRange(new byte[0], new byte[0], false, 10)); - * - * ColumnParent parent = new ColumnParent("Standard1", null); - * - * List results = client.get_slice("Keyspace1", - * key_user_id, parent, predicate, ConsistencyLevel.ONE); - * - * for (ColumnOrSuperColumn result : results) { - * - * Column column = result.column; - * - * System.out.println(new String(column.name, "UTF-8") + " -> " + new - * String(column.value, "UTF-8")); - * - * } - * - * - * - * - * } - */ -} diff --git a/db/cassandra-1.0.6/src/com/yahoo/ycsb/db/CassandraClient10.java b/db/cassandra-1.0.6/src/com/yahoo/ycsb/db/CassandraClient10.java deleted file mode 100644 index a374f82c..00000000 --- a/db/cassandra-1.0.6/src/com/yahoo/ycsb/db/CassandraClient10.java +++ /dev/null @@ -1,619 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package com.yahoo.ycsb.db; - -import com.yahoo.ycsb.*; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Vector; -import java.util.Random; -import java.util.Properties; -import java.nio.ByteBuffer; - -import org.apache.thrift.transport.TTransport; -import org.apache.thrift.transport.TFramedTransport; -import org.apache.thrift.transport.TSocket; -import org.apache.thrift.protocol.TProtocol; -import org.apache.thrift.protocol.TBinaryProtocol; -import org.apache.cassandra.thrift.*; - - -//XXXX if we do replication, fix the consistency levels -/** - * Cassandra 1.0.6 client for YCSB framework - */ -public class CassandraClient10 extends DB -{ - static Random random = new Random(); - public static final int Ok = 0; - public static final int Error = -1; - public static final ByteBuffer emptyByteBuffer = ByteBuffer.wrap(new byte[0]); - - public int ConnectionRetries; - public int OperationRetries; - public String column_family; - - public static final String CONNECTION_RETRY_PROPERTY = "cassandra.connectionretries"; - public static final String CONNECTION_RETRY_PROPERTY_DEFAULT = "300"; - - public static final String OPERATION_RETRY_PROPERTY = "cassandra.operationretries"; - public static final String OPERATION_RETRY_PROPERTY_DEFAULT = "300"; - - public static final String USERNAME_PROPERTY = "cassandra.username"; - public static final String PASSWORD_PROPERTY = "cassandra.password"; - - public static final String COLUMN_FAMILY_PROPERTY = "cassandra.columnfamily"; - public static final String COLUMN_FAMILY_PROPERTY_DEFAULT = "data"; - - TTransport tr; - Cassandra.Client client; - - boolean _debug = false; - - String _table = ""; - Exception errorexception = null; - - List mutations = new ArrayList(); - Map> mutationMap = new HashMap>(); - Map>> record = new HashMap>>(); - - ColumnParent parent; - - /** - * Initialize any state for this DB. Called once per DB instance; there is one - * DB instance per client thread. - */ - public void init() throws DBException - { - String hosts = getProperties().getProperty("hosts"); - if (hosts == null) - { - throw new DBException("Required property \"hosts\" missing for CassandraClient"); - } - - column_family = getProperties().getProperty(COLUMN_FAMILY_PROPERTY, COLUMN_FAMILY_PROPERTY_DEFAULT); - parent = new ColumnParent(column_family); - - ConnectionRetries = Integer.parseInt(getProperties().getProperty(CONNECTION_RETRY_PROPERTY, - CONNECTION_RETRY_PROPERTY_DEFAULT)); - OperationRetries = Integer.parseInt(getProperties().getProperty(OPERATION_RETRY_PROPERTY, - OPERATION_RETRY_PROPERTY_DEFAULT)); - - String username = getProperties().getProperty(USERNAME_PROPERTY); - String password = getProperties().getProperty(PASSWORD_PROPERTY); - - _debug = Boolean.parseBoolean(getProperties().getProperty("debug", "false")); - - String[] allhosts = hosts.split(","); - String myhost = allhosts[random.nextInt(allhosts.length)]; - - Exception connectexception = null; - - for (int retry = 0; retry < ConnectionRetries; retry++) - { - tr = new TFramedTransport(new TSocket(myhost, 9160)); - TProtocol proto = new TBinaryProtocol(tr); - client = new Cassandra.Client(proto); - try - { - tr.open(); - connectexception = null; - break; - } catch (Exception e) - { - connectexception = e; - } - try - { - Thread.sleep(1000); - } catch (InterruptedException e) - { - } - } - if (connectexception != null) - { - System.err.println("Unable to connect to " + myhost + " after " + ConnectionRetries - + " tries"); - System.out.println("Unable to connect to " + myhost + " after " + ConnectionRetries - + " tries"); - throw new DBException(connectexception); - } - - if (username != null && password != null) - { - Map cred = new HashMap(); - cred.put("username", username); - cred.put("password", password); - AuthenticationRequest req = new AuthenticationRequest(cred); - try - { - client.login(req); - } - catch (Exception e) - { - throw new DBException(e); - } - } - } - - /** - * Cleanup any state for this DB. Called once per DB instance; there is one DB - * instance per client thread. - */ - public void cleanup() throws DBException - { - tr.close(); - } - - /** - * Read a record from the database. Each field/value pair from the result will - * be stored in a HashMap. - * - * @param table - * The name of the table - * @param key - * The record key of the record to read. - * @param fields - * The list of fields to read, or null for all of them - * @param result - * A HashMap of field/value pairs for the result - * @return Zero on success, a non-zero error code on error - */ - public int read(String table, String key, Set fields, HashMap result) - { - if (!_table.equals(table)) { - try - { - client.set_keyspace(table); - _table = table; - } - catch (Exception e) - { - e.printStackTrace(); - e.printStackTrace(System.out); - return Error; - } - } - - for (int i = 0; i < OperationRetries; i++) - { - - try - { - SlicePredicate predicate; - if (fields == null) - { - predicate = new SlicePredicate().setSlice_range(new SliceRange(emptyByteBuffer, emptyByteBuffer, false, 1000000)); - - } else { - ArrayList fieldlist = new ArrayList(fields.size()); - for (String s : fields) - { - fieldlist.add(ByteBuffer.wrap(s.getBytes("UTF-8"))); - } - - predicate = new SlicePredicate().setColumn_names(fieldlist); - } - - List results = client.get_slice(ByteBuffer.wrap(key.getBytes("UTF-8")), parent, predicate, ConsistencyLevel.ONE); - - if (_debug) - { - System.out.print("Reading key: " + key); - } - - Column column; - String name; - ByteIterator value; - for (ColumnOrSuperColumn oneresult : results) - { - - column = oneresult.column; - name = new String(column.name.array(), column.name.position()+column.name.arrayOffset(), column.name.remaining()); - value = new ByteArrayByteIterator(column.value.array(), column.value.position()+column.value.arrayOffset(), column.value.remaining()); - - result.put(name,value); - - if (_debug) - { - System.out.print("(" + name + "=" + value + ")"); - } - } - - if (_debug) - { - System.out.println(); - } - - return Ok; - } catch (Exception e) - { - errorexception = e; - } - - try - { - Thread.sleep(500); - } catch (InterruptedException e) - { - } - } - errorexception.printStackTrace(); - errorexception.printStackTrace(System.out); - return Error; - - } - - /** - * Perform a range scan for a set of records in the database. Each field/value - * pair from the result will be stored in a HashMap. - * - * @param table - * The name of the table - * @param startkey - * The record key of the first record to read. - * @param recordcount - * The number of records to read - * @param fields - * The list of fields to read, or null for all of them - * @param result - * A Vector of HashMaps, where each HashMap is a set field/value - * pairs for one record - * @return Zero on success, a non-zero error code on error - */ - public int scan(String table, String startkey, int recordcount, Set fields, - Vector> result) - { - if (!_table.equals(table)) { - try - { - client.set_keyspace(table); - _table = table; - } - catch (Exception e) - { - e.printStackTrace(); - e.printStackTrace(System.out); - return Error; - } - } - - for (int i = 0; i < OperationRetries; i++) - { - - try - { - SlicePredicate predicate; - if (fields == null) - { - predicate = new SlicePredicate().setSlice_range(new SliceRange(emptyByteBuffer, emptyByteBuffer, false, 1000000)); - - } else { - ArrayList fieldlist = new ArrayList(fields.size()); - for (String s : fields) - { - fieldlist.add(ByteBuffer.wrap(s.getBytes("UTF-8"))); - } - - predicate = new SlicePredicate().setColumn_names(fieldlist); - } - - KeyRange kr = new KeyRange().setStart_key(startkey.getBytes("UTF-8")).setEnd_key(new byte[] {}).setCount(recordcount); - - List results = client.get_range_slices(parent, predicate, kr, ConsistencyLevel.ONE); - - if (_debug) - { - System.out.println("Scanning startkey: " + startkey); - } - - HashMap tuple; - for (KeySlice oneresult : results) - { - tuple = new HashMap(); - - Column column; - String name; - ByteIterator value; - for (ColumnOrSuperColumn onecol : oneresult.columns) - { - column = onecol.column; - name = new String(column.name.array(), column.name.position()+column.name.arrayOffset(), column.name.remaining()); - value = new ByteArrayByteIterator(column.value.array(), column.value.position()+column.value.arrayOffset(), column.value.remaining()); - - tuple.put(name, value); - - if (_debug) - { - System.out.print("(" + name + "=" + value + ")"); - } - } - - result.add(tuple); - if (_debug) - { - System.out.println(); - } - } - - return Ok; - } catch (Exception e) - { - errorexception = e; - } - try - { - Thread.sleep(500); - } catch (InterruptedException e) - { - } - } - errorexception.printStackTrace(); - errorexception.printStackTrace(System.out); - return Error; - } - - /** - * Update a record in the database. Any field/value pairs in the specified - * values HashMap will be written into the record with the specified record - * key, overwriting any existing values with the same field name. - * - * @param table - * The name of the table - * @param key - * The record key of the record to write. - * @param values - * A HashMap of field/value pairs to update in the record - * @return Zero on success, a non-zero error code on error - */ - public int update(String table, String key, HashMap values) - { - return insert(table, key, values); - } - - /** - * Insert a record in the database. Any field/value pairs in the specified - * values HashMap will be written into the record with the specified record - * key. - * - * @param table - * The name of the table - * @param key - * The record key of the record to insert. - * @param values - * A HashMap of field/value pairs to insert in the record - * @return Zero on success, a non-zero error code on error - */ - public int insert(String table, String key, HashMap values) - { - if (!_table.equals(table)) { - try - { - client.set_keyspace(table); - _table = table; - } - catch (Exception e) - { - e.printStackTrace(); - e.printStackTrace(System.out); - return Error; - } - } - - for (int i = 0; i < OperationRetries; i++) - { - if (_debug) - { - System.out.println("Inserting key: " + key); - } - - try - { - ByteBuffer wrappedKey = ByteBuffer.wrap(key.getBytes("UTF-8")); - - Column col; - ColumnOrSuperColumn column; - for (Map.Entry entry : values.entrySet()) - { - col = new Column(); - col.setName(ByteBuffer.wrap(entry.getKey().getBytes("UTF-8"))); - col.setValue(ByteBuffer.wrap(entry.getValue().toArray())); - col.setTimestamp(System.currentTimeMillis()); - - column = new ColumnOrSuperColumn(); - column.setColumn(col); - - mutations.add(new Mutation().setColumn_or_supercolumn(column)); - } - - mutationMap.put(column_family, mutations); - record.put(wrappedKey, mutationMap); - - client.batch_mutate(record, ConsistencyLevel.ONE); - - mutations.clear(); - mutationMap.clear(); - record.clear(); - - return Ok; - } catch (Exception e) - { - errorexception = e; - } - try - { - Thread.sleep(500); - } catch (InterruptedException e) - { - } - } - - errorexception.printStackTrace(); - errorexception.printStackTrace(System.out); - return Error; - } - - /** - * Delete a record from the database. - * - * @param table - * The name of the table - * @param key - * The record key of the record to delete. - * @return Zero on success, a non-zero error code on error - */ - public int delete(String table, String key) - { - if (!_table.equals(table)) { - try - { - client.set_keyspace(table); - _table = table; - } - catch (Exception e) - { - e.printStackTrace(); - e.printStackTrace(System.out); - return Error; - } - } - - for (int i = 0; i < OperationRetries; i++) - { - try - { - client.remove(ByteBuffer.wrap(key.getBytes("UTF-8")), - new ColumnPath(column_family), - System.currentTimeMillis(), - ConsistencyLevel.ONE); - - if (_debug) - { - System.out.println("Delete key: " + key); - } - - return Ok; - } catch (Exception e) - { - errorexception = e; - } - try - { - Thread.sleep(500); - } catch (InterruptedException e) - { - } - } - errorexception.printStackTrace(); - errorexception.printStackTrace(System.out); - return Error; - } - - public static void main(String[] args) - { - CassandraClient10 cli = new CassandraClient10(); - - Properties props = new Properties(); - - props.setProperty("hosts", args[0]); - cli.setProperties(props); - - try - { - cli.init(); - } catch (Exception e) - { - e.printStackTrace(); - System.exit(0); - } - - HashMap vals = new HashMap(); - vals.put("age", new StringByteIterator("57")); - vals.put("middlename", new StringByteIterator("bradley")); - vals.put("favoritecolor", new StringByteIterator("blue")); - int res = cli.insert("usertable", "BrianFrankCooper", vals); - System.out.println("Result of insert: " + res); - - HashMap result = new HashMap(); - HashSet fields = new HashSet(); - fields.add("middlename"); - fields.add("age"); - fields.add("favoritecolor"); - res = cli.read("usertable", "BrianFrankCooper", null, result); - System.out.println("Result of read: " + res); - for (String s : result.keySet()) - { - System.out.println("[" + s + "]=[" + result.get(s) + "]"); - } - - res = cli.delete("usertable", "BrianFrankCooper"); - System.out.println("Result of delete: " + res); - } - - /* - * public static void main(String[] args) throws TException, - * InvalidRequestException, UnavailableException, - * UnsupportedEncodingException, NotFoundException { - * - * - * - * String key_user_id = "1"; - * - * - * - * - * client.insert("Keyspace1", key_user_id, new ColumnPath("Standard1", null, - * "age".getBytes("UTF-8")), "24".getBytes("UTF-8"), timestamp, - * ConsistencyLevel.ONE); - * - * - * // read single column ColumnPath path = new ColumnPath("Standard1", null, - * "name".getBytes("UTF-8")); - * - * System.out.println(client.get("Keyspace1", key_user_id, path, - * ConsistencyLevel.ONE)); - * - * - * // read entire row SlicePredicate predicate = new SlicePredicate(null, new - * SliceRange(new byte[0], new byte[0], false, 10)); - * - * ColumnParent parent = new ColumnParent("Standard1", null); - * - * List results = client.get_slice("Keyspace1", - * key_user_id, parent, predicate, ConsistencyLevel.ONE); - * - * for (ColumnOrSuperColumn result : results) { - * - * Column column = result.column; - * - * System.out.println(new String(column.name, "UTF-8") + " -> " + new - * String(column.value, "UTF-8")); - * - * } - * - * - * - * - * } - */ -} diff --git a/db/gemfire/lib/README.txt b/db/gemfire/lib/README.txt deleted file mode 100644 index 46697a10..00000000 --- a/db/gemfire/lib/README.txt +++ /dev/null @@ -1,25 +0,0 @@ -This directory should contain gemfire.jar for compiling GemFireClient. - -GemFireClient can be compiled using target: -$ ant dbcompile-gemfire - -Running benchmark. -1. Copy cache.xml from this dir to your GemFire install directory ($GEMFIRE_HOME) -2. start GemFire cache server - - $ cd $GEMFIRE_HOME - - $ bin/cacheserver start -J-Xms42g -J-Xmx42g -J-XX:+UseConcMarkSweepGC -J-XX:CMSInitiatingOccupancyFraction=70 -3. Add ycsb.jar and gemfire.jar to CLASSPATH. -4. run YCSB workload. - -GemFire can be run either in client-server or peer-to-peer mode. -By default com.yahoo.ycsb.db.GemFireClient connects as a client to GemFire server running on localhost on default port (40404). host name and port of a GemFire server running elsewhere can be specified by properties "gemfire.serverhost" and "gemfire.serverport" respectively. Example: -$ java com.yahoo.ycsb.Client -load -db com.yahoo.ycsb.db.GemFireClient -P workloads/workloada -p gemfire.serverhost=host2 -p gemfire.serverport=3333 - -To run com.yahoo.ycsb.db.GemFireClient as a peer to existing GemFire members, use property "gemfire.topology" like so: -$ java com.yahoo.ycsb.Client -load -db com.yahoo.ycsb.db.GemFireClient -P workloads/workloada -p gemfire.topology=p2p - -Locators can be used for member discovery, either in client-server or peer-to-peer mode. Please see GemFire docs for details. locators can be specified like so: -$ java com.yahoo.ycsb.Client -load -db com.yahoo.ycsb.db.GemFireClient -P workloads/workloada -p gemfire.locator=host1[port1],host2[port2] - -Please refer to GemFire docs here: https://www.vmware.com/support/pubs/vfabric-gemfire.html. -Questions? visit: http://forums.gemstone.com diff --git a/db/gemfire/lib/cache.xml b/db/gemfire/lib/cache.xml deleted file mode 100644 index 90254df0..00000000 --- a/db/gemfire/lib/cache.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - diff --git a/db/gemfire/src/com/yahoo/ycsb/db/GemFireClient.java b/db/gemfire/src/com/yahoo/ycsb/db/GemFireClient.java deleted file mode 100644 index 9fe598a6..00000000 --- a/db/gemfire/src/com/yahoo/ycsb/db/GemFireClient.java +++ /dev/null @@ -1,200 +0,0 @@ -package com.yahoo.ycsb.db; - -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; -import java.util.Set; -import java.util.Vector; - -import com.gemstone.gemfire.cache.Cache; -import com.gemstone.gemfire.cache.CacheFactory; -import com.gemstone.gemfire.cache.GemFireCache; -import com.gemstone.gemfire.cache.Region; -import com.gemstone.gemfire.cache.RegionExistsException; -import com.gemstone.gemfire.cache.RegionFactory; -import com.gemstone.gemfire.cache.RegionShortcut; -import com.gemstone.gemfire.cache.client.ClientCache; -import com.gemstone.gemfire.cache.client.ClientCacheFactory; -import com.gemstone.gemfire.cache.client.ClientRegionFactory; -import com.gemstone.gemfire.cache.client.ClientRegionShortcut; -import com.gemstone.gemfire.internal.admin.remote.DistributionLocatorId; -import com.yahoo.ycsb.ByteArrayByteIterator; -import com.yahoo.ycsb.ByteIterator; -import com.yahoo.ycsb.DB; -import com.yahoo.ycsb.DBException; -import com.yahoo.ycsb.StringByteIterator; - -/** - * VMware vFabric GemFire client for the YCSB benchmark.
- *

By default acts as a GemFire client and tries to connect - * to GemFire cache server running on localhost with default - * cache server port. Hostname and port of a GemFire cacheServer - * can be provided using gemfire.serverport=port and - * gemfire.serverhost=host properties on YCSB command line. - * A locator may also be used for discovering a cacheServer - * by using the property gemfire.locator=host[port]

- * - *

To run this client in a peer-to-peer topology with other GemFire - * nodes, use the property gemfire.topology=p2p. Running - * in p2p mode will enable embedded caching in this client.

- * - *

YCSB by default does its operations against "usertable". When running - * as a client this is a ClientRegionShortcut.PROXY region, - * when running in p2p mode it is a RegionShortcut.PARTITION - * region. A cache.xml defining "usertable" region can be placed in the - * working directory to override these region definitions.

- * - * @author Swapnil Bawaskar (sbawaska at vmware) - * - */ -public class GemFireClient extends DB { - - /** Return code when operation succeeded */ - private static final int SUCCESS = 0; - - /** Return code when operation did not succeed */ - private static final int ERROR = -1; - - /** property name of the port where GemFire server is listening for connections */ - private static final String SERVERPORT_PROPERTY_NAME = "gemfire.serverport"; - - /** property name of the host where GemFire server is running */ - private static final String SERVERHOST_PROPERTY_NAME = "gemfire.serverhost"; - - /** default value of {@link #SERVERHOST_PROPERTY_NAME} */ - private static final String SERVERHOST_PROPERTY_DEFAULT = "localhost"; - - /** property name to specify a GemFire locator. This property can be used in both - * client server and p2p topology */ - private static final String LOCATOR_PROPERTY_NAME = "gemfire.locator"; - - /** property name to specify GemFire topology */ - private static final String TOPOLOGY_PROPERTY_NAME = "gemfire.topology"; - - /** value of {@value #TOPOLOGY_PROPERTY_NAME} when peer to peer topology should be used. - * (client-server topology is default) */ - private static final String TOPOLOGY_P2P_VALUE = "p2p"; - - private GemFireCache cache; - - /** - * true if ycsb client runs as a client to a - * GemFire cache server - */ - private boolean isClient; - - @Override - public void init() throws DBException { - Properties props = getProperties(); - // hostName where GemFire cacheServer is running - String serverHost = null; - // port of GemFire cacheServer - int serverPort = 0; - String locatorStr = null; - - if (props != null && !props.isEmpty()) { - String serverPortStr = props.getProperty(SERVERPORT_PROPERTY_NAME); - if (serverPortStr != null) { - serverPort = Integer.parseInt(serverPortStr); - } - serverHost = props.getProperty(SERVERHOST_PROPERTY_NAME, SERVERHOST_PROPERTY_DEFAULT); - locatorStr = props.getProperty(LOCATOR_PROPERTY_NAME); - - String topology = props.getProperty(TOPOLOGY_PROPERTY_NAME); - if (topology != null && topology.equals(TOPOLOGY_P2P_VALUE)) { - CacheFactory cf = new CacheFactory(); - if (locatorStr != null) { - cf.set("locators", locatorStr); - } - cache = cf.create(); - isClient = false; - return; - } - } - isClient = true; - DistributionLocatorId locator = null; - if (locatorStr != null) { - locator = new DistributionLocatorId(locatorStr); - } - ClientCacheFactory ccf = new ClientCacheFactory(); - if (serverPort != 0) { - ccf.addPoolServer(serverHost, serverPort); - } else if (locator != null) { - ccf.addPoolLocator(locator.getHost().getCanonicalHostName(), locator.getPort()); - } - cache = ccf.create(); - } - - @Override - public int read(String table, String key, Set fields, - HashMap result) { - Region> r = getRegion(table); - Map val = r.get(key); - if (val != null) { - if (fields == null) { - for (String k : val.keySet()) { - result.put(key, new ByteArrayByteIterator(val.get(key))); - } - } else { - for (String field : fields) { - result.put(field, new ByteArrayByteIterator(val.get(field))); - } - } - return SUCCESS; - } - return ERROR; - } - - @Override - public int scan(String table, String startkey, int recordcount, - Set fields, Vector> result) { - // GemFire does not support scan - return ERROR; - } - - @Override - public int update(String table, String key, HashMap values) { - getRegion(table).put(key, convertToBytearrayMap(values)); - return 0; - } - - @Override - public int insert(String table, String key, HashMap values) { - getRegion(table).put(key, convertToBytearrayMap(values)); - return 0; - } - - @Override - public int delete(String table, String key) { - getRegion(table).destroy(key); - return 0; - } - - private Map convertToBytearrayMap(Map values) { - Map retVal = new HashMap(); - for (String key : values.keySet()) { - retVal.put(key, values.get(key).toArray()); - } - return retVal; - } - - private Region> getRegion(String table) { - Region> r = cache.getRegion(table); - if (r == null) { - try { - if (isClient) { - ClientRegionFactory> crf = ((ClientCache) cache).createClientRegionFactory(ClientRegionShortcut.PROXY); - r = crf.create(table); - } else { - RegionFactory> rf = ((Cache)cache).createRegionFactory(RegionShortcut.PARTITION); - r = rf.create(table); - } - } catch (RegionExistsException e) { - // another thread created the region - r = cache.getRegion(table); - } - } - return r; - } - -} diff --git a/db/hbase/conf/hbase-site.xml b/db/hbase/conf/hbase-site.xml deleted file mode 100644 index 9c95a473..00000000 --- a/db/hbase/conf/hbase-site.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - hbase.rootdir - hdfs://:20001/hbase - The directory shared by region servers. - - - - hbase.master - :60000 - The host and port that the HBase master runs at. - - - - hbase.zookeeper.quorum - - - diff --git a/db/hbase/lib/README b/db/hbase/lib/README deleted file mode 100644 index dd9411b1..00000000 --- a/db/hbase/lib/README +++ /dev/null @@ -1 +0,0 @@ -This directory should contain jars for building and running HBase diff --git a/db/hbase/src/com/yahoo/ycsb/db/HBaseClient.java b/db/hbase/src/com/yahoo/ycsb/db/HBaseClient.java deleted file mode 100644 index 90b8db9a..00000000 --- a/db/hbase/src/com/yahoo/ycsb/db/HBaseClient.java +++ /dev/null @@ -1,520 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package com.yahoo.ycsb.db; - - -import com.yahoo.ycsb.DBException; -import com.yahoo.ycsb.ByteIterator; -import com.yahoo.ycsb.ByteArrayByteIterator; -import com.yahoo.ycsb.StringByteIterator; - -import java.io.IOException; -import java.util.*; -//import java.util.HashMap; -//import java.util.Properties; -//import java.util.Set; -//import java.util.Vector; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.client.HTable; -//import org.apache.hadoop.hbase.client.Scanner; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -//import org.apache.hadoop.hbase.io.Cell; -//import org.apache.hadoop.hbase.io.RowResult; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.HBaseConfiguration; - -/** - * HBase client for YCSB framework - */ -public class HBaseClient extends com.yahoo.ycsb.DB -{ - // BFC: Change to fix broken build (with HBase 0.20.6) - //private static final Configuration config = HBaseConfiguration.create(); - private static final Configuration config = HBaseConfiguration.create(); //new HBaseConfiguration(); - - public boolean _debug=false; - - public String _table=""; - public HTable _hTable=null; - public String _columnFamily=""; - public byte _columnFamilyBytes[]; - - public static final int Ok=0; - public static final int ServerError=-1; - public static final int HttpError=-2; - public static final int NoMatchingRecord=-3; - - public static final Object tableLock = new Object(); - - /** - * Initialize any state for this DB. - * Called once per DB instance; there is one DB instance per client thread. - */ - public void init() throws DBException - { - if ( (getProperties().getProperty("debug")!=null) && - (getProperties().getProperty("debug").compareTo("true")==0) ) - { - _debug=true; - } - - _columnFamily = getProperties().getProperty("columnfamily"); - if (_columnFamily == null) - { - System.err.println("Error, must specify a columnfamily for HBase table"); - throw new DBException("No columnfamily specified"); - } - _columnFamilyBytes = Bytes.toBytes(_columnFamily); - - } - - /** - * Cleanup any state for this DB. - * Called once per DB instance; there is one DB instance per client thread. - */ - public void cleanup() throws DBException - { - try { - if (_hTable != null) { - _hTable.flushCommits(); - } - } catch (IOException e) { - throw new DBException(e); - } - } - - public void getHTable(String table) throws IOException - { - synchronized (tableLock) { - _hTable = new HTable(config, table); - //2 suggestions from http://ryantwopointoh.blogspot.com/2009/01/performance-of-hbase-importing.html - _hTable.setAutoFlush(false); - _hTable.setWriteBufferSize(1024*1024*12); - //return hTable; - } - - } - - /** - * Read a record from the database. Each field/value pair from the result will be stored in a HashMap. - * - * @param table The name of the table - * @param key The record key of the record to read. - * @param fields The list of fields to read, or null for all of them - * @param result A HashMap of field/value pairs for the result - * @return Zero on success, a non-zero error code on error - */ - public int read(String table, String key, Set fields, HashMap result) - { - //if this is a "new" table, init HTable object. Else, use existing one - if (!_table.equals(table)) { - _hTable = null; - try - { - getHTable(table); - _table = table; - } - catch (IOException e) - { - System.err.println("Error accessing HBase table: "+e); - return ServerError; - } - } - - Result r = null; - try - { - if (_debug) { - System.out.println("Doing read from HBase columnfamily "+_columnFamily); - System.out.println("Doing read for key: "+key); - } - Get g = new Get(Bytes.toBytes(key)); - if (fields == null) { - g.addFamily(_columnFamilyBytes); - } else { - for (String field : fields) { - g.addColumn(_columnFamilyBytes, Bytes.toBytes(field)); - } - } - r = _hTable.get(g); - } - catch (IOException e) - { - System.err.println("Error doing get: "+e); - return ServerError; - } - catch (ConcurrentModificationException e) - { - //do nothing for now...need to understand HBase concurrency model better - return ServerError; - } - - for (KeyValue kv : r.raw()) { - result.put( - Bytes.toString(kv.getQualifier()), - new ByteArrayByteIterator(kv.getValue())); - if (_debug) { - System.out.println("Result for field: "+Bytes.toString(kv.getQualifier())+ - " is: "+Bytes.toString(kv.getValue())); - } - - } - return Ok; - } - - /** - * Perform a range scan for a set of records in the database. Each field/value pair from the result will be stored in a HashMap. - * - * @param table The name of the table - * @param startkey The record key of the first record to read. - * @param recordcount The number of records to read - * @param fields The list of fields to read, or null for all of them - * @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record - * @return Zero on success, a non-zero error code on error - */ - public int scan(String table, String startkey, int recordcount, Set fields, Vector> result) - { - //if this is a "new" table, init HTable object. Else, use existing one - if (!_table.equals(table)) { - _hTable = null; - try - { - getHTable(table); - _table = table; - } - catch (IOException e) - { - System.err.println("Error accessing HBase table: "+e); - return ServerError; - } - } - - Scan s = new Scan(Bytes.toBytes(startkey)); - //HBase has no record limit. Here, assume recordcount is small enough to bring back in one call. - //We get back recordcount records - s.setCaching(recordcount); - - //add specified fields or else all fields - if (fields == null) - { - s.addFamily(_columnFamilyBytes); - } - else - { - for (String field : fields) - { - s.addColumn(_columnFamilyBytes,Bytes.toBytes(field)); - } - } - - //get results - ResultScanner scanner = null; - try { - scanner = _hTable.getScanner(s); - int numResults = 0; - for (Result rr = scanner.next(); rr != null; rr = scanner.next()) - { - //get row key - String key = Bytes.toString(rr.getRow()); - if (_debug) - { - System.out.println("Got scan result for key: "+key); - } - - HashMap rowResult = new HashMap(); - - for (KeyValue kv : rr.raw()) { - rowResult.put( - Bytes.toString(kv.getQualifier()), - new ByteArrayByteIterator(kv.getValue())); - } - //add rowResult to result vector - result.add(rowResult); - numResults++; - if (numResults >= recordcount) //if hit recordcount, bail out - { - break; - } - } //done with row - - } - - catch (IOException e) { - if (_debug) - { - System.out.println("Error in getting/parsing scan result: "+e); - } - return ServerError; - } - - finally { - scanner.close(); - } - - return Ok; - } - - /** - * Update a record in the database. Any field/value pairs in the specified values HashMap will be written into the record with the specified - * record key, overwriting any existing values with the same field name. - * - * @param table The name of the table - * @param key The record key of the record to write - * @param values A HashMap of field/value pairs to update in the record - * @return Zero on success, a non-zero error code on error - */ - public int update(String table, String key, HashMap values) - { - //if this is a "new" table, init HTable object. Else, use existing one - if (!_table.equals(table)) { - _hTable = null; - try - { - getHTable(table); - _table = table; - } - catch (IOException e) - { - System.err.println("Error accessing HBase table: "+e); - return ServerError; - } - } - - - if (_debug) { - System.out.println("Setting up put for key: "+key); - } - Put p = new Put(Bytes.toBytes(key)); - for (Map.Entry entry : values.entrySet()) - { - if (_debug) { - System.out.println("Adding field/value " + entry.getKey() + "/"+ - entry.getValue() + " to put request"); - } - p.add(_columnFamilyBytes,Bytes.toBytes(entry.getKey()),entry.getValue().toArray()); - } - - try - { - _hTable.put(p); - } - catch (IOException e) - { - if (_debug) { - System.err.println("Error doing put: "+e); - } - return ServerError; - } - catch (ConcurrentModificationException e) - { - //do nothing for now...hope this is rare - return ServerError; - } - - return Ok; - } - - /** - * Insert a record in the database. Any field/value pairs in the specified values HashMap will be written into the record with the specified - * record key. - * - * @param table The name of the table - * @param key The record key of the record to insert. - * @param values A HashMap of field/value pairs to insert in the record - * @return Zero on success, a non-zero error code on error - */ - public int insert(String table, String key, HashMap values) - { - return update(table,key,values); - } - - /** - * Delete a record from the database. - * - * @param table The name of the table - * @param key The record key of the record to delete. - * @return Zero on success, a non-zero error code on error - */ - public int delete(String table, String key) - { - //if this is a "new" table, init HTable object. Else, use existing one - if (!_table.equals(table)) { - _hTable = null; - try - { - getHTable(table); - _table = table; - } - catch (IOException e) - { - System.err.println("Error accessing HBase table: "+e); - return ServerError; - } - } - - if (_debug) { - System.out.println("Doing delete for key: "+key); - } - - Delete d = new Delete(Bytes.toBytes(key)); - try - { - _hTable.delete(d); - } - catch (IOException e) - { - if (_debug) { - System.err.println("Error doing delete: "+e); - } - return ServerError; - } - - return Ok; - } - - public static void main(String[] args) - { - if (args.length!=3) - { - System.out.println("Please specify a threadcount, columnfamily and operation count"); - System.exit(0); - } - - final int keyspace=10000; //120000000; - - final int threadcount=Integer.parseInt(args[0]); - - final String columnfamily=args[1]; - - - final int opcount=Integer.parseInt(args[2])/threadcount; - - Vector allthreads=new Vector(); - - for (int i=0; i result=new HashMap(); - - long accum=0; - - for (int i=0; i s = new HashSet(); - s.add("field1"); - s.add("field2"); - - rescode=cli.read("table1", key, s, result); - //rescode=cli.delete("table1",key); - rescode=cli.read("table1", key, s, result); - */ - HashSet scanFields = new HashSet(); - scanFields.add("field1"); - scanFields.add("field3"); - Vector> scanResults = new Vector>(); - rescode = cli.scan("table1","user2",20,null,scanResults); - - long en=System.currentTimeMillis(); - - accum+=(en-st); - - if (rescode!=Ok) - { - System.out.println("Error "+rescode+" for "+key); - } - - if (i%1==0) - { - System.out.println(i+" operations, average latency: "+(((double)accum)/((double)i))); - } - } - - //System.out.println("Average latency: "+(((double)accum)/((double)opcount))); - //System.out.println("Average get latency: "+(((double)cli.TotalGetTime)/((double)cli.TotalGetOps))); - } - catch (Exception e) - { - e.printStackTrace(); - } - } - }; - allthreads.add(t); - } - - long st=System.currentTimeMillis(); - for (Thread t: allthreads) - { - t.start(); - } - - for (Thread t: allthreads) - { - try - { - t.join(); - } - catch (InterruptedException e) - { - } - } - long en=System.currentTimeMillis(); - - System.out.println("Throughput: "+((1000.0)*(((double)(opcount*threadcount))/((double)(en-st))))+" ops/sec"); - - } -} - -/* For customized vim control - * set autoindent - * set si - * set shiftwidth=4 -*/ - diff --git a/db/infinispan-5.0/conf/infinispan-config.xml b/db/infinispan-5.0/conf/infinispan-config.xml deleted file mode 100644 index 15b9978c..00000000 --- a/db/infinispan-5.0/conf/infinispan-config.xml +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - \ No newline at end of file diff --git a/db/infinispan-5.0/lib/README b/db/infinispan-5.0/lib/README deleted file mode 100644 index 93bd19d9..00000000 --- a/db/infinispan-5.0/lib/README +++ /dev/null @@ -1,25 +0,0 @@ -This directory should contain the Infinispan jars. - -You do this by downloading the latest Infinispan 5.x release from http://sourceforge.net/projects/infinispan/files/infinispan - -It is recommended that you download the bin.ZIP archive, such as infinispan-5.0.0.CR7-bin.zip - -Extract this archive somewhere, and copy the necessary jar files to this directory. - - $ unzip -a infinispan-5.0.0.CR7-bin.zip -d /tmp - $ cp /tmp/infinispan-5.0.0.CR7/infinispan-core.jar . - $ cp /tmp/infinispan-5.0.0.CR7/lib/* . - -The jar files you will need are: - - * infinispan-core.jar - * jboss-logging.jar - * jboss-marshalling.jar - * jboss-marshalling-river.jar - * jboss-transaction-api.jar - * jgroups.jar - - - - - diff --git a/db/infinispan-5.0/src/com/yahoo/ycsb/db/InfinispanClient.java b/db/infinispan-5.0/src/com/yahoo/ycsb/db/InfinispanClient.java deleted file mode 100644 index 528ded5c..00000000 --- a/db/infinispan-5.0/src/com/yahoo/ycsb/db/InfinispanClient.java +++ /dev/null @@ -1,136 +0,0 @@ -package com.yahoo.ycsb.db; - -import com.yahoo.ycsb.DB; -import com.yahoo.ycsb.DBException; -import com.yahoo.ycsb.ByteIterator; -import com.yahoo.ycsb.StringByteIterator; - -import org.infinispan.Cache; -import org.infinispan.atomic.AtomicMap; -import org.infinispan.atomic.AtomicMapLookup; -import org.infinispan.manager.DefaultCacheManager; -import org.infinispan.manager.EmbeddedCacheManager; -import org.infinispan.util.logging.Log; -import org.infinispan.util.logging.LogFactory; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import java.util.Vector; - -/** - * This is a client implementation for Infinispan 5.x. - * - * Some settings: - * - * @author Manik Surtani (manik AT jboss DOT org) - */ -public class InfinispanClient extends DB { - - private static final int OK = 0; - private static final int ERROR = -1; - private static final int NOT_FOUND = -2; - - // An optimisation for clustered mode - private final boolean clustered; - - private EmbeddedCacheManager infinispanManager; - - private static final Log logger = LogFactory.getLog(InfinispanClient.class); - - public InfinispanClient() { - clustered = Boolean.getBoolean("infinispan.clustered"); - } - - public void init() throws DBException { - try { - infinispanManager = new DefaultCacheManager("infinispan-config.xml"); - } catch (IOException e) { - throw new DBException(e); - } - } - - public void cleanup() { - infinispanManager.stop(); - infinispanManager = null; - } - - public int read(String table, String key, Set fields, HashMap result) { - try { - Map row; - if (clustered) { - row = AtomicMapLookup.getAtomicMap(infinispanManager.getCache(table), key, false); - } else { - Cache> cache = infinispanManager.getCache(table); - row = cache.get(key); - } - if (row != null) { - result.clear(); - if (fields == null || fields.isEmpty()) { - StringByteIterator.putAllAsByteIterators(result, row); - } else { - for (String field : fields) result.put(field, new StringByteIterator(row.get(field))); - } - } - return OK; - } catch (Exception e) { - return ERROR; - } - } - - public int scan(String table, String startkey, int recordcount, Set fields, Vector> result) { - logger.warn("Infinispan does not support scan semantics"); - return OK; - } - - public int update(String table, String key, HashMap values) { - try { - if (clustered) { - AtomicMap row = AtomicMapLookup.getAtomicMap(infinispanManager.getCache(table), key); - StringByteIterator.putAllAsStrings(row, values); - } else { - Cache> cache = infinispanManager.getCache(table); - Map row = cache.get(key); - if (row == null) { - row = StringByteIterator.getStringMap(values); - cache.put(key, row); - } else { - StringByteIterator.putAllAsStrings(row, values); - } - } - - return OK; - } catch (Exception e) { - return ERROR; - } - } - - public int insert(String table, String key, HashMap values) { - try { - if (clustered) { - AtomicMap row = AtomicMapLookup.getAtomicMap(infinispanManager.getCache(table), key); - row.clear(); - StringByteIterator.putAllAsStrings(row, values); - } else { - infinispanManager.getCache(table).put(key, values); - } - - return OK; - } catch (Exception e) { - return ERROR; - } - } - - public int delete(String table, String key) { - try { - if (clustered) - AtomicMapLookup.removeAtomicMap(infinispanManager.getCache(table), key); - else - infinispanManager.getCache(table).remove(key); - return OK; - } catch (Exception e) { - return ERROR; - } - } -} diff --git a/db/jdbc/conf/db.properties b/db/jdbc/conf/db.properties deleted file mode 100644 index c985d57b..00000000 --- a/db/jdbc/conf/db.properties +++ /dev/null @@ -1,6 +0,0 @@ -# Properties file that contains database connection information. - -jdbc.driver=org.h2.Driver -db.url=jdbc:h2:tcp://foo.com:9092/~/h2/ycsb -db.user=sa -db.passwd= diff --git a/db/jdbc/conf/h2.properties b/db/jdbc/conf/h2.properties deleted file mode 100644 index c985d57b..00000000 --- a/db/jdbc/conf/h2.properties +++ /dev/null @@ -1,6 +0,0 @@ -# Properties file that contains database connection information. - -jdbc.driver=org.h2.Driver -db.url=jdbc:h2:tcp://foo.com:9092/~/h2/ycsb -db.user=sa -db.passwd= diff --git a/db/jdbc/lib/README b/db/jdbc/lib/README deleted file mode 100755 index b57cd254..00000000 --- a/db/jdbc/lib/README +++ /dev/null @@ -1 +0,0 @@ -This directory should contain jars for building and running the JDBC database. diff --git a/db/jdbc/sql/README b/db/jdbc/sql/README deleted file mode 100644 index 3afd7047..00000000 --- a/db/jdbc/sql/README +++ /dev/null @@ -1 +0,0 @@ -Contains all the SQL statements used by the JDBC client. diff --git a/db/jdbc/sql/create_table.mysql b/db/jdbc/sql/create_table.mysql deleted file mode 100644 index 7b89e3e5..00000000 --- a/db/jdbc/sql/create_table.mysql +++ /dev/null @@ -1,12 +0,0 @@ --- Creates a Table. - --- Drop the table if it exists; -DROP TABLE IF EXISTS usertable; - --- Create the user table with 5 fields. -CREATE TABLE usertable(YCSB_KEY VARCHAR (255) PRIMARY KEY, - FIELD1 TEXT, FIELD2 TEXT, - FIELD3 TEXT, FIELD4 TEXT, - FIELD5 TEXT, FIELD6 TEXT, - FIELD7 TEXT, FIELD8 TEXT, - FIELD9 TEXT, FIELD10 TEXT); diff --git a/db/jdbc/sql/create_table.sql b/db/jdbc/sql/create_table.sql deleted file mode 100644 index c394f26f..00000000 --- a/db/jdbc/sql/create_table.sql +++ /dev/null @@ -1,12 +0,0 @@ --- Creates a Table. - --- Drop the table if it exists; -DROP TABLE IF EXISTS usertable; - --- Create the user table with 5 fields. -CREATE TABLE usertable(YCSB_KEY VARCHAR PRIMARY KEY, - FIELD1 VARCHAR, FIELD2 VARCHAR, - FIELD3 VARCHAR, FIELD4 VARCHAR, - FIELD5 VARCHAR, FIELD6 VARCHAR, - FIELD7 VARCHAR, FIELD8 VARCHAR, - FIELD9 VARCHAR, FIELD10 VARCHAR); diff --git a/db/jdbc/src/com/yahoo/ycsb/db/JdbcDBCli.java b/db/jdbc/src/com/yahoo/ycsb/db/JdbcDBCli.java deleted file mode 100644 index d736eaf8..00000000 --- a/db/jdbc/src/com/yahoo/ycsb/db/JdbcDBCli.java +++ /dev/null @@ -1,178 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ -package com.yahoo.ycsb.db; - -import java.io.FileInputStream; -import java.io.IOException; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Enumeration; -import java.util.Properties; - -/** - * Execute a JDBC command line. - * - * @author sudipto - * - */ -public class JdbcDBCli implements JdbcDBClientConstants { - - private static void usageMessage() { - System.out.println("JdbcCli. Options:"); - System.out.println(" -p key=value properties defined."); - System.out.println(" -P location of the properties file to load."); - System.out.println(" -c SQL command to execute."); - } - - private static void executeCommand(Properties props, String sql) - throws SQLException { - String driver = props.getProperty(DRIVER_CLASS); - String username = props.getProperty(CONNECTION_USER); - String password = props.getProperty(CONNECTION_PASSWD, ""); - String url = props.getProperty(CONNECTION_URL); - if (driver == null || username == null || url == null) { - throw new SQLException("Missing connection information."); - } - - Connection conn = null; - - try { - Class.forName(driver); - - conn = DriverManager.getConnection(url, username, password); - Statement stmt = conn.createStatement(); - stmt.execute(sql); - System.out.println("Command \"" + sql + "\" successfully executed."); - } catch (ClassNotFoundException e) { - throw new SQLException("JDBC Driver class not found."); - } finally { - if (conn != null) { - System.out.println("Closing database connection."); - conn.close(); - } - } - } - - /** - * @param args - */ - public static void main(String[] args) { - - if (args.length == 0) { - usageMessage(); - System.exit(0); - } - - Properties props = new Properties(); - Properties fileprops = new Properties(); - String sql = null; - - // parse arguments - int argindex = 0; - while (args[argindex].startsWith("-")) { - if (args[argindex].compareTo("-P") == 0) { - argindex++; - if (argindex >= args.length) { - usageMessage(); - System.exit(0); - } - String propfile = args[argindex]; - argindex++; - - Properties myfileprops = new Properties(); - try { - myfileprops.load(new FileInputStream(propfile)); - } catch (IOException e) { - System.out.println(e.getMessage()); - System.exit(0); - } - - // Issue #5 - remove call to stringPropertyNames to make compilable - // under Java 1.5 - for (Enumeration e = myfileprops.propertyNames(); e - .hasMoreElements();) { - String prop = (String) e.nextElement(); - - fileprops.setProperty(prop, myfileprops.getProperty(prop)); - } - - } else if (args[argindex].compareTo("-p") == 0) { - argindex++; - if (argindex >= args.length) { - usageMessage(); - System.exit(0); - } - int eq = args[argindex].indexOf('='); - if (eq < 0) { - usageMessage(); - System.exit(0); - } - - String name = args[argindex].substring(0, eq); - String value = args[argindex].substring(eq + 1); - props.put(name, value); - argindex++; - } else if (args[argindex].compareTo("-c") == 0) { - argindex++; - if (argindex >= args.length) { - usageMessage(); - System.exit(0); - } - sql = args[argindex++]; - } else { - System.out.println("Unknown option " + args[argindex]); - usageMessage(); - System.exit(0); - } - - if (argindex >= args.length) { - break; - } - } - - if (argindex != args.length) { - usageMessage(); - System.exit(0); - } - - // overwrite file properties with properties from the command line - - // Issue #5 - remove call to stringPropertyNames to make compilable under - // Java 1.5 - for (Enumeration e = props.propertyNames(); e.hasMoreElements();) { - String prop = (String) e.nextElement(); - - fileprops.setProperty(prop, props.getProperty(prop)); - } - - if (sql == null) { - System.err.println("Missing command."); - usageMessage(); - System.exit(1); - } - - try { - executeCommand(fileprops, sql); - } catch (SQLException e) { - System.err.println("Error in executing command. " + e); - System.exit(1); - } - } - -} diff --git a/db/jdbc/src/com/yahoo/ycsb/db/JdbcDBClient.java b/db/jdbc/src/com/yahoo/ycsb/db/JdbcDBClient.java deleted file mode 100644 index 8d1a6a7e..00000000 --- a/db/jdbc/src/com/yahoo/ycsb/db/JdbcDBClient.java +++ /dev/null @@ -1,449 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package com.yahoo.ycsb.db; - -import com.yahoo.ycsb.DB; -import com.yahoo.ycsb.DBException; -import com.yahoo.ycsb.ByteIterator; -import com.yahoo.ycsb.StringByteIterator; - -import java.sql.*; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; - -/** - * A class that wraps a JDBC compliant database to allow it to be interfaced with YCSB. - * This class extends {@link DB} and implements the database interface used by YCSB client. - * - *
Each client will have its own instance of this class. This client is - * not thread safe. - * - *
This interface expects a schema ... - * All attributes are of type VARCHAR. All accesses are through the primary key. Therefore, - * only one index on the primary key is needed. - * - *

The following options must be passed when using this database client. - * - *

    - *
  • db.driver The JDBC driver class to use.
    • - *
  • db.url The Database connection URL.
    • - *
  • db.user User name for the connection.
    • - *
  • db.passwd Password for the connection.
    • - *
- * - * @author sudipto - * - */ -public class JdbcDBClient extends DB implements JdbcDBClientConstants { - - private ArrayList conns; - private boolean initialized = false; - private Properties props; - private static final String DEFAULT_PROP = ""; - private ConcurrentMap cachedStatements; - - /** - * The statement type for the prepared statements. - */ - private static class StatementType { - - enum Type { - INSERT(1), - DELETE(2), - READ(3), - UPDATE(4), - SCAN(5), - ; - int internalType; - private Type(int type) { - internalType = type; - } - - int getHashCode() { - final int prime = 31; - int result = 1; - result = prime * result + internalType; - return result; - } - } - - Type type; - int shardIndex; - int numFields; - String tableName; - - StatementType(Type type, String tableName, int numFields, int _shardIndex) { - this.type = type; - this.tableName = tableName; - this.numFields = numFields; - this.shardIndex = _shardIndex; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + numFields + 100 * shardIndex; - result = prime * result - + ((tableName == null) ? 0 : tableName.hashCode()); - result = prime * result + ((type == null) ? 0 : type.getHashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - StatementType other = (StatementType) obj; - if (numFields != other.numFields) - return false; - if (shardIndex != other.shardIndex) - return false; - if (tableName == null) { - if (other.tableName != null) - return false; - } else if (!tableName.equals(other.tableName)) - return false; - if (type != other.type) - return false; - return true; - } - } - - /** - * For the given key, returns what shard contains data for this key - * - * @param key Data key to do operation on - * @return Shard index - */ - private int getShardIndexByKey(String key) { - int ret = Math.abs(key.hashCode()) % conns.size(); - //System.out.println(conns.size() + ": Shard instance for "+ key + " (hash " + key.hashCode()+ " ) " + " is " + ret); - return ret; - } - - /** - * For the given key, returns Connection object that holds connection - * to the shard that contains this key - * - * @param key Data key to get information for - * @return Connection object - */ - private Connection getShardConnectionByKey(String key) { - return conns.get(getShardIndexByKey(key)); - } - - private void cleanupAllConnections() throws SQLException { - for(Connection conn: conns) { - conn.close(); - } - } - - /** - * Initialize the database connection and set it up for sending requests to the database. - * This must be called once per client. - * @throws - */ - @Override - public void init() throws DBException { - if (initialized) { - System.err.println("Client connection already initialized."); - return; - } - props = getProperties(); - String urls = props.getProperty(CONNECTION_URL, DEFAULT_PROP); - String user = props.getProperty(CONNECTION_USER, DEFAULT_PROP); - String passwd = props.getProperty(CONNECTION_PASSWD, DEFAULT_PROP); - String driver = props.getProperty(DRIVER_CLASS); - - try { - if (driver != null) { - Class.forName(driver); - } - int shardCount = 0; - conns = new ArrayList(3); - for (String url: urls.split(",")) { - System.out.println("Adding shard node URL: " + url); - Connection conn = DriverManager.getConnection(url, user, passwd); - // Since there is no explicit commit method in the DB interface, all - // operations should auto commit. - conn.setAutoCommit(true); - shardCount++; - conns.add(conn); - } - - System.out.println("Using " + shardCount + " shards"); - - cachedStatements = new ConcurrentHashMap(); - } catch (ClassNotFoundException e) { - System.err.println("Error in initializing the JDBS driver: " + e); - throw new DBException(e); - } catch (SQLException e) { - System.err.println("Error in database operation: " + e); - throw new DBException(e); - } catch (NumberFormatException e) { - System.err.println("Invalid value for fieldcount property. " + e); - throw new DBException(e); - } - initialized = true; - } - - @Override - public void cleanup() throws DBException { - try { - cleanupAllConnections(); - } catch (SQLException e) { - System.err.println("Error in closing the connection. " + e); - throw new DBException(e); - } - } - - private PreparedStatement createAndCacheInsertStatement(StatementType insertType, String key) - throws SQLException { - StringBuilder insert = new StringBuilder("INSERT INTO "); - insert.append(insertType.tableName); - insert.append(" VALUES(?"); - for (int i = 0; i < insertType.numFields; i++) { - insert.append(",?"); - } - insert.append(");"); - PreparedStatement insertStatement = getShardConnectionByKey(key).prepareStatement(insert.toString()); - PreparedStatement stmt = cachedStatements.putIfAbsent(insertType, insertStatement); - if (stmt == null) return insertStatement; - else return stmt; - } - - private PreparedStatement createAndCacheReadStatement(StatementType readType, String key) - throws SQLException { - StringBuilder read = new StringBuilder("SELECT * FROM "); - read.append(readType.tableName); - read.append(" WHERE "); - read.append(PRIMARY_KEY); - read.append(" = "); - read.append("?;"); - PreparedStatement readStatement = getShardConnectionByKey(key).prepareStatement(read.toString()); - PreparedStatement stmt = cachedStatements.putIfAbsent(readType, readStatement); - if (stmt == null) return readStatement; - else return stmt; - } - - private PreparedStatement createAndCacheDeleteStatement(StatementType deleteType, String key) - throws SQLException { - StringBuilder delete = new StringBuilder("DELETE FROM "); - delete.append(deleteType.tableName); - delete.append(" WHERE "); - delete.append(PRIMARY_KEY); - delete.append(" = ?;"); - PreparedStatement deleteStatement = getShardConnectionByKey(key).prepareStatement(delete.toString()); - PreparedStatement stmt = cachedStatements.putIfAbsent(deleteType, deleteStatement); - if (stmt == null) return deleteStatement; - else return stmt; - } - - private PreparedStatement createAndCacheUpdateStatement(StatementType updateType, String key) - throws SQLException { - StringBuilder update = new StringBuilder("UPDATE "); - update.append(updateType.tableName); - update.append(" SET "); - for (int i = 1; i <= updateType.numFields; i++) { - update.append(COLUMN_PREFIX); - update.append(i); - update.append("=?"); - if (i < updateType.numFields) update.append(", "); - } - update.append(" WHERE "); - update.append(PRIMARY_KEY); - update.append(" = ?;"); - PreparedStatement insertStatement = getShardConnectionByKey(key).prepareStatement(update.toString()); - PreparedStatement stmt = cachedStatements.putIfAbsent(updateType, insertStatement); - if (stmt == null) return insertStatement; - else return stmt; - } - - private PreparedStatement createAndCacheScanStatement(StatementType scanType, String key) - throws SQLException { - StringBuilder select = new StringBuilder("SELECT * FROM "); - select.append(scanType.tableName); - select.append(" WHERE "); - select.append(PRIMARY_KEY); - select.append(" >= "); - select.append("?;"); - PreparedStatement scanStatement = getShardConnectionByKey(key).prepareStatement(select.toString()); - PreparedStatement stmt = cachedStatements.putIfAbsent(scanType, scanStatement); - if (stmt == null) return scanStatement; - else return stmt; - } - - @Override - public int read(String tableName, String key, Set fields, - HashMap result) { - if (tableName == null) { - return -1; - } - if (key == null) { - return -1; - } - try { - StatementType type = new StatementType(StatementType.Type.READ, tableName, 1, getShardIndexByKey(key)); - PreparedStatement readStatement = cachedStatements.get(type); - if (readStatement == null) { - readStatement = createAndCacheReadStatement(type, key); - } - readStatement.setString(1, key); - ResultSet resultSet = readStatement.executeQuery(); - if (!resultSet.next()) { - resultSet.close(); - return 1; - } - if (result != null && fields != null) { - for (String field : fields) { - String value = resultSet.getString(field); - result.put(field, new StringByteIterator(value)); - } - } - resultSet.close(); - return SUCCESS; - } catch (SQLException e) { - System.err.println("Error in processing read of table " + tableName + ": "+e); - return -2; - } - } - - @Override - public int scan(String tableName, String startKey, int recordcount, - Set fields, Vector> result) { - if (tableName == null) { - return -1; - } - if (startKey == null) { - return -1; - } - try { - StatementType type = new StatementType(StatementType.Type.SCAN, tableName, 1, getShardIndexByKey(startKey)); - PreparedStatement scanStatement = cachedStatements.get(type); - if (scanStatement == null) { - scanStatement = createAndCacheScanStatement(type, startKey); - } - scanStatement.setString(1, startKey); - ResultSet resultSet = scanStatement.executeQuery(); - for (int i = 0; i < recordcount && resultSet.next(); i++) { - if (result != null && fields != null) { - HashMap values = new HashMap(); - for (String field : fields) { - String value = resultSet.getString(field); - values.put(field, new StringByteIterator(value)); - } - result.add(values); - } - } - resultSet.close(); - return SUCCESS; - } catch (SQLException e) { - System.err.println("Error in processing scan of table: " + tableName + e); - return -2; - } - } - - @Override - public int update(String tableName, String key, HashMap values) { - if (tableName == null) { - return -1; - } - if (key == null) { - return -1; - } - try { - int numFields = values.size(); - StatementType type = new StatementType(StatementType.Type.UPDATE, tableName, numFields, getShardIndexByKey(key)); - PreparedStatement updateStatement = cachedStatements.get(type); - if (updateStatement == null) { - updateStatement = createAndCacheUpdateStatement(type, key); - } - int index = 1; - for (Map.Entry entry : values.entrySet()) { - updateStatement.setString(index++, entry.getValue().toString()); - } - updateStatement.setString(index, key); - int result = updateStatement.executeUpdate(); - if (result == 1) return SUCCESS; - else return 1; - } catch (SQLException e) { - System.err.println("Error in processing update to table: " + tableName + e); - return -1; - } - } - - @Override - public int insert(String tableName, String key, HashMap values) { - if (tableName == null) { - return -1; - } - if (key == null) { - return -1; - } - try { - int numFields = values.size(); - StatementType type = new StatementType(StatementType.Type.INSERT, tableName, numFields, getShardIndexByKey(key)); - PreparedStatement insertStatement = cachedStatements.get(type); - if (insertStatement == null) { - insertStatement = createAndCacheInsertStatement(type, key); - } - insertStatement.setString(1, key); - int index = 2; - for (Map.Entry entry : values.entrySet()) { - String field = entry.getValue().toString(); - insertStatement.setString(index++, field); - } - int result = insertStatement.executeUpdate(); - if (result == 1) return SUCCESS; - else return 1; - } catch (SQLException e) { - System.err.println("Error in processing insert to table: " + tableName + e); - return -1; - } - } - - @Override - public int delete(String tableName, String key) { - if (tableName == null) { - return -1; - } - if (key == null) { - return -1; - } - try { - StatementType type = new StatementType(StatementType.Type.DELETE, tableName, 1, getShardIndexByKey(key)); - PreparedStatement deleteStatement = cachedStatements.get(type); - if (deleteStatement == null) { - deleteStatement = createAndCacheDeleteStatement(type, key); - } - deleteStatement.setString(1, key); - int result = deleteStatement.executeUpdate(); - if (result == 1) return SUCCESS; - else return 1; - } catch (SQLException e) { - System.err.println("Error in processing delete to table: " + tableName + e); - return -1; - } - } -} diff --git a/db/jdbc/src/com/yahoo/ycsb/db/JdbcDBClientConstants.java b/db/jdbc/src/com/yahoo/ycsb/db/JdbcDBClientConstants.java deleted file mode 100644 index 79e0525e..00000000 --- a/db/jdbc/src/com/yahoo/ycsb/db/JdbcDBClientConstants.java +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ -package com.yahoo.ycsb.db; - -/** - * Constants used by the JDBC client. - * - * @author sudipto - * - */ -public interface JdbcDBClientConstants { - - /** The class to use as the jdbc driver. */ - public static final String DRIVER_CLASS = "db.driver"; - - /** The URL to connect to the database. */ - public static final String CONNECTION_URL = "db.url"; - - /** The user name to use to connect to the database. */ - public static final String CONNECTION_USER = "db.user"; - - /** The password to use for establishing the connection. */ - public static final String CONNECTION_PASSWD = "db.passwd"; - - /** The name of the property for the number of fields in a record. */ - public static final String FIELD_COUNT_PROPERTY="fieldcount"; - - /** Default number of fields in a record. */ - public static final String FIELD_COUNT_PROPERTY_DEFAULT="10"; - - /** Representing a NULL value. */ - public static final String NULL_VALUE = "NULL"; - - /** The code to return when the call succeeds. */ - public static final int SUCCESS = 0; - - /** The primary key in the user table.*/ - public static String PRIMARY_KEY = "YCSB_KEY"; - - /** The field name prefix in the table.*/ - public static String COLUMN_PREFIX = "FIELD"; -} diff --git a/db/jdbc/src/com/yahoo/ycsb/db/JdbcDBCreateTable.java b/db/jdbc/src/com/yahoo/ycsb/db/JdbcDBCreateTable.java deleted file mode 100644 index 7416d05a..00000000 --- a/db/jdbc/src/com/yahoo/ycsb/db/JdbcDBCreateTable.java +++ /dev/null @@ -1,220 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ -package com.yahoo.ycsb.db; - -import java.io.FileInputStream; -import java.io.IOException; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Enumeration; -import java.util.Properties; - -/** - * Utility class to create the table to be used by the benchmark. - * - * @author sudipto - * - */ -public class JdbcDBCreateTable implements JdbcDBClientConstants { - - private static void usageMessage() { - System.out.println("Create Table Client. Options:"); - System.out.println(" -p key=value properties defined."); - System.out.println(" -P location of the properties file to load."); - System.out.println(" -n name of the table."); - System.out.println(" -f number of fields (default 10)."); - } - - private static void createTable(Properties props, String tablename) - throws SQLException { - String driver = props.getProperty(DRIVER_CLASS); - String username = props.getProperty(CONNECTION_USER); - String password = props.getProperty(CONNECTION_PASSWD, ""); - String url = props.getProperty(CONNECTION_URL); - int fieldcount = Integer.parseInt(props.getProperty(FIELD_COUNT_PROPERTY, - FIELD_COUNT_PROPERTY_DEFAULT)); - - if (driver == null || username == null || url == null) { - throw new SQLException("Missing connection information."); - } - - Connection conn = null; - - try { - Class.forName(driver); - - conn = DriverManager.getConnection(url, username, password); - Statement stmt = conn.createStatement(); - - StringBuilder sql = new StringBuilder("DROP TABLE IF EXISTS "); - sql.append(tablename); - sql.append(";"); - - stmt.execute(sql.toString()); - - sql = new StringBuilder("CREATE TABLE "); - sql.append(tablename); - sql.append(" (KEY VARCHAR PRIMARY KEY"); - - for (int idx = 0; idx < fieldcount; idx++) { - sql.append(", FIELD"); - sql.append(idx); - sql.append(" VARCHAR"); - } - sql.append(");"); - - stmt.execute(sql.toString()); - - System.out.println("Table " + tablename + " created.."); - } catch (ClassNotFoundException e) { - throw new SQLException("JDBC Driver class not found."); - } finally { - if (conn != null) { - System.out.println("Closing database connection."); - conn.close(); - } - } - } - - /** - * @param args - */ - public static void main(String[] args) { - - if (args.length == 0) { - usageMessage(); - System.exit(0); - } - - String tablename = null; - int fieldcount = -1; - Properties props = new Properties(); - Properties fileprops = new Properties(); - - // parse arguments - int argindex = 0; - while (args[argindex].startsWith("-")) { - if (args[argindex].compareTo("-P") == 0) { - argindex++; - if (argindex >= args.length) { - usageMessage(); - System.exit(0); - } - String propfile = args[argindex]; - argindex++; - - Properties myfileprops = new Properties(); - try { - myfileprops.load(new FileInputStream(propfile)); - } catch (IOException e) { - System.out.println(e.getMessage()); - System.exit(0); - } - - // Issue #5 - remove call to stringPropertyNames to make compilable - // under Java 1.5 - for (Enumeration e = myfileprops.propertyNames(); e - .hasMoreElements();) { - String prop = (String) e.nextElement(); - - fileprops.setProperty(prop, myfileprops.getProperty(prop)); - } - - } else if (args[argindex].compareTo("-p") == 0) { - argindex++; - if (argindex >= args.length) { - usageMessage(); - System.exit(0); - } - int eq = args[argindex].indexOf('='); - if (eq < 0) { - usageMessage(); - System.exit(0); - } - - String name = args[argindex].substring(0, eq); - String value = args[argindex].substring(eq + 1); - props.put(name, value); - argindex++; - } else if (args[argindex].compareTo("-n") == 0) { - argindex++; - if (argindex >= args.length) { - usageMessage(); - System.exit(0); - } - tablename = args[argindex++]; - } else if (args[argindex].compareTo("-f") == 0) { - argindex++; - if (argindex >= args.length) { - usageMessage(); - System.exit(0); - } - try { - fieldcount = Integer.parseInt(args[argindex++]); - } catch (NumberFormatException e) { - System.err.println("Invalid number for field count"); - usageMessage(); - System.exit(1); - } - } else { - System.out.println("Unknown option " + args[argindex]); - usageMessage(); - System.exit(0); - } - - if (argindex >= args.length) { - break; - } - } - - if (argindex != args.length) { - usageMessage(); - System.exit(0); - } - - // overwrite file properties with properties from the command line - - // Issue #5 - remove call to stringPropertyNames to make compilable under - // Java 1.5 - for (Enumeration e = props.propertyNames(); e.hasMoreElements();) { - String prop = (String) e.nextElement(); - - fileprops.setProperty(prop, props.getProperty(prop)); - } - - props = fileprops; - - if (tablename == null) { - System.err.println("table name missing."); - usageMessage(); - System.exit(1); - } - - if (fieldcount > 0) { - props.setProperty(FIELD_COUNT_PROPERTY, String.valueOf(fieldcount)); - } - - try { - createTable(props, tablename); - } catch (SQLException e) { - System.err.println("Error in creating table. " + e); - System.exit(1); - } - } -} diff --git a/db/mapkeeper/lib/README b/db/mapkeeper/lib/README deleted file mode 100644 index a59623a1..00000000 --- a/db/mapkeeper/lib/README +++ /dev/null @@ -1,6 +0,0 @@ -This directory should contain 2 jars; mapkeeper.jar and libthrift-0.6.1.jar. -To get them, do: - -$ wget https://raw.github.com/m1ch1/mapkeeper/master/lib/mapkeeper.jar -$ wget https://raw.github.com/m1ch1/mapkeeper/master/lib/libthrift-0.6.1.jar - diff --git a/db/mapkeeper/src/com/yahoo/ycsb/db/MapKeeperClient.java b/db/mapkeeper/src/com/yahoo/ycsb/db/MapKeeperClient.java deleted file mode 100644 index 594c9308..00000000 --- a/db/mapkeeper/src/com/yahoo/ycsb/db/MapKeeperClient.java +++ /dev/null @@ -1,196 +0,0 @@ -package com.yahoo.ycsb.db; - -import java.nio.ByteBuffer; -import java.util.HashMap; -import java.util.Properties; -import java.util.Set; -import java.util.Vector; - -import org.apache.thrift.TException; -import org.apache.thrift.protocol.TBinaryProtocol; -import org.apache.thrift.protocol.TProtocol; -import org.apache.thrift.transport.TFramedTransport; -import org.apache.thrift.transport.TSocket; -import org.apache.thrift.transport.TTransport; - -import com.yahoo.mapkeeper.BinaryResponse; -import com.yahoo.mapkeeper.MapKeeper; -import com.yahoo.mapkeeper.Record; -import com.yahoo.mapkeeper.RecordListResponse; -import com.yahoo.mapkeeper.ResponseCode; -import com.yahoo.mapkeeper.ScanOrder; -import com.yahoo.ycsb.ByteIterator; -import com.yahoo.ycsb.DB; -import com.yahoo.ycsb.StringByteIterator; -import com.yahoo.ycsb.workloads.CoreWorkload; - -public class MapKeeperClient extends DB { - MapKeeper.Client c; - boolean writeallfields; - static boolean initteddb = false; - private synchronized static void initDB(Properties p, MapKeeper.Client c) throws TException { - if(!initteddb) { - initteddb = true; - c.addMap(p.getProperty(CoreWorkload.TABLENAME_PROPERTY, CoreWorkload.TABLENAME_PROPERTY_DEFAULT)); - } - } - - public void init() { - TTransport tr = new TFramedTransport(new TSocket("localhost", 9090)); - TProtocol proto = new TBinaryProtocol(tr); - c = new MapKeeper.Client(proto); - try { - tr.open(); - initDB(getProperties(), c); - } catch(TException e) { - throw new RuntimeException(e); - } - writeallfields = Boolean.parseBoolean(getProperties().getProperty(CoreWorkload.WRITE_ALL_FIELDS_PROPERTY, - CoreWorkload.WRITE_ALL_FIELDS_PROPERTY_DEFAULT)); - } - - ByteBuffer encode(HashMap values) { - int len = 0; - for(String k : values.keySet()) { - len += (k.length() + 1 + values.get(k).bytesLeft() + 1); - } - byte[] array = new byte[len]; - int i = 0; - for(String k : values.keySet()) { - for(int j = 0; j < k.length(); j++) { - array[i] = (byte)k.charAt(j); - i++; - } - array[i] = '\t'; // XXX would like to use sane delimiter (null, 254, 255, ...) but java makes this nearly impossible - i++; - ByteIterator v = values.get(k); - i = v.nextBuf(array, i); - array[i] = '\t'; - i++; - } - array[array.length-1] = 0; - ByteBuffer buf = ByteBuffer.wrap(array); - buf.rewind(); - return buf; - } - void decode(Set fields, String tups, HashMap tup) { - String[] tok = tups.split("\\t"); - if(tok.length == 0) { throw new IllegalStateException("split returned empty array!"); } - for(int i = 0; i < tok.length; i+=2) { - if(fields == null || fields.contains(tok[i])) { - if(tok.length < i+2) { throw new IllegalStateException("Couldn't parse tuple <" + tups + "> at index " + i); } - if(tok[i] == null || tok[i+1] == null) throw new NullPointerException("Key is " + tok[i] + " val is + " + tok[i+1]); - tup.put(tok[i], new StringByteIterator(tok[i+1])); - } - } - if(tok.length == 0) { - System.err.println("Empty tuple: " + tups); - } - } - - int ycsbThriftRet(BinaryResponse succ, ResponseCode zero, ResponseCode one) { - return ycsbThriftRet(succ.responseCode, zero, one); - } - int ycsbThriftRet(ResponseCode rc, ResponseCode zero, ResponseCode one) { - return - rc == zero ? 0 : - rc == one ? 1 : 2; - } - ByteBuffer bufStr(String str) { - ByteBuffer buf = ByteBuffer.wrap(str.getBytes()); - return buf; - } - String strResponse(BinaryResponse buf) { - return new String(buf.value.array()); - } - - @Override - public int read(String table, String key, Set fields, - HashMap result) { - try { - ByteBuffer buf = bufStr(key); - - BinaryResponse succ = c.get(table, buf); - - int ret = ycsbThriftRet( - succ, - ResponseCode.RecordExists, - ResponseCode.RecordNotFound); - - if(ret == 0) { - decode(fields, strResponse(succ), result); - } - return ret; - } catch(TException e) { - e.printStackTrace(); - return 2; - } - } - - @Override - public int scan(String table, String startkey, int recordcount, - Set fields, Vector> result) { - try { - //XXX what to pass in for nulls / zeros? - RecordListResponse res = c.scan(table, ScanOrder.Ascending, bufStr(startkey), true, null, false, recordcount, 0); - int ret = ycsbThriftRet(res.responseCode, ResponseCode.Success, ResponseCode.ScanEnded); - if(ret == 0) { - for(Record r : res.records) { - HashMap tuple = new HashMap(); - // Note: r.getKey() and r.getValue() call special helper methods that trim the buffer - // to an appropriate length, and memcpy it to a byte[]. Trying to manipulate the ByteBuffer - // directly leads to trouble. - tuple.put("key", new StringByteIterator(new String(r.getKey()))); - decode(fields, new String(r.getValue())/*strBuf(r.bufferForValue())*/, tuple); - result.add(tuple); - } - } - return ret; - } catch(TException e) { - e.printStackTrace(); - return 2; - } - } - - @Override - public int update(String table, String key, - HashMap values) { - try { - if(!writeallfields) { - HashMap oldval = new HashMap(); - read(table, key, null, oldval); - for(String k: values.keySet()) { - oldval.put(k, values.get(k)); - } - values = oldval; - } - ResponseCode succ = c.update(table, bufStr(key), encode(values)); - return ycsbThriftRet(succ, ResponseCode.RecordExists, ResponseCode.RecordNotFound); - } catch(TException e) { - e.printStackTrace(); - return 2; - } - } - - @Override - public int insert(String table, String key, - HashMap values) { - try { - int ret = ycsbThriftRet(c.insert(table, bufStr(key), encode(values)), ResponseCode.Success, ResponseCode.RecordExists); - return ret; - } catch(TException e) { - e.printStackTrace(); - return 2; - } - } - - @Override - public int delete(String table, String key) { - try { - return ycsbThriftRet(c.remove(table, bufStr(key)), ResponseCode.Success, ResponseCode.RecordExists); - } catch(TException e) { - e.printStackTrace(); - return 2; - } - } -} diff --git a/db/mongodb/lib/README b/db/mongodb/lib/README deleted file mode 100644 index d255a2ec..00000000 --- a/db/mongodb/lib/README +++ /dev/null @@ -1 +0,0 @@ -This directory should contain jars for building and running MongoDB diff --git a/db/mongodb/src/com/yahoo/ycsb/db/MongoDbClient.java b/db/mongodb/src/com/yahoo/ycsb/db/MongoDbClient.java deleted file mode 100644 index df70e148..00000000 --- a/db/mongodb/src/com/yahoo/ycsb/db/MongoDbClient.java +++ /dev/null @@ -1,318 +0,0 @@ -/** - * MongoDB client binding for YCSB. - * - * Submitted by Yen Pai on 5/11/2010. - * - * https://gist.github.com/000a66b8db2caf42467b#file_mongo_db.java - * - */ - -package com.yahoo.ycsb.db; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Properties; -import java.util.Set; -import java.util.Map; -import java.util.Vector; - -import org.bson.types.ObjectId; - -import com.mongodb.BasicDBObject; -import com.mongodb.DBAddress; -import com.mongodb.DBCollection; -import com.mongodb.DBCursor; -import com.mongodb.DBObject; -import com.mongodb.Mongo; -import com.mongodb.ServerAddress; -import com.mongodb.WriteConcern; - -import com.yahoo.ycsb.DB; -import com.yahoo.ycsb.DBException; -import com.yahoo.ycsb.ByteIterator; -import com.yahoo.ycsb.StringByteIterator; - -/** - * MongoDB client for YCSB framework. - * - * Properties to set: - * - * mongodb.url=mongodb://localhost:27017 - * mongodb.database=ycsb - * mongodb.writeConcern=normal - * - * @author ypai - * - */ -public class MongoDbClient extends DB { - - private Mongo mongo; - private WriteConcern writeConcern; - private String database; - - /** - * Initialize any state for this DB. Called once per DB instance; there is - * one DB instance per client thread. - */ - public void init() throws DBException { - // initialize MongoDb driver - Properties props = getProperties(); - String url = props.getProperty("mongodb.url"); - database = props.getProperty("mongodb.database"); - String writeConcernType = props.getProperty("mongodb.writeConcern"); - - if ("none".equals(writeConcernType)) { - writeConcern = WriteConcern.NONE; - } else if ("strict".equals(writeConcernType)) { - writeConcern = WriteConcern.SAFE; - } else if ("normal".equals(writeConcernType)) { - writeConcern = WriteConcern.NORMAL; - } - - try { - // strip out prefix since Java driver doesn't currently support - // standard connection format URL yet - // http://www.mongodb.org/display/DOCS/Connections - if (url.startsWith("mongodb://")) { - url = url.substring(10); - } - - // need to append db to url. - url += "/"+database; - System.out.println("new database url = "+url); - mongo = new Mongo(new DBAddress(url)); - System.out.println("mongo connection created with "+url); - } catch (Exception e1) { - System.err.println( - "Could not initialize MongoDB connection pool for Loader: " - + e1.toString()); - return; - } - - } - - @Override - /** - * Delete a record from the database. - * - * @param table The name of the table - * @param key The record key of the record to delete. - * @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes. - */ - public int delete(String table, String key) { - com.mongodb.DB db=null; - try { - db = mongo.getDB(database); - db.requestStart(); - DBCollection collection = db.getCollection(table); - DBObject q = new BasicDBObject().append("_id", key); - if (writeConcern.equals(WriteConcern.SAFE)) { - q.put("$atomic", true); - } - collection.remove(q); - - // see if record was deleted - DBObject errors = db.getLastError(); - - return ((Integer) errors.get("n")) == 1 ? 0 : 1; - } catch (Exception e) { - System.err.println(e.toString()); - return 1; - } - finally - { - if (db!=null) - { - db.requestDone(); - } - } - } - - @Override - /** - * Insert a record in the database. Any field/value pairs in the specified values HashMap will be written into the record with the specified - * record key. - * - * @param table The name of the table - * @param key The record key of the record to insert. - * @param values A HashMap of field/value pairs to insert in the record - * @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes. - */ - public int insert(String table, String key, HashMap values) { - com.mongodb.DB db = null; - try { - db = mongo.getDB(database); - - db.requestStart(); - - DBCollection collection = db.getCollection(table); - DBObject r = new BasicDBObject().append("_id", key); - for(String k: values.keySet()) { - r.put(k, values.get(k).toString()); - } - collection.setWriteConcern(writeConcern); - - collection.insert(r); - - // determine if record was inserted, does not seem to return - // n= for insert - DBObject errors = db.getLastError(); - - return ((Double) errors.get("ok") == 1.0) && errors.get("err") == null ? 0 : 1; - } catch (Exception e) { - System.err.println(e.toString()); - return 1; - } finally { - if (db!=null) - { - db.requestDone(); - } - } - } - - @Override - @SuppressWarnings("unchecked") - /** - * Read a record from the database. Each field/value pair from the result will be stored in a HashMap. - * - * @param table The name of the table - * @param key The record key of the record to read. - * @param fields The list of fields to read, or null for all of them - * @param result A HashMap of field/value pairs for the result - * @return Zero on success, a non-zero error code on error or "not found". - */ - public int read(String table, String key, Set fields, - HashMap result) { - com.mongodb.DB db = null; - try { - db = mongo.getDB(database); - - db.requestStart(); - - DBCollection collection = db.getCollection(table); - DBObject q = new BasicDBObject().append("_id", key); - DBObject fieldsToReturn = new BasicDBObject(); - boolean returnAllFields = fields == null; - - DBObject queryResult = null; - if (!returnAllFields) { - Iterator iter = fields.iterator(); - while (iter.hasNext()) { - fieldsToReturn.put(iter.next(), 1); - } - queryResult = collection.findOne(q, fieldsToReturn); - } else { - queryResult = collection.findOne(q); - } - - if (queryResult != null) { - result.putAll(queryResult.toMap()); - } - return queryResult != null ? 0 : 1; - } catch (Exception e) { - System.err.println(e.toString()); - return 1; - } finally { - if (db!=null) - { - db.requestDone(); - } - } - } - - - @Override - /** - * Update a record in the database. Any field/value pairs in the specified values HashMap will be written into the record with the specified - * record key, overwriting any existing values with the same field name. - * - * @param table The name of the table - * @param key The record key of the record to write. - * @param values A HashMap of field/value pairs to update in the record - * @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes. - */ - public int update(String table, String key, HashMap values) { - com.mongodb.DB db = null; - try { - db = mongo.getDB(database); - - db.requestStart(); - - DBCollection collection = db.getCollection(table); - DBObject q = new BasicDBObject().append("_id", key); - DBObject u = new BasicDBObject(); - DBObject fieldsToSet = new BasicDBObject(); - Iterator keys = values.keySet().iterator(); - String tmpKey = null, tmpVal = null; - while (keys.hasNext()) { - tmpKey = keys.next(); - tmpVal = values.get(tmpKey).toString(); - fieldsToSet.put(tmpKey, tmpVal); - - } - u.put("$set", fieldsToSet); - - collection.setWriteConcern(writeConcern); - - collection.update(q, u); - - // determine if record was inserted - DBObject errors = db.getLastError(); - - return (Integer) errors.get("n") == 1 ? 0 : 1; - } catch (Exception e) { - System.err.println(e.toString()); - return 1; - } finally { - if (db!=null) - { - db.requestDone(); - } - } - } - - @Override - @SuppressWarnings("unchecked") - /** - * Perform a range scan for a set of records in the database. Each field/value pair from the result will be stored in a HashMap. - * - * @param table The name of the table - * @param startkey The record key of the first record to read. - * @param recordcount The number of records to read - * @param fields The list of fields to read, or null for all of them - * @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record - * @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes. - */ - public int scan(String table, String startkey, int recordcount, - Set fields, Vector> result) { - com.mongodb.DB db=null; - try { - db = mongo.getDB(database); - db.requestStart(); - DBCollection collection = db.getCollection(table); - // { "_id":{"$gte":startKey, "$lte":{"appId":key+"\uFFFF"}} } - DBObject scanRange = new BasicDBObject().append("$gte", startkey); - DBObject q = new BasicDBObject().append("_id", scanRange); - DBCursor cursor = collection.find(q).limit(recordcount); - while (cursor.hasNext()) { - //toMap() returns a Map, but result.add() expects a Map. Hence, the suppress warnings. - result.add(StringByteIterator.getByteIteratorMap((Map)cursor.next().toMap())); - } - - return 0; - } catch (Exception e) { - System.err.println(e.toString()); - return 1; - } - finally - { - if (db!=null) - { - db.requestDone(); - } - } - - } -} - diff --git a/db/redis/lib/README b/db/redis/lib/README deleted file mode 100644 index 773da82f..00000000 --- a/db/redis/lib/README +++ /dev/null @@ -1,4 +0,0 @@ -This directory should contain jars for building and running Redis, namely -jedis.jar (tested with 1.5.2). - -Jedis is available from https://github.com/xetorthio/jedis/downloads diff --git a/db/redis/src/com/yahoo/ycsb/db/RedisClient.java b/db/redis/src/com/yahoo/ycsb/db/RedisClient.java deleted file mode 100644 index 0d5c3ff1..00000000 --- a/db/redis/src/com/yahoo/ycsb/db/RedisClient.java +++ /dev/null @@ -1,131 +0,0 @@ -/** - * Redis client binding for YCSB. - * - * All YCSB records are mapped to a Redis *hash field*. For scanning - * operations, all keys are saved (by an arbitrary hash) in a sorted set. - */ - -package com.yahoo.ycsb.db; -import com.yahoo.ycsb.DB; -import com.yahoo.ycsb.DBException; -import com.yahoo.ycsb.ByteIterator; -import com.yahoo.ycsb.StringByteIterator; - -import java.util.Map; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Properties; -import java.util.Set; -import java.util.Vector; - -import redis.clients.jedis.Jedis; -import redis.clients.jedis.Protocol; - -public class RedisClient extends DB { - - private Jedis jedis; - - public static final String HOST_PROPERTY = "redis.host"; - public static final String PORT_PROPERTY = "redis.port"; - public static final String PASSWORD_PROPERTY = "redis.password"; - - public static final String INDEX_KEY = "_indices"; - - public void init() throws DBException { - Properties props = getProperties(); - int port; - - String portString = props.getProperty(PORT_PROPERTY); - if (portString != null) { - port = Integer.parseInt(portString); - } - else { - port = Protocol.DEFAULT_PORT; - } - String host = props.getProperty(HOST_PROPERTY); - - jedis = new Jedis(host, port); - jedis.connect(); - - String password = props.getProperty(PASSWORD_PROPERTY); - if (password != null) { - jedis.auth(password); - } - } - - public void cleanup() throws DBException { - jedis.disconnect(); - } - - /* Calculate a hash for a key to store it in an index. The actual return - * value of this function is not interesting -- it primarily needs to be - * fast and scattered along the whole space of doubles. In a real world - * scenario one would probably use the ASCII values of the keys. - */ - private double hash(String key) { - return key.hashCode(); - } - - //XXX jedis.select(int index) to switch to `table` - - @Override - public int read(String table, String key, Set fields, - HashMap result) { - if (fields == null) { - StringByteIterator.putAllAsByteIterators(result, jedis.hgetAll(key)); - } - else { - String[] fieldArray = (String[])fields.toArray(new String[fields.size()]); - List values = jedis.hmget(key, fieldArray); - - Iterator fieldIterator = fields.iterator(); - Iterator valueIterator = values.iterator(); - - while (fieldIterator.hasNext() && valueIterator.hasNext()) { - result.put(fieldIterator.next(), - new StringByteIterator(valueIterator.next())); - } - assert !fieldIterator.hasNext() && !valueIterator.hasNext(); - } - return result.isEmpty() ? 1 : 0; - } - - @Override - public int insert(String table, String key, HashMap values) { - if (jedis.hmset(key, StringByteIterator.getStringMap(values)).equals("OK")) { - jedis.zadd(INDEX_KEY, hash(key), key); - return 0; - } - return 1; - } - - @Override - public int delete(String table, String key) { - return jedis.del(key) == 0 - && jedis.zrem(INDEX_KEY, key) == 0 - ? 1 : 0; - } - - @Override - public int update(String table, String key, HashMap values) { - return jedis.hmset(key, StringByteIterator.getStringMap(values)).equals("OK") ? 0 : 1; - } - - @Override - public int scan(String table, String startkey, int recordcount, - Set fields, Vector> result) { - Set keys = jedis.zrangeByScore(INDEX_KEY, hash(startkey), - Double.POSITIVE_INFINITY, 0, recordcount); - - HashMap values; - for (String key : keys) { - values = new HashMap(); - read(table, key, fields, values); - result.add(values); - } - - return 0; - } - -} diff --git a/db/voldemort/config/cluster.xml b/db/voldemort/config/cluster.xml deleted file mode 100644 index 175f69ad..00000000 --- a/db/voldemort/config/cluster.xml +++ /dev/null @@ -1,11 +0,0 @@ - - mycluster - - 0 - localhost - 8081 - 6666 - 0, 1 - - - diff --git a/db/voldemort/config/server.properties b/db/voldemort/config/server.properties deleted file mode 100644 index fce5ee43..00000000 --- a/db/voldemort/config/server.properties +++ /dev/null @@ -1,26 +0,0 @@ -# The ID of *this* particular cluster node -node.id=0 - -max.threads=100 - -############### DB options ###################### - -http.enable=true -socket.enable=true - -# BDB -bdb.write.transactions=false -bdb.flush.transactions=false -bdb.cache.size=1G - -# Mysql -mysql.host=localhost -mysql.port=1521 -mysql.user=root -mysql.password=3306 -mysql.database=test - -#NIO connector settings. -enable.nio.connector=true - -storage.configs=voldemort.store.bdb.BdbStorageConfiguration, voldemort.store.readonly.ReadOnlyStorageConfiguration diff --git a/db/voldemort/config/stores.xml b/db/voldemort/config/stores.xml deleted file mode 100644 index b2d892d7..00000000 --- a/db/voldemort/config/stores.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - usertable - bdb - client - 1 - 1 - 1 - - string - - - java-serialization - - - diff --git a/db/voldemort/lib/README b/db/voldemort/lib/README deleted file mode 100644 index 9dbdbc92..00000000 --- a/db/voldemort/lib/README +++ /dev/null @@ -1,26 +0,0 @@ -This directory should contain jars for building and running Voldemort - -HOWTO -a) Get Voldemort -> mkdir voldemort-0.81 ; cd voldemort-0.81 -> git clone git@github.com:voldemort/voldemort.git -> git checkout -b 81 origin/release-081 -OR -> wget http://github.com/downloads/voldemort/voldemort/voldemort-0.81.tar.gz -> tar xvfz voldemort-0.81.tar.gz -> cd voldemort-0.81 - -b) Copy libs -> ant # inside voldemort-0.81 folder -> cp dist/voldemort-*0.81.jar $YCSB_PATH/db/voldemort/lib/. -> cp lib/*.jar $YCSB_PATH/db/voldemort/lib/. - -c) Compile -> cd $YCSB_PATH -> ant ; ant dbcompile-voldemort - -d) Start test voldemort -> voldemort-0.81/bin/voldemort-server.sh $YCSB_PATH/db/voldemort/ & # Reads the files in config folder - -e) Run YCSB -> java com.yahoo.ycsb.CommandLine -db com.yahoo.ycsb.db.VoldemortClient -p bootstrap_urls=tcp://localhost:6666 diff --git a/db/voldemort/src/com/yahoo/ycsb/db/VoldemortClient.java b/db/voldemort/src/com/yahoo/ycsb/db/VoldemortClient.java deleted file mode 100644 index 84093b91..00000000 --- a/db/voldemort/src/com/yahoo/ycsb/db/VoldemortClient.java +++ /dev/null @@ -1,151 +0,0 @@ -package com.yahoo.ycsb.db; - -import java.util.HashMap; -import java.util.Set; -import java.util.Vector; -import java.util.Map.Entry; - -import org.apache.log4j.Logger; - -import voldemort.client.ClientConfig; -import voldemort.client.SocketStoreClientFactory; -import voldemort.client.StoreClient; -import voldemort.versioning.VectorClock; -import voldemort.versioning.Versioned; - -import com.yahoo.ycsb.DB; -import com.yahoo.ycsb.DBException; -import com.yahoo.ycsb.ByteIterator; -import com.yahoo.ycsb.StringByteIterator; - - -public class VoldemortClient extends DB { - - private StoreClient> storeClient; - private SocketStoreClientFactory socketFactory; - private String storeName; - private final Logger logger = Logger.getLogger(VoldemortClient.class); - - public static final int OK = 0; - public static final int ERROR = -1; - public static final int NOT_FOUND = -2; - - /** - * Initialize the DB layer. This accepts all properties allowed by the Voldemort client. - * A store maps to a table. - * Required : bootstrap_urls - * Additional property : store_name -> to preload once, should be same as -t - * - * {@linktourl http://project-voldemort.com/javadoc/client/voldemort/client/ClientConfig.html} - */ - public void init() throws DBException { - ClientConfig clientConfig = new ClientConfig(getProperties()); - socketFactory = new SocketStoreClientFactory(clientConfig); - - // Retrieve store name - storeName = getProperties().getProperty("store_name", "usertable"); - - // Use store name to retrieve client - storeClient = socketFactory.getStoreClient(storeName); - if ( storeClient == null ) - throw new DBException("Unable to instantiate store client"); - - } - - public void cleanup() throws DBException { - socketFactory.close(); - } - - @Override - public int delete(String table, String key) { - if ( checkStore(table) == ERROR ) { - return ERROR; - } - - if ( storeClient.delete(key) ) - return OK; - else - return ERROR; - } - - @Override - public int insert(String table, String key, HashMap values) { - if ( checkStore(table) == ERROR ) { - return ERROR; - } - storeClient.put(key, (HashMap)StringByteIterator.getStringMap(values)); - return OK; - } - - @Override - public int read(String table, String key, Set fields, - HashMap result) { - if ( checkStore(table) == ERROR ) { - return ERROR; - } - - Versioned> versionedValue = storeClient.get(key); - - if ( versionedValue == null ) - return NOT_FOUND; - - if ( fields != null ) { - for (String field : fields) { - String val = versionedValue.getValue().get(field); - if ( val != null ) - result.put(field, new StringByteIterator(val)); - } - } else { - StringByteIterator.putAllAsByteIterators(result, versionedValue.getValue()); - } - return OK; - } - - @Override - public int scan(String table, String startkey, int recordcount, - Set fields, Vector> result) { - logger.warn("Voldemort does not support Scan semantics"); - return OK; - } - - @Override - public int update(String table, String key, HashMap values) { - if ( checkStore(table) == ERROR ) { - return ERROR; - } - - Versioned> versionedValue = storeClient.get(key); - HashMap value = new HashMap(); - VectorClock version; - if ( versionedValue != null ) { - version = ((VectorClock) versionedValue.getVersion()).incremented(0, 1); - value = versionedValue.getValue(); - for (Entry entry : values.entrySet()) { - value.put(entry.getKey(), entry.getValue().toString()); - } - } else { - version = new VectorClock(); - StringByteIterator.putAllAsStrings(value, values); - } - - storeClient.put(key, Versioned.value(value, version)); - return OK; - } - - private int checkStore(String table) { - if ( table.compareTo(storeName) != 0 ) { - try { - storeClient = socketFactory.getStoreClient(table); - if ( storeClient == null ) { - logger.error("Could not instantiate storeclient for " + table); - return ERROR; - } - storeName = table; - } catch ( Exception e ) { - return ERROR; - } - } - return OK; - } - -} diff --git a/lib/jackson-core-asl-1.5.2.jar b/lib/jackson-core-asl-1.5.2.jar deleted file mode 100644 index 24107e68..00000000 Binary files a/lib/jackson-core-asl-1.5.2.jar and /dev/null differ diff --git a/lib/jackson-mapper-asl-1.5.2.jar b/lib/jackson-mapper-asl-1.5.2.jar deleted file mode 100644 index ebf9b5ef..00000000 Binary files a/lib/jackson-mapper-asl-1.5.2.jar and /dev/null differ diff --git a/src/com/yahoo/ycsb/BasicDB.java b/src/com/yahoo/ycsb/BasicDB.java deleted file mode 100644 index 9490451e..00000000 --- a/src/com/yahoo/ycsb/BasicDB.java +++ /dev/null @@ -1,267 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package com.yahoo.ycsb; - -import java.util.HashMap; -import java.util.Properties; -import java.util.Set; -import java.util.Enumeration; -import java.util.Vector; - - -/** - * Basic DB that just prints out the requested operations, instead of doing them against a database. - */ -public class BasicDB extends DB -{ - public static final String VERBOSE="basicdb.verbose"; - public static final String VERBOSE_DEFAULT="true"; - - public static final String SIMULATE_DELAY="basicdb.simulatedelay"; - public static final String SIMULATE_DELAY_DEFAULT="0"; - - - boolean verbose; - int todelay; - - public BasicDB() - { - todelay=0; - } - - - void delay() - { - if (todelay>0) - { - try - { - Thread.sleep((long)Utils.random().nextInt(todelay)); - } - catch (InterruptedException e) - { - //do nothing - } - } - } - - /** - * Initialize any state for this DB. - * Called once per DB instance; there is one DB instance per client thread. - */ - @SuppressWarnings("unchecked") - public void init() - { - verbose=Boolean.parseBoolean(getProperties().getProperty(VERBOSE, VERBOSE_DEFAULT)); - todelay=Integer.parseInt(getProperties().getProperty(SIMULATE_DELAY, SIMULATE_DELAY_DEFAULT)); - - if (verbose) - { - System.out.println("***************** properties *****************"); - Properties p=getProperties(); - if (p!=null) - { - for (Enumeration e=p.propertyNames(); e.hasMoreElements(); ) - { - String k=(String)e.nextElement(); - System.out.println("\""+k+"\"=\""+p.getProperty(k)+"\""); - } - } - System.out.println("**********************************************"); - } - } - - /** - * Read a record from the database. Each field/value pair from the result will be stored in a HashMap. - * - * @param table The name of the table - * @param key The record key of the record to read. - * @param fields The list of fields to read, or null for all of them - * @param result A HashMap of field/value pairs for the result - * @return Zero on success, a non-zero error code on error - */ - public int read(String table, String key, Set fields, HashMap result) - { - delay(); - - if (verbose) - { - System.out.print("READ "+table+" "+key+" [ "); - if (fields!=null) - { - for (String f : fields) - { - System.out.print(f+" "); - } - } - else - { - System.out.print(""); - } - - System.out.println("]"); - } - - return 0; - } - - /** - * Perform a range scan for a set of records in the database. Each field/value pair from the result will be stored in a HashMap. - * - * @param table The name of the table - * @param startkey The record key of the first record to read. - * @param recordcount The number of records to read - * @param fields The list of fields to read, or null for all of them - * @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record - * @return Zero on success, a non-zero error code on error - */ - public int scan(String table, String startkey, int recordcount, Set fields, Vector> result) - { - delay(); - - if (verbose) - { - System.out.print("SCAN "+table+" "+startkey+" "+recordcount+" [ "); - if (fields!=null) - { - for (String f : fields) - { - System.out.print(f+" "); - } - } - else - { - System.out.print(""); - } - - System.out.println("]"); - } - - return 0; - } - - /** - * Update a record in the database. Any field/value pairs in the specified values HashMap will be written into the record with the specified - * record key, overwriting any existing values with the same field name. - * - * @param table The name of the table - * @param key The record key of the record to write. - * @param values A HashMap of field/value pairs to update in the record - * @return Zero on success, a non-zero error code on error - */ - public int update(String table, String key, HashMap values) - { - delay(); - - if (verbose) - { - System.out.print("UPDATE "+table+" "+key+" [ "); - if (values!=null) - { - for (String k : values.keySet()) - { - System.out.print(k+"="+values.get(k)+" "); - } - } - System.out.println("]"); - } - - return 0; - } - - /** - * Insert a record in the database. Any field/value pairs in the specified values HashMap will be written into the record with the specified - * record key. - * - * @param table The name of the table - * @param key The record key of the record to insert. - * @param values A HashMap of field/value pairs to insert in the record - * @return Zero on success, a non-zero error code on error - */ - public int insert(String table, String key, HashMap values) - { - delay(); - - if (verbose) - { - System.out.print("INSERT "+table+" "+key+" [ "); - if (values!=null) - { - for (String k : values.keySet()) - { - System.out.print(k+"="+values.get(k)+" "); - } - } - - System.out.println("]"); - } - - return 0; - } - - - /** - * Delete a record from the database. - * - * @param table The name of the table - * @param key The record key of the record to delete. - * @return Zero on success, a non-zero error code on error - */ - public int delete(String table, String key) - { - delay(); - - if (verbose) - { - System.out.println("DELETE "+table+" "+key); - } - - return 0; - } - - /** - * Short test of BasicDB - */ - /* - public static void main(String[] args) - { - BasicDB bdb=new BasicDB(); - - Properties p=new Properties(); - p.setProperty("Sky","Blue"); - p.setProperty("Ocean","Wet"); - - bdb.setProperties(p); - - bdb.init(); - - HashMap fields=new HashMap(); - fields.put("A","X"); - fields.put("B","Y"); - - bdb.read("table","key",null,null); - bdb.insert("table","key",fields); - - fields=new HashMap(); - fields.put("C","Z"); - - bdb.update("table","key",fields); - - bdb.delete("table","key"); - }*/ -} diff --git a/src/com/yahoo/ycsb/ByteArrayByteIterator.java b/src/com/yahoo/ycsb/ByteArrayByteIterator.java deleted file mode 100644 index 3e36f988..00000000 --- a/src/com/yahoo/ycsb/ByteArrayByteIterator.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ -package com.yahoo.ycsb; - -public class ByteArrayByteIterator extends ByteIterator { - byte[] str; - int off; - final int len; - public ByteArrayByteIterator(byte[] s) { - this.str = s; - this.off = 0; - this.len = s.length; - } - - public ByteArrayByteIterator(byte[] s, int off, int len) { - this.str = s; - this.off = off; - this.len = off + len; - } - - @Override - public boolean hasNext() { - return off < len; - } - - @Override - public byte nextByte() { - byte ret = str[off]; - off++; - return ret; - } - - @Override - public long bytesLeft() { - return len - off; - } - -} diff --git a/src/com/yahoo/ycsb/ByteIterator.java b/src/com/yahoo/ycsb/ByteIterator.java deleted file mode 100644 index 9bb3b63e..00000000 --- a/src/com/yahoo/ycsb/ByteIterator.java +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ -package com.yahoo.ycsb; - -import java.util.Iterator; -import java.util.ArrayList; -/** - * YCSB-specific buffer class. ByteIterators are designed to support - * efficient field generation, and to allow backend drivers that can stream - * fields (instead of materializing them in RAM) to do so. - *

- * YCSB originially used String objects to represent field values. This led to - * two performance issues. - *

- * First, it leads to unnecessary conversions between UTF-16 and UTF-8, both - * during field generation, and when passing data to byte-based backend - * drivers. - *

- * Second, Java strings are represented internally using UTF-16, and are - * built by appending to a growable array type (StringBuilder or - * StringBuffer), then calling a toString() method. This leads to a 4x memory - * overhead as field values are being built, which prevented YCSB from - * driving large object stores. - *

- * The StringByteIterator class contains a number of convenience methods for - * backend drivers that convert between Map<String,String> and - * Map<String,ByteBuffer>. - * - * @author sears - */ -public abstract class ByteIterator implements Iterator { - - @Override - public abstract boolean hasNext(); - - @Override - public Byte next() { - throw new UnsupportedOperationException(); - //return nextByte(); - } - - public abstract byte nextByte(); - /** @return byte offset immediately after the last valid byte */ - public int nextBuf(byte[] buf, int buf_off) { - int sz = buf_off; - while(sz < buf.length && hasNext()) { - buf[sz] = nextByte(); - sz++; - } - return sz; - } - - public abstract long bytesLeft(); - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - - /** Consumes remaining contents of this object, and returns them as a string. */ - public String toString() { - StringBuilder sb = new StringBuilder(); - while(this.hasNext()) { sb.append((char)nextByte()); } - return sb.toString(); - } - /** Consumes remaining contents of this object, and returns them as a byte array. */ - public byte[] toArray() { - long left = bytesLeft(); - if(left != (int)left) { throw new ArrayIndexOutOfBoundsException("Too much data to fit in one array!"); } - byte[] ret = new byte[(int)left]; - int off = 0; - while(off < ret.length) { - off = nextBuf(ret, off); - } - return ret; - } - -} diff --git a/src/com/yahoo/ycsb/Client.java b/src/com/yahoo/ycsb/Client.java deleted file mode 100644 index 64585f81..00000000 --- a/src/com/yahoo/ycsb/Client.java +++ /dev/null @@ -1,802 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package com.yahoo.ycsb; - - -import java.io.*; -import java.text.DecimalFormat; -import java.util.*; - -import com.yahoo.ycsb.measurements.Measurements; -import com.yahoo.ycsb.measurements.exporter.MeasurementsExporter; -import com.yahoo.ycsb.measurements.exporter.TextMeasurementsExporter; - -//import org.apache.log4j.BasicConfigurator; - -/** - * A thread to periodically show the status of the experiment, to reassure you that progress is being made. - * - * @author cooperb - * - */ -class StatusThread extends Thread -{ - Vector _threads; - String _label; - boolean _standardstatus; - - /** - * The interval for reporting status. - */ - public static final long sleeptime=10000; - - public StatusThread(Vector threads, String label, boolean standardstatus) - { - _threads=threads; - _label=label; - _standardstatus=standardstatus; - } - - /** - * Run and periodically report status. - */ - public void run() - { - long st=System.currentTimeMillis(); - - long lasten=st; - long lasttotalops=0; - - boolean alldone; - - do - { - alldone=true; - - int totalops=0; - - //terminate this thread when all the worker threads are done - for (Thread t : _threads) - { - if (t.getState()!=Thread.State.TERMINATED) - { - alldone=false; - } - - ClientThread ct=(ClientThread)t; - totalops+=ct.getOpsDone(); - } - - long en=System.currentTimeMillis(); - - long interval=en-st; - //double throughput=1000.0*((double)totalops)/((double)interval); - - double curthroughput=1000.0*(((double)(totalops-lasttotalops))/((double)(en-lasten))); - - lasttotalops=totalops; - lasten=en; - - DecimalFormat d = new DecimalFormat("#.##"); - - if (totalops==0) - { - System.err.println(_label+" "+(interval/1000)+" sec: "+totalops+" operations; "+Measurements.getMeasurements().getSummary()); - } - else - { - System.err.println(_label+" "+(interval/1000)+" sec: "+totalops+" operations; "+d.format(curthroughput)+" current ops/sec; "+Measurements.getMeasurements().getSummary()); - } - - if (_standardstatus) - { - if (totalops==0) - { - System.out.println(_label+" "+(interval/1000)+" sec: "+totalops+" operations; "+Measurements.getMeasurements().getSummary()); - } - else - { - System.out.println(_label+" "+(interval/1000)+" sec: "+totalops+" operations; "+d.format(curthroughput)+" current ops/sec; "+Measurements.getMeasurements().getSummary()); - } - } - - try - { - sleep(sleeptime); - } - catch (InterruptedException e) - { - //do nothing - } - - } - while (!alldone); - } -} - -/** - * A thread for executing transactions or data inserts to the database. - * - * @author cooperb - * - */ -class ClientThread extends Thread -{ - DB _db; - boolean _dotransactions; - Workload _workload; - int _opcount; - double _target; - - int _opsdone; - int _threadid; - int _threadcount; - Object _workloadstate; - Properties _props; - - - /** - * Constructor. - * - * @param db the DB implementation to use - * @param dotransactions true to do transactions, false to insert data - * @param workload the workload to use - * @param threadid the id of this thread - * @param threadcount the total number of threads - * @param props the properties defining the experiment - * @param opcount the number of operations (transactions or inserts) to do - * @param targetperthreadperms target number of operations per thread per ms - */ - public ClientThread(DB db, boolean dotransactions, Workload workload, int threadid, int threadcount, Properties props, int opcount, double targetperthreadperms) - { - //TODO: consider removing threadcount and threadid - _db=db; - _dotransactions=dotransactions; - _workload=workload; - _opcount=opcount; - _opsdone=0; - _target=targetperthreadperms; - _threadid=threadid; - _threadcount=threadcount; - _props=props; - //System.out.println("Interval = "+interval); - } - - public int getOpsDone() - { - return _opsdone; - } - - public void run() - { - try - { - _db.init(); - } - catch (DBException e) - { - e.printStackTrace(); - e.printStackTrace(System.out); - return; - } - - try - { - _workloadstate=_workload.initThread(_props,_threadid,_threadcount); - } - catch (WorkloadException e) - { - e.printStackTrace(); - e.printStackTrace(System.out); - return; - } - - //spread the thread operations out so they don't all hit the DB at the same time - try - { - //GH issue 4 - throws exception if _target>1 because random.nextInt argument must be >0 - //and the sleep() doesn't make sense for granularities < 1 ms anyway - if ( (_target>0) && (_target<=1.0) ) - { - sleep(Utils.random().nextInt((int)(1.0/_target))); - } - } - catch (InterruptedException e) - { - // do nothing. - } - - try - { - if (_dotransactions) - { - long st=System.currentTimeMillis(); - - while (((_opcount == 0) || (_opsdone < _opcount)) && !_workload.isStopRequested()) - { - - if (!_workload.doTransaction(_db,_workloadstate)) - { - break; - } - - _opsdone++; - - //throttle the operations - if (_target>0) - { - //this is more accurate than other throttling approaches we have tried, - //like sleeping for (1/target throughput)-operation latency, - //because it smooths timing inaccuracies (from sleep() taking an int, - //current time in millis) over many operations - while (System.currentTimeMillis()-st<((double)_opsdone)/_target) - { - try - { - sleep(1); - } - catch (InterruptedException e) - { - // do nothing. - } - - } - } - } - } - else - { - long st=System.currentTimeMillis(); - - while (((_opcount == 0) || (_opsdone < _opcount)) && !_workload.isStopRequested()) - { - - if (!_workload.doInsert(_db,_workloadstate)) - { - break; - } - - _opsdone++; - - //throttle the operations - if (_target>0) - { - //this is more accurate than other throttling approaches we have tried, - //like sleeping for (1/target throughput)-operation latency, - //because it smooths timing inaccuracies (from sleep() taking an int, - //current time in millis) over many operations - while (System.currentTimeMillis()-st<((double)_opsdone)/_target) - { - try - { - sleep(1); - } - catch (InterruptedException e) - { - // do nothing. - } - } - } - } - } - } - catch (Exception e) - { - e.printStackTrace(); - e.printStackTrace(System.out); - System.exit(0); - } - - try - { - _db.cleanup(); - } - catch (DBException e) - { - e.printStackTrace(); - e.printStackTrace(System.out); - return; - } - } -} - -/** - * Main class for executing YCSB. - */ -public class Client -{ - - public static final String OPERATION_COUNT_PROPERTY="operationcount"; - - public static final String RECORD_COUNT_PROPERTY="recordcount"; - - public static final String WORKLOAD_PROPERTY="workload"; - - /** - * Indicates how many inserts to do, if less than recordcount. Useful for partitioning - * the load among multiple servers, if the client is the bottleneck. Additionally, workloads - * should support the "insertstart" property, which tells them which record to start at. - */ - public static final String INSERT_COUNT_PROPERTY="insertcount"; - - /** - * The maximum amount of time (in seconds) for which the benchmark will be run. - */ - public static final String MAX_EXECUTION_TIME = "maxexecutiontime"; - - public static void usageMessage() - { - System.out.println("Usage: java com.yahoo.ycsb.Client [options]"); - System.out.println("Options:"); - System.out.println(" -threads n: execute using n threads (default: 1) - can also be specified as the \n" + - " \"threadcount\" property using -p"); - System.out.println(" -target n: attempt to do n operations per second (default: unlimited) - can also\n" + - " be specified as the \"target\" property using -p"); - System.out.println(" -load: run the loading phase of the workload"); - System.out.println(" -t: run the transactions phase of the workload (default)"); - System.out.println(" -db dbname: specify the name of the DB to use (default: com.yahoo.ycsb.BasicDB) - \n" + - " can also be specified as the \"db\" property using -p"); - System.out.println(" -P propertyfile: load properties from the given file. Multiple files can"); - System.out.println(" be specified, and will be processed in the order specified"); - System.out.println(" -p name=value: specify a property to be passed to the DB and workloads;"); - System.out.println(" multiple properties can be specified, and override any"); - System.out.println(" values in the propertyfile"); - System.out.println(" -s: show status during run (default: no status)"); - System.out.println(" -l label: use label for status (e.g. to label one experiment out of a whole batch)"); - System.out.println(""); - System.out.println("Required properties:"); - System.out.println(" "+WORKLOAD_PROPERTY+": the name of the workload class to use (e.g. com.yahoo.ycsb.workloads.CoreWorkload)"); - System.out.println(""); - System.out.println("To run the transaction phase from multiple servers, start a separate client on each."); - System.out.println("To run the load phase from multiple servers, start a separate client on each; additionally,"); - System.out.println("use the \"insertcount\" and \"insertstart\" properties to divide up the records to be inserted"); - } - - public static boolean checkRequiredProperties(Properties props) - { - if (props.getProperty(WORKLOAD_PROPERTY)==null) - { - System.out.println("Missing property: "+WORKLOAD_PROPERTY); - return false; - } - - return true; - } - - - /** - * Exports the measurements to either sysout or a file using the exporter - * loaded from conf. - * @throws IOException Either failed to write to output stream or failed to close it. - */ - private static void exportMeasurements(Properties props, int opcount, long runtime) - throws IOException - { - MeasurementsExporter exporter = null; - try - { - // if no destination file is provided the results will be written to stdout - OutputStream out; - String exportFile = props.getProperty("exportfile"); - if (exportFile == null) - { - out = System.out; - } else - { - out = new FileOutputStream(exportFile); - } - - // if no exporter is provided the default text one will be used - String exporterStr = props.getProperty("exporter", "com.yahoo.ycsb.measurements.exporter.TextMeasurementsExporter"); - try - { - exporter = (MeasurementsExporter) Class.forName(exporterStr).getConstructor(OutputStream.class).newInstance(out); - } catch (Exception e) - { - System.err.println("Could not find exporter " + exporterStr - + ", will use default text reporter."); - e.printStackTrace(); - exporter = new TextMeasurementsExporter(out); - } - - exporter.write("OVERALL", "RunTime(ms)", runtime); - double throughput = 1000.0 * ((double) opcount) / ((double) runtime); - exporter.write("OVERALL", "Throughput(ops/sec)", throughput); - - Measurements.getMeasurements().exportMeasurements(exporter); - } finally - { - if (exporter != null) - { - exporter.close(); - } - } - } - - @SuppressWarnings("unchecked") - public static void main(String[] args) - { - String dbname; - Properties props=new Properties(); - Properties fileprops=new Properties(); - boolean dotransactions=true; - int threadcount=1; - int target=0; - boolean status=false; - String label=""; - - //parse arguments - int argindex=0; - - if (args.length==0) - { - usageMessage(); - System.exit(0); - } - - while (args[argindex].startsWith("-")) - { - if (args[argindex].compareTo("-threads")==0) - { - argindex++; - if (argindex>=args.length) - { - usageMessage(); - System.exit(0); - } - int tcount=Integer.parseInt(args[argindex]); - props.setProperty("threadcount", tcount+""); - argindex++; - } - else if (args[argindex].compareTo("-target")==0) - { - argindex++; - if (argindex>=args.length) - { - usageMessage(); - System.exit(0); - } - int ttarget=Integer.parseInt(args[argindex]); - props.setProperty("target", ttarget+""); - argindex++; - } - else if (args[argindex].compareTo("-load")==0) - { - dotransactions=false; - argindex++; - } - else if (args[argindex].compareTo("-t")==0) - { - dotransactions=true; - argindex++; - } - else if (args[argindex].compareTo("-s")==0) - { - status=true; - argindex++; - } - else if (args[argindex].compareTo("-db")==0) - { - argindex++; - if (argindex>=args.length) - { - usageMessage(); - System.exit(0); - } - props.setProperty("db",args[argindex]); - argindex++; - } - else if (args[argindex].compareTo("-l")==0) - { - argindex++; - if (argindex>=args.length) - { - usageMessage(); - System.exit(0); - } - label=args[argindex]; - argindex++; - } - else if (args[argindex].compareTo("-P")==0) - { - argindex++; - if (argindex>=args.length) - { - usageMessage(); - System.exit(0); - } - String propfile=args[argindex]; - argindex++; - - Properties myfileprops=new Properties(); - try - { - myfileprops.load(new FileInputStream(propfile)); - } - catch (IOException e) - { - System.out.println(e.getMessage()); - System.exit(0); - } - - //Issue #5 - remove call to stringPropertyNames to make compilable under Java 1.5 - for (Enumeration e=myfileprops.propertyNames(); e.hasMoreElements(); ) - { - String prop=(String)e.nextElement(); - - fileprops.setProperty(prop,myfileprops.getProperty(prop)); - } - - } - else if (args[argindex].compareTo("-p")==0) - { - argindex++; - if (argindex>=args.length) - { - usageMessage(); - System.exit(0); - } - int eq=args[argindex].indexOf('='); - if (eq<0) - { - usageMessage(); - System.exit(0); - } - - String name=args[argindex].substring(0,eq); - String value=args[argindex].substring(eq+1); - props.put(name,value); - //System.out.println("["+name+"]=["+value+"]"); - argindex++; - } - else - { - System.out.println("Unknown option "+args[argindex]); - usageMessage(); - System.exit(0); - } - - if (argindex>=args.length) - { - break; - } - } - - if (argindex!=args.length) - { - usageMessage(); - System.exit(0); - } - - //set up logging - //BasicConfigurator.configure(); - - //overwrite file properties with properties from the command line - - //Issue #5 - remove call to stringPropertyNames to make compilable under Java 1.5 - for (Enumeration e=props.propertyNames(); e.hasMoreElements(); ) - { - String prop=(String)e.nextElement(); - - fileprops.setProperty(prop,props.getProperty(prop)); - } - - props=fileprops; - - if (!checkRequiredProperties(props)) - { - System.exit(0); - } - - long maxExecutionTime = Integer.parseInt(props.getProperty(MAX_EXECUTION_TIME, "0")); - - //get number of threads, target and db - threadcount=Integer.parseInt(props.getProperty("threadcount","1")); - dbname=props.getProperty("db","com.yahoo.ycsb.BasicDB"); - target=Integer.parseInt(props.getProperty("target","0")); - - //compute the target throughput - double targetperthreadperms=-1; - if (target>0) - { - double targetperthread=((double)target)/((double)threadcount); - targetperthreadperms=targetperthread/1000.0; - } - - System.out.println("YCSB Client 0.1"); - System.out.print("Command line:"); - for (int i=0; i threads=new Vector(); - - for (int threadid=0; threadid 0) { - terminator = new TerminatorThread(maxExecutionTime, threads, workload); - terminator.start(); - } - - int opsDone = 0; - - for (Thread t : threads) - { - try - { - t.join(); - opsDone += ((ClientThread)t).getOpsDone(); - } - catch (InterruptedException e) - { - } - } - - long en=System.currentTimeMillis(); - - if (terminator != null && !terminator.isInterrupted()) { - terminator.interrupt(); - } - - if (status) - { - statusthread.interrupt(); - } - - try - { - workload.cleanup(); - } - catch (WorkloadException e) - { - e.printStackTrace(); - e.printStackTrace(System.out); - System.exit(0); - } - - try - { - exportMeasurements(props, opsDone, en - st); - } catch (IOException e) - { - System.err.println("Could not export measurements, error: " + e.getMessage()); - e.printStackTrace(); - System.exit(-1); - } - - System.exit(0); - } -} diff --git a/src/com/yahoo/ycsb/CommandLine.java b/src/com/yahoo/ycsb/CommandLine.java deleted file mode 100644 index c177426b..00000000 --- a/src/com/yahoo/ycsb/CommandLine.java +++ /dev/null @@ -1,410 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package com.yahoo.ycsb; - -import java.util.Properties; -import java.io.FileInputStream; -import java.io.IOException; -import java.util.Enumeration; -import java.io.BufferedReader; -import java.io.InputStreamReader; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import java.util.HashSet; -import java.util.Vector; - -import com.yahoo.ycsb.workloads.*; - -/** - * A simple command line client to a database, using the appropriate com.yahoo.ycsb.DB implementation. - */ -public class CommandLine -{ - public static final String DEFAULT_DB="com.yahoo.ycsb.BasicDB"; - - public static void usageMessage() - { - System.out.println("YCSB Command Line Client"); - System.out.println("Usage: java com.yahoo.ycsb.CommandLine [options]"); - System.out.println("Options:"); - System.out.println(" -P filename: Specify a property file"); - System.out.println(" -p name=value: Specify a property value"); - System.out.println(" -db classname: Use a specified DB class (can also set the \"db\" property)"); - System.out.println(" -table tablename: Use the table name instead of the default \""+CoreWorkload.TABLENAME_PROPERTY_DEFAULT+"\""); - System.out.println(); - } - - public static void help() - { - System.out.println("Commands:"); - System.out.println(" read key [field1 field2 ...] - Read a record"); - System.out.println(" scan key recordcount [field1 field2 ...] - Scan starting at key"); - System.out.println(" insert key name1=value1 [name2=value2 ...] - Insert a new record"); - System.out.println(" update key name1=value1 [name2=value2 ...] - Update a record"); - System.out.println(" delete key - Delete a record"); - System.out.println(" table [tablename] - Get or [set] the name of the table"); - System.out.println(" quit - Quit"); - } - - public static void main(String[] args) - { - int argindex=0; - - Properties props=new Properties(); - Properties fileprops=new Properties(); - String table=CoreWorkload.TABLENAME_PROPERTY_DEFAULT; - - while ( (argindex=args.length) - { - usageMessage(); - System.exit(0); - } - props.setProperty("db",args[argindex]); - argindex++; - } - else if (args[argindex].compareTo("-P")==0) - { - argindex++; - if (argindex>=args.length) - { - usageMessage(); - System.exit(0); - } - String propfile=args[argindex]; - argindex++; - - Properties myfileprops=new Properties(); - try - { - myfileprops.load(new FileInputStream(propfile)); - } - catch (IOException e) - { - System.out.println(e.getMessage()); - System.exit(0); - } - - for (Enumeration e=myfileprops.propertyNames(); e.hasMoreElements(); ) - { - String prop=(String)e.nextElement(); - - fileprops.setProperty(prop,myfileprops.getProperty(prop)); - } - - } - else if (args[argindex].compareTo("-p")==0) - { - argindex++; - if (argindex>=args.length) - { - usageMessage(); - System.exit(0); - } - int eq=args[argindex].indexOf('='); - if (eq<0) - { - usageMessage(); - System.exit(0); - } - - String name=args[argindex].substring(0,eq); - String value=args[argindex].substring(eq+1); - props.put(name,value); - //System.out.println("["+name+"]=["+value+"]"); - argindex++; - } - else if (args[argindex].compareTo("-table")==0) - { - argindex++; - if (argindex>=args.length) - { - usageMessage(); - System.exit(0); - } - table=args[argindex]; - argindex++; - } - else - { - System.out.println("Unknown option "+args[argindex]); - usageMessage(); - System.exit(0); - } - - if (argindex>=args.length) - { - break; - } - } - - if (argindex!=args.length) - { - usageMessage(); - System.exit(0); - } - - for (Enumeration e=props.propertyNames(); e.hasMoreElements(); ) - { - String prop=(String)e.nextElement(); - - fileprops.setProperty(prop,props.getProperty(prop)); - } - - props=fileprops; - - System.out.println("YCSB Command Line client"); - System.out.println("Type \"help\" for command line help"); - System.out.println("Start with \"-help\" for usage info"); - - //create a DB - String dbname=props.getProperty("db",DEFAULT_DB); - - ClassLoader classLoader = CommandLine.class.getClassLoader(); - - DB db=null; - - try - { - Class dbclass = classLoader.loadClass(dbname); - db=(DB)dbclass.newInstance(); - } - catch (Exception e) - { - e.printStackTrace(); - System.exit(0); - } - - db.setProperties(props); - try - { - db.init(); - } - catch (DBException e) - { - e.printStackTrace(); - System.exit(0); - } - - System.out.println("Connected."); - - //main loop - BufferedReader br = new BufferedReader(new InputStreamReader(System.in)); - - for (;;) - { - //get user input - System.out.print("> "); - - String input=null; - - try - { - input=br.readLine(); - } - catch (IOException e) - { - e.printStackTrace(); - System.exit(1); - } - - if (input.compareTo("")==0) - { - continue; - } - - if (input.compareTo("help")==0) - { - help(); - continue; - } - - if (input.compareTo("quit")==0) - { - break; - } - - String[] tokens=input.split(" "); - - long st=System.currentTimeMillis(); - //handle commands - if (tokens[0].compareTo("table")==0) - { - if (tokens.length==1) - { - System.out.println("Using table \""+table+"\""); - } - else if (tokens.length==2) - { - table=tokens[1]; - System.out.println("Using table \""+table+"\""); - } - else - { - System.out.println("Error: syntax is \"table tablename\""); - } - } - else if (tokens[0].compareTo("read")==0) - { - if (tokens.length==1) - { - System.out.println("Error: syntax is \"read keyname [field1 field2 ...]\""); - } - else - { - Set fields=null; - - if (tokens.length>2) - { - fields=new HashSet(); - - for (int i=2; i result=new HashMap(); - int ret=db.read(table,tokens[1],fields,result); - System.out.println("Return code: "+ret); - for (Map.Entry ent : result.entrySet()) - { - System.out.println(ent.getKey()+"="+ent.getValue()); - } - } - } - else if (tokens[0].compareTo("scan")==0) - { - if (tokens.length<3) - { - System.out.println("Error: syntax is \"scan keyname scanlength [field1 field2 ...]\""); - } - else - { - Set fields=null; - - if (tokens.length>3) - { - fields=new HashSet(); - - for (int i=3; i> results=new Vector>(); - int ret=db.scan(table,tokens[1],Integer.parseInt(tokens[2]),fields,results); - System.out.println("Return code: "+ret); - int record=0; - if (results.size()==0) - { - System.out.println("0 records"); - } - else - { - System.out.println("--------------------------------"); - } - for (HashMap result : results) - { - System.out.println("Record "+(record++)); - for (Map.Entry ent : result.entrySet()) - { - System.out.println(ent.getKey()+"="+ent.getValue()); - } - System.out.println("--------------------------------"); - } - } - } - else if (tokens[0].compareTo("update")==0) - { - if (tokens.length<3) - { - System.out.println("Error: syntax is \"update keyname name1=value1 [name2=value2 ...]\""); - } - else - { - HashMap values=new HashMap(); - - for (int i=2; i values=new HashMap(); - - for (int i=2; i fields, HashMap result); - - /** - * Perform a range scan for a set of records in the database. Each field/value pair from the result will be stored in a HashMap. - * - * @param table The name of the table - * @param startkey The record key of the first record to read. - * @param recordcount The number of records to read - * @param fields The list of fields to read, or null for all of them - * @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record - * @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes. - */ - public abstract int scan(String table, String startkey, int recordcount, Set fields, Vector> result); - - /** - * Update a record in the database. Any field/value pairs in the specified values HashMap will be written into the record with the specified - * record key, overwriting any existing values with the same field name. - * - * @param table The name of the table - * @param key The record key of the record to write. - * @param values A HashMap of field/value pairs to update in the record - * @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes. - */ - public abstract int update(String table, String key, HashMap values); - - /** - * Insert a record in the database. Any field/value pairs in the specified values HashMap will be written into the record with the specified - * record key. - * - * @param table The name of the table - * @param key The record key of the record to insert. - * @param values A HashMap of field/value pairs to insert in the record - * @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes. - */ - public abstract int insert(String table, String key, HashMap values); - - /** - * Delete a record from the database. - * - * @param table The name of the table - * @param key The record key of the record to delete. - * @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes. - */ - public abstract int delete(String table, String key); -} diff --git a/src/com/yahoo/ycsb/DBException.java b/src/com/yahoo/ycsb/DBException.java deleted file mode 100644 index c598938a..00000000 --- a/src/com/yahoo/ycsb/DBException.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package com.yahoo.ycsb; - -/** - * Something bad happened while interacting with the database. - */ -public class DBException extends Exception -{ - /** - * - */ - private static final long serialVersionUID = 6646883591588721475L; - - public DBException(String message) - { - super(message); - } - - public DBException() - { - super(); - } - - public DBException(String message, Throwable cause) - { - super(message,cause); - } - - public DBException(Throwable cause) - { - super(cause); - } - -} diff --git a/src/com/yahoo/ycsb/DBFactory.java b/src/com/yahoo/ycsb/DBFactory.java deleted file mode 100644 index 18f7f5e1..00000000 --- a/src/com/yahoo/ycsb/DBFactory.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package com.yahoo.ycsb; - -import java.util.Properties; - -/** - * Creates a DB layer by dynamically classloading the specified DB class. - */ -public class DBFactory -{ - @SuppressWarnings("unchecked") - public static DB newDB(String dbname, Properties properties) throws UnknownDBException - { - ClassLoader classLoader = DBFactory.class.getClassLoader(); - - DB ret=null; - - try - { - Class dbclass = classLoader.loadClass(dbname); - //System.out.println("dbclass.getName() = " + dbclass.getName()); - - ret=(DB)dbclass.newInstance(); - } - catch (Exception e) - { - e.printStackTrace(); - return null; - } - - ret.setProperties(properties); - - return new DBWrapper(ret); - } - -} diff --git a/src/com/yahoo/ycsb/DBWrapper.java b/src/com/yahoo/ycsb/DBWrapper.java deleted file mode 100644 index 4516e7b0..00000000 --- a/src/com/yahoo/ycsb/DBWrapper.java +++ /dev/null @@ -1,168 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package com.yahoo.ycsb; - -import java.util.HashMap; -import java.util.Properties; -import java.util.Set; -import java.util.Vector; - -import com.yahoo.ycsb.measurements.Measurements; - -/** - * Wrapper around a "real" DB that measures latencies and counts return codes. - */ -public class DBWrapper extends DB -{ - DB _db; - Measurements _measurements; - - public DBWrapper(DB db) - { - _db=db; - _measurements=Measurements.getMeasurements(); - } - - /** - * Set the properties for this DB. - */ - public void setProperties(Properties p) - { - _db.setProperties(p); - } - - /** - * Get the set of properties for this DB. - */ - public Properties getProperties() - { - return _db.getProperties(); - } - - /** - * Initialize any state for this DB. - * Called once per DB instance; there is one DB instance per client thread. - */ - public void init() throws DBException - { - _db.init(); - } - - /** - * Cleanup any state for this DB. - * Called once per DB instance; there is one DB instance per client thread. - */ - public void cleanup() throws DBException - { - _db.cleanup(); - } - - /** - * Read a record from the database. Each field/value pair from the result will be stored in a HashMap. - * - * @param table The name of the table - * @param key The record key of the record to read. - * @param fields The list of fields to read, or null for all of them - * @param result A HashMap of field/value pairs for the result - * @return Zero on success, a non-zero error code on error - */ - public int read(String table, String key, Set fields, HashMap result) - { - long st=System.nanoTime(); - int res=_db.read(table,key,fields,result); - long en=System.nanoTime(); - _measurements.measure("READ",(int)((en-st)/1000)); - _measurements.reportReturnCode("READ",res); - return res; - } - - /** - * Perform a range scan for a set of records in the database. Each field/value pair from the result will be stored in a HashMap. - * - * @param table The name of the table - * @param startkey The record key of the first record to read. - * @param recordcount The number of records to read - * @param fields The list of fields to read, or null for all of them - * @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record - * @return Zero on success, a non-zero error code on error - */ - public int scan(String table, String startkey, int recordcount, Set fields, Vector> result) - { - long st=System.nanoTime(); - int res=_db.scan(table,startkey,recordcount,fields,result); - long en=System.nanoTime(); - _measurements.measure("SCAN",(int)((en-st)/1000)); - _measurements.reportReturnCode("SCAN",res); - return res; - } - - /** - * Update a record in the database. Any field/value pairs in the specified values HashMap will be written into the record with the specified - * record key, overwriting any existing values with the same field name. - * - * @param table The name of the table - * @param key The record key of the record to write. - * @param values A HashMap of field/value pairs to update in the record - * @return Zero on success, a non-zero error code on error - */ - public int update(String table, String key, HashMap values) - { - long st=System.nanoTime(); - int res=_db.update(table,key,values); - long en=System.nanoTime(); - _measurements.measure("UPDATE",(int)((en-st)/1000)); - _measurements.reportReturnCode("UPDATE",res); - return res; - } - - /** - * Insert a record in the database. Any field/value pairs in the specified values HashMap will be written into the record with the specified - * record key. - * - * @param table The name of the table - * @param key The record key of the record to insert. - * @param values A HashMap of field/value pairs to insert in the record - * @return Zero on success, a non-zero error code on error - */ - public int insert(String table, String key, HashMap values) - { - long st=System.nanoTime(); - int res=_db.insert(table,key,values); - long en=System.nanoTime(); - _measurements.measure("INSERT",(int)((en-st)/1000)); - _measurements.reportReturnCode("INSERT",res); - return res; - } - - /** - * Delete a record from the database. - * - * @param table The name of the table - * @param key The record key of the record to delete. - * @return Zero on success, a non-zero error code on error - */ - public int delete(String table, String key) - { - long st=System.nanoTime(); - int res=_db.delete(table,key); - long en=System.nanoTime(); - _measurements.measure("DELETE",(int)((en-st)/1000)); - _measurements.reportReturnCode("DELETE",res); - return res; - } -} diff --git a/src/com/yahoo/ycsb/InputStreamByteIterator.java b/src/com/yahoo/ycsb/InputStreamByteIterator.java deleted file mode 100644 index 5e90496c..00000000 --- a/src/com/yahoo/ycsb/InputStreamByteIterator.java +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ -package com.yahoo.ycsb; - -import java.io.InputStream; - -public class InputStreamByteIterator extends ByteIterator { - long len; - InputStream ins; - long off; - - public InputStreamByteIterator(InputStream ins, long len) { - this.len = len; - this.ins = ins; - off = 0; - } - - @Override - public boolean hasNext() { - return off < len; - } - - @Override - public byte nextByte() { - int ret; - try { - ret = ins.read(); - } catch(Exception e) { - throw new IllegalStateException(e); - } - if(ret == -1) { throw new IllegalStateException("Past EOF!"); } - off++; - return (byte)ret; - } - - @Override - public long bytesLeft() { - return len - off; - } - -} diff --git a/src/com/yahoo/ycsb/RandomByteIterator.java b/src/com/yahoo/ycsb/RandomByteIterator.java deleted file mode 100644 index 99239f85..00000000 --- a/src/com/yahoo/ycsb/RandomByteIterator.java +++ /dev/null @@ -1,84 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ -package com.yahoo.ycsb; - -public class RandomByteIterator extends ByteIterator { - long len; - long off; - int buf_off; - byte[] buf; - - @Override - public boolean hasNext() { - return (off + buf_off) < len; - } - - private void fillBytesImpl(byte[] buf, int base) { - int bytes = Utils.random().nextInt(); - try { - buf[base+0] = (byte)(((bytes ) & 31) + ' '); - buf[base+1] = (byte)(((bytes >> 5 ) & 31) + ' '); - buf[base+2] = (byte)(((bytes >> 10) & 31) + ' '); - buf[base+3] = (byte)(((bytes >> 15) & 31) + ' '); - buf[base+4] = (byte)(((bytes >> 20) & 31) + ' '); - buf[base+5] = (byte)(((bytes >> 25) & 31) + ' '); - } catch (ArrayIndexOutOfBoundsException e) { /* ignore it */ } - } - - private void fillBytes() { - if(buf_off == buf.length) { - fillBytesImpl(buf, 0); - buf_off = 0; - off += buf.length; - } - } - - public RandomByteIterator(long len) { - this.len = len; - this.buf = new byte[6]; - this.buf_off = buf.length; - fillBytes(); - this.off = 0; - } - - public byte nextByte() { - fillBytes(); - buf_off++; - return buf[buf_off-1]; - } - @Override - public int nextBuf(byte[] b, int buf_off) { - int ret; - if(len - off < b.length - buf_off) { - ret = (int)(len - off); - } else { - ret = b.length - buf_off; - } - int i; - for(i = 0; i < ret; i+=6) { - fillBytesImpl(b, i+buf_off); - } - off+=ret; - return ret + buf_off; - } - - - @Override - public long bytesLeft() { - return len - off; - } -} diff --git a/src/com/yahoo/ycsb/StringByteIterator.java b/src/com/yahoo/ycsb/StringByteIterator.java deleted file mode 100644 index 84465cba..00000000 --- a/src/com/yahoo/ycsb/StringByteIterator.java +++ /dev/null @@ -1,108 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package com.yahoo.ycsb; - -import java.util.Map; -import java.util.HashMap; - -public class StringByteIterator extends ByteIterator { - String str; - int off; - - /** - * Put all of the entries of one map into the other, converting - * String values into ByteIterators. - */ - public static void putAllAsByteIterators(Map out, Map in) { - for(String s: in.keySet()) { out.put(s, new StringByteIterator(in.get(s))); } - } - - /** - * Put all of the entries of one map into the other, converting - * ByteIterator values into Strings. - */ - public static void putAllAsStrings(Map out, Map in) { - for(String s: in.keySet()) { out.put(s, in.get(s).toString()); } - } - - /** - * Create a copy of a map, converting the values from Strings to - * StringByteIterators. - */ - public static HashMap getByteIteratorMap(Map m) { - HashMap ret = - new HashMap(); - - for(String s: m.keySet()) { - ret.put(s, new StringByteIterator(m.get(s))); - } - return ret; - } - - /** - * Create a copy of a map, converting the values from - * StringByteIterators to Strings. - */ - public static HashMap getStringMap(Map m) { - HashMap ret = new HashMap(); - - for(String s: m.keySet()) { - ret.put(s, m.get(s).toString());; - } - return ret; - } - - public StringByteIterator(String s) { - this.str = s; - this.off = 0; - } - @Override - public boolean hasNext() { - return off < str.length(); - } - - @Override - public byte nextByte() { - byte ret = (byte)str.charAt(off); - off++; - return ret; - } - - @Override - public long bytesLeft() { - return str.length() - off; - } - - /** - * Specialization of general purpose toString() to avoid unnecessary - * copies. - *

- * Creating a new StringByteIterator, then calling toString() - * yields the original String object, and does not perform any copies - * or String conversion operations. - *

- */ - @Override - public String toString() { - if(off > 0) { - return super.toString(); - } else { - return str; - } - } -} diff --git a/src/com/yahoo/ycsb/TerminatorThread.java b/src/com/yahoo/ycsb/TerminatorThread.java deleted file mode 100644 index 51deab82..00000000 --- a/src/com/yahoo/ycsb/TerminatorThread.java +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Copyright (c) 2011 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ -package com.yahoo.ycsb; - -import java.util.Vector; - -/** - * A thread that waits for the maximum specified time and then interrupts all the client - * threads passed as the Vector at initialization of this thread. - * - * The maximum execution time passed is assumed to be in seconds. - * - * @author sudipto - * - */ -public class TerminatorThread extends Thread { - - private Vector threads; - private long maxExecutionTime; - private Workload workload; - private long waitTimeOutInMS; - - public TerminatorThread(long maxExecutionTime, Vector threads, - Workload workload) { - this.maxExecutionTime = maxExecutionTime; - this.threads = threads; - this.workload = workload; - waitTimeOutInMS = 2000; - System.err.println("Maximum execution time specified as: " + maxExecutionTime + " secs"); - } - - public void run() { - try { - Thread.sleep(maxExecutionTime * 1000); - } catch (InterruptedException e) { - System.err.println("Could not wait until max specified time, TerminatorThread interrupted."); - return; - } - System.err.println("Maximum time elapsed. Requesting stop for the workload."); - workload.requestStop(); - System.err.println("Stop requested for workload. Now Joining!"); - for (Thread t : threads) { - while (t.isAlive()) { - try { - t.join(waitTimeOutInMS); - if (t.isAlive()) { - System.err.println("Still waiting for thread " + t.getName() + " to complete. " + - "Workload status: " + workload.isStopRequested()); - } - } catch (InterruptedException e) { - // Do nothing. Don't know why I was interrupted. - } - } - } - } -} diff --git a/src/com/yahoo/ycsb/UnknownDBException.java b/src/com/yahoo/ycsb/UnknownDBException.java deleted file mode 100644 index 9e20a351..00000000 --- a/src/com/yahoo/ycsb/UnknownDBException.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package com.yahoo.ycsb; - -/** - * Could not create the specified DB. - */ -public class UnknownDBException extends Exception -{ - /** - * - */ - private static final long serialVersionUID = 459099842269616836L; - - public UnknownDBException(String message) - { - super(message); - } - - public UnknownDBException() - { - super(); - } - - public UnknownDBException(String message, Throwable cause) - { - super(message,cause); - } - - public UnknownDBException(Throwable cause) - { - super(cause); - } - -} diff --git a/src/com/yahoo/ycsb/Utils.java b/src/com/yahoo/ycsb/Utils.java deleted file mode 100644 index f49bc0f5..00000000 --- a/src/com/yahoo/ycsb/Utils.java +++ /dev/null @@ -1,116 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package com.yahoo.ycsb; - -import java.util.Random; - -/** - * Utility functions. - */ -public class Utils -{ - private static final Random rand = new Random(); - private static final ThreadLocal rng = new ThreadLocal(); - - public static Random random() { - Random ret = rng.get(); - if(ret == null) { - ret = new Random(rand.nextLong()); - rng.set(ret); - } - return ret; - } - /** - * Generate a random ASCII string of a given length. - */ - public static String ASCIIString(int length) - { - int interval='~'-' '+1; - - byte []buf = new byte[length]; - random().nextBytes(buf); - for (int i = 0; i < length; i++) { - if (buf[i] < 0) { - buf[i] = (byte)((-buf[i] % interval) + ' '); - } else { - buf[i] = (byte)((buf[i] % interval) + ' '); - } - } - return new String(buf); - } - - /** - * Hash an integer value. - */ - public static long hash(long val) - { - return FNVhash64(val); - } - - public static final int FNV_offset_basis_32=0x811c9dc5; - public static final int FNV_prime_32=16777619; - - /** - * 32 bit FNV hash. Produces more "random" hashes than (say) String.hashCode(). - * - * @param val The value to hash. - * @return The hash value - */ - public static int FNVhash32(int val) - { - //from http://en.wikipedia.org/wiki/Fowler_Noll_Vo_hash - int hashval = FNV_offset_basis_32; - - for (int i=0; i<4; i++) - { - int octet=val&0x00ff; - val=val>>8; - - hashval = hashval ^ octet; - hashval = hashval * FNV_prime_32; - //hashval = hashval ^ octet; - } - return Math.abs(hashval); - } - - public static final long FNV_offset_basis_64=0xCBF29CE484222325L; - public static final long FNV_prime_64=1099511628211L; - - /** - * 64 bit FNV hash. Produces more "random" hashes than (say) String.hashCode(). - * - * @param val The value to hash. - * @return The hash value - */ - public static long FNVhash64(long val) - { - //from http://en.wikipedia.org/wiki/Fowler_Noll_Vo_hash - long hashval = FNV_offset_basis_64; - - for (int i=0; i<8; i++) - { - long octet=val&0x00ff; - val=val>>8; - - hashval = hashval ^ octet; - hashval = hashval * FNV_prime_64; - //hashval = hashval ^ octet; - } - return Math.abs(hashval); - } -} diff --git a/src/com/yahoo/ycsb/Workload.java b/src/com/yahoo/ycsb/Workload.java deleted file mode 100644 index 6b3e8ba7..00000000 --- a/src/com/yahoo/ycsb/Workload.java +++ /dev/null @@ -1,112 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package com.yahoo.ycsb; - -import java.util.Properties; -import java.util.concurrent.atomic.AtomicBoolean; - -/** - * One experiment scenario. One object of this type will - * be instantiated and shared among all client threads. This class - * should be constructed using a no-argument constructor, so we can - * load it dynamically. Any argument-based initialization should be - * done by init(). - * - * If you extend this class, you should support the "insertstart" property. This - * allows the load phase to proceed from multiple clients on different machines, in case - * the client is the bottleneck. For example, if we want to load 1 million records from - * 2 machines, the first machine should have insertstart=0 and the second insertstart=500000. Additionally, - * the "insertcount" property, which is interpreted by Client, can be used to tell each instance of the - * client how many inserts to do. In the example above, both clients should have insertcount=500000. - */ -public abstract class Workload -{ - public static final String INSERT_START_PROPERTY="insertstart"; - - public static final String INSERT_START_PROPERTY_DEFAULT="0"; - - private volatile AtomicBoolean stopRequested = new AtomicBoolean(false); - - /** - * Initialize the scenario. Create any generators and other shared objects here. - * Called once, in the main client thread, before any operations are started. - */ - public void init(Properties p) throws WorkloadException - { - } - - /** - * Initialize any state for a particular client thread. Since the scenario object - * will be shared among all threads, this is the place to create any state that is specific - * to one thread. To be clear, this means the returned object should be created anew on each - * call to initThread(); do not return the same object multiple times. - * The returned object will be passed to invocations of doInsert() and doTransaction() - * for this thread. There should be no side effects from this call; all state should be encapsulated - * in the returned object. If you have no state to retain for this thread, return null. (But if you have - * no state to retain for this thread, probably you don't need to override initThread().) - * - * @return false if the workload knows it is done for this thread. Client will terminate the thread. Return true otherwise. Return true for workloads that rely on operationcount. For workloads that read traces from a file, return true when there are more to do, false when you are done. - */ - public Object initThread(Properties p, int mythreadid, int threadcount) throws WorkloadException - { - return null; - } - - /** - * Cleanup the scenario. Called once, in the main client thread, after all operations have completed. - */ - public void cleanup() throws WorkloadException - { - } - - /** - * Do one insert operation. Because it will be called concurrently from multiple client threads, this - * function must be thread safe. However, avoid synchronized, or the threads will block waiting for each - * other, and it will be difficult to reach the target throughput. Ideally, this function would have no side - * effects other than DB operations and mutations on threadstate. Mutations to threadstate do not need to be - * synchronized, since each thread has its own threadstate instance. - */ - public abstract boolean doInsert(DB db, Object threadstate); - - /** - * Do one transaction operation. Because it will be called concurrently from multiple client threads, this - * function must be thread safe. However, avoid synchronized, or the threads will block waiting for each - * other, and it will be difficult to reach the target throughput. Ideally, this function would have no side - * effects other than DB operations and mutations on threadstate. Mutations to threadstate do not need to be - * synchronized, since each thread has its own threadstate instance. - * - * @return false if the workload knows it is done for this thread. Client will terminate the thread. Return true otherwise. Return true for workloads that rely on operationcount. For workloads that read traces from a file, return true when there are more to do, false when you are done. - */ - public abstract boolean doTransaction(DB db, Object threadstate); - - /** - * Allows scheduling a request to stop the workload. - */ - public void requestStop() { - stopRequested.set(true); - } - - /** - * Check the status of the stop request flag. - * @return true if stop was requested, false otherwise. - */ - public boolean isStopRequested() { - if (stopRequested.get() == true) return true; - else return false; - } -} diff --git a/src/com/yahoo/ycsb/WorkloadException.java b/src/com/yahoo/ycsb/WorkloadException.java deleted file mode 100644 index 0704881c..00000000 --- a/src/com/yahoo/ycsb/WorkloadException.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package com.yahoo.ycsb; - -/** - * The workload tried to do something bad. - */ -public class WorkloadException extends Exception -{ - /** - * - */ - private static final long serialVersionUID = 8844396756042772132L; - - public WorkloadException(String message) - { - super(message); - } - - public WorkloadException() - { - super(); - } - - public WorkloadException(String message, Throwable cause) - { - super(message,cause); - } - - public WorkloadException(Throwable cause) - { - super(cause); - } - -} diff --git a/src/com/yahoo/ycsb/generator/ConstantIntegerGenerator.java b/src/com/yahoo/ycsb/generator/ConstantIntegerGenerator.java deleted file mode 100644 index 94dd3a6a..00000000 --- a/src/com/yahoo/ycsb/generator/ConstantIntegerGenerator.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ -package com.yahoo.ycsb.generator; - -/** - * A trivial integer generator that always returns the same value. - * - * @author sears - * - */ -public class ConstantIntegerGenerator extends IntegerGenerator { - private final int i; - /** - * @param i The integer that this generator will always return. - */ - public ConstantIntegerGenerator(int i) { - this.i = i; - } - - @Override - public int nextInt() { - return i; - } - - @Override - public double mean() { - return i; - } - -} diff --git a/src/com/yahoo/ycsb/generator/CounterGenerator.java b/src/com/yahoo/ycsb/generator/CounterGenerator.java deleted file mode 100644 index df37b239..00000000 --- a/src/com/yahoo/ycsb/generator/CounterGenerator.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package com.yahoo.ycsb.generator; - -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Generates a sequence of integers 0, 1, ... - */ -public class CounterGenerator extends IntegerGenerator -{ - final AtomicInteger counter; - - /** - * Create a counter that starts at countstart - */ - public CounterGenerator(int countstart) - { - counter=new AtomicInteger(countstart); - setLastInt(counter.get()-1); - } - - /** - * If the generator returns numeric (integer) values, return the next value as an int. Default is to return -1, which - * is appropriate for generators that do not return numeric values. - */ - public int nextInt() - { - int ret = counter.getAndIncrement(); - setLastInt(ret); - return ret; - } - @Override - public int lastInt() - { - return counter.get() - 1; - } - @Override - public double mean() { - throw new UnsupportedOperationException("Can't compute mean of non-stationary distribution!"); - } -} diff --git a/src/com/yahoo/ycsb/generator/DiscreteGenerator.java b/src/com/yahoo/ycsb/generator/DiscreteGenerator.java deleted file mode 100644 index c28f3ae7..00000000 --- a/src/com/yahoo/ycsb/generator/DiscreteGenerator.java +++ /dev/null @@ -1,114 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package com.yahoo.ycsb.generator; - -import java.util.Vector; -import java.util.Random; - -import com.yahoo.ycsb.Utils; -import com.yahoo.ycsb.WorkloadException; - -/** - * Generates a distribution by choosing from a discrete set of values. - */ -public class DiscreteGenerator extends Generator -{ - class Pair - { - public double _weight; - public String _value; - - Pair(double weight, String value) - { - _weight=weight; - _value=value; - } - } - - Vector _values; - String _lastvalue; - - public DiscreteGenerator() - { - _values=new Vector(); - _lastvalue=null; - } - - /** - * Generate the next string in the distribution. - */ - public String nextString() - { - double sum=0; - - for (Pair p : _values) - { - sum+=p._weight; - } - - double val=Utils.random().nextDouble(); - - for (Pair p : _values) - { - if (val a = new ArrayList(); - - str = in.readLine(); - if(str == null) { - throw new IOException("Empty input file!\n"); - } - line = str.split("\t"); - if(line[0].compareTo("BlockSize") != 0) { - throw new IOException("First line of histogram is not the BlockSize!\n"); - } - block_size = Integer.parseInt(line[1]); - - while((str = in.readLine()) != null){ - // [0] is the bucket, [1] is the value - line = str.split("\t"); - - a.add(Integer.parseInt(line[0]), Integer.parseInt(line[1])); - } - buckets = new long[a.size()]; - for(int i = 0; i < a.size(); i++) { - buckets[i] = a.get(i); - } - - in.close(); - init(); - } - - public HistogramGenerator(long[] buckets, int block_size) { - this.block_size = block_size; - this.buckets = buckets; - init(); - } - private void init() { - for(int i = 0; i < buckets.length; i++) { - area += buckets[i]; - weighted_area = i * buckets[i]; - } - // calculate average file size - mean_size = ((double)block_size) * ((double)weighted_area) / (double)(area); - } - - @Override - public int nextInt() { - int number = Utils.random().nextInt((int)area); - int i; - - for(i = 0; i < (buckets.length - 1); i++){ - number -= buckets[i]; - if(number <= 0){ - return (int)((i+1)*block_size); - } - } - - return (int)(i * block_size); - } - - @Override - public double mean() { - return mean_size; - } -} diff --git a/src/com/yahoo/ycsb/generator/HotspotIntegerGenerator.java b/src/com/yahoo/ycsb/generator/HotspotIntegerGenerator.java deleted file mode 100644 index 969ae770..00000000 --- a/src/com/yahoo/ycsb/generator/HotspotIntegerGenerator.java +++ /dev/null @@ -1,124 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ -package com.yahoo.ycsb.generator; - -import java.util.Random; - -import com.yahoo.ycsb.Utils; - -/** - * Generate integers resembling a hotspot distribution where x% of operations - * access y% of data items. The parameters specify the bounds for the numbers, - * the percentage of the of the interval which comprises the hot set and - * the percentage of operations that access the hot set. Numbers of the hot set are - * always smaller than any number in the cold set. Elements from the hot set and - * the cold set are chose using a uniform distribution. - * - * @author sudipto - * - */ -public class HotspotIntegerGenerator extends IntegerGenerator { - - private final int lowerBound; - private final int upperBound; - private final int hotInterval; - private final int coldInterval; - private final double hotsetFraction; - private final double hotOpnFraction; - - /** - * Create a generator for Hotspot distributions. - * - * @param lowerBound lower bound of the distribution. - * @param upperBound upper bound of the distribution. - * @param hotsetFraction percentage of data item - * @param hotOpnFraction percentage of operations accessing the hot set. - */ - public HotspotIntegerGenerator(int lowerBound, int upperBound, - double hotsetFraction, double hotOpnFraction) { - if (hotsetFraction < 0.0 || hotsetFraction > 1.0) { - System.err.println("Hotset fraction out of range. Setting to 0.0"); - hotsetFraction = 0.0; - } - if (hotOpnFraction < 0.0 || hotOpnFraction > 1.0) { - System.err.println("Hot operation fraction out of range. Setting to 0.0"); - hotOpnFraction = 0.0; - } - if (lowerBound > upperBound) { - System.err.println("Upper bound of Hotspot generator smaller than the lower bound. " + - "Swapping the values."); - int temp = lowerBound; - lowerBound = upperBound; - upperBound = temp; - } - this.lowerBound = lowerBound; - this.upperBound = upperBound; - this.hotsetFraction = hotsetFraction; - int interval = upperBound - lowerBound + 1; - this.hotInterval = (int)(interval * hotsetFraction); - this.coldInterval = interval - hotInterval; - this.hotOpnFraction = hotOpnFraction; - } - - @Override - public int nextInt() { - int value = 0; - Random random = Utils.random(); - if (random.nextDouble() < hotOpnFraction) { - // Choose a value from the hot set. - value = lowerBound + random.nextInt(hotInterval); - } else { - // Choose a value from the cold set. - value = lowerBound + hotInterval + random.nextInt(coldInterval); - } - setLastInt(value); - return value; - } - - /** - * @return the lowerBound - */ - public int getLowerBound() { - return lowerBound; - } - - /** - * @return the upperBound - */ - public int getUpperBound() { - return upperBound; - } - - /** - * @return the hotsetFraction - */ - public double getHotsetFraction() { - return hotsetFraction; - } - - /** - * @return the hotOpnFraction - */ - public double getHotOpnFraction() { - return hotOpnFraction; - } - @Override - public double mean() { - return hotOpnFraction * (lowerBound + hotInterval/2.0) - + (1 - hotOpnFraction) * (lowerBound + hotInterval + coldInterval/2.0); - } -} diff --git a/src/com/yahoo/ycsb/generator/IntegerGenerator.java b/src/com/yahoo/ycsb/generator/IntegerGenerator.java deleted file mode 100644 index 13d36cd1..00000000 --- a/src/com/yahoo/ycsb/generator/IntegerGenerator.java +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package com.yahoo.ycsb.generator; - -/** - * A generator that is capable of generating ints as well as strings - * - * @author cooperb - * - */ -public abstract class IntegerGenerator extends Generator -{ - int lastint; - - /** - * Set the last value generated. IntegerGenerator subclasses must use this call - * to properly set the last string value, or the lastString() and lastInt() calls won't work. - */ - protected void setLastInt(int last) - { - lastint=last; - } - - /** - * Return the next value as an int. When overriding this method, be sure to call setLastString() properly, or the lastString() call won't work. - */ - public abstract int nextInt(); - - /** - * Generate the next string in the distribution. - */ - public String nextString() - { - return ""+nextInt(); - } - - /** - * Return the previous string generated by the distribution; e.g., returned from the last nextString() call. - * Calling lastString() should not advance the distribution or have any side effects. If nextString() has not yet - * been called, lastString() should return something reasonable. - */ - @Override - public String lastString() - { - return ""+lastInt(); - } - - /** - * Return the previous int generated by the distribution. This call is unique to IntegerGenerator subclasses, and assumes - * IntegerGenerator subclasses always return ints for nextInt() (e.g. not arbitrary strings). - */ - public int lastInt() - { - return lastint; - } - /** - * Return the expected value (mean) of the values this generator will return. - */ - public abstract double mean(); -} diff --git a/src/com/yahoo/ycsb/generator/ScrambledZipfianGenerator.java b/src/com/yahoo/ycsb/generator/ScrambledZipfianGenerator.java deleted file mode 100644 index 8b62c4c2..00000000 --- a/src/com/yahoo/ycsb/generator/ScrambledZipfianGenerator.java +++ /dev/null @@ -1,137 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package com.yahoo.ycsb.generator; - -import com.yahoo.ycsb.Utils; - -/** - * A generator of a zipfian distribution. It produces a sequence of items, such that some items are more popular than others, according - * to a zipfian distribution. When you construct an instance of this class, you specify the number of items in the set to draw from, either - * by specifying an itemcount (so that the sequence is of items from 0 to itemcount-1) or by specifying a min and a max (so that the sequence is of - * items from min to max inclusive). After you construct the instance, you can change the number of items by calling nextInt(itemcount) or nextLong(itemcount). - * - * Unlike @ZipfianGenerator, this class scatters the "popular" items across the itemspace. Use this, instead of @ZipfianGenerator, if you - * don't want the head of the distribution (the popular items) clustered together. - */ -public class ScrambledZipfianGenerator extends IntegerGenerator -{ - public static final double ZETAN=26.46902820178302; - public static final double USED_ZIPFIAN_CONSTANT=0.99; - public static final long ITEM_COUNT=10000000000L; - - ZipfianGenerator gen; - long _min,_max,_itemcount; - - /******************************* Constructors **************************************/ - - /** - * Create a zipfian generator for the specified number of items. - * @param _items The number of items in the distribution. - */ - public ScrambledZipfianGenerator(long _items) - { - this(0,_items-1); - } - - /** - * Create a zipfian generator for items between min and max. - * @param _min The smallest integer to generate in the sequence. - * @param _max The largest integer to generate in the sequence. - */ - public ScrambledZipfianGenerator(long _min, long _max) - { - this(_min,_max,ZipfianGenerator.ZIPFIAN_CONSTANT); - } - - /** - * Create a zipfian generator for the specified number of items using the specified zipfian constant. - * - * @param _items The number of items in the distribution. - * @param _zipfianconstant The zipfian constant to use. - */ - /* -// not supported, as the value of zeta depends on the zipfian constant, and we have only precomputed zeta for one zipfian constant - public ScrambledZipfianGenerator(long _items, double _zipfianconstant) - { - this(0,_items-1,_zipfianconstant); - } -*/ - - /** - * Create a zipfian generator for items between min and max (inclusive) for the specified zipfian constant. If you - * use a zipfian constant other than 0.99, this will take a long time to complete because we need to recompute zeta. - * @param min The smallest integer to generate in the sequence. - * @param max The largest integer to generate in the sequence. - * @param _zipfianconstant The zipfian constant to use. - */ - public ScrambledZipfianGenerator(long min, long max, double _zipfianconstant) - { - _min=min; - _max=max; - _itemcount=_max-_min+1; - if (_zipfianconstant == USED_ZIPFIAN_CONSTANT) - { - gen=new ZipfianGenerator(0,ITEM_COUNT,_zipfianconstant,ZETAN); - } else { - gen=new ZipfianGenerator(0,ITEM_COUNT,_zipfianconstant); - } - } - - /**************************************************************************************************/ - - /** - * Return the next int in the sequence. - */ - @Override - public int nextInt() { - return (int)nextLong(); - } - - /** - * Return the next long in the sequence. - */ - public long nextLong() - { - long ret=gen.nextLong(); - ret=_min+Utils.FNVhash64(ret)%_itemcount; - setLastInt((int)ret); - return ret; - } - - public static void main(String[] args) - { - double newzetan = ZipfianGenerator.zetastatic(ITEM_COUNT,ZipfianGenerator.ZIPFIAN_CONSTANT); - System.out.println("zetan: "+newzetan); - System.exit(0); - - ScrambledZipfianGenerator gen=new ScrambledZipfianGenerator(10000); - - for (int i=0; i<1000000; i++) - { - System.out.println(""+gen.nextInt()); - } - } - - /** - * since the values are scrambled (hopefully uniformly), the mean is simply the middle of the range. - */ - @Override - public double mean() { - return ((double)(((long)_min) +(long)_max))/2.0; - } -} diff --git a/src/com/yahoo/ycsb/generator/SkewedLatestGenerator.java b/src/com/yahoo/ycsb/generator/SkewedLatestGenerator.java deleted file mode 100644 index 376eb777..00000000 --- a/src/com/yahoo/ycsb/generator/SkewedLatestGenerator.java +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package com.yahoo.ycsb.generator; - -/** - * Generate a popularity distribution of items, skewed to favor recent items significantly more than older items. - */ -public class SkewedLatestGenerator extends IntegerGenerator -{ - CounterGenerator _basis; - ZipfianGenerator _zipfian; - - public SkewedLatestGenerator(CounterGenerator basis) - { - _basis=basis; - _zipfian=new ZipfianGenerator(_basis.lastInt()); - nextInt(); - } - - /** - * Generate the next string in the distribution, skewed Zipfian favoring the items most recently returned by the basis generator. - */ - public int nextInt() - { - int max=_basis.lastInt(); - int nextint=max-_zipfian.nextInt(max); - setLastInt(nextint); - return nextint; - } - - public static void main(String[] args) - { - SkewedLatestGenerator gen=new SkewedLatestGenerator(new CounterGenerator(1000)); - for (int i=0; i _values; - String _laststring; - UniformIntegerGenerator _gen; - - - /** - * Creates a generator that will return strings from the specified set uniformly randomly - */ - @SuppressWarnings( "unchecked" ) - public UniformGenerator(Vector values) - { - _values=(Vector)values.clone(); - _laststring=null; - _gen=new UniformIntegerGenerator(0,values.size()-1); - } - - /** - * Generate the next string in the distribution. - */ - public String nextString() - { - _laststring=_values.elementAt(_gen.nextInt()); - return _laststring; - } - - /** - * Return the previous string generated by the distribution; e.g., returned from the last nextString() call. - * Calling lastString() should not advance the distribution or have any side effects. If nextString() has not yet - * been called, lastString() should return something reasonable. - */ - public String lastString() - { - if (_laststring==null) - { - nextString(); - } - return _laststring; - } -} - diff --git a/src/com/yahoo/ycsb/generator/UniformIntegerGenerator.java b/src/com/yahoo/ycsb/generator/UniformIntegerGenerator.java deleted file mode 100644 index fa6af131..00000000 --- a/src/com/yahoo/ycsb/generator/UniformIntegerGenerator.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package com.yahoo.ycsb.generator; - -import java.util.Random; - -import com.yahoo.ycsb.Utils; - -/** - * Generates integers randomly uniform from an interval. - */ -public class UniformIntegerGenerator extends IntegerGenerator -{ - int _lb,_ub,_interval; - - /** - * Creates a generator that will return integers uniformly randomly from the interval [lb,ub] inclusive (that is, lb and ub are possible values) - * - * @param lb the lower bound (inclusive) of generated values - * @param ub the upper bound (inclusive) of generated values - */ - public UniformIntegerGenerator(int lb, int ub) - { - _lb=lb; - _ub=ub; - _interval=_ub-_lb+1; - } - - @Override - public int nextInt() - { - int ret=Utils.random().nextInt(_interval)+_lb; - setLastInt(ret); - - return ret; - } - - @Override - public double mean() { - return ((double)((long)(_lb + (long)_ub))) / 2.0; - } -} diff --git a/src/com/yahoo/ycsb/generator/ZipfianGenerator.java b/src/com/yahoo/ycsb/generator/ZipfianGenerator.java deleted file mode 100644 index 8a70f086..00000000 --- a/src/com/yahoo/ycsb/generator/ZipfianGenerator.java +++ /dev/null @@ -1,325 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package com.yahoo.ycsb.generator; - -import java.util.Random; - -import com.yahoo.ycsb.Utils; - -/** - * A generator of a zipfian distribution. It produces a sequence of items, such that some items are more popular than others, according - * to a zipfian distribution. When you construct an instance of this class, you specify the number of items in the set to draw from, either - * by specifying an itemcount (so that the sequence is of items from 0 to itemcount-1) or by specifying a min and a max (so that the sequence is of - * items from min to max inclusive). After you construct the instance, you can change the number of items by calling nextInt(itemcount) or nextLong(itemcount). - * - * Note that the popular items will be clustered together, e.g. item 0 is the most popular, item 1 the second most popular, and so on (or min is the most - * popular, min+1 the next most popular, etc.) If you don't want this clustering, and instead want the popular items scattered throughout the - * item space, then use ScrambledZipfianGenerator instead. - * - * Be aware: initializing this generator may take a long time if there are lots of items to choose from (e.g. over a minute - * for 100 million objects). This is because certain mathematical values need to be computed to properly generate a zipfian skew, and one of those - * values (zeta) is a sum sequence from 1 to n, where n is the itemcount. Note that if you increase the number of items in the set, we can compute - * a new zeta incrementally, so it should be fast unless you have added millions of items. However, if you decrease the number of items, we recompute - * zeta from scratch, so this can take a long time. - * - * The algorithm used here is from "Quickly Generating Billion-Record Synthetic Databases", Jim Gray et al, SIGMOD 1994. - */ -public class ZipfianGenerator extends IntegerGenerator -{ - public static final double ZIPFIAN_CONSTANT=0.99; - - /** - * Number of items. - */ - long items; - - /** - * Min item to generate. - */ - long base; - - /** - * The zipfian constant to use. - */ - double zipfianconstant; - - /** - * Computed parameters for generating the distribution. - */ - double alpha,zetan,eta,theta,zeta2theta; - - /** - * The number of items used to compute zetan the last time. - */ - long countforzeta; - - /** - * Flag to prevent problems. If you increase the number of items the zipfian generator is allowed to choose from, this code will incrementally compute a new zeta - * value for the larger itemcount. However, if you decrease the number of items, the code computes zeta from scratch; this is expensive for large itemsets. - * Usually this is not intentional; e.g. one thread thinks the number of items is 1001 and calls "nextLong()" with that item count; then another thread who thinks the - * number of items is 1000 calls nextLong() with itemcount=1000 triggering the expensive recomputation. (It is expensive for 100 million items, not really for 1000 items.) Why - * did the second thread think there were only 1000 items? maybe it read the item count before the first thread incremented it. So this flag allows you to say if you really do - * want that recomputation. If true, then the code will recompute zeta if the itemcount goes down. If false, the code will assume itemcount only goes up, and never recompute. - */ - boolean allowitemcountdecrease=false; - - /******************************* Constructors **************************************/ - - /** - * Create a zipfian generator for the specified number of items. - * @param _items The number of items in the distribution. - */ - public ZipfianGenerator(long _items) - { - this(0,_items-1); - } - - /** - * Create a zipfian generator for items between min and max. - * @param _min The smallest integer to generate in the sequence. - * @param _max The largest integer to generate in the sequence. - */ - public ZipfianGenerator(long _min, long _max) - { - this(_min,_max,ZIPFIAN_CONSTANT); - } - - /** - * Create a zipfian generator for the specified number of items using the specified zipfian constant. - * - * @param _items The number of items in the distribution. - * @param _zipfianconstant The zipfian constant to use. - */ - public ZipfianGenerator(long _items, double _zipfianconstant) - { - this(0,_items-1,_zipfianconstant); - } - - /** - * Create a zipfian generator for items between min and max (inclusive) for the specified zipfian constant. - * @param min The smallest integer to generate in the sequence. - * @param max The largest integer to generate in the sequence. - * @param _zipfianconstant The zipfian constant to use. - */ - public ZipfianGenerator(long min, long max, double _zipfianconstant) - { - this(min,max,_zipfianconstant,zetastatic(max-min+1,_zipfianconstant)); - } - - /** - * Create a zipfian generator for items between min and max (inclusive) for the specified zipfian constant, using the precomputed value of zeta. - * - * @param min The smallest integer to generate in the sequence. - * @param max The largest integer to generate in the sequence. - * @param _zipfianconstant The zipfian constant to use. - * @param _zetan The precomputed zeta constant. - */ - public ZipfianGenerator(long min, long max, double _zipfianconstant, double _zetan) - { - - items=max-min+1; - base=min; - zipfianconstant=_zipfianconstant; - - theta=zipfianconstant; - - zeta2theta=zeta(2,theta); - - - alpha=1.0/(1.0-theta); - //zetan=zeta(items,theta); - zetan=_zetan; - countforzeta=items; - eta=(1-Math.pow(2.0/items,1-theta))/(1-zeta2theta/zetan); - - //System.out.println("XXXX 3 XXXX"); - nextInt(); - //System.out.println("XXXX 4 XXXX"); - } - - /**************************************************************************/ - - /** - * Compute the zeta constant needed for the distribution. Do this from scratch for a distribution with n items, using the - * zipfian constant theta. Remember the value of n, so if we change the itemcount, we can recompute zeta. - * - * @param n The number of items to compute zeta over. - * @param theta The zipfian constant. - */ - double zeta(long n, double theta) - { - countforzeta=n; - return zetastatic(n,theta); - } - - /** - * Compute the zeta constant needed for the distribution. Do this from scratch for a distribution with n items, using the - * zipfian constant theta. This is a static version of the function which will not remember n. - * @param n The number of items to compute zeta over. - * @param theta The zipfian constant. - */ - static double zetastatic(long n, double theta) - { - return zetastatic(0,n,theta,0); - } - - /** - * Compute the zeta constant needed for the distribution. Do this incrementally for a distribution that - * has n items now but used to have st items. Use the zipfian constant theta. Remember the new value of - * n so that if we change the itemcount, we'll know to recompute zeta. - * - * @param st The number of items used to compute the last initialsum - * @param n The number of items to compute zeta over. - * @param theta The zipfian constant. - * @param initialsum The value of zeta we are computing incrementally from. - */ - double zeta(long st, long n, double theta, double initialsum) - { - countforzeta=n; - return zetastatic(st,n,theta,initialsum); - } - - /** - * Compute the zeta constant needed for the distribution. Do this incrementally for a distribution that - * has n items now but used to have st items. Use the zipfian constant theta. Remember the new value of - * n so that if we change the itemcount, we'll know to recompute zeta. - * @param st The number of items used to compute the last initialsum - * @param n The number of items to compute zeta over. - * @param theta The zipfian constant. - * @param initialsum The value of zeta we are computing incrementally from. - */ - static double zetastatic(long st, long n, double theta, double initialsum) - { - double sum=initialsum; - for (long i=st; icountforzeta) - { - //System.err.println("WARNING: Incrementally recomputing Zipfian distribtion. (itemcount="+itemcount+" countforzeta="+countforzeta+")"); - - //we have added more items. can compute zetan incrementally, which is cheaper - zetan=zeta(countforzeta,itemcount,theta,zetan); - eta=(1-Math.pow(2.0/items,1-theta))/(1-zeta2theta/zetan); - } - else if ( (itemcount data; - boolean histogram=true; - - private Properties _props; - - /** - * Create a new object with the specified properties. - */ - public Measurements(Properties props) - { - data=new HashMap(); - - _props=props; - - if (_props.getProperty(MEASUREMENT_TYPE, MEASUREMENT_TYPE_DEFAULT).compareTo("histogram")==0) - { - histogram=true; - } - else - { - histogram=false; - } - } - - OneMeasurement constructOneMeasurement(String name) - { - if (histogram) - { - return new OneMeasurementHistogram(name,_props); - } - else - { - return new OneMeasurementTimeSeries(name,_props); - } - } - - /** - * Report a single value of a single metric. E.g. for read latency, operation="READ" and latency is the measured value. - */ - public synchronized void measure(String operation, int latency) - { - if (!data.containsKey(operation)) - { - synchronized(this) - { - if (!data.containsKey(operation)) - { - data.put(operation,constructOneMeasurement(operation)); - } - } - } - try - { - data.get(operation).measure(latency); - } - catch (java.lang.ArrayIndexOutOfBoundsException e) - { - System.out.println("ERROR: java.lang.ArrayIndexOutOfBoundsException - ignoring and continuing"); - e.printStackTrace(); - e.printStackTrace(System.out); - } - } - - /** - * Report a return code for a single DB operaiton. - */ - public void reportReturnCode(String operation, int code) - { - if (!data.containsKey(operation)) - { - synchronized(this) - { - if (!data.containsKey(operation)) - { - data.put(operation,constructOneMeasurement(operation)); - } - } - } - data.get(operation).reportReturnCode(code); - } - - /** - * Export the current measurements to a suitable format. - * - * @param exporter Exporter representing the type of format to write to. - * @throws IOException Thrown if the export failed. - */ - public void exportMeasurements(MeasurementsExporter exporter) throws IOException - { - for (OneMeasurement measurement : data.values()) - { - measurement.exportMeasurements(exporter); - } - } - - /** - * Return a one line summary of the measurements. - */ - public String getSummary() - { - String ret=""; - for (OneMeasurement m : data.values()) - { - ret+=m.getSummary()+" "; - } - - return ret; - } -} diff --git a/src/com/yahoo/ycsb/measurements/OneMeasurement.java b/src/com/yahoo/ycsb/measurements/OneMeasurement.java deleted file mode 100644 index 90720ef4..00000000 --- a/src/com/yahoo/ycsb/measurements/OneMeasurement.java +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package com.yahoo.ycsb.measurements; - -import java.io.IOException; - -import com.yahoo.ycsb.measurements.exporter.MeasurementsExporter; - -/** - * A single measured metric (such as READ LATENCY) - */ -public abstract class OneMeasurement { - - String _name; - - public String getName() { - return _name; - } - - /** - * @param _name - */ - public OneMeasurement(String _name) { - this._name = _name; - } - - public abstract void reportReturnCode(int code); - - public abstract void measure(int latency); - - public abstract String getSummary(); - - /** - * Export the current measurements to a suitable format. - * - * @param exporter Exporter representing the type of format to write to. - * @throws IOException Thrown if the export failed. - */ - public abstract void exportMeasurements(MeasurementsExporter exporter) throws IOException; -} diff --git a/src/com/yahoo/ycsb/measurements/OneMeasurementHistogram.java b/src/com/yahoo/ycsb/measurements/OneMeasurementHistogram.java deleted file mode 100644 index 2cba955a..00000000 --- a/src/com/yahoo/ycsb/measurements/OneMeasurementHistogram.java +++ /dev/null @@ -1,165 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package com.yahoo.ycsb.measurements; - -import java.io.IOException; -import java.text.DecimalFormat; -import java.util.HashMap; -import java.util.Properties; - -import com.yahoo.ycsb.measurements.exporter.MeasurementsExporter; - - -/** - * Take measurements and maintain a histogram of a given metric, such as READ LATENCY. - * - * @author cooperb - * - */ -public class OneMeasurementHistogram extends OneMeasurement -{ - public static final String BUCKETS="histogram.buckets"; - public static final String BUCKETS_DEFAULT="1000"; - - int _buckets; - int[] histogram; - int histogramoverflow; - int operations; - long totallatency; - - //keep a windowed version of these stats for printing status - int windowoperations; - long windowtotallatency; - - int min; - int max; - HashMap returncodes; - - public OneMeasurementHistogram(String name, Properties props) - { - super(name); - _buckets=Integer.parseInt(props.getProperty(BUCKETS, BUCKETS_DEFAULT)); - histogram=new int[_buckets]; - histogramoverflow=0; - operations=0; - totallatency=0; - windowoperations=0; - windowtotallatency=0; - min=-1; - max=-1; - returncodes=new HashMap(); - } - - /* (non-Javadoc) - * @see com.yahoo.ycsb.OneMeasurement#reportReturnCode(int) - */ - public synchronized void reportReturnCode(int code) - { - Integer Icode=code; - if (!returncodes.containsKey(Icode)) - { - int[] val=new int[1]; - val[0]=0; - returncodes.put(Icode,val); - } - returncodes.get(Icode)[0]++; - } - - - /* (non-Javadoc) - * @see com.yahoo.ycsb.OneMeasurement#measure(int) - */ - public synchronized void measure(int latency) - { - if (latency/1000>=_buckets) - { - histogramoverflow++; - } - else - { - histogram[latency/1000]++; - } - operations++; - totallatency+=latency; - windowoperations++; - windowtotallatency+=latency; - - if ( (min<0) || (latencymax) ) - { - max=latency; - } - } - - - @Override - public void exportMeasurements(MeasurementsExporter exporter) throws IOException - { - exporter.write(getName(), "Operations", operations); - exporter.write(getName(), "AverageLatency(us)", (((double)totallatency)/((double)operations))); - exporter.write(getName(), "MinLatency(us)", min); - exporter.write(getName(), "MaxLatency(us)", max); - - int opcounter=0; - boolean done95th=false; - for (int i=0; i<_buckets; i++) - { - opcounter+=histogram[i]; - if ( (!done95th) && (((double)opcounter)/((double)operations)>=0.95) ) - { - exporter.write(getName(), "95thPercentileLatency(ms)", i); - done95th=true; - } - if (((double)opcounter)/((double)operations)>=0.99) - { - exporter.write(getName(), "99thPercentileLatency(ms)", i); - break; - } - } - - for (Integer I : returncodes.keySet()) - { - int[] val=returncodes.get(I); - exporter.write(getName(), "Return="+I, val[0]); - } - - for (int i=0; i<_buckets; i++) - { - exporter.write(getName(), Integer.toString(i), histogram[i]); - } - exporter.write(getName(), ">"+_buckets, histogramoverflow); - } - - @Override - public String getSummary() { - if (windowoperations==0) - { - return ""; - } - DecimalFormat d = new DecimalFormat("#.##"); - double report=((double)windowtotallatency)/((double)windowoperations); - windowtotallatency=0; - windowoperations=0; - return "["+getName()+" AverageLatency(us)="+d.format(report)+"]"; - } - -} diff --git a/src/com/yahoo/ycsb/measurements/OneMeasurementTimeSeries.java b/src/com/yahoo/ycsb/measurements/OneMeasurementTimeSeries.java deleted file mode 100644 index 25c77b76..00000000 --- a/src/com/yahoo/ycsb/measurements/OneMeasurementTimeSeries.java +++ /dev/null @@ -1,179 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package com.yahoo.ycsb.measurements; - -import java.io.IOException; -import java.text.DecimalFormat; -import java.util.HashMap; -import java.util.Properties; -import java.util.Vector; - -import com.yahoo.ycsb.measurements.exporter.MeasurementsExporter; - -class SeriesUnit -{ - /** - * @param time - * @param average - */ - public SeriesUnit(long time, double average) { - this.time = time; - this.average = average; - } - public long time; - public double average; -} - -/** - * A time series measurement of a metric, such as READ LATENCY. - */ -public class OneMeasurementTimeSeries extends OneMeasurement -{ - /** - * Granularity for time series; measurements will be averaged in chunks of this granularity. Units are milliseconds. - */ - public static final String GRANULARITY="timeseries.granularity"; - - public static final String GRANULARITY_DEFAULT="1000"; - - int _granularity; - Vector _measurements; - - long start=-1; - long currentunit=-1; - int count=0; - int sum=0; - int operations=0; - long totallatency=0; - - //keep a windowed version of these stats for printing status - int windowoperations=0; - long windowtotallatency=0; - - int min=-1; - int max=-1; - - private HashMap returncodes; - - public OneMeasurementTimeSeries(String name, Properties props) - { - super(name); - _granularity=Integer.parseInt(props.getProperty(GRANULARITY,GRANULARITY_DEFAULT)); - _measurements=new Vector(); - returncodes=new HashMap(); - } - - void checkEndOfUnit(boolean forceend) - { - long now=System.currentTimeMillis(); - - if (start<0) - { - currentunit=0; - start=now; - } - - long unit=((now-start)/_granularity)*_granularity; - - if ( (unit>currentunit) || (forceend) ) - { - double avg=((double)sum)/((double)count); - _measurements.add(new SeriesUnit(currentunit,avg)); - - currentunit=unit; - - count=0; - sum=0; - } - } - - @Override - public void measure(int latency) - { - checkEndOfUnit(false); - - count++; - sum+=latency; - totallatency+=latency; - operations++; - windowoperations++; - windowtotallatency+=latency; - - if (latency>max) - { - max=latency; - } - - if ( (latency - * Properties to control the client: - *

- *
    - *
  • disksize: how many bytes of storage can the disk store? (default 100,000,000) - *
  • occupancy: what fraction of the available storage should be used? (default 0.9) - *
  • requestdistribution: what distribution should be used to select the records to operate on - uniform, zipfian or latest (default: histogram) - *
- * - * - *

See also: - * Russell Sears, Catharine van Ingen. - * Fragmentation in Large Object Repositories, - * CIDR 2006. [Presentation] - *

- * - * - * @author sears - * - */ -public class ConstantOccupancyWorkload extends CoreWorkload { - long disksize; - long storageages; - IntegerGenerator objectsizes; - double occupancy; - - long object_count; - - public static final String STORAGE_AGE_PROPERTY = "storageages"; - public static final long STORAGE_AGE_PROPERTY_DEFAULT = 10; - - public static final String DISK_SIZE_PROPERTY = "disksize"; - public static final long DISK_SIZE_PROPERTY_DEFAULT = 100 * 1000 * 1000; - - public static final String OCCUPANCY_PROPERTY = "occupancy"; - public static final double OCCUPANCY_PROPERTY_DEFAULT = 0.9; - - @Override - public void init(Properties p) throws WorkloadException - { - disksize = Long.parseLong( p.getProperty(DISK_SIZE_PROPERTY, DISK_SIZE_PROPERTY_DEFAULT+"")); - storageages = Long.parseLong( p.getProperty(STORAGE_AGE_PROPERTY, STORAGE_AGE_PROPERTY_DEFAULT+"")); - occupancy = Double.parseDouble(p.getProperty(OCCUPANCY_PROPERTY, OCCUPANCY_PROPERTY_DEFAULT+"")); - - if(p.getProperty(Client.RECORD_COUNT_PROPERTY) != null || - p.getProperty(Client.INSERT_COUNT_PROPERTY) != null || - p.getProperty(Client.OPERATION_COUNT_PROPERTY) != null) { - System.err.println("Warning: record, insert or operation count was set prior to initting ConstantOccupancyWorkload. Overriding old values."); - } - IntegerGenerator g = CoreWorkload.getFieldLengthGenerator(p); - double fieldsize = g.mean(); - int fieldcount = Integer.parseInt(p.getProperty(FIELD_COUNT_PROPERTY, FIELD_COUNT_PROPERTY_DEFAULT)); - - object_count = (long)(occupancy * ((double)disksize / (fieldsize * (double)fieldcount))); - if(object_count == 0) { - throw new IllegalStateException("Object count was zero. Perhaps disksize is too low?"); - } - p.setProperty(Client.RECORD_COUNT_PROPERTY, object_count+""); - p.setProperty(Client.OPERATION_COUNT_PROPERTY, (storageages*object_count)+""); - p.setProperty(Client.INSERT_COUNT_PROPERTY, object_count+""); - - super.init(p); - } - -} diff --git a/src/com/yahoo/ycsb/workloads/CoreWorkload.java b/src/com/yahoo/ycsb/workloads/CoreWorkload.java deleted file mode 100644 index 2d178893..00000000 --- a/src/com/yahoo/ycsb/workloads/CoreWorkload.java +++ /dev/null @@ -1,641 +0,0 @@ -/** - * Copyright (c) 2010 Yahoo! Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package com.yahoo.ycsb.workloads; - -import java.util.Properties; -import com.yahoo.ycsb.*; -import com.yahoo.ycsb.generator.CounterGenerator; -import com.yahoo.ycsb.generator.DiscreteGenerator; -import com.yahoo.ycsb.generator.ExponentialGenerator; -import com.yahoo.ycsb.generator.Generator; -import com.yahoo.ycsb.generator.ConstantIntegerGenerator; -import com.yahoo.ycsb.generator.HotspotIntegerGenerator; -import com.yahoo.ycsb.generator.HistogramGenerator; -import com.yahoo.ycsb.generator.IntegerGenerator; -import com.yahoo.ycsb.generator.ScrambledZipfianGenerator; -import com.yahoo.ycsb.generator.SkewedLatestGenerator; -import com.yahoo.ycsb.generator.UniformIntegerGenerator; -import com.yahoo.ycsb.generator.ZipfianGenerator; -import com.yahoo.ycsb.measurements.Measurements; - -import java.io.IOException; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Vector; - -/** - * The core benchmark scenario. Represents a set of clients doing simple CRUD operations. The relative - * proportion of different kinds of operations, and other properties of the workload, are controlled - * by parameters specified at runtime. - * - * Properties to control the client: - *
    - *
  • fieldcount: the number of fields in a record (default: 10) - *
  • fieldlength: the size of each field (default: 100) - *
  • readallfields: should reads read all fields (true) or just one (false) (default: true) - *
  • writeallfields: should updates and read/modify/writes update all fields (true) or just one (false) (default: false) - *
  • readproportion: what proportion of operations should be reads (default: 0.95) - *
  • updateproportion: what proportion of operations should be updates (default: 0.05) - *
  • insertproportion: what proportion of operations should be inserts (default: 0) - *
  • scanproportion: what proportion of operations should be scans (default: 0) - *
  • readmodifywriteproportion: what proportion of operations should be read a record, modify it, write it back (default: 0) - *
  • requestdistribution: what distribution should be used to select the records to operate on - uniform, zipfian, hotspot, or latest (default: uniform) - *
  • maxscanlength: for scans, what is the maximum number of records to scan (default: 1000) - *
  • scanlengthdistribution: for scans, what distribution should be used to choose the number of records to scan, for each scan, between 1 and maxscanlength (default: uniform) - *
  • insertorder: should records be inserted in order by key ("ordered"), or in hashed order ("hashed") (default: hashed) - *
- */ -public class CoreWorkload extends Workload -{ - - /** - * The name of the database table to run queries against. - */ - public static final String TABLENAME_PROPERTY="table"; - - /** - * The default name of the database table to run queries against. - */ - public static final String TABLENAME_PROPERTY_DEFAULT="usertable"; - - public static String table; - - - /** - * The name of the property for the number of fields in a record. - */ - public static final String FIELD_COUNT_PROPERTY="fieldcount"; - - /** - * Default number of fields in a record. - */ - public static final String FIELD_COUNT_PROPERTY_DEFAULT="10"; - - int fieldcount; - - /** - * The name of the property for the field length distribution. Options are "uniform", "zipfian" (favoring short records), "constant", and "histogram". - * - * If "uniform", "zipfian" or "constant", the maximum field length will be that specified by the fieldlength property. If "histogram", then the - * histogram will be read from the filename specified in the "fieldlengthhistogram" property. - */ - public static final String FIELD_LENGTH_DISTRIBUTION_PROPERTY="fieldlengthdistribution"; - /** - * The default field length distribution. - */ - public static final String FIELD_LENGTH_DISTRIBUTION_PROPERTY_DEFAULT = "constant"; - - /** - * The name of the property for the length of a field in bytes. - */ - public static final String FIELD_LENGTH_PROPERTY="fieldlength"; - /** - * The default maximum length of a field in bytes. - */ - public static final String FIELD_LENGTH_PROPERTY_DEFAULT="100"; - - /** - * The name of a property that specifies the filename containing the field length histogram (only used if fieldlengthdistribution is "histogram"). - */ - public static final String FIELD_LENGTH_HISTOGRAM_FILE_PROPERTY = "fieldlengthhistogram"; - /** - * The default filename containing a field length histogram. - */ - public static final String FIELD_LENGTH_HISTOGRAM_FILE_PROPERTY_DEFAULT = "hist.txt"; - - /** - * Generator object that produces field lengths. The value of this depends on the properties that start with "FIELD_LENGTH_". - */ - IntegerGenerator fieldlengthgenerator; - - /** - * The name of the property for deciding whether to read one field (false) or all fields (true) of a record. - */ - public static final String READ_ALL_FIELDS_PROPERTY="readallfields"; - - /** - * The default value for the readallfields property. - */ - public static final String READ_ALL_FIELDS_PROPERTY_DEFAULT="true"; - - boolean readallfields; - - /** - * The name of the property for deciding whether to write one field (false) or all fields (true) of a record. - */ - public static final String WRITE_ALL_FIELDS_PROPERTY="writeallfields"; - - /** - * The default value for the writeallfields property. - */ - public static final String WRITE_ALL_FIELDS_PROPERTY_DEFAULT="false"; - - boolean writeallfields; - - - /** - * The name of the property for the proportion of transactions that are reads. - */ - public static final String READ_PROPORTION_PROPERTY="readproportion"; - - /** - * The default proportion of transactions that are reads. - */ - public static final String READ_PROPORTION_PROPERTY_DEFAULT="0.95"; - - /** - * The name of the property for the proportion of transactions that are updates. - */ - public static final String UPDATE_PROPORTION_PROPERTY="updateproportion"; - - /** - * The default proportion of transactions that are updates. - */ - public static final String UPDATE_PROPORTION_PROPERTY_DEFAULT="0.05"; - - /** - * The name of the property for the proportion of transactions that are inserts. - */ - public static final String INSERT_PROPORTION_PROPERTY="insertproportion"; - - /** - * The default proportion of transactions that are inserts. - */ - public static final String INSERT_PROPORTION_PROPERTY_DEFAULT="0.0"; - - /** - * The name of the property for the proportion of transactions that are scans. - */ - public static final String SCAN_PROPORTION_PROPERTY="scanproportion"; - - /** - * The default proportion of transactions that are scans. - */ - public static final String SCAN_PROPORTION_PROPERTY_DEFAULT="0.0"; - - /** - * The name of the property for the proportion of transactions that are read-modify-write. - */ - public static final String READMODIFYWRITE_PROPORTION_PROPERTY="readmodifywriteproportion"; - - /** - * The default proportion of transactions that are scans. - */ - public static final String READMODIFYWRITE_PROPORTION_PROPERTY_DEFAULT="0.0"; - - /** - * The name of the property for the the distribution of requests across the keyspace. Options are "uniform", "zipfian" and "latest" - */ - public static final String REQUEST_DISTRIBUTION_PROPERTY="requestdistribution"; - - /** - * The default distribution of requests across the keyspace - */ - public static final String REQUEST_DISTRIBUTION_PROPERTY_DEFAULT="uniform"; - - /** - * The name of the property for the max scan length (number of records) - */ - public static final String MAX_SCAN_LENGTH_PROPERTY="maxscanlength"; - - /** - * The default max scan length. - */ - public static final String MAX_SCAN_LENGTH_PROPERTY_DEFAULT="1000"; - - /** - * The name of the property for the scan length distribution. Options are "uniform" and "zipfian" (favoring short scans) - */ - public static final String SCAN_LENGTH_DISTRIBUTION_PROPERTY="scanlengthdistribution"; - - /** - * The default max scan length. - */ - public static final String SCAN_LENGTH_DISTRIBUTION_PROPERTY_DEFAULT="uniform"; - - /** - * The name of the property for the order to insert records. Options are "ordered" or "hashed" - */ - public static final String INSERT_ORDER_PROPERTY="insertorder"; - - /** - * Default insert order. - */ - public static final String INSERT_ORDER_PROPERTY_DEFAULT="hashed"; - - /** - * Percentage data items that constitute the hot set. - */ - public static final String HOTSPOT_DATA_FRACTION = "hotspotdatafraction"; - - /** - * Default value of the size of the hot set. - */ - public static final String HOTSPOT_DATA_FRACTION_DEFAULT = "0.2"; - - /** - * Percentage operations that access the hot set. - */ - public static final String HOTSPOT_OPN_FRACTION = "hotspotopnfraction"; - - /** - * Default value of the percentage operations accessing the hot set. - */ - public static final String HOTSPOT_OPN_FRACTION_DEFAULT = "0.8"; - - IntegerGenerator keysequence; - - DiscreteGenerator operationchooser; - - IntegerGenerator keychooser; - - Generator fieldchooser; - - CounterGenerator transactioninsertkeysequence; - - IntegerGenerator scanlength; - - boolean orderedinserts; - - int recordcount; - - protected static IntegerGenerator getFieldLengthGenerator(Properties p) throws WorkloadException{ - IntegerGenerator fieldlengthgenerator; - String fieldlengthdistribution = p.getProperty(FIELD_LENGTH_DISTRIBUTION_PROPERTY, FIELD_LENGTH_DISTRIBUTION_PROPERTY_DEFAULT); - int fieldlength=Integer.parseInt(p.getProperty(FIELD_LENGTH_PROPERTY,FIELD_LENGTH_PROPERTY_DEFAULT)); - String fieldlengthhistogram = p.getProperty(FIELD_LENGTH_HISTOGRAM_FILE_PROPERTY, FIELD_LENGTH_HISTOGRAM_FILE_PROPERTY_DEFAULT); - if(fieldlengthdistribution.compareTo("constant") == 0) { - fieldlengthgenerator = new ConstantIntegerGenerator(fieldlength); - } else if(fieldlengthdistribution.compareTo("uniform") == 0) { - fieldlengthgenerator = new UniformIntegerGenerator(1, fieldlength); - } else if(fieldlengthdistribution.compareTo("zipfian") == 0) { - fieldlengthgenerator = new ZipfianGenerator(1, fieldlength); - } else if(fieldlengthdistribution.compareTo("histogram") == 0) { - try { - fieldlengthgenerator = new HistogramGenerator(fieldlengthhistogram); - } catch(IOException e) { - throw new WorkloadException("Couldn't read field length histogram file: "+fieldlengthhistogram, e); - } - } else { - throw new WorkloadException("Unknown field length distribution \""+fieldlengthdistribution+"\""); - } - return fieldlengthgenerator; - } - - /** - * Initialize the scenario. - * Called once, in the main client thread, before any operations are started. - */ - public void init(Properties p) throws WorkloadException - { - table = p.getProperty(TABLENAME_PROPERTY,TABLENAME_PROPERTY_DEFAULT); - - fieldcount=Integer.parseInt(p.getProperty(FIELD_COUNT_PROPERTY,FIELD_COUNT_PROPERTY_DEFAULT)); - fieldlengthgenerator = CoreWorkload.getFieldLengthGenerator(p); - - double readproportion=Double.parseDouble(p.getProperty(READ_PROPORTION_PROPERTY,READ_PROPORTION_PROPERTY_DEFAULT)); - double updateproportion=Double.parseDouble(p.getProperty(UPDATE_PROPORTION_PROPERTY,UPDATE_PROPORTION_PROPERTY_DEFAULT)); - double insertproportion=Double.parseDouble(p.getProperty(INSERT_PROPORTION_PROPERTY,INSERT_PROPORTION_PROPERTY_DEFAULT)); - double scanproportion=Double.parseDouble(p.getProperty(SCAN_PROPORTION_PROPERTY,SCAN_PROPORTION_PROPERTY_DEFAULT)); - double readmodifywriteproportion=Double.parseDouble(p.getProperty(READMODIFYWRITE_PROPORTION_PROPERTY,READMODIFYWRITE_PROPORTION_PROPERTY_DEFAULT)); - recordcount=Integer.parseInt(p.getProperty(Client.RECORD_COUNT_PROPERTY)); - String requestdistrib=p.getProperty(REQUEST_DISTRIBUTION_PROPERTY,REQUEST_DISTRIBUTION_PROPERTY_DEFAULT); - int maxscanlength=Integer.parseInt(p.getProperty(MAX_SCAN_LENGTH_PROPERTY,MAX_SCAN_LENGTH_PROPERTY_DEFAULT)); - String scanlengthdistrib=p.getProperty(SCAN_LENGTH_DISTRIBUTION_PROPERTY,SCAN_LENGTH_DISTRIBUTION_PROPERTY_DEFAULT); - - int insertstart=Integer.parseInt(p.getProperty(INSERT_START_PROPERTY,INSERT_START_PROPERTY_DEFAULT)); - - readallfields=Boolean.parseBoolean(p.getProperty(READ_ALL_FIELDS_PROPERTY,READ_ALL_FIELDS_PROPERTY_DEFAULT)); - writeallfields=Boolean.parseBoolean(p.getProperty(WRITE_ALL_FIELDS_PROPERTY,WRITE_ALL_FIELDS_PROPERTY_DEFAULT)); - - if (p.getProperty(INSERT_ORDER_PROPERTY,INSERT_ORDER_PROPERTY_DEFAULT).compareTo("hashed")==0) - { - orderedinserts=false; - } - else if (requestdistrib.compareTo("exponential")==0) - { - double percentile = Double.parseDouble(p.getProperty(ExponentialGenerator.EXPONENTIAL_PERCENTILE_PROPERTY, - ExponentialGenerator.EXPONENTIAL_PERCENTILE_DEFAULT)); - double frac = Double.parseDouble(p.getProperty(ExponentialGenerator.EXPONENTIAL_FRAC_PROPERTY, - ExponentialGenerator.EXPONENTIAL_FRAC_DEFAULT)); - keychooser = new ExponentialGenerator(percentile, recordcount*frac); - } - else - { - orderedinserts=true; - } - - keysequence=new CounterGenerator(insertstart); - operationchooser=new DiscreteGenerator(); - if (readproportion>0) - { - operationchooser.addValue(readproportion,"READ"); - } - - if (updateproportion>0) - { - operationchooser.addValue(updateproportion,"UPDATE"); - } - - if (insertproportion>0) - { - operationchooser.addValue(insertproportion,"INSERT"); - } - - if (scanproportion>0) - { - operationchooser.addValue(scanproportion,"SCAN"); - } - - if (readmodifywriteproportion>0) - { - operationchooser.addValue(readmodifywriteproportion,"READMODIFYWRITE"); - } - - transactioninsertkeysequence=new CounterGenerator(recordcount); - if (requestdistrib.compareTo("uniform")==0) - { - keychooser=new UniformIntegerGenerator(0,recordcount-1); - } - else if (requestdistrib.compareTo("zipfian")==0) - { - //it does this by generating a random "next key" in part by taking the modulus over the number of keys - //if the number of keys changes, this would shift the modulus, and we don't want that to change which keys are popular - //so we'll actually construct the scrambled zipfian generator with a keyspace that is larger than exists at the beginning - //of the test. that is, we'll predict the number of inserts, and tell the scrambled zipfian generator the number of existing keys - //plus the number of predicted keys as the total keyspace. then, if the generator picks a key that hasn't been inserted yet, will - //just ignore it and pick another key. this way, the size of the keyspace doesn't change from the perspective of the scrambled zipfian generator - - int opcount=Integer.parseInt(p.getProperty(Client.OPERATION_COUNT_PROPERTY)); - int expectednewkeys=(int)(((double)opcount)*insertproportion*2.0); //2 is fudge factor - - keychooser=new ScrambledZipfianGenerator(recordcount+expectednewkeys); - } - else if (requestdistrib.compareTo("latest")==0) - { - keychooser=new SkewedLatestGenerator(transactioninsertkeysequence); - } - else if (requestdistrib.equals("hotspot")) - { - double hotsetfraction = Double.parseDouble(p.getProperty( - HOTSPOT_DATA_FRACTION, HOTSPOT_DATA_FRACTION_DEFAULT)); - double hotopnfraction = Double.parseDouble(p.getProperty( - HOTSPOT_OPN_FRACTION, HOTSPOT_OPN_FRACTION_DEFAULT)); - keychooser = new HotspotIntegerGenerator(0, recordcount - 1, - hotsetfraction, hotopnfraction); - } - else - { - throw new WorkloadException("Unknown request distribution \""+requestdistrib+"\""); - } - - fieldchooser=new UniformIntegerGenerator(0,fieldcount-1); - - if (scanlengthdistrib.compareTo("uniform")==0) - { - scanlength=new UniformIntegerGenerator(1,maxscanlength); - } - else if (scanlengthdistrib.compareTo("zipfian")==0) - { - scanlength=new ZipfianGenerator(1,maxscanlength); - } - else - { - throw new WorkloadException("Distribution \""+scanlengthdistrib+"\" not allowed for scan length"); - } - } - - public String buildKeyName(long keynum) { - if (!orderedinserts) - { - keynum=Utils.hash(keynum); - } - return "user"+keynum; - } - HashMap buildValues() { - HashMap values=new HashMap(); - - for (int i=0; i buildUpdate() { - //update a random field - HashMap values=new HashMap(); - String fieldname="field"+fieldchooser.nextString(); - ByteIterator data = new RandomByteIterator(fieldlengthgenerator.nextInt()); - values.put(fieldname,data); - return values; - } - - /** - * Do one insert operation. Because it will be called concurrently from multiple client threads, this - * function must be thread safe. However, avoid synchronized, or the threads will block waiting for each - * other, and it will be difficult to reach the target throughput. Ideally, this function would have no side - * effects other than DB operations. - */ - public boolean doInsert(DB db, Object threadstate) - { - int keynum=keysequence.nextInt(); - String dbkey = buildKeyName(keynum); - HashMap values = buildValues(); - if (db.insert(table,dbkey,values) == 0) - return true; - else - return false; - } - - /** - * Do one transaction operation. Because it will be called concurrently from multiple client threads, this - * function must be thread safe. However, avoid synchronized, or the threads will block waiting for each - * other, and it will be difficult to reach the target throughput. Ideally, this function would have no side - * effects other than DB operations. - */ - public boolean doTransaction(DB db, Object threadstate) - { - String op=operationchooser.nextString(); - - if (op.compareTo("READ")==0) - { - doTransactionRead(db); - } - else if (op.compareTo("UPDATE")==0) - { - doTransactionUpdate(db); - } - else if (op.compareTo("INSERT")==0) - { - doTransactionInsert(db); - } - else if (op.compareTo("SCAN")==0) - { - doTransactionScan(db); - } - else - { - doTransactionReadModifyWrite(db); - } - - return true; - } - - int nextKeynum() { - int keynum; - if(keychooser instanceof ExponentialGenerator) { - do - { - keynum=transactioninsertkeysequence.lastInt() - keychooser.nextInt(); - } - while(keynum < 0); - } else { - do - { - keynum=keychooser.nextInt(); - } - while (keynum > transactioninsertkeysequence.lastInt()); - } - return keynum; - } - - public void doTransactionRead(DB db) - { - //choose a random key - int keynum = nextKeynum(); - - String keyname = buildKeyName(keynum); - - HashSet fields=null; - - if (!readallfields) - { - //read a random field - String fieldname="field"+fieldchooser.nextString(); - - fields=new HashSet(); - fields.add(fieldname); - } - - db.read(table,keyname,fields,new HashMap()); - } - - public void doTransactionReadModifyWrite(DB db) - { - //choose a random key - int keynum = nextKeynum(); - - String keyname = buildKeyName(keynum); - - HashSet fields=null; - - if (!readallfields) - { - //read a random field - String fieldname="field"+fieldchooser.nextString(); - - fields=new HashSet(); - fields.add(fieldname); - } - - HashMap values; - - if (writeallfields) - { - //new data for all the fields - values = buildValues(); - } - else - { - //update a random field - values = buildUpdate(); - } - - //do the transaction - - long st=System.currentTimeMillis(); - - db.read(table,keyname,fields,new HashMap()); - - db.update(table,keyname,values); - - long en=System.currentTimeMillis(); - - Measurements.getMeasurements().measure("READ-MODIFY-WRITE", (int)(en-st)); - } - - public void doTransactionScan(DB db) - { - //choose a random key - int keynum = nextKeynum(); - - String startkeyname = buildKeyName(keynum); - - //choose a random scan length - int len=scanlength.nextInt(); - - HashSet fields=null; - - if (!readallfields) - { - //read a random field - String fieldname="field"+fieldchooser.nextString(); - - fields=new HashSet(); - fields.add(fieldname); - } - - db.scan(table,startkeyname,len,fields,new Vector>()); - } - - public void doTransactionUpdate(DB db) - { - //choose a random key - int keynum = nextKeynum(); - - String keyname=buildKeyName(keynum); - - HashMap values; - - if (writeallfields) - { - //new data for all the fields - values = buildValues(); - } - else - { - //update a random field - values = buildUpdate(); - } - - db.update(table,keyname,values); - } - - public void doTransactionInsert(DB db) - { - //choose the next key - int keynum=transactioninsertkeysequence.nextInt(); - - String dbkey = buildKeyName(keynum); - - HashMap values = buildValues(); - db.insert(table,dbkey,values); - } -} diff --git a/workloads/workloada b/workloads/workloada deleted file mode 100644 index 8d4afb64..00000000 --- a/workloads/workloada +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright (c) 2010 Yahoo! Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you -# may not use this file except in compliance with the License. You -# may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. See the License for the specific language governing -# permissions and limitations under the License. See accompanying -# LICENSE file. - - -# Yahoo! Cloud System Benchmark -# Workload A: Update heavy workload -# Application example: Session store recording recent actions -# -# Read/update ratio: 50/50 -# Default data size: 1 KB records (10 fields, 100 bytes each, plus key) -# Request distribution: zipfian - -recordcount=1000 -operationcount=1000 -workload=com.yahoo.ycsb.workloads.CoreWorkload - -readallfields=true - -readproportion=0.5 -updateproportion=0.5 -scanproportion=0 -insertproportion=0 - -requestdistribution=zipfian - diff --git a/workloads/workloadb b/workloads/workloadb deleted file mode 100644 index 408a6bfd..00000000 --- a/workloads/workloadb +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright (c) 2010 Yahoo! Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you -# may not use this file except in compliance with the License. You -# may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. See the License for the specific language governing -# permissions and limitations under the License. See accompanying -# LICENSE file. - -# Yahoo! Cloud System Benchmark -# Workload B: Read mostly workload -# Application example: photo tagging; add a tag is an update, but most operations are to read tags -# -# Read/update ratio: 95/5 -# Default data size: 1 KB records (10 fields, 100 bytes each, plus key) -# Request distribution: zipfian - -recordcount=1000 -operationcount=1000 -workload=com.yahoo.ycsb.workloads.CoreWorkload - -readallfields=true - -readproportion=0.95 -updateproportion=0.05 -scanproportion=0 -insertproportion=0 - -requestdistribution=zipfian - diff --git a/workloads/workloadc b/workloads/workloadc deleted file mode 100644 index 1afe62c7..00000000 --- a/workloads/workloadc +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (c) 2010 Yahoo! Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you -# may not use this file except in compliance with the License. You -# may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. See the License for the specific language governing -# permissions and limitations under the License. See accompanying -# LICENSE file. - -# Yahoo! Cloud System Benchmark -# Workload C: Read only -# Application example: user profile cache, where profiles are constructed elsewhere (e.g., Hadoop) -# -# Read/update ratio: 100/0 -# Default data size: 1 KB records (10 fields, 100 bytes each, plus key) -# Request distribution: zipfian - -recordcount=1000 -operationcount=1000 -workload=com.yahoo.ycsb.workloads.CoreWorkload - -readallfields=true - -readproportion=1 -updateproportion=0 -scanproportion=0 -insertproportion=0 - -requestdistribution=zipfian - - - diff --git a/workloads/workloadd b/workloads/workloadd deleted file mode 100644 index 3b911f3c..00000000 --- a/workloads/workloadd +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) 2010 Yahoo! Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you -# may not use this file except in compliance with the License. You -# may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. See the License for the specific language governing -# permissions and limitations under the License. See accompanying -# LICENSE file. - -# Yahoo! Cloud System Benchmark -# Workload D: Read latest workload -# Application example: user status updates; people want to read the latest -# -# Read/update/insert ratio: 95/0/5 -# Default data size: 1 KB records (10 fields, 100 bytes each, plus key) -# Request distribution: latest - -# The insert order for this is hashed, not ordered. The "latest" items may be -# scattered around the keyspace if they are keyed by userid.timestamp. A workload -# which orders items purely by time, and demands the latest, is very different than -# workload here (which we believe is more typical of how people build systems.) - -recordcount=1000 -operationcount=1000 -workload=com.yahoo.ycsb.workloads.CoreWorkload - -readallfields=true - -readproportion=0.95 -updateproportion=0 -scanproportion=0 -insertproportion=0.05 - -requestdistribution=latest - diff --git a/workloads/workloade b/workloads/workloade deleted file mode 100644 index 64304d25..00000000 --- a/workloads/workloade +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright (c) 2010 Yahoo! Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you -# may not use this file except in compliance with the License. You -# may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. See the License for the specific language governing -# permissions and limitations under the License. See accompanying -# LICENSE file. - -# Yahoo! Cloud System Benchmark -# Workload E: Short ranges -# Application example: threaded conversations, where each scan is for the posts in a given thread (assumed to be clustered by thread id) -# -# Scan/insert ratio: 95/5 -# Default data size: 1 KB records (10 fields, 100 bytes each, plus key) -# Request distribution: zipfian - -# The insert order is hashed, not ordered. Although the scans are ordered, it does not necessarily -# follow that the data is inserted in order. For example, posts for thread 342 may not be inserted contiguously, but -# instead interspersed with posts from lots of other threads. The way the YCSB client works is that it will pick a start -# key, and then request a number of records; this works fine even for hashed insertion. - -recordcount=1000 -operationcount=1000 -workload=com.yahoo.ycsb.workloads.CoreWorkload - -readallfields=true - -readproportion=0 -updateproportion=0 -scanproportion=0.95 -insertproportion=0.05 - -requestdistribution=zipfian - -maxscanlength=100 - -scanlengthdistribution=uniform - - diff --git a/workloads/workloadf b/workloads/workloadf deleted file mode 100644 index 6c7455d3..00000000 --- a/workloads/workloadf +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright (c) 2010 Yahoo! Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you -# may not use this file except in compliance with the License. You -# may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. See the License for the specific language governing -# permissions and limitations under the License. See accompanying -# LICENSE file. - -# Yahoo! Cloud System Benchmark -# Workload F: Read-modify-write workload -# Application example: user database, where user records are read and modified by the user or to record user activity. -# -# Read/read-modify-write ratio: 50/50 -# Default data size: 1 KB records (10 fields, 100 bytes each, plus key) -# Request distribution: zipfian - -recordcount=1000 -operationcount=1000 -workload=com.yahoo.ycsb.workloads.CoreWorkload - -readallfields=true - -readproportion=0.5 -updateproportion=0 -scanproportion=0 -insertproportion=0 -readmodifywriteproportion=0.5 - -requestdistribution=zipfian -