diff --git a/distribution/src/main/bin/ycsb b/distribution/src/main/bin/ycsb index 8ca3b098..50b52fe8 100755 --- a/distribution/src/main/bin/ycsb +++ b/distribution/src/main/bin/ycsb @@ -1,114 +1,98 @@ #!/usr/bin/env python import os import sys import subprocess COMMANDS = { "load" : { "command" : "-load", "description" : "Execute the load phase", }, "run" : { "command" : "-t", "description" : "Execute the transaction phase", }, } DATABASES = { "basic" : "com.yahoo.ycsb.BasicDB", "cassandra-7" : "com.yahoo.ycsb.db.CassandraClient7", "cassandra-8" : "com.yahoo.ycsb.db.CassandraClient8", "cassandra-10" : "com.yahoo.ycsb.db.CassandraClient10", + "gemfire" : "com.yahoo.ycsb.db.GemFireClient", "hbase" : "com.yahoo.ycsb.db.HBaseClient", "infinispan" : "com.yahoo.ycsb.db.InfinispanClient", "jdbc" : "com.yahoo.ycsb.db.JdbcDBClient", "mapkeeper" : "com.yahoo.ycsb.db.MapKeeperClient", "mongodb" : "com.yahoo.ycsb.db.MongoDbClient", "redis" : "com.yahoo.ycsb.db.RedisClient", "voldemort" : "com.yahoo.ycsb.db.VoldemortClient", } OPTIONS = { "-p key=value" : "Override workload property", "-s" : "Print status to stderr", "-target n" : "Target ops/sec (default: unthrottled)", "-threads n" : "Number of client threads (default: 1)", } + def usage(): - usage = "Usage: %s command database workload-file [options]" % sys.argv[0] - print usage + print "Usage: %s command database workload-file [options]" % sys.argv[0] - print "Commands:" + print "\nCommands:" for command in sorted(COMMANDS.keys()): print " {0:13} {1}".format(command, COMMANDS[command]["description"]) - print - print "Databases:" + print "\nDatabases:" for db in sorted(DATABASES.keys()): print " %s" % db - print - print """Workload Files: + print """\nWorkload Files: There are various predefined workloads under workloads/ directory. See https://github.com/brianfrankcooper/YCSB/wiki/Core-Properties - for the list of workload properties. - """ + for the list of workload properties.""" - print "Options:" + print "\nOptions:" for option in sorted(OPTIONS.keys()): print " {0:13} {1}".format(option, OPTIONS[option]) sys.exit(1) def find_jars(dir, database): jars = [] - print database for (dirpath, dirnames, filenames) in os.walk(dir): + if dirpath.endswith("conf"): + jars.append(dirpath) for filename in filenames: - print filename if filename.endswith(".jar") and \ - filename.startswith("core") or filename.startswith(database.split("-")[0]): - print filename + filename.startswith("core") or \ + filename.startswith(database.split("-")[0]): jars.append(os.path.join(dirpath, filename)) return jars def get_ycsb_home(): dir = os.path.abspath(os.path.dirname(sys.argv[0])) while "CHANGELOG" not in os.listdir(dir): dir = os.path.join(dir, os.path.pardir) - return os.path.abspath(dir ) - -def get_command(): - if len(sys.argv) < 2: - usage() - if sys.argv[1] not in COMMANDS: - print "ERROR: Command '%s' not found" % sys.argv[1] - usage() - return COMMANDS[sys.argv[1]]["command"] - -def get_database(): - if len(sys.argv) < 3: - usage() - if sys.argv[2] not in DATABASES: - print "ERROR: Database '%s' not found" % sys.argv[2] - usage() - return sys.argv[2], DATABASES[sys.argv[2]] + return os.path.abspath(dir) -def get_workload(): - if len(sys.argv) < 4: - usage() - return sys.argv[3] - -def get_options(): - return sys.argv[4:] +if len(sys.argv) < 4: + usage() +if sys.argv[1] not in COMMANDS: + print "ERROR: Command '%s' not found" % sys.argv[1] + usage() +if sys.argv[2] not in DATABASES: + print "ERROR: Database '%s' not found" % sys.argv[2] + usage() ycsb_home = get_ycsb_home() -command = get_command() -database, db_classname = get_database() -workload = get_workload() -options = get_options() +command = COMMANDS[sys.argv[1]]["command"] +database = sys.argv[2] +db_classname = DATABASES[database] +workload = sys.argv[3] +options = sys.argv[4:] + ycsb_command = ["java", "-cp", ":".join(find_jars(ycsb_home, database)), \ "com.yahoo.ycsb.Client", command, "-db", db_classname, \ "-P", workload] + options -print " ".join(ycsb_command) subprocess.call(ycsb_command) diff --git a/gemfire/pom.xml b/gemfire/pom.xml index ba979440..63acb5d2 100644 --- a/gemfire/pom.xml +++ b/gemfire/pom.xml @@ -1,63 +1,58 @@ 4.0.0 org.apache.ycsb root 0.1.3 gemfire-binding Gemfire DB Binding jar com.gemstone.gemfire gemfire 6.6 - - org.springframework - spring-core - ${gemfire.version} - org.apache.ycsb core ${project.version} gemstone http://dist.gemstone.com.s3.amazonaws.com/maven/release/ org.apache.maven.plugins maven-assembly-plugin ${maven.assembly.version} jar-with-dependencies false package single diff --git a/mapkeeper/src/main/java/com/yahoo/ycsb/db/MapKeeperClient.java b/mapkeeper/src/main/java/com/yahoo/ycsb/db/MapKeeperClient.java index 594c9308..5af5ed9e 100644 --- a/mapkeeper/src/main/java/com/yahoo/ycsb/db/MapKeeperClient.java +++ b/mapkeeper/src/main/java/com/yahoo/ycsb/db/MapKeeperClient.java @@ -1,196 +1,202 @@ package com.yahoo.ycsb.db; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Properties; import java.util.Set; import java.util.Vector; import org.apache.thrift.TException; import org.apache.thrift.protocol.TBinaryProtocol; import org.apache.thrift.protocol.TProtocol; import org.apache.thrift.transport.TFramedTransport; import org.apache.thrift.transport.TSocket; import org.apache.thrift.transport.TTransport; import com.yahoo.mapkeeper.BinaryResponse; import com.yahoo.mapkeeper.MapKeeper; import com.yahoo.mapkeeper.Record; import com.yahoo.mapkeeper.RecordListResponse; import com.yahoo.mapkeeper.ResponseCode; import com.yahoo.mapkeeper.ScanOrder; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.StringByteIterator; import com.yahoo.ycsb.workloads.CoreWorkload; public class MapKeeperClient extends DB { + private static final String HOST = "mapkeeper.host"; + private static final String HOST_DEFAULT = "localhost"; + private static final String PORT = "mapkeeper.port"; + private static final String PORT_DEFAULT = "9090"; MapKeeper.Client c; boolean writeallfields; static boolean initteddb = false; private synchronized static void initDB(Properties p, MapKeeper.Client c) throws TException { if(!initteddb) { initteddb = true; c.addMap(p.getProperty(CoreWorkload.TABLENAME_PROPERTY, CoreWorkload.TABLENAME_PROPERTY_DEFAULT)); } } public void init() { - TTransport tr = new TFramedTransport(new TSocket("localhost", 9090)); + String host = getProperties().getProperty(HOST, HOST_DEFAULT); + int port = Integer.parseInt(getProperties().getProperty(PORT, PORT_DEFAULT)); + TTransport tr = new TFramedTransport(new TSocket(host, port)); TProtocol proto = new TBinaryProtocol(tr); c = new MapKeeper.Client(proto); try { tr.open(); initDB(getProperties(), c); } catch(TException e) { throw new RuntimeException(e); } writeallfields = Boolean.parseBoolean(getProperties().getProperty(CoreWorkload.WRITE_ALL_FIELDS_PROPERTY, CoreWorkload.WRITE_ALL_FIELDS_PROPERTY_DEFAULT)); } ByteBuffer encode(HashMap values) { int len = 0; for(String k : values.keySet()) { len += (k.length() + 1 + values.get(k).bytesLeft() + 1); } byte[] array = new byte[len]; int i = 0; for(String k : values.keySet()) { for(int j = 0; j < k.length(); j++) { array[i] = (byte)k.charAt(j); i++; } array[i] = '\t'; // XXX would like to use sane delimiter (null, 254, 255, ...) but java makes this nearly impossible i++; ByteIterator v = values.get(k); i = v.nextBuf(array, i); array[i] = '\t'; i++; } array[array.length-1] = 0; ByteBuffer buf = ByteBuffer.wrap(array); buf.rewind(); return buf; } void decode(Set fields, String tups, HashMap tup) { String[] tok = tups.split("\\t"); if(tok.length == 0) { throw new IllegalStateException("split returned empty array!"); } for(int i = 0; i < tok.length; i+=2) { if(fields == null || fields.contains(tok[i])) { if(tok.length < i+2) { throw new IllegalStateException("Couldn't parse tuple <" + tups + "> at index " + i); } if(tok[i] == null || tok[i+1] == null) throw new NullPointerException("Key is " + tok[i] + " val is + " + tok[i+1]); tup.put(tok[i], new StringByteIterator(tok[i+1])); } } if(tok.length == 0) { System.err.println("Empty tuple: " + tups); } } int ycsbThriftRet(BinaryResponse succ, ResponseCode zero, ResponseCode one) { return ycsbThriftRet(succ.responseCode, zero, one); } int ycsbThriftRet(ResponseCode rc, ResponseCode zero, ResponseCode one) { return rc == zero ? 0 : rc == one ? 1 : 2; } ByteBuffer bufStr(String str) { ByteBuffer buf = ByteBuffer.wrap(str.getBytes()); return buf; } String strResponse(BinaryResponse buf) { return new String(buf.value.array()); } @Override public int read(String table, String key, Set fields, HashMap result) { try { ByteBuffer buf = bufStr(key); BinaryResponse succ = c.get(table, buf); int ret = ycsbThriftRet( succ, ResponseCode.RecordExists, ResponseCode.RecordNotFound); if(ret == 0) { decode(fields, strResponse(succ), result); } return ret; } catch(TException e) { e.printStackTrace(); return 2; } } @Override public int scan(String table, String startkey, int recordcount, Set fields, Vector> result) { try { //XXX what to pass in for nulls / zeros? RecordListResponse res = c.scan(table, ScanOrder.Ascending, bufStr(startkey), true, null, false, recordcount, 0); int ret = ycsbThriftRet(res.responseCode, ResponseCode.Success, ResponseCode.ScanEnded); if(ret == 0) { for(Record r : res.records) { HashMap tuple = new HashMap(); // Note: r.getKey() and r.getValue() call special helper methods that trim the buffer // to an appropriate length, and memcpy it to a byte[]. Trying to manipulate the ByteBuffer // directly leads to trouble. tuple.put("key", new StringByteIterator(new String(r.getKey()))); decode(fields, new String(r.getValue())/*strBuf(r.bufferForValue())*/, tuple); result.add(tuple); } } return ret; } catch(TException e) { e.printStackTrace(); return 2; } } @Override public int update(String table, String key, HashMap values) { try { if(!writeallfields) { HashMap oldval = new HashMap(); read(table, key, null, oldval); for(String k: values.keySet()) { oldval.put(k, values.get(k)); } values = oldval; } ResponseCode succ = c.update(table, bufStr(key), encode(values)); return ycsbThriftRet(succ, ResponseCode.RecordExists, ResponseCode.RecordNotFound); } catch(TException e) { e.printStackTrace(); return 2; } } @Override public int insert(String table, String key, HashMap values) { try { int ret = ycsbThriftRet(c.insert(table, bufStr(key), encode(values)), ResponseCode.Success, ResponseCode.RecordExists); return ret; } catch(TException e) { e.printStackTrace(); return 2; } } @Override public int delete(String table, String key) { try { return ycsbThriftRet(c.remove(table, bufStr(key)), ResponseCode.Success, ResponseCode.RecordExists); } catch(TException e) { e.printStackTrace(); return 2; } } } diff --git a/pom.xml b/pom.xml index 286d23c5..f0d88561 100644 --- a/pom.xml +++ b/pom.xml @@ -1,77 +1,76 @@ 4.0.0 org.apache.ycsb root 0.1.3 pom YCSB Root This is the top level project that builds, packages the core and all the DB bindings for YCSB infrastructure. com.google.collections google-collections 1.0 org.slf4j slf4j-api 1.6.4 2.2.1 0.90.5 0.7.0 - 1.0.0.M3 7.1.0.CR1 2.1.1 1.0 2.7.2 2.0.0 0.81 UTF-8 core hbase cassandra - + gemfire infinispan jdbc mapkeeper mongodb redis voldemort distribution org.apache.maven.plugins maven-compiler-plugin 2.3.2 1.6 1.6