diff --git a/distribution/src/main/bin/ycsb b/distribution/src/main/bin/ycsb
index 8ca3b098..50b52fe8 100755
--- a/distribution/src/main/bin/ycsb
+++ b/distribution/src/main/bin/ycsb
@@ -1,114 +1,98 @@
#!/usr/bin/env python
import os
import sys
import subprocess
COMMANDS = {
"load" : {
"command" : "-load",
"description" : "Execute the load phase",
},
"run" : {
"command" : "-t",
"description" : "Execute the transaction phase",
},
}
DATABASES = {
"basic" : "com.yahoo.ycsb.BasicDB",
"cassandra-7" : "com.yahoo.ycsb.db.CassandraClient7",
"cassandra-8" : "com.yahoo.ycsb.db.CassandraClient8",
"cassandra-10" : "com.yahoo.ycsb.db.CassandraClient10",
+ "gemfire" : "com.yahoo.ycsb.db.GemFireClient",
"hbase" : "com.yahoo.ycsb.db.HBaseClient",
"infinispan" : "com.yahoo.ycsb.db.InfinispanClient",
"jdbc" : "com.yahoo.ycsb.db.JdbcDBClient",
"mapkeeper" : "com.yahoo.ycsb.db.MapKeeperClient",
"mongodb" : "com.yahoo.ycsb.db.MongoDbClient",
"redis" : "com.yahoo.ycsb.db.RedisClient",
"voldemort" : "com.yahoo.ycsb.db.VoldemortClient",
}
OPTIONS = {
"-p key=value" : "Override workload property",
"-s" : "Print status to stderr",
"-target n" : "Target ops/sec (default: unthrottled)",
"-threads n" : "Number of client threads (default: 1)",
}
+
def usage():
- usage = "Usage: %s command database workload-file [options]" % sys.argv[0]
- print usage
+ print "Usage: %s command database workload-file [options]" % sys.argv[0]
- print "Commands:"
+ print "\nCommands:"
for command in sorted(COMMANDS.keys()):
print " {0:13} {1}".format(command, COMMANDS[command]["description"])
- print
- print "Databases:"
+ print "\nDatabases:"
for db in sorted(DATABASES.keys()):
print " %s" % db
- print
- print """Workload Files:
+ print """\nWorkload Files:
There are various predefined workloads under workloads/ directory.
See https://github.com/brianfrankcooper/YCSB/wiki/Core-Properties
- for the list of workload properties.
- """
+ for the list of workload properties."""
- print "Options:"
+ print "\nOptions:"
for option in sorted(OPTIONS.keys()):
print " {0:13} {1}".format(option, OPTIONS[option])
sys.exit(1)
def find_jars(dir, database):
jars = []
- print database
for (dirpath, dirnames, filenames) in os.walk(dir):
+ if dirpath.endswith("conf"):
+ jars.append(dirpath)
for filename in filenames:
- print filename
if filename.endswith(".jar") and \
- filename.startswith("core") or filename.startswith(database.split("-")[0]):
- print filename
+ filename.startswith("core") or \
+ filename.startswith(database.split("-")[0]):
jars.append(os.path.join(dirpath, filename))
return jars
def get_ycsb_home():
dir = os.path.abspath(os.path.dirname(sys.argv[0]))
while "CHANGELOG" not in os.listdir(dir):
dir = os.path.join(dir, os.path.pardir)
- return os.path.abspath(dir )
-
-def get_command():
- if len(sys.argv) < 2:
- usage()
- if sys.argv[1] not in COMMANDS:
- print "ERROR: Command '%s' not found" % sys.argv[1]
- usage()
- return COMMANDS[sys.argv[1]]["command"]
-
-def get_database():
- if len(sys.argv) < 3:
- usage()
- if sys.argv[2] not in DATABASES:
- print "ERROR: Database '%s' not found" % sys.argv[2]
- usage()
- return sys.argv[2], DATABASES[sys.argv[2]]
+ return os.path.abspath(dir)
-def get_workload():
- if len(sys.argv) < 4:
- usage()
- return sys.argv[3]
-
-def get_options():
- return sys.argv[4:]
+if len(sys.argv) < 4:
+ usage()
+if sys.argv[1] not in COMMANDS:
+ print "ERROR: Command '%s' not found" % sys.argv[1]
+ usage()
+if sys.argv[2] not in DATABASES:
+ print "ERROR: Database '%s' not found" % sys.argv[2]
+ usage()
ycsb_home = get_ycsb_home()
-command = get_command()
-database, db_classname = get_database()
-workload = get_workload()
-options = get_options()
+command = COMMANDS[sys.argv[1]]["command"]
+database = sys.argv[2]
+db_classname = DATABASES[database]
+workload = sys.argv[3]
+options = sys.argv[4:]
+
ycsb_command = ["java", "-cp", ":".join(find_jars(ycsb_home, database)), \
"com.yahoo.ycsb.Client", command, "-db", db_classname, \
"-P", workload] + options
-print " ".join(ycsb_command)
subprocess.call(ycsb_command)
diff --git a/gemfire/pom.xml b/gemfire/pom.xml
index ba979440..63acb5d2 100644
--- a/gemfire/pom.xml
+++ b/gemfire/pom.xml
@@ -1,63 +1,58 @@
4.0.0
org.apache.ycsb
root
0.1.3
gemfire-binding
Gemfire DB Binding
jar
com.gemstone.gemfire
gemfire
6.6
-
- org.springframework
- spring-core
- ${gemfire.version}
-
org.apache.ycsb
core
${project.version}
gemstone
http://dist.gemstone.com.s3.amazonaws.com/maven/release/
org.apache.maven.plugins
maven-assembly-plugin
${maven.assembly.version}
jar-with-dependencies
false
package
single
diff --git a/mapkeeper/src/main/java/com/yahoo/ycsb/db/MapKeeperClient.java b/mapkeeper/src/main/java/com/yahoo/ycsb/db/MapKeeperClient.java
index 594c9308..5af5ed9e 100644
--- a/mapkeeper/src/main/java/com/yahoo/ycsb/db/MapKeeperClient.java
+++ b/mapkeeper/src/main/java/com/yahoo/ycsb/db/MapKeeperClient.java
@@ -1,196 +1,202 @@
package com.yahoo.ycsb.db;
import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.Properties;
import java.util.Set;
import java.util.Vector;
import org.apache.thrift.TException;
import org.apache.thrift.protocol.TBinaryProtocol;
import org.apache.thrift.protocol.TProtocol;
import org.apache.thrift.transport.TFramedTransport;
import org.apache.thrift.transport.TSocket;
import org.apache.thrift.transport.TTransport;
import com.yahoo.mapkeeper.BinaryResponse;
import com.yahoo.mapkeeper.MapKeeper;
import com.yahoo.mapkeeper.Record;
import com.yahoo.mapkeeper.RecordListResponse;
import com.yahoo.mapkeeper.ResponseCode;
import com.yahoo.mapkeeper.ScanOrder;
import com.yahoo.ycsb.ByteIterator;
import com.yahoo.ycsb.DB;
import com.yahoo.ycsb.StringByteIterator;
import com.yahoo.ycsb.workloads.CoreWorkload;
public class MapKeeperClient extends DB {
+ private static final String HOST = "mapkeeper.host";
+ private static final String HOST_DEFAULT = "localhost";
+ private static final String PORT = "mapkeeper.port";
+ private static final String PORT_DEFAULT = "9090";
MapKeeper.Client c;
boolean writeallfields;
static boolean initteddb = false;
private synchronized static void initDB(Properties p, MapKeeper.Client c) throws TException {
if(!initteddb) {
initteddb = true;
c.addMap(p.getProperty(CoreWorkload.TABLENAME_PROPERTY, CoreWorkload.TABLENAME_PROPERTY_DEFAULT));
}
}
public void init() {
- TTransport tr = new TFramedTransport(new TSocket("localhost", 9090));
+ String host = getProperties().getProperty(HOST, HOST_DEFAULT);
+ int port = Integer.parseInt(getProperties().getProperty(PORT, PORT_DEFAULT));
+ TTransport tr = new TFramedTransport(new TSocket(host, port));
TProtocol proto = new TBinaryProtocol(tr);
c = new MapKeeper.Client(proto);
try {
tr.open();
initDB(getProperties(), c);
} catch(TException e) {
throw new RuntimeException(e);
}
writeallfields = Boolean.parseBoolean(getProperties().getProperty(CoreWorkload.WRITE_ALL_FIELDS_PROPERTY,
CoreWorkload.WRITE_ALL_FIELDS_PROPERTY_DEFAULT));
}
ByteBuffer encode(HashMap values) {
int len = 0;
for(String k : values.keySet()) {
len += (k.length() + 1 + values.get(k).bytesLeft() + 1);
}
byte[] array = new byte[len];
int i = 0;
for(String k : values.keySet()) {
for(int j = 0; j < k.length(); j++) {
array[i] = (byte)k.charAt(j);
i++;
}
array[i] = '\t'; // XXX would like to use sane delimiter (null, 254, 255, ...) but java makes this nearly impossible
i++;
ByteIterator v = values.get(k);
i = v.nextBuf(array, i);
array[i] = '\t';
i++;
}
array[array.length-1] = 0;
ByteBuffer buf = ByteBuffer.wrap(array);
buf.rewind();
return buf;
}
void decode(Set fields, String tups, HashMap tup) {
String[] tok = tups.split("\\t");
if(tok.length == 0) { throw new IllegalStateException("split returned empty array!"); }
for(int i = 0; i < tok.length; i+=2) {
if(fields == null || fields.contains(tok[i])) {
if(tok.length < i+2) { throw new IllegalStateException("Couldn't parse tuple <" + tups + "> at index " + i); }
if(tok[i] == null || tok[i+1] == null) throw new NullPointerException("Key is " + tok[i] + " val is + " + tok[i+1]);
tup.put(tok[i], new StringByteIterator(tok[i+1]));
}
}
if(tok.length == 0) {
System.err.println("Empty tuple: " + tups);
}
}
int ycsbThriftRet(BinaryResponse succ, ResponseCode zero, ResponseCode one) {
return ycsbThriftRet(succ.responseCode, zero, one);
}
int ycsbThriftRet(ResponseCode rc, ResponseCode zero, ResponseCode one) {
return
rc == zero ? 0 :
rc == one ? 1 : 2;
}
ByteBuffer bufStr(String str) {
ByteBuffer buf = ByteBuffer.wrap(str.getBytes());
return buf;
}
String strResponse(BinaryResponse buf) {
return new String(buf.value.array());
}
@Override
public int read(String table, String key, Set fields,
HashMap result) {
try {
ByteBuffer buf = bufStr(key);
BinaryResponse succ = c.get(table, buf);
int ret = ycsbThriftRet(
succ,
ResponseCode.RecordExists,
ResponseCode.RecordNotFound);
if(ret == 0) {
decode(fields, strResponse(succ), result);
}
return ret;
} catch(TException e) {
e.printStackTrace();
return 2;
}
}
@Override
public int scan(String table, String startkey, int recordcount,
Set fields, Vector> result) {
try {
//XXX what to pass in for nulls / zeros?
RecordListResponse res = c.scan(table, ScanOrder.Ascending, bufStr(startkey), true, null, false, recordcount, 0);
int ret = ycsbThriftRet(res.responseCode, ResponseCode.Success, ResponseCode.ScanEnded);
if(ret == 0) {
for(Record r : res.records) {
HashMap tuple = new HashMap();
// Note: r.getKey() and r.getValue() call special helper methods that trim the buffer
// to an appropriate length, and memcpy it to a byte[]. Trying to manipulate the ByteBuffer
// directly leads to trouble.
tuple.put("key", new StringByteIterator(new String(r.getKey())));
decode(fields, new String(r.getValue())/*strBuf(r.bufferForValue())*/, tuple);
result.add(tuple);
}
}
return ret;
} catch(TException e) {
e.printStackTrace();
return 2;
}
}
@Override
public int update(String table, String key,
HashMap values) {
try {
if(!writeallfields) {
HashMap oldval = new HashMap();
read(table, key, null, oldval);
for(String k: values.keySet()) {
oldval.put(k, values.get(k));
}
values = oldval;
}
ResponseCode succ = c.update(table, bufStr(key), encode(values));
return ycsbThriftRet(succ, ResponseCode.RecordExists, ResponseCode.RecordNotFound);
} catch(TException e) {
e.printStackTrace();
return 2;
}
}
@Override
public int insert(String table, String key,
HashMap values) {
try {
int ret = ycsbThriftRet(c.insert(table, bufStr(key), encode(values)), ResponseCode.Success, ResponseCode.RecordExists);
return ret;
} catch(TException e) {
e.printStackTrace();
return 2;
}
}
@Override
public int delete(String table, String key) {
try {
return ycsbThriftRet(c.remove(table, bufStr(key)), ResponseCode.Success, ResponseCode.RecordExists);
} catch(TException e) {
e.printStackTrace();
return 2;
}
}
}
diff --git a/pom.xml b/pom.xml
index 286d23c5..f0d88561 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1,77 +1,76 @@
4.0.0
org.apache.ycsb
root
0.1.3
pom
YCSB Root
This is the top level project that builds, packages the core and all the DB bindings for YCSB infrastructure.
com.google.collections
google-collections
1.0
org.slf4j
slf4j-api
1.6.4
2.2.1
0.90.5
0.7.0
- 1.0.0.M3
7.1.0.CR1
2.1.1
1.0
2.7.2
2.0.0
0.81
UTF-8
core
hbase
cassandra
-
+ gemfire
infinispan
jdbc
mapkeeper
mongodb
redis
voldemort
distribution
org.apache.maven.plugins
maven-compiler-plugin
2.3.2
1.6