diff --git a/README.md b/README.md index 774510c5..9fa5ba44 100644 --- a/README.md +++ b/README.md @@ -1,49 +1,67 @@ + + Yahoo! Cloud System Benchmark (YCSB) ==================================== [![Build Status](https://travis-ci.org/brianfrankcooper/YCSB.png?branch=master)](https://travis-ci.org/brianfrankcooper/YCSB) Links ----- http://wiki.github.com/brianfrankcooper/YCSB/ http://research.yahoo.com/Web_Information_Management/YCSB/ ycsb-users@yahoogroups.com Getting Started --------------- 1. Download the latest release of YCSB: ```sh curl -O https://github.com/brianfrankcooper/YCSB/releases/download/0.2.0/ycsb-0.2.0.tar.gz tar xfvz ycsb-0.2.0.tar.gz cd ycsb-0.2.0 ``` 2. Set up a database to benchmark. There is a README file under each binding directory. 3. Run YCSB command. ```sh bin/ycsb load basic -P workloads/workloada bin/ycsb run basic -P workloads/workloada ``` Running the `ycsb` command without any argument will print the usage. See https://github.com/brianfrankcooper/YCSB/wiki/Running-a-Workload for a detailed documentation on how to run a workload. See https://github.com/brianfrankcooper/YCSB/wiki/Core-Properties for the list of available workload properties. Building from source -------------------- To build the full distribution, with all database bindings: mvn clean package To build a single database binding: mvn -pl com.yahoo.ycsb:mongodb-binding -am clean package diff --git a/accumulo/pom.xml b/accumulo/pom.xml index d20bd9e7..2d7a33cc 100644 --- a/accumulo/pom.xml +++ b/accumulo/pom.xml @@ -1,49 +1,67 @@ + + 4.0.0 com.yahoo.ycsb binding-parent 0.3.0-RC4-SNAPSHOT ../binding-parent accumulo-binding Accumulo DB Binding org.apache.accumulo accumulo-core ${accumulo.version} org.apache.hadoop hadoop-common org.apache.thrift thrift org.apache.zookeeper zookeeper org.apache.zookeeper zookeeper 3.3.1 org.apache.hadoop hadoop-core 0.20.203.0 com.yahoo.ycsb core ${project.version} provided diff --git a/accumulo/src/main/java/com/yahoo/ycsb/db/AccumuloClient.java b/accumulo/src/main/java/com/yahoo/ycsb/db/AccumuloClient.java index 0551849c..d661908e 100644 --- a/accumulo/src/main/java/com/yahoo/ycsb/db/AccumuloClient.java +++ b/accumulo/src/main/java/com/yahoo/ycsb/db/AccumuloClient.java @@ -1,435 +1,453 @@ +/** + * Copyright (c) 2011 YCSB++ project, 2014 YCSB contributors. + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + package com.yahoo.ycsb.db; import java.util.HashMap; import java.util.HashSet; import java.util.Hashtable; import java.util.Map; import java.util.Map.Entry; import java.util.Random; import java.util.Set; import java.util.TreeSet; import java.util.Vector; import java.util.concurrent.TimeUnit; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.BatchWriterConfig; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.MutationsRejectedException; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.ZooKeeperInstance; import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.util.CleanUp; import org.apache.hadoop.io.Text; import org.apache.zookeeper.KeeperException; import com.yahoo.ycsb.ByteArrayByteIterator; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; public class AccumuloClient extends DB { // Error code constants. public static final int Ok = 0; public static final int ServerError = -1; public static final int HttpError = -2; public static final int NoMatchingRecord = -3; private ZooKeeperInstance _inst; private Connector _connector; private String _table = ""; private BatchWriter _bw = null; private Text _colFam = new Text(""); private Scanner _singleScanner = null; // A scanner for reads/deletes. private Scanner _scanScanner = null; // A scanner for use by scan() private static final String PC_PRODUCER = "producer"; private static final String PC_CONSUMER = "consumer"; private String _PC_FLAG = ""; private ZKProducerConsumer.Queue q = null; private static Hashtable hmKeyReads = null; private static Hashtable hmKeyNumReads = null; private Random r = null; @Override public void init() throws DBException { _colFam = new Text(getProperties().getProperty("accumulo.columnFamily")); _inst = new ZooKeeperInstance(getProperties().getProperty("accumulo.instanceName"), getProperties().getProperty("accumulo.zooKeepers")); try { String principal = getProperties().getProperty("accumulo.username"); AuthenticationToken token = new PasswordToken(getProperties().getProperty("accumulo.password")); _connector = _inst.getConnector(principal, token); } catch (AccumuloException e) { throw new DBException(e); } catch (AccumuloSecurityException e) { throw new DBException(e); } _PC_FLAG = getProperties().getProperty("accumulo.PC_FLAG","none"); if (_PC_FLAG.equals(PC_PRODUCER) || _PC_FLAG.equals(PC_CONSUMER)) { System.out.println("*** YCSB Client is "+_PC_FLAG); String address = getProperties().getProperty("accumulo.PC_SERVER"); String root = getProperties().getProperty("accumulo.PC_ROOT_IN_ZK"); System.out.println("*** PC_INFO(server:"+address+";root="+root+")"); q = new ZKProducerConsumer.Queue(address, root); r = new Random(); } if (_PC_FLAG.equals(PC_CONSUMER)) { hmKeyReads = new Hashtable(); hmKeyNumReads = new Hashtable(); keyNotification(null); } } @Override public void cleanup() throws DBException { try { if (_bw != null) { _bw.close(); } } catch (MutationsRejectedException e) { throw new DBException(e); } CleanUp.shutdownNow(); } /** * Commonly repeated functionality: Before doing any operation, make sure * we're working on the correct table. If not, open the correct one. * * @param table */ public void checkTable(String table) throws TableNotFoundException { if (!_table.equals(table)) { getTable(table); } } /** * Called when the user specifies a table that isn't the same as the * existing table. Connect to it and if necessary, close our current * connection. * * @param table */ public void getTable(String table) throws TableNotFoundException { if (_bw != null) { // Close the existing writer if necessary. try { _bw.close(); } catch (MutationsRejectedException e) { // Couldn't spit out the mutations we wanted. // Ignore this for now. } } BatchWriterConfig bwc = new BatchWriterConfig(); bwc.setMaxLatency(Long.parseLong(getProperties().getProperty("accumulo.batchWriterMaxLatency", "30000")), TimeUnit.MILLISECONDS); bwc.setMaxMemory(Long.parseLong(getProperties().getProperty("accumulo.batchWriterSize", "100000"))); bwc.setMaxWriteThreads(Integer.parseInt(getProperties().getProperty("accumulo.batchWriterThreads", "1"))); _bw = _connector.createBatchWriter(table, bwc); // Create our scanners _singleScanner = _connector.createScanner(table, Authorizations.EMPTY); _scanScanner = _connector.createScanner(table, Authorizations.EMPTY); _table = table; // Store the name of the table we have open. } /** * Gets a scanner from Accumulo over one row * * @param row the row to scan * @param fields the set of columns to scan * @return an Accumulo {@link Scanner} bound to the given row and columns */ private Scanner getRow(Text row, Set fields) { _singleScanner.clearColumns(); _singleScanner.setRange(new Range(row)); if (fields != null) { for(String field:fields) { _singleScanner.fetchColumn(_colFam, new Text(field)); } } return _singleScanner; } @Override public int read(String table, String key, Set fields, HashMap result) { try { checkTable(table); } catch (TableNotFoundException e) { System.err.println("Error trying to connect to Accumulo table." + e); return ServerError; } try { // Pick out the results we care about. for (Entry entry : getRow(new Text(key), null)) { Value v = entry.getValue(); byte[] buf = v.get(); result.put(entry.getKey().getColumnQualifier().toString(), new ByteArrayByteIterator(buf)); } } catch (Exception e) { System.err.println("Error trying to reading Accumulo table" + key + e); return ServerError; } return Ok; } @Override public int scan(String table, String startkey, int recordcount, Set fields, Vector> result) { try { checkTable(table); } catch (TableNotFoundException e) { System.err.println("Error trying to connect to Accumulo table." + e); return ServerError; } // There doesn't appear to be a way to create a range for a given // LENGTH. Just start and end keys. So we'll do this the hard way for now: // Just make the end 'infinity' and only read as much as we need. _scanScanner.clearColumns(); _scanScanner.setRange(new Range(new Text(startkey), null)); // Batch size is how many key/values to try to get per call. Here, I'm // guessing that the number of keys in a row is equal to the number of fields // we're interested in. // We try to fetch one more so as to tell when we've run out of fields. if (fields != null) { // And add each of them as fields we want. for(String field:fields) { _scanScanner.fetchColumn(_colFam, new Text(field)); } } else { // If no fields are provided, we assume one column/row. } String rowKey = ""; HashMap currentHM = null; int count = 0; // Begin the iteration. for (Entry entry : _scanScanner) { // Check for a new row. if (!rowKey.equals(entry.getKey().getRow().toString())) { if (count++ == recordcount) { // Done reading the last row. break; } rowKey = entry.getKey().getRow().toString(); if (fields != null) { // Initial Capacity for all keys. currentHM = new HashMap(fields.size()); } else { // An empty result map. currentHM = new HashMap(); } result.add(currentHM); } // Now add the key to the hashmap. Value v = entry.getValue(); byte[] buf = v.get(); currentHM.put(entry.getKey().getColumnQualifier().toString(), new ByteArrayByteIterator(buf)); } return Ok; } @Override public int update(String table, String key, HashMap values) { try { checkTable(table); } catch (TableNotFoundException e) { System.err.println("Error trying to connect to Accumulo table." + e); return ServerError; } Mutation mutInsert = new Mutation(new Text(key)); for (Map.Entry entry : values.entrySet()) { mutInsert.put(_colFam, new Text(entry.getKey()), System .currentTimeMillis(), new Value(entry.getValue().toArray())); } try { _bw.addMutation(mutInsert); // Distributed YCSB co-ordination: YCSB on a client produces the key to // be stored in the shared queue in ZooKeeper. if (_PC_FLAG.equals(PC_PRODUCER)) { if (r.nextFloat() < 0.01) keyNotification(key); } } catch (MutationsRejectedException e) { System.err.println("Error performing update."); e.printStackTrace(); return ServerError; } return Ok; } @Override public int insert(String table, String key, HashMap values) { return update(table, key, values); } @Override public int delete(String table, String key) { try { checkTable(table); } catch (TableNotFoundException e) { System.err.println("Error trying to connect to Accumulo table." + e); return ServerError; } try { deleteRow(new Text(key)); } catch (RuntimeException e) { System.err.println("Error performing delete."); e.printStackTrace(); return ServerError; } return Ok; } // These functions are adapted from RowOperations.java: private void deleteRow(Text row) { deleteRow(getRow(row, null)); } /** * Deletes a row, given a Scanner of JUST that row * */ private void deleteRow(Scanner scanner) { Mutation deleter = null; // iterate through the keys for (Entry entry : scanner) { // create a mutation for the row if (deleter == null) deleter = new Mutation(entry.getKey().getRow()); // the remove function adds the key with the delete flag set to true deleter.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier()); } try { _bw.addMutation(deleter); } catch (MutationsRejectedException e) { // TODO Auto-generated catch block e.printStackTrace(); } } private void keyNotification(String key) { if (_PC_FLAG.equals(PC_PRODUCER)) { try { q.produce(key); } catch (KeeperException e) { } catch (InterruptedException e) { } } else { //XXX: do something better to keep the loop going (while??) for (int i = 0; i < 10000000; i++) { try { String strKey = q.consume(); if ((hmKeyReads.containsKey(strKey) == false) && (hmKeyNumReads.containsKey(strKey) == false)) { hmKeyReads.put(strKey, new Long(System.currentTimeMillis())); hmKeyNumReads.put(strKey, new Integer(1)); } //YCSB Consumer will read the key that was fetched from the //queue in ZooKeeper. //(current way is kind of ugly but works, i think) //TODO : Get table name from configuration or argument String table = "usertable"; HashSet fields = new HashSet(); for (int j=0; j<9; j++) fields.add("field"+j); HashMap result = new HashMap(); int retval = read(table, strKey, fields, result); //If the results are empty, the key is enqueued in Zookeeper //and tried again, until the results are found. if (result.size() == 0) { q.produce(strKey); int count = ((Integer)hmKeyNumReads.get(strKey)).intValue(); hmKeyNumReads.put(strKey, new Integer(count+1)); } else { if (((Integer)hmKeyNumReads.get(strKey)).intValue() > 1) { long currTime = System.currentTimeMillis(); long writeTime = ((Long)hmKeyReads.get(strKey)).longValue(); System.out.println("Key="+strKey+ //";StartSearch="+writeTime+ //";EndSearch="+currTime+ ";TimeLag="+(currTime-writeTime)); } } } catch (KeeperException e) { } catch (InterruptedException e) { } } } } public int presplit(String table, String[] keys) { TreeSet splits = new TreeSet(); for (int i = 0;i < keys.length; i ++) { splits.add(new Text(keys[i])); } try { _connector.tableOperations().addSplits(table, splits); } catch (TableNotFoundException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (AccumuloException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (AccumuloSecurityException e) { // TODO Auto-generated catch block e.printStackTrace(); } return Ok; } } diff --git a/accumulo/src/main/java/com/yahoo/ycsb/db/ZKProducerConsumer.java b/accumulo/src/main/java/com/yahoo/ycsb/db/ZKProducerConsumer.java index 2daec519..09741641 100644 --- a/accumulo/src/main/java/com/yahoo/ycsb/db/ZKProducerConsumer.java +++ b/accumulo/src/main/java/com/yahoo/ycsb/db/ZKProducerConsumer.java @@ -1,122 +1,140 @@ +/** + * Copyright (c) 2011 YCSB++ project, 2014 YCSB contributors. + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + package com.yahoo.ycsb.db; import java.io.IOException; import java.util.List; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.Watcher; import org.apache.zookeeper.ZooKeeper; import org.apache.zookeeper.ZooDefs.Ids; import org.apache.zookeeper.data.Stat; // Implementing the PC Queue in ZooKeeper // public class ZKProducerConsumer implements Watcher { static ZooKeeper zk = null; static Integer mutex; String root; // Constructor that takes tha address of the ZK server // ZKProducerConsumer(String address) { if(zk == null){ try { System.out.println("Starting ZK:"); zk = new ZooKeeper(address, 3000, this); mutex = new Integer(-1); System.out.println("Finished starting ZK: " + zk); } catch (IOException e) { System.out.println(e.toString()); zk = null; } } //else mutex = new Integer(-1); } synchronized public void process(WatchedEvent event) { synchronized (mutex) { //System.out.println("Process: " + event.getType()); mutex.notify(); } } static public class QueueElement { public String key; public long writeTime; QueueElement(String key, long writeTime) { this.key = key; this.writeTime = writeTime; } } // Producer-Consumer queue static public class Queue extends ZKProducerConsumer { // Constructor of producer-consumer queue Queue(String address, String name) { super(address); this.root = name; // Create ZK node name if (zk != null) { try { Stat s = zk.exists(root, false); if (s == null) { zk.create(root, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); } } catch (KeeperException e) { System.out .println("Keeper exception when instantiating queue: " + e.toString()); } catch (InterruptedException e) { System.out.println("Interrupted exception"); } } } // Producer calls this method to insert the key in the queue // boolean produce(String key) throws KeeperException, InterruptedException{ byte[] value; value = key.getBytes(); zk.create(root + "/key", value, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT_SEQUENTIAL); return true; } // Consumer calls this method to "wait" for the key to the available // String consume() throws KeeperException, InterruptedException { String retvalue = null; Stat stat = null; // Get the first element available while (true) { synchronized (mutex) { List list = zk.getChildren(root, true); if (list.size() == 0) { System.out.println("Going to wait"); mutex.wait(); } else { String path = root+"/"+list.get(0); byte[] b = zk.getData(path, false, stat); retvalue = new String(b); zk.delete(path, -1); return retvalue; } } } } } } diff --git a/aerospike/README.md b/aerospike/README.md index 4c5e5dc6..c5bb13ef 100644 --- a/aerospike/README.md +++ b/aerospike/README.md @@ -1,41 +1,58 @@ + + ## Quick Start This section describes how to run YCSB on Aerospike. ### 1. Start Aerospike ### 2. Install Java and Maven ### 3. Set Up YCSB Git clone YCSB and compile: git clone http://github.com/brianfrankcooper/YCSB.git cd YCSB mvn -pl com.yahoo.ycsb:aerospike-binding -am clean package ### 4. Provide Aerospike Connection Parameters The following connection parameters are available. * `as.host` - The Aerospike cluster to connect to (default: `localhost`) * `as.port` - The port to connect to (default: `3000`) * `as.user` - The user to connect as (no default) * `as.password` - The password for the user (no default) * `as.timeout` - The transaction and connection timeout (in ms, default: `10000`) * `as.namespace` - The namespace to be used for the benchmark (default: `ycsb`) Add them to the workload or set them with the shell command, as in: ./bin/ycsb load aerospike -s -P workloads/workloada -p as.timeout=5000 >outputLoad.txt ### 5. Load Data and Run Tests Load the data: ./bin/ycsb load aerospike -s -P workloads/workloada >outputLoad.txt Run the workload test: ./bin/ycsb run aerospike -s -P workloads/workloada >outputRun.txt diff --git a/aerospike/pom.xml b/aerospike/pom.xml index 736201fe..8cf954b4 100644 --- a/aerospike/pom.xml +++ b/aerospike/pom.xml @@ -1,28 +1,45 @@ + + 4.0.0 com.yahoo.ycsb binding-parent 0.3.0-RC4-SNAPSHOT ../binding-parent aerospike-binding Aerospike DB Binding jar com.aerospike aerospike-client ${aerospike.version} com.yahoo.ycsb core ${project.version} provided diff --git a/aerospike/src/main/java/com/yahoo/ycsb/db/AerospikeClient.java b/aerospike/src/main/java/com/yahoo/ycsb/db/AerospikeClient.java index 73ccd386..dc872061 100644 --- a/aerospike/src/main/java/com/yahoo/ycsb/db/AerospikeClient.java +++ b/aerospike/src/main/java/com/yahoo/ycsb/db/AerospikeClient.java @@ -1,199 +1,216 @@ +/** + * Copyright (c) 2015 YCSB contributors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + package com.yahoo.ycsb.db; import java.util.HashMap; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.Vector; import com.aerospike.client.AerospikeException; import com.aerospike.client.Bin; import com.aerospike.client.Key; import com.aerospike.client.Record; import com.aerospike.client.ResultCode; import com.aerospike.client.policy.ClientPolicy; import com.aerospike.client.policy.Policy; import com.aerospike.client.policy.RecordExistsAction; import com.aerospike.client.policy.WritePolicy; import com.yahoo.ycsb.ByteArrayByteIterator; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DBException; public class AerospikeClient extends com.yahoo.ycsb.DB { private static final boolean DEBUG = false; private static final String DEFAULT_HOST = "localhost"; private static final String DEFAULT_PORT = "3000"; private static final String DEFAULT_TIMEOUT = "10000"; private static final String DEFAULT_NAMESPACE = "ycsb"; private static final int RESULT_OK = 0; private static final int RESULT_ERROR = -1; private static final int WRITE_OVERLOAD_DELAY = 5; private static final int WRITE_OVERLOAD_TRIES = 3; private String namespace = null; private com.aerospike.client.AerospikeClient client = null; private Policy readPolicy = new Policy(); private WritePolicy insertPolicy = new WritePolicy(); private WritePolicy updatePolicy = new WritePolicy(); private WritePolicy deletePolicy = new WritePolicy(); @Override public void init() throws DBException { insertPolicy.recordExistsAction = RecordExistsAction.CREATE_ONLY; updatePolicy.recordExistsAction = RecordExistsAction.UPDATE_ONLY; Properties props = getProperties(); namespace = props.getProperty("as.namespace", DEFAULT_NAMESPACE); String host = props.getProperty("as.host", DEFAULT_HOST); String user = props.getProperty("as.user"); String password = props.getProperty("as.password"); int port = Integer.parseInt(props.getProperty("as.port", DEFAULT_PORT)); int timeout = Integer.parseInt(props.getProperty("as.timeout", DEFAULT_TIMEOUT)); readPolicy.timeout = timeout; insertPolicy.timeout = timeout; updatePolicy.timeout = timeout; deletePolicy.timeout = timeout; ClientPolicy clientPolicy = new ClientPolicy(); if (user != null && password != null) { clientPolicy.user = user; clientPolicy.password = password; } try { client = new com.aerospike.client.AerospikeClient(clientPolicy, host, port); } catch (AerospikeException e) { throw new DBException(String.format("Error while creating Aerospike " + "client for %s:%d.", host, port), e); } } @Override public void cleanup() throws DBException { client.close(); } @Override public int read(String table, String key, Set fields, HashMap result) { try { Record record; if (fields != null) { record = client.get(readPolicy, new Key(namespace, table, key), fields.toArray(new String[fields.size()])); } else { record = client.get(readPolicy, new Key(namespace, table, key)); } if (record == null) { if (DEBUG) { System.err.println("Record key " + key + " not found (read)"); } return RESULT_ERROR; } for (Map.Entry entry: record.bins.entrySet()) { result.put(entry.getKey(), new ByteArrayByteIterator((byte[])entry.getValue())); } return RESULT_OK; } catch (AerospikeException e) { System.err.println("Error while reading key " + key + ": " + e); return RESULT_ERROR; } } @Override public int scan(String table, String start, int count, Set fields, Vector> result) { System.err.println("Scan not implemented"); return RESULT_ERROR; } private int write(String table, String key, WritePolicy writePolicy, HashMap values) { Bin[] bins = new Bin[values.size()]; int index = 0; for (Map.Entry entry: values.entrySet()) { bins[index] = new Bin(entry.getKey(), entry.getValue().toArray()); ++index; } int delay = WRITE_OVERLOAD_DELAY; Key keyObj = new Key(namespace, table, key); for (int tries = 0; tries < WRITE_OVERLOAD_TRIES; ++tries) { try { client.put(writePolicy, keyObj, bins); return RESULT_OK; } catch (AerospikeException e) { if (e.getResultCode() != ResultCode.DEVICE_OVERLOAD) { System.err.println("Error while writing key " + key + ": " + e); return RESULT_ERROR; } try { Thread.sleep(delay); } catch (InterruptedException e2) { if (DEBUG) { System.err.println("Interrupted: " + e2); } } delay *= 2; } } if (DEBUG) { System.err.println("Device overload"); } return RESULT_ERROR; } @Override public int update(String table, String key, HashMap values) { return write(table, key, updatePolicy, values); } @Override public int insert(String table, String key, HashMap values) { return write(table, key, insertPolicy, values); } @Override public int delete(String table, String key) { try { if (!client.delete(deletePolicy, new Key(namespace, table, key))) { if (DEBUG) { System.err.println("Record key " + key + " not found (delete)"); } return RESULT_ERROR; } return RESULT_OK; } catch (AerospikeException e) { System.err.println("Error while deleting key " + key + ": " + e); return RESULT_ERROR; } } } diff --git a/bin/ycsb b/bin/ycsb index ea1f6530..3f32ea9a 100755 --- a/bin/ycsb +++ b/bin/ycsb @@ -1,209 +1,225 @@ #!/usr/bin/env python +# +# Copyright (c) 2012 - 2015 YCSB contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. See accompanying +# LICENSE file. +# import argparse import fnmatch import io import os import shlex import sys import subprocess BASE_URL = "https://github.com/brianfrankcooper/YCSB/tree/master/" COMMANDS = { "shell" : { "command" : "", "description" : "Interactive mode", "main" : "com.yahoo.ycsb.CommandLine", }, "load" : { "command" : "-load", "description" : "Execute the load phase", "main" : "com.yahoo.ycsb.Client", }, "run" : { "command" : "-t", "description" : "Execute the transaction phase", "main" : "com.yahoo.ycsb.Client", }, } DATABASES = { "accumulo" : "com.yahoo.ycsb.db.AccumuloClient", "aerospike" : "com.yahoo.ycsb.db.AerospikeClient", "basic" : "com.yahoo.ycsb.BasicDB", "cassandra-7" : "com.yahoo.ycsb.db.CassandraClient7", "cassandra-8" : "com.yahoo.ycsb.db.CassandraClient8", "cassandra-10" : "com.yahoo.ycsb.db.CassandraClient10", "cassandra-cql": "com.yahoo.ycsb.db.CassandraCQLClient", "couchbase" : "com.yahoo.ycsb.db.CouchbaseClient", "dynamodb" : "com.yahoo.ycsb.db.DynamoDBClient", "elasticsearch": "com.yahoo.ycsb.db.ElasticSearchClient", "gemfire" : "com.yahoo.ycsb.db.GemFireClient", "hbase" : "com.yahoo.ycsb.db.HBaseClient", "hbase-10" : "com.yahoo.ycsb.db.HBaseClient10", "hypertable" : "com.yahoo.ycsb.db.HypertableClient", "infinispan-cs": "com.yahoo.ycsb.db.InfinispanRemoteClient", "infinispan" : "com.yahoo.ycsb.db.InfinispanClient", "jdbc" : "com.yahoo.ycsb.db.JdbcDBClient", "mapkeeper" : "com.yahoo.ycsb.db.MapKeeperClient", "mongodb" : "com.yahoo.ycsb.db.MongoDbClient", "mongodb-async": "com.yahoo.ycsb.db.AsyncMongoDbClient", "nosqldb" : "com.yahoo.ycsb.db.NoSqlDbClient", "orientdb" : "com.yahoo.ycsb.db.OrientDBClient", "redis" : "com.yahoo.ycsb.db.RedisClient", "tarantool" : "com.yahoo.ycsb.db.TarantoolClient", "voldemort" : "com.yahoo.ycsb.db.VoldemortClient" } OPTIONS = { "-P file" : "Specify workload file", "-p key=value" : "Override workload property", "-s" : "Print status to stderr", "-target n" : "Target ops/sec (default: unthrottled)", "-threads n" : "Number of client threads (default: 1)", "-cp path" : "Additional Java classpath entries", "-jvm-args args" : "Additional arguments to the JVM", } def usage(): output = io.BytesIO() print >> output, "%s command database [options]" % sys.argv[0] print >> output, "\nCommands:" for command in sorted(COMMANDS.keys()): print >> output, " %s %s" % (command.ljust(14), COMMANDS[command]["description"]) print >> output, "\nDatabases:" for db in sorted(DATABASES.keys()): print >> output, " %s %s" % (db.ljust(14), BASE_URL + db.split("-")[0]) print >> output, "\nOptions:" for option in sorted(OPTIONS.keys()): print >> output, " %s %s" % (option.ljust(14), OPTIONS[option]) print >> output, """\nWorkload Files: There are various predefined workloads under workloads/ directory. See https://github.com/brianfrankcooper/YCSB/wiki/Core-Properties for the list of workload properties.""" return output.getvalue() def debug(message): print >> sys.stderr, "[DEBUG] ", message def warn(message): print >> sys.stderr, "[WARN] ", message def error(message): print >> sys.stderr, "[ERROR] ", message def find_jars(dir, glob='*.jar'): jars = [] for (dirpath, dirnames, filenames) in os.walk(dir): for filename in fnmatch.filter(filenames, glob): jars.append(os.path.join(dirpath, filename)) return jars def get_ycsb_home(): dir = os.path.abspath(os.path.dirname(sys.argv[0])) while "LICENSE.txt" not in os.listdir(dir): dir = os.path.join(dir, os.path.pardir) return os.path.abspath(dir) def is_distribution(): # If there's a top level pom, we're a source checkout. otherwise a dist artifact return "pom.xml" not in os.listdir(get_ycsb_home()) # Run the maven dependency plugin to get the local jar paths. # presumes maven can run, so should only be run on source checkouts # will invoke the 'package' goal for the given binding in order to resolve intra-project deps # presumes maven properly handles system-specific path separators def get_classpath_from_maven(database): try: debug("Running 'mvn -pl com.yahoo.ycsb:"+database+"-binding -am package -DskipTests " "dependency:build-classpath -DincludeScope=compile -Dmdep.outputFilterFile=true'") mvn_output = subprocess.check_output(["mvn", "-pl", "com.yahoo.ycsb:"+database+"-binding", "-am", "package", "-DskipTests", "dependency:build-classpath", "-DincludeScope=compile", "-Dmdep.outputFilterFile=true"]) # the above outputs a "classpath=/path/tojar:/path/to/other/jar" for each module # the last module will be the datastore binding line = [x for x in mvn_output.splitlines() if x.startswith("classpath=")][-1:] return line[0][len("classpath="):] except subprocess.CalledProcessError, err: error("Attempting to generate a classpath from Maven failed " "with return code '" + str(err.returncode) + "'. The output from " "Maven follows, try running " "'mvn -DskipTests package dependency:build=classpath' on your " "own and correct errors." + os.linesep + os.linesep + "mvn output:" + os.linesep + err.output) sys.exit(err.returncode) def main(): p = argparse.ArgumentParser( usage=usage(), formatter_class=argparse.RawDescriptionHelpFormatter) p.add_argument('-cp', dest='classpath', help="""Additional classpath entries, e.g. '-cp /tmp/hbase-1.0.1.1/conf'. Will be prepended to the YCSB classpath.""") p.add_argument("-jvm-args", default=[], type=shlex.split, help="""Additional arguments to pass to 'java', e.g. '-Xmx4g'""") p.add_argument("command", choices=sorted(COMMANDS), help="""Command to run.""") p.add_argument("database", choices=sorted(DATABASES), help="""Database to test.""") args, remaining = p.parse_known_args() ycsb_home = get_ycsb_home() # Use JAVA_HOME to find java binary if set, otherwise just use PATH. java = "java" java_home = os.getenv("JAVA_HOME") if java_home: java = os.path.join(java_home, "bin", "java") db_classname = DATABASES[args.database] command = COMMANDS[args.command]["command"] main_classname = COMMANDS[args.command]["main"] # Classpath set up binding = args.database.split("-")[0] if is_distribution(): db_dir = os.path.join(ycsb_home, binding + "-binding") # include top-level conf for when we're a binding-specific artifact. # If we add top-level conf to the general artifact, starting here # will allow binding-specific conf to override (because it's prepended) cp = [os.path.join(ycsb_home, "conf")] cp.extend(find_jars(os.path.join(ycsb_home, "lib"))) cp.extend(find_jars(os.path.join(db_dir, "lib"))) else: warn("Running against a source checkout. In order to get our runtime " "dependencies we'll have to invoke Maven. Depending on the state " "of your system, this may take ~30-45 seconds") db_dir = os.path.join(ycsb_home, binding) # goes first so we can rely on side-effect of package maven_says = get_classpath_from_maven(binding) # TODO when we have a version property, skip the glob cp = find_jars(os.path.join(db_dir, "target"), binding + "-binding*.jar") # alredy in jar:jar:jar form cp.append(maven_says) cp.insert(0, os.path.join(db_dir, "conf")) classpath = os.pathsep.join(cp) if args.classpath: classpath = os.pathsep.join([args.classpath, classpath]) ycsb_command = ([java] + args.jvm_args + ["-cp", classpath, main_classname, "-db", db_classname] + remaining) if command: ycsb_command.append(command) print >> sys.stderr, " ".join(ycsb_command) return subprocess.call(ycsb_command) if __name__ == '__main__': sys.exit(main()) diff --git a/binding-parent/datastore-specific-descriptor/pom.xml b/binding-parent/datastore-specific-descriptor/pom.xml index 597b0740..1edd9549 100644 --- a/binding-parent/datastore-specific-descriptor/pom.xml +++ b/binding-parent/datastore-specific-descriptor/pom.xml @@ -1,27 +1,44 @@ + + 4.0.0 com.yahoo.ycsb root 0.3.0-RC4-SNAPSHOT ../../ datastore-specific-descriptor Per Datastore Binding descriptor jar This module contains the assembly descriptor used by the individual components to build binding-specific distributions. com.yahoo.ycsb core ${project.version} diff --git a/binding-parent/datastore-specific-descriptor/src/main/resources/assemblies/datastore-specific-assembly.xml b/binding-parent/datastore-specific-descriptor/src/main/resources/assemblies/datastore-specific-assembly.xml index 0d667dff..5783421c 100644 --- a/binding-parent/datastore-specific-descriptor/src/main/resources/assemblies/datastore-specific-assembly.xml +++ b/binding-parent/datastore-specific-descriptor/src/main/resources/assemblies/datastore-specific-assembly.xml @@ -1,60 +1,77 @@ + + dist true ycsb-${artifactId}-${version} README.md .. 0644 LICENSE.txt NOTICE.txt ../bin bin 0755 ycsb ../workloads workloads 0644 src/main/conf conf 0644 lib com.yahoo.ycsb:core provided true lib *:jar:* *:sources diff --git a/binding-parent/pom.xml b/binding-parent/pom.xml index 54820372..da8da1ae 100644 --- a/binding-parent/pom.xml +++ b/binding-parent/pom.xml @@ -1,102 +1,119 @@ + + 4.0.0 com.yahoo.ycsb root 0.3.0-RC4-SNAPSHOT binding-parent YCSB Datastore Binding Parent pom This module acts as the parent for new datastore bindings. It creates a datastore specific binary artifact. datastore-specific-descriptor org.apache.maven.plugins maven-assembly-plugin ${maven.assembly.version} com.yahoo.ycsb datastore-specific-descriptor ${project.version} datastore-specific-assembly ycsb-${project.artifactId}-${project.version} tar.gz false package single org.apache.maven.plugins maven-dependency-plugin ${maven.dependency.version} org.apache.maven.plugins maven-dependency-plugin stage-dependencies package copy-dependencies runtime datastore-binding README.md org.apache.maven.plugins maven-assembly-plugin diff --git a/cassandra/pom.xml b/cassandra/pom.xml index 34bb6df8..53c46c97 100644 --- a/cassandra/pom.xml +++ b/cassandra/pom.xml @@ -1,34 +1,51 @@ + + 4.0.0 com.yahoo.ycsb binding-parent 0.3.0-RC4-SNAPSHOT ../binding-parent cassandra-binding Cassandra DB Binding jar org.apache.cassandra cassandra-all ${cassandra.version} com.datastax.cassandra cassandra-driver-core ${cassandra.cql.version} com.yahoo.ycsb core ${project.version} provided diff --git a/checkstyle.xml b/checkstyle.xml index eda4a471..029f0abd 100644 --- a/checkstyle.xml +++ b/checkstyle.xml @@ -1,169 +1,186 @@ + + diff --git a/core/CHANGES.md b/core/CHANGES.md index d6a53d48..05a4b38e 100644 --- a/core/CHANGES.md +++ b/core/CHANGES.md @@ -1,67 +1,84 @@ + + When used as a latency under load benchmark YCSB in it's original form suffers from Coordinated Omission[1] and related measurement issue: * Load is controlled by response time * Measurement does not account for missing time * Measurement starts at beginning of request rather than at intended beginning * Measurement is limited in scope as the histogram does not provide data on overflow values To provide a minimal correction patch the following were implemented: 1. Replace internal histogram implementation with HdrHistogram[2]: HdrHistogram offers a dynamic range of measurement at a given precision and will improve the fidelity of reporting. It allows capturing a much wider range of latencies. HdrHistogram also supports compressed loss-less serialization which enable capturing snapshot histograms from which lower resolution histograms can be constructed for plotting latency over time. Snapshot interval histograms are serialized on status reporting which must be enabled using the '-s' option. 2. Track intended operation start and report latencies from that point in time: Assuming the benchmark sets a target schedule of execution in which every operation is supposed to happen at a given time the benchmark should measure the latency between intended start time and operation completion. This required the introduction of a new measurement point and inevitably includes measuring some of the internal preparation steps of the load generator. These overhead should be negligible in the context of a network hop, but could be corrected for by estimating the load-generator overheads (e.g. by measuring a no-op DB or by measuring the setup time for an operation and deducting that from total). This intended measurement point is only used when there is a target load (specified by the -target paramaeter) This branch supports the following new options: * -p measurementtype=[histogram|hdrhistogram|hdrhistogram+histogram|timeseries] (default=histogram) The new measurement types are hdrhistogram and hdrhistogram+histogram. Default is still histogram, which is the old histogram. Ultimately we would remove the old measurement types and use only HdrHistogram but the old measurement is left in there for comparison sake. * -p measurement.interval=[op|intended|both] (default=op) This new option deferentiates between measured intervals and adds the intended interval(as described) above, and the option to record both the op and intended for comparison. * -p hdrhistogram.fileoutput=[true|false] (default=false) This new option will enable periodical writes of the interval histogram into an output file. The path can be set using '-p hdrhistogram.output.path='. Example parameters: -target 1000 -s -p workload=com.yahoo.ycsb.workloads.CoreWorkload -p basicdb.verbose=false -p basicdb.simulatedelay=4 -p measurement.interval=both -p measurementtype=hdrhistogram -p hdrhistogram.fileoutput=true -p maxexecutiontime=60 Further changes made: * -p status.interval= (default=10) Controls the number of seconds between status reports and therefore between HdrHistogram snapshots reported. * -p basicdb.randomizedelay=[true|false] (default=true) Controls weather the delay simulated by the mock DB is uniformly random or not. Further suggestions: 1. Correction load control: currently after a pause the load generator will do operations back to back to catchup, this leads to a flat out throughput mode of testing as opposed to controlled load. 2. Move to async model: Scenarios where Ops have no dependency could delegate the Op execution to a threadpool and thus separate the request rate control from the synchronous execution of Ops. Measurement would start on queuing for execution. 1. https://groups.google.com/forum/#!msg/mechanical-sympathy/icNZJejUHfE/BfDekfBEs_sJ 2. https://github.com/HdrHistogram/HdrHistogram \ No newline at end of file diff --git a/core/pom.xml b/core/pom.xml index f68fb3cc..39af107c 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -1,64 +1,81 @@ + + 4.0.0 com.yahoo.ycsb root 0.3.0-RC4-SNAPSHOT core Core YCSB jar 1.9.4 org.codehaus.jackson jackson-mapper-asl ${jackson.api.version} org.codehaus.jackson jackson-core-asl ${jackson.api.version} org.testng testng 6.1.1 test org.hdrhistogram HdrHistogram 2.1.4 org.apache.maven.plugins maven-assembly-plugin ${maven.assembly.version} jar-with-dependencies false package single diff --git a/core/src/test/java/com/yahoo/ycsb/TestByteIterator.java b/core/src/test/java/com/yahoo/ycsb/TestByteIterator.java index b95415b0..7edbc947 100644 --- a/core/src/test/java/com/yahoo/ycsb/TestByteIterator.java +++ b/core/src/test/java/com/yahoo/ycsb/TestByteIterator.java @@ -1,22 +1,39 @@ +/** + * Copyright (c) 2012 YCSB contributors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + package com.yahoo.ycsb; import org.testng.annotations.Test; import static org.testng.AssertJUnit.*; public class TestByteIterator { @Test public void testRandomByteIterator() { int size = 100; ByteIterator itor = new RandomByteIterator(size); assertTrue(itor.hasNext()); assertEquals(size, itor.bytesLeft()); assertEquals(size, itor.toString().getBytes().length); assertFalse(itor.hasNext()); assertEquals(0, itor.bytesLeft()); itor = new RandomByteIterator(size); assertEquals(size, itor.toArray().length); assertFalse(itor.hasNext()); assertEquals(0, itor.bytesLeft()); } } diff --git a/couchbase/README.md b/couchbase/README.md index d688d8c9..4004cb7f 100644 --- a/couchbase/README.md +++ b/couchbase/README.md @@ -1,50 +1,67 @@ + + # Couchbase Driver for YCSB This driver is a binding for the YCSB facilities to operate against a Couchbase Server cluster. It uses the official Couchbase Java SDK and provides a rich set of configuration options. ## Quickstart ### 1. Start Couchbase Server You need to start a single node or a cluster to point the client at. Please see [http://couchbase.com](couchbase.com) for more details and instructions. ### 2. Set up YCSB You need to clone the repository and compile everything. ``` git clone git://github.com/brianfrankcooper/YCSB.git cd YCSB mvn clean package ``` ### 3. Run the Workload Before you can actually run the workload, you need to "load" the data first. ``` bin/ycsb load couchbase -s -P workloads/workloada ``` Then, you can run the workload: ``` bin/ycsb run couchbase -s -P workloads/workloada ``` Please see the general instructions in the `doc` folder if you are not sure how it all works. You can apply a property (as seen in the next section) like this: ``` bin/ycsb run couchbase -s -P workloads/workloada -p couchbase.useJson=false ``` ## Configuration Options Since no setup is the same and the goal of YCSB is to deliver realistic benchmarks, here are some setups that you can tune. Note that if you need more flexibility (let's say a custom transcoder), you still need to extend this driver and implement the facilities on your own. You can set the following properties (with the default settings applied): - couchbase.url=http://127.0.0.1:8091/pools => The connection URL from one server. - couchbase.bucket=default => The bucket name to use. - couchbase.password= => The password of the bucket. - couchbase.checkFutures=true => If the futures should be inspected (makes ops sync). - couchbase.persistTo=0 => Observe Persistence ("PersistTo" constraint). - couchbase.replicateTo=0 => Observe Replication ("ReplicateTo" constraint). - couchbase.json=true => Use json or java serialization as target format. diff --git a/couchbase/pom.xml b/couchbase/pom.xml index 33e634e8..bb13d30c 100644 --- a/couchbase/pom.xml +++ b/couchbase/pom.xml @@ -1,45 +1,62 @@ + + 4.0.0 com.yahoo.ycsb binding-parent 0.3.0-RC4-SNAPSHOT ../binding-parent couchbase-binding Couchbase Binding jar couchbase couchbase-client ${couchbase.version} com.yahoo.ycsb core ${project.version} provided com.fasterxml.jackson.core jackson-databind 2.2.2 org.slf4j slf4j-api couchbase Couchbase Maven Repository http://files.couchbase.com/maven2/ diff --git a/distribution/pom.xml b/distribution/pom.xml index e7cf6285..715e576d 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -1,134 +1,151 @@ + + 4.0.0 com.yahoo.ycsb root 0.3.0-RC4-SNAPSHOT ycsb YCSB Release Distribution Builder pom This module creates the release package of the YCSB with all DB library bindings. It is only used by the build process and does not contain any real code of itself. com.yahoo.ycsb core ${project.version} com.yahoo.ycsb accumulo-binding ${project.version} com.yahoo.ycsb aerospike-binding ${project.version} com.yahoo.ycsb cassandra-binding ${project.version} com.yahoo.ycsb couchbase-binding ${project.version} com.yahoo.ycsb dynamodb-binding ${project.version} com.yahoo.ycsb elasticsearch-binding ${project.version} com.yahoo.ycsb gemfire-binding ${project.version} com.yahoo.ycsb hbase-binding ${project.version} com.yahoo.ycsb hypertable-binding ${project.version} com.yahoo.ycsb infinispan-binding ${project.version} com.yahoo.ycsb jdbc-binding ${project.version} com.yahoo.ycsb mongodb-binding ${project.version} com.yahoo.ycsb orientdb-binding ${project.version} com.yahoo.ycsb redis-binding ${project.version} com.yahoo.ycsb tarantool-binding ${project.version} org.apache.maven.plugins maven-assembly-plugin ${maven.assembly.version} src/main/assembly/distribution.xml false package single diff --git a/distribution/src/main/assembly/distribution.xml b/distribution/src/main/assembly/distribution.xml index 75eb0097..618cee5c 100644 --- a/distribution/src/main/assembly/distribution.xml +++ b/distribution/src/main/assembly/distribution.xml @@ -1,81 +1,98 @@ + + package tar.gz true .. . 0644 README LICENSE.txt NOTICE.txt ../bin bin 0755 ycsb ../workloads workloads 0644 lib com.yahoo.ycsb:core runtime false false true true true true com.yahoo.ycsb:core com.yahoo.ycsb:binding-parent com.yahoo.ycsb:datastore-specific-descriptor com.yahoo.ycsb:ycsb com.yahoo.ycsb:root README.md conf src/main/conf lib target/dependency false ${module.artifactId}/lib false diff --git a/doc/coreproperties.html b/doc/coreproperties.html index 81b2ede0..a03e9673 100644 --- a/doc/coreproperties.html +++ b/doc/coreproperties.html @@ -1,30 +1,47 @@ + + YCSB - Core workload package properties

Yahoo! Cloud Serving Benchmark

Version 0.1.2


Home - Core workloads - Tips and FAQ

Core workload package properties

The property files used with the core workload generator can specify values for the following properties:

  • fieldcount: the number of fields in a record (default: 10)
  • fieldlength: the size of each field (default: 100)
  • readallfields: should reads read all fields (true) or just one (false) (default: true)
  • readproportion: what proportion of operations should be reads (default: 0.95)
  • updateproportion: what proportion of operations should be updates (default: 0.05)
  • insertproportion: what proportion of operations should be inserts (default: 0)
  • scanproportion: what proportion of operations should be scans (default: 0)
  • readmodifywriteproportion: what proportion of operations should be read a record, modify it, write it back (default: 0)
  • requestdistribution: what distribution should be used to select the records to operate on - uniform, zipfian or latest (default: uniform)
  • maxscanlength: for scans, what is the maximum number of records to scan (default: 1000)
  • scanlengthdistribution: for scans, what distribution should be used to choose the number of records to scan, for each scan, between 1 and maxscanlength (default: uniform)
  • insertorder: should records be inserted in order by key ("ordered"), or in hashed order ("hashed") (default: hashed)

YCSB - Yahoo! Research - Contact cooperb@yahoo-inc.com. diff --git a/doc/coreworkloads.html b/doc/coreworkloads.html index d039ac86..e6f195a8 100644 --- a/doc/coreworkloads.html +++ b/doc/coreworkloads.html @@ -1,59 +1,76 @@ + + YCSB - Core workloads

Yahoo! Cloud Serving Benchmark

Version 0.1.2


Home - Core workloads - Tips and FAQ

Core workloads

YCSB includes a set of core workloads that define a basic benchmark for cloud systems. Of course, you can define your own workloads, as described here. However, the core workloads are a useful first step, and obtaining these benchmark numbers for a variety of different systems would allow you to understand the performance tradeoffs of different systems.

The core workloads consist of six different workloads:

Workload A: Update heavy workload

This workload has a mix of 50/50 reads and writes. An application example is a session store recording recent actions.

Workload B: Read mostly workload

This workload has a 95/5 reads/write mix. Application example: photo tagging; add a tag is an update, but most operations are to read tags.

Workload C: Read only

This workload is 100% read. Application example: user profile cache, where profiles are constructed elsewhere (e.g., Hadoop).

Workload D: Read latest workload

In this workload, new records are inserted, and the most recently inserted records are the most popular. Application example: user status updates; people want to read the latest.

Workload E: Short ranges

In this workload, short ranges of records are queried, instead of individual records. Application example: threaded conversations, where each scan is for the posts in a given thread (assumed to be clustered by thread id).

Workload F: Read-modify-write

In this workload, the client will read a record, modify it, and write back the changes. Application example: user database, where user records are read and modified by the user or to record user activity.


Running the workloads

All six workloads have a data set which is similar. Workloads D and E insert records during the test run. Thus, to keep the database size consistent, we recommend the following sequence:
  1. Load the database, using workload A's parameter file (workloads/workloada) and the "-load" switch to the client.
  2. Run workload A (using workloads/workloada and "-t") for a variety of throughputs.
  3. Run workload B (using workloads/workloadb and "-t") for a variety of throughputs.
  4. Run workload C (using workloads/workloadc and "-t") for a variety of throughputs.
  5. Run workload F (using workloads/workloadf and "-t") for a variety of throughputs.
  6. Run workload D (using workloads/workloadd and "-t") for a variety of throughputs. This workload inserts records, increasing the size of the database.
  7. Delete the data in the database.
  8. Reload the database, using workload E's parameter file (workloads/workloade) and the "-load switch to the client.
  9. Run workload E (using workloads/workloadd and "-t") for a variety of throughputs. This workload inserts records, increasing the size of the database.

YCSB - Yahoo! Research - Contact cooperb@yahoo-inc.com. diff --git a/doc/dblayer.html b/doc/dblayer.html index 35cdfdbf..e75b6814 100644 --- a/doc/dblayer.html +++ b/doc/dblayer.html @@ -1,98 +1,115 @@ + + YCSB - DB Interface Layer

Yahoo! Cloud Serving Benchmark

Version 0.1.2


Home - Core workloads - Tips and FAQ

Implementing a database interface layer - overview

The database interface layer hides the details of the specific database you are benchmarking from the YCSB Client. This allows the client to generate operations like "read record" or "update record" without having to understand the specific API of your database. Thus, it is very easy to benchmark new database systems; once you have created the database interface layer, the rest of the benchmark framework runs without having to change.

The database interface layer is a simple abstract class that provides read, insert, update, delete and scan operations for your database. Implementing a database interface layer for your database means filling out the body of each of those methods. Once you have compiled your layer, you can specify the name of your implemented class on the command line (or as a property) to the YCSB Client. The YCSB Client will load your implementation dynamically when it starts. Thus, you do not need to recompile the YCSB Client itself to add or change a database interface layer.


Creating a new layer step-by-step

Step 1 - Extend com.yahoo.ycsb.DB

The base class of all database interface layer implementations is com.yahoo.ycsb.DB. This is an abstract class, so you need to create a new class which extends the DB class. Your class must have a public no-argument constructor, because the instances will be constructed inside a factory which will use the no-argument constructor.

The YCSB Client framework will create one instance of your DB class per worker thread, but there might be multiple worker threads generating the workload, so there might be multiple instances of your DB class created.

Step 2 - Implement init() if necessary

You can perform any initialization of your DB object by implementing the following method
 
 public void init() throws DBException
 
to perform any initialization actions. The init() method will be called once per DB instance; so if there are multiple threads, each DB instance will have init() called separately.

The init() method should be used to set up the connection to the database and do any other initialization. In particular, you can configure your database layer using properties passed to the YCSB Client at runtime. In fact, the YCSB Client will pass to the DB interface layer all of the properties specified in all parameter files specified when the Client starts up. Thus, you can create new properties for configuring your DB interface layer, set them in your parameter files (or on the command line), and then retrieve them inside your implementation of the DB interface layer.

These properties will be passed to the DB instance after the constructor, so it is important to retrieve them only in the init() method and not the constructor. You can get the set of properties using the

 public Properties getProperties()
 
method which is already implemented and inherited from the DB base class.

Step 3 - Implement the database query and update methods

The methods that you need to implement are:
   //Read a single record
   public int read(String table, String key, Set fields, HashMap result);
 
   //Perform a range scan
   public int scan(String table, String startkey, int recordcount, Set fields, Vector> result);
 	
   //Update a single record
   public int update(String table, String key, HashMap values);
 
   //Insert a single record
   public int insert(String table, String key, HashMap values);
 
   //Delete a single record
   public int delete(String table, String key);
 
In each case, the method takes a table name and record key. (In the case of scan, the record key is the first key in the range to scan.) For the read methods (read() and scan()) the methods additionally take a set of fields to be read, and provide a structure (HashMap or Vector of HashMaps) to store the returned data. For the write methods (insert() and update()) the methods take HashMap which maps field names to values.

The database should have the appropriate tables created before you run the benchmark. So you can assume in your implementation of the above methods that the appropriate tables already exist, and just write code to read or write from the tables named in the "table" parameter.

Step 4 - Compile your database interface layer

Your code can be compiled separately from the compilation of the YCSB Client and framework. In particular, you can make changes to your DB class and recompile without having to recompile the YCSB Client.

Step 5 - Use it with the YCSB Client

Make sure that the classes for your implementation (or a jar containing those classes) are available on your CLASSPATH, as well as any libraries/jar files used by your implementation. Now, when you run the YCSB Client, specify the "-db" argument on the command line and provide the fully qualified classname of your DB class. For example, to run workloada with your DB class:
 %  java -cp build/ycsb.jar:yourjarpath com.yahoo.ycsb.Client -t -db com.foo.YourDBClass -P workloads/workloada -P large.dat -s > transactions.dat
 
You can also specify the DB interface layer using the DB property in your parameter file:
 db=com.foo.YourDBClass
 

YCSB - Yahoo! Research - Contact cooperb@yahoo-inc.com. diff --git a/doc/index.html b/doc/index.html index 0aeb3eef..6e2a20db 100644 --- a/doc/index.html +++ b/doc/index.html @@ -1,76 +1,93 @@ + + YCSB - Yahoo! Cloud Serving Benchmark

Yahoo! Cloud Serving Benchmark

Version 0.1.2


Home - Core workloads - Tips and FAQ

Overview

There are many new serving databases available, including:
It is difficult to decide which system is right for your application, partially because the features differ between systems, and partially because there is not an easy way to compare the performance of one system versus another.

The goal of the YCSB project is to develop a framework and common set of workloads for evaluating the performance of different "key-value" and "cloud" serving stores. The project comprises two things:

  • The YCSB Client, an extensible workload generator
  • The Core workloads, a set of workload scenarios to be executed by the generator
Although the core workloads provide a well rounded picture of a system's performance, the Client is extensible so that you can define new and different workloads to examine system aspects, or application scenarios, not adequately covered by the core workload. Similarly, the Client is extensible to support benchmarking different databases. Although we include sample code for benchmarking HBase and Cassandra, it is straightforward to write a new interface layer to benchmark your favorite database.

A common use of the tool is to benchmark multiple systems and compare them. For example, you can install multiple systems on the same hardward configuration, and run the same workloads against each system. Then you can plot the performance of each system (for example, as latency versus throughput curves) to see when one system does better than another.


Download YCSB

YCSB is available at
http://wiki.github.com/brianfrankcooper/YCSB/.

Getting started

Detailed instructions for using YCSB are available on the GitHub wiki:
http://wiki.github.com/brianfrankcooper/YCSB/getting-started.

Extending YCSB

YCSB is designed to be extensible. It is easy to add a new database interface layer to support benchmarking a new database. It is also easy to define new workloads.
More details about the entire class structure of YCSB is available here:
YCSB - Yahoo! Research - Contact cooperb@yahoo-inc.com. diff --git a/doc/parallelclients.html b/doc/parallelclients.html index 4735bb5b..3de79cac 100644 --- a/doc/parallelclients.html +++ b/doc/parallelclients.html @@ -1,46 +1,63 @@ + + YCSB - Parallel clients

Yahoo! Cloud Serving Benchmark

Version 0.1.2


Home - Core workloads - Tips and FAQ

Running multiple clients in parallel

It is straightforward to run the transaction phase of the workload from multiple servers - just start up clients on different servers, each running the same workload. Each client will produce performance statistics when it is done, and you'll have to aggregate these individual files into a single set of results.

In some cases it makes sense to load the database using multiple servers. In this case, you will want to partition the records to be loaded among the clients. Normally, YCSB just loads all of the records (as defined by the recordcount property). However, if you want to partition the load you need to additionally specify two other properties for each client:

  • insertstart: The index of the record to start at.
  • insertcount: The number of records to insert.
These properties can be specified in a property file or on the command line using the -p option.

For example, imagine you want to load 100 million records (so recordcount=100000000). Imagine you want to load with four clients. For the first client:

 insertstart=0
 insertcount=25000000
 
For the second client:
 insertstart=25000000
 insertcount=25000000
 
For the third client:
 insertstart=50000000
 insertcount=25000000
 
And for the fourth client:
 insertstart=75000000
 insertcount=25000000
 

YCSB - Yahoo! Research - Contact cooperb@yahoo-inc.com. diff --git a/doc/tipsfaq.html b/doc/tipsfaq.html index 27f4a5e9..3bd5a590 100644 --- a/doc/tipsfaq.html +++ b/doc/tipsfaq.html @@ -1,31 +1,48 @@ + + YCSB - Tips and FAQ

Yahoo! Cloud Serving Benchmark

Version 0.1.2


Home - Core workloads - Tips and FAQ

Tips

Tip 1 - Carefully adjust the number of threads

The number of threads determines how much workload you can generate against the database. Imagine that you are trying to run a test with 10,000 operations per second, but you are only achieving 8,000 operations per second. Is this because the database can't keep up with the load? Not necessarily. Imagine that you are running with 100 client threads (e.g. "-threads 100") and each operation is taking 12 milliseconds on average. Each thread will only be able to generate 83 operations per second, because each thread operates sequentially. Over 100 threads, your client will only generate 8300 operations per second, even if the database can support more. Increasing the number of threads ensures there are enough parallel clients hitting the database so that the database, not the client, is the bottleneck.

To calculate the number of threads needed, you should have some idea of the expected latency. For example, at 10,000 operations per second, we might expect the database to have a latency of 10-30 milliseconds on average. So you to generate 10,000 operations per second, you will need (Ops per sec / (1000 / avg latency in ms) ), or (10000/(1000/30))=300 threads. In fact, to be conservative, you might consider having 400 threads. Although this is a lot of threads, each thread will spend most of its time waiting for the database to respond, so the context switching overhead will be low.

Experiment with increasing the number of threads, especially if you find you are not reaching your target throughput. Eventually, of course, you will saturate the database and there will be no way to increase the number of threads to get more throughput (in fact, increasing the number of client threads may make things worse) but you need to have enough threads to ensure it is the database, not the client, that is the bottleneck.


YCSB - Yahoo! Research - Contact cooperb@yahoo-inc.com. diff --git a/doc/workload.html b/doc/workload.html index 305d0204..02c41e92 100644 --- a/doc/workload.html +++ b/doc/workload.html @@ -1,129 +1,146 @@ + + YCSB - Implementing new workloads

Yahoo! Cloud Serving Benchmark

Version 0.1.2


Home - Core workloads - Tips and FAQ

Implementing new workloads - overview

A workload represents the load that a given application will put on the database system. For benchmarking purposes, we must define workloads that are relatively simple compared to real applications, so that we can better reason about the benchmarking results we get. However, a workload should be detailed enough so that once we measure the database's performance, we know what kinds of applications might experience similar performance.

In the context of YCSB, a workload defines both a data set, which is a set of records to be loaded into the database, and a transaction set, which are the set of read and write operations against the database. Creating the transactions requires understanding the structure of the records, which is why both the data and the transactions must be defined in the workload.

For a complete benchmark, multiple important (but distinct) workloads might be grouped together into a workload package. The CoreWorkload package included with the YCSB client is an example of such a collection of workloads.

Typically a workload consists of two files:

  • A java class which contains the code to create data records and generate transactions against them
  • A parameter file which tunes the specifics of the workload
For example, a workload class file might generate some combination of read and update operations against the database. The parameter file might specify whether the mix of reads and updates is 50/50, 80/20, etc.

There are two ways to create a new workload or package of workloads.

Option 1: new parameter files

The core workloads included with YCSB are defined by a set of parameter files (workloada, workloadb, etc.) You can create your own parameter file with new values for the read/write mix, request distribution, etc. For example, the workloada file has the following contents:

 workload=com.yahoo.ycsb.workloads.CoreWorkload
 
 readallfields=false
 
 readproportion=0.5
 updateproportion=0.5
 scanproportion=0
 insertproportion=0
 
 requestdistribution=zipfian
 
Creating a new file that changes any of these values will produce a new workload with different characteristics. The set of properties that can be specified is here.

Option 2: new java class

The workload java class will be created by the YCSB Client at runtime, and will use an instance of the DB interface layer to generate the actual operations against the database. Thus, the java class only needs to decide (based on settings in the parameter file) what records to create for the data set, and what reads, updates etc. to generate for the transaction phase. The YCSB Client will take care of creating the workload java class, passing it to a worker thread for executing, deciding how many records to create or how many operations to execute, and measuring the resulting performance.

If the CoreWorkload (or some other existing package) does not have the ability to generate the workload you desire, you can create a new workload java class. This is done using the following steps:

Step 1. Extend com.yahoo.ycsb.Workload

The base class of all workload classes is com.yahoo.ycsb.Workload. This is an abstract class, so you create a new workload that extends this base class. Your class must have a public no-argument constructor, because the workload will be created in a factory using the no-argument constructor. The YCSB Client will create one Workload object for each worker thread, so if you run the Client with multiple threads, multiple workload objects will be created.

Step 2. Write code to initialize your workload class

The parameter fill will be passed to the workload object after the constructor has been called, so if you are using any parameter properties, you must use them to initialize your workload using either the init() or initThread() methods.
  • init() - called once for all workload instances. Used to initialize any objects shared by all threads.
  • initThread() - called once per workload instance in the context of the worker thread. Used to initialize any objects specific to a single Workload instance and single worker thread.
In either case, you can access the parameter properties using the Properties object passed in to both methods. These properties will include all properties defined in any property file passed to the YCSB Client or defined on the client command line.

Step 3. Write any cleanup code

The cleanup() method is called once for all workload instances, after the workload has completed.

Step 4. Define the records to be inserted

The YCSB Client will call the doInsert() method once for each record to be inserted into the database. So you should implement this method to create and insert a single record. The DB object you can use to perform the insert will be passed to the doInsert() method.

Step 5. Define the transactions

The YCSB Client will call the doTransaction() method once for every transaction that is to be executed. So you should implement this method to execute a single transaction, using the DB object passed in to access the database. Your implementation of this method can choose between different types of transactions, and can make multiple calls to the DB interface layer. However, each invocation of the method should be a logical transaction. In particular, when you run the client, you'll specify the number of operations to execute; if you request 1000 operations then doTransaction() will be executed 1000 times.

Note that you do not have to do any throttling of your transactions (or record insertions) to achieve the target throughput. The YCSB Client will do the throttling for you.

Note also that it is allowable to insert records inside the doTransaction() method. You might do this if you wish the database to grow during the workload. In this case, the initial dataset will be constructed using calls to the doInsert() method, while additional records would be inserted using calls to the doTransaction() method.

Step 6 - Measure latency, if necessary

The YCSB client will automatically measure the latency and throughput of database operations, even for workloads that you define. However, the client will only measure the latency of individual calls to the database, not of more complex transactions. Consider for example a workload that reads a record, modifies it, and writes the changes back to the database. The YCSB client will automatically measure the latency of the read operation to the database; and separately will automatically measure the latency of the update operation. However, if you would like to measure the latency of the entire read-modify-write transaction, you will need to add an additional timing step to your code.

Measurements are gathered using the Measurements.measure() call. There is a singleton instance of Measurements, which can be obtained using the Measurements.getMeasurements() static method. For each metric you are measuring, you need to assign a string tag; this tag will label the resulting average, min, max, histogram etc. measurements output by the tool at the end of the workload. For example, consider the following code:

 long st=System.currentTimeMillis();
 db.read(TABLENAME,keyname,fields,new HashMap());
 db.update(TABLENAME,keyname,values);
 long en=System.currentTimeMillis();
 Measurements.getMeasurements().measure("READ-MODIFY-WRITE", (int)(en-st));
 
In this code, the calls to System.currentTimeMillis() are used to time the read and write transaction. Then, the call to measure() reports the latency to the measurement component.

Using this pattern, your custom measurements will be gathered and aggregated using the same mechanism that is used to gather measurements for individual READ, UPDATE etc. operations.

Step 7 - Use it with the YCSB Client

Make sure that the classes for your implementation (or a jar containing those classes) are available on your CLASSPATH, as well as any libraries/jar files used by your implementation. Now, when you run the YCSB Client, specify the "workload" property to provide the fully qualified classname of your DB class. For example:
 workload=com.foo.YourWorkloadClass
 

YCSB - Yahoo! Research - Contact cooperb@yahoo-inc.com. diff --git a/dynamodb/conf/AWSCredentials.properties b/dynamodb/conf/AWSCredentials.properties index dab032a1..e337b391 100644 --- a/dynamodb/conf/AWSCredentials.properties +++ b/dynamodb/conf/AWSCredentials.properties @@ -1,4 +1,19 @@ +# Copyright (c) 2012 YCSB contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. See accompanying +# LICENSE file. + # Fill in your AWS Access Key ID and Secret Access Key # http://aws.amazon.com/security-credentials #accessKey = #secretKey = diff --git a/dynamodb/conf/dynamodb.properties b/dynamodb/conf/dynamodb.properties index a510e4c9..870d3dc5 100644 --- a/dynamodb/conf/dynamodb.properties +++ b/dynamodb/conf/dynamodb.properties @@ -1,37 +1,52 @@ +# Copyright (c) 2012 YCSB contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. See accompanying +# LICENSE file. + # # Sample property file for Amazon DynamoDB database client ## Mandatory parameters # AWS credentials associated with your aws account. #dynamodb.awsCredentialsFile = # Primarykey of table 'usertable' #dynamodb.primaryKey = ## Optional parameters # Endpoint to connect to.For best latency, it is recommended # to choose the endpoint which is closer to the client. # Default is us-east-1 #dynamodb.endpoint = http://dynamodb.us-east-1.amazonaws.com # Strongly recommended to set to uniform.Refer FAQs in README #requestdistribution = uniform # Enable/disable debug messages.Defaults to false # "true" or "false" #dynamodb.debug = false # Maximum number of concurrent connections #dynamodb.connectMax = 50 # Read consistency.Consistent reads are expensive and consume twice # as many resources as eventually consistent reads. Defaults to false. # "true" or "false" #dynamodb.consistentReads = false # Workload size has implications on provisioned read and write # capacity units.Refer FAQs in README #fieldcount = 10 #fieldlength = 90 diff --git a/dynamodb/pom.xml b/dynamodb/pom.xml index fbe04273..d60ddf85 100644 --- a/dynamodb/pom.xml +++ b/dynamodb/pom.xml @@ -1,32 +1,49 @@ + + 4.0.0 com.yahoo.ycsb binding-parent 0.3.0-RC4-SNAPSHOT ../binding-parent dynamodb-binding DynamoDB DB Binding com.amazonaws aws-java-sdk 1.3.14 log4j log4j 1.2.17 com.yahoo.ycsb core ${project.version} provided diff --git a/dynamodb/src/main/resources/log4j.properties b/dynamodb/src/main/resources/log4j.properties index 8b4120f4..a9f3d668 100644 --- a/dynamodb/src/main/resources/log4j.properties +++ b/dynamodb/src/main/resources/log4j.properties @@ -1,10 +1,25 @@ +# Copyright (c) 2012 YCSB contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. See accompanying +# LICENSE file. + #define the console appender log4j.appender.consoleAppender = org.apache.log4j.ConsoleAppender # now define the layout for the appender log4j.appender.consoleAppender.layout = org.apache.log4j.PatternLayout log4j.appender.consoleAppender.layout.ConversionPattern=%-4r [%t] %-5p %c %x -%m%n # now map our console appender as a root logger, means all log messages will go # to this appender log4j.rootLogger = INFO, consoleAppender diff --git a/elasticsearch/README.md b/elasticsearch/README.md index 047a9ea3..32c6d2b2 100644 --- a/elasticsearch/README.md +++ b/elasticsearch/README.md @@ -1,65 +1,82 @@ + + ## Quick Start This section describes how to run YCSB on ElasticSearch running locally. ### 1. Set Up YCSB Clone the YCSB git repository and compile: git clone git://github.com/brianfrankcooper/YCSB.git cd YCSB mvn clean package ### 2. Run YCSB Now you are ready to run! First, load the data: ./bin/ycsb load elasticsearch -s -P workloads/workloada Then, run the workload: ./bin/ycsb run elasticsearch -s -P workloads/workloada For further configuration see below: ### Defaults Configuration The default setting for the ElasticSearch node that is created is as follows: cluster.name=es.ycsb.cluster node.local=true path.data=$TEMP_DIR/esdata discovery.zen.ping.multicast.enabled=false index.mapping._id.indexed=true index.gateway.type=none gateway.type=none index.number_of_shards=1 index.number_of_replicas=0 es.index.key=es.ycsb ### Custom Configuration If you wish to customize the settings used to create the ElasticSerach node you can created a new property file that contains your desired ElasticSearch node settings and pass it in via the parameter to 'bin/ycsb' script. Note that the default properties will be kept if you don't explicitly overwrite them. Assuming that we have a properties file named "myproperties.data" that contains custom ElasticSearch node configuration you can execute the following to pass it into the ElasticSearch client: ./bin/ycsb run elasticsearch -P workloads/workloada -P myproperties.data -s If you wish to use a in-memory store type rather than the default disk store add the following properties to your custom properties file. For a large number of insert operations insure that you have sufficient memory on your test system otherwise you will run out of memory. index.store.type=memory index.store.fs.memory.enabled=true cache.memory.small_buffer_size=4mb cache.memory.large_cache_size=1024mb If you wish to change the default index name you can set the following property: es.index.key=my_index_key diff --git a/elasticsearch/pom.xml b/elasticsearch/pom.xml index ba687b72..e46571da 100644 --- a/elasticsearch/pom.xml +++ b/elasticsearch/pom.xml @@ -1,43 +1,60 @@ + + 4.0.0 com.yahoo.ycsb binding-parent 0.3.0-RC4-SNAPSHOT ../binding-parent elasticsearch-binding ElasticSearch Binding jar 0.19.8 sonatype-nexus-snapshots Sonatype releases https://oss.sonatype.org/content/repositories/releases com.yahoo.ycsb core ${project.version} provided org.elasticsearch elasticsearch ${elasticsearch-version} org.testng testng 6.1.1 test diff --git a/elasticsearch/src/main/java/com/yahoo/ycsb/db/ElasticSearchClient.java b/elasticsearch/src/main/java/com/yahoo/ycsb/db/ElasticSearchClient.java index 30349bb3..c86baf7f 100644 --- a/elasticsearch/src/main/java/com/yahoo/ycsb/db/ElasticSearchClient.java +++ b/elasticsearch/src/main/java/com/yahoo/ycsb/db/ElasticSearchClient.java @@ -1,296 +1,313 @@ +/** + * Copyright (c) 2012 YCSB contributors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + package com.yahoo.ycsb.db; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.StringByteIterator; import java.util.HashMap; import java.util.Map.Entry; import java.util.Properties; import java.util.Set; import java.util.Vector; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; import org.elasticsearch.client.Requests; import static org.elasticsearch.common.settings.ImmutableSettings.*; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.settings.ImmutableSettings.Builder; import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.common.xcontent.XContentBuilder; import static org.elasticsearch.common.xcontent.XContentFactory.*; import static org.elasticsearch.index.query.FilterBuilders.*; import static org.elasticsearch.index.query.QueryBuilders.*; import org.elasticsearch.index.query.RangeFilterBuilder; import org.elasticsearch.node.Node; import static org.elasticsearch.node.NodeBuilder.*; import org.elasticsearch.search.SearchHit; /** * ElasticSearch client for YCSB framework. * *

Default properties to set:

  • es.cluster.name = es.ycsb.cluster *
  • es.client = true
  • es.index.key = es.ycsb
* * @author Sharmarke Aden * */ public class ElasticSearchClient extends DB { public static final String DEFAULT_CLUSTER_NAME = "es.ycsb.cluster"; public static final String DEFAULT_INDEX_KEY = "es.ycsb"; public static final String DEFAULT_REMOTE_HOST = "localhost:9300"; private Node node; private Client client; private String indexKey; private Boolean remoteMode; /** * Initialize any state for this DB. Called once per DB instance; there is * one DB instance per client thread. */ @Override public void init() throws DBException { // initialize OrientDB driver Properties props = getProperties(); this.indexKey = props.getProperty("es.index.key", DEFAULT_INDEX_KEY); String clusterName = props.getProperty("cluster.name", DEFAULT_CLUSTER_NAME); //Check if transport client needs to be used (To connect to multiple elasticsearch nodes) remoteMode = Boolean.parseBoolean(props.getProperty("elasticsearch.remote", "false")); Boolean newdb = Boolean.parseBoolean(props.getProperty("elasticsearch.newdb", "false")); Builder settings = settingsBuilder() .put("node.local", "true") .put("path.data", System.getProperty("java.io.tmpdir") + "/esdata") .put("discovery.zen.ping.multicast.enabled", "false") .put("index.mapping._id.indexed", "true") .put("index.gateway.type", "none") .put("gateway.type", "none") .put("index.number_of_shards", "1") .put("index.number_of_replicas", "0"); //if properties file contains elasticsearch user defined properties //add it to the settings file (will overwrite the defaults). settings.put(props); System.out.println("ElasticSearch starting node = " + settings.get("cluster.name")); System.out.println("ElasticSearch node data path = " + settings.get("path.data")); System.out.println("ElasticSearch Remote Mode = " +remoteMode); //Remote mode support for connecting to remote elasticsearch cluster if(remoteMode) { settings.put("client.transport.sniff", true) .put("client.transport.ignore_cluster_name", false) .put("client.transport.ping_timeout", "30s") .put("client.transport.nodes_sampler_interval", "30s"); //Default it to localhost:9300 String nodeList[] = props.getProperty("elasticsearch.hosts.list", DEFAULT_REMOTE_HOST).split(","); System.out.println("ElasticSearch Remote Hosts = " +props.getProperty("elasticsearch.hosts.list", DEFAULT_REMOTE_HOST)); TransportClient tClient = new TransportClient(settings); for(String h : nodeList) { String node[] = h.split(":"); tClient.addTransportAddress(new InetSocketTransportAddress(node[0], Integer.parseInt(node[1]))); } client = tClient; } else { //Start node only if transport client mode is disabled node = nodeBuilder().clusterName(clusterName).settings(settings).node(); node.start(); client = node.client(); } if (newdb) { client.admin().indices().prepareDelete(indexKey).execute().actionGet(); client.admin().indices().prepareCreate(indexKey).execute().actionGet(); } else { boolean exists = client.admin().indices().exists(Requests.indicesExistsRequest(indexKey)).actionGet().isExists(); if (!exists) { client.admin().indices().prepareCreate(indexKey).execute().actionGet(); } } } @Override public void cleanup() throws DBException { if(!remoteMode) { if (!node.isClosed()) { client.close(); node.stop(); node.close(); } } else { client.close(); } } /** * Insert a record in the database. Any field/value pairs in the specified * values HashMap will be written into the record with the specified record * key. * * @param table The name of the table * @param key The record key of the record to insert. * @param values A HashMap of field/value pairs to insert in the record * @return Zero on success, a non-zero error code on error. See this class's * description for a discussion of error codes. */ @Override public int insert(String table, String key, HashMap values) { try { final XContentBuilder doc = jsonBuilder().startObject(); for (Entry entry : StringByteIterator.getStringMap(values).entrySet()) { doc.field(entry.getKey(), entry.getValue()); } doc.endObject(); client.prepareIndex(indexKey, table, key) .setSource(doc) .execute() .actionGet(); return 0; } catch (Exception e) { e.printStackTrace(); } return 1; } /** * Delete a record from the database. * * @param table The name of the table * @param key The record key of the record to delete. * @return Zero on success, a non-zero error code on error. See this class's * description for a discussion of error codes. */ @Override public int delete(String table, String key) { try { client.prepareDelete(indexKey, table, key) .execute() .actionGet(); return 0; } catch (Exception e) { e.printStackTrace(); } return 1; } /** * Read a record from the database. Each field/value pair from the result * will be stored in a HashMap. * * @param table The name of the table * @param key The record key of the record to read. * @param fields The list of fields to read, or null for all of them * @param result A HashMap of field/value pairs for the result * @return Zero on success, a non-zero error code on error or "not found". */ @Override public int read(String table, String key, Set fields, HashMap result) { try { final GetResponse response = client.prepareGet(indexKey, table, key) .execute() .actionGet(); if (response.isExists()) { if (fields != null) { for (String field : fields) { result.put(field, new StringByteIterator((String) response.getSource().get(field))); } } else { for (String field : response.getSource().keySet()) { result.put(field, new StringByteIterator((String) response.getSource().get(field))); } } return 0; } } catch (Exception e) { e.printStackTrace(); } return 1; } /** * Update a record in the database. Any field/value pairs in the specified * values HashMap will be written into the record with the specified record * key, overwriting any existing values with the same field name. * * @param table The name of the table * @param key The record key of the record to write. * @param values A HashMap of field/value pairs to update in the record * @return Zero on success, a non-zero error code on error. See this class's * description for a discussion of error codes. */ @Override public int update(String table, String key, HashMap values) { try { final GetResponse response = client.prepareGet(indexKey, table, key) .execute() .actionGet(); if (response.isExists()) { for (Entry entry : StringByteIterator.getStringMap(values).entrySet()) { response.getSource().put(entry.getKey(), entry.getValue()); } client.prepareIndex(indexKey, table, key) .setSource(response.getSource()) .execute() .actionGet(); return 0; } } catch (Exception e) { e.printStackTrace(); } return 1; } /** * Perform a range scan for a set of records in the database. Each * field/value pair from the result will be stored in a HashMap. * * @param table The name of the table * @param startkey The record key of the first record to read. * @param recordcount The number of records to read * @param fields The list of fields to read, or null for all of them * @param result A Vector of HashMaps, where each HashMap is a set * field/value pairs for one record * @return Zero on success, a non-zero error code on error. See this class's * description for a discussion of error codes. */ @Override public int scan(String table, String startkey, int recordcount, Set fields, Vector> result) { try { final RangeFilterBuilder filter = rangeFilter("_id").gte(startkey); final SearchResponse response = client.prepareSearch(indexKey) .setTypes(table) .setQuery(matchAllQuery()) .setFilter(filter) .setSize(recordcount) .execute() .actionGet(); HashMap entry; for (SearchHit hit : response.getHits()) { entry = new HashMap(fields.size()); for (String field : fields) { entry.put(field, new StringByteIterator((String) hit.getSource().get(field))); } result.add(entry); } return 0; } catch (Exception e) { e.printStackTrace(); } return 1; } } diff --git a/elasticsearch/src/test/java/com/yahoo/ycsb/db/ElasticSearchClientTest.java b/elasticsearch/src/test/java/com/yahoo/ycsb/db/ElasticSearchClientTest.java index 3ef8ed9c..72177f49 100644 --- a/elasticsearch/src/test/java/com/yahoo/ycsb/db/ElasticSearchClientTest.java +++ b/elasticsearch/src/test/java/com/yahoo/ycsb/db/ElasticSearchClientTest.java @@ -1,137 +1,154 @@ +/** + * Copyright (c) 2012 YCSB contributors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + /* * To change this template, choose Tools | Templates * and open the template in the editor. */ package com.yahoo.ycsb.db; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.StringByteIterator; import java.util.HashMap; import java.util.Set; import java.util.Vector; import static org.testng.AssertJUnit.*; import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; /** * * @author saden */ public class ElasticSearchClientTest { protected final static ElasticSearchClient instance = new ElasticSearchClient(); protected final static HashMap MOCK_DATA; protected final static String MOCK_TABLE = "MOCK_TABLE"; protected final static String MOCK_KEY0 = "0"; protected final static String MOCK_KEY1 = "1"; protected final static String MOCK_KEY2 = "2"; static { MOCK_DATA = new HashMap(10); for (int i = 1; i <= 10; i++) { MOCK_DATA.put("field" + i, new StringByteIterator("value" + i)); } } @BeforeClass public static void setUpClass() throws DBException { instance.init(); } @AfterClass public static void tearDownClass() throws DBException { instance.cleanup(); } @BeforeMethod public void setUp() { instance.insert(MOCK_TABLE, MOCK_KEY1, MOCK_DATA); instance.insert(MOCK_TABLE, MOCK_KEY2, MOCK_DATA); } @AfterMethod public void tearDown() { instance.delete(MOCK_TABLE, MOCK_KEY1); instance.delete(MOCK_TABLE, MOCK_KEY2); } /** * Test of insert method, of class ElasticSearchClient. */ @Test public void testInsert() { System.out.println("insert"); int expResult = 0; int result = instance.insert(MOCK_TABLE, MOCK_KEY0, MOCK_DATA); assertEquals(expResult, result); } /** * Test of delete method, of class ElasticSearchClient. */ @Test public void testDelete() { System.out.println("delete"); int expResult = 0; int result = instance.delete(MOCK_TABLE, MOCK_KEY1); assertEquals(expResult, result); } /** * Test of read method, of class ElasticSearchClient. */ @Test public void testRead() { System.out.println("read"); Set fields = MOCK_DATA.keySet(); HashMap resultParam = new HashMap(10); int expResult = 0; int result = instance.read(MOCK_TABLE, MOCK_KEY1, fields, resultParam); assertEquals(expResult, result); } /** * Test of update method, of class ElasticSearchClient. */ @Test public void testUpdate() { System.out.println("update"); int i; HashMap newValues = new HashMap(10); for (i = 1; i <= 10; i++) { newValues.put("field" + i, new StringByteIterator("newvalue" + i)); } int expResult = 0; int result = instance.update(MOCK_TABLE, MOCK_KEY1, newValues); assertEquals(expResult, result); //validate that the values changed HashMap resultParam = new HashMap(10); instance.read(MOCK_TABLE, MOCK_KEY1, MOCK_DATA.keySet(), resultParam); for (i = 1; i <= 10; i++) { assertEquals("newvalue" + i, resultParam.get("field" + i).toString()); } } /** * Test of scan method, of class ElasticSearchClient. */ @Test public void testScan() { System.out.println("scan"); int recordcount = 10; Set fields = MOCK_DATA.keySet(); Vector> resultParam = new Vector>(10); int expResult = 0; int result = instance.scan(MOCK_TABLE, MOCK_KEY1, recordcount, fields, resultParam); assertEquals(expResult, result); } } diff --git a/gemfire/pom.xml b/gemfire/pom.xml index 82036a44..857b792e 100644 --- a/gemfire/pom.xml +++ b/gemfire/pom.xml @@ -1,35 +1,52 @@ + + 4.0.0 com.yahoo.ycsb binding-parent 0.3.0-RC4-SNAPSHOT ../binding-parent gemfire-binding Gemfire DB Binding jar com.gemstone.gemfire gemfire ${gemfire.version} com.yahoo.ycsb core ${project.version} provided gemstone http://dist.gemstone.com.s3.amazonaws.com/maven/release diff --git a/gemfire/src/main/conf/cache.xml b/gemfire/src/main/conf/cache.xml index 90254df0..372e1c45 100644 --- a/gemfire/src/main/conf/cache.xml +++ b/gemfire/src/main/conf/cache.xml @@ -1,9 +1,25 @@ + diff --git a/gemfire/src/main/java/com/yahoo/ycsb/db/GemFireClient.java b/gemfire/src/main/java/com/yahoo/ycsb/db/GemFireClient.java index 6815c44e..7e61bbeb 100644 --- a/gemfire/src/main/java/com/yahoo/ycsb/db/GemFireClient.java +++ b/gemfire/src/main/java/com/yahoo/ycsb/db/GemFireClient.java @@ -1,200 +1,217 @@ +/** + * Copyright (c) 2013 - 2014 YCSB Contributors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + package com.yahoo.ycsb.db; import java.util.HashMap; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.Vector; import com.gemstone.gemfire.cache.Cache; import com.gemstone.gemfire.cache.CacheFactory; import com.gemstone.gemfire.cache.GemFireCache; import com.gemstone.gemfire.cache.Region; import com.gemstone.gemfire.cache.RegionExistsException; import com.gemstone.gemfire.cache.RegionFactory; import com.gemstone.gemfire.cache.RegionShortcut; import com.gemstone.gemfire.cache.client.ClientCache; import com.gemstone.gemfire.cache.client.ClientCacheFactory; import com.gemstone.gemfire.cache.client.ClientRegionFactory; import com.gemstone.gemfire.cache.client.ClientRegionShortcut; import com.gemstone.gemfire.internal.admin.remote.DistributionLocatorId; import com.yahoo.ycsb.ByteArrayByteIterator; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.StringByteIterator; /** * VMware vFabric GemFire client for the YCSB benchmark.
*

By default acts as a GemFire client and tries to connect * to GemFire cache server running on localhost with default * cache server port. Hostname and port of a GemFire cacheServer * can be provided using gemfire.serverport=port and * gemfire.serverhost=host properties on YCSB command line. * A locator may also be used for discovering a cacheServer * by using the property gemfire.locator=host[port]

* *

To run this client in a peer-to-peer topology with other GemFire * nodes, use the property gemfire.topology=p2p. Running * in p2p mode will enable embedded caching in this client.

* *

YCSB by default does its operations against "usertable". When running * as a client this is a ClientRegionShortcut.PROXY region, * when running in p2p mode it is a RegionShortcut.PARTITION * region. A cache.xml defining "usertable" region can be placed in the * working directory to override these region definitions.

* * @author Swapnil Bawaskar (sbawaska at vmware) * */ public class GemFireClient extends DB { /** Return code when operation succeeded */ private static final int SUCCESS = 0; /** Return code when operation did not succeed */ private static final int ERROR = -1; /** property name of the port where GemFire server is listening for connections */ private static final String SERVERPORT_PROPERTY_NAME = "gemfire.serverport"; /** property name of the host where GemFire server is running */ private static final String SERVERHOST_PROPERTY_NAME = "gemfire.serverhost"; /** default value of {@link #SERVERHOST_PROPERTY_NAME} */ private static final String SERVERHOST_PROPERTY_DEFAULT = "localhost"; /** property name to specify a GemFire locator. This property can be used in both * client server and p2p topology */ private static final String LOCATOR_PROPERTY_NAME = "gemfire.locator"; /** property name to specify GemFire topology */ private static final String TOPOLOGY_PROPERTY_NAME = "gemfire.topology"; /** value of {@value #TOPOLOGY_PROPERTY_NAME} when peer to peer topology should be used. * (client-server topology is default) */ private static final String TOPOLOGY_P2P_VALUE = "p2p"; private GemFireCache cache; /** * true if ycsb client runs as a client to a * GemFire cache server */ private boolean isClient; @Override public void init() throws DBException { Properties props = getProperties(); // hostName where GemFire cacheServer is running String serverHost = null; // port of GemFire cacheServer int serverPort = 0; String locatorStr = null; if (props != null && !props.isEmpty()) { String serverPortStr = props.getProperty(SERVERPORT_PROPERTY_NAME); if (serverPortStr != null) { serverPort = Integer.parseInt(serverPortStr); } serverHost = props.getProperty(SERVERHOST_PROPERTY_NAME, SERVERHOST_PROPERTY_DEFAULT); locatorStr = props.getProperty(LOCATOR_PROPERTY_NAME); String topology = props.getProperty(TOPOLOGY_PROPERTY_NAME); if (topology != null && topology.equals(TOPOLOGY_P2P_VALUE)) { CacheFactory cf = new CacheFactory(); if (locatorStr != null) { cf.set("locators", locatorStr); } cache = cf.create(); isClient = false; return; } } isClient = true; DistributionLocatorId locator = null; if (locatorStr != null) { locator = new DistributionLocatorId(locatorStr); } ClientCacheFactory ccf = new ClientCacheFactory(); if (serverPort != 0) { ccf.addPoolServer(serverHost, serverPort); } else if (locator != null) { ccf.addPoolLocator(locator.getHost().getCanonicalHostName(), locator.getPort()); } cache = ccf.create(); } @Override public int read(String table, String key, Set fields, HashMap result) { Region> r = getRegion(table); Map val = r.get(key); if (val != null) { if (fields == null) { for (String k : val.keySet()) { result.put(k, new ByteArrayByteIterator(val.get(k))); } } else { for (String field : fields) { result.put(field, new ByteArrayByteIterator(val.get(field))); } } return SUCCESS; } return ERROR; } @Override public int scan(String table, String startkey, int recordcount, Set fields, Vector> result) { // GemFire does not support scan return ERROR; } @Override public int update(String table, String key, HashMap values) { getRegion(table).put(key, convertToBytearrayMap(values)); return 0; } @Override public int insert(String table, String key, HashMap values) { getRegion(table).put(key, convertToBytearrayMap(values)); return 0; } @Override public int delete(String table, String key) { getRegion(table).destroy(key); return 0; } private Map convertToBytearrayMap(Map values) { Map retVal = new HashMap(); for (String key : values.keySet()) { retVal.put(key, values.get(key).toArray()); } return retVal; } private Region> getRegion(String table) { Region> r = cache.getRegion(table); if (r == null) { try { if (isClient) { ClientRegionFactory> crf = ((ClientCache) cache).createClientRegionFactory(ClientRegionShortcut.PROXY); r = crf.create(table); } else { RegionFactory> rf = ((Cache)cache).createRegionFactory(RegionShortcut.PARTITION); r = rf.create(table); } } catch (RegionExistsException e) { // another thread created the region r = cache.getRegion(table); } } return r; } } diff --git a/hbase/README.md b/hbase/README.md index 096d144b..7316c620 100644 --- a/hbase/README.md +++ b/hbase/README.md @@ -1,56 +1,73 @@ + + # HBase Driver for YCSB This driver is a binding for the YCSB facilities to operate against a HBase Server cluster. ## Quickstart ### 1. Start a HBase Server You need to start a single node or a cluster to point the client at. Please see [Apache HBase Reference Guide](http://hbase.apache.org/book.html) for more details and instructions. ### 2. Set up YCSB You need to clone the repository and compile everything. ``` git clone git://github.com/brianfrankcooper/YCSB.git cd YCSB mvn clean package ``` ### 3. Create a HBase table for testing For best results, use the pre-splitting strategy recommended in [HBASE-4163](https://issues.apache.org/jira/browse/HBASE-4163): ``` hbase(main):001:0> n_splits = 200 # HBase recommends (10 * number of regionservers) hbase(main):002:0> create 'usertable', 'family', {SPLITS => (1..n_splits).map {|i| "user#{1000+i*(9999-1000)/n_splits}"}} ``` *Failing to do so will cause all writes to initially target a single region server*. ### 4. Run the Workload Before you can actually run the workload, you need to "load" the data first. You should specify a HBase config directory(or any other directory containing your hbase-site.xml) and a table name and a column family(-cp is used to set java classpath and -p is used to set various properties). ``` bin/ycsb load hbase -P workloads/workloada -cp /HBASE-HOME-DIR/conf -p table=usertable -p columnfamily=family ``` Then, you can run the workload: ``` bin/ycsb run hbase -P workloads/workloada -cp /HBASE-HOME-DIR/conf -p table=usertable -p columnfamily=family ``` Please see the general instructions in the `doc` folder if you are not sure how it all works. You can apply additional properties (as seen in the next section) like this: ``` bin/ycsb run hbase -P workloads/workloada -cp /HBASE-HOME-DIR/conf -p table=usertable -p columnfamily=family -p clientbuffering=true ``` ## Configuration Options Following options can be configurable using `-p`. * `columnfamily`: The HBase column family to target. * `clientbuffering` : If true, buffer mutations on the client. The default is false. * `writebuffersize` : Buffer size to be used when `clientbuffering` is activated. The default is 12MB. * `debug` : If true, debugging logs are activated. The default is false. diff --git a/hbase/pom.xml b/hbase/pom.xml index 76d567c2..c263082c 100644 --- a/hbase/pom.xml +++ b/hbase/pom.xml @@ -1,27 +1,44 @@ + + 4.0.0 com.yahoo.ycsb binding-parent 0.3.0-RC4-SNAPSHOT ../binding-parent/ hbase-binding HBase DB Binding org.apache.hbase hbase-client ${hbase.version} com.yahoo.ycsb core ${project.version} provided diff --git a/hypertable/pom.xml b/hypertable/pom.xml index 41fa86a9..f7abe322 100644 --- a/hypertable/pom.xml +++ b/hypertable/pom.xml @@ -1,40 +1,57 @@ + + 4.0.0 com.yahoo.ycsb binding-parent 0.3.0-RC4-SNAPSHOT ../binding-parent hypertable-binding Hypertable DB Binding jar com.yahoo.ycsb core ${project.version} provided org.apache.thrift libthrift ${thrift.version} org.hypertable hypertable ${hypertable.version} clojars.org http://clojars.org/repo diff --git a/infinispan/README.md b/infinispan/README.md index d1303ceb..c4673aae 100644 --- a/infinispan/README.md +++ b/infinispan/README.md @@ -1,41 +1,58 @@ + + ## Quick Start This section describes how to run YCSB on infinispan. ### 1. Install Java and Maven ### 2. Set Up YCSB 1. Git clone YCSB and compile: ``` git clone http://github.com/brianfrankcooper/YCSB.git cd YCSB mvn clean package ``` 2. Copy and untar YCSB distribution in distribution/target/ycsb-x.x.x.tar.gz to target machine ### 4. Load data and run tests ####4.1 embedded mode with cluster or not Load the data: ``` ./bin/ycsb load infinispan -P workloads/workloada -p infinispan.clustered= ``` Run the workload test: ``` ./bin/ycsb run infinispan -s -P workloads/workloada -p infinispan.clustered= ``` ####4.2 client-server mode 1. start infinispan server 2. read [RemoteCacheManager](http://docs.jboss.org/infinispan/7.2/apidocs/org/infinispan/client/hotrod/RemoteCacheManager.html) doc and customize hotrod client properties in infinispan-binding/conf/remote-cache.properties 3. Load the data with specified cache: ``` ./bin/ycsb load infinispan-cs -s -P workloads/workloada -P infinispan-binding/conf/remote-cache.properties -p cache= ``` 4. Run the workload test with specified cache: ``` ./bin/ycsb run infinispan-cs -s -P workloads/workloada -P infinispan-binding/conf/remote-cache.properties -p cache= ``` \ No newline at end of file diff --git a/infinispan/pom.xml b/infinispan/pom.xml index 87538dc1..c64652c8 100644 --- a/infinispan/pom.xml +++ b/infinispan/pom.xml @@ -1,33 +1,50 @@ + + 4.0.0 com.yahoo.ycsb binding-parent 0.3.0-RC4-SNAPSHOT ../binding-parent infinispan-binding Infinispan DB Binding jar org.infinispan infinispan-client-hotrod ${infinispan.version} org.infinispan infinispan-core ${infinispan.version} com.yahoo.ycsb core ${project.version} provided diff --git a/infinispan/src/main/conf/infinispan-config.xml b/infinispan/src/main/conf/infinispan-config.xml index 15b9978c..fedf2816 100644 --- a/infinispan/src/main/conf/infinispan-config.xml +++ b/infinispan/src/main/conf/infinispan-config.xml @@ -1,17 +1,33 @@ + \ No newline at end of file diff --git a/infinispan/src/main/conf/remote-cache.properties b/infinispan/src/main/conf/remote-cache.properties index 04e50054..8e96ff5e 100644 --- a/infinispan/src/main/conf/remote-cache.properties +++ b/infinispan/src/main/conf/remote-cache.properties @@ -1,9 +1,24 @@ +# Copyright (c) 2015 YCSB contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. See accompanying +# LICENSE file. + infinispan.client.hotrod.server_list=192.168.101.17:11222 infinispan.client.hotrod.force_return_values=false maxActive=-1 maxIdle=-1 minIdle=1 maxTotal=-1 whenExhaustedAction=1 diff --git a/infinispan/src/main/java/com/yahoo/ycsb/db/InfinispanClient.java b/infinispan/src/main/java/com/yahoo/ycsb/db/InfinispanClient.java index 528ded5c..efc0deb0 100644 --- a/infinispan/src/main/java/com/yahoo/ycsb/db/InfinispanClient.java +++ b/infinispan/src/main/java/com/yahoo/ycsb/db/InfinispanClient.java @@ -1,136 +1,153 @@ +/** + * Copyright (c) 2012 YCSB contributors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + package com.yahoo.ycsb.db; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.StringByteIterator; import org.infinispan.Cache; import org.infinispan.atomic.AtomicMap; import org.infinispan.atomic.AtomicMapLookup; import org.infinispan.manager.DefaultCacheManager; import org.infinispan.manager.EmbeddedCacheManager; import org.infinispan.util.logging.Log; import org.infinispan.util.logging.LogFactory; import java.io.IOException; import java.util.HashMap; import java.util.Map; import java.util.Set; import java.util.Vector; /** * This is a client implementation for Infinispan 5.x. * * Some settings: * * @author Manik Surtani (manik AT jboss DOT org) */ public class InfinispanClient extends DB { private static final int OK = 0; private static final int ERROR = -1; private static final int NOT_FOUND = -2; // An optimisation for clustered mode private final boolean clustered; private EmbeddedCacheManager infinispanManager; private static final Log logger = LogFactory.getLog(InfinispanClient.class); public InfinispanClient() { clustered = Boolean.getBoolean("infinispan.clustered"); } public void init() throws DBException { try { infinispanManager = new DefaultCacheManager("infinispan-config.xml"); } catch (IOException e) { throw new DBException(e); } } public void cleanup() { infinispanManager.stop(); infinispanManager = null; } public int read(String table, String key, Set fields, HashMap result) { try { Map row; if (clustered) { row = AtomicMapLookup.getAtomicMap(infinispanManager.getCache(table), key, false); } else { Cache> cache = infinispanManager.getCache(table); row = cache.get(key); } if (row != null) { result.clear(); if (fields == null || fields.isEmpty()) { StringByteIterator.putAllAsByteIterators(result, row); } else { for (String field : fields) result.put(field, new StringByteIterator(row.get(field))); } } return OK; } catch (Exception e) { return ERROR; } } public int scan(String table, String startkey, int recordcount, Set fields, Vector> result) { logger.warn("Infinispan does not support scan semantics"); return OK; } public int update(String table, String key, HashMap values) { try { if (clustered) { AtomicMap row = AtomicMapLookup.getAtomicMap(infinispanManager.getCache(table), key); StringByteIterator.putAllAsStrings(row, values); } else { Cache> cache = infinispanManager.getCache(table); Map row = cache.get(key); if (row == null) { row = StringByteIterator.getStringMap(values); cache.put(key, row); } else { StringByteIterator.putAllAsStrings(row, values); } } return OK; } catch (Exception e) { return ERROR; } } public int insert(String table, String key, HashMap values) { try { if (clustered) { AtomicMap row = AtomicMapLookup.getAtomicMap(infinispanManager.getCache(table), key); row.clear(); StringByteIterator.putAllAsStrings(row, values); } else { infinispanManager.getCache(table).put(key, values); } return OK; } catch (Exception e) { return ERROR; } } public int delete(String table, String key) { try { if (clustered) AtomicMapLookup.removeAtomicMap(infinispanManager.getCache(table), key); else infinispanManager.getCache(table).remove(key); return OK; } catch (Exception e) { return ERROR; } } } diff --git a/infinispan/src/main/java/com/yahoo/ycsb/db/InfinispanRemoteClient.java b/infinispan/src/main/java/com/yahoo/ycsb/db/InfinispanRemoteClient.java index f670357a..5f79f417 100644 --- a/infinispan/src/main/java/com/yahoo/ycsb/db/InfinispanRemoteClient.java +++ b/infinispan/src/main/java/com/yahoo/ycsb/db/InfinispanRemoteClient.java @@ -1,128 +1,145 @@ +/** + * Copyright (c) 2015 YCSB contributors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + package com.yahoo.ycsb.db; import java.util.HashMap; import java.util.Map; import java.util.Set; import java.util.Vector; import org.infinispan.client.hotrod.RemoteCache; import org.infinispan.client.hotrod.RemoteCacheManager; import org.infinispan.util.logging.Log; import org.infinispan.util.logging.LogFactory; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.StringByteIterator; /** * This is a client implementation for Infinispan 5.x in client-server mode. * * @author mylesjao * */ public class InfinispanRemoteClient extends DB { private RemoteCacheManager remoteIspnManager; private String cacheName = null; private static final Log logger = LogFactory.getLog(InfinispanRemoteClient.class); public InfinispanRemoteClient() { } @Override public void init() throws DBException { remoteIspnManager = RemoteCacheManagerHolder.getInstance(getProperties()); cacheName = getProperties().getProperty("cache"); } @Override public void cleanup() { remoteIspnManager.stop(); remoteIspnManager = null; } @Override public int insert(String table, String recordKey, HashMap values) { String compositKey = createKey(table, recordKey); Map stringValues = new HashMap(); StringByteIterator.putAllAsStrings(stringValues, values); try { cache().put(compositKey, stringValues); return Status.OK; } catch (Exception e) { return Status.ERROR; } } @Override public int read(String table, String recordKey, Set fields, HashMap result) { String compositKey = createKey(table, recordKey); try { Map values = cache().get(compositKey); if(values == null || values.isEmpty()){ return Status.NOT_FOUND; } if(fields == null){ //get all field/value pairs StringByteIterator.putAllAsByteIterators(result, values); }else{ for(String field: fields){ String value = values.get(field); if(value != null){ result.put(field, new StringByteIterator(value) ); } } } return Status.OK; } catch (Exception e) { return Status.ERROR; } } @Override public int scan(String table, String startkey, int recordcount, Set fields, Vector> result) { logger.warn("Infinispan does not support scan semantics"); return Status.NOT_SUPPORT; } @Override public int update(String table, String recordKey, HashMap values) { String compositKey = createKey(table, recordKey); try { Map stringValues = new HashMap(); StringByteIterator.putAllAsStrings(stringValues, values); cache().put(compositKey, stringValues); return Status.OK; } catch (Exception e) { return Status.ERROR; } } @Override public int delete(String table, String recordKey) { String compositKey = createKey(table, recordKey); try { cache().remove(compositKey); return Status.OK; } catch (Exception e) { return Status.ERROR; } } private RemoteCache> cache(){ if(this.cacheName != null){ return remoteIspnManager.getCache(cacheName); }else{ return remoteIspnManager.getCache(); } } private String createKey(String table, String recordKey){ return table + "-" + recordKey; } } diff --git a/infinispan/src/main/java/com/yahoo/ycsb/db/RemoteCacheManagerHolder.java b/infinispan/src/main/java/com/yahoo/ycsb/db/RemoteCacheManagerHolder.java index ffdc9371..b166f6b8 100644 --- a/infinispan/src/main/java/com/yahoo/ycsb/db/RemoteCacheManagerHolder.java +++ b/infinispan/src/main/java/com/yahoo/ycsb/db/RemoteCacheManagerHolder.java @@ -1,25 +1,42 @@ +/** + * Copyright (c) 2015 YCSB contributors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + package com.yahoo.ycsb.db; import java.util.Properties; import org.infinispan.client.hotrod.RemoteCacheManager; public class RemoteCacheManagerHolder { private static volatile RemoteCacheManager cacheManager = null; private RemoteCacheManagerHolder() {} public static RemoteCacheManager getInstance(Properties props){ RemoteCacheManager result = cacheManager; if(result == null){ synchronized (RemoteCacheManagerHolder.class) { result = cacheManager; if (result == null) { cacheManager = result = new RemoteCacheManager(props); } } } return result; } } diff --git a/infinispan/src/main/java/com/yahoo/ycsb/db/Status.java b/infinispan/src/main/java/com/yahoo/ycsb/db/Status.java index 661dc723..00a08a02 100644 --- a/infinispan/src/main/java/com/yahoo/ycsb/db/Status.java +++ b/infinispan/src/main/java/com/yahoo/ycsb/db/Status.java @@ -1,9 +1,26 @@ +/** + * Copyright (c) 2015 YCSB contributors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + package com.yahoo.ycsb.db; public class Status { public static final int OK = 0; public static final int ERROR = 1; public static final int NOT_FOUND = 2; public static final int CONFLICT = 3; public static final int NOT_SUPPORT = 4; } diff --git a/jdbc/pom.xml b/jdbc/pom.xml index 64d2f013..78455f06 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -1,28 +1,45 @@ + + 4.0.0 com.yahoo.ycsb binding-parent 0.3.0-RC4-SNAPSHOT ../binding-parent jdbc-binding JDBC DB Binding jar org.apache.openjpa openjpa-jdbc ${openjpa.jdbc.version} com.yahoo.ycsb core ${project.version} provided diff --git a/jdbc/src/main/conf/db.properties b/jdbc/src/main/conf/db.properties index dac28e23..81849a63 100644 --- a/jdbc/src/main/conf/db.properties +++ b/jdbc/src/main/conf/db.properties @@ -1,7 +1,22 @@ +# Copyright (c) 2012 YCSB contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. See accompanying +# LICENSE file. + # Properties file that contains database connection information. jdbc.driver=org.h2.Driver # jdbc.fetchsize=20 db.url=jdbc:h2:tcp://foo.com:9092/~/h2/ycsb db.user=sa db.passwd= diff --git a/jdbc/src/main/conf/h2.properties b/jdbc/src/main/conf/h2.properties index c985d57b..d698edec 100644 --- a/jdbc/src/main/conf/h2.properties +++ b/jdbc/src/main/conf/h2.properties @@ -1,6 +1,21 @@ +# Copyright (c) 2012 YCSB contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. See accompanying +# LICENSE file. + # Properties file that contains database connection information. jdbc.driver=org.h2.Driver db.url=jdbc:h2:tcp://foo.com:9092/~/h2/ycsb db.user=sa db.passwd= diff --git a/mapkeeper/pom.xml b/mapkeeper/pom.xml index d55c62d9..36818e4f 100644 --- a/mapkeeper/pom.xml +++ b/mapkeeper/pom.xml @@ -1,35 +1,52 @@ + + 4.0.0 com.yahoo.ycsb binding-parent 0.3.0-RC4-SNAPSHOT ../binding-parent mapkeeper-binding Mapkeeper DB Binding jar com.yahoo.mapkeeper mapkeeper ${mapkeeper.version} com.yahoo.ycsb core ${project.version} provided mapkeeper-releases https://raw.github.com/m1ch1/m1ch1-mvn-repo/master/releases diff --git a/mapkeeper/src/main/java/com/yahoo/ycsb/db/MapKeeperClient.java b/mapkeeper/src/main/java/com/yahoo/ycsb/db/MapKeeperClient.java index 5af5ed9e..763d98e3 100644 --- a/mapkeeper/src/main/java/com/yahoo/ycsb/db/MapKeeperClient.java +++ b/mapkeeper/src/main/java/com/yahoo/ycsb/db/MapKeeperClient.java @@ -1,202 +1,219 @@ +/** + * Copyright (c) 2012 YCSB contributors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + package com.yahoo.ycsb.db; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Properties; import java.util.Set; import java.util.Vector; import org.apache.thrift.TException; import org.apache.thrift.protocol.TBinaryProtocol; import org.apache.thrift.protocol.TProtocol; import org.apache.thrift.transport.TFramedTransport; import org.apache.thrift.transport.TSocket; import org.apache.thrift.transport.TTransport; import com.yahoo.mapkeeper.BinaryResponse; import com.yahoo.mapkeeper.MapKeeper; import com.yahoo.mapkeeper.Record; import com.yahoo.mapkeeper.RecordListResponse; import com.yahoo.mapkeeper.ResponseCode; import com.yahoo.mapkeeper.ScanOrder; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.StringByteIterator; import com.yahoo.ycsb.workloads.CoreWorkload; public class MapKeeperClient extends DB { private static final String HOST = "mapkeeper.host"; private static final String HOST_DEFAULT = "localhost"; private static final String PORT = "mapkeeper.port"; private static final String PORT_DEFAULT = "9090"; MapKeeper.Client c; boolean writeallfields; static boolean initteddb = false; private synchronized static void initDB(Properties p, MapKeeper.Client c) throws TException { if(!initteddb) { initteddb = true; c.addMap(p.getProperty(CoreWorkload.TABLENAME_PROPERTY, CoreWorkload.TABLENAME_PROPERTY_DEFAULT)); } } public void init() { String host = getProperties().getProperty(HOST, HOST_DEFAULT); int port = Integer.parseInt(getProperties().getProperty(PORT, PORT_DEFAULT)); TTransport tr = new TFramedTransport(new TSocket(host, port)); TProtocol proto = new TBinaryProtocol(tr); c = new MapKeeper.Client(proto); try { tr.open(); initDB(getProperties(), c); } catch(TException e) { throw new RuntimeException(e); } writeallfields = Boolean.parseBoolean(getProperties().getProperty(CoreWorkload.WRITE_ALL_FIELDS_PROPERTY, CoreWorkload.WRITE_ALL_FIELDS_PROPERTY_DEFAULT)); } ByteBuffer encode(HashMap values) { int len = 0; for(String k : values.keySet()) { len += (k.length() + 1 + values.get(k).bytesLeft() + 1); } byte[] array = new byte[len]; int i = 0; for(String k : values.keySet()) { for(int j = 0; j < k.length(); j++) { array[i] = (byte)k.charAt(j); i++; } array[i] = '\t'; // XXX would like to use sane delimiter (null, 254, 255, ...) but java makes this nearly impossible i++; ByteIterator v = values.get(k); i = v.nextBuf(array, i); array[i] = '\t'; i++; } array[array.length-1] = 0; ByteBuffer buf = ByteBuffer.wrap(array); buf.rewind(); return buf; } void decode(Set fields, String tups, HashMap tup) { String[] tok = tups.split("\\t"); if(tok.length == 0) { throw new IllegalStateException("split returned empty array!"); } for(int i = 0; i < tok.length; i+=2) { if(fields == null || fields.contains(tok[i])) { if(tok.length < i+2) { throw new IllegalStateException("Couldn't parse tuple <" + tups + "> at index " + i); } if(tok[i] == null || tok[i+1] == null) throw new NullPointerException("Key is " + tok[i] + " val is + " + tok[i+1]); tup.put(tok[i], new StringByteIterator(tok[i+1])); } } if(tok.length == 0) { System.err.println("Empty tuple: " + tups); } } int ycsbThriftRet(BinaryResponse succ, ResponseCode zero, ResponseCode one) { return ycsbThriftRet(succ.responseCode, zero, one); } int ycsbThriftRet(ResponseCode rc, ResponseCode zero, ResponseCode one) { return rc == zero ? 0 : rc == one ? 1 : 2; } ByteBuffer bufStr(String str) { ByteBuffer buf = ByteBuffer.wrap(str.getBytes()); return buf; } String strResponse(BinaryResponse buf) { return new String(buf.value.array()); } @Override public int read(String table, String key, Set fields, HashMap result) { try { ByteBuffer buf = bufStr(key); BinaryResponse succ = c.get(table, buf); int ret = ycsbThriftRet( succ, ResponseCode.RecordExists, ResponseCode.RecordNotFound); if(ret == 0) { decode(fields, strResponse(succ), result); } return ret; } catch(TException e) { e.printStackTrace(); return 2; } } @Override public int scan(String table, String startkey, int recordcount, Set fields, Vector> result) { try { //XXX what to pass in for nulls / zeros? RecordListResponse res = c.scan(table, ScanOrder.Ascending, bufStr(startkey), true, null, false, recordcount, 0); int ret = ycsbThriftRet(res.responseCode, ResponseCode.Success, ResponseCode.ScanEnded); if(ret == 0) { for(Record r : res.records) { HashMap tuple = new HashMap(); // Note: r.getKey() and r.getValue() call special helper methods that trim the buffer // to an appropriate length, and memcpy it to a byte[]. Trying to manipulate the ByteBuffer // directly leads to trouble. tuple.put("key", new StringByteIterator(new String(r.getKey()))); decode(fields, new String(r.getValue())/*strBuf(r.bufferForValue())*/, tuple); result.add(tuple); } } return ret; } catch(TException e) { e.printStackTrace(); return 2; } } @Override public int update(String table, String key, HashMap values) { try { if(!writeallfields) { HashMap oldval = new HashMap(); read(table, key, null, oldval); for(String k: values.keySet()) { oldval.put(k, values.get(k)); } values = oldval; } ResponseCode succ = c.update(table, bufStr(key), encode(values)); return ycsbThriftRet(succ, ResponseCode.RecordExists, ResponseCode.RecordNotFound); } catch(TException e) { e.printStackTrace(); return 2; } } @Override public int insert(String table, String key, HashMap values) { try { int ret = ycsbThriftRet(c.insert(table, bufStr(key), encode(values)), ResponseCode.Success, ResponseCode.RecordExists); return ret; } catch(TException e) { e.printStackTrace(); return 2; } } @Override public int delete(String table, String key) { try { return ycsbThriftRet(c.remove(table, bufStr(key)), ResponseCode.Success, ResponseCode.RecordExists); } catch(TException e) { e.printStackTrace(); return 2; } } } diff --git a/mongodb/README.md b/mongodb/README.md index 26c59fda..16717ae2 100644 --- a/mongodb/README.md +++ b/mongodb/README.md @@ -1,129 +1,146 @@ + + ## Quick Start This section describes how to run YCSB on MongoDB. ### 1. Start MongoDB First, download MongoDB and start `mongod`. For example, to start MongoDB on x86-64 Linux box: wget http://fastdl.mongodb.org/linux/mongodb-linux-x86_64-x.x.x.tgz tar xfvz mongodb-linux-x86_64-*.tgz mkdir /tmp/mongodb cd mongodb-linux-x86_64-* ./bin/mongod --dbpath /tmp/mongodb Replace x.x.x above with the latest stable release version for MongoDB. See http://docs.mongodb.org/manual/installation/ for installation steps for various operating systems. ### 2. Install Java and Maven Go to http://www.oracle.com/technetwork/java/javase/downloads/index.html and get the url to download the rpm into your server. For example: wget http://download.oracle.com/otn-pub/java/jdk/7u40-b43/jdk-7u40-linux-x64.rpm?AuthParam=11232426132 -o jdk-7u40-linux-x64.rpm rpm -Uvh jdk-7u40-linux-x64.rpm Or install via yum/apt-get sudo yum install java-devel Download MVN from http://maven.apache.org/download.cgi wget http://ftp.heanet.ie/mirrors/www.apache.org/dist/maven/maven-3/3.1.1/binaries/apache-maven-3.1.1-bin.tar.gz sudo tar xzf apache-maven-*-bin.tar.gz -C /usr/local cd /usr/local sudo ln -s apache-maven-* maven sudo vi /etc/profile.d/maven.sh Add the following to `maven.sh` export M2_HOME=/usr/local/maven export PATH=${M2_HOME}/bin:${PATH} Reload bash and test mvn bash mvn -version ### 3. Set Up YCSB Download the YCSB zip file and compile: git clone git://github.com/brianfrankcooper/YCSB.git cd YCSB mvn -pl com.yahoo.ycsb:mongodb-binding -am clean package ### 4. Run YCSB Now you are ready to run! First, use the asynchronous driver to load the data: ./bin/ycsb load mongodb-async -s -P workloads/workloada > outputLoad.txt Then, run the workload: ./bin/ycsb run mongodb-async -s -P workloads/workloada > outputRun.txt Similarly, to use the synchronous driver from MongoDB Inc. we load the data: ./bin/ycsb load mongodb -s -P workloads/workloada > outputLoad.txt Then, run the workload: ./bin/ycsb run mongodb -s -P workloads/workloada > outputRun.txt See the next section for the list of configuration parameters for MongoDB. ## MongoDB Configuration Parameters - `mongodb.url` - This should be a MongoDB URI or connection string. - See http://docs.mongodb.org/manual/reference/connection-string/ for the standard options. - For the complete set of options for the asynchronous driver see: - http://www.allanbank.com/mongodb-async-driver/apidocs/index.html?com/allanbank/mongodb/MongoDbUri.html - For the complete set of options for the synchronous driver see: - http://api.mongodb.org/java/current/index.html?com/mongodb/MongoClientURI.html - Default value is `mongodb://localhost:27017/ycsb?w=1` - Default value of database is `ycsb` - `mongodb.batchsize` - Useful for the insert workload as it will submit the inserts in batches inproving throughput. - Default value is `1`. - `mongodb.writeConcern` - **Deprecated** - Use the `w` and `journal` options on the MongoDB URI provided by the `mongodb.uri`. - Allowed values are : - `errors_ignored` - `unacknowledged` - `acknowledged` - `journaled` - `replica_acknowledged` - `majority` - Default value is `acknowledged`. - `mongodb.readPreference` - **Deprecated** - Use the `readPreference` options on the MongoDB URI provided by the `mongodb.uri`. - Allowed values are : - `primary` - `primary_preferred` - `secondary` - `secondary_preferred` - `nearest` - Default value is `primary`. - `mongodb.maxconnections` - **Deprecated** - Use the `maxPoolSize` options on the MongoDB URI provided by the `mongodb.uri`. - Default value is `100`. - `mongodb.threadsAllowedToBlockForConnectionMultiplier` - **Deprecated** - Use the `waitQueueMultiple` options on the MongoDB URI provided by the `mongodb.uri`. - Default value is `5`. For example: ./bin/ycsb load mongodb-async -s -P workloads/workloada -p mongodb.url=mongodb://localhost:27017/ycsb?w=0 To run with the synchronous driver from MongoDB Inc.: ./bin/ycsb load mongodb -s -P workloads/workloada -p mongodb.url=mongodb://localhost:27017/ycsb?w=0 diff --git a/mongodb/pom.xml b/mongodb/pom.xml index 8b9a01d0..b800ec86 100644 --- a/mongodb/pom.xml +++ b/mongodb/pom.xml @@ -1,64 +1,82 @@ - - 4.0.0 - - com.yahoo.ycsb - binding-parent - 0.3.0-RC4-SNAPSHOT - ../binding-parent - + + + + 4.0.0 + + com.yahoo.ycsb + binding-parent + 0.3.0-RC4-SNAPSHOT + ../binding-parent + mongodb-binding MongoDB Binding jar org.mongodb mongo-java-driver ${mongodb.version} com.allanbank mongodb-async-driver ${mongodb.async.version} com.yahoo.ycsb core ${project.version} provided ch.qos.logback logback-classic 1.1.2 runtime junit junit 4.12 test true always warn false never fail allanbank Allanbank Releases http://www.allanbank.com/repo/ default diff --git a/mongodb/src/main/java/com/yahoo/ycsb/db/MongoDbClient.java b/mongodb/src/main/java/com/yahoo/ycsb/db/MongoDbClient.java index 644c9fcb..7dacff85 100644 --- a/mongodb/src/main/java/com/yahoo/ycsb/db/MongoDbClient.java +++ b/mongodb/src/main/java/com/yahoo/ycsb/db/MongoDbClient.java @@ -1,449 +1,466 @@ /** + * Copyright (c) 2012 - 2015 YCSB contributors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +/* * MongoDB client binding for YCSB. * * Submitted by Yen Pai on 5/11/2010. * * https://gist.github.com/000a66b8db2caf42467b#file_mongo_database.java * */ package com.yahoo.ycsb.db; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.Vector; import java.util.concurrent.atomic.AtomicInteger; import com.mongodb.client.model.Filters; import com.mongodb.client.model.InsertManyOptions; import org.bson.Document; import org.bson.types.Binary; import com.mongodb.MongoClient; import com.mongodb.MongoClientURI; import com.mongodb.ReadPreference; import com.mongodb.WriteConcern; import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; import com.mongodb.client.MongoDatabase; import com.mongodb.client.model.UpdateOptions; import com.mongodb.client.result.DeleteResult; import com.mongodb.client.result.UpdateResult; import com.yahoo.ycsb.ByteArrayByteIterator; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; /** * MongoDB client for YCSB framework. * * Properties to set: * * mongodb.url=mongodb://localhost:27017 * mongodb.writeConcern=acknowledged * * @author ypai */ public class MongoDbClient extends DB { /** Used to include a field in a response. */ private static final Integer INCLUDE = Integer.valueOf(1); /** The options to use for inserting many documents */ private static final InsertManyOptions INSERT_UNORDERED = new InsertManyOptions().ordered(false); /** The options to use for inserting a single document */ private static final UpdateOptions UPDATE_WITH_UPSERT = new UpdateOptions().upsert(true); /** * The database name to access. */ private static String databaseName; /** The database name to access. */ private static MongoDatabase database; /** * Count the number of times initialized to teardown on the last * {@link #cleanup()}. */ private static final AtomicInteger initCount = new AtomicInteger(0); /** A singleton Mongo instance. */ private static MongoClient mongoClient; /** The default read preference for the test */ private static ReadPreference readPreference; /** The default write concern for the test. */ private static WriteConcern writeConcern; /** The batch size to use for inserts. */ private static int batchSize; /** The bulk inserts pending for the thread. */ private final List bulkInserts = new ArrayList(); /** * Cleanup any state for this DB. Called once per DB instance; there is one * DB instance per client thread. */ @Override public void cleanup() throws DBException { if (initCount.decrementAndGet() == 0) { try { mongoClient.close(); } catch (Exception e1) { System.err.println("Could not close MongoDB connection pool: " + e1.toString()); e1.printStackTrace(); return; } finally { database = null; mongoClient = null; } } } /** * Delete a record from the database. * * @param table * The name of the table * @param key * The record key of the record to delete. * @return Zero on success, a non-zero error code on error. See the * {@link DB} class's description for a discussion of error codes. */ @Override public int delete(String table, String key) { try { MongoCollection collection = database .getCollection(table); Document query = new Document("_id", key); DeleteResult result = collection.withWriteConcern(writeConcern) .deleteOne(query); if (result.wasAcknowledged() && result.getDeletedCount() == 0) { System.err.println("Nothing deleted for key " + key); return 1; } return 0; } catch (Exception e) { System.err.println(e.toString()); return 1; } } /** * Initialize any state for this DB. Called once per DB instance; there is * one DB instance per client thread. */ @Override public void init() throws DBException { initCount.incrementAndGet(); synchronized (INCLUDE) { if (mongoClient != null) { return; } Properties props = getProperties(); // Set insert batchsize, default 1 - to be YCSB-original equivalent batchSize = Integer.parseInt(props.getProperty("batchsize", "1")); // Just use the standard connection format URL // http://docs.mongodb.org/manual/reference/connection-string/ // to configure the client. String url = props.getProperty("mongodb.url", null); boolean defaultedUrl = false; if (url == null) { defaultedUrl = true; url = "mongodb://localhost:27017/ycsb?w=1"; } url = OptionsSupport.updateUrl(url, props); if (!url.startsWith("mongodb://")) { System.err .println("ERROR: Invalid URL: '" + url + "'. Must be of the form " + "'mongodb://:,:/database?options'. " + "http://docs.mongodb.org/manual/reference/connection-string/"); System.exit(1); } try { MongoClientURI uri = new MongoClientURI(url); String uriDb = uri.getDatabase(); if (!defaultedUrl && (uriDb != null) && !uriDb.isEmpty() && !"admin".equals(uriDb)) { databaseName = uriDb; } else { //If no database is specified in URI, use "ycsb" databaseName = "ycsb"; } readPreference = uri.getOptions().getReadPreference(); writeConcern = uri.getOptions().getWriteConcern(); mongoClient = new MongoClient(uri); database = mongoClient.getDatabase(databaseName) .withReadPreference(readPreference) .withWriteConcern(writeConcern); System.out.println("mongo client connection created with " + url); } catch (Exception e1) { System.err .println("Could not initialize MongoDB connection pool for Loader: " + e1.toString()); e1.printStackTrace(); return; } } } /** * Insert a record in the database. Any field/value pairs in the specified * values HashMap will be written into the record with the specified record * key. * * @param table * The name of the table * @param key * The record key of the record to insert. * @param values * A HashMap of field/value pairs to insert in the record * @return Zero on success, a non-zero error code on error. See the * {@link DB} class's description for a discussion of error codes. */ @Override public int insert(String table, String key, HashMap values) { try { MongoCollection collection = database .getCollection(table); Document toInsert = new Document("_id", key); for (Map.Entry entry : values.entrySet()) { toInsert.put(entry.getKey(), entry.getValue().toArray()); } if (batchSize == 1) { // this is effectively an insert, but using an upsert instead due // to current inability of the framework to clean up after itself // between test runs. collection.replaceOne(new Document("_id", toInsert.get("_id")), toInsert, UPDATE_WITH_UPSERT); } else { bulkInserts.add(toInsert); if (bulkInserts.size() == batchSize) { collection.insertMany(bulkInserts, INSERT_UNORDERED); bulkInserts.clear(); } } return 0; } catch (Exception e) { System.err.println("Exception while trying bulk insert with " + bulkInserts.size()); e.printStackTrace(); return 1; } } /** * Read a record from the database. Each field/value pair from the result * will be stored in a HashMap. * * @param table * The name of the table * @param key * The record key of the record to read. * @param fields * The list of fields to read, or null for all of them * @param result * A HashMap of field/value pairs for the result * @return Zero on success, a non-zero error code on error or "not found". */ @Override public int read(String table, String key, Set fields, HashMap result) { try { MongoCollection collection = database .getCollection(table); Document query = new Document("_id", key); FindIterable findIterable = collection.find(query); if (fields != null) { Document projection = new Document(); for (String field : fields) { projection.put(field, INCLUDE); } findIterable.projection(projection); } Document queryResult = findIterable.first(); if (queryResult != null) { fillMap(result, queryResult); } return queryResult != null ? 0 : 1; } catch (Exception e) { System.err.println(e.toString()); return 1; } } /** * Perform a range scan for a set of records in the database. Each * field/value pair from the result will be stored in a HashMap. * * @param table * The name of the table * @param startkey * The record key of the first record to read. * @param recordcount * The number of records to read * @param fields * The list of fields to read, or null for all of them * @param result * A Vector of HashMaps, where each HashMap is a set field/value * pairs for one record * @return Zero on success, a non-zero error code on error. See the * {@link DB} class's description for a discussion of error codes. */ @Override public int scan(String table, String startkey, int recordcount, Set fields, Vector> result) { MongoCursor cursor = null; try { MongoCollection collection = database .getCollection(table); Document scanRange = new Document("$gte", startkey); Document query = new Document("_id", scanRange); Document sort = new Document("_id", INCLUDE); FindIterable findIterable = collection.find(query) .sort(sort) .limit(recordcount); if (fields != null) { Document projection = new Document(); for (String fieldName : fields) { projection.put(fieldName, INCLUDE); } findIterable.projection(projection); } cursor = findIterable.iterator(); if (!cursor.hasNext()) { System.err.println("Nothing found in scan for key " + startkey); return 1; } result.ensureCapacity(recordcount); while (cursor.hasNext()) { HashMap resultMap = new HashMap(); Document obj = cursor.next(); fillMap(resultMap, obj); result.add(resultMap); } return 0; } catch (Exception e) { System.err.println(e.toString()); return 1; } finally { if (cursor != null) { cursor.close(); } } } /** * Update a record in the database. Any field/value pairs in the specified * values HashMap will be written into the record with the specified record * key, overwriting any existing values with the same field name. * * @param table * The name of the table * @param key * The record key of the record to write. * @param values * A HashMap of field/value pairs to update in the record * @return Zero on success, a non-zero error code on error. See this class's * description for a discussion of error codes. */ @Override public int update(String table, String key, HashMap values) { try { MongoCollection collection = database .getCollection(table); Document query = new Document("_id", key); Document fieldsToSet = new Document(); for (Map.Entry entry : values.entrySet()) { fieldsToSet.put(entry.getKey(), entry.getValue().toArray()); } Document update = new Document("$set", fieldsToSet); UpdateResult result = collection.updateOne(query, update); if (result.wasAcknowledged() && result.getMatchedCount() == 0) { System.err.println("Nothing updated for key " + key); return 1; } return 0; } catch (Exception e) { System.err.println(e.toString()); return 1; } } /** * Fills the map with the values from the DBObject. * * @param resultMap * The map to fill/ * @param obj * The object to copy values from. */ protected void fillMap(HashMap resultMap, Document obj) { for (Map.Entry entry : obj.entrySet()) { if (entry.getValue() instanceof Binary) { resultMap.put(entry.getKey(), new ByteArrayByteIterator( ((Binary) entry.getValue()).getData())); } } } } diff --git a/nosqldb/pom.xml b/nosqldb/pom.xml index 8f220f7c..6028c3a5 100644 --- a/nosqldb/pom.xml +++ b/nosqldb/pom.xml @@ -1,27 +1,44 @@ + + 4.0.0 com.yahoo.ycsb binding-parent 0.3.0-RC4-SNAPSHOT ../binding-parent nosqldb-binding Oracle NoSQL Database Binding com.oracle kvclient 1.2.123 com.yahoo.ycsb core ${project.version} provided diff --git a/nosqldb/src/main/conf/nosqldb.properties b/nosqldb/src/main/conf/nosqldb.properties index 22f2504e..9ab5cd1c 100644 --- a/nosqldb/src/main/conf/nosqldb.properties +++ b/nosqldb/src/main/conf/nosqldb.properties @@ -1,28 +1,43 @@ +# Copyright (c) 2012 YCSB contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. See accompanying +# LICENSE file. + # # Sample property file for Oracle NoSQL Database client # # Refer to the Javadoc of oracle.kv.KVStoreConfig class # for more details. # # Store name #storeName=kvstore # Comma-separated list of helper host/port pairs #helperHost=localhost:5000 # Read consistency # "ABSOLUTE" or "NONE_REQUIRED" #consistency=NONE_REQUIRED # Write durability # "COMMIT_NO_SYNC", "COMMIT_SYNC" or "COMMIT_WRITE_NO_SYNC" #durability=COMMIT_NO_SYNC # Limitations on the number of active requests to a node #requestLimit.maxActiveRequests=100 #requestLimit.requestThresholdPercent=90 #requestLimit.nodeLimitPercent=80 # Request timeout in seconds (positive integer) #requestTimeout=5 diff --git a/nosqldb/src/main/conf/script.txt b/nosqldb/src/main/conf/script.txt index 87f1c8af..679d7621 100644 --- a/nosqldb/src/main/conf/script.txt +++ b/nosqldb/src/main/conf/script.txt @@ -1,9 +1,24 @@ +# Copyright (c) 2012 YCSB contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. See accompanying +# LICENSE file. + # Simple configuration file; only one node in a system configure kvstore plan -execute -name "Deploy DC" deploy-datacenter "Local" plan -execute -name "Deploy n01" deploy-sn 1 localhost 5000 plan -execute -name "Deploy admin" deploy-admin 1 5001 addpool LocalPool joinpool LocalPool 1 plan -execute -name "Deploy the store" deploy-store LocalPool 1 100 quit diff --git a/nosqldb/src/main/java/com/yahoo/ycsb/db/NoSqlDbClient.java b/nosqldb/src/main/java/com/yahoo/ycsb/db/NoSqlDbClient.java index a19d8f8f..1426678d 100644 --- a/nosqldb/src/main/java/com/yahoo/ycsb/db/NoSqlDbClient.java +++ b/nosqldb/src/main/java/com/yahoo/ycsb/db/NoSqlDbClient.java @@ -1,221 +1,238 @@ +/** + * Copyright (c) 2012 YCSB contributors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + package com.yahoo.ycsb.db; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.SortedMap; import java.util.Vector; import java.util.concurrent.TimeUnit; import oracle.kv.Consistency; import oracle.kv.Durability; import oracle.kv.FaultException; import oracle.kv.KVStore; import oracle.kv.KVStoreConfig; import oracle.kv.KVStoreFactory; import oracle.kv.Key; import oracle.kv.RequestLimitConfig; import oracle.kv.Value; import oracle.kv.ValueVersion; import com.yahoo.ycsb.ByteArrayByteIterator; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; /** * A database interface layer for Oracle NoSQL Database. */ public class NoSqlDbClient extends DB { public static final int OK = 0; public static final int ERROR = -1; KVStore store; private int getPropertyInt(Properties properties, String key, int defaultValue) throws DBException { String p = properties.getProperty(key); int i = defaultValue; if (p != null) { try { i = Integer.parseInt(p); } catch (NumberFormatException e) { throw new DBException("Illegal number format in " + key + " property"); } } return i; } @Override public void init() throws DBException { Properties properties = getProperties(); /* Mandatory properties */ String storeName = properties.getProperty("storeName", "kvstore"); String[] helperHosts = properties.getProperty("helperHost", "localhost:5000").split(","); KVStoreConfig config = new KVStoreConfig(storeName, helperHosts); /* Optional properties */ String p; p = properties.getProperty("consistency"); if (p != null) { if (p.equalsIgnoreCase("ABSOLUTE")) { config.setConsistency(Consistency.ABSOLUTE); } else if (p.equalsIgnoreCase("NONE_REQUIRED")) { config.setConsistency(Consistency.NONE_REQUIRED); } else { throw new DBException("Illegal value in consistency property"); } } p = properties.getProperty("durability"); if (p != null) { if (p.equalsIgnoreCase("COMMIT_NO_SYNC")) { config.setDurability(Durability.COMMIT_NO_SYNC); } else if (p.equalsIgnoreCase("COMMIT_SYNC")) { config.setDurability(Durability.COMMIT_SYNC); } else if (p.equalsIgnoreCase("COMMIT_WRITE_NO_SYNC")) { config.setDurability(Durability.COMMIT_WRITE_NO_SYNC); } else { throw new DBException("Illegal value in durability property"); } } int maxActiveRequests = getPropertyInt(properties, "requestLimit.maxActiveRequests", RequestLimitConfig.DEFAULT_MAX_ACTIVE_REQUESTS); int requestThresholdPercent = getPropertyInt(properties, "requestLimit.requestThresholdPercent", RequestLimitConfig.DEFAULT_REQUEST_THRESHOLD_PERCENT); int nodeLimitPercent = getPropertyInt(properties, "requestLimit.nodeLimitPercent", RequestLimitConfig.DEFAULT_NODE_LIMIT_PERCENT); RequestLimitConfig requestLimitConfig; /* It is said that the constructor could throw NodeRequestLimitException in Javadoc, the exception is not provided */ // try { requestLimitConfig = new RequestLimitConfig(maxActiveRequests, requestThresholdPercent, nodeLimitPercent); // } catch (NodeRequestLimitException e) { // throw new DBException(e); // } config.setRequestLimit(requestLimitConfig); p = properties.getProperty("requestTimeout"); if (p != null) { long timeout = 1; try { timeout = Long.parseLong(p); } catch (NumberFormatException e) { throw new DBException("Illegal number format in requestTimeout property"); } try { // TODO Support other TimeUnit config.setRequestTimeout(timeout, TimeUnit.SECONDS); } catch (IllegalArgumentException e) { throw new DBException(e); } } try { store = KVStoreFactory.getStore(config); } catch (FaultException e) { throw new DBException(e); } } @Override public void cleanup() throws DBException { store.close(); } /** * Create a key object. * We map "table" and (YCSB's) "key" to a major component of the oracle.kv.Key, * and "field" to a minor component. * * @return An oracle.kv.Key object. */ private static Key createKey(String table, String key, String field) { List majorPath = new ArrayList(); majorPath.add(table); majorPath.add(key); if (field == null) { return Key.createKey(majorPath); } return Key.createKey(majorPath, field); } private static Key createKey(String table, String key) { return createKey(table, key, null); } private static String getFieldFromKey(Key key) { return key.getMinorPath().get(0); } @Override public int read(String table, String key, Set fields, HashMap result) { Key kvKey = createKey(table, key); SortedMap kvResult; try { kvResult = store.multiGet(kvKey, null, null); } catch (FaultException e) { System.err.println(e); return ERROR; } for (Map.Entry entry : kvResult.entrySet()) { /* If fields is null, read all fields */ String field = getFieldFromKey(entry.getKey()); if (fields != null && !fields.contains(field)) { continue; } result.put(field, new ByteArrayByteIterator(entry.getValue().getValue().getValue())); } return OK; } @Override public int scan(String table, String startkey, int recordcount, Set fields, Vector> result) { System.err.println("Oracle NoSQL Database does not support Scan semantics"); return ERROR; } @Override public int update(String table, String key, HashMap values) { for (Map.Entry entry : values.entrySet()) { Key kvKey = createKey(table, key, entry.getKey()); Value kvValue = Value.createValue(entry.getValue().toArray()); try { store.put(kvKey, kvValue); } catch (FaultException e) { System.err.println(e); return ERROR; } } return OK; } @Override public int insert(String table, String key, HashMap values) { return update(table, key, values); } @Override public int delete(String table, String key) { Key kvKey = createKey(table, key); try { store.multiDelete(kvKey, null, null); } catch (FaultException e) { System.err.println(e); return ERROR; } return OK; } } diff --git a/orientdb/README.md b/orientdb/README.md index 05211935..02d1a23e 100644 --- a/orientdb/README.md +++ b/orientdb/README.md @@ -1,31 +1,48 @@ + + ## Quick Start This section describes how to run YCSB on OrientDB running locally. ### 1. Set Up YCSB Clone the YCSB git repository and compile: git clone git://github.com/nuvolabase/YCSB.git cd YCSB mvn clean package ### 2. Run YCSB Now you are ready to run! First, load the data: ./bin/ycsb load orientdb -s -P workloads/workloada Then, run the workload: ./bin/ycsb run orientdb -s -P workloads/workloada See the next section for the list of configuration parameters for OrientDB. ## OrientDB Configuration Parameters ### `OrientDB.url` (default: `local:C:/temp/databases/ycsb`) ### `OrientDB.user` (default `admin`) ### `OrientDB.password` (default `admin`) diff --git a/orientdb/pom.xml b/orientdb/pom.xml index 51eac8c2..aa7a0494 100644 --- a/orientdb/pom.xml +++ b/orientdb/pom.xml @@ -1,34 +1,51 @@ + + 4.0.0 com.yahoo.ycsb binding-parent 0.3.0-RC4-SNAPSHOT ../binding-parent orientdb-binding OrientDB Binding jar sonatype-nexus-snapshots Sonatype Nexus Snapshots https://oss.sonatype.org/content/repositories/snapshots com.yahoo.ycsb core ${project.version} provided com.orientechnologies orientdb-core 1.7.10 diff --git a/orientdb/src/main/java/com/yahoo/ycsb/db/OrientDBClient.java b/orientdb/src/main/java/com/yahoo/ycsb/db/OrientDBClient.java index 692d56d6..fb0d7dd6 100644 --- a/orientdb/src/main/java/com/yahoo/ycsb/db/OrientDBClient.java +++ b/orientdb/src/main/java/com/yahoo/ycsb/db/OrientDBClient.java @@ -1,233 +1,250 @@ +/** + * Copyright (c) 2012 - 2015 YCSB contributors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + /** * OrientDB client binding for YCSB. * * Submitted by Luca Garulli on 5/10/2012. * */ package com.yahoo.ycsb.db; import com.orientechnologies.orient.core.config.OGlobalConfiguration; import com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx; import com.orientechnologies.orient.core.db.record.OIdentifiable; import com.orientechnologies.orient.core.dictionary.ODictionary; import com.orientechnologies.orient.core.index.OIndexCursor; import com.orientechnologies.orient.core.intent.OIntentMassiveInsert; import com.orientechnologies.orient.core.record.ORecordInternal; import com.orientechnologies.orient.core.record.impl.ODocument; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.StringByteIterator; import java.util.HashMap; import java.util.Map.Entry; import java.util.Properties; import java.util.Set; import java.util.Vector; /** * OrientDB client for YCSB framework. * * Properties to set: * * orientdb.url=local:C:/temp/databases or remote:localhost:2424
* orientdb.database=ycsb
* orientdb.user=admin
* orientdb.password=admin
* * @author Luca Garulli * */ public class OrientDBClient extends DB { private static final String CLASS = "usertable"; private ODatabaseDocumentTx db; private ODictionary> dictionary; /** * Initialize any state for this DB. Called once per DB instance; there is one DB instance per client thread. */ public void init() throws DBException { // initialize OrientDB driver Properties props = getProperties(); String url; if (System.getProperty("os.name").toLowerCase().contains("win")) url = props.getProperty("orientdb.url", "plocal:C:/temp/databases/ycsb"); else url = props.getProperty("orientdb.url", "plocal:/temp/databases/ycsb"); String user = props.getProperty("orientdb.user", "admin"); String password = props.getProperty("orientdb.password", "admin"); Boolean newdb = Boolean.parseBoolean(props.getProperty("orientdb.newdb", "false")); try { System.out.println("OrientDB loading database url = " + url); OGlobalConfiguration.STORAGE_KEEP_OPEN.setValue(false); db = new ODatabaseDocumentTx(url); if (db.exists()) { db.open(user, password); if (newdb) { System.out.println("OrientDB drop and recreate fresh db"); db.drop(); db.create(); } } else { System.out.println("OrientDB database not found, create fresh db"); db.create(); } System.out.println("OrientDB connection created with " + url); dictionary = db.getMetadata().getIndexManager().getDictionary(); if (!db.getMetadata().getSchema().existsClass(CLASS)) db.getMetadata().getSchema().createClass(CLASS); db.declareIntent(new OIntentMassiveInsert()); } catch (Exception e1) { System.err.println("Could not initialize OrientDB connection pool for Loader: " + e1.toString()); e1.printStackTrace(); return; } } @Override public void cleanup() throws DBException { if (db != null) { db.close(); db = null; } } @Override /** * Insert a record in the database. Any field/value pairs in the specified values HashMap will be written into the record with the specified * record key. * * @param table The name of the table * @param key The record key of the record to insert. * @param values A HashMap of field/value pairs to insert in the record * @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes. */ public int insert(String table, String key, HashMap values) { try { final ODocument document = new ODocument(CLASS); for (Entry entry : StringByteIterator.getStringMap(values).entrySet()) document.field(entry.getKey(), entry.getValue()); document.save(); dictionary.put(key, document); return 0; } catch (Exception e) { e.printStackTrace(); } return 1; } @Override /** * Delete a record from the database. * * @param table The name of the table * @param key The record key of the record to delete. * @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes. */ public int delete(String table, String key) { try { dictionary.remove(key); return 0; } catch (Exception e) { e.printStackTrace(); } return 1; } @Override /** * Read a record from the database. Each field/value pair from the result will be stored in a HashMap. * * @param table The name of the table * @param key The record key of the record to read. * @param fields The list of fields to read, or null for all of them * @param result A HashMap of field/value pairs for the result * @return Zero on success, a non-zero error code on error or "not found". */ public int read(String table, String key, Set fields, HashMap result) { try { final ODocument document = dictionary.get(key); if (document != null) { if (fields != null) for (String field : fields) result.put(field, new StringByteIterator((String) document.field(field))); else for (String field : document.fieldNames()) result.put(field, new StringByteIterator((String) document.field(field))); return 0; } } catch (Exception e) { e.printStackTrace(); } return 1; } @Override /** * Update a record in the database. Any field/value pairs in the specified values HashMap will be written into the record with the specified * record key, overwriting any existing values with the same field name. * * @param table The name of the table * @param key The record key of the record to write. * @param values A HashMap of field/value pairs to update in the record * @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes. */ public int update(String table, String key, HashMap values) { try { final ODocument document = dictionary.get(key); if (document != null) { for (Entry entry : StringByteIterator.getStringMap(values).entrySet()) document.field(entry.getKey(), entry.getValue()); document.save(); return 0; } } catch (Exception e) { e.printStackTrace(); } return 1; } @Override /** * Perform a range scan for a set of records in the database. Each field/value pair from the result will be stored in a HashMap. * * @param table The name of the table * @param startkey The record key of the first record to read. * @param recordcount The number of records to read * @param fields The list of fields to read, or null for all of them * @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record * @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes. */ public int scan(String table, String startkey, int recordcount, Set fields, Vector> result) { try { final OIndexCursor entries = dictionary.getIndex().iterateEntriesMajor(startkey, true, true); while (entries.hasNext()) { final Entry entry = entries.nextEntry(); final ODocument document = entry.getValue().getRecord(); final HashMap map = new HashMap(); result.add(map); for (String field : fields) map.put(field, new StringByteIterator((String) document.field(field))); } return 0; } catch (Exception e) { e.printStackTrace(); } return 1; } } diff --git a/pom.xml b/pom.xml index b219a8f1..4b0fdfee 100644 --- a/pom.xml +++ b/pom.xml @@ -1,133 +1,150 @@ + + 4.0.0 com.yahoo.ycsb root 0.3.0-RC4-SNAPSHOT pom YCSB Root This is the top level project that builds, packages the core and all the DB bindings for YCSB infrastructure. scm:git:git://github.com/brianfrankcooper/YCSB.git master https://github.com/brianfrankcooper/YCSB checkstyle checkstyle 5.0 org.jdom jdom 1.1 com.google.collections google-collections 1.0 org.slf4j slf4j-api 1.6.4 2.5.5 2.10 1.0.0 1.6.0 1.2.9 1.0.3 8.1.0 7.2.2.Final 2.1.1 3.0.2 2.0.1 1.0.1 2.0.0 0.81 UTF-8 0.8.0 0.9.5.6 1.1.8 1.6.1 3.1.2 core binding-parent accumulo aerospike cassandra couchbase distribution dynamodb elasticsearch gemfire hbase hypertable infinispan jdbc mongodb orientdb redis tarantool org.apache.maven.plugins maven-compiler-plugin 3.3 1.6 1.6 org.apache.maven.plugins maven-checkstyle-plugin 2.15 true checkstyle.xml validate validate checkstyle diff --git a/redis/README.md b/redis/README.md index d2ef806a..ab11cfd0 100644 --- a/redis/README.md +++ b/redis/README.md @@ -1,39 +1,56 @@ + + ## Quick Start This section describes how to run YCSB on Redis. ### 1. Start Redis ### 2. Install Java and Maven ### 3. Set Up YCSB Git clone YCSB and compile: git clone http://github.com/brianfrankcooper/YCSB.git cd YCSB mvn -pl com.yahoo.ycsb:redis-binding -am clean package ### 4. Provide Redis Connection Parameters Set the host, port, and password (do not redis auth is not turned on) in the workload you plan to run. - `redis.url` - `redis.port` - `redis.password` Or, you can set configs with the shell command, EG: ./bin/ycsb load redis -s -P workloads/workloada -p "redis.host=127.0.0.1" -p "redis.port=6379" > outputLoad.txt ### 5. Load data and run tests Load the data: ./bin/ycsb load redis -s -P workloads/workloada > outputLoad.txt Run the workload test: ./bin/ycsb run redis -s -P workloads/workloada > outputRun.txt diff --git a/redis/pom.xml b/redis/pom.xml index 09c5c6a1..7dcf3f02 100644 --- a/redis/pom.xml +++ b/redis/pom.xml @@ -1,28 +1,45 @@ + + 4.0.0 com.yahoo.ycsb binding-parent 0.3.0-RC4-SNAPSHOT ../binding-parent redis-binding Redis DB Binding jar redis.clients jedis ${redis.version} com.yahoo.ycsb core ${project.version} provided diff --git a/redis/src/main/java/com/yahoo/ycsb/db/RedisClient.java b/redis/src/main/java/com/yahoo/ycsb/db/RedisClient.java index 0d5c3ff1..ed6f1ec5 100644 --- a/redis/src/main/java/com/yahoo/ycsb/db/RedisClient.java +++ b/redis/src/main/java/com/yahoo/ycsb/db/RedisClient.java @@ -1,131 +1,148 @@ +/** + * Copyright (c) 2012 YCSB contributors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + /** * Redis client binding for YCSB. * * All YCSB records are mapped to a Redis *hash field*. For scanning * operations, all keys are saved (by an arbitrary hash) in a sorted set. */ package com.yahoo.ycsb.db; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.StringByteIterator; import java.util.Map; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Properties; import java.util.Set; import java.util.Vector; import redis.clients.jedis.Jedis; import redis.clients.jedis.Protocol; public class RedisClient extends DB { private Jedis jedis; public static final String HOST_PROPERTY = "redis.host"; public static final String PORT_PROPERTY = "redis.port"; public static final String PASSWORD_PROPERTY = "redis.password"; public static final String INDEX_KEY = "_indices"; public void init() throws DBException { Properties props = getProperties(); int port; String portString = props.getProperty(PORT_PROPERTY); if (portString != null) { port = Integer.parseInt(portString); } else { port = Protocol.DEFAULT_PORT; } String host = props.getProperty(HOST_PROPERTY); jedis = new Jedis(host, port); jedis.connect(); String password = props.getProperty(PASSWORD_PROPERTY); if (password != null) { jedis.auth(password); } } public void cleanup() throws DBException { jedis.disconnect(); } /* Calculate a hash for a key to store it in an index. The actual return * value of this function is not interesting -- it primarily needs to be * fast and scattered along the whole space of doubles. In a real world * scenario one would probably use the ASCII values of the keys. */ private double hash(String key) { return key.hashCode(); } //XXX jedis.select(int index) to switch to `table` @Override public int read(String table, String key, Set fields, HashMap result) { if (fields == null) { StringByteIterator.putAllAsByteIterators(result, jedis.hgetAll(key)); } else { String[] fieldArray = (String[])fields.toArray(new String[fields.size()]); List values = jedis.hmget(key, fieldArray); Iterator fieldIterator = fields.iterator(); Iterator valueIterator = values.iterator(); while (fieldIterator.hasNext() && valueIterator.hasNext()) { result.put(fieldIterator.next(), new StringByteIterator(valueIterator.next())); } assert !fieldIterator.hasNext() && !valueIterator.hasNext(); } return result.isEmpty() ? 1 : 0; } @Override public int insert(String table, String key, HashMap values) { if (jedis.hmset(key, StringByteIterator.getStringMap(values)).equals("OK")) { jedis.zadd(INDEX_KEY, hash(key), key); return 0; } return 1; } @Override public int delete(String table, String key) { return jedis.del(key) == 0 && jedis.zrem(INDEX_KEY, key) == 0 ? 1 : 0; } @Override public int update(String table, String key, HashMap values) { return jedis.hmset(key, StringByteIterator.getStringMap(values)).equals("OK") ? 0 : 1; } @Override public int scan(String table, String startkey, int recordcount, Set fields, Vector> result) { Set keys = jedis.zrangeByScore(INDEX_KEY, hash(startkey), Double.POSITIVE_INFINITY, 0, recordcount); HashMap values; for (String key : keys) { values = new HashMap(); read(table, key, fields, values); result.add(values); } return 0; } } diff --git a/tarantool/README.md b/tarantool/README.md index 4ad4c9cc..26854e43 100644 --- a/tarantool/README.md +++ b/tarantool/README.md @@ -1,62 +1,79 @@ + + # Tarantool ## Introduction Tarantool is a NoSQL In-Memory database. It's distributed under BSD licence and is hosted on [github][tarantool-github]. Tarantool features: * Defferent index types with iterators: - HASH (the fastest) - TREE (range and ordered retreival) - BITSET (bit mask search) - RTREE (geo search) * multipart keys for HASH and TREE indexes * Data persistence with by Write Ahead Log (WAL) and snapshots. * asynchronous master-master replication, hot standby. * coroutines and async. IO are used to implement high-performance lock-free access to data. - socket-io/file-io with yeilds from lua * stored procedures in Lua (Using LuaJIT) * supports plugins written on C/C++ (Have two basic plugins for working with MySQL and PostgreSQL) * Authentication and access control ## Quick start This section descrives how to run YCSB against a local Tarantool instance ### 1. Start Tarantool First, clone Tarantool from it's own git repo and build it (described in our [README.md][tarantool-readme]): cp %YCSB%/tarantool/conf/tarantool-tree.lua /tarantool.lua cp %TNT%/src/box/tarantool cd ./tarantool tarantool.lua OR you can simply download ans install a binary package for your GNU/Linux or BSD distro from http://tarantool.org/download.html ### 2. Run YCSB Now you are ready to run! First, load the data: ./bin/ycsb load tarantool -s -P workloads/workloada Then, run the workload: ./bin/ycsb run tarantool -s -P workloads/workloada See the next section for the list of configuration parameters for Tarantool. ## Tarantool Configuration Parameters #### 'tarantool.host' (default : 'localhost') Which host YCSB must use for connection with Tarantool #### 'tarantool.port' (default : 3301) Which port YCSB must use for connection with Tarantool #### 'tarantool.space' (default : 1024) (possible values: 0 .. 255) Which space YCSB must use for benchmark Tarantool [tarantool-github]: https://github.com/tarantool/tarantool/ [tarantool-readme]: https://github.com/tarantool/tarantool/blob/master/README.md diff --git a/tarantool/pom.xml b/tarantool/pom.xml index 69568f17..18418fbe 100644 --- a/tarantool/pom.xml +++ b/tarantool/pom.xml @@ -1,38 +1,55 @@ + + 4.0.0 com.yahoo.ycsb binding-parent 0.3.0-RC4-SNAPSHOT ../binding-parent/ tarantool-binding Tarantool DB Binding jar org.tarantool connector ${tarantool.version} com.yahoo.ycsb core ${project.version} provided dgreenru-repo dgreenru repository http://dgreenru.github.com/repo/ diff --git a/voldemort/pom.xml b/voldemort/pom.xml index f1896b11..5fbeab9e 100644 --- a/voldemort/pom.xml +++ b/voldemort/pom.xml @@ -1,41 +1,58 @@ + + 4.0.0 com.yahoo.ycsb binding-parent 0.3.0-RC4-SNAPSHOT ../binding-parent voldemort-binding Voldemort DB Binding jar voldemort voldemort ${voldemort.version} log4j log4j 1.2.16 com.yahoo.ycsb core ${project.version} provided clojars.org http://clojars.org/repo diff --git a/voldemort/src/main/conf/cluster.xml b/voldemort/src/main/conf/cluster.xml index 175f69ad..a34e5009 100644 --- a/voldemort/src/main/conf/cluster.xml +++ b/voldemort/src/main/conf/cluster.xml @@ -1,11 +1,28 @@ + + mycluster 0 localhost 8081 6666 0, 1 diff --git a/voldemort/src/main/conf/server.properties b/voldemort/src/main/conf/server.properties index fce5ee43..783d234b 100644 --- a/voldemort/src/main/conf/server.properties +++ b/voldemort/src/main/conf/server.properties @@ -1,26 +1,41 @@ +# Copyright (c) 2012 YCSB contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. See accompanying +# LICENSE file. + # The ID of *this* particular cluster node node.id=0 max.threads=100 ############### DB options ###################### http.enable=true socket.enable=true # BDB bdb.write.transactions=false bdb.flush.transactions=false bdb.cache.size=1G # Mysql mysql.host=localhost mysql.port=1521 mysql.user=root mysql.password=3306 mysql.database=test #NIO connector settings. enable.nio.connector=true storage.configs=voldemort.store.bdb.BdbStorageConfiguration, voldemort.store.readonly.ReadOnlyStorageConfiguration diff --git a/voldemort/src/main/conf/stores.xml b/voldemort/src/main/conf/stores.xml index b2d892d7..87abe52e 100644 --- a/voldemort/src/main/conf/stores.xml +++ b/voldemort/src/main/conf/stores.xml @@ -1,16 +1,33 @@ + + usertable bdb client 1 1 1 string java-serialization diff --git a/voldemort/src/main/java/com/yahoo/ycsb/db/VoldemortClient.java b/voldemort/src/main/java/com/yahoo/ycsb/db/VoldemortClient.java index 84093b91..40858b08 100644 --- a/voldemort/src/main/java/com/yahoo/ycsb/db/VoldemortClient.java +++ b/voldemort/src/main/java/com/yahoo/ycsb/db/VoldemortClient.java @@ -1,151 +1,168 @@ +/** + * Copyright (c) 2012 YCSB contributors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + package com.yahoo.ycsb.db; import java.util.HashMap; import java.util.Set; import java.util.Vector; import java.util.Map.Entry; import org.apache.log4j.Logger; import voldemort.client.ClientConfig; import voldemort.client.SocketStoreClientFactory; import voldemort.client.StoreClient; import voldemort.versioning.VectorClock; import voldemort.versioning.Versioned; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.StringByteIterator; public class VoldemortClient extends DB { private StoreClient> storeClient; private SocketStoreClientFactory socketFactory; private String storeName; private final Logger logger = Logger.getLogger(VoldemortClient.class); public static final int OK = 0; public static final int ERROR = -1; public static final int NOT_FOUND = -2; /** * Initialize the DB layer. This accepts all properties allowed by the Voldemort client. * A store maps to a table. * Required : bootstrap_urls * Additional property : store_name -> to preload once, should be same as -t * * {@linktourl http://project-voldemort.com/javadoc/client/voldemort/client/ClientConfig.html} */ public void init() throws DBException { ClientConfig clientConfig = new ClientConfig(getProperties()); socketFactory = new SocketStoreClientFactory(clientConfig); // Retrieve store name storeName = getProperties().getProperty("store_name", "usertable"); // Use store name to retrieve client storeClient = socketFactory.getStoreClient(storeName); if ( storeClient == null ) throw new DBException("Unable to instantiate store client"); } public void cleanup() throws DBException { socketFactory.close(); } @Override public int delete(String table, String key) { if ( checkStore(table) == ERROR ) { return ERROR; } if ( storeClient.delete(key) ) return OK; else return ERROR; } @Override public int insert(String table, String key, HashMap values) { if ( checkStore(table) == ERROR ) { return ERROR; } storeClient.put(key, (HashMap)StringByteIterator.getStringMap(values)); return OK; } @Override public int read(String table, String key, Set fields, HashMap result) { if ( checkStore(table) == ERROR ) { return ERROR; } Versioned> versionedValue = storeClient.get(key); if ( versionedValue == null ) return NOT_FOUND; if ( fields != null ) { for (String field : fields) { String val = versionedValue.getValue().get(field); if ( val != null ) result.put(field, new StringByteIterator(val)); } } else { StringByteIterator.putAllAsByteIterators(result, versionedValue.getValue()); } return OK; } @Override public int scan(String table, String startkey, int recordcount, Set fields, Vector> result) { logger.warn("Voldemort does not support Scan semantics"); return OK; } @Override public int update(String table, String key, HashMap values) { if ( checkStore(table) == ERROR ) { return ERROR; } Versioned> versionedValue = storeClient.get(key); HashMap value = new HashMap(); VectorClock version; if ( versionedValue != null ) { version = ((VectorClock) versionedValue.getVersion()).incremented(0, 1); value = versionedValue.getValue(); for (Entry entry : values.entrySet()) { value.put(entry.getKey(), entry.getValue().toString()); } } else { version = new VectorClock(); StringByteIterator.putAllAsStrings(value, values); } storeClient.put(key, Versioned.value(value, version)); return OK; } private int checkStore(String table) { if ( table.compareTo(storeName) != 0 ) { try { storeClient = socketFactory.getStoreClient(table); if ( storeClient == null ) { logger.error("Could not instantiate storeclient for " + table); return ERROR; } storeName = table; } catch ( Exception e ) { return ERROR; } } return OK; } } diff --git a/voldemort/src/main/resources/config/cluster.xml b/voldemort/src/main/resources/config/cluster.xml index 175f69ad..a34e5009 100644 --- a/voldemort/src/main/resources/config/cluster.xml +++ b/voldemort/src/main/resources/config/cluster.xml @@ -1,11 +1,28 @@ + + mycluster 0 localhost 8081 6666 0, 1 diff --git a/voldemort/src/main/resources/config/server.properties b/voldemort/src/main/resources/config/server.properties index fce5ee43..783d234b 100644 --- a/voldemort/src/main/resources/config/server.properties +++ b/voldemort/src/main/resources/config/server.properties @@ -1,26 +1,41 @@ +# Copyright (c) 2012 YCSB contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. See accompanying +# LICENSE file. + # The ID of *this* particular cluster node node.id=0 max.threads=100 ############### DB options ###################### http.enable=true socket.enable=true # BDB bdb.write.transactions=false bdb.flush.transactions=false bdb.cache.size=1G # Mysql mysql.host=localhost mysql.port=1521 mysql.user=root mysql.password=3306 mysql.database=test #NIO connector settings. enable.nio.connector=true storage.configs=voldemort.store.bdb.BdbStorageConfiguration, voldemort.store.readonly.ReadOnlyStorageConfiguration diff --git a/voldemort/src/main/resources/config/stores.xml b/voldemort/src/main/resources/config/stores.xml index b2d892d7..87abe52e 100644 --- a/voldemort/src/main/resources/config/stores.xml +++ b/voldemort/src/main/resources/config/stores.xml @@ -1,16 +1,33 @@ + + usertable bdb client 1 1 1 string java-serialization diff --git a/workloads/workload_template b/workloads/workload_template index 1d55cb0d..e59bace1 100644 --- a/workloads/workload_template +++ b/workloads/workload_template @@ -1,108 +1,123 @@ +# Copyright (c) 2012 YCSB contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. See accompanying +# LICENSE file. + # Yahoo! Cloud System Benchmark # Workload Template: Default Values # # File contains all properties that can be set to define a # YCSB session. All properties are set to their default # value if one exists. If not, the property is commented # out. When a property has a finite number of settings, # the default is enabled and the alternates are shown in # comments below it. # # Use of most explained through comments in Client.java or # CoreWorkload.java or on the YCSB wiki page: # https://github.com/brianfrankcooper/YCSB/wiki/Core-Properties # The name of the workload class to use workload=com.yahoo.ycsb.workloads.CoreWorkload # There is no default setting for recordcount but it is # required to be set. # The number of records in the table to be inserted in # the load phase or the number of records already in the # table before the run phase. recordcount=1000000 # There is no default setting for operationcount but it is # required to be set. # The number of operations to use during the run phase. operationcount=3000000 # The number of insertions to do, if different from recordcount. # Used with insertstart to grow an existing table. #insertcount= # The offset of the first insertion insertstart=0 # The number of fields in a record fieldcount=10 # The size of each field (in bytes) fieldlength=100 # Should read all fields readallfields=true # Should write all fields on update writeallfields=false # The distribution used to choose the length of a field fieldlengthdistribution=constant #fieldlengthdistribution=uniform #fieldlengthdistribution=zipfian # What proportion of operations are reads readproportion=0.95 # What proportion of operations are updates updateproportion=0.05 # What proportion of operations are inserts insertproportion=0 # What proportion of operations read then modify a record readmodifywriteproportion=0 # What proportion of operations are scans scanproportion=0 # On a single scan, the maximum number of records to access maxscanlength=1000 # The distribution used to choose the number of records to access on a scan scanlengthdistribution=uniform #scanlengthdistribution=zipfian # Should records be inserted in order or pseudo-randomly insertorder=hashed #insertorder=ordered # The distribution of requests across the keyspace requestdistribution=zipfian #requestdistribution=uniform #requestdistribution=latest # Percentage of data items that constitute the hot set hotspotdatafraction=0.2 # Percentage of operations that access the hot set hotspotopnfraction=0.8 # Maximum execution time in seconds #maxexecutiontime= # The name of the database table to run queries against table=usertable # The column family of fields (required by some databases) #columnfamily= # How the latency measurements are presented measurementtype=histogram #measurementtype=timeseries # The range of latencies to track in the histogram (milliseconds) histogram.buckets=1000 # Granularity for time series (in milliseconds) timeseries.granularity=1000