>(10);
Status result = instance.scan(MOCK_TABLE, MOCK_KEY1, recordcount, fields, resultParam);
assertEquals(Status.OK, result);
}
}
diff --git a/geode/README.md b/geode/README.md
index 9d4fe3a3..59690bfe 100644
--- a/geode/README.md
+++ b/geode/README.md
@@ -1,67 +1,68 @@
## Quick Start
This section describes how to run YCSB on Apache Geode (incubating).
### Get Apache Geode
You can download Geode from http://geode.incubator.apache.org/releases/
#### Start Geode Cluster
Use the Geode shell (gfsh) to start the cluster. You will need to start
at-least one locator which is a member discovery service and one or more
Geode servers.
Launch gfsh:
```
$ cd $GEODE_HOME
$ ./bin/gfsh
```
Start a locator and two servers:
```
gfsh> start locator --name=locator1
gfsh> start server --name=server1 --server-port=40404
gfsh> start server --name=server2 --server-port=40405
+gfsh> configure pdx --read-serialized=true
```
Create the "usertable" region required by YCSB driver:
```
gfsh>create region --name=usertable --type=PARTITION
```
gfsh has tab autocompletion, so you can play around with various options.
### Start YCSB workload
From your YCSB directory, you can run the ycsb workload as follows
```
./bin/ycsb load geode -P workloads/workloada -p geode.locator=host[port]
```
(default port of locator is 10334).
In the default mode, ycsb geode driver will connect as a client to the geode
cluster. To make the ycsb driver a peer member of the distributed system
use the property
`-p geode.topology=p2p -p geode.locator=host[port]`
Note:
For update workloads, please use the property `-p writeallfields=true`
diff --git a/geode/pom.xml b/geode/pom.xml
index bf8a65a8..4e20d9e2 100644
--- a/geode/pom.xml
+++ b/geode/pom.xml
@@ -1,74 +1,74 @@
4.0.0
com.yahoo.ycsb
binding-parent
- 0.11.0-SNAPSHOT
+ 0.12.0-SNAPSHOT
../binding-parent
geode-binding
Geode DB Binding
jar
false
org.apache.geode
geode-core
${geode.version}
com.yahoo.ycsb
core
${project.version}
provided
org.apache.maven.plugins
maven-checkstyle-plugin
2.15
true
../checkstyle.xml
true
true
validate
validate
checkstyle
diff --git a/geode/src/main/java/com/yahoo/ycsb/db/GeodeClient.java b/geode/src/main/java/com/yahoo/ycsb/db/GeodeClient.java
index f6bcc01a..603b7b73 100644
--- a/geode/src/main/java/com/yahoo/ycsb/db/GeodeClient.java
+++ b/geode/src/main/java/com/yahoo/ycsb/db/GeodeClient.java
@@ -1,191 +1,210 @@
/**
* Copyright (c) 2013 - 2016 YCSB Contributors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
* may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License. See accompanying
* LICENSE file.
*/
package com.yahoo.ycsb.db;
import com.gemstone.gemfire.cache.*;
import com.gemstone.gemfire.cache.client.ClientCache;
import com.gemstone.gemfire.cache.client.ClientCacheFactory;
import com.gemstone.gemfire.cache.client.ClientRegionFactory;
import com.gemstone.gemfire.cache.client.ClientRegionShortcut;
import com.gemstone.gemfire.internal.admin.remote.DistributionLocatorId;
+import com.gemstone.gemfire.internal.cache.GemFireCacheImpl;
+import com.gemstone.gemfire.pdx.JSONFormatter;
+import com.gemstone.gemfire.pdx.PdxInstance;
+import com.gemstone.gemfire.pdx.PdxInstanceFactory;
import com.yahoo.ycsb.*;
import java.util.*;
/**
* Apache Geode (incubating) client for the YCSB benchmark.
*
By default acts as a Geode client and tries to connect
* to Geode cache server running on localhost with default
* cache server port. Hostname and port of a Geode cacheServer
* can be provided using geode.serverport=port and
* geode.serverhost=host properties on YCSB command line.
* A locator may also be used for discovering a cacheServer
* by using the property geode.locator=host[port]
- *
+ *
*
To run this client in a peer-to-peer topology with other Geode
* nodes, use the property geode.topology=p2p. Running
* in p2p mode will enable embedded caching in this client.
- *
+ *
*
YCSB by default does its operations against "usertable". When running
* as a client this is a ClientRegionShortcut.PROXY region,
* when running in p2p mode it is a RegionShortcut.PARTITION
* region. A cache.xml defining "usertable" region can be placed in the
* working directory to override these region definitions.
- *
*/
public class GeodeClient extends DB {
- /** property name of the port where Geode server is listening for connections. */
+ /**
+ * property name of the port where Geode server is listening for connections.
+ */
private static final String SERVERPORT_PROPERTY_NAME = "geode.serverport";
- /** property name of the host where Geode server is running. */
+ /**
+ * property name of the host where Geode server is running.
+ */
private static final String SERVERHOST_PROPERTY_NAME = "geode.serverhost";
- /** default value of {@link #SERVERHOST_PROPERTY_NAME}. */
+ /**
+ * default value of {@link #SERVERHOST_PROPERTY_NAME}.
+ */
private static final String SERVERHOST_PROPERTY_DEFAULT = "localhost";
- /** property name to specify a Geode locator. This property can be used in both
- * client server and p2p topology */
+ /**
+ * property name to specify a Geode locator. This property can be used in both
+ * client server and p2p topology
+ */
private static final String LOCATOR_PROPERTY_NAME = "geode.locator";
- /** property name to specify Geode topology. */
+ /**
+ * property name to specify Geode topology.
+ */
private static final String TOPOLOGY_PROPERTY_NAME = "geode.topology";
- /** value of {@value #TOPOLOGY_PROPERTY_NAME} when peer to peer topology should be used.
- * (client-server topology is default) */
+ /**
+ * value of {@value #TOPOLOGY_PROPERTY_NAME} when peer to peer topology should be used.
+ * (client-server topology is default)
+ */
private static final String TOPOLOGY_P2P_VALUE = "p2p";
private GemFireCache cache;
- /** true if ycsb client runs as a client to a Geode cache server. */
+ /**
+ * true if ycsb client runs as a client to a Geode cache server.
+ */
private boolean isClient;
@Override
public void init() throws DBException {
Properties props = getProperties();
// hostName where Geode cacheServer is running
String serverHost = null;
// port of Geode cacheServer
int serverPort = 0;
String locatorStr = null;
if (props != null && !props.isEmpty()) {
String serverPortStr = props.getProperty(SERVERPORT_PROPERTY_NAME);
if (serverPortStr != null) {
serverPort = Integer.parseInt(serverPortStr);
}
serverHost = props.getProperty(SERVERHOST_PROPERTY_NAME, SERVERHOST_PROPERTY_DEFAULT);
locatorStr = props.getProperty(LOCATOR_PROPERTY_NAME);
String topology = props.getProperty(TOPOLOGY_PROPERTY_NAME);
if (topology != null && topology.equals(TOPOLOGY_P2P_VALUE)) {
CacheFactory cf = new CacheFactory();
if (locatorStr != null) {
cf.set("locators", locatorStr);
}
cache = cf.create();
isClient = false;
return;
}
}
isClient = true;
DistributionLocatorId locator = null;
if (locatorStr != null) {
locator = new DistributionLocatorId(locatorStr);
}
ClientCacheFactory ccf = new ClientCacheFactory();
if (serverPort != 0) {
ccf.addPoolServer(serverHost, serverPort);
} else if (locator != null) {
ccf.addPoolLocator(locator.getHost().getCanonicalHostName(), locator.getPort());
}
cache = ccf.create();
}
@Override
public Status read(String table, String key, Set fields,
HashMap result) {
- Region> r = getRegion(table);
- Map val = r.get(key);
+ Region r = getRegion(table);
+ PdxInstance val = r.get(key);
if (val != null) {
if (fields == null) {
- for (Map.Entry entry : val.entrySet()) {
- result.put(entry.getKey(), new ByteArrayByteIterator(entry.getValue()));
+ for (String fieldName : val.getFieldNames()) {
+ result.put(fieldName, new ByteArrayByteIterator((byte[]) val.getField(fieldName)));
}
} else {
for (String field : fields) {
- result.put(field, new ByteArrayByteIterator(val.get(field)));
+ result.put(field, new ByteArrayByteIterator((byte[]) val.getField(field)));
}
}
return Status.OK;
}
return Status.ERROR;
}
@Override
public Status scan(String table, String startkey, int recordcount,
Set fields, Vector> result) {
// Geode does not support scan
return Status.ERROR;
}
@Override
public Status update(String table, String key, HashMap values) {
getRegion(table).put(key, convertToBytearrayMap(values));
return Status.OK;
}
@Override
public Status insert(String table, String key, HashMap values) {
getRegion(table).put(key, convertToBytearrayMap(values));
return Status.OK;
}
@Override
public Status delete(String table, String key) {
getRegion(table).destroy(key);
return Status.OK;
}
- private Map convertToBytearrayMap(Map values) {
- Map retVal = new HashMap();
+ private PdxInstance convertToBytearrayMap(Map values) {
+ GemFireCacheImpl gci = (GemFireCacheImpl) CacheFactory.getAnyInstance();
+ PdxInstanceFactory pdxInstanceFactory = gci.createPdxInstanceFactory(JSONFormatter.JSON_CLASSNAME);
+
for (Map.Entry entry : values.entrySet()) {
- retVal.put(entry.getKey(), entry.getValue().toArray());
+ pdxInstanceFactory.writeByteArray(entry.getKey(), entry.getValue().toArray());
}
- return retVal;
+ return pdxInstanceFactory.create();
}
- private Region> getRegion(String table) {
- Region> r = cache.getRegion(table);
+ private Region getRegion(String table) {
+ Region r = cache.getRegion(table);
if (r == null) {
try {
if (isClient) {
- ClientRegionFactory> crf =
+ ClientRegionFactory crf =
((ClientCache) cache).createClientRegionFactory(ClientRegionShortcut.PROXY);
r = crf.create(table);
} else {
- RegionFactory> rf = ((Cache) cache).createRegionFactory(RegionShortcut.PARTITION);
+ RegionFactory rf = ((Cache) cache).createRegionFactory(RegionShortcut.PARTITION);
r = rf.create(table);
}
} catch (RegionExistsException e) {
// another thread created the region
r = cache.getRegion(table);
}
}
return r;
}
}
\ No newline at end of file
diff --git a/googlebigtable/pom.xml b/googlebigtable/pom.xml
index d6ade78b..a1c0e143 100644
--- a/googlebigtable/pom.xml
+++ b/googlebigtable/pom.xml
@@ -1,47 +1,47 @@
4.0.0
com.yahoo.ycsb
binding-parent
- 0.11.0-SNAPSHOT
+ 0.12.0-SNAPSHOT
../binding-parent/
googlebigtable-binding
Google Cloud Bigtable Binding
jar
com.google.cloud.bigtable
bigtable-hbase-1.0
${googlebigtable.version}
com.yahoo.ycsb
core
${project.version}
provided
\ No newline at end of file
diff --git a/googledatastore/pom.xml b/googledatastore/pom.xml
index 3da08353..554193a6 100644
--- a/googledatastore/pom.xml
+++ b/googledatastore/pom.xml
@@ -1,50 +1,50 @@
4.0.0
com.yahoo.ycsb
binding-parent
- 0.11.0-SNAPSHOT
+ 0.12.0-SNAPSHOT
../binding-parent
googledatastore-binding
Google Cloud Datastore Binding
https://github.com/GoogleCloudPlatform/google-cloud-datastore
com.google.cloud.datastore
- datastore-v1beta3-proto-client
- 1.0.0-beta.1
+ datastore-v1-proto-client
+ 1.1.0
log4j
log4j
1.2.17
com.yahoo.ycsb
core
${project.version}
provided
diff --git a/googledatastore/src/main/java/com/yahoo/ycsb/db/GoogleDatastoreClient.java b/googledatastore/src/main/java/com/yahoo/ycsb/db/GoogleDatastoreClient.java
index a3f65534..7eb35b1e 100644
--- a/googledatastore/src/main/java/com/yahoo/ycsb/db/GoogleDatastoreClient.java
+++ b/googledatastore/src/main/java/com/yahoo/ycsb/db/GoogleDatastoreClient.java
@@ -1,335 +1,335 @@
/*
* Copyright 2015 YCSB contributors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
* may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License. See accompanying
* LICENSE file.
*/
package com.yahoo.ycsb.db;
import com.google.api.client.auth.oauth2.Credential;
-import com.google.datastore.v1beta3.*;
-import com.google.datastore.v1beta3.CommitRequest.Mode;
-import com.google.datastore.v1beta3.ReadOptions.ReadConsistency;
-import com.google.datastore.v1beta3.client.Datastore;
-import com.google.datastore.v1beta3.client.DatastoreException;
-import com.google.datastore.v1beta3.client.DatastoreFactory;
-import com.google.datastore.v1beta3.client.DatastoreHelper;
-import com.google.datastore.v1beta3.client.DatastoreOptions;
+import com.google.datastore.v1.*;
+import com.google.datastore.v1.CommitRequest.Mode;
+import com.google.datastore.v1.ReadOptions.ReadConsistency;
+import com.google.datastore.v1.client.Datastore;
+import com.google.datastore.v1.client.DatastoreException;
+import com.google.datastore.v1.client.DatastoreFactory;
+import com.google.datastore.v1.client.DatastoreHelper;
+import com.google.datastore.v1.client.DatastoreOptions;
import com.yahoo.ycsb.ByteIterator;
import com.yahoo.ycsb.DB;
import com.yahoo.ycsb.DBException;
import com.yahoo.ycsb.Status;
import com.yahoo.ycsb.StringByteIterator;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import java.io.IOException;
import java.security.GeneralSecurityException;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.Vector;
import javax.annotation.Nullable;
/**
* Google Cloud Datastore Client for YCSB.
*/
public class GoogleDatastoreClient extends DB {
/**
* Defines a MutationType used in this class.
*/
private enum MutationType {
UPSERT,
UPDATE,
DELETE
}
/**
* Defines a EntityGroupingMode enum used in this class.
*/
private enum EntityGroupingMode {
ONE_ENTITY_PER_GROUP,
MULTI_ENTITY_PER_GROUP
}
private static Logger logger =
Logger.getLogger(GoogleDatastoreClient.class);
// Read consistency defaults to "STRONG" per YCSB guidance.
// User can override this via configure.
private ReadConsistency readConsistency = ReadConsistency.STRONG;
private EntityGroupingMode entityGroupingMode =
EntityGroupingMode.ONE_ENTITY_PER_GROUP;
private String rootEntityName;
private Datastore datastore = null;
/**
* Initialize any state for this DB. Called once per DB instance; there is
* one DB instance per client thread.
*/
@Override
public void init() throws DBException {
String debug = getProperties().getProperty("googledatastore.debug", null);
if (null != debug && "true".equalsIgnoreCase(debug)) {
logger.setLevel(Level.DEBUG);
}
// We need the following 3 essential properties to initialize datastore:
//
// - DatasetId,
// - Path to private key file,
// - Service account email address.
String datasetId = getProperties().getProperty(
"googledatastore.datasetId", null);
if (datasetId == null) {
throw new DBException(
"Required property \"datasetId\" missing.");
}
String privateKeyFile = getProperties().getProperty(
"googledatastore.privateKeyFile", null);
if (privateKeyFile == null) {
throw new DBException(
"Required property \"privateKeyFile\" missing.");
}
String serviceAccountEmail = getProperties().getProperty(
"googledatastore.serviceAccountEmail", null);
if (serviceAccountEmail == null) {
throw new DBException(
"Required property \"serviceAccountEmail\" missing.");
}
// Below are properties related to benchmarking.
String readConsistencyConfig = getProperties().getProperty(
"googledatastore.readConsistency", null);
if (readConsistencyConfig != null) {
try {
this.readConsistency = ReadConsistency.valueOf(
readConsistencyConfig.trim().toUpperCase());
} catch (IllegalArgumentException e) {
throw new DBException("Invalid read consistency specified: " +
readConsistencyConfig + ". Expecting STRONG or EVENTUAL.");
}
}
//
// Entity Grouping Mode (googledatastore.entitygroupingmode), see
// documentation in conf/googledatastore.properties.
//
String entityGroupingConfig = getProperties().getProperty(
"googledatastore.entityGroupingMode", null);
if (entityGroupingConfig != null) {
try {
this.entityGroupingMode = EntityGroupingMode.valueOf(
entityGroupingConfig.trim().toUpperCase());
} catch (IllegalArgumentException e) {
throw new DBException("Invalid entity grouping mode specified: " +
entityGroupingConfig + ". Expecting ONE_ENTITY_PER_GROUP or " +
"MULTI_ENTITY_PER_GROUP.");
}
}
this.rootEntityName = getProperties().getProperty(
"googledatastore.rootEntityName", "YCSB_ROOT_ENTITY");
try {
// Setup the connection to Google Cloud Datastore with the credentials
// obtained from the configure.
DatastoreOptions.Builder options = new DatastoreOptions.Builder();
Credential credential = DatastoreHelper.getServiceAccountCredential(
serviceAccountEmail, privateKeyFile);
logger.info("Using JWT Service Account credential.");
logger.info("DatasetID: " + datasetId + ", Service Account Email: " +
serviceAccountEmail + ", Private Key File Path: " + privateKeyFile);
datastore = DatastoreFactory.get().create(
options.credential(credential).projectId(datasetId).build());
} catch (GeneralSecurityException exception) {
throw new DBException("Security error connecting to the datastore: " +
exception.getMessage(), exception);
} catch (IOException exception) {
throw new DBException("I/O error connecting to the datastore: " +
exception.getMessage(), exception);
}
logger.info("Datastore client instance created: " +
datastore.toString());
}
@Override
public Status read(String table, String key, Set fields,
HashMap result) {
LookupRequest.Builder lookupRequest = LookupRequest.newBuilder();
lookupRequest.addKeys(buildPrimaryKey(table, key));
lookupRequest.getReadOptionsBuilder().setReadConsistency(
this.readConsistency);
// Note above, datastore lookupRequest always reads the entire entity, it
// does not support reading a subset of "fields" (properties) of an entity.
logger.debug("Built lookup request as: " + lookupRequest.toString());
LookupResponse response = null;
try {
response = datastore.lookup(lookupRequest.build());
} catch (DatastoreException exception) {
logger.error(
String.format("Datastore Exception when reading (%s): %s %s",
exception.getMessage(),
exception.getMethodName(),
exception.getCode()));
// DatastoreException.getCode() returns an HTTP response code which we
// will bubble up to the user as part of the YCSB Status "name".
return new Status("ERROR-" + exception.getCode(), exception.getMessage());
}
if (response.getFoundCount() == 0) {
return new Status("ERROR-404", "Not Found, key is: " + key);
} else if (response.getFoundCount() > 1) {
// We only asked to lookup for one key, shouldn't have got more than one
// entity back. Unexpected State.
return Status.UNEXPECTED_STATE;
}
Entity entity = response.getFound(0).getEntity();
logger.debug("Read entity: " + entity.toString());
Map properties = entity.getProperties();
Set propertiesToReturn =
(fields == null ? properties.keySet() : fields);
for (String name : propertiesToReturn) {
if (properties.containsKey(name)) {
result.put(name, new StringByteIterator(properties.get(name)
.getStringValue()));
}
}
return Status.OK;
}
@Override
public Status scan(String table, String startkey, int recordcount,
Set fields, Vector> result) {
// TODO: Implement Scan as query on primary key.
return Status.NOT_IMPLEMENTED;
}
@Override
public Status update(String table, String key,
HashMap values) {
return doSingleItemMutation(table, key, values, MutationType.UPDATE);
}
@Override
public Status insert(String table, String key,
HashMap