diff --git a/mongodb/pom.xml b/mongodb/pom.xml index c78ee63c..f004871b 100644 --- a/mongodb/pom.xml +++ b/mongodb/pom.xml @@ -1,64 +1,89 @@ - - 4.0.0 - - com.yahoo.ycsb - binding-parent - 0.4.0-SNAPSHOT - ../binding-parent - + + 4.0.0 + + com.yahoo.ycsb + binding-parent + 0.4.0-SNAPSHOT + ../binding-parent + - mongodb-binding - MongoDB Binding - jar + mongodb-binding + MongoDB Binding + jar - - - org.mongodb - mongo-java-driver - ${mongodb.version} - - - com.allanbank - mongodb-async-driver - ${mongodb.async.version} - - - com.yahoo.ycsb - core - ${project.version} - provided - - - ch.qos.logback - logback-classic - 1.1.2 - runtime - + + + org.mongodb + mongo-java-driver + ${mongodb.version} + + + com.allanbank + mongodb-async-driver + ${mongodb.async.version} + + + com.yahoo.ycsb + core + ${project.version} + provided + + + ch.qos.logback + logback-classic + 1.1.2 + runtime + - - junit - junit - 4.12 - test - - - - - - true - always - warn - - - false - never - fail - - allanbank - Allanbank Releases - http://www.allanbank.com/repo/ - default - - - + + junit + junit + 4.12 + test + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + 2.15 + + true + ../checkstyle.xml + true + true + + + + validate + validate + + checkstyle + + + + + + + + + + true + always + warn + + + false + never + fail + + allanbank + Allanbank Releases + http://www.allanbank.com/repo/ + default + + + diff --git a/mongodb/src/main/java/com/yahoo/ycsb/db/AsyncMongoDbClient.java b/mongodb/src/main/java/com/yahoo/ycsb/db/AsyncMongoDbClient.java index 63ae2a7e..7db9b0b5 100644 --- a/mongodb/src/main/java/com/yahoo/ycsb/db/AsyncMongoDbClient.java +++ b/mongodb/src/main/java/com/yahoo/ycsb/db/AsyncMongoDbClient.java @@ -1,543 +1,531 @@ -/** - * Copyright (c) 2014, Yahoo!, Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. +/* + * Copyright (c) 2014, Yahoo!, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. */ package com.yahoo.ycsb.db; import static com.allanbank.mongodb.builder.QueryBuilder.where; import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.Vector; import java.util.concurrent.atomic.AtomicInteger; import com.allanbank.mongodb.Durability; import com.allanbank.mongodb.LockType; import com.allanbank.mongodb.MongoClient; import com.allanbank.mongodb.MongoClientConfiguration; import com.allanbank.mongodb.MongoCollection; import com.allanbank.mongodb.MongoDatabase; import com.allanbank.mongodb.MongoDbUri; import com.allanbank.mongodb.MongoFactory; import com.allanbank.mongodb.MongoIterator; import com.allanbank.mongodb.ReadPreference; import com.allanbank.mongodb.bson.Document; import com.allanbank.mongodb.bson.Element; import com.allanbank.mongodb.bson.ElementType; import com.allanbank.mongodb.bson.builder.BuilderFactory; import com.allanbank.mongodb.bson.builder.DocumentBuilder; import com.allanbank.mongodb.bson.element.BinaryElement; import com.allanbank.mongodb.builder.BatchedWrite; import com.allanbank.mongodb.builder.BatchedWriteMode; import com.allanbank.mongodb.builder.Find; import com.allanbank.mongodb.builder.Sort; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; /** - * MongoDB asynchronous client for YCSB framework. - * - * Properties to set: - * - * mongodb.url=mongodb://localhost:27017 - * mongodb.writeConcern=normal - * + * MongoDB asynchronous client for YCSB framework using the Asynchronous Java + * Driver + *

+ * See the README.md for configuration information. + *

+ * * @author rjm + * @see Asynchronous + * Java Driver */ public class AsyncMongoDbClient extends DB { - /** Used to include a field in a response. */ - protected static final int INCLUDE = 1; + /** Used to include a field in a response. */ + protected static final int INCLUDE = 1; - /** The database to use. */ - private static String databaseName; + /** The database to use. */ + private static String databaseName; - /** Thread local document builder. */ - private static final ThreadLocal DOCUMENT_BUILDER = new ThreadLocal() { + /** Thread local document builder. */ + private static final ThreadLocal DOCUMENT_BUILDER = + new ThreadLocal() { @Override protected DocumentBuilder initialValue() { - return BuilderFactory.start(); + return BuilderFactory.start(); + } + }; + + /** The write concern for the requests. */ + private static final AtomicInteger INIT_COUNT = new AtomicInteger(0); + + /** The connection to MongoDB. */ + private static MongoClient mongoClient; + + /** The write concern for the requests. */ + private static Durability writeConcern; + + /** Which servers to use for reads. */ + private static ReadPreference readPreference; + + /** The database to MongoDB. */ + private MongoDatabase database; + + /** The batch size to use for inserts. */ + private static int batchSize; + + /** The bulk inserts pending for the thread. */ + private final BatchedWrite.Builder batchedWrite = BatchedWrite.builder() + .mode(BatchedWriteMode.REORDERED); + + /** The number of writes in the batchedWrite. */ + private int batchedWriteCount = 0; + + /** + * Cleanup any state for this DB. Called once per DB instance; there is one DB + * instance per client thread. + */ + @Override + public final void cleanup() throws DBException { + if (INIT_COUNT.decrementAndGet() == 0) { + try { + mongoClient.close(); + } catch (final Exception e1) { + System.err.println("Could not close MongoDB connection pool: " + + e1.toString()); + e1.printStackTrace(); + return; + } finally { + mongoClient = null; + database = null; + } + } + } + + /** + * Delete a record from the database. + * + * @param table + * The name of the table + * @param key + * The record key of the record to delete. + * @return Zero on success, a non-zero error code on error. See this class's + * description for a discussion of error codes. + */ + @Override + public final int delete(final String table, final String key) { + try { + final MongoCollection collection = database.getCollection(table); + final Document q = BuilderFactory.start().add("_id", key).build(); + final long res = collection.delete(q, writeConcern); + if (res == 0) { + System.err.println("Nothing deleted for key " + key); + return 1; + } + return 0; + } catch (final Exception e) { + System.err.println(e.toString()); + return 1; + } + } + + /** + * Initialize any state for this DB. Called once per DB instance; there is one + * DB instance per client thread. + */ + @Override + public final void init() throws DBException { + final int count = INIT_COUNT.incrementAndGet(); + + synchronized (AsyncMongoDbClient.class) { + final Properties props = getProperties(); + + if (mongoClient != null) { + database = mongoClient.getDatabase(databaseName); + + // If there are more threads (count) than connections then the + // Low latency spin lock is not really needed as we will keep + // the connections occupied. + if (count > mongoClient.getConfig().getMaxConnectionCount()) { + mongoClient.getConfig().setLockType(LockType.MUTEX); } - }; - - /** The write concern for the requests. */ - private static final AtomicInteger initCount = new AtomicInteger(0); - - /** The connection to MongoDB. */ - private static MongoClient mongoClient; - - /** The write concern for the requests. */ - private static Durability writeConcern; - /** Which servers to use for reads. */ - private static ReadPreference readPreference; + return; + } + + // Set insert batchsize, default 1 - to be YCSB-original equivalent + batchSize = Integer.parseInt(props.getProperty("mongodb.batchsize", "1")); + + // Just use the standard connection format URL + // http://docs.mongodb.org/manual/reference/connection-string/ + // to configure the client. + String url = + props + .getProperty("mongodb.url", "mongodb://localhost:27017/ycsb?w=1"); + if (!url.startsWith("mongodb://")) { + System.err.println("ERROR: Invalid URL: '" + url + + "'. Must be of the form " + + "'mongodb://:,:/database?" + + "options'. See " + + "http://docs.mongodb.org/manual/reference/connection-string/."); + System.exit(1); + } + + MongoDbUri uri = new MongoDbUri(url); + + try { + databaseName = uri.getDatabase(); + if ((databaseName == null) || databaseName.isEmpty()) { + // Default database is "ycsb" if database is not + // specified in URL + databaseName = "ycsb"; + } - /** The database to MongoDB. */ - private MongoDatabase database; + mongoClient = MongoFactory.createClient(uri); - /** The batch size to use for inserts. */ - private static int batchSize; + MongoClientConfiguration config = mongoClient.getConfig(); + if (!url.toLowerCase().contains("locktype=")) { + config.setLockType(LockType.LOW_LATENCY_SPIN); // assumed... + } - /** The bulk inserts pending for the thread. */ - private final BatchedWrite.Builder batchedWrite = BatchedWrite.builder() - .mode(BatchedWriteMode.REORDERED); + readPreference = config.getDefaultReadPreference(); + writeConcern = config.getDefaultDurability(); - /** The number of writes in the batchedWrite. */ - private int batchedWriteCount = 0; + database = mongoClient.getDatabase(databaseName); - /** - * Cleanup any state for this DB. Called once per DB instance; there is one - * DB instance per client thread. - */ - @Override - public final void cleanup() throws DBException { - if (initCount.decrementAndGet() == 0) { - try { - mongoClient.close(); - } - catch (final Exception e1) { - System.err.println("Could not close MongoDB connection pool: " - + e1.toString()); - e1.printStackTrace(); - return; - } - finally { - mongoClient = null; - database = null; - } - } + System.out.println("mongo connection created with " + url); + } catch (final Exception e1) { + System.err + .println("Could not initialize MongoDB connection pool for Loader: " + + e1.toString()); + e1.printStackTrace(); + return; + } } - - /** - * Delete a record from the database. - * - * @param table - * The name of the table - * @param key - * The record key of the record to delete. - * @return Zero on success, a non-zero error code on error. See this class's - * description for a discussion of error codes. - */ - @Override - public final int delete(final String table, final String key) { - try { - final MongoCollection collection = database.getCollection(table); - final Document q = BuilderFactory.start().add("_id", key).build(); - final long res = collection.delete(q, writeConcern); - if (res == 0) { - System.err.println("Nothing deleted for key " + key); - return 1; - } - return 0; + } + + /** + * Insert a record in the database. Any field/value pairs in the specified + * values HashMap will be written into the record with the specified record + * key. + * + * @param table + * The name of the table + * @param key + * The record key of the record to insert. + * @param values + * A HashMap of field/value pairs to insert in the record + * @return Zero on success, a non-zero error code on error. See the {@link DB} + * class's description for a discussion of error codes. + */ + @Override + public final int insert(final String table, final String key, + final HashMap values) { + try { + final MongoCollection collection = database.getCollection(table); + final DocumentBuilder toInsert = + DOCUMENT_BUILDER.get().reset().add("_id", key); + final Document query = toInsert.build(); + for (final Map.Entry entry : values.entrySet()) { + toInsert.add(entry.getKey(), entry.getValue().toArray()); + } + + // Do an upsert. + if (batchSize <= 1) { + long result = collection.update(query, toInsert, + /* multi= */false, /* upsert= */true, writeConcern); + + return result == 1 ? 0 : 1; + } + + // Use a bulk insert. + try { + batchedWrite.insert(toInsert); + batchedWriteCount += 1; + + if (batchedWriteCount < batchSize) { + return 0; } - catch (final Exception e) { - System.err.println(e.toString()); - return 1; - } - } - /** - * Initialize any state for this DB. Called once per DB instance; there is - * one DB instance per client thread. - */ - @Override - public final void init() throws DBException { - final int count = initCount.incrementAndGet(); - - synchronized (AsyncMongoDbClient.class) { - final Properties props = getProperties(); - - if (mongoClient != null) { - database = mongoClient.getDatabase(databaseName); - - // If there are more threads (count) than connections then the - // Low latency spin lock is not really needed as we will keep - // the connections occupied. - if (count > mongoClient.getConfig().getMaxConnectionCount()) { - mongoClient.getConfig().setLockType(LockType.MUTEX); - } - - return; - } - - // Set insert batchsize, default 1 - to be YCSB-original equivalent - batchSize = Integer.parseInt(props.getProperty("mongodb.batchsize", "1")); - - // Just use the standard connection format URL - // http://docs.mongodb.org/manual/reference/connection-string/ - // to configure the client. - String url = props.getProperty("mongodb.url", - "mongodb://localhost:27017/ycsb?w=1"); - if (!url.startsWith("mongodb://")) { - System.err - .println("ERROR: Invalid URL: '" - + url - + "'. Must be of the form " - + "'mongodb://:,:/database?options'. " - + "See http://docs.mongodb.org/manual/reference/connection-string/."); - System.exit(1); - } - - MongoDbUri uri = new MongoDbUri(url); - - try { - databaseName = uri.getDatabase(); - if ((databaseName == null) || databaseName.isEmpty()) { - // Default database is "ycsb" if database is not - // specified in URL - databaseName="ycsb"; - } - - mongoClient = MongoFactory.createClient(uri); - - MongoClientConfiguration config = mongoClient.getConfig(); - if (!url.toLowerCase().contains("locktype=")) { - config.setLockType(LockType.LOW_LATENCY_SPIN); // assumed... - } - - readPreference = config.getDefaultReadPreference(); - writeConcern = config.getDefaultDurability(); - - database = mongoClient.getDatabase(databaseName); - - System.out.println("mongo connection created with " + url); - } - catch (final Exception e1) { - System.err - .println("Could not initialize MongoDB connection pool for Loader: " - + e1.toString()); - e1.printStackTrace(); - return; - } + long count = collection.write(batchedWrite); + if (count == batchedWriteCount) { + batchedWrite.reset().mode(BatchedWriteMode.REORDERED); + batchedWriteCount = 0; + return 0; } - } - /** - * Insert a record in the database. Any field/value pairs in the specified - * values HashMap will be written into the record with the specified record - * key. - * - * @param table - * The name of the table - * @param key - * The record key of the record to insert. - * @param values - * A HashMap of field/value pairs to insert in the record - * @return Zero on success, a non-zero error code on error. See the - * {@link DB} class's description for a discussion of error codes. - */ - @Override - public final int insert(final String table, final String key, - final HashMap values) { - try { - final MongoCollection collection = database.getCollection(table); - final DocumentBuilder toInsert = DOCUMENT_BUILDER.get().reset() - .add("_id", key); - final Document query = toInsert.build(); - for (final Map.Entry entry : values - .entrySet()) { - toInsert.add(entry.getKey(), entry.getValue().toArray()); - } - - // Do an upsert. - if (batchSize <= 1) { - long result = collection.update(query, toInsert, - /* multi= */false, /* upsert= */true, writeConcern); - - return result == 1 ? 0 : 1; - } - - // Use a bulk insert. - try { - batchedWrite.insert(toInsert); - batchedWriteCount += 1; - - if (batchedWriteCount < batchSize) { - return 0; - } - - long count = collection.write(batchedWrite); - if (count == batchedWriteCount) { - batchedWrite.reset().mode(BatchedWriteMode.REORDERED); - batchedWriteCount = 0; - return 0; - } - - System.err - .println("Number of inserted documents doesn't match the number sent, " - + count - + " inserted, sent " - + batchedWriteCount); - batchedWrite.reset().mode(BatchedWriteMode.REORDERED); - batchedWriteCount = 0; - return 1; - } - catch (Exception e) { - System.err.println("Exception while trying bulk insert with " - + batchedWriteCount); - e.printStackTrace(); - return 1; - } + System.err.println("Number of inserted documents doesn't match the " + + "number sent, " + count + " inserted, sent " + batchedWriteCount); + batchedWrite.reset().mode(BatchedWriteMode.REORDERED); + batchedWriteCount = 0; + return 1; + } catch (Exception e) { + System.err.println("Exception while trying bulk insert with " + + batchedWriteCount); + e.printStackTrace(); + return 1; + } + } catch (final Exception e) { + e.printStackTrace(); + return 1; + } + } + + /** + * Read a record from the database. Each field/value pair from the result will + * be stored in a HashMap. + * + * @param table + * The name of the table + * @param key + * The record key of the record to read. + * @param fields + * The list of fields to read, or null for all of them + * @param result + * A HashMap of field/value pairs for the result + * @return Zero on success, a non-zero error code on error or "not found". + */ + @Override + public final int read(final String table, final String key, + final Set fields, final HashMap result) { + try { + final MongoCollection collection = database.getCollection(table); + final DocumentBuilder query = + DOCUMENT_BUILDER.get().reset().add("_id", key); + + Document queryResult = null; + if (fields != null) { + final DocumentBuilder fieldsToReturn = BuilderFactory.start(); + final Iterator iter = fields.iterator(); + while (iter.hasNext()) { + fieldsToReturn.add(iter.next(), 1); } - catch (final Exception e) { - e.printStackTrace(); - return 1; + + final Find.Builder fb = new Find.Builder(query); + fb.projection(fieldsToReturn); + fb.setLimit(1); + fb.setBatchSize(1); + fb.readPreference(readPreference); + + final MongoIterator ci = collection.find(fb.build()); + if (ci.hasNext()) { + queryResult = ci.next(); + ci.close(); } + } else { + queryResult = collection.findOne(query); + } + + if (queryResult != null) { + fillMap(result, queryResult); + } + return queryResult != null ? 0 : 1; + } catch (final Exception e) { + System.err.println(e.toString()); + return 1; } - /** - * Read a record from the database. Each field/value pair from the result - * will be stored in a HashMap. - * - * @param table - * The name of the table - * @param key - * The record key of the record to read. - * @param fields - * The list of fields to read, or null for all of them - * @param result - * A HashMap of field/value pairs for the result - * @return Zero on success, a non-zero error code on error or "not found". - */ - @Override - public final int read(final String table, final String key, - final Set fields, final HashMap result) { - try { - final MongoCollection collection = database.getCollection(table); - final DocumentBuilder query = DOCUMENT_BUILDER.get().reset() - .add("_id", key); - - Document queryResult = null; - if (fields != null) { - final DocumentBuilder fieldsToReturn = BuilderFactory.start(); - final Iterator iter = fields.iterator(); - while (iter.hasNext()) { - fieldsToReturn.add(iter.next(), 1); - } - - final Find.Builder fb = new Find.Builder(query); - fb.projection(fieldsToReturn); - fb.setLimit(1); - fb.setBatchSize(1); - fb.readPreference(readPreference); - - final MongoIterator ci = collection.find(fb.build()); - if (ci.hasNext()) { - queryResult = ci.next(); - ci.close(); - } - } - else { - queryResult = collection.findOne(query); - } - - if (queryResult != null) { - fillMap(result, queryResult); - } - return queryResult != null ? 0 : 1; - } - catch (final Exception e) { - System.err.println(e.toString()); - return 1; + } + + /** + * Perform a range scan for a set of records in the database. Each field/value + * pair from the result will be stored in a HashMap. + * + * @param table + * The name of the table + * @param startkey + * The record key of the first record to read. + * @param recordcount + * The number of records to read + * @param fields + * The list of fields to read, or null for all of them + * @param result + * A Vector of HashMaps, where each HashMap is a set field/value + * pairs for one record + * @return Zero on success, a non-zero error code on error. See the {@link DB} + * class's description for a discussion of error codes. + */ + @Override + public final int scan(final String table, final String startkey, + final int recordcount, final Set fields, + final Vector> result) { + try { + final MongoCollection collection = database.getCollection(table); + + final Find.Builder find = + Find.builder().query(where("_id").greaterThanOrEqualTo(startkey)) + .limit(recordcount).batchSize(recordcount).sort(Sort.asc("_id")) + .readPreference(readPreference); + + if (fields != null) { + final DocumentBuilder fieldsDoc = BuilderFactory.start(); + for (final String field : fields) { + fieldsDoc.add(field, INCLUDE); } + find.projection(fieldsDoc); + } + + result.ensureCapacity(recordcount); + + final MongoIterator cursor = collection.find(find); + if (!cursor.hasNext()) { + System.err.println("Nothing found in scan for key " + startkey); + return 1; + } + while (cursor.hasNext()) { + // toMap() returns a Map but result.add() expects a + // Map. Hence, the suppress warnings. + final Document doc = cursor.next(); + final HashMap docAsMap = + new HashMap(); + + fillMap(docAsMap, doc); + + result.add(docAsMap); + } + + return 0; + } catch (final Exception e) { + System.err.println(e.toString()); + return 1; + } + } + + /** + * Update a record in the database. Any field/value pairs in the specified + * values HashMap will be written into the record with the specified record + * key, overwriting any existing values with the same field name. + * + * @param table + * The name of the table + * @param key + * The record key of the record to write. + * @param values + * A HashMap of field/value pairs to update in the record + * @return Zero on success, a non-zero error code on error. See the {@link DB} + * class's description for a discussion of error codes. + */ + @Override + public final int update(final String table, final String key, + final HashMap values) { + try { + final MongoCollection collection = database.getCollection(table); + final DocumentBuilder query = BuilderFactory.start().add("_id", key); + final DocumentBuilder update = BuilderFactory.start(); + final DocumentBuilder fieldsToSet = update.push("$set"); + + for (final Map.Entry entry : values.entrySet()) { + fieldsToSet.add(entry.getKey(), entry.getValue().toArray()); + } + final long res = + collection.update(query, update, false, false, writeConcern); + return res == 1 ? 0 : 1; + } catch (final Exception e) { + System.err.println(e.toString()); + return 1; } + } + + /** + * Fills the map with the ByteIterators from the document. + * + * @param result + * The map to fill. + * @param queryResult + * The document to fill from. + */ + protected final void fillMap(final HashMap result, + final Document queryResult) { + for (final Element be : queryResult) { + if (be.getType() == ElementType.BINARY) { + result.put(be.getName(), + new BinaryByteArrayIterator((BinaryElement) be)); + } + } + } + + /** + * BinaryByteArrayIterator provides an adapter from a {@link BinaryElement} to + * a {@link ByteIterator}. + */ + private static final class BinaryByteArrayIterator extends ByteIterator { + + /** The binary data. */ + private final BinaryElement binaryElement; + + /** The current offset into the binary element. */ + private int offset; /** - * Perform a range scan for a set of records in the database. Each - * field/value pair from the result will be stored in a HashMap. + * Creates a new BinaryByteArrayIterator. * - * @param table - * The name of the table - * @param startkey - * The record key of the first record to read. - * @param recordcount - * The number of records to read - * @param fields - * The list of fields to read, or null for all of them - * @param result - * A Vector of HashMaps, where each HashMap is a set field/value - * pairs for one record - * @return Zero on success, a non-zero error code on error. See the - * {@link DB} class's description for a discussion of error codes. + * @param element + * The {@link BinaryElement} to iterate over. */ - @Override - public final int scan(final String table, final String startkey, - final int recordcount, final Set fields, - final Vector> result) { - try { - final MongoCollection collection = database.getCollection(table); - - final Find.Builder find = Find.builder() - .query(where("_id").greaterThanOrEqualTo(startkey)) - .limit(recordcount).batchSize(recordcount) - .sort(Sort.asc("_id")).readPreference(readPreference); - - if (fields != null) { - final DocumentBuilder fieldsDoc = BuilderFactory.start(); - for (final String field : fields) { - fieldsDoc.add(field, INCLUDE); - } - - find.projection(fieldsDoc); - } - - result.ensureCapacity(recordcount); - - final MongoIterator cursor = collection.find(find); - if (!cursor.hasNext()) { - System.err.println("Nothing found in scan for key " + startkey); - return 1; - } - while (cursor.hasNext()) { - // toMap() returns a Map but result.add() expects a - // Map. Hence, the suppress warnings. - final Document doc = cursor.next(); - final HashMap docAsMap = new HashMap(); - - fillMap(docAsMap, doc); - - result.add(docAsMap); - } - - return 0; - } - catch (final Exception e) { - System.err.println(e.toString()); - return 1; - } + public BinaryByteArrayIterator(final BinaryElement element) { + this.binaryElement = element; + this.offset = 0; } /** - * Update a record in the database. Any field/value pairs in the specified - * values HashMap will be written into the record with the specified record - * key, overwriting any existing values with the same field name. - * - * @param table - * The name of the table - * @param key - * The record key of the record to write. - * @param values - * A HashMap of field/value pairs to update in the record - * @return Zero on success, a non-zero error code on error. See the - * {@link DB} class's description for a discussion of error codes. + * {@inheritDoc} + *

+ * Overridden to return the number of bytes remaining in the iterator. + *

*/ @Override - public final int update(final String table, final String key, - final HashMap values) { - try { - final MongoCollection collection = database.getCollection(table); - final DocumentBuilder query = BuilderFactory.start() - .add("_id", key); - final DocumentBuilder update = BuilderFactory.start(); - final DocumentBuilder fieldsToSet = update.push("$set"); - - for (final Map.Entry entry : values - .entrySet()) { - fieldsToSet.add(entry.getKey(), entry.getValue().toArray()); - } - final long res = collection.update(query, update, false, false, - writeConcern); - return res == 1 ? 0 : 1; - } - catch (final Exception e) { - System.err.println(e.toString()); - return 1; - } + public long bytesLeft() { + return Math.max(0, binaryElement.length() - offset); } /** - * Fills the map with the ByteIterators from the document. - * - * @param result - * The map to fill. - * @param queryResult - * The document to fill from. + * {@inheritDoc} + *

+ * Overridden to return true if there is more data in the + * {@link BinaryElement}. + *

*/ - protected final void fillMap(final HashMap result, - final Document queryResult) { - for (final Element be : queryResult) { - if (be.getType() == ElementType.BINARY) { - result.put(be.getName(), new BinaryByteArrayIterator( - (BinaryElement) be)); - } - } + @Override + public boolean hasNext() { + return (offset < binaryElement.length()); } /** - * BinaryByteArrayIterator provides an adapter from a {@link BinaryElement} - * to a {@link ByteIterator}. + * {@inheritDoc} + *

+ * Overridden to return the next value and advance the iterator. + *

*/ - private final static class BinaryByteArrayIterator extends ByteIterator { - - /** The binary data. */ - private final BinaryElement binaryElement; - - /** The current offset into the binary element. */ - private int offset; - - /** - * Creates a new BinaryByteArrayIterator. - * - * @param element - * The {@link BinaryElement} to iterate over. - */ - public BinaryByteArrayIterator(final BinaryElement element) { - this.binaryElement = element; - this.offset = 0; - } - - /** - * {@inheritDoc} - *

- * Overridden to return the number of bytes remaining in the iterator. - *

- */ - @Override - public long bytesLeft() { - return Math.max(0, binaryElement.length() - offset); - } - - /** - * {@inheritDoc} - *

- * Overridden to return true if there is more data in the - * {@link BinaryElement}. - *

- */ - @Override - public boolean hasNext() { - return (offset < binaryElement.length()); - } - - /** - * {@inheritDoc} - *

- * Overridden to return the next value and advance the iterator. - *

- */ - @Override - public byte nextByte() { - final byte value = binaryElement.get(offset); - offset += 1; + @Override + public byte nextByte() { + final byte value = binaryElement.get(offset); + offset += 1; - return value; - } + return value; } + } } diff --git a/mongodb/src/main/java/com/yahoo/ycsb/db/MongoDbClient.java b/mongodb/src/main/java/com/yahoo/ycsb/db/MongoDbClient.java index 644c9fcb..e1aac645 100644 --- a/mongodb/src/main/java/com/yahoo/ycsb/db/MongoDbClient.java +++ b/mongodb/src/main/java/com/yahoo/ycsb/db/MongoDbClient.java @@ -1,449 +1,427 @@ -/** +/* * MongoDB client binding for YCSB. * * Submitted by Yen Pai on 5/11/2010. * * https://gist.github.com/000a66b8db2caf42467b#file_mongo_database.java - * */ - package com.yahoo.ycsb.db; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.Vector; import java.util.concurrent.atomic.AtomicInteger; -import com.mongodb.client.model.Filters; import com.mongodb.client.model.InsertManyOptions; import org.bson.Document; import org.bson.types.Binary; import com.mongodb.MongoClient; import com.mongodb.MongoClientURI; import com.mongodb.ReadPreference; import com.mongodb.WriteConcern; import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; import com.mongodb.client.MongoDatabase; import com.mongodb.client.model.UpdateOptions; import com.mongodb.client.result.DeleteResult; import com.mongodb.client.result.UpdateResult; import com.yahoo.ycsb.ByteArrayByteIterator; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; /** - * MongoDB client for YCSB framework. - * - * Properties to set: - * - * mongodb.url=mongodb://localhost:27017 - * mongodb.writeConcern=acknowledged + * MongoDB asynchronous client for YCSB framework using the MongoDB Inc. driver + *

+ * See the README.md for configuration information. + *

* * @author ypai + * @see MongoDB Inc. + * driver */ public class MongoDbClient extends DB { - /** Used to include a field in a response. */ - private static final Integer INCLUDE = Integer.valueOf(1); - - /** The options to use for inserting many documents */ - private static final InsertManyOptions INSERT_UNORDERED = - new InsertManyOptions().ordered(false); - - /** The options to use for inserting a single document */ - private static final UpdateOptions UPDATE_WITH_UPSERT = - new UpdateOptions().upsert(true); - - /** - * The database name to access. - */ - private static String databaseName; - - /** The database name to access. */ - private static MongoDatabase database; - - /** - * Count the number of times initialized to teardown on the last - * {@link #cleanup()}. - */ - private static final AtomicInteger initCount = new AtomicInteger(0); - - /** A singleton Mongo instance. */ - private static MongoClient mongoClient; - - /** The default read preference for the test */ - private static ReadPreference readPreference; - - /** The default write concern for the test. */ - private static WriteConcern writeConcern; - - /** The batch size to use for inserts. */ - private static int batchSize; - - /** The bulk inserts pending for the thread. */ - private final List bulkInserts = new ArrayList(); - - /** - * Cleanup any state for this DB. Called once per DB instance; there is one - * DB instance per client thread. - */ - @Override - public void cleanup() throws DBException { - if (initCount.decrementAndGet() == 0) { - try { - mongoClient.close(); - } - catch (Exception e1) { - System.err.println("Could not close MongoDB connection pool: " - + e1.toString()); - e1.printStackTrace(); - return; - } - finally { - database = null; - mongoClient = null; - } - } + /** Used to include a field in a response. */ + private static final Integer INCLUDE = Integer.valueOf(1); + + /** The options to use for inserting many documents. */ + private static final InsertManyOptions INSERT_UNORDERED = + new InsertManyOptions().ordered(false); + + /** The options to use for inserting a single document. */ + private static final UpdateOptions UPDATE_WITH_UPSERT = new UpdateOptions() + .upsert(true); + + /** + * The database name to access. + */ + private static String databaseName; + + /** The database name to access. */ + private static MongoDatabase database; + + /** + * Count the number of times initialized to teardown on the last + * {@link #cleanup()}. + */ + private static final AtomicInteger INIT_COUNT = new AtomicInteger(0); + + /** A singleton Mongo instance. */ + private static MongoClient mongoClient; + + /** The default read preference for the test. */ + private static ReadPreference readPreference; + + /** The default write concern for the test. */ + private static WriteConcern writeConcern; + + /** The batch size to use for inserts. */ + private static int batchSize; + + /** The bulk inserts pending for the thread. */ + private final List bulkInserts = new ArrayList(); + + /** + * Cleanup any state for this DB. Called once per DB instance; there is one DB + * instance per client thread. + */ + @Override + public void cleanup() throws DBException { + if (INIT_COUNT.decrementAndGet() == 0) { + try { + mongoClient.close(); + } catch (Exception e1) { + System.err.println("Could not close MongoDB connection pool: " + + e1.toString()); + e1.printStackTrace(); + return; + } finally { + database = null; + mongoClient = null; + } + } + } + + /** + * Delete a record from the database. + * + * @param table + * The name of the table + * @param key + * The record key of the record to delete. + * @return Zero on success, a non-zero error code on error. See the {@link DB} + * class's description for a discussion of error codes. + */ + @Override + public int delete(String table, String key) { + try { + MongoCollection collection = database.getCollection(table); + + Document query = new Document("_id", key); + DeleteResult result = + collection.withWriteConcern(writeConcern).deleteOne(query); + if (result.wasAcknowledged() && result.getDeletedCount() == 0) { + System.err.println("Nothing deleted for key " + key); + return 1; + } + return 0; + } catch (Exception e) { + System.err.println(e.toString()); + return 1; } + } + + /** + * Initialize any state for this DB. Called once per DB instance; there is one + * DB instance per client thread. + */ + @Override + public void init() throws DBException { + INIT_COUNT.incrementAndGet(); + synchronized (INCLUDE) { + if (mongoClient != null) { + return; + } + + Properties props = getProperties(); + + // Set insert batchsize, default 1 - to be YCSB-original equivalent + batchSize = Integer.parseInt(props.getProperty("batchsize", "1")); + + // Just use the standard connection format URL + // http://docs.mongodb.org/manual/reference/connection-string/ + // to configure the client. + String url = props.getProperty("mongodb.url", null); + boolean defaultedUrl = false; + if (url == null) { + defaultedUrl = true; + url = "mongodb://localhost:27017/ycsb?w=1"; + } + + url = OptionsSupport.updateUrl(url, props); + + if (!url.startsWith("mongodb://")) { + System.err.println("ERROR: Invalid URL: '" + url + + "'. Must be of the form " + + "'mongodb://:,:/database?options'. " + + "http://docs.mongodb.org/manual/reference/connection-string/"); + System.exit(1); + } + + try { + MongoClientURI uri = new MongoClientURI(url); + + String uriDb = uri.getDatabase(); + if (!defaultedUrl && (uriDb != null) && !uriDb.isEmpty() + && !"admin".equals(uriDb)) { + databaseName = uriDb; + } else { + // If no database is specified in URI, use "ycsb" + databaseName = "ycsb"; - /** - * Delete a record from the database. - * - * @param table - * The name of the table - * @param key - * The record key of the record to delete. - * @return Zero on success, a non-zero error code on error. See the - * {@link DB} class's description for a discussion of error codes. - */ - @Override - public int delete(String table, String key) { - try { - MongoCollection collection = database - .getCollection(table); - - Document query = new Document("_id", key); - DeleteResult result = collection.withWriteConcern(writeConcern) - .deleteOne(query); - if (result.wasAcknowledged() && result.getDeletedCount() == 0) { - System.err.println("Nothing deleted for key " + key); - return 1; - } - return 0; } - catch (Exception e) { - System.err.println(e.toString()); - return 1; - } - } - /** - * Initialize any state for this DB. Called once per DB instance; there is - * one DB instance per client thread. - */ - @Override - public void init() throws DBException { - initCount.incrementAndGet(); - synchronized (INCLUDE) { - if (mongoClient != null) { - return; - } - - Properties props = getProperties(); - - // Set insert batchsize, default 1 - to be YCSB-original equivalent - batchSize = Integer.parseInt(props.getProperty("batchsize", "1")); - - // Just use the standard connection format URL - // http://docs.mongodb.org/manual/reference/connection-string/ - // to configure the client. - String url = props.getProperty("mongodb.url", null); - boolean defaultedUrl = false; - if (url == null) { - defaultedUrl = true; - url = "mongodb://localhost:27017/ycsb?w=1"; - } - - url = OptionsSupport.updateUrl(url, props); - - if (!url.startsWith("mongodb://")) { - System.err - .println("ERROR: Invalid URL: '" - + url - + "'. Must be of the form " - + "'mongodb://:,:/database?options'. " - + "http://docs.mongodb.org/manual/reference/connection-string/"); - System.exit(1); - } - - try { - MongoClientURI uri = new MongoClientURI(url); - - String uriDb = uri.getDatabase(); - if (!defaultedUrl && (uriDb != null) && !uriDb.isEmpty() - && !"admin".equals(uriDb)) { - databaseName = uriDb; - } - else { - //If no database is specified in URI, use "ycsb" - databaseName = "ycsb"; - - } - - - readPreference = uri.getOptions().getReadPreference(); - writeConcern = uri.getOptions().getWriteConcern(); - - mongoClient = new MongoClient(uri); - database = mongoClient.getDatabase(databaseName) - .withReadPreference(readPreference) - .withWriteConcern(writeConcern); - - System.out.println("mongo client connection created with " - + url); - } - catch (Exception e1) { - System.err - .println("Could not initialize MongoDB connection pool for Loader: " - + e1.toString()); - e1.printStackTrace(); - return; - } + readPreference = uri.getOptions().getReadPreference(); + writeConcern = uri.getOptions().getWriteConcern(); + + mongoClient = new MongoClient(uri); + database = + mongoClient.getDatabase(databaseName) + .withReadPreference(readPreference) + .withWriteConcern(writeConcern); + + System.out.println("mongo client connection created with " + url); + } catch (Exception e1) { + System.err + .println("Could not initialize MongoDB connection pool for Loader: " + + e1.toString()); + e1.printStackTrace(); + return; + } + } + } + + /** + * Insert a record in the database. Any field/value pairs in the specified + * values HashMap will be written into the record with the specified record + * key. + * + * @param table + * The name of the table + * @param key + * The record key of the record to insert. + * @param values + * A HashMap of field/value pairs to insert in the record + * @return Zero on success, a non-zero error code on error. See the {@link DB} + * class's description for a discussion of error codes. + */ + @Override + public int insert(String table, String key, + HashMap values) { + try { + MongoCollection collection = database.getCollection(table); + Document toInsert = new Document("_id", key); + for (Map.Entry entry : values.entrySet()) { + toInsert.put(entry.getKey(), entry.getValue().toArray()); + } + + if (batchSize == 1) { + // this is effectively an insert, but using an upsert instead due + // to current inability of the framework to clean up after itself + // between test runs. + collection.replaceOne(new Document("_id", toInsert.get("_id")), + toInsert, UPDATE_WITH_UPSERT); + } else { + bulkInserts.add(toInsert); + if (bulkInserts.size() == batchSize) { + collection.insertMany(bulkInserts, INSERT_UNORDERED); + bulkInserts.clear(); } + } + return 0; + } catch (Exception e) { + System.err.println("Exception while trying bulk insert with " + + bulkInserts.size()); + e.printStackTrace(); + return 1; } - /** - * Insert a record in the database. Any field/value pairs in the specified - * values HashMap will be written into the record with the specified record - * key. - * - * @param table - * The name of the table - * @param key - * The record key of the record to insert. - * @param values - * A HashMap of field/value pairs to insert in the record - * @return Zero on success, a non-zero error code on error. See the - * {@link DB} class's description for a discussion of error codes. - */ - @Override - public int insert(String table, String key, - HashMap values) { - try { - MongoCollection collection = database - .getCollection(table); - Document toInsert = new Document("_id", key); - for (Map.Entry entry : values.entrySet()) { - toInsert.put(entry.getKey(), entry.getValue().toArray()); - } - - if (batchSize == 1) { - // this is effectively an insert, but using an upsert instead due - // to current inability of the framework to clean up after itself - // between test runs. - collection.replaceOne(new Document("_id", toInsert.get("_id")), - toInsert, - UPDATE_WITH_UPSERT); - } - else { - bulkInserts.add(toInsert); - if (bulkInserts.size() == batchSize) { - collection.insertMany(bulkInserts, INSERT_UNORDERED); - bulkInserts.clear(); - } - } - return 0; + } + + /** + * Read a record from the database. Each field/value pair from the result will + * be stored in a HashMap. + * + * @param table + * The name of the table + * @param key + * The record key of the record to read. + * @param fields + * The list of fields to read, or null for all of them + * @param result + * A HashMap of field/value pairs for the result + * @return Zero on success, a non-zero error code on error or "not found". + */ + @Override + public int read(String table, String key, Set fields, + HashMap result) { + try { + MongoCollection collection = database.getCollection(table); + Document query = new Document("_id", key); + + FindIterable findIterable = collection.find(query); + + if (fields != null) { + Document projection = new Document(); + for (String field : fields) { + projection.put(field, INCLUDE); } - catch (Exception e) { - System.err.println("Exception while trying bulk insert with " - + bulkInserts.size()); - e.printStackTrace(); - return 1; + findIterable.projection(projection); + } + + Document queryResult = findIterable.first(); + + if (queryResult != null) { + fillMap(result, queryResult); + } + return queryResult != null ? 0 : 1; + } catch (Exception e) { + System.err.println(e.toString()); + return 1; + } + } + + /** + * Perform a range scan for a set of records in the database. Each field/value + * pair from the result will be stored in a HashMap. + * + * @param table + * The name of the table + * @param startkey + * The record key of the first record to read. + * @param recordcount + * The number of records to read + * @param fields + * The list of fields to read, or null for all of them + * @param result + * A Vector of HashMaps, where each HashMap is a set field/value + * pairs for one record + * @return Zero on success, a non-zero error code on error. See the {@link DB} + * class's description for a discussion of error codes. + */ + @Override + public int scan(String table, String startkey, int recordcount, + Set fields, Vector> result) { + MongoCursor cursor = null; + try { + MongoCollection collection = database.getCollection(table); + + Document scanRange = new Document("$gte", startkey); + Document query = new Document("_id", scanRange); + Document sort = new Document("_id", INCLUDE); + + FindIterable findIterable = + collection.find(query).sort(sort).limit(recordcount); + + if (fields != null) { + Document projection = new Document(); + for (String fieldName : fields) { + projection.put(fieldName, INCLUDE); } + findIterable.projection(projection); + } - } + cursor = findIterable.iterator(); - /** - * Read a record from the database. Each field/value pair from the result - * will be stored in a HashMap. - * - * @param table - * The name of the table - * @param key - * The record key of the record to read. - * @param fields - * The list of fields to read, or null for all of them - * @param result - * A HashMap of field/value pairs for the result - * @return Zero on success, a non-zero error code on error or "not found". - */ - @Override - public int read(String table, String key, Set fields, - HashMap result) { - try { - MongoCollection collection = database - .getCollection(table); - Document query = new Document("_id", key); - - FindIterable findIterable = collection.find(query); - - if (fields != null) { - Document projection = new Document(); - for (String field : fields) { - projection.put(field, INCLUDE); - } - findIterable.projection(projection); - } - - Document queryResult = findIterable.first(); - - if (queryResult != null) { - fillMap(result, queryResult); - } - return queryResult != null ? 0 : 1; - } - catch (Exception e) { - System.err.println(e.toString()); - return 1; - } - } + if (!cursor.hasNext()) { + System.err.println("Nothing found in scan for key " + startkey); + return 1; + } - /** - * Perform a range scan for a set of records in the database. Each - * field/value pair from the result will be stored in a HashMap. - * - * @param table - * The name of the table - * @param startkey - * The record key of the first record to read. - * @param recordcount - * The number of records to read - * @param fields - * The list of fields to read, or null for all of them - * @param result - * A Vector of HashMaps, where each HashMap is a set field/value - * pairs for one record - * @return Zero on success, a non-zero error code on error. See the - * {@link DB} class's description for a discussion of error codes. - */ - @Override - public int scan(String table, String startkey, int recordcount, - Set fields, Vector> result) { - MongoCursor cursor = null; - try { - MongoCollection collection = database - .getCollection(table); - - Document scanRange = new Document("$gte", startkey); - Document query = new Document("_id", scanRange); - Document sort = new Document("_id", INCLUDE); - - FindIterable findIterable = collection.find(query) - .sort(sort) - .limit(recordcount); - - if (fields != null) { - Document projection = new Document(); - for (String fieldName : fields) { - projection.put(fieldName, INCLUDE); - } - findIterable.projection(projection); - } - - cursor = findIterable.iterator(); - - if (!cursor.hasNext()) { - System.err.println("Nothing found in scan for key " + startkey); - return 1; - } - - result.ensureCapacity(recordcount); - - while (cursor.hasNext()) { - HashMap resultMap = new HashMap(); - - Document obj = cursor.next(); - fillMap(resultMap, obj); - - result.add(resultMap); - } - - return 0; - } - catch (Exception e) { - System.err.println(e.toString()); - return 1; - } - finally { - if (cursor != null) { - cursor.close(); - } - } - } + result.ensureCapacity(recordcount); - /** - * Update a record in the database. Any field/value pairs in the specified - * values HashMap will be written into the record with the specified record - * key, overwriting any existing values with the same field name. - * - * @param table - * The name of the table - * @param key - * The record key of the record to write. - * @param values - * A HashMap of field/value pairs to update in the record - * @return Zero on success, a non-zero error code on error. See this class's - * description for a discussion of error codes. - */ - @Override - public int update(String table, String key, - HashMap values) { - try { - MongoCollection collection = database - .getCollection(table); - - Document query = new Document("_id", key); - Document fieldsToSet = new Document(); - for (Map.Entry entry : values.entrySet()) { - fieldsToSet.put(entry.getKey(), entry.getValue().toArray()); - } - Document update = new Document("$set", fieldsToSet); - - UpdateResult result = collection.updateOne(query, update); - if (result.wasAcknowledged() && result.getMatchedCount() == 0) { - System.err.println("Nothing updated for key " + key); - return 1; - } - return 0; - } - catch (Exception e) { - System.err.println(e.toString()); - return 1; - } - } + while (cursor.hasNext()) { + HashMap resultMap = + new HashMap(); - /** - * Fills the map with the values from the DBObject. - * - * @param resultMap - * The map to fill/ - * @param obj - * The object to copy values from. - */ - protected void fillMap(HashMap resultMap, Document obj) { - for (Map.Entry entry : obj.entrySet()) { - if (entry.getValue() instanceof Binary) { - resultMap.put(entry.getKey(), new ByteArrayByteIterator( - ((Binary) entry.getValue()).getData())); - } - } + Document obj = cursor.next(); + fillMap(resultMap, obj); + + result.add(resultMap); + } + + return 0; + } catch (Exception e) { + System.err.println(e.toString()); + return 1; + } finally { + if (cursor != null) { + cursor.close(); + } + } + } + + /** + * Update a record in the database. Any field/value pairs in the specified + * values HashMap will be written into the record with the specified record + * key, overwriting any existing values with the same field name. + * + * @param table + * The name of the table + * @param key + * The record key of the record to write. + * @param values + * A HashMap of field/value pairs to update in the record + * @return Zero on success, a non-zero error code on error. See this class's + * description for a discussion of error codes. + */ + @Override + public int update(String table, String key, + HashMap values) { + try { + MongoCollection collection = database.getCollection(table); + + Document query = new Document("_id", key); + Document fieldsToSet = new Document(); + for (Map.Entry entry : values.entrySet()) { + fieldsToSet.put(entry.getKey(), entry.getValue().toArray()); + } + Document update = new Document("$set", fieldsToSet); + + UpdateResult result = collection.updateOne(query, update); + if (result.wasAcknowledged() && result.getMatchedCount() == 0) { + System.err.println("Nothing updated for key " + key); + return 1; + } + return 0; + } catch (Exception e) { + System.err.println(e.toString()); + return 1; + } + } + + /** + * Fills the map with the values from the DBObject. + * + * @param resultMap + * The map to fill/ + * @param obj + * The object to copy values from. + */ + protected void fillMap(Map resultMap, Document obj) { + for (Map.Entry entry : obj.entrySet()) { + if (entry.getValue() instanceof Binary) { + resultMap.put(entry.getKey(), + new ByteArrayByteIterator(((Binary) entry.getValue()).getData())); + } } + } } diff --git a/mongodb/src/main/java/com/yahoo/ycsb/db/OptionsSupport.java b/mongodb/src/main/java/com/yahoo/ycsb/db/OptionsSupport.java index 4b142960..62092a0d 100644 --- a/mongodb/src/main/java/com/yahoo/ycsb/db/OptionsSupport.java +++ b/mongodb/src/main/java/com/yahoo/ycsb/db/OptionsSupport.java @@ -1,149 +1,145 @@ /* - * #%L - * OptionsSupport.java - mongodb-binding - Yahoo!, Inc. - * %% - * Copyright (C) 2015 Yahoo!, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * Copyright (c) 2014, Yahoo!, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. */ - package com.yahoo.ycsb.db; import java.util.Properties; /** * OptionsSupport provides methods for handling legacy options. - * + * * @author rjm */ public final class OptionsSupport { - /** Value for an unavailable property. */ - private static final String UNAVAILABLE = "n/a"; - - /** - * Updates the URL with the appropriate attributes if legacy properties are - * set and the URL does not have the property already set. - * - * @param url - * The URL to update. - * @param props - * The legacy properties. - * @return The updated URL. - */ - public static String updateUrl(String url, Properties props) { - String result = url; + /** Value for an unavailable property. */ + private static final String UNAVAILABLE = "n/a"; - // max connections. - final String maxConnections = props.getProperty( - "mongodb.maxconnections", UNAVAILABLE).toLowerCase(); - if (!UNAVAILABLE.equals(maxConnections)) { - result = addUrlOption(result, "maxPoolSize", maxConnections); - } + /** + * Updates the URL with the appropriate attributes if legacy properties are + * set and the URL does not have the property already set. + * + * @param url + * The URL to update. + * @param props + * The legacy properties. + * @return The updated URL. + */ + public static String updateUrl(String url, Properties props) { + String result = url; - // Blocked thread multiplier. - final String threadsAllowedToBlockForConnectionMultiplier = props - .getProperty( - "mongodb.threadsAllowedToBlockForConnectionMultiplier", - UNAVAILABLE).toLowerCase(); - if (!UNAVAILABLE.equals(threadsAllowedToBlockForConnectionMultiplier)) { - result = addUrlOption(result, "waitQueueMultiple", - threadsAllowedToBlockForConnectionMultiplier); - } + // max connections. + final String maxConnections = + props.getProperty("mongodb.maxconnections", UNAVAILABLE).toLowerCase(); + if (!UNAVAILABLE.equals(maxConnections)) { + result = addUrlOption(result, "maxPoolSize", maxConnections); + } - // write concern - String writeConcernType = props.getProperty("mongodb.writeConcern", + // Blocked thread multiplier. + final String threadsAllowedToBlockForConnectionMultiplier = + props + .getProperty( + "mongodb.threadsAllowedToBlockForConnectionMultiplier", UNAVAILABLE).toLowerCase(); - if (!UNAVAILABLE.equals(writeConcernType)) { - if ("errors_ignored".equals(writeConcernType)) { - result = addUrlOption(result, "w", "0"); - } - else if ("unacknowledged".equals(writeConcernType)) { - result = addUrlOption(result, "w", "0"); - } - else if ("acknowledged".equals(writeConcernType)) { - result = addUrlOption(result, "w", "1"); - } - else if ("journaled".equals(writeConcernType)) { - result = addUrlOption(result, "journal", "true"); // this is the documented option name - result = addUrlOption(result, "j", "true"); // but keep this until MongoDB Java driver supports "journal" option - } - else if ("replica_acknowledged".equals(writeConcernType)) { - result = addUrlOption(result, "w", "2"); - } - else if ("majority".equals(writeConcernType)) { - result = addUrlOption(result, "w", "majority"); - } - else { - System.err.println("WARNING: Invalid writeConcern: '" - + writeConcernType + "' will be ignored. " - + "Must be one of [ unacknowledged | acknowledged | " - + "journaled | replica_acknowledged | majority ]"); - } - } + if (!UNAVAILABLE.equals(threadsAllowedToBlockForConnectionMultiplier)) { + result = + addUrlOption(result, "waitQueueMultiple", + threadsAllowedToBlockForConnectionMultiplier); + } - // read preference - String readPreferenceType = props.getProperty("mongodb.readPreference", - UNAVAILABLE).toLowerCase(); - if (!UNAVAILABLE.equals(readPreferenceType)) { - if ("primary".equals(readPreferenceType)) { - result = addUrlOption(result, "readPreference", "primary"); - } - else if ("primary_preferred".equals(readPreferenceType)) { - result = addUrlOption(result, "readPreference", - "primaryPreferred"); - } - else if ("secondary".equals(readPreferenceType)) { - result = addUrlOption(result, "readPreference", "secondary"); - } - else if ("secondary_preferred".equals(readPreferenceType)) { - result = addUrlOption(result, "readPreference", - "secondaryPreferred"); - } - else if ("nearest".equals(readPreferenceType)) { - result = addUrlOption(result, "readPreference", "nearest"); - } - else { - System.err.println("WARNING: Invalid readPreference: '" - + readPreferenceType + "' will be ignored. " - + "Must be one of [ primary | primary_preferred | " - + "secondary | secondary_preferred | nearest ]"); - } - } + // write concern + String writeConcernType = + props.getProperty("mongodb.writeConcern", UNAVAILABLE).toLowerCase(); + if (!UNAVAILABLE.equals(writeConcernType)) { + if ("errors_ignored".equals(writeConcernType)) { + result = addUrlOption(result, "w", "0"); + } else if ("unacknowledged".equals(writeConcernType)) { + result = addUrlOption(result, "w", "0"); + } else if ("acknowledged".equals(writeConcernType)) { + result = addUrlOption(result, "w", "1"); + } else if ("journaled".equals(writeConcernType)) { + result = addUrlOption(result, "journal", "true"); // this is the + // documented option + // name + result = addUrlOption(result, "j", "true"); // but keep this until + // MongoDB Java driver + // supports "journal" option + } else if ("replica_acknowledged".equals(writeConcernType)) { + result = addUrlOption(result, "w", "2"); + } else if ("majority".equals(writeConcernType)) { + result = addUrlOption(result, "w", "majority"); + } else { + System.err.println("WARNING: Invalid writeConcern: '" + + writeConcernType + "' will be ignored. " + + "Must be one of [ unacknowledged | acknowledged | " + + "journaled | replica_acknowledged | majority ]"); + } + } - return result; + // read preference + String readPreferenceType = + props.getProperty("mongodb.readPreference", UNAVAILABLE).toLowerCase(); + if (!UNAVAILABLE.equals(readPreferenceType)) { + if ("primary".equals(readPreferenceType)) { + result = addUrlOption(result, "readPreference", "primary"); + } else if ("primary_preferred".equals(readPreferenceType)) { + result = addUrlOption(result, "readPreference", "primaryPreferred"); + } else if ("secondary".equals(readPreferenceType)) { + result = addUrlOption(result, "readPreference", "secondary"); + } else if ("secondary_preferred".equals(readPreferenceType)) { + result = addUrlOption(result, "readPreference", "secondaryPreferred"); + } else if ("nearest".equals(readPreferenceType)) { + result = addUrlOption(result, "readPreference", "nearest"); + } else { + System.err.println("WARNING: Invalid readPreference: '" + + readPreferenceType + "' will be ignored. " + + "Must be one of [ primary | primary_preferred | " + + "secondary | secondary_preferred | nearest ]"); + } } - /** - * Adds an option to the url if it does not already contain the option. - * - * @param url - * The URL to append the options to. - * @param name - * The name of the option. - * @param value - * The value for the option. - * @return The updated URL. - */ - private static String addUrlOption(String url, String name, String value) { - String fullName = name + "="; - if (!url.contains(fullName)) { - if (url.contains("?")) { - return url + "&" + fullName + value; - } - return url + "?" + fullName + value; - } - return url; + return result; + } + + /** + * Adds an option to the url if it does not already contain the option. + * + * @param url + * The URL to append the options to. + * @param name + * The name of the option. + * @param value + * The value for the option. + * @return The updated URL. + */ + private static String addUrlOption(String url, String name, String value) { + String fullName = name + "="; + if (!url.contains(fullName)) { + if (url.contains("?")) { + return url + "&" + fullName + value; + } + return url + "?" + fullName + value; } + return url; + } + + /** + * Hidden Constructor. + */ + private OptionsSupport() { + // Nothing. + } } diff --git a/mongodb/src/main/java/com/yahoo/ycsb/db/package-info.java b/mongodb/src/main/java/com/yahoo/ycsb/db/package-info.java new file mode 100644 index 00000000..6c64e3fa --- /dev/null +++ b/mongodb/src/main/java/com/yahoo/ycsb/db/package-info.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2014, Yahoo!, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +/** + * The YCSB binding for MongoDB. + * For additional details on using and configuring the binding see the + * accompanying README.md. + *

+ * A YCSB binding is provided for both the the + * Asynchronous + * Java Driver and the MongoDB Inc. + * driver. + *

+ */ +package com.yahoo.ycsb.db; + diff --git a/mongodb/src/test/java/com/yahoo/ycsb/db/AbstractDBTestCases.java b/mongodb/src/test/java/com/yahoo/ycsb/db/AbstractDBTestCases.java index 79340e80..a2ced3f5 100644 --- a/mongodb/src/test/java/com/yahoo/ycsb/db/AbstractDBTestCases.java +++ b/mongodb/src/test/java/com/yahoo/ycsb/db/AbstractDBTestCases.java @@ -1,267 +1,263 @@ -/* - * Copyright (c) 2014, Yahoo!, Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. +/* + * Copyright (c) 2014, Yahoo!, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. */ package com.yahoo.ycsb.db; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.CoreMatchers.notNullValue; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeNoException; import java.io.IOException; import java.net.InetAddress; import java.net.Socket; import java.util.Collections; import java.util.HashMap; import java.util.Set; import java.util.Vector; import org.junit.BeforeClass; import org.junit.Test; import com.yahoo.ycsb.ByteArrayByteIterator; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; /** * MongoDbClientTest provides runs the basic DB test cases. *

* The tests will be skipped if MongoDB is not running on port 27017 on the * local machine. See the README.md for how to get MongoDB running. *

*/ @SuppressWarnings("boxing") public abstract class AbstractDBTestCases { - /** The default port for MongoDB. */ - private static final int MONGODB_DEFAULT_PORT = 27017; + /** The default port for MongoDB. */ + private static final int MONGODB_DEFAULT_PORT = 27017; - /** - * Verifies the mongod process (or some process) is running on port 27017, - * if not the tests are skipped. - */ - @BeforeClass - public static void setUpBeforeClass() { - // Test if we can connect. - Socket socket = null; + /** + * Verifies the mongod process (or some process) is running on port 27017, if + * not the tests are skipped. + */ + @BeforeClass + public static void setUpBeforeClass() { + // Test if we can connect. + Socket socket = null; + try { + // Connect + socket = new Socket(InetAddress.getLocalHost(), MONGODB_DEFAULT_PORT); + assertThat("Socket is not bound.", socket.getLocalPort(), not(-1)); + } catch (IOException connectFailed) { + assumeNoException("MongoDB is not running. Skipping tests.", + connectFailed); + } finally { + if (socket != null) { try { - // Connect - socket = new Socket(InetAddress.getLocalHost(), - MONGODB_DEFAULT_PORT); - assertThat("Socket is not bound.", socket.getLocalPort(), not(-1)); - } - catch (IOException connectFailed) { - assumeNoException("MongoDB is not running. Skipping tests.", - connectFailed); - } - finally { - if (socket != null) { - try { - socket.close(); - } - catch (IOException ignore) { - // Ignore. - } - } - socket = null; + socket.close(); + } catch (IOException ignore) { + // Ignore. } + } + socket = null; } + } - /** - * Test method for {@link DB#insert}, {@link DB#read}, and {@link DB#delete} - * . - */ - @Test - public void testInsertReadDelete() { - final DB client = getDB(); + /** + * Test method for {@link DB#insert}, {@link DB#read}, and {@link DB#delete} . + */ + @Test + public void testInsertReadDelete() { + final DB client = getDB(); - final String table = "test"; - final String id = "delete"; + final String table = getClass().getSimpleName(); + final String id = "delete"; - HashMap inserted = new HashMap(); - inserted.put("a", new ByteArrayByteIterator(new byte[] { 1, 2, 3, 4 })); - int result = client.insert(table, id, inserted); - assertThat("Insert did not return success (0).", result, is(0)); + HashMap inserted = + new HashMap(); + inserted.put("a", new ByteArrayByteIterator(new byte[] { 1, 2, 3, 4 })); + int result = client.insert(table, id, inserted); + assertThat("Insert did not return success (0).", result, is(0)); - HashMap read = new HashMap(); - Set keys = Collections.singleton("a"); - result = client.read(table, id, keys, read); - assertThat("Read did not return success (0).", result, is(0)); - for (String key : keys) { - ByteIterator iter = read.get(key); + HashMap read = new HashMap(); + Set keys = Collections.singleton("a"); + result = client.read(table, id, keys, read); + assertThat("Read did not return success (0).", result, is(0)); + for (String key : keys) { + ByteIterator iter = read.get(key); - assertThat("Did not read the inserted field: " + key, iter, - notNullValue()); - assertTrue(iter.hasNext()); - assertThat(iter.nextByte(), is(Byte.valueOf((byte) 1))); - assertTrue(iter.hasNext()); - assertThat(iter.nextByte(), is(Byte.valueOf((byte) 2))); - assertTrue(iter.hasNext()); - assertThat(iter.nextByte(), is(Byte.valueOf((byte) 3))); - assertTrue(iter.hasNext()); - assertThat(iter.nextByte(), is(Byte.valueOf((byte) 4))); - assertFalse(iter.hasNext()); - } + assertThat("Did not read the inserted field: " + key, iter, + notNullValue()); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 1))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 2))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 3))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 4))); + assertFalse(iter.hasNext()); + } - result = client.delete(table, id); - assertThat("Delete did not return success (0).", result, is(0)); + result = client.delete(table, id); + assertThat("Delete did not return success (0).", result, is(0)); - read.clear(); - result = client.read(table, id, null, read); - assertThat("Read, after delete, did not return not found (1).", result, - is(1)); - assertThat("Found the deleted fields.", read.size(), is(0)); + read.clear(); + result = client.read(table, id, null, read); + assertThat("Read, after delete, did not return not found (1).", result, + is(1)); + assertThat("Found the deleted fields.", read.size(), is(0)); - result = client.delete(table, id); - assertThat("Delete did not return not found (1).", result, is(1)); - } + result = client.delete(table, id); + assertThat("Delete did not return not found (1).", result, is(1)); + } - /** - * Test method for {@link DB#insert}, {@link DB#read}, and {@link DB#update} - * . - */ - @Test - public void testInsertReadUpdate() { - DB client = getDB(); + /** + * Test method for {@link DB#insert}, {@link DB#read}, and {@link DB#update} . + */ + @Test + public void testInsertReadUpdate() { + DB client = getDB(); - final String table = "test"; - final String id = "update"; + final String table = getClass().getSimpleName(); + final String id = "update"; - HashMap inserted = new HashMap(); - inserted.put("a", new ByteArrayByteIterator(new byte[] { 1, 2, 3, 4 })); - int result = client.insert(table, id, inserted); - assertThat("Insert did not return success (0).", result, is(0)); + HashMap inserted = + new HashMap(); + inserted.put("a", new ByteArrayByteIterator(new byte[] { 1, 2, 3, 4 })); + int result = client.insert(table, id, inserted); + assertThat("Insert did not return success (0).", result, is(0)); - HashMap read = new HashMap(); - Set keys = Collections.singleton("a"); - result = client.read(table, id, keys, read); - assertThat("Read did not return success (0).", result, is(0)); - for (String key : keys) { - ByteIterator iter = read.get(key); + HashMap read = new HashMap(); + Set keys = Collections.singleton("a"); + result = client.read(table, id, keys, read); + assertThat("Read did not return success (0).", result, is(0)); + for (String key : keys) { + ByteIterator iter = read.get(key); - assertThat("Did not read the inserted field: " + key, iter, - notNullValue()); - assertTrue(iter.hasNext()); - assertThat(iter.nextByte(), is(Byte.valueOf((byte) 1))); - assertTrue(iter.hasNext()); - assertThat(iter.nextByte(), is(Byte.valueOf((byte) 2))); - assertTrue(iter.hasNext()); - assertThat(iter.nextByte(), is(Byte.valueOf((byte) 3))); - assertTrue(iter.hasNext()); - assertThat(iter.nextByte(), is(Byte.valueOf((byte) 4))); - assertFalse(iter.hasNext()); - } + assertThat("Did not read the inserted field: " + key, iter, + notNullValue()); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 1))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 2))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 3))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 4))); + assertFalse(iter.hasNext()); + } - HashMap updated = new HashMap(); - updated.put("a", new ByteArrayByteIterator(new byte[] { 5, 6, 7, 8 })); - result = client.update(table, id, updated); - assertThat("Update did not return success (0).", result, is(0)); + HashMap updated = new HashMap(); + updated.put("a", new ByteArrayByteIterator(new byte[] { 5, 6, 7, 8 })); + result = client.update(table, id, updated); + assertThat("Update did not return success (0).", result, is(0)); - read.clear(); - result = client.read(table, id, null, read); - assertThat("Read, after update, did not return success (0).", result, - is(0)); - for (String key : keys) { - ByteIterator iter = read.get(key); + read.clear(); + result = client.read(table, id, null, read); + assertThat("Read, after update, did not return success (0).", result, is(0)); + for (String key : keys) { + ByteIterator iter = read.get(key); - assertThat("Did not read the inserted field: " + key, iter, - notNullValue()); - assertTrue(iter.hasNext()); - assertThat(iter.nextByte(), is(Byte.valueOf((byte) 5))); - assertTrue(iter.hasNext()); - assertThat(iter.nextByte(), is(Byte.valueOf((byte) 6))); - assertTrue(iter.hasNext()); - assertThat(iter.nextByte(), is(Byte.valueOf((byte) 7))); - assertTrue(iter.hasNext()); - assertThat(iter.nextByte(), is(Byte.valueOf((byte) 8))); - assertFalse(iter.hasNext()); - } + assertThat("Did not read the inserted field: " + key, iter, + notNullValue()); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 5))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 6))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 7))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 8))); + assertFalse(iter.hasNext()); } + } - /** - * Test method for {@link DB#scan}. - */ - @Test - public void testScan() { - final DB client = getDB(); + /** + * Test method for {@link DB#scan}. + */ + @Test + public void testScan() { + final DB client = getDB(); - final String table = "test"; + final String table = getClass().getSimpleName(); - // Insert a bunch of documents. - for (int i = 0; i < 100; ++i) { - HashMap inserted = new HashMap(); - inserted.put("a", new ByteArrayByteIterator(new byte[] { - (byte) (i & 0xFF), (byte) (i >> 8 & 0xFF), - (byte) (i >> 16 & 0xFF), (byte) (i >> 24 & 0xFF) })); - int result = client.insert(table, padded(i), inserted); - assertThat("Insert did not return success (0).", result, is(0)); - } + // Insert a bunch of documents. + for (int i = 0; i < 100; ++i) { + HashMap inserted = + new HashMap(); + inserted.put("a", new ByteArrayByteIterator(new byte[] { + (byte) (i & 0xFF), (byte) (i >> 8 & 0xFF), (byte) (i >> 16 & 0xFF), + (byte) (i >> 24 & 0xFF) })); + int result = client.insert(table, padded(i), inserted); + assertThat("Insert did not return success (0).", result, is(0)); + } - Set keys = Collections.singleton("a"); - Vector> results = new Vector>(); - int result = client.scan(table, "00050", 5, null, results); - assertThat("Read did not return success (0).", result, is(0)); - assertThat(results.size(), is(5)); - for (int i = 0; i < 5; ++i) { - HashMap read = results.get(i); - for (String key : keys) { - ByteIterator iter = read.get(key); + Set keys = Collections.singleton("a"); + Vector> results = + new Vector>(); + int result = client.scan(table, "00050", 5, null, results); + assertThat("Read did not return success (0).", result, is(0)); + assertThat(results.size(), is(5)); + for (int i = 0; i < 5; ++i) { + HashMap read = results.get(i); + for (String key : keys) { + ByteIterator iter = read.get(key); - assertThat("Did not read the inserted field: " + key, iter, - notNullValue()); - assertTrue(iter.hasNext()); - assertThat(iter.nextByte(), - is(Byte.valueOf((byte) ((i + 50) & 0xFF)))); - assertTrue(iter.hasNext()); - assertThat(iter.nextByte(), - is(Byte.valueOf((byte) ((i + 50) >> 8 & 0xFF)))); - assertTrue(iter.hasNext()); - assertThat(iter.nextByte(), - is(Byte.valueOf((byte) ((i + 50) >> 16 & 0xFF)))); - assertTrue(iter.hasNext()); - assertThat(iter.nextByte(), - is(Byte.valueOf((byte) ((i + 50) >> 24 & 0xFF)))); - assertFalse(iter.hasNext()); - } - } + assertThat("Did not read the inserted field: " + key, iter, + notNullValue()); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) ((i + 50) & 0xFF)))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), + is(Byte.valueOf((byte) ((i + 50) >> 8 & 0xFF)))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), + is(Byte.valueOf((byte) ((i + 50) >> 16 & 0xFF)))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), + is(Byte.valueOf((byte) ((i + 50) >> 24 & 0xFF)))); + assertFalse(iter.hasNext()); + } } + } - /** - * Gets the test DB. - * - * @return The test DB. - */ - protected abstract DB getDB(); + /** + * Gets the test DB. + * + * @return The test DB. + */ + protected abstract DB getDB(); - /** - * Creates a zero padded integer. - * - * @param i - * The integer to padd. - * @return The padded integer. - */ - private String padded(int i) { - String result = String.valueOf(i); - while (result.length() < 5) { - result = "0" + result; - } - return result; + /** + * Creates a zero padded integer. + * + * @param i + * The integer to padd. + * @return The padded integer. + */ + private String padded(int i) { + String result = String.valueOf(i); + while (result.length() < 5) { + result = "0" + result; } + return result; + } } \ No newline at end of file diff --git a/mongodb/src/test/java/com/yahoo/ycsb/db/AsyncMongoDbClientTest.java b/mongodb/src/test/java/com/yahoo/ycsb/db/AsyncMongoDbClientTest.java index f5ee7b65..11b45b2d 100644 --- a/mongodb/src/test/java/com/yahoo/ycsb/db/AsyncMongoDbClientTest.java +++ b/mongodb/src/test/java/com/yahoo/ycsb/db/AsyncMongoDbClientTest.java @@ -1,77 +1,74 @@ -/* - * Copyright (c) 2014, Yahoo!, Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. +/* + * Copyright (c) 2014, Yahoo!, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. */ package com.yahoo.ycsb.db; import static org.junit.Assume.assumeNoException; import java.util.Properties; import org.junit.After; import org.junit.Before; import com.yahoo.ycsb.DB; /** * AsyncMongoDbClientTest provides runs the basic workload operations. */ public class AsyncMongoDbClientTest extends AbstractDBTestCases { - /** The client to use. */ - private AsyncMongoDbClient myClient = null; + /** The client to use. */ + private AsyncMongoDbClient myClient = null; - /** - * Start a test client. - */ - @Before - public void setUp() { - myClient = new AsyncMongoDbClient(); - myClient.setProperties(new Properties()); - try { - myClient.init(); - } - catch (Exception error) { - assumeNoException(error); - } + /** + * Start a test client. + */ + @Before + public void setUp() { + myClient = new AsyncMongoDbClient(); + myClient.setProperties(new Properties()); + try { + myClient.init(); + } catch (Exception error) { + assumeNoException(error); } + } - /** - * Stops the test client. - */ - @After - public void tearDown() { - try { - myClient.cleanup(); - } - catch (Exception error) { - // Ignore. - } - finally { - myClient = null; - } + /** + * Stops the test client. + */ + @After + public void tearDown() { + try { + myClient.cleanup(); + } catch (Exception error) { + // Ignore. + } finally { + myClient = null; } + } - /** - * {@inheritDoc} - *

- * Overriden to return the {@link AsyncMongoDbClient}. - *

- */ - @Override - protected DB getDB() { - return myClient; - } + /** + * {@inheritDoc} + *

+ * Overriden to return the {@link AsyncMongoDbClient}. + *

+ */ + @Override + protected DB getDB() { + return myClient; + } } diff --git a/mongodb/src/test/java/com/yahoo/ycsb/db/MongoDbClientTest.java b/mongodb/src/test/java/com/yahoo/ycsb/db/MongoDbClientTest.java index 515fb802..47a926e4 100644 --- a/mongodb/src/test/java/com/yahoo/ycsb/db/MongoDbClientTest.java +++ b/mongodb/src/test/java/com/yahoo/ycsb/db/MongoDbClientTest.java @@ -1,77 +1,74 @@ -/* - * Copyright (c) 2014, Yahoo!, Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. +/* + * Copyright (c) 2014, Yahoo!, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. */ package com.yahoo.ycsb.db; import static org.junit.Assume.assumeNoException; import java.util.Properties; import org.junit.After; import org.junit.Before; import com.yahoo.ycsb.DB; /** * MongoDbClientTest provides runs the basic workload operations. */ public class MongoDbClientTest extends AbstractDBTestCases { - /** The client to use. */ - private MongoDbClient myClient = null; + /** The client to use. */ + private MongoDbClient myClient = null; - /** - * Start a test client. - */ - @Before - public void setUp() { - myClient = new MongoDbClient(); - myClient.setProperties(new Properties()); - try { - myClient.init(); - } - catch (Exception error) { - assumeNoException(error); - } + /** + * Start a test client. + */ + @Before + public void setUp() { + myClient = new MongoDbClient(); + myClient.setProperties(new Properties()); + try { + myClient.init(); + } catch (Exception error) { + assumeNoException(error); } + } - /** - * Stops the test client. - */ - @After - public void tearDown() { - try { - myClient.cleanup(); - } - catch (Exception error) { - // Ignore. - } - finally { - myClient = null; - } + /** + * Stops the test client. + */ + @After + public void tearDown() { + try { + myClient.cleanup(); + } catch (Exception error) { + // Ignore. + } finally { + myClient = null; } + } - /** - * {@inheritDoc} - *

- * Overriden to return the {@link MongoDbClient}. - *

- */ - @Override - protected DB getDB() { - return myClient; - } + /** + * {@inheritDoc} + *

+ * Overriden to return the {@link MongoDbClient}. + *

+ */ + @Override + protected DB getDB() { + return myClient; + } } diff --git a/mongodb/src/test/java/com/yahoo/ycsb/db/OptionsSupportTest.java b/mongodb/src/test/java/com/yahoo/ycsb/db/OptionsSupportTest.java index a1b9de65..be7f52fe 100644 --- a/mongodb/src/test/java/com/yahoo/ycsb/db/OptionsSupportTest.java +++ b/mongodb/src/test/java/com/yahoo/ycsb/db/OptionsSupportTest.java @@ -1,193 +1,184 @@ /* - * #%L - * OptionsSupport.java - mongodb-binding - Yahoo!, Inc. - * %% - * Copyright (C) 2015 Yahoo!, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * Copyright (c) 2014, Yahoo!, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. */ - package com.yahoo.ycsb.db; import static com.yahoo.ycsb.db.OptionsSupport.updateUrl; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertThat; import java.util.Properties; import org.junit.Test; /** * OptionsSupportTest provides tests for the OptionsSupport class. - * + * * @author rjm */ public class OptionsSupportTest { - /** - * Test method for {@link OptionsSupport#updateUrl(String, Properties)} for - * {@code mongodb.maxconnections}. - */ - @Test - public void testUpdateUrlMaxConnections() { - assertThat( - updateUrl("mongodb://locahost:27017/", - props("mongodb.maxconnections", "1234")), - is("mongodb://locahost:27017/?maxPoolSize=1234")); - assertThat( - updateUrl("mongodb://locahost:27017/?foo=bar", - props("mongodb.maxconnections", "1234")), - is("mongodb://locahost:27017/?foo=bar&maxPoolSize=1234")); - assertThat( - updateUrl("mongodb://locahost:27017/?maxPoolSize=1", - props("mongodb.maxconnections", "1234")), - is("mongodb://locahost:27017/?maxPoolSize=1")); - assertThat( - updateUrl("mongodb://locahost:27017/?foo=bar", - props("foo", "1234")), - is("mongodb://locahost:27017/?foo=bar")); - } - - /** - * Test method for {@link OptionsSupport#updateUrl(String, Properties)} for - * {@code mongodb.threadsAllowedToBlockForConnectionMultiplier}. - */ - @Test - public void testUpdateUrlWaitQueueMultiple() { - assertThat( - updateUrl( - "mongodb://locahost:27017/", - props("mongodb.threadsAllowedToBlockForConnectionMultiplier", - "1234")), - is("mongodb://locahost:27017/?waitQueueMultiple=1234")); - assertThat( - updateUrl( - "mongodb://locahost:27017/?foo=bar", - props("mongodb.threadsAllowedToBlockForConnectionMultiplier", - "1234")), - is("mongodb://locahost:27017/?foo=bar&waitQueueMultiple=1234")); - assertThat( - updateUrl( - "mongodb://locahost:27017/?waitQueueMultiple=1", - props("mongodb.threadsAllowedToBlockForConnectionMultiplier", - "1234")), - is("mongodb://locahost:27017/?waitQueueMultiple=1")); - assertThat( - updateUrl("mongodb://locahost:27017/?foo=bar", - props("foo", "1234")), - is("mongodb://locahost:27017/?foo=bar")); - } - - /** - * Test method for {@link OptionsSupport#updateUrl(String, Properties)} for - * {@code mongodb.threadsAllowedToBlockForConnectionMultiplier}. - */ - @Test - public void testUpdateUrlWriteConcern() { - assertThat( - updateUrl("mongodb://locahost:27017/", - props("mongodb.writeConcern", "errors_ignored")), - is("mongodb://locahost:27017/?w=0")); - assertThat( - updateUrl("mongodb://locahost:27017/?foo=bar", - props("mongodb.writeConcern", "unacknowledged")), - is("mongodb://locahost:27017/?foo=bar&w=0")); - assertThat( - updateUrl("mongodb://locahost:27017/?foo=bar", - props("mongodb.writeConcern", "acknowledged")), - is("mongodb://locahost:27017/?foo=bar&w=1")); - assertThat( - updateUrl("mongodb://locahost:27017/?foo=bar", - props("mongodb.writeConcern", "journaled")), - is("mongodb://locahost:27017/?foo=bar&journal=true&j=true")); - assertThat( - updateUrl("mongodb://locahost:27017/?foo=bar", - props("mongodb.writeConcern", "replica_acknowledged")), - is("mongodb://locahost:27017/?foo=bar&w=2")); - assertThat( - updateUrl("mongodb://locahost:27017/?foo=bar", - props("mongodb.writeConcern", "majority")), - is("mongodb://locahost:27017/?foo=bar&w=majority")); - - // w already exists. - assertThat( - updateUrl("mongodb://locahost:27017/?w=1", - props("mongodb.writeConcern", "acknowledged")), - is("mongodb://locahost:27017/?w=1")); - - // Unknown options - assertThat( - updateUrl("mongodb://locahost:27017/?foo=bar", - props("foo", "1234")), - is("mongodb://locahost:27017/?foo=bar")); - } - - /** - * Test method for {@link OptionsSupport#updateUrl(String, Properties)} for - * {@code mongodb.threadsAllowedToBlockForConnectionMultiplier}. - */ - @Test - public void testUpdateUrlReadPreference() { - assertThat( - updateUrl("mongodb://locahost:27017/", - props("mongodb.readPreference", "primary")), - is("mongodb://locahost:27017/?readPreference=primary")); - assertThat( - updateUrl("mongodb://locahost:27017/?foo=bar", - props("mongodb.readPreference", "primary_preferred")), - is("mongodb://locahost:27017/?foo=bar&readPreference=primaryPreferred")); - assertThat( - updateUrl("mongodb://locahost:27017/?foo=bar", - props("mongodb.readPreference", "secondary")), - is("mongodb://locahost:27017/?foo=bar&readPreference=secondary")); - assertThat( - updateUrl("mongodb://locahost:27017/?foo=bar", - props("mongodb.readPreference", "secondary_preferred")), - is("mongodb://locahost:27017/?foo=bar&readPreference=secondaryPreferred")); - assertThat( - updateUrl("mongodb://locahost:27017/?foo=bar", - props("mongodb.readPreference", "nearest")), - is("mongodb://locahost:27017/?foo=bar&readPreference=nearest")); - - // readPreference already exists. - assertThat( - updateUrl("mongodb://locahost:27017/?readPreference=primary", - props("mongodb.readPreference", "secondary")), - is("mongodb://locahost:27017/?readPreference=primary")); - - // Unknown options - assertThat( - updateUrl("mongodb://locahost:27017/?foo=bar", - props("foo", "1234")), - is("mongodb://locahost:27017/?foo=bar")); - } - - /** - * Factory method for a {@link Properties} object. - * - * @param key - * The key for the property to set. - * @param value - * The value for the property to set. - * @return The {@link Properties} with the property added. - */ - private Properties props(String key, String value) { - Properties props = new Properties(); - - props.setProperty(key, value); - - return props; - } + /** + * Test method for {@link OptionsSupport#updateUrl(String, Properties)} for + * {@code mongodb.maxconnections}. + */ + @Test + public void testUpdateUrlMaxConnections() { + assertThat( + updateUrl("mongodb://locahost:27017/", + props("mongodb.maxconnections", "1234")), + is("mongodb://locahost:27017/?maxPoolSize=1234")); + assertThat( + updateUrl("mongodb://locahost:27017/?foo=bar", + props("mongodb.maxconnections", "1234")), + is("mongodb://locahost:27017/?foo=bar&maxPoolSize=1234")); + assertThat( + updateUrl("mongodb://locahost:27017/?maxPoolSize=1", + props("mongodb.maxconnections", "1234")), + is("mongodb://locahost:27017/?maxPoolSize=1")); + assertThat( + updateUrl("mongodb://locahost:27017/?foo=bar", props("foo", "1234")), + is("mongodb://locahost:27017/?foo=bar")); + } + + /** + * Test method for {@link OptionsSupport#updateUrl(String, Properties)} for + * {@code mongodb.threadsAllowedToBlockForConnectionMultiplier}. + */ + @Test + public void testUpdateUrlWaitQueueMultiple() { + assertThat( + updateUrl( + "mongodb://locahost:27017/", + props("mongodb.threadsAllowedToBlockForConnectionMultiplier", + "1234")), + is("mongodb://locahost:27017/?waitQueueMultiple=1234")); + assertThat( + updateUrl( + "mongodb://locahost:27017/?foo=bar", + props("mongodb.threadsAllowedToBlockForConnectionMultiplier", + "1234")), + is("mongodb://locahost:27017/?foo=bar&waitQueueMultiple=1234")); + assertThat( + updateUrl( + "mongodb://locahost:27017/?waitQueueMultiple=1", + props("mongodb.threadsAllowedToBlockForConnectionMultiplier", + "1234")), is("mongodb://locahost:27017/?waitQueueMultiple=1")); + assertThat( + updateUrl("mongodb://locahost:27017/?foo=bar", props("foo", "1234")), + is("mongodb://locahost:27017/?foo=bar")); + } + + /** + * Test method for {@link OptionsSupport#updateUrl(String, Properties)} for + * {@code mongodb.threadsAllowedToBlockForConnectionMultiplier}. + */ + @Test + public void testUpdateUrlWriteConcern() { + assertThat( + updateUrl("mongodb://locahost:27017/", + props("mongodb.writeConcern", "errors_ignored")), + is("mongodb://locahost:27017/?w=0")); + assertThat( + updateUrl("mongodb://locahost:27017/?foo=bar", + props("mongodb.writeConcern", "unacknowledged")), + is("mongodb://locahost:27017/?foo=bar&w=0")); + assertThat( + updateUrl("mongodb://locahost:27017/?foo=bar", + props("mongodb.writeConcern", "acknowledged")), + is("mongodb://locahost:27017/?foo=bar&w=1")); + assertThat( + updateUrl("mongodb://locahost:27017/?foo=bar", + props("mongodb.writeConcern", "journaled")), + is("mongodb://locahost:27017/?foo=bar&journal=true&j=true")); + assertThat( + updateUrl("mongodb://locahost:27017/?foo=bar", + props("mongodb.writeConcern", "replica_acknowledged")), + is("mongodb://locahost:27017/?foo=bar&w=2")); + assertThat( + updateUrl("mongodb://locahost:27017/?foo=bar", + props("mongodb.writeConcern", "majority")), + is("mongodb://locahost:27017/?foo=bar&w=majority")); + + // w already exists. + assertThat( + updateUrl("mongodb://locahost:27017/?w=1", + props("mongodb.writeConcern", "acknowledged")), + is("mongodb://locahost:27017/?w=1")); + + // Unknown options + assertThat( + updateUrl("mongodb://locahost:27017/?foo=bar", props("foo", "1234")), + is("mongodb://locahost:27017/?foo=bar")); + } + + /** + * Test method for {@link OptionsSupport#updateUrl(String, Properties)} for + * {@code mongodb.threadsAllowedToBlockForConnectionMultiplier}. + */ + @Test + public void testUpdateUrlReadPreference() { + assertThat( + updateUrl("mongodb://locahost:27017/", + props("mongodb.readPreference", "primary")), + is("mongodb://locahost:27017/?readPreference=primary")); + assertThat( + updateUrl("mongodb://locahost:27017/?foo=bar", + props("mongodb.readPreference", "primary_preferred")), + is("mongodb://locahost:27017/?foo=bar&readPreference=primaryPreferred")); + assertThat( + updateUrl("mongodb://locahost:27017/?foo=bar", + props("mongodb.readPreference", "secondary")), + is("mongodb://locahost:27017/?foo=bar&readPreference=secondary")); + assertThat( + updateUrl("mongodb://locahost:27017/?foo=bar", + props("mongodb.readPreference", "secondary_preferred")), + is("mongodb://locahost:27017/?foo=bar&readPreference=secondaryPreferred")); + assertThat( + updateUrl("mongodb://locahost:27017/?foo=bar", + props("mongodb.readPreference", "nearest")), + is("mongodb://locahost:27017/?foo=bar&readPreference=nearest")); + + // readPreference already exists. + assertThat( + updateUrl("mongodb://locahost:27017/?readPreference=primary", + props("mongodb.readPreference", "secondary")), + is("mongodb://locahost:27017/?readPreference=primary")); + + // Unknown options + assertThat( + updateUrl("mongodb://locahost:27017/?foo=bar", props("foo", "1234")), + is("mongodb://locahost:27017/?foo=bar")); + } + + /** + * Factory method for a {@link Properties} object. + * + * @param key + * The key for the property to set. + * @param value + * The value for the property to set. + * @return The {@link Properties} with the property added. + */ + private Properties props(String key, String value) { + Properties props = new Properties(); + + props.setProperty(key, value); + + return props; + } }