diff --git a/bin/bindings.properties b/bin/bindings.properties index 063a90cb..f6cd9fef 100644 --- a/bin/bindings.properties +++ b/bin/bindings.properties @@ -1,81 +1,82 @@ # # Copyright (c) 2012 - 2016 YCSB contributors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # #DATABASE BINDINGS # # Available bindings should be listed here in the form of # name:class # # - the name must start in column 0. # - the name is also the directory where the class can be found. # - if the directory contains multiple versions with different classes, # use a dash with the version. (e.g. cassandra-7, cassandra-cql) # accumulo:com.yahoo.ycsb.db.accumulo.AccumuloClient accumulo1.6:com.yahoo.ycsb.db.accumulo.AccumuloClient accumulo1.7:com.yahoo.ycsb.db.accumulo.AccumuloClient accumulo1.8:com.yahoo.ycsb.db.accumulo.AccumuloClient aerospike:com.yahoo.ycsb.db.AerospikeClient asynchbase:com.yahoo.ycsb.db.AsyncHBaseClient arangodb:com.yahoo.ycsb.db.arangodb.ArangoDBClient arangodb3:com.yahoo.ycsb.db.arangodb.ArangoDBClient azuredocumentdb:com.yahoo.ycsb.db.azuredocumentdb.AzureDocumentDBClient azuretablestorage:com.yahoo.ycsb.db.azuretablestorage.AzureClient basic:com.yahoo.ycsb.BasicDB basicts:com.yahoo.ycsb.BasicTSDB cassandra-cql:com.yahoo.ycsb.db.CassandraCQLClient cassandra2-cql:com.yahoo.ycsb.db.CassandraCQLClient cloudspanner:com.yahoo.ycsb.db.cloudspanner.CloudSpannerClient couchbase:com.yahoo.ycsb.db.CouchbaseClient couchbase2:com.yahoo.ycsb.db.couchbase2.Couchbase2Client dynamodb:com.yahoo.ycsb.db.DynamoDBClient elasticsearch:com.yahoo.ycsb.db.ElasticsearchClient elasticsearch5:com.yahoo.ycsb.db.elasticsearch5.ElasticsearchClient elasticsearch5-rest:com.yahoo.ycsb.db.elasticsearch5.ElasticsearchRestClient foundationdb:com.yahoo.ycsb.db.foundationdb.FoundationDBClient geode:com.yahoo.ycsb.db.GeodeClient googlebigtable:com.yahoo.ycsb.db.GoogleBigtableClient googledatastore:com.yahoo.ycsb.db.GoogleDatastoreClient hbase098:com.yahoo.ycsb.db.HBaseClient hbase10:com.yahoo.ycsb.db.HBaseClient10 hbase12:com.yahoo.ycsb.db.hbase12.HBaseClient12 hbase14:com.yahoo.ycsb.db.hbase14.HBaseClient14 hbase20:com.yahoo.ycsb.db.hbase14.HBaseClient20 hypertable:com.yahoo.ycsb.db.HypertableClient ignite:com.yahoo.ycsb.db.ignite.IgniteClient ignite-sql:com.yahoo.ycsb.db.ignite.IgniteSqlClient infinispan-cs:com.yahoo.ycsb.db.InfinispanRemoteClient infinispan:com.yahoo.ycsb.db.InfinispanClient jdbc:com.yahoo.ycsb.db.JdbcDBClient kudu:com.yahoo.ycsb.db.KuduYCSBClient +leveldb:com.yahoo.ycsb.db.leveldb.LevelDBClient memcached:com.yahoo.ycsb.db.MemcachedClient mongodb:com.yahoo.ycsb.db.MongoDbClient mongodb-async:com.yahoo.ycsb.db.AsyncMongoDbClient mongodb-asyncnative:com.yahoo.ycsb.db.NativeAsyncMongoDbClient mysql:com.yahoo.ycsb.db.JAsyncMySQLDBClient nosqldb:com.yahoo.ycsb.db.NoSqlDbClient orientdb:com.yahoo.ycsb.db.OrientDBClient rados:com.yahoo.ycsb.db.RadosClient redis:com.yahoo.ycsb.db.RedisClient rest:com.yahoo.ycsb.webservice.rest.RestClient riak:com.yahoo.ycsb.db.riak.RiakKVClient rocksdb:com.yahoo.ycsb.db.rocksdb.RocksDBClient s3:com.yahoo.ycsb.db.S3Client solr:com.yahoo.ycsb.db.solr.SolrClient solr6:com.yahoo.ycsb.db.solr6.SolrClient tarantool:com.yahoo.ycsb.db.TarantoolClient diff --git a/bin/ycsb b/bin/ycsb index d77d8e8b..40ed0657 100755 --- a/bin/ycsb +++ b/bin/ycsb @@ -1,344 +1,345 @@ #!/usr/bin/env python # # Copyright (c) 2012 - 2015 YCSB contributors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # import errno import fnmatch import io import os import shlex import sys import subprocess try: mod = __import__('argparse') import argparse except ImportError: print >> sys.stderr, '[ERROR] argparse not found. Try installing it via "pip".' exit(1) BASE_URL = "https://github.com/brianfrankcooper/YCSB/tree/master/" COMMANDS = { "shell" : { "command" : "", "description" : "Interactive mode", "main" : "com.yahoo.ycsb.CommandLine", }, "load" : { "command" : "-load", "description" : "Execute the load phase", "main" : "com.yahoo.ycsb.Client", }, "run" : { "command" : "-t", "description" : "Execute the transaction phase", "main" : "com.yahoo.ycsb.Client", }, } DATABASES = { "accumulo" : "com.yahoo.ycsb.db.accumulo.AccumuloClient", "accumulo1.6" : "com.yahoo.ycsb.db.accumulo.AccumuloClient", "accumulo1.7" : "com.yahoo.ycsb.db.accumulo.AccumuloClient", "accumulo1.8" : "com.yahoo.ycsb.db.accumulo.AccumuloClient", "aerospike" : "com.yahoo.ycsb.db.AerospikeClient", "arangodb" : "com.yahoo.ycsb.db.arangodb.ArangoDBClient", "arangodb3" : "com.yahoo.ycsb.db.arangodb.ArangoDBClient", "asynchbase" : "com.yahoo.ycsb.db.AsyncHBaseClient", "azuredocumentdb" : "com.yahoo.ycsb.db.azuredocumentdb.AzureDocumentDBClient", "azuretablestorage" : "com.yahoo.ycsb.db.azuretablestorage.AzureClient", "basic" : "com.yahoo.ycsb.BasicDB", "basicts" : "com.yahoo.ycsb.BasicTSDB", "cassandra-cql": "com.yahoo.ycsb.db.CassandraCQLClient", "cassandra2-cql": "com.yahoo.ycsb.db.CassandraCQLClient", "cloudspanner" : "com.yahoo.ycsb.db.cloudspanner.CloudSpannerClient", "couchbase" : "com.yahoo.ycsb.db.CouchbaseClient", "couchbase2" : "com.yahoo.ycsb.db.couchbase2.Couchbase2Client", "dynamodb" : "com.yahoo.ycsb.db.DynamoDBClient", "elasticsearch": "com.yahoo.ycsb.db.ElasticsearchClient", "elasticsearch5": "com.yahoo.ycsb.db.elasticsearch5.ElasticsearchClient", "foundationdb" : "com.yahoo.ycsb.db.foundationdb.FoundationDBClient", "geode" : "com.yahoo.ycsb.db.GeodeClient", "googlebigtable" : "com.yahoo.ycsb.db.GoogleBigtableClient", "googledatastore" : "com.yahoo.ycsb.db.GoogleDatastoreClient", "hbase098" : "com.yahoo.ycsb.db.HBaseClient", "hbase10" : "com.yahoo.ycsb.db.HBaseClient10", "hbase12" : "com.yahoo.ycsb.db.hbase12.HBaseClient12", "hbase14" : "com.yahoo.ycsb.db.hbase14.HBaseClient14", "hbase20" : "com.yahoo.ycsb.db.hbase20.HBaseClient20", "hypertable" : "com.yahoo.ycsb.db.HypertableClient", "ignite" : "com.yahoo.ycsb.db.ignite.IgniteClient", "ignite-sql" : "com.yahoo.ycsb.db.ignite.IgniteSqlClient", "infinispan-cs": "com.yahoo.ycsb.db.InfinispanRemoteClient", "infinispan" : "com.yahoo.ycsb.db.InfinispanClient", "jdbc" : "com.yahoo.ycsb.db.JdbcDBClient", "kudu" : "com.yahoo.ycsb.db.KuduYCSBClient", + "leveldb" : "com.yahoo.ycsb.db.leveldb.LevelDBClient", "memcached" : "com.yahoo.ycsb.db.MemcachedClient", "maprdb" : "com.yahoo.ycsb.db.mapr.MapRDBClient", "maprjsondb" : "com.yahoo.ycsb.db.mapr.MapRJSONDBClient", "mongodb" : "com.yahoo.ycsb.db.MongoDbClient", "mongodb-async": "com.yahoo.ycsb.db.AsyncMongoDbClient", "mongodb-asyncnative": "com.yahoo.ycsb.db.NativeAsyncMongoDbClient", "mysql" : "com.yahoo.ycsb.db.JAsyncMySQLDBClient", "nosqldb" : "com.yahoo.ycsb.db.NoSqlDbClient", "orientdb" : "com.yahoo.ycsb.db.OrientDBClient", "rados" : "com.yahoo.ycsb.db.RadosClient", "redis" : "com.yahoo.ycsb.db.RedisClient", "rest" : "com.yahoo.ycsb.webservice.rest.RestClient", "riak" : "com.yahoo.ycsb.db.riak.RiakKVClient", "rocksdb" : "com.yahoo.ycsb.db.rocksdb.RocksDBClient", "s3" : "com.yahoo.ycsb.db.S3Client", "solr" : "com.yahoo.ycsb.db.solr.SolrClient", "solr6" : "com.yahoo.ycsb.db.solr6.SolrClient", "tarantool" : "com.yahoo.ycsb.db.TarantoolClient", } OPTIONS = { "-P file" : "Specify workload file", "-p key=value" : "Override workload property", "-s" : "Print status to stderr", "-target n" : "Target ops/sec (default: unthrottled)", "-threads n" : "Number of client threads (default: 1)", "-cp path" : "Additional Java classpath entries", "-jvm-args args" : "Additional arguments to the JVM", } def usage(): output = io.BytesIO() print >> output, "%s command database [options]" % sys.argv[0] print >> output, "\nCommands:" for command in sorted(COMMANDS.keys()): print >> output, " %s %s" % (command.ljust(14), COMMANDS[command]["description"]) print >> output, "\nDatabases:" for db in sorted(DATABASES.keys()): print >> output, " %s %s" % (db.ljust(14), BASE_URL + db.split("-")[0]) print >> output, "\nOptions:" for option in sorted(OPTIONS.keys()): print >> output, " %s %s" % (option.ljust(14), OPTIONS[option]) print >> output, """\nWorkload Files: There are various predefined workloads under workloads/ directory. See https://github.com/brianfrankcooper/YCSB/wiki/Core-Properties for the list of workload properties.""" return output.getvalue() # Python 2.6 doesn't have check_output. Add the method as it is in Python 2.7 # Based on https://github.com/python/cpython/blob/2.7/Lib/subprocess.py#L545 def check_output(*popenargs, **kwargs): r"""Run command with arguments and return its output as a byte string. If the exit code was non-zero it raises a CalledProcessError. The CalledProcessError object will have the return code in the returncode attribute and output in the output attribute. The arguments are the same as for the Popen constructor. Example: >>> check_output(["ls", "-l", "/dev/null"]) 'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n' The stdout argument is not allowed as it is used internally. To capture standard error in the result, use stderr=STDOUT. >>> check_output(["/bin/sh", "-c", ... "ls -l non_existent_file ; exit 0"], ... stderr=STDOUT) 'ls: non_existent_file: No such file or directory\n' """ if 'stdout' in kwargs: raise ValueError('stdout argument not allowed, it will be overridden.') process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs) output, unused_err = process.communicate() retcode = process.poll() if retcode: cmd = kwargs.get("args") if cmd is None: cmd = popenargs[0] error = subprocess.CalledProcessError(retcode, cmd) error.output = output raise error return output def debug(message): print >> sys.stderr, "[DEBUG] ", message def warn(message): print >> sys.stderr, "[WARN] ", message def error(message): print >> sys.stderr, "[ERROR] ", message def find_jars(dir, glob='*.jar'): jars = [] for (dirpath, dirnames, filenames) in os.walk(dir): for filename in fnmatch.filter(filenames, glob): jars.append(os.path.join(dirpath, filename)) return jars def get_ycsb_home(): dir = os.path.abspath(os.path.dirname(sys.argv[0])) while "LICENSE.txt" not in os.listdir(dir): dir = os.path.join(dir, os.path.pardir) return os.path.abspath(dir) def is_distribution(): # If there's a top level pom, we're a source checkout. otherwise a dist artifact return "pom.xml" not in os.listdir(get_ycsb_home()) # Run the maven dependency plugin to get the local jar paths. # presumes maven can run, so should only be run on source checkouts # will invoke the 'package' goal for the given binding in order to resolve intra-project deps # presumes maven properly handles system-specific path separators # Given module is full module name eg. 'core' or 'couchbase-binding' def get_classpath_from_maven(module): try: debug("Running 'mvn -pl com.yahoo.ycsb:" + module + " -am package -DskipTests " "dependency:build-classpath -DincludeScope=compile -Dmdep.outputFilterFile=true'") mvn_output = check_output(["mvn", "-pl", "com.yahoo.ycsb:" + module, "-am", "package", "-DskipTests", "dependency:build-classpath", "-DincludeScope=compile", "-Dmdep.outputFilterFile=true"]) # the above outputs a "classpath=/path/tojar:/path/to/other/jar" for each module # the last module will be the datastore binding line = [x for x in mvn_output.splitlines() if x.startswith("classpath=")][-1:] return line[0][len("classpath="):] except subprocess.CalledProcessError, err: error("Attempting to generate a classpath from Maven failed " "with return code '" + str(err.returncode) + "'. The output from " "Maven follows, try running " "'mvn -DskipTests package dependency:build-classpath' on your " "own and correct errors." + os.linesep + os.linesep + "mvn output:" + os.linesep + err.output) sys.exit(err.returncode) def main(): p = argparse.ArgumentParser( usage=usage(), formatter_class=argparse.RawDescriptionHelpFormatter) p.add_argument('-cp', dest='classpath', help="""Additional classpath entries, e.g. '-cp /tmp/hbase-1.0.1.1/conf'. Will be prepended to the YCSB classpath.""") p.add_argument("-jvm-args", default=[], type=shlex.split, help="""Additional arguments to pass to 'java', e.g. '-Xmx4g'""") p.add_argument("command", choices=sorted(COMMANDS), help="""Command to run.""") p.add_argument("database", choices=sorted(DATABASES), help="""Database to test.""") args, remaining = p.parse_known_args() ycsb_home = get_ycsb_home() # Use JAVA_HOME to find java binary if set, otherwise just use PATH. java = "java" java_home = os.getenv("JAVA_HOME") if java_home: java = os.path.join(java_home, "bin", "java") db_classname = DATABASES[args.database] command = COMMANDS[args.command]["command"] main_classname = COMMANDS[args.command]["main"] # Classpath set up binding = args.database.split("-")[0] if binding == "accumulo": warn("The 'accumulo' client has been deprecated in favor of version " "specific bindings. This name still maps to the binding for " "Accumulo 1.6, which is named 'accumulo-1.6'. This alias will " "be removed in a future YCSB release.") binding = "accumulo1.6" if binding == "accumulo1.6": warn("The 'accumulo1.6' client has been deprecated because Accumulo 1.6 " "is EOM. If you are using Accumulo 1.7+ try using the 'accumulo1.7' " "client instead.") if binding == "cassandra2": warn("The 'cassandra2-cql' client has been deprecated. It has been " "renamed to simply 'cassandra-cql'. This alias will be removed" " in the next YCSB release.") binding = "cassandra" if binding == "couchbase": warn("The 'couchbase' client has been deprecated. If you are using " "Couchbase 4.0+ try using the 'couchbase2' client instead.") if binding == "hbase098": warn("The 'hbase098' client has been deprecated because HBase 0.98 " "is EOM. If you are using HBase 1.2+ try using the 'hbase12' " "client instead.") if binding == "hbase10": warn("The 'hbase10' client has been deprecated because HBase 1.0 " "is EOM. If you are using HBase 1.2+ try using the 'hbase12' " "client instead.") if binding == "arangodb3": warn("The 'arangodb3' client has been deprecated. The binding 'arangodb' " "now covers every ArangoDB version. This alias will be removed " "in the next YCSB release.") binding = "arangodb" if is_distribution(): db_dir = os.path.join(ycsb_home, binding + "-binding") # include top-level conf for when we're a binding-specific artifact. # If we add top-level conf to the general artifact, starting here # will allow binding-specific conf to override (because it's prepended) cp = [os.path.join(ycsb_home, "conf")] cp.extend(find_jars(os.path.join(ycsb_home, "lib"))) cp.extend(find_jars(os.path.join(db_dir, "lib"))) else: warn("Running against a source checkout. In order to get our runtime " "dependencies we'll have to invoke Maven. Depending on the state " "of your system, this may take ~30-45 seconds") db_location = "core" if (binding == "basic" or binding == "basicts") else binding project = "core" if (binding == "basic" or binding == "basicts") else binding + "-binding" db_dir = os.path.join(ycsb_home, db_location) # goes first so we can rely on side-effect of package maven_says = get_classpath_from_maven(project) # TODO when we have a version property, skip the glob cp = find_jars(os.path.join(db_dir, "target"), project + "*.jar") # alredy in jar:jar:jar form cp.append(maven_says) cp.insert(0, os.path.join(db_dir, "conf")) classpath = os.pathsep.join(cp) if args.classpath: classpath = os.pathsep.join([args.classpath, classpath]) ycsb_command = ([java] + args.jvm_args + ["-cp", classpath, main_classname, "-db", db_classname] + remaining) if command: ycsb_command.append(command) print >> sys.stderr, " ".join(ycsb_command) try: return subprocess.call(ycsb_command) except OSError as e: if e.errno == errno.ENOENT: error('Command failed. Is java installed and on your PATH?') return 1 else: raise if __name__ == '__main__': sys.exit(main()) diff --git a/distribution/pom.xml b/distribution/pom.xml index 939ba1a4..61b8d085 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -1,302 +1,307 @@ 4.0.0 com.yahoo.ycsb root 0.15.0 ycsb YCSB Release Distribution Builder pom This module creates the release package of the YCSB with all DB library bindings. It is only used by the build process and does not contain any real code of itself. com.yahoo.ycsb core ${project.version} com.yahoo.ycsb accumulo1.6-binding ${project.version} com.yahoo.ycsb accumulo1.7-binding ${project.version} com.yahoo.ycsb accumulo1.8-binding ${project.version} com.yahoo.ycsb aerospike-binding ${project.version} com.yahoo.ycsb arangodb-binding ${project.version} com.yahoo.ycsb asynchbase-binding ${project.version} com.yahoo.ycsb cassandra-binding ${project.version} com.yahoo.ycsb cloudspanner-binding ${project.version} com.yahoo.ycsb couchbase-binding ${project.version} com.yahoo.ycsb couchbase2-binding ${project.version} com.yahoo.ycsb azuredocumentdb-binding ${project.version} com.yahoo.ycsb azuretablestorage-binding ${project.version} com.yahoo.ycsb dynamodb-binding ${project.version} com.yahoo.ycsb elasticsearch-binding ${project.version} com.yahoo.ycsb elasticsearch5-binding ${project.version} com.yahoo.ycsb foundationdb-binding ${project.version} com.yahoo.ycsb geode-binding ${project.version} com.yahoo.ycsb googledatastore-binding ${project.version} com.yahoo.ycsb googlebigtable-binding ${project.version} com.yahoo.ycsb hbase098-binding ${project.version} com.yahoo.ycsb hbase10-binding ${project.version} com.yahoo.ycsb hbase12-binding ${project.version} com.yahoo.ycsb hbase14-binding ${project.version} com.yahoo.ycsb hbase20-binding ${project.version} com.yahoo.ycsb hypertable-binding ${project.version} com.yahoo.ycsb ignite-binding ${project.version} com.yahoo.ycsb infinispan-binding ${project.version} com.yahoo.ycsb jdbc-binding ${project.version} com.yahoo.ycsb kudu-binding ${project.version} + + com.yahoo.ycsb + leveldb-binding + ${project.version} + com.yahoo.ycsb memcached-binding ${project.version} com.yahoo.ycsb maprdb-binding ${project.version} com.yahoo.ycsb maprjsondb-binding ${project.version} com.yahoo.ycsb mongodb-binding ${project.version} com.yahoo.ycsb mysql-binding ${project.version} com.yahoo.ycsb nosqldb-binding ${project.version} com.yahoo.ycsb orientdb-binding ${project.version} com.yahoo.ycsb rados-binding ${project.version} com.yahoo.ycsb redis-binding ${project.version} com.yahoo.ycsb rest-binding ${project.version} com.yahoo.ycsb riak-binding ${project.version} com.yahoo.ycsb rocksdb-binding ${project.version} com.yahoo.ycsb s3-binding ${project.version} com.yahoo.ycsb solr-binding ${project.version} com.yahoo.ycsb solr6-binding ${project.version} com.yahoo.ycsb tarantool-binding ${project.version} org.apache.maven.plugins maven-assembly-plugin ${maven.assembly.version} src/main/assembly/distribution.xml false posix package single diff --git a/leveldb/dependency/org/fusesource/leveldbjni/leveldbjni-linux64/99-master-SNAPSHOT/leveldbjni-linux64-99-master-SNAPSHOT.jar b/leveldb/dependency/org/fusesource/leveldbjni/leveldbjni-linux64/99-master-SNAPSHOT/leveldbjni-linux64-99-master-SNAPSHOT.jar new file mode 100644 index 00000000..bafbf7f4 Binary files /dev/null and b/leveldb/dependency/org/fusesource/leveldbjni/leveldbjni-linux64/99-master-SNAPSHOT/leveldbjni-linux64-99-master-SNAPSHOT.jar differ diff --git a/leveldb/dependency/org/fusesource/leveldbjni/leveldbjni-linux64/99-master-SNAPSHOT/leveldbjni-linux64-99-master-SNAPSHOT.pom b/leveldb/dependency/org/fusesource/leveldbjni/leveldbjni-linux64/99-master-SNAPSHOT/leveldbjni-linux64-99-master-SNAPSHOT.pom new file mode 100644 index 00000000..7b1486cf --- /dev/null +++ b/leveldb/dependency/org/fusesource/leveldbjni/leveldbjni-linux64/99-master-SNAPSHOT/leveldbjni-linux64-99-master-SNAPSHOT.pom @@ -0,0 +1,9 @@ + + + 4.0.0 + org.fusesource.leveldbjni + leveldbjni-linux64 + 99-master-SNAPSHOT + POM was created from install:install-file + diff --git a/leveldb/dependency/org/fusesource/leveldbjni/leveldbjni-linux64/99-master-SNAPSHOT/maven-metadata-local.xml b/leveldb/dependency/org/fusesource/leveldbjni/leveldbjni-linux64/99-master-SNAPSHOT/maven-metadata-local.xml new file mode 100644 index 00000000..701e0ce3 --- /dev/null +++ b/leveldb/dependency/org/fusesource/leveldbjni/leveldbjni-linux64/99-master-SNAPSHOT/maven-metadata-local.xml @@ -0,0 +1,24 @@ + + + org.fusesource.leveldbjni + leveldbjni-linux64 + 99-master-SNAPSHOT + + + true + + 20181017194611 + + + jar + 99-master-SNAPSHOT + 20181017194611 + + + pom + 99-master-SNAPSHOT + 20181017194611 + + + + diff --git a/leveldb/dependency/org/fusesource/leveldbjni/leveldbjni-linux64/maven-metadata-local.xml b/leveldb/dependency/org/fusesource/leveldbjni/leveldbjni-linux64/maven-metadata-local.xml new file mode 100644 index 00000000..1bac5855 --- /dev/null +++ b/leveldb/dependency/org/fusesource/leveldbjni/leveldbjni-linux64/maven-metadata-local.xml @@ -0,0 +1,11 @@ + + + org.fusesource.leveldbjni + leveldbjni-linux64 + + + 99-master-SNAPSHOT + + 20181017194611 + + diff --git a/leveldb/dependency/org/fusesource/leveldbjni/leveldbjni/99-master-SNAPSHOT/leveldbjni-99-master-SNAPSHOT.jar b/leveldb/dependency/org/fusesource/leveldbjni/leveldbjni/99-master-SNAPSHOT/leveldbjni-99-master-SNAPSHOT.jar new file mode 100644 index 00000000..fbdb96a2 Binary files /dev/null and b/leveldb/dependency/org/fusesource/leveldbjni/leveldbjni/99-master-SNAPSHOT/leveldbjni-99-master-SNAPSHOT.jar differ diff --git a/leveldb/dependency/org/fusesource/leveldbjni/leveldbjni/99-master-SNAPSHOT/leveldbjni-99-master-SNAPSHOT.pom b/leveldb/dependency/org/fusesource/leveldbjni/leveldbjni/99-master-SNAPSHOT/leveldbjni-99-master-SNAPSHOT.pom new file mode 100644 index 00000000..5b12596e --- /dev/null +++ b/leveldb/dependency/org/fusesource/leveldbjni/leveldbjni/99-master-SNAPSHOT/leveldbjni-99-master-SNAPSHOT.pom @@ -0,0 +1,9 @@ + + + 4.0.0 + org.fusesource.leveldbjni + leveldbjni + 99-master-SNAPSHOT + POM was created from install:install-file + diff --git a/leveldb/dependency/org/fusesource/leveldbjni/leveldbjni/99-master-SNAPSHOT/maven-metadata-local.xml b/leveldb/dependency/org/fusesource/leveldbjni/leveldbjni/99-master-SNAPSHOT/maven-metadata-local.xml new file mode 100644 index 00000000..b2117c81 --- /dev/null +++ b/leveldb/dependency/org/fusesource/leveldbjni/leveldbjni/99-master-SNAPSHOT/maven-metadata-local.xml @@ -0,0 +1,24 @@ + + + org.fusesource.leveldbjni + leveldbjni + 99-master-SNAPSHOT + + + true + + 20181017194556 + + + jar + 99-master-SNAPSHOT + 20181017194556 + + + pom + 99-master-SNAPSHOT + 20181017194404 + + + + diff --git a/leveldb/dependency/org/fusesource/leveldbjni/leveldbjni/maven-metadata-local.xml b/leveldb/dependency/org/fusesource/leveldbjni/leveldbjni/maven-metadata-local.xml new file mode 100644 index 00000000..fef050b6 --- /dev/null +++ b/leveldb/dependency/org/fusesource/leveldbjni/leveldbjni/maven-metadata-local.xml @@ -0,0 +1,11 @@ + + + org.fusesource.leveldbjni + leveldbjni + + + 99-master-SNAPSHOT + + 20181017194556 + + diff --git a/leveldb/pom.xml b/leveldb/pom.xml new file mode 100644 index 00000000..23adf975 --- /dev/null +++ b/leveldb/pom.xml @@ -0,0 +1,71 @@ + + + 4.0.0 + + com.yahoo.ycsb + binding-parent + 0.15.0 + ../binding-parent + + + leveldb-binding + LevelDB Binding + jar + + + + my-local-repo + file://${project.basedir}/dependency + + + + + + org.iq80.leveldb + leveldb-api + 0.10 + + + org.fusesource.leveldbjni + leveldbjni + 99-master-SNAPSHOT + + + org.fusesource.leveldbjni + leveldbjni-linux64 + 99-master-SNAPSHOT + runtime + + + com.yahoo.ycsb + core + ${project.version} + compile + + + net.jcip + jcip-annotations + 1.0 + + + org.slf4j + slf4j-api + 1.7.25 + + + org.slf4j + slf4j-simple + 1.7.25 + runtime + + + junit + junit + 4.12 + test + + + + diff --git a/leveldb/src/main/java/com/yahoo/ycsb/db/leveldb/LevelDBClient.java b/leveldb/src/main/java/com/yahoo/ycsb/db/leveldb/LevelDBClient.java new file mode 100644 index 00000000..d036b70f --- /dev/null +++ b/leveldb/src/main/java/com/yahoo/ycsb/db/leveldb/LevelDBClient.java @@ -0,0 +1,283 @@ +/* + * Copyright (c) 2018 YCSB contributors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package com.yahoo.ycsb.db.leveldb; + +import com.yahoo.ycsb.AsyncDB; +import com.yahoo.ycsb.ByteArrayByteIterator; +import com.yahoo.ycsb.ByteIterator; +import com.yahoo.ycsb.DBException; +import com.yahoo.ycsb.Status; +import net.jcip.annotations.GuardedBy; +import org.fusesource.leveldbjni.JniDBFactory; +import org.iq80.leveldb.CompressionType; +import org.iq80.leveldb.DBIterator; +import org.iq80.leveldb.Options; +import org.iq80.leveldb.ReadOptions; +import org.iq80.leveldb.WriteBatch; +import org.iq80.leveldb.WriteOptions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.Vector; +import java.util.concurrent.CompletableFuture; + +import static org.fusesource.leveldbjni.JniDBFactory.bytes; + +/** + * LevelDB binding for LevelDB. + * + */ +public class LevelDBClient extends AsyncDB { + + private static final String PROPERTY_LEVELDB_DIR = "leveldb.dir"; + + private static final String PROPERTY_SYNC_WRITES = "leveldb.sync"; + + private static final String PROPERTY_VERIFY_CHECKSUMS = "leveldb.verify_checksums"; + + private static final String PROPERTY_COMPRESSION = "leveldb.compression"; + + private static final String PROPERTY_WRITE_BATCH_SIZE = "leveldb.write_batch_size"; + + private static final Logger LOGGER = LoggerFactory.getLogger(LevelDBClient.class); + + @GuardedBy("LevelDBClient.class") private static ReadOptions readOptions = null; + @GuardedBy("LevelDBClient.class") private static WriteOptions writeOptions = null; + @GuardedBy("LevelDBClient.class") private static org.iq80.leveldb.DB levelDb = null; + @GuardedBy("LevelDBClient.class") private static int references = 0; + @GuardedBy("LevelDBClient.class") private static int batchWriteSize; + private static ThreadLocal batchWriteCount = ThreadLocal.withInitial(() -> 0); + private static ThreadLocal batchWrites; + + @Override + public void init() throws DBException { + synchronized(LevelDBClient.class) { + if (levelDb == null) { + Path levelDbDir = Paths.get(getProperties().getProperty(PROPERTY_LEVELDB_DIR)); + LOGGER.info("LevelDB data dir: " + levelDbDir); + + CompressionType compressionType = CompressionType.valueOf(getProperties().getProperty(PROPERTY_COMPRESSION, + "NONE")); + + Options dbOptions = new Options().createIfMissing(true).compressionType(compressionType); + + try { + levelDb = JniDBFactory.factory.open(levelDbDir.toFile(), dbOptions); + } catch (final IOException e) { + throw new DBException(e); + } + + boolean verifyChecksums = Boolean.parseBoolean(getProperties().getProperty(PROPERTY_VERIFY_CHECKSUMS, + "false")); + readOptions = new ReadOptions().fillCache(true).verifyChecksums(verifyChecksums); + boolean syncWrites = Boolean.parseBoolean(getProperties().getProperty(PROPERTY_SYNC_WRITES, + "false")); + writeOptions = new WriteOptions().sync(syncWrites); + + batchWriteSize = Integer.parseInt(getProperties().getProperty(PROPERTY_WRITE_BATCH_SIZE, "1")); + + batchWrites = ThreadLocal.withInitial(levelDb::createWriteBatch); + } + + references++; + } + } + + @Override + public void cleanup() throws DBException { + super.cleanup(); + + synchronized (LevelDBClient.class) { + try { + if (references == 1) { + levelDb.close(); + } + } catch (final IOException e) { + throw new DBException(e); + } finally { + references--; + } + } + } + + @Override + public CompletableFuture read(final String table, final String key, final Set fields, + final Map result) { + + final byte[] values; + + try { + values = levelDb.get(bytes(key)); + } catch (org.iq80.leveldb.DBException e) { + LOGGER.error("Exception thrown reading from DB: " + e); + return CompletableFuture.completedFuture(Status.ERROR); + } + + if (values == null) { + LOGGER.warn("Record with key " + key + " not found."); + return CompletableFuture.completedFuture(Status.NOT_FOUND); + } + + deserializeValues(values, fields, result); + return CompletableFuture.completedFuture(Status.OK); + } + + @Override + public CompletableFuture scan(final String table, final String startkey, final int recordcount, + final Set fields, final Vector> result) { + try (final DBIterator iterator = levelDb.iterator(readOptions)) { + + iterator.seek(bytes(startkey)); + if (!iterator.hasNext()) { + return CompletableFuture.completedFuture(Status.NOT_FOUND); + } + + int records = 0; + while (records < recordcount) { + if (!iterator.hasNext()) { + LOGGER.warn("Tried retrieving " + recordcount + " records, only found " + (records + 1) + "."); + return CompletableFuture.completedFuture(Status.NOT_FOUND); + } + Map.Entry record = iterator.next(); + final HashMap values = new HashMap<>(); + deserializeValues(record.getValue(), fields, values); + result.add(values); + records++; + } + return CompletableFuture.completedFuture(Status.OK); + + } catch (org.iq80.leveldb.DBException | IOException e) { + LOGGER.error("Exception thrown scanning DB: " + e); + return CompletableFuture.completedFuture(Status.ERROR); + } + } + + @Override + public CompletableFuture update(final String table, final String key, + final Map values) { + + final Map newValues = new HashMap<>(); + return read(table, key, null, newValues).thenApply(r -> { + if (!r.isOk()) { + return r; + } + newValues.putAll(values); + return insert(table, key, newValues).join(); + }); + } + + @Override + public CompletableFuture insert(final String table, final String key, + final Map values) { + try { + if (batchWriteSize > 1) { + batchWriteCount.set(batchWriteCount.get() + 1); + batchWrites.get().put(bytes(key), serializeValues(values)); + if (batchWriteCount.get() < batchWriteSize) { + return CompletableFuture.completedFuture(Status.BATCHED_OK); + } else { + levelDb.write(batchWrites.get(), writeOptions); + batchWriteCount.set(0); + batchWrites.set(levelDb.createWriteBatch()); + return CompletableFuture.completedFuture(Status.OK); + } + } else { + levelDb.put(bytes(key), serializeValues(values)); + return CompletableFuture.completedFuture(Status.OK); + } + } catch (org.iq80.leveldb.DBException | IOException e) { + LOGGER.error("Exception thrown writing to DB: " + e); + return CompletableFuture.completedFuture(Status.ERROR); + } + } + + @Override + public CompletableFuture delete(final String table, final String key) { + try { + levelDb.delete(bytes(key), writeOptions); + return CompletableFuture.completedFuture(Status.OK); + } catch (org.iq80.leveldb.DBException e) { + LOGGER.error("Exception thrown deleting from DB: " + e); + return CompletableFuture.completedFuture(Status.ERROR); + } + } + + private Map deserializeValues(final byte[] values, final Set fields, + final Map result) { + final ByteBuffer buf = ByteBuffer.allocate(4); + + int offset = 0; + while(offset < values.length) { + buf.put(values, offset, 4); + buf.flip(); + final int keyLen = buf.getInt(); + buf.clear(); + offset += 4; + + final String key = new String(values, offset, keyLen); + offset += keyLen; + + buf.put(values, offset, 4); + buf.flip(); + final int valueLen = buf.getInt(); + buf.clear(); + offset += 4; + + if(fields == null || fields.contains(key)) { + result.put(key, new ByteArrayByteIterator(values, offset, valueLen)); + } + + offset += valueLen; + } + + return result; + } + + private byte[] serializeValues(final Map values) throws IOException { + try(final ByteArrayOutputStream baos = new ByteArrayOutputStream()) { + final ByteBuffer buf = ByteBuffer.allocate(4); + + for(final Map.Entry value : values.entrySet()) { + final byte[] keyBytes = bytes(value.getKey()); + final byte[] valueBytes = value.getValue().toArray(); + + buf.putInt(keyBytes.length); + baos.write(buf.array()); + baos.write(keyBytes); + + buf.clear(); + + buf.putInt(valueBytes.length); + baos.write(buf.array()); + baos.write(valueBytes); + + buf.clear(); + } + return baos.toByteArray(); + } + } + +} diff --git a/leveldb/src/main/java/com/yahoo/ycsb/db/leveldb/package-info.java b/leveldb/src/main/java/com/yahoo/ycsb/db/leveldb/package-info.java new file mode 100644 index 00000000..ef4994f5 --- /dev/null +++ b/leveldb/src/main/java/com/yahoo/ycsb/db/leveldb/package-info.java @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2014 - 2016, Yahoo!, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +/** + * The YCSB binding for stores that can be accessed via LevelDB's API. + */ +package com.yahoo.ycsb.db.leveldb; + diff --git a/pom.xml b/pom.xml index da1d3d7c..12fb4c3a 100644 --- a/pom.xml +++ b/pom.xml @@ -1,225 +1,226 @@ 4.0.0 com.yahoo.ycsb root 0.15.0 pom YCSB Root This is the top level project that builds, packages the core and all the DB bindings for YCSB infrastructure. scm:git:git://github.com/brianfrankcooper/YCSB.git master https://github.com/brianfrankcooper/YCSB com.puppycrawl.tools checkstyle 7.7.1 org.jdom jdom 1.1 com.google.collections google-collections 1.0 org.slf4j slf4j-api 1.7.25 2.5.5 2.10 1.6.6 1.7.4 1.9.1 3.1.2 4.4.1 1.7.1 1.8.1 4.0.0 3.0.0 0.24.0-beta 1.4.10 2.3.1 5.5.1 5.2.5 1.2.0 1.3.0 0.98.14-hadoop2 1.0.2 1.2.5 1.4.2 2.0.0 0.9.5.6 2.6.0 7.2.2.Final 1.6.0 1.1.8-mapr-1710 3.8.2 2.0.1 2.1.1 2.2.10 UTF-8 2.9.0 2.0.5 5.11.3 1.10.20 5.5.3 6.4.1 1.6.5 0.8.0 0.81 core binding-parent distribution accumulo1.6 accumulo1.7 accumulo1.8 aerospike arangodb asynchbase azuredocumentdb azuretablestorage cassandra cloudspanner couchbase couchbase2 dynamodb elasticsearch elasticsearch5 foundationdb geode googlebigtable googledatastore hbase098 hbase10 hbase12 hbase14 hbase20 hypertable ignite infinispan jdbc kudu + leveldb maprdb maprjsondb memcached mongodb mysql nosqldb orientdb rados redis rest riak rocksdb s3 solr solr6 tarantool org.apache.maven.plugins maven-checkstyle-plugin 2.16 org.apache.maven.plugins maven-enforcer-plugin 3.0.0-M1 enforce-maven enforce 3.1.0 org.apache.maven.plugins maven-compiler-plugin 3.7.0 1.8 1.8 org.apache.maven.plugins maven-checkstyle-plugin validate validate check checkstyle.xml