[prev in list] [next in list] [prev in thread] [next in thread]
List: forgerock-opendj-dev
Subject: [Opendj-dev] [10846] trunk/scratch/matthew_swift/scratch-matt-be: Fix OPENDJ-1485: Initial PoC backe
From: noreply () forgerock ! org
Date: 2014-06-30 15:40:09
Message-ID: 20140630154009.CB73B41337 () sources ! internal ! forgerock ! com
[Download RAW message or body]
[Attachment #2 (text/html)]
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>[10846] trunk/scratch/matthew_swift/scratch-matt-be: Fix OPENDJ-1485: Initial \
PoC backend based on LevelDB or variant</title> </head>
<body>
<style type="text/css"><!--
#msg dl.meta { border: 1px #006 solid; background: #369; padding: 6px; color: #fff; }
#msg dl.meta dt { float: left; width: 6em; font-weight: bold; }
#msg dt:after { content:':';}
#msg dl, #msg dt, #msg ul, #msg li, #header, #footer, #logmsg { font-family: \
verdana,arial,helvetica,sans-serif; font-size: 10pt; } #msg dl a { font-weight: \
bold} #msg dl a:link { color:#fc3; }
#msg dl a:active { color:#ff0; }
#msg dl a:visited { color:#cc6; }
h3 { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; font-weight: \
bold; } #msg pre { overflow: auto; background: #ffc; border: 1px #fa0 solid; padding: \
6px; } #logmsg { background: #ffc; border: 1px #fa0 solid; padding: 1em 1em 0 1em; }
#logmsg p, #logmsg pre, #logmsg blockquote { margin: 0 0 1em 0; }
#logmsg p, #logmsg li, #logmsg dt, #logmsg dd { line-height: 14pt; }
#logmsg h1, #logmsg h2, #logmsg h3, #logmsg h4, #logmsg h5, #logmsg h6 { margin: .5em \
0; } #logmsg h1:first-child, #logmsg h2:first-child, #logmsg h3:first-child, #logmsg \
h4:first-child, #logmsg h5:first-child, #logmsg h6:first-child { margin-top: 0; } \
#logmsg ul, #logmsg ol { padding: 0; list-style-position: inside; margin: 0 0 0 1em; \
} #logmsg ul { text-indent: -1em; padding-left: 1em; }#logmsg ol { text-indent: \
-1.5em; padding-left: 1.5em; } #logmsg > ul, #logmsg > ol { margin: 0 0 1em 0; }
#logmsg pre { background: #eee; padding: 1em; }
#logmsg blockquote { border: 1px solid #fa0; border-left-width: 10px; padding: 1em \
1em 0 1em; background: white;} #logmsg dl { margin: 0; }
#logmsg dt { font-weight: bold; }
#logmsg dd { margin: 0; padding: 0 0 0.5em 0; }
#logmsg dd:before { content:'\00bb';}
#logmsg table { border-spacing: 0px; border-collapse: collapse; border-top: 4px solid \
#fa0; border-bottom: 1px solid #fa0; background: #fff; } #logmsg table th { \
text-align: left; font-weight: normal; padding: 0.2em 0.5em; border-top: 1px dotted \
#fa0; } #logmsg table td { text-align: right; border-top: 1px dotted #fa0; padding: \
0.2em 0.5em; } #logmsg table thead th { text-align: center; border-bottom: 1px solid \
#fa0; } #logmsg table th.Corner { text-align: left; }
#logmsg hr { border: none 0; border-top: 2px dashed #fa0; height: 1px; }
#header, #footer { color: #fff; background: #636; border: 1px #300 solid; padding: \
6px; } #patch { width: 100%; }
#patch h4 {font-family: \
verdana,arial,helvetica,sans-serif;font-size:10pt;padding:8px;background:#369;color:#fff;margin:0;}
#patch .propset h4, #patch .binary h4 {margin:0;}
#patch pre {padding:0;line-height:1.2em;margin:0;}
#patch .diff {width:100%;background:#eee;padding: 0 0 10px 0;overflow:auto;}
#patch .propset .diff, #patch .binary .diff {padding:10px 0;}
#patch span {display:block;padding:0 10px;}
#patch .modfile, #patch .addfile, #patch .delfile, #patch .propset, #patch .binary, \
#patch .copfile {border:1px solid #ccc;margin:10px 0;} #patch ins \
{background:#dfd;text-decoration:none;display:block;padding:0 10px;} #patch del \
{background:#fdd;text-decoration:none;display:block;padding:0 10px;} #patch .lines, \
.info {color:#888;background:#fff;}
--></style>
<div id="msg">
<dl class="meta">
<dt>Revision</dt> <dd><a \
href="http://sources.forgerock.org/changelog/opendj/?cs=10846">10846</a></dd> \
<dt>Author</dt> <dd>matthew</dd> <dt>Date</dt> <dd>2014-06-30 16:40:09 +0100 (Mon, 30 \
Jun 2014)</dd> </dl>
<h3>Log Message</h3>
<pre>Fix <a href="https://bugster.forgerock.org/jira/browse/OPENDJ-1485">OPENDJ-1485</a>: \
Initial PoC backend based on LevelDB or variant
* PoC based on RocksDB
* uses native library
* does not provide native record locking, e.g. RMW, so this needs to be done by the \
application
* to run:
java -Djava.library.path=lib-`uname -i` -cp \
target/scratch-matt-be-0.0.1-SNAPSHOT.jar:lib/rocksdbjni.jar \
org.opendj.scratch.be.Server 1389 ROCKS 10000</pre>
<h3>Modified Paths</h3>
<ul>
<li><a href="#trunkscratchmatthew_swiftscratchmattbepomxml">trunk/scratch/matthew_swift/scratch-matt-be/pom.xml</a></li>
<li><a href="#trunkscratchmatthew_swiftscratchmattbesrcmainjavaorgopendjscratchbeJEBa \
ckendjava">trunk/scratch/matthew_swift/scratch-matt-be/src/main/java/org/opendj/scratch/be/JEBackend.java</a></li>
<li><a href="#trunkscratchmatthew_swiftscratchmattbesrcmainjavaorgopendjscratchbeMapD \
BBackendjava">trunk/scratch/matthew_swift/scratch-matt-be/src/main/java/org/opendj/scratch/be/MapDBBackend.java</a></li>
<li><a href="#trunkscratchmatthew_swiftscratchmattbesrcmainjavaorgopendjscratchbeOrie \
ntBackendjava">trunk/scratch/matthew_swift/scratch-matt-be/src/main/java/org/opendj/scratch/be/OrientBackend.java</a></li>
<li><a href="#trunkscratchmatthew_swiftscratchmattbesrcmainjavaorgopendjscratchbeServ \
erjava">trunk/scratch/matthew_swift/scratch-matt-be/src/main/java/org/opendj/scratch/be/Server.java</a></li>
</ul>
<h3>Added Paths</h3>
<ul>
<li>trunk/scratch/matthew_swift/scratch-matt-be/lib/</li>
<li><a href="#trunkscratchmatthew_swiftscratchmattbelibrocksdbjnijar">trunk/scratch/matthew_swift/scratch-matt-be/lib/rocksdbjni.jar</a></li>
<li>trunk/scratch/matthew_swift/scratch-matt-be/lib-x86_64/</li>
<li><a href="#trunkscratchmatthew_swiftscratchmattbelibx86_64librocksdbjniso">trunk/scratch/matthew_swift/scratch-matt-be/lib-x86_64/librocksdbjni.so</a></li>
<li><a href="#trunkscratchmatthew_swiftscratchmattbesrcmainjavaorgopendjscratchbeRock \
sDBBackendjava">trunk/scratch/matthew_swift/scratch-matt-be/src/main/java/org/opendj/scratch/be/RocksDBBackend.java</a></li>
</ul>
</div>
<div id="patch">
<h3>Diff</h3>
<a id="trunkscratchmatthew_swiftscratchmattbelibrocksdbjnijar"></a>
<div class="binary"><h4>Added: \
trunk/scratch/matthew_swift/scratch-matt-be/lib/rocksdbjni.jar</h4> <pre \
class="diff"><span> <span class="cx">(Binary files differ)
</span></span></pre></div>
<span class="cx">Property changes on: \
trunk/scratch/matthew_swift/scratch-matt-be/lib/rocksdbjni.jar </span><span \
class="cx">___________________________________________________________________ \
</span><a id="svnmimetype"></a> <div class="addfile"><h4>Added: \
svn:mime-type</h4></div> <a \
id="trunkscratchmatthew_swiftscratchmattbelibx86_64librocksdbjniso"></a> <div \
class="binary"><h4>Added: \
trunk/scratch/matthew_swift/scratch-matt-be/lib-x86_64/librocksdbjni.so</h4> <pre \
class="diff"><span> <span class="cx">(Binary files differ)
</span></span></pre></div>
<span class="cx">Property changes on: \
trunk/scratch/matthew_swift/scratch-matt-be/lib-x86_64/librocksdbjni.so </span><span \
class="cx">___________________________________________________________________ \
</span><a id="svnexecutable"></a> <div class="addfile"><h4>Added: \
svn:executable</h4></div> <a id="svnmimetype"></a>
<div class="addfile"><h4>Added: svn:mime-type</h4></div>
<a id="trunkscratchmatthew_swiftscratchmattbepomxml"></a>
<div class="modfile"><h4>Modified: \
trunk/scratch/matthew_swift/scratch-matt-be/pom.xml (10845 => 10846)</h4> <pre \
class="diff"><span> <span class="info">--- \
trunk/scratch/matthew_swift/scratch-matt-be/pom.xml 2014-06-30 12:49:28 UTC (rev \
10845)
+++ trunk/scratch/matthew_swift/scratch-matt-be/pom.xml 2014-06-30 15:40:09 UTC (rev \
10846) </span><span class="lines">@@ -36,6 +36,13 @@
</span><span class="cx"> <artifactId>mapdb</artifactId>
</span><span class="cx"> <version>1.0.1</version>
</span><span class="cx"> </dependency>
</span><ins>+ <dependency>
+ <groupId>org.rocksdb</groupId>
+ <artifactId>rocksdb</artifactId>
+ <version>0.1</version>
+ <scope>system</scope>
+ <systemPath>${basedir}/lib/rocksdbjni.jar</systemPath>
+ </dependency>
</ins><span class="cx"> </dependencies>
</span><span class="cx">
</span><span class="cx"> <build>
</span></span></pre></div>
<a id="trunkscratchmatthew_swiftscratchmattbesrcmainjavaorgopendjscratchbeJEBackendjava"></a>
<div class="modfile"><h4>Modified: \
trunk/scratch/matthew_swift/scratch-matt-be/src/main/java/org/opendj/scratch/be/JEBackend.java \
(10845 => 10846)</h4> <pre class="diff"><span>
<span class="info">--- \
trunk/scratch/matthew_swift/scratch-matt-be/src/main/java/org/opendj/scratch/be/JEBackend.java 2014-06-30 \
12:49:28 UTC (rev 10845)
+++ trunk/scratch/matthew_swift/scratch-matt-be/src/main/java/org/opendj/scratch/be/JEBackend.java 2014-06-30 \
15:40:09 UTC (rev 10846) </span><span class="lines">@@ -32,7 +32,7 @@
</span><span class="cx"> import com.sleepycat.je.TransactionConfig;
</span><span class="cx">
</span><span class="cx"> public final class JEBackend implements Backend {
</span><del>- private static final File DB_DIR = new File("jeBackend");
</del><ins>+ private static final File DB_DIR = new \
File("target/jeBackend"); </ins><span class="cx">
</span><span class="cx"> private Environment env = null;
</span><span class="cx"> private Database description2id = null;
</span><span class="lines">@@ -144,7 +144,7 @@
</span><span class="cx"> }
</span><span class="cx"> }
</span><span class="cx">
</span><del>- private DatabaseEntry encodeDn(final DN dn) throws \
ErrorResultException { </del><ins>+ private DatabaseEntry encodeDn(final DN dn) {
</ins><span class="cx"> return new \
DatabaseEntry(Util.encodeDn(dn).toByteArray()); </span><span class="cx"> }
</span><span class="cx">
</span></span></pre></div>
<a id="trunkscratchmatthew_swiftscratchmattbesrcmainjavaorgopendjscratchbeMapDBBackendjava"></a>
<div class="modfile"><h4>Modified: \
trunk/scratch/matthew_swift/scratch-matt-be/src/main/java/org/opendj/scratch/be/MapDBBackend.java \
(10845 => 10846)</h4> <pre class="diff"><span>
<span class="info">--- \
trunk/scratch/matthew_swift/scratch-matt-be/src/main/java/org/opendj/scratch/be/MapDBBackend.java 2014-06-30 \
12:49:28 UTC (rev 10845)
+++ trunk/scratch/matthew_swift/scratch-matt-be/src/main/java/org/opendj/scratch/be/MapDBBackend.java 2014-06-30 \
15:40:09 UTC (rev 10846) </span><span class="lines">@@ -135,7 +135,7 @@
</span><span class="cx">
</span><span class="cx"> private static final \
BTreeKeySerializer<ByteString> BYTE_STRING_KEY_SERIALIZER = </span><span \
class="cx"> new ByteStringKeySerializer(); </span><del>- private \
static final File DB_DIR = new File("mapBackend"); </del><ins>+ private \
static final File DB_DIR = new File("target/mapBackend"); </ins><span \
class="cx"> private static final File DB_FILE = new File(DB_DIR, "db"); \
</span><span class="cx"> </span><span class="cx"> /**
</span></span></pre></div>
<a id="trunkscratchmatthew_swiftscratchmattbesrcmainjavaorgopendjscratchbeOrientBackendjava"></a>
<div class="modfile"><h4>Modified: \
trunk/scratch/matthew_swift/scratch-matt-be/src/main/java/org/opendj/scratch/be/OrientBackend.java \
(10845 => 10846)</h4> <pre class="diff"><span>
<span class="info">--- \
trunk/scratch/matthew_swift/scratch-matt-be/src/main/java/org/opendj/scratch/be/OrientBackend.java 2014-06-30 \
12:49:28 UTC (rev 10845)
+++ trunk/scratch/matthew_swift/scratch-matt-be/src/main/java/org/opendj/scratch/be/OrientBackend.java 2014-06-30 \
15:40:09 UTC (rev 10846) </span><span class="lines">@@ -40,7 +40,7 @@
</span><span class="cx">
</span><span class="cx"> }
</span><span class="cx">
</span><del>- private static final File DB_DIR = new \
File("orientBackend"); </del><ins>+ private static final File DB_DIR = \
new File("target/orientBackend"); </ins><span class="cx"> private \
static final String DB_URL = "plocal:" + DB_DIR.getAbsolutePath(); \
</span><span class="cx"> </span><span class="cx"> private final \
Queue<ODatabaseDocumentTx> activeDbConnections = </span></span></pre></div>
<a id="trunkscratchmatthew_swiftscratchmattbesrcmainjavaorgopendjscratchbeRocksDBBackendjava"></a>
<div class="addfile"><h4>Added: \
trunk/scratch/matthew_swift/scratch-matt-be/src/main/java/org/opendj/scratch/be/RocksDBBackend.java \
(0 => 10846)</h4> <pre class="diff"><span>
<span class="info">--- \
trunk/scratch/matthew_swift/scratch-matt-be/src/main/java/org/opendj/scratch/be/RocksDBBackend.java \
(rev 0)
+++ trunk/scratch/matthew_swift/scratch-matt-be/src/main/java/org/opendj/scratch/be/RocksDBBackend.java 2014-06-30 \
15:40:09 UTC (rev 10846) </span><span class="lines">@@ -0,0 +1,158 @@
</span><ins>+package org.opendj.scratch.be;
+
+import static org.opendj.scratch.be.Util.*;
+
+import java.io.File;
+import java.util.Map;
+
+import org.forgerock.opendj.ldap.ByteString;
+import org.forgerock.opendj.ldap.ByteStringBuilder;
+import org.forgerock.opendj.ldap.DN;
+import org.forgerock.opendj.ldap.Entries;
+import org.forgerock.opendj.ldap.Entry;
+import org.forgerock.opendj.ldap.ErrorResultException;
+import org.forgerock.opendj.ldap.requests.ModifyRequest;
+import org.forgerock.opendj.ldif.EntryReader;
+import org.rocksdb.Options;
+import org.rocksdb.RocksDB;
+import org.rocksdb.WriteBatch;
+import org.rocksdb.WriteOptions;
+
+public final class RocksDBBackend implements Backend {
+ private static final File DB_DIR = new File("target/rocksBackend");
+ private static final byte PREFIX_DESCRIPTION2ID = 2;
+ private static final byte PREFIX_DN2ID = 1;
+ private static final byte PREFIX_ID2ENTRY = 0;
+
+ private RocksDB db;
+ private Options dbOptions;
+
+ @Override
+ public void close() {
+ if (db != null) {
+ db.close();
+ db = null;
+ dbOptions.dispose();
+ dbOptions = null;
+ }
+ }
+
+ @Override
+ public void importEntries(final EntryReader entries, final Map<String, \
String> options) + throws Exception {
+ clearAndCreateDbDir(DB_DIR);
+
+ RocksDB.loadLibrary();
+ dbOptions = new Options().setCreateIfMissing(true);
+ db = RocksDB.open(dbOptions, DB_DIR.toString());
+
+ try {
+ final ByteStringBuilder builder = new ByteStringBuilder();
+ for (int nextEntryId = 0; entries.hasNext(); nextEntryId++) {
+ builder.clear();
+ builder.append(PREFIX_ID2ENTRY);
+ builder.append(nextEntryId);
+ final byte[] id = builder.toByteArray();
+
+ final Entry entry = entries.readEntry();
+ builder.clear();
+ builder.append(encodeEntry(entry));
+ db.put(id, builder.toByteArray());
+
+ builder.clear();
+ builder.append(PREFIX_DN2ID);
+ builder.append(encodeDn(entry.getName()));
+ db.put(builder.toByteArray(), id);
+
+ final ByteString encodedDescription = encodeDescription(entry);
+ if (encodedDescription != null) {
+ builder.clear();
+ builder.append(PREFIX_DESCRIPTION2ID);
+ builder.append(encodedDescription);
+ db.put(builder.toByteArray(), id);
+ }
+ }
+ } finally {
+ close();
+ }
+ }
+
+ @Override
+ public void initialize(final Map<String, String> options) throws Exception \
{ + RocksDB.loadLibrary();
+ dbOptions = new Options().setCreateIfMissing(true);
+ db = RocksDB.open(dbOptions, DB_DIR.toString());
+ }
+
+ @Override
+ public void modifyEntry(final ModifyRequest request) throws ErrorResultException \
{ + final WriteBatch batchUpdate = new WriteBatch();
+ final WriteOptions writeOptions = new WriteOptions();
+ try {
+ // Read entry and apply updates.
+ final byte[] entryId = db.get(encodeDnKey(request.getName()));
+ final Entry entry = decodeEntry(db.get(entryId));
+ final ByteString oldDescriptionKey = encodeDescription(entry);
+ Entries.modifyEntry(entry, request);
+ final ByteString newDescriptionKey = encodeDescription(entry);
+ // Update description index.
+ final int comparison = oldDescriptionKey.compareTo(newDescriptionKey);
+ if (comparison != 0) {
+ final byte[] oldKey = toDescriptionKey(oldDescriptionKey);
+ final byte[] newKey = toDescriptionKey(newDescriptionKey);
+ if (comparison < 0) {
+ batchUpdate.remove(oldKey);
+ batchUpdate.put(newKey, entryId);
+ } else {
+ batchUpdate.put(newKey, entryId);
+ batchUpdate.remove(oldKey);
+ }
+ }
+ // Update id2entry index.
+ batchUpdate.put(entryId, encodeEntry(entry));
+ db.write(writeOptions, batchUpdate);
+ } catch (final Exception e) {
+ throw internalError(e);
+ } finally {
+ writeOptions.dispose();
+ batchUpdate.dispose();
+ }
+ }
+
+ @Override
+ public Entry readEntryByDescription(final ByteString description) throws \
ErrorResultException { + try {
+ return decodeEntry(db.get(db.get(toDescriptionKey(encodeDescription(description)))));
+ } catch (final Exception e) {
+ throw internalError(e);
+ }
+ }
+
+ @Override
+ public Entry readEntryByDN(final DN name) throws ErrorResultException {
+ try {
+ return decodeEntry(db.get(db.get(encodeDnKey(name))));
+ } catch (final Exception e) {
+ throw internalError(e);
+ }
+ }
+
+ private byte[] encodeDnKey(final DN name) {
+ return toDnKey(encodeDn(name));
+ }
+
+ private byte[] toDescriptionKey(final ByteString encodedDescription) {
+ final byte[] key = new byte[encodedDescription.length() + 1];
+ key[0] = PREFIX_DESCRIPTION2ID;
+ encodedDescription.copyTo(key, 1);
+ return key;
+ }
+
+ private byte[] toDnKey(final ByteString encodedDn) {
+ final byte[] key = new byte[encodedDn.length() + 1];
+ key[0] = PREFIX_DN2ID;
+ encodedDn.copyTo(key, 1);
+ return key;
+ }
+
+}
</ins><span class="cx">Property changes on: \
trunk/scratch/matthew_swift/scratch-matt-be/src/main/java/org/opendj/scratch/be/RocksDBBackend.java
</span><span class="cx">___________________________________________________________________
</span></span></pre></div>
<a id="svneolstyle"></a>
<div class="addfile"><h4>Added: svn:eol-style</h4></div>
<a id="trunkscratchmatthew_swiftscratchmattbesrcmainjavaorgopendjscratchbeServerjava"></a>
<div class="modfile"><h4>Modified: \
trunk/scratch/matthew_swift/scratch-matt-be/src/main/java/org/opendj/scratch/be/Server.java \
(10845 => 10846)</h4> <pre class="diff"><span>
<span class="info">--- \
trunk/scratch/matthew_swift/scratch-matt-be/src/main/java/org/opendj/scratch/be/Server.java 2014-06-30 \
12:49:28 UTC (rev 10845)
+++ trunk/scratch/matthew_swift/scratch-matt-be/src/main/java/org/opendj/scratch/be/Server.java 2014-06-30 \
15:40:09 UTC (rev 10846) </span><span class="lines">@@ -161,7 +161,8 @@
</span><span class="cx"> }
</span><span class="cx">
</span><span class="cx"> private static enum BackendType {
</span><del>- JE(JEBackend.class), MAP(MapDBBackend.class), \
ORIENT(OrientBackend.class); </del><ins>+ JE(JEBackend.class), \
MAP(MapDBBackend.class), ORIENT(OrientBackend.class), ROCKS( + \
RocksDBBackend.class); </ins><span class="cx">
</span><span class="cx"> private final Class<? extends Backend> \
backendClass; </span><span class="cx">
</span></span></pre>
</div>
</div>
<div id="footer">Copyright (c) by ForgeRock. All rights reserved.</div>
</body>
</html>
_______________________________________________
OpenDJ-dev mailing list
OpenDJ-dev@forgerock.org
https://lists.forgerock.org/mailman/listinfo/opendj-dev
[prev in list] [next in list] [prev in thread] [next in thread]
Configure |
About |
News |
Add a list |
Sponsored by KoreLogic