[prev in list] [next in list] [prev in thread] [next in thread]
List: kde-commits
Subject: playground/base/strigi/src
From: Jos van den Oever <jos () vandenoever ! info>
Date: 2006-09-07 16:06:08
Message-ID: 1157645168.034221.3339.nullmailer () svn ! kde ! org
[Download RAW message or body]
SVN commit 581810 by vandenoever:
Add more tests and enable quick erasing of all index data.
M +1 -1 daemon/daemon.cpp
M +4 -2 daemon/indexscheduler.cpp
M +1 -0 dummyindexer/dummyindexwriter.h
M +60 -34 estraierindexer/estraierindexmanager.cpp
M +2 -0 estraierindexer/estraierindexmanager.h
M +8 -1 estraierindexer/estraierindexwriter.cpp
M +1 -0 estraierindexer/estraierindexwriter.h
M +7 -2 estraierindexer/tests/EstraierTest.cpp
M +74 -11 indexertests/indexmanagertests.cpp
M +1 -1 indexertests/indexmanagertests.h
M +2 -1 indexertests/indexwritertests.cpp
M +8 -2 luceneindexer/cluceneindexmanager.cpp
M +2 -1 luceneindexer/cluceneindexmanager.h
M +4 -0 luceneindexer/cluceneindexwriter.cpp
M +1 -0 luceneindexer/cluceneindexwriter.h
M +1 -0 luceneindexer/tests/CLuceneTest.cpp
M +5 -0 sqliteindexer/sqliteindexwriter.cpp
M +1 -0 sqliteindexer/sqliteindexwriter.h
M +1 -1 streamindexer/indexwriter.cpp
M +1 -0 streamindexer/indexwriter.h
--- trunk/playground/base/strigi/src/daemon/daemon.cpp #581809:581810
@@ -271,8 +271,8 @@
set<string> dirs = readdirstoindex(dirsfile);
+ scheduler.setIndexManager(index);
scheduler.setIndexedDirectories(dirs);
- scheduler.setIndexManager(index);
EventListenerQueue listenerEventQueue;
scheduler.setEventListenerQueue (&listenerEventQueue);
--- trunk/playground/base/strigi/src/daemon/indexscheduler.cpp #581809:581810
@@ -124,7 +124,6 @@
IndexWriter* writer = indexmanager->getIndexWriter();
StreamIndexer* streamindexer = new StreamIndexer(writer);
-
if (dbfiles.size() == 0 && toindex.size() == 0) {
// retrieve the list of real files currently in the database
dbfiles = reader->getFiles(0);
@@ -132,7 +131,7 @@
char buff [20];
snprintf(buff, 20* sizeof (char), "%i", dbfiles.size());
STRIGI_LOG_DEBUG ("strigi.IndexScheduler", string(buff) + " real files in \
the database")
-
+
// first loop through all files
FileLister lister;
lister.setCallbackFunction(&addFileCallback);
@@ -255,4 +254,7 @@
dirstoindex.insert(dir);
}
}
+ if (dirstoindex.size() == 0) {
+ indexmanager->getIndexWriter()->deleteAllEntries();
+ }
}
--- trunk/playground/base/strigi/src/dummyindexer/dummyindexwriter.h #581809:581810
@@ -52,6 +52,7 @@
~DummyIndexWriter() {}
void commit() {}
void deleteEntries(const std::vector<std::string>& entries) {}
+ void deleteAllEntries() {}
};
#endif
--- trunk/playground/base/strigi/src/estraierindexer/estraierindexmanager.cpp \
#581809:581810 @@ -17,10 +17,14 @@
* the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
+#include "jstreamsconfig.h"
#include "estraierindexmanager.h"
#include "estraierindexreader.h"
#include "estraierindexwriter.h"
#include <assert.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include "stgdirent.h" //our dirent compatibility header... uses native if available
using namespace std;
using namespace jstreams;
@@ -34,7 +38,8 @@
EstraierIndexManager::EstraierIndexManager(const char* dbd)
: dblock(lock), dbdir(dbd) {
int errorcode;
- db = est_db_open(dbdir.c_str(), ESTDBWRITER|ESTDBCREAT|ESTDBPERFNG,
+ db = est_db_open(dbdir.c_str(),
+ ESTDBCREAT|ESTDBWRITER|ESTDBREADER|ESTDBNOLCK|ESTDBPERFNG,
&errorcode);
if (db == 0) printf("could not open db %s: %s\n", dbdir.c_str(),
est_err_msg(errorcode));
@@ -70,40 +75,7 @@
}
return w;
}
-/*ESTDB*
-EstraierIndexManager::getWriteDB() {
- pthread_mutex_lock(&dblock);
- // check if a writable db is already opened
- if (db) {
- if (writing) return db;
- closedb();
- }
- writing = true;
- int errorcode;
- db = est_db_open(dbdir.c_str(), ESTDBWRITER|ESTDBCREAT, &errorcode);
- if (db == 0) printf("could not open db: error %i\n", errorcode);
- return db;
-}
void
-EstraierIndexManager::returnWriteDB() {
- pthread_mutex_unlock(&dblock);
-}
-ESTDB*
-EstraierIndexManager::getReadDB() {
- pthread_mutex_lock(&dblock);
- // check if a readable db is already opened
- if (db) {
- if (!writing) return db;
- closedb();
- }
- writing = false;
- return db;
-}
-void
-EstraierIndexManager::returnReadDB() {
- pthread_mutex_unlock(&dblock);
-}*/
-void
EstraierIndexManager::ref() {
pthread_mutex_lock(&dblock);
}
@@ -119,3 +91,57 @@
if (!ok) printf("could not close db: error %i\n", errorcode);
db = 0;
}
+void
+removefiles(const string& d, bool rmd = false) {
+ // remove all entries from the subdir
+ DIR* dir = opendir(d.c_str());
+ if (dir == 0) {
+ fprintf(stderr, "could not open index directory.\n");
+ return;
+ }
+ // delete all the index files
+ struct dirent* e = readdir(dir);
+ while (e != 0) {
+ // skip the directories '.' and '..'
+ char c1 = e->d_name[0];
+ if (c1 == '.') {
+ char c2 = e->d_name[1];
+ if (c2 == '.' || c2 == '\0') {
+ e = readdir(dir);
+ continue;
+ }
+ }
+ string filename = d+'/'+e->d_name;
+ struct stat s;
+ // use lstat, we don't want to follow into symlinked directories
+ int r = lstat(filename.c_str(), &s);
+ if (r == 0) {
+ if (S_ISDIR(s.st_mode)) {
+ removefiles(filename, true);
+ } else {
+ printf("unlink %s\n", filename.c_str());
+ unlink(filename.c_str());
+ }
+ } else {
+ fprintf(stderr, "could not open file %s\n", filename.c_str());
+ }
+ e = readdir(dir);
+ }
+ closedir(dir);
+ if (rmd) {
+ rmdir(d.c_str());
+ }
+}
+void
+EstraierIndexManager::deleteIndex() {
+ ref();
+ closedb();
+
+ removefiles(dbdir);
+ // open the index again
+ int errorcode;
+ db = est_db_open(dbdir.c_str(), \
ESTDBCREAT|ESTDBWRITER|ESTDBREADER|ESTDBNOLCK|ESTDBPERFNG, &errorcode); + if (db \
== 0) printf("could not reopen db %s: %s\n", dbdir.c_str(), + \
est_err_msg(errorcode)); + deref();
+}
--- trunk/playground/base/strigi/src/estraierindexer/estraierindexmanager.h \
#581809:581810 @@ -52,6 +52,8 @@
jstreams::IndexWriter* getIndexWriter();
void ref();
void deref();
+
+ void deleteIndex();
};
jstreams::IndexManager*
--- trunk/playground/base/strigi/src/estraierindexer/estraierindexwriter.cpp \
#581809:581810 @@ -72,7 +72,10 @@
manager->ref();
int ok = est_db_put_doc(db, doc, 0);
- if (!ok) printf("could not write document\n");
+ if (!ok) {
+ fprintf(stderr, "error writing document: %s\n",
+ est_err_msg(est_db_error(db)));
+ }
// deallocate the estraier document
est_doc_delete(doc);
manager->deref();
@@ -117,3 +120,7 @@
free(all);
manager->deref();
}
+void
+EstraierIndexWriter::deleteAllEntries() {
+ manager->deleteIndex();
+}
--- trunk/playground/base/strigi/src/estraierindexer/estraierindexwriter.h \
#581809:581810 @@ -43,6 +43,7 @@
public:
void commit();
void deleteEntries(const std::vector<std::string>& entries);
+ void deleteAllEntries();
int itemsInCache() { return 0; };
};
--- trunk/playground/base/strigi/src/estraierindexer/tests/EstraierTest.cpp \
#581809:581810 @@ -1,5 +1,6 @@
#include "jstreamsconfig.h"
#include "estraierindexmanager.h"
+#include "indexmanagertests.h"
#include "indexwritertests.h"
#include "indexreadertests.h"
#include <sys/stat.h>
@@ -14,10 +15,14 @@
mkdir(path, S_IRUSR|S_IWUSR|S_IXUSR);
EstraierIndexManager* manager = new EstraierIndexManager(path);
- jstreams::IndexWriter* writer = manager->getIndexWriter();
- IndexWriterTests tests(writer);
+ IndexManagerTests tests(manager);
errors += tests.testAll();
+ errors += tests.testAllInThreads(20);
+ jstreams::IndexWriter* writer = manager->getIndexWriter();
+ IndexWriterTests wtests(writer);
+ errors += wtests.testAll();
+
jstreams::IndexReader* reader = manager->getIndexReader();
IndexReaderTests rtests(reader);
errors += rtests.testAll();
--- trunk/playground/base/strigi/src/indexertests/indexmanagertests.cpp \
#581809:581810 @@ -1,29 +1,92 @@
#include "indexmanagertests.h"
#include "indexreader.h"
+#include "indexwriter.h"
#include "indexmanager.h"
+#include <sstream>
+using namespace std;
using namespace jstreams;
class IndexManagerTester {
private:
- IndexManager* reader;
+ pthread_mutex_t lock;
+ int errors;
+ IndexManager* manager;
+ IndexWriter* writer;
+ IndexReader* reader;
public:
- IndexManagerTester(IndexManager* w) :reader(w) {}
+ IndexManagerTester(IndexManager* m) :errors(0), manager(m) {
+ writer = manager->getIndexWriter();
+ reader = manager->getIndexReader();
+ pthread_mutex_init(&lock, 0);
+ }
+ ~IndexManagerTester() {
+ pthread_mutex_destroy(&lock);
+ }
+ int testAll(bool threaded = false);
+ void cleanErrors() {
+ pthread_mutex_lock(&lock);
+ errors = 0;
+ pthread_mutex_unlock(&lock);
+ }
+ int getErrors() {
+ int n;
+ pthread_mutex_lock(&lock);
+ n = errors;
+ pthread_mutex_unlock(&lock);
+ return errors;
+ }
+ void addErrors(int n) {
+ pthread_mutex_lock(&lock);
+ errors += n;
+ pthread_mutex_unlock(&lock);
+ }
+ int addAndCount();
};
+int
+IndexManagerTester::testAll(bool threaded) {
+ int n = 0;
-IndexManagerTests::IndexManagerTests(jstreams::IndexManager* m)
- :tester (new IndexManagerTester(m)) {
+ // tests that only need return 0 when not threaded
+ int nt = 0;
+ nt += addAndCount();
+ if (!threaded) {
+ n += nt;
+ }
+
+ addErrors(n);
+ return n;
}
-IndexManagerTests::~IndexManagerTests() {
- delete tester;
+int
+IndexManagerTester::addAndCount() {
+ int m = 20;
+ ostringstream str;
+ for (int i=0; i<m; ++i) {
+ str << i;
+ string s(str.str());
+ { Indexable idx(s, 0, writer, 0); }
+ str.str("");
+ }
+ writer->commit();
+ int n = reader->countDocuments();
+ printf("%i %i\n", n, m);
+ return n != m;
}
+/* below here the threading plumbing is done */
void*
threadstarter(void *d) {
IndexManagerTests* tester = static_cast<IndexManagerTests*>(d);
- tester->testAll();
+ tester->testAll(true);
pthread_exit(0);
}
+IndexManagerTests::IndexManagerTests(jstreams::IndexManager* m)
+ :tester (new IndexManagerTester(m)) {
+}
+IndexManagerTests::~IndexManagerTests() {
+ delete tester;
+}
int
IndexManagerTests::testAllInThreads(int n) {
+ tester->cleanErrors();
pthread_t* thread = new pthread_t[n];
for (int i=0; i<n; ++i) {
pthread_create(thread+i, NULL, threadstarter, this);
@@ -33,11 +96,11 @@
}
delete [] thread;
- return 0;
+ return tester->getErrors();
}
int
-IndexManagerTests::testAll() {
- int n = 0;
- return n;
+IndexManagerTests::testAll(bool threaded) {
+ tester->cleanErrors();
+ return tester->testAll(threaded);
}
--- trunk/playground/base/strigi/src/indexertests/indexmanagertests.h #581809:581810
@@ -13,7 +13,7 @@
IndexManagerTests(jstreams::IndexManager* w);
~IndexManagerTests();
int testAllInThreads(int n);
- int testAll();
+ int testAll(bool threaded=false);
};
#endif
--- trunk/playground/base/strigi/src/indexertests/indexwritertests.cpp #581809:581810
@@ -12,8 +12,9 @@
return 0;
}
int add() {
+ std::string s("a"); // we must pass a string, not a const char*
{
- Indexable i("", 0, writer, 0);
+ Indexable i(s, 0, writer, 0);
}
writer->commit();
--- trunk/playground/base/strigi/src/luceneindexer/cluceneindexmanager.cpp \
#581809:581810 @@ -123,11 +123,11 @@
indexreader = 0;
}
void
-CLuceneIndexManager::openWriter() {
+CLuceneIndexManager::openWriter(bool truncate) {
version++;
try {
printf("writer at %s\n", dbdir.c_str());
- if (IndexReader::indexExists(dbdir.c_str())) {
+ if (!truncate && IndexReader::indexExists(dbdir.c_str())) {
if (IndexReader::isLocked(dbdir.c_str())) {
IndexReader::unlock(dbdir.c_str());
}
@@ -190,3 +190,9 @@
closedir(dir);
return size;
}
+void
+CLuceneIndexManager::deleteIndex() {
+ closeReader();
+ closeWriter();
+ openWriter(true);
+}
--- trunk/playground/base/strigi/src/luceneindexer/cluceneindexmanager.h \
#581809:581810 @@ -60,7 +60,7 @@
void openReader();
void closeReader();
- void openWriter();
+ void openWriter(bool truncate=false);
void closeWriter();
public:
CLuceneIndexManager(const std::string& path);
@@ -76,6 +76,7 @@
int32_t docCount();
int64_t getIndexSize();
int getVersion() const { return version; }
+ void deleteIndex();
};
jstreams::IndexManager*
--- trunk/playground/base/strigi/src/luceneindexer/cluceneindexwriter.cpp \
#581809:581810 @@ -170,3 +170,7 @@
_CLDECDELETE(term);
manager->derefReader();
}
+void
+CLuceneIndexWriter::deleteAllEntries() {
+ manager->deleteIndex();
+}
--- trunk/playground/base/strigi/src/luceneindexer/cluceneindexwriter.h \
#581809:581810 @@ -38,6 +38,7 @@
~CLuceneIndexWriter();
void commit() {};
void deleteEntries(const std::vector<std::string>& entries);
+ void deleteAllEntries();
void deleteEntry(const std::string& entry);
};
--- trunk/playground/base/strigi/src/luceneindexer/tests/CLuceneTest.cpp \
#581809:581810 @@ -16,6 +16,7 @@
jstreams::IndexManager* manager = createCLuceneIndexManager(path);
IndexManagerTests tests(manager);
+ errors += tests.testAll();
errors += tests.testAllInThreads(20);
jstreams::IndexWriter* writer = manager->getIndexWriter();
--- trunk/playground/base/strigi/src/sqliteindexer/sqliteindexwriter.cpp \
#581809:581810 @@ -276,3 +276,8 @@
}
manager->deref();
}
+void
+SqliteIndexWriter::deleteAllEntries() {
+ manager->ref();
+ manager->deref();
+}
--- trunk/playground/base/strigi/src/sqliteindexer/sqliteindexwriter.h #581809:581810
@@ -53,6 +53,7 @@
public:
void commit();
void deleteEntries(const std::vector<std::string>& entries);
+ void deleteAllEntries();
int itemsInCache() { return temprows; };
};
--- trunk/playground/base/strigi/src/streamindexer/indexwriter.cpp #581809:581810
@@ -36,5 +36,5 @@
if (p != string::npos) {
return name.substr(p+1);
}
- return "";
+ return name;
}
--- trunk/playground/base/strigi/src/streamindexer/indexwriter.h #581809:581810
@@ -54,6 +54,7 @@
virtual ~IndexWriter() {}
virtual void commit() { return; }
virtual void deleteEntries(const std::vector<std::string>& entries) = 0;
+ virtual void deleteAllEntries() = 0;
virtual int itemsInCache() { return 0; }
virtual void optimize() {}
};
[prev in list] [next in list] [prev in thread] [next in thread]
Configure |
About |
News |
Add a list |
Sponsored by KoreLogic