[prev in list] [next in list] [prev in thread] [next in thread]
List: kmail-devel
Subject: [PATCH] full text indexing for kmail
From: Luís_Pedro_Coelho <luis () luispedro ! org>
Date: 2005-07-31 18:28:31
Message-ID: 200507311128.31699.luis () luispedro ! org
[Download RAW message or body]
Hi,
this is a heads-up. Either tomorrow evening or tue morning, I'll commit this
to kmail unless I get a "No" from the maintainers.
This depends on indexlib which has been on kdesupport for some time, so please
compile that: it's a very little library.
(I wanted to do this two weeks ago, but kdepim has been completely FUBAR for
me. I had to fix a couple of portability issues to get it to compile)
bye,
--
luispedro
http://blog.luispedro.org/
http://luispedro.org/
["kmail-patch" (text/x-diff)]
Index: kmfoldersearch.h
===================================================================
--- kmfoldersearch.h (revision 440058)
+++ kmfoldersearch.h (working copy)
@@ -71,9 +71,11 @@
bool running() const { return mRunning; }
void stop();
int foundCount() const { return mFoundCount; }
- int searchedCount() const { return mSearchedCount; }
QString currentFolder() const { return mLastFolder; }
+public slots:
+ void indexFinished();
+
signals:
void found(Q_UINT32 serNum);
void finished(bool success);
@@ -87,7 +89,6 @@
friend class ::KMIndexSearchTarget;
void setRunning(bool b) { mRunning = b; }
void setFoundCount(int f) { mFoundCount = f; }
- void setSearchedCount(int f) { mSearchedCount = f; }
void setCurrentFolder(const QString &f) { mLastFolder = f; }
private:
@@ -99,7 +100,7 @@
QValueList<QGuardedPtr<KMFolderImap> > mIncompleteFolders;
SerNumList mSerNums;
QString mLastFolder;
- int mSearchedCount, mFoundCount;
+ int mFoundCount;
QTimer *mProcessNextBatchTimer;
};
Index: kmmessage.h
===================================================================
--- kmmessage.h (revision 440058)
+++ kmmessage.h (working copy)
@@ -818,7 +818,6 @@
*/
QCString mboxMessageSeparator();
-private:
/** Returns message body with quoting header and indented by the
given indentation string. This is suitable for including the message
in another message of for replies, forwards. The header string is
@@ -843,6 +842,7 @@
/** Return the textual content of the message as plain text,
converting HTML to plain text if necessary. */
QString asPlainText( bool stripSignature, bool allowDecryption ) const;
+private:
/** Initialization shared by the ctors. */
void init();
Index: Makefile.am
===================================================================
--- Makefile.am (revision 440058)
+++ Makefile.am (working copy)
@@ -3,6 +3,8 @@
SUBDIRS = interfaces . about pics profiles avscripts tests
+LIB_INDEX = -lindex
+
INCLUDES = -I$(top_srcdir)/libkmime \
-I$(top_srcdir)/libkpgp \
-I$(top_srcdir)/libkdenetwork \
@@ -18,7 +20,7 @@
lib_LTLIBRARIES = libkmailprivate.la
libkmailprivate_la_LDFLAGS = $(all_libraries) -avoid-version -no-undefined
-libkmailprivate_la_LIBADD = $(LIB_KHTML) $(LIB_KSPELL) $(LIB_KABC) \
+libkmailprivate_la_LIBADD = $(LIB_KHTML) $(LIB_KSPELL) $(LIB_KABC) $(LIB_INDEX) \
../libkmime/libkmime.la ../libkpgp/libkpgp.la ../libkdepim/libkdepim.la \
../libkpimidentities/libkpimidentities.la ../mimelib/libmimelib.la \
../libksieve/libksieve.la ../libemailfunctions/libemailfunctions.la \
@@ -45,6 +47,7 @@
libkmailprivate_la_SOURCES = kmmessage.cpp kmmainwin.cpp configuredialog.cpp \
configuredialog_p.cpp \
+ index.cpp klistviewindexedsearchline.cpp \
simplestringlisteditor.cpp kmmsgindex.cpp \
identitydrag.cpp identitylistview.cpp identitydialog.cpp \
kmfolderdia.cpp kmfoldertree.cpp kmtransport.cpp \
Index: headeritem.cpp
===================================================================
--- headeritem.cpp (revision 440058)
+++ headeritem.cpp (working copy)
@@ -98,6 +98,7 @@
}
KMMsgBase *mMsgBase = headers->folder()->getMsgBase( mMsgId );
+ mSerNum = mMsgBase->getMsgSerNum();
if (mMsgBase->isNew() || mMsgBase->isUnread()
|| mMsgBase->isImportant() || mMsgBase->isTodo() || mMsgBase->isWatched() ) {
setOpen(true);
@@ -114,6 +115,12 @@
return mMsgId;
}
+// Return the serial number
+Q_UINT32 HeaderItem::msgSerNum() const
+{
+ return mSerNum;
+}
+
// Update this item to summarise a new folder and message
//Opens all children in the thread
Index: kmsearchpattern.cpp
===================================================================
--- kmsearchpattern.cpp (revision 440058)
+++ kmsearchpattern.cpp (working copy)
@@ -10,6 +10,9 @@
#include "kmmsgindex.h"
#include "kmmsgdict.h"
#include "filterlog.h"
+#include "kmkernel.h"
+#include "kmmsgdict.h"
+#include "kmfolder.h"
using KMail::FilterLog;
#include <libemailfunctions/email.h>
Index: searchwindow.cpp
===================================================================
--- searchwindow.cpp (revision 440058)
+++ searchwindow.cpp (working copy)
@@ -334,7 +334,6 @@
QString folderName;
if (search) {
numMatches = search->foundCount();
- count = search->searchedCount();
folderName = search->currentFolder();
}
Index: kmkernel.cpp
===================================================================
--- kmkernel.cpp (revision 440058)
+++ kmkernel.cpp (working copy)
@@ -11,7 +11,7 @@
#include "globalsettings.h"
#include "kmstartup.h"
-#include "kmmsgindex.h"
+#include "index.h"
#include "kmmainwin.h"
#include "composer.h"
#include "kmmsgpart.h"
@@ -1385,13 +1385,7 @@
readConfig();
mICalIface->readConfig();
// filterMgr->dump();
-#if 0 //disabled for now..
- the_msgIndex = new KMMsgIndex(this, "the_index"); //create the indexer
- the_msgIndex->init();
- the_msgIndex->remove();
- delete the_msgIndex;
- the_msgIndex = 0;
-#endif
+ the_msgIndex = new KMMsgIndex(this); //create the indexer
#if 0
the_weaver = new KPIM::ThreadWeaver::Weaver( this );
Index: kmail.kcfg
===================================================================
--- kmail.kcfg (revision 440058)
+++ kmail.kcfg (working copy)
@@ -433,4 +433,11 @@
</group>
+ <group name="TextIndex">
+ <entry name="automaticDecrypt" type="Bool" key="automaticDecrypt">
+ <default>true</default>
+ <label></label>
+ </entry>
+ </group>
+
</kcfg>
Index: kmmsgindex.h
===================================================================
--- kmmsgindex.h (revision 440058)
+++ kmmsgindex.h (working copy)
@@ -1,3 +1,5 @@
+#include "index.h"
+#if 0
/* Message indexing
*
* Author: Sam Magnuson <zachsman@wiw.org>
@@ -167,4 +169,5 @@
+#endif
Index: kmmainwidget.cpp
===================================================================
--- kmmainwidget.cpp (revision 440058)
+++ kmmainwidget.cpp (working copy)
@@ -88,6 +88,7 @@
#include "filterlogdlg.h"
using KMail::FilterLogDialog;
#include <headerlistquicksearch.h>
+#include "klistviewindexedsearchline.h"
using KMail::HeaderListQuickSearch;
#include "kmheaders.h"
#include "mailinglistpropertiesdialog.h"
@@ -519,7 +520,7 @@
mHeaders = new KMHeaders(this, mSearchAndHeaders, "headers");
- mQuickSearchLine = new HeaderListQuickSearch( mSearchToolBar, mHeaders,
+ mQuickSearchLine = new KListViewIndexedSearchLine( mSearchToolBar, mHeaders,
actionCollection(), "headers \
quick search line" ); label->setBuddy( mQuickSearchLine );
mSearchToolBar->setStretchableWidget( mQuickSearchLine );
Index: headeritem.h
===================================================================
--- headeritem.h (revision 440058)
+++ headeritem.h (working copy)
@@ -179,6 +179,9 @@
/** Return the msgId of the message associated with this item. */
int msgId() const;
+ // Return the serial number of the message associated with this item;
+ Q_UINT32 msgSerNum() const;
+
/** Expands all children of the list view item. */
void setOpenRecursive( bool open );
@@ -226,6 +229,7 @@
private:
int mMsgId;
+ Q_UINT32 mSerNum;
QString mKey;
bool mAboutToBeDeleted;
SortCacheItem *mSortCacheItem;
Index: kmfoldersearch.cpp
===================================================================
--- kmfoldersearch.cpp (revision 440058)
+++ kmfoldersearch.cpp (working copy)
@@ -25,7 +25,7 @@
#include "kmfoldermgr.h"
#include "kmsearchpattern.h"
#include "kmmsgdict.h"
-#include "kmmsgindex.h"
+#include "index.h"
#include "jobscheduler.h"
#include <kdebug.h>
@@ -77,12 +77,10 @@
mRunByIndex = mRunning = false;
mRoot = 0;
mSearchPattern = 0;
- mSearchedCount = 0;
mFoundCount = 0;
mProcessNextBatchTimer = new QTimer();
- connect(mProcessNextBatchTimer, SIGNAL(timeout()),
- this, SLOT(slotProcessNextBatch()));
+ connect(mProcessNextBatchTimer, SIGNAL(timeout()), this, \
SLOT(slotProcessNextBatch())); }
KMSearch::~KMSearch()
@@ -155,7 +153,6 @@
return;
}
- mSearchedCount = 0;
mFoundCount = 0;
mRunning = true;
mRunByIndex = false;
@@ -231,6 +228,11 @@
emit finished(false);
}
+void KMSearch::indexFinished() {
+ mRunning = false;
+ mRunByIndex = false;
+}
+
void KMSearch::slotProcessNextBatch()
{
if ( !running() )
@@ -246,11 +248,9 @@
folder->open();
mOpenedFolders.append( folder );
connect( folder->storage(),
- SIGNAL( searchResult( KMFolder*, QValueList<Q_UINT32>,
- const KMSearchPattern*, bool ) ),
+ SIGNAL( searchResult( KMFolder*, QValueList<Q_UINT32>, const \
KMSearchPattern*, bool ) ), this,
- SLOT( slotSearchFolderResult( KMFolder*, QValueList<Q_UINT32>,
- const KMSearchPattern*, bool ) ) );
+ SLOT( slotSearchFolderResult( KMFolder*, QValueList<Q_UINT32>, const \
KMSearchPattern*, bool ) ) ); folder->storage()->search( mSearchPattern );
} else
--mRemainingFolders;
@@ -281,7 +281,6 @@
this,
SLOT( slotSearchFolderResult( KMFolder*, QValueList<Q_UINT32>,
const KMSearchPattern*, bool ) ) );
- mSearchedCount += folder->count();
--mRemainingFolders;
folder->close();
mOpenedFolders.remove( folder );
Index: kmmsgindex.cpp
===================================================================
--- kmmsgindex.cpp (revision 440058)
+++ kmmsgindex.cpp (working copy)
@@ -1,6 +1,7 @@
// Author: Sam Magnuson <zachsman@wiw.org>
// License GPL -- indexing logic for KMail
+#if 0
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
@@ -1242,7 +1243,6 @@
if (!folder || (idx == -1))
continue;
if(mSearch->inScope(folder)) {
- mSearch->setSearchedCount(mSearch->searchedCount()+1);
mSearch->setCurrentFolder(folder->label());
if(mVerifyResult) { //full phrase..
if(mOpenedFolders.findIndex(folder) == -1) {
@@ -1297,3 +1297,4 @@
return stopQuery(id);
}
#include "kmmsgindex.moc"
+#endif
--- /dev/null Sun Jul 31 11:23:40 2005
+++ klistviewindexedsearchline.cpp Fri Jul 29 11:31:03 2005
@@ -0,0 +1,75 @@
+
+/* This file is part of KMail
+ * Copyright (C) 2005 Luís Pedro Coelho <luis@luispedro.org>
+ *
+ * KMail is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License, version 2, as
+ * published by the Free Software Foundation.
+ *
+ * KMail is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of this program with any edition of
+ * the Qt library by Trolltech AS, Norway (or with modified versions
+ * of Qt that use the same license as Qt), and distribute linked
+ * combinations including the two. You must obey the GNU General
+ * Public License in all respects for all of the code used other than
+ * Qt. If you modify this file, you may extend this exception to
+ * your version of the file, but you are not obligated to do so. If
+ * you do not wish to do so, delete this exception statement from
+ * your version.
+ */
+
+#include "klistviewindexedsearchline.h"
+#include <kdebug.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include "headeritem.h"
+#include "kmheaders.h"
+#include "kmfolder.h"
+#include "index.h"
+
+using KMail::HeaderListQuickSearch;
+
+KListViewIndexedSearchLine::KListViewIndexedSearchLine( QWidget* parent, KListView* \
listView, KActionCollection* actionCollection, const char* name ): \
+ HeaderListQuickSearch( parent, listView, actionCollection, name ), + mFiltering( \
false ) +{
+}
+
+KListViewIndexedSearchLine::~KListViewIndexedSearchLine() {
+}
+
+
+//std::string dirbase = std::string( getenv( "HOME" ) ) + "/.indexes/" + \
folder->location().latin1(); +
+void KListViewIndexedSearchLine::updateSearch( const QString& s ) {
+ kdDebug( 5006 ) << "updateSearch( -" << s << "- )" << endl;
+ mFiltering = false;
+ if ( !s.isNull() && !s.isEmpty() ) {
+ KMMsgIndex* index = kmkernel->msgIndex();
+ mResults = index->simpleSearch( s );
+ std::sort( mResults.begin(), mResults.end() );
+ mFiltering = true;
+ }
+ KListViewSearchLine::updateSearch( s );
+}
+#include <iostream>
+#define kdDebug( x ) std::cerr
+#define endl std::endl
+
+bool KListViewIndexedSearchLine::itemMatches( const QListViewItem* item, const \
QString& s ) const { + return !mFiltering ||
+ std::binary_search( mResults.begin(), mResults.end(), static_cast<const \
KMail::HeaderItem*>( item )->msgSerNum() ) + || KListViewSearchLine::itemMatches( \
item, s ); +}
+
+#include "klistviewindexedsearchline.moc"
+
--- /dev/null Sun Jul 31 11:23:40 2005
+++ klistviewindexedsearchline.h Fri Jul 29 11:31:03 2005
@@ -0,0 +1,73 @@
+#ifndef LPC_KLISTVIEWINDEXEDSEARCHLINE_H1107549660_INCLUDE_GUARD_
+#define LPC_KLISTVIEWINDEXEDSEARCHLINE_H1107549660_INCLUDE_GUARD_
+
+/* This file is part of KMail
+ * Copyright (C) 2005 Luís Pedro Coelho <luis@luispedro.org>
+ *
+ * KMail is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License, version 2, as
+ * published by the Free Software Foundation.
+ *
+ * KMail is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of this program with any edition of
+ * the Qt library by Trolltech AS, Norway (or with modified versions
+ * of Qt that use the same license as Qt), and distribute linked
+ * combinations including the two. You must obey the GNU General
+ * Public License in all respects for all of the code used other than
+ * Qt. If you modify this file, you may extend this exception to
+ * your version of the file, but you are not obligated to do so. If
+ * you do not wish to do so, delete this exception statement from
+ * your version.
+ */
+
+
+#include <klineedit.h>
+#include <klistviewsearchline.h>
+#include "headerlistquicksearch.h"
+#include <qhbox.h>
+
+#include <vector>
+
+class KListView;
+class QListViewItem;
+class QToolButton;
+class KListViewSearchLine;
+class indexer;
+
+class KDEUI_EXPORT KListViewIndexedSearchLine: public KMail::HeaderListQuickSearch
+{
+ Q_OBJECT
+
+public:
+
+ explicit KListViewIndexedSearchLine(QWidget *parent, KListView *listView, \
KActionCollection* action, const char *name = 0); + ~KListViewIndexedSearchLine();
+
+public slots:
+ /**
+ * Updates search to only make visible the items that match \a s. If
+ * \a s is null then the line edit's text will be used.
+ */
+ virtual void updateSearch(const QString &s = QString::null);
+
+protected:
+ virtual bool itemMatches(const QListViewItem *item, const QString &s) const;
+
+private:
+ std::vector<unsigned> mResults;
+ bool mFiltering;
+
+};
+
+
+
+#endif /* LPC_KLISTVIEWINDEXEDSEARCHLINE_H1107549660_INCLUDE_GUARD_ */
--- /dev/null Sun Jul 31 11:23:40 2005
+++ index.h Fri Jul 29 11:31:03 2005
@@ -0,0 +1,151 @@
+#ifndef LPC_INDEX_H1110724080_INCLUDE_GUARD_
+#define LPC_INDEX_H1110724080_INCLUDE_GUARD_
+
+/* This file is part of KMail
+ * Copyright (C) 2005 Luís Pedro Coelho <luis@luispedro.org>
+ *
+ * KMail is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License, version 2, as
+ * published by the Free Software Foundation.
+ *
+ * KMail is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of this program with any edition of
+ * the Qt library by Trolltech AS, Norway (or with modified versions
+ * of Qt that use the same license as Qt), and distribute linked
+ * combinations including the two. You must obey the GNU General
+ * Public License in all respects for all of the code used other than
+ * Qt. If you modify this file, you may extend this exception to
+ * your version of the file, but you are not obligated to do so. If
+ * you do not wish to do so, delete this exception statement from
+ * your version.
+ */
+
+
+#include <qobject.h>
+#include <qcstring.h>
+#include <qvaluelist.h>
+#include <qtimer.h>
+#include <index/index.h>
+#include <index/lockfile.h>
+#include <vector>
+#include <set>
+
+class KMFolder;
+
+class KMSearch;
+class KMSearchRule;
+class KMSearchPattern;
+
+class KMMsgIndex : public QObject {
+ Q_OBJECT
+ public:
+ explicit KMMsgIndex( QObject* parent );
+ ~KMMsgIndex();
+
+ public:
+
+ bool startQuery( KMSearch* );
+ bool stopQuery( KMSearch* );
+
+ std::vector<Q_UINT32> simpleSearch( QString ) const;
+ public slots:
+ void clear();
+ void create();
+ void cleanUp();
+
+ private slots:
+ void act();
+ void removeSearch( QObject* );
+
+ //void syncIndex();
+ //void startSync();
+ void continueCreation();
+
+ void slotAddMessage( KMFolder*, Q_UINT32 message );
+ void slotRemoveMessage( KMFolder*, Q_UINT32 message );
+ private:
+ static QString defaultPath();
+
+ bool canHandleQuery( const KMSearchPattern* ) const;
+ bool isIndexed( const KMFolder* ) const;
+ int addMessage( Q_UINT32 );
+ void removeMessage( Q_UINT32 );
+
+ void scheduleAction();
+ bool creating() const;
+
+ public:
+ /**
+ * @internal
+ * DO NOT USE THIS CLASS
+ *
+ * It is conceptually a private class. Just needs to be public because of moc \
limitations + */
+ class Search;
+ private:
+
+ std::vector<Q_UINT32> mPendingMsgs;
+ std::vector<KMFolder*> mPendingFolders;
+ std::vector<Q_UINT32> mAddedMsgs;
+ std::vector<Q_UINT32> mRemovedMsgs;
+ std::vector<Q_UINT32> mExisting;
+
+ enum e_state {
+ s_idle, // doing nothing
+ s_willcreate, // just constructed, create() scheduled (mIndex == 0)
+ s_creating, // creating the index from the messages
+ s_processing // has messages to process
+ } mState;
+
+ unsigned mMaintenanceCount;
+
+ /**
+ * The lock below should be moved down into libindex itself
+ * where things can be handled with a lot more granularity
+ */
+ indexlib::detail::lockfile mLockFile;
+ //enum e_syncState { ss_none, ss_started, ss_synced } mSyncState;
+ //QTimer* mSyncTimer;
+
+ std::set<KMFolder*> mOpenedFolders;
+ std::vector<Search*> mSearches;
+ indexlib::index* mIndex;
+ QCString mIndexPath;
+ QTimer* mTimer;
+ bool mSlowDown;
+};
+
+
+class KMMsgIndex::Search : public QObject {
+ Q_OBJECT
+ public:
+ explicit Search( KMSearch* s );
+ ~Search();
+ KMSearch* search() const { return mSearch; }
+ signals:
+ void found( Q_UINT32 );
+ void finished( bool );
+ private slots:
+ void act();
+ private:
+ KMSearch* mSearch;
+ QTimer* mTimer;
+ /**
+ * mResidual is the part of the pattern which is not
+ * handled by the index
+ */
+ KMSearchPattern* mResidual;
+ std::vector<Q_UINT32> mValues;
+ enum { s_none = 0, s_starting, s_emitting, s_emitstopped, s_done } mState;
+};
+
+#endif /* LPC_INDEX_H1110724080_INCLUDE_GUARD_ */
--- /dev/null Sun Jul 31 11:23:40 2005
+++ index.cpp Fri Jul 29 11:31:03 2005
@@ -0,0 +1,435 @@
+/* This file is part of KMail
+ * Copyright (C) 2005 Luís Pedro Coelho <luis@luispedro.org>
+ *
+ * KMail is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License, version 2, as
+ * published by the Free Software Foundation.
+ *
+ * KMail is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of this program with any edition of
+ * the Qt library by Trolltech AS, Norway (or with modified versions
+ * of Qt that use the same license as Qt), and distribute linked
+ * combinations including the two. You must obey the GNU General
+ * Public License in all respects for all of the code used other than
+ * Qt. If you modify this file, you may extend this exception to
+ * your version of the file, but you are not obligated to do so. If
+ * you do not wish to do so, delete this exception statement from
+ * your version.
+ */
+
+#include "index.h"
+
+#include "kmkernel.h"
+#include "kmfoldermgr.h"
+#include "kmmsgdict.h"
+#include "kmfolder.h"
+#include "kmsearchpattern.h"
+#include "kmfoldersearch.h"
+
+#include <kdebug.h>
+#include <kapplication.h>
+#include <qfile.h>
+#include <qtimer.h>
+#include <qvaluestack.h>
+#include <qfileinfo.h>
+#include <index/create.h>
+
+#include <sys/types.h>
+#include <sys/stat.h>
+
+#include <iostream>
+#include <algorithm>
+#include <cstdlib>
+
+const unsigned int MaintenanceLimit = 1000;
+
+static
+QValueList<int> vectorToQValueList( const std::vector<Q_UINT32>& input ) {
+ QValueList<int> res;
+ std::copy( input.begin(), input.end(), std::back_inserter( res ) );
+ return res;
+}
+
+
+static
+std::vector<Q_UINT32> QValueListToVector( const QValueList<int>& input ) {
+ std::vector<Q_UINT32> res;
+ // res.assign( input.begin(), input.end() ) doesn't work for some reason
+ for ( QValueList<int>::const_iterator first = input.begin(), past = input.end(); \
first != past; ++first ) { + res.push_back( *first );
+ }
+ return res;
+}
+
+KMMsgIndex::KMMsgIndex( QObject* parent ):
+ QObject( parent, "index" ),
+ mIndex( 0 ),
+ mState( s_idle ),
+ mIndexPath( QFile::encodeName( defaultPath() ) ),
+ mTimer( new QTimer( this ) ),
+ mLockFile( std::string( static_cast<const char*>( QFile::encodeName( defaultPath() \
) + "/lock" ) ) ), + //mSyncState( ss_none ),
+ //mSyncTimer( new QTimer( this ) ),
+ mSlowDown( false ) {
+ kdDebug( 5006 ) << "KMMsgIndex::KMMsgIndex()" << endl;
+
+ if ( !QFileInfo( mIndexPath ).exists() ) {
+ ::mkdir( mIndexPath, S_IRWXU );
+ }
+
+ connect( kmkernel->folderMgr(), SIGNAL( msgRemoved( KMFolder*, Q_UINT32 ) ), SLOT( \
slotRemoveMessage( KMFolder*, Q_UINT32 ) ) ); + connect( kmkernel->folderMgr(), \
SIGNAL( msgAdded( KMFolder*, Q_UINT32 ) ), SLOT( slotAddMessage( KMFolder*, Q_UINT32 \
) ) ); + connect( kmkernel->dimapFolderMgr(), SIGNAL( msgRemoved( KMFolder*, Q_UINT32 \
) ), SLOT( slotRemoveMessage( KMFolder*, Q_UINT32 ) ) ); + connect( \
kmkernel->dimapFolderMgr(), SIGNAL( msgAdded( KMFolder*, Q_UINT32 ) ), SLOT( \
slotAddMessage( KMFolder*, Q_UINT32 ) ) ); +
+ connect( mTimer, SIGNAL( timeout() ), SLOT( act() ) );
+ //connect( mSyncTimer, SIGNAL( timeout() ), SLOT( syncIndex() ) );
+
+ if ( !mLockFile.trylock() ) {
+ indexlib::remove( mIndexPath );
+ mLockFile.force_unlock();
+ mLockFile.trylock();
+ } else {
+ mIndex = indexlib::open( mIndexPath, indexlib::open_flags::fail_if_nonexistant \
).release(); + }
+ if ( !mIndex ) {
+ QTimer::singleShot( 8000, this, SLOT( create() ) );
+ mState = s_willcreate;
+ } else {
+ KConfigGroup cfg( KMKernel::config(), "text-index" );
+ if ( cfg.readBoolEntry( "creating" ) ) {
+ QTimer::singleShot( 8000, this, SLOT( continueCreation() ) );
+ mState = s_creating;
+ } else {
+ mPendingMsgs = QValueListToVector( cfg.readIntListEntry( "pending" ) );
+ mRemovedMsgs = QValueListToVector( cfg.readIntListEntry( "removed" ) );
+ }
+ }
+ //if ( mState == s_idle ) mSyncState = ss_synced;
+}
+
+
+KMMsgIndex::~KMMsgIndex() {
+ kdDebug( 5006 ) << "KMMsgIndex::~KMMsgIndex()" << endl;
+ KConfigGroup cfg( KMKernel::config(), "text-index" );
+ cfg.writeEntry( "creating", mState == s_creating );
+ QValueList<int> pendingMsg;
+ if ( mState == s_processing ) {
+ Q_ASSERT( mAddedMsgs.empty() );
+ pendingMsg = vectorToQValueList( mPendingMsgs );
+ }
+ cfg.writeEntry( "pending", pendingMsg );
+ cfg.writeEntry( "removed", vectorToQValueList( mRemovedMsgs ) );
+ delete mIndex;
+}
+
+void KMMsgIndex::clear() {
+ delete mIndex;
+ mLockFile.force_unlock();
+ mIndex = 0;
+ indexlib::remove( mIndexPath );
+}
+
+void KMMsgIndex::cleanUp() {
+ if ( mState != s_idle || kapp->hasPendingEvents() ) {
+ QTimer::singleShot( 8000, this, SLOT( cleanUp() ) );
+ return;
+ }
+ mIndex->maintenance();
+}
+
+int KMMsgIndex::addMessage( Q_UINT32 serNum ) {
+ kdDebug( 5006 ) << "KMMsgIndex::addMessage( " << serNum << " )" << endl;
+ assert( mIndex );
+ if ( !mExisting.empty() && std::binary_search( mExisting.begin(), mExisting.end(), \
serNum ) ) return 0; +
+ int idx = -1;
+ KMFolder* folder = 0;
+ KMMsgDict::instance()->getLocation( serNum, &folder, &idx );
+ if ( !folder || idx == -1 ) return -1;
+ if ( !mOpenedFolders.count( folder ) ) {
+ mOpenedFolders.insert( folder );
+ folder->open();
+ }
+ KMMessage* msg = folder->getMsg( idx );
+ /* I still don't know whether we should allow decryption or not.
+ * Setting to false which makes more sense.
+ * We keep signature to get the person's name
+ */
+ QString body = msg->asPlainText( false, false );
+ if ( !body.isEmpty() && static_cast<const char*>( body.latin1() ) ) {
+ mIndex->add( body.latin1(), QString::number( serNum ).latin1() );
+ } else {
+ kdDebug( 5006 ) << "Funny, no body" << endl;
+ }
+ folder->unGetMsg( idx );
+ return 0;
+}
+
+void KMMsgIndex::act() {
+ kdDebug( 5006 ) << "KMMsgIndex::act()" << endl;
+ if ( kapp->hasPendingEvents() ) {
+ //nah, some other time..
+ mTimer->start( 500 );
+ mSlowDown = true;
+ return;
+ }
+ if ( mSlowDown ) {
+ mSlowDown = false;
+ mTimer->start( 0 );
+ }
+ if ( !mPendingMsgs.empty() ) {
+ addMessage( mPendingMsgs.back() );
+ mPendingMsgs.pop_back();
+ return;
+ }
+ if ( !mPendingFolders.empty() ) {
+ KMFolder *f = mPendingFolders.back();
+ mPendingFolders.pop_back();
+ if ( !mOpenedFolders.count( f ) ) {
+ mOpenedFolders.insert( f );
+ f->open();
+ }
+ KMMsgDict* dict = KMMsgDict::instance();
+ for ( int i = 0; i < f->count(); ++i) {
+ mPendingMsgs.push_back( dict->getMsgSerNum( f, i ) );
+ }
+ return;
+ }
+ if ( !mAddedMsgs.empty() ) {
+ std::swap( mAddedMsgs, mPendingMsgs );
+ mState = s_processing;
+ return;
+ }
+ for ( std::set<KMFolder*>::const_iterator first = mOpenedFolders.begin(), past = \
mOpenedFolders.end(); + first != past;
+ ++first ) {
+ ( *first )->close();
+ }
+ mOpenedFolders.clear();
+ mState = s_idle;
+ mTimer->stop();
+}
+
+void KMMsgIndex::continueCreation() {
+ kdDebug( 5006 ) << "KMMsgIndex::continueCreation()" << endl;
+ create();
+ unsigned count = mIndex->ndocs();
+ mExisting.clear();
+ mExisting.reserve( count );
+ for ( unsigned i = 0; i != count; ++i ) {
+ mExisting.push_back( std::atoi( mIndex->lookup_docname( i ).c_str() ) );
+ }
+ std::sort( mExisting.begin(), mExisting.end() );
+}
+
+void KMMsgIndex::create() {
+ kdDebug( 5006 ) << "KMMsgIndex::create()" << endl;
+ mState = s_creating;
+ if ( !mIndex ) mIndex = indexlib::create( mIndexPath ).release();
+ Q_ASSERT( mIndex );
+ QValueStack<KMFolderDir*> folders;
+ folders.push(&(kmkernel->folderMgr()->dir()));
+ folders.push(&(kmkernel->dimapFolderMgr()->dir()));
+ while ( !folders.empty() ) {
+ KMFolderDir *dir = folders.pop();
+ for(KMFolderNode *child = dir->first(); child; child = dir->next()) {
+ if ( child->isDir() )
+ folders.push((KMFolderDir*)child);
+ else
+ mPendingFolders.push_back( (KMFolder*)child );
+ }
+ }
+ mTimer->start( 4000 ); // wait a couple of seconds before starting up...
+ mSlowDown = true;
+}
+
+bool KMMsgIndex::startQuery( KMSearch* s ) {
+ kdDebug( 5006 ) << "KMMsgIndex::startQuery( . )" << endl;
+ if ( mState == s_creating ) return false;
+ if ( !isIndexed( s->root() ) || !canHandleQuery( s->searchPattern() ) ) return \
false; +
+ kdDebug( 5006 ) << "KMMsgIndex::startQuery( . ) starting query" << endl;
+ Search* search = new Search( s );
+ connect( search, SIGNAL( finished( bool ) ), s, SIGNAL( finished( bool ) ) );
+ connect( search, SIGNAL( finished( bool ) ), s, SLOT( indexFinished() ) );
+ connect( search, SIGNAL( destroyed( QObject* ) ), SLOT( removeSearch( QObject* ) ) \
); + connect( search, SIGNAL( found( Q_UINT32 ) ), s, SIGNAL( found( Q_UINT32 ) ) );
+ mSearches.push_back( search );
+ return true;
+}
+
+
+//void KMMsgIndex::startSync() {
+// switch ( mSyncState ) {
+// case ss_none:
+// mIndex->start_sync();
+// mSyncState = ss_started;
+// mSyncTimer.start( 4000, true );
+// break;
+// case ss_started:
+// mIndex->sync_now();
+// mSyncState = ss_synced;
+// mLockFile.unlock();
+// break;
+// }
+//}
+//
+//void KMMsgIndex::finishSync() {
+//
+//}
+
+void KMMsgIndex::removeSearch( QObject* destroyed ) {
+ mSearches.erase( std::find( mSearches.begin(), mSearches.end(), destroyed ) );
+}
+
+bool KMMsgIndex::isIndexed( const KMFolder* folder ) const {
+ if ( mState == s_creating || mState == s_willcreate ) return false;
+ const KMFolderMgr* manager = folder->parent()->manager();
+ return manager == kmkernel->folderMgr() || manager == kmkernel->dimapFolderMgr();
+}
+
+bool KMMsgIndex::stopQuery( KMSearch* s ) {
+ kdDebug( 5006 ) << "KMMsgIndex::stopQuery( . )" << endl;
+ for ( std::vector<Search*>::iterator iter = mSearches.begin(), past = \
mSearches.end(); iter != past; ++iter ) { + if ( ( *iter )->search() == s ) {
+ delete *iter;
+ mSearches.erase( iter );
+ return true;
+ }
+ }
+ return false;
+}
+
+std::vector<Q_UINT32> KMMsgIndex::simpleSearch( QString s ) const {
+ kdDebug( 5006 ) << "KMMsgIndex::simpleSearch( -" << s.latin1() << "- )" << endl;
+ std::vector<Q_UINT32> res;
+ assert( mIndex );
+ std::vector<unsigned> residx = mIndex->search( s.latin1() )->list();
+ res.reserve( residx.size() );
+ for ( std::vector<unsigned>::const_iterator first = residx.begin(), past = \
residx.end();first != past; ++first ) { + res.push_back( std::atoi( \
mIndex->lookup_docname( *first ).c_str() ) ); + }
+ return res;
+}
+
+bool KMMsgIndex::canHandleQuery( const KMSearchPattern* pat ) const {
+ kdDebug( 5006 ) << "KMMsgIndex::canHandleQuery( . )" << endl;
+ for ( KMSearchRule* rule = pat->first(); rule; rule = pat->next() ) {
+ if ( !rule->field().isEmpty() && !rule->contents().isEmpty() &&
+ rule->function() == KMSearchRule::FuncContains &&
+ rule->field() == "<body>" ) return true;
+ }
+ return false;
+}
+
+void KMMsgIndex::slotAddMessage( KMFolder* folder, Q_UINT32 serNum ) {
+ kdDebug( 5006 ) << "KMMsgIndex::slotAddMessage( . , " << serNum << " )" << endl;
+
+ if ( mState == s_creating ) mAddedMsgs.push_back( serNum );
+ else mPendingMsgs.push_back( serNum );
+
+ if ( mState == s_idle ) mState = s_processing;
+ scheduleAction();
+}
+
+void KMMsgIndex::slotRemoveMessage( KMFolder* folder, Q_UINT32 serNum ) {
+ kdDebug( 5006 ) << "KMMsgIndex::slotRemoveMessage( . , " << serNum << " )" << endl;
+ if ( mState == s_idle ) mState = s_processing;
+ mRemovedMsgs.push_back( serNum );
+ scheduleAction();
+}
+
+void KMMsgIndex::scheduleAction() {
+ if ( mState == s_willcreate || !mIndex ) return;
+ if ( !mSlowDown ) mTimer->start( 0 );
+}
+
+void KMMsgIndex::removeMessage( Q_UINT32 serNum ) {
+ mIndex->remove_doc( QString::number( serNum ).latin1() );
+ ++mMaintenanceCount;
+ if ( mMaintenanceCount > MaintenanceLimit && mRemovedMsgs.empty() ) {
+ QTimer::singleShot( 100, this, SLOT( cleanUp() ) );
+ }
+}
+
+QString KMMsgIndex::defaultPath() {
+ return KMKernel::localDataPath() + "text-index";
+}
+
+bool KMMsgIndex::creating() const {
+ return !mPendingMsgs.empty() || !mPendingFolders.empty();
+}
+
+KMMsgIndex::Search::Search( KMSearch* s ):
+ mSearch( s ),
+ mTimer( new QTimer( this ) ),
+ mState( s_starting ),
+ mResidual( new KMSearchPattern ) {
+ connect( mTimer, SIGNAL( timeout() ), SLOT( act() ) );
+ mTimer->start( 0 );
+}
+
+KMMsgIndex::Search::~Search() {
+ delete mTimer;
+}
+
+void KMMsgIndex::Search::act() {
+ switch ( mState ) {
+ case s_starting: {
+ KMSearchPattern* pat = mSearch->searchPattern();
+ QString terms;
+ for ( KMSearchRule* rule = pat->first(); rule; rule = pat->next() ) {
+ Q_ASSERT( rule->function() == KMSearchRule::FuncContains );
+ terms += QString::fromLatin1( " %1 " ).arg( rule->contents() );
+ }
+
+ mValues = kmkernel->msgIndex()->simpleSearch( terms );
+ mState = s_emitting;
+ break;
+ }
+ case s_emitstopped:
+ mTimer->start( 0 );
+ mState = s_emitting;
+ // fall throu
+ case s_emitting:
+ if ( kapp->hasPendingEvents() ) {
+ //nah, some other time..
+ mTimer->start( 250 );
+ mState = s_emitstopped;
+ return;
+ }
+ for ( int i = 0; i != 16 && !mValues.empty(); ++i ) {
+ KMFolder* folder;
+ int index;
+ KMMsgDict::instance()->getLocation( mValues.back(), &folder, &index );
+ if ( folder &&
+ mSearch->inScope( folder ) &&
+ ( !mResidual || mResidual->matches( mValues.back() ) ) ) {
+
+ emit found( mValues.back() );
+ }
+ mValues.pop_back();
+ }
+ if ( mValues.empty() ) {
+ emit finished( true );
+ mState = s_done;
+ mTimer->stop();
+ delete this;
+ }
+ break;
+ default:
+ Q_ASSERT( 0 );
+ }
+}
+#include "index.moc"
+
_______________________________________________
KMail developers mailing list
KMail-devel@kde.org
https://mail.kde.org/mailman/listinfo/kmail-devel
[prev in list] [next in list] [prev in thread] [next in thread]
Configure |
About |
News |
Add a list |
Sponsored by KoreLogic