[prev in list] [next in list] [prev in thread] [next in thread]
List: kde-commits
Subject: branches/KDE/4.6/kdelibs/nepomuk/query
From: Sebastian Trueg <sebastian () trueg ! de>
Date: 2011-01-20 16:02:49
Message-ID: 20110120160249.862E3AC8B8 () svn ! kde ! org
[Download RAW message or body]
SVN commit 1215973 by trueg:
Backport:
* Fixed the handling of quotes and keywords such as "AND", "OR", and "NOT" in \
LiteralTerm. Now correct bif:contains or regex filters are created for values.
* Made the query parser merge LiteralTerms into a single one to improve query \
performance. While merging two LiteralTerms into one does not yield the exact same \
query (when merged both literal tokens need to appear in the same property value \
while with separate LiteralTerms the tokens can appear in different properties) it \
should cover close to all typical use cases while increasing the performance \
significantly and getting rid of the nasty "Virtuoso goes crazy when I use KRunner" \
bug.
CCBUG: 246678
M +91 -48 literalterm.cpp
M +40 -5 queryparser.cpp
--- branches/KDE/4.6/kdelibs/nepomuk/query/literalterm.cpp #1215972:1215973
@@ -73,41 +73,6 @@
namespace {
-QString prepareQueryText( const QString& text )
-{
- //
- // we try to be a little smart about creating the query text
- // by following a few simple rules:
- //
- // 1. enclose everything in quotes to be safe
- // 2. quotes in search terms are not handled. replace them with spaces
- // 3. replace double quotes with single quotes
- // [4. wildcards can only be used if they are preceeded by at least 4 chars]
- //
-
- QString s = text.simplified();
- if( s.isEmpty() )
- return s;
-
- // strip quotes
- if( s[0] == '"' || s[0] == '\'' ) {
- s = s.mid(1);
- }
- if( !s.isEmpty() &&
- ( s[s.length()-1] == '"' || s[s.length()-1] == '\'' ) ) {
- s.truncate(s.length()-1);
- }
-
- // replace quotes with spaces
- s.replace( '"', ' ' );
- s.replace( '\'', ' ' );
-
- // add quotes
- s = '\'' + s + '\'';
-
- return s;
-}
-
QString prepareRegexText( const QString& text )
{
QString filterRxStr = QRegExp::escape( text );
@@ -121,29 +86,107 @@
QString Nepomuk::Query::LiteralTermPrivate::createContainsPattern( const QString& \
varName, const QString& text, Nepomuk::Query::QueryBuilderData* qbd ) {
- const int i = text.indexOf( QRegExp(QLatin1String("[\\?\\*]")) );
+ // each token with a negation flag
+ QList<QPair<QString, bool> > containsTokens;
+ QList<QPair<QString, bool> > regexTokens;
+ // we only support AND xor OR, not both at the same time
+ bool isUnion = false;
+
+ // gather all the tokens
+ bool inQuotes = false;
+ QString currentToken;
+ bool nextIsNegated = false;
+ int i = 0;
+ while( i < text.length() ) {
+ const QChar& c = text[i];
+ bool tokenEnd = false;
+
+ if( c == QChar('"') || c == QChar('\'') ) {
+ inQuotes = !inQuotes;
+ tokenEnd = !inQuotes;
+ }
+ else if( c.isSpace() && !inQuotes ) {
+ tokenEnd = true;
+ }
+ else {
+ currentToken.append(c);
+ }
+
+ if( i == text.count()-1 ) {
+ tokenEnd = true;
+ }
+
+ if( tokenEnd && !currentToken.isEmpty() ) {
//
+ // Handle the three special tokens supported in Virtuoso's full text \
search engine we support (there is also "near" which we do not handle yet) + \
// + if( currentToken.toLower() == QLatin1String("and") ) {
+ isUnion = false;
+ }
+ else if( currentToken.toLower() == QLatin1String("or") ) {
+ isUnion = true;
+ }
+ else if( currentToken.toLower() == QLatin1String("not") ) {
+ nextIsNegated = true;
+ }
+ else {
+ QPair<QString, bool> currentTokenPair = qMakePair( currentToken, \
nextIsNegated ); +
+ //
// Virtuoso needs four leading chars when using wildcards. Thus, if there is \
less (this includes 0) we fall back to the slower regex filter //
- if( i < 0 || i > 3 ) {
- const QString finalText = prepareQueryText( text );
+ const QStringList subTokens = currentToken.split( QLatin1Char(' '), \
QString::SkipEmptyParts ); + bool needsRegex = false;
+ Q_FOREACH( const QString& subToken, subTokens ) {
+ const int i = subToken.indexOf( \
QRegExp(QLatin1String("[\\?\\*]")) ); + if( i >= 0 && i < 4 ) {
+ needsRegex = true;
+ break;
+ }
+ }
+ if( !needsRegex ) {
+ containsTokens << currentTokenPair;
+ }
+ else {
+ regexTokens << currentTokenPair;
+ }
+ }
- QString scoringPattern;
- if( qbd->query()->m_fullTextScoringEnabled ) {
- scoringPattern = QString::fromLatin1("OPTION (score %1) \
").arg(qbd->createScoringVariable()); + nextIsNegated = false;
+ currentToken.clear();
}
- qbd->addFullTextSearchTerm( varName, finalText );
- return QString::fromLatin1( "%1 bif:contains \"%2\" %3. " )
+ ++i;
+ }
+
+ // convert the tokens into SPARQL filters
+ QStringList filters;
+ QStringList containsFilterTokens;
+ for( int i = 0; i < containsTokens.count(); ++i ) {
+ QString containsFilterToken;
+ if( containsTokens[i].second )
+ containsFilterToken += QLatin1String("NOT ");
+ containsFilterToken += \
QString::fromLatin1("'%1'").arg(containsTokens[i].first); + \
containsFilterTokens << containsFilterToken; + }
+ if( !containsFilterTokens.isEmpty() ) {
+ filters << QString::fromLatin1("bif:contains(%1, \"%2\")")
.arg( varName,
- finalText,
- scoringPattern );
+ containsFilterTokens.join( isUnion ? QLatin1String(" OR ") \
: QLatin1String(" AND ")) ); }
- else {
- return QString::fromLatin1( "FILTER(REGEX(%1, \"%2\")) . " )
- .arg( varName, prepareRegexText(text) );
+ QStringList regexFilters;
+ for( int i = 0; i < regexTokens.count(); ++i ) {
+ QString regexFilter;
+ if( regexTokens[i].second )
+ regexFilter += QLatin1Char('!');
+ regexFilter += QString::fromLatin1( "REGEX(%1, \"%2\")" )
+ .arg( varName,
+ prepareRegexText(regexTokens[i].first) );
+ filters << regexFilter;
}
+
+ return QString( QLatin1String("FILTER(") + filters.join( isUnion ? \
QLatin1String(" || ") : QLatin1String(" && ") ) + QLatin1String(") . ") ); }
--- branches/KDE/4.6/kdelibs/nepomuk/query/queryparser.cpp #1215972:1215973
@@ -130,9 +130,14 @@
}
}
- Soprano::LiteralValue createLiteral( const QString& s_, bool globbing ) {
- bool hadQuotes = false;
- QString s = stripQuotes( s_, &hadQuotes );
+ Soprano::LiteralValue createLiteral( const QString& s, bool globbing ) {
+ // no globbing if we have quotes or if there already is a wildcard
+ if ( s[0] == QLatin1Char('\'') ||
+ s[0] == QLatin1Char('\"') ) {
+ return s;
+ }
+
+ // at this point we should have a string without spaces in it
bool b = false;
int i = s.toInt( &b );
if ( b )
@@ -144,7 +149,7 @@
//
// we can only do query term globbing for strings longer than 3 chars
//
- if( !hadQuotes && globbing && s.length() > 3 && !s.endsWith('*') && \
!s.endsWith('?') ) + if( globbing && s.length() > 3 && !s.endsWith('*') && \
!s.endsWith('?') ) return QString(s + '*');
else
return s;
@@ -250,6 +255,36 @@
\
Nepomuk::Query::ComparisonTerm::Regexp ); }
+ /**
+ * Merging literal terms is an optimization which is based on the assumption \
that most + * users want to search for the full text terms they enter in the \
value of the same + * property.
+ * Since merging two literals "foo" and "bar" into one term "foo AND bar" \
effectively + * changes the result set (the former allows that "foo" occurs in a \
property value + * different from "bar" while the latter forces them to occur in \
the same.) + * But the resulting query is much faster.
+ */
+ Nepomuk::Query::Term mergeLiteralTerms( const Nepomuk::Query::Term& term )
+ {
+ if( term.isAndTerm() ) {
+ AndTerm mergedTerm;
+ QStringList fullTextTerms;
+ Q_FOREACH( const Term& st, term.toAndTerm().subTerms() ) {
+ if( st.isLiteralTerm() ) {
+ fullTextTerms << st.toLiteralTerm().value().toString();
+ }
+ else {
+ mergedTerm.addSubTerm( st );
+ }
+ }
+ mergedTerm.addSubTerm( LiteralTerm( fullTextTerms.join( \
QString::fromLatin1(" AND ") ) ) ); + return mergedTerm.optimized();
+ }
+ else {
+ return term;
+ }
+ }
+
#ifndef Q_CC_MSVC
#warning Make the parser handle different data, time, and datetime encodings as well \
as suffixes like MB or GB #endif
@@ -620,7 +655,7 @@
final.setTerm( t );
}
- final.setTerm( resolveFields( final.term(), this ) );
+ final.setTerm( mergeLiteralTerms( resolveFields( final.term(), this ) ) );
return final;
}
[prev in list] [next in list] [prev in thread] [next in thread]
Configure |
About |
News |
Add a list |
Sponsored by KoreLogic