--f46d04426bd6dea2b704bc8ed81f Content-Type: text/plain; charset=ISO-8859-1 On Sun, Apr 1, 2012 at 12:30 AM, Sebastian Trueg wrote: > Git commit 9476df60687ecb133ebc9dd2f9c893edc1ab097a by Sebastian Trueg. > Committed on 31/03/2012 at 20:46. > Pushed by trueg into branch 'feature/virtuosoInference'. > > New resource visibility handling. > > Until now the query api would use the nao:userVisible values stored for > each resource to determine if a resource should be returned. > This requires a lot of nao:userVisible value maintenance and makes for > slower queries. > The new solution is much simpler: only types have nao:userVisible values > and the query api simply checks if the result candidate has a type > which is visible. > In contrast to the old one the new solution does not allow to hide a > sub-class of a visible class. But since we never did that anyway and > the resulting queries are faster and it allows us to remove a lot > maintenance code (which eats CPU cycles and disk space) the change is > worth it. > I'm not sure about this. I kinda liked being able to hide certain resources from "normal" view. I always thought it could be used to hold Artem's decision resources. But then it wasn't really used. > > M +8 -13 libnepomukcore/query/query.cpp > M +52 -0 services/storage/ontologyloader.cpp > M +2 -1 services/storage/ontologyloader.h > > > http://commits.kde.org/nepomuk-core/9476df60687ecb133ebc9dd2f9c893edc1ab097a > > diff --git a/libnepomukcore/query/query.cpp > b/libnepomukcore/query/query.cpp > index 943705d..6e242f4 100644 > --- a/libnepomukcore/query/query.cpp > +++ b/libnepomukcore/query/query.cpp > @@ -1,6 +1,6 @@ > /* > This file is part of the Nepomuk KDE project. > - Copyright (C) 2008-2010 Sebastian Trueg > + Copyright (C) 2008-2012 Sebastian Trueg > > This library is free software; you can redistribute it and/or > modify it under the terms of the GNU Library General Public > @@ -450,22 +450,17 @@ QString Nepomuk::Query::Query::toSparqlQuery( > SparqlFlags sparqlFlags ) const > QueryBuilderData qbd( d.constData(), sparqlFlags ); > > > - // We restrict results to user visible types only. There is no need > for this with file queries as normally all files are visible. > // > - // Here we use an optimizations: > - // The storage service creates "nao:userVisible 1" entries for all > visible resources. This contains two optimizations: > - // - We use an integer instead of a boolean because Virtuoso does not > support booleans and, thus, Soprano converts > - // booleans into a fake type which is stored as a string. This > makes comparision much slower. > - // - Instead of using crappy inference via "?r a ?t . ?t > nao:userVisible true" we can directly check the resources. > - // - Instead of restricting the visibility to 1 we trick the Virtuoso > query optimizer so it won't try to use the visibility > - // as priority. We do this with a filter that does not test for > equality. Using "?r nao:userVisible 1" has a drastic > - // performance impact which the filter has not. > + // We restrict results to user visible types only. There is no need > for this with file queries as normally all files are visible. > + // We need the additional type pattern to ensure that the variable ?r > is actually available to the filter (in case there is only > + // UNIONs before. > // > QString userVisibilityRestriction; > if( !(queryFlags()&NoResultRestrictions) && !d->m_isFileQuery ) { > - userVisibilityRestriction = QString::fromLatin1("?r %1 %2 . > FILTER(%2>0) . ") > - > .arg(Soprano::Node::resourceToN3(Soprano::Vocabulary::NAO::userVisible()), > - qbd.uniqueVarName()); > + userVisibilityRestriction = QString::fromLatin1("?r a %1 . FILTER > EXISTS { ?r a [ %2 %3 ] . } . ") > + .arg(qbd.uniqueVarName(), > + > Soprano::Node::resourceToN3(Soprano::Vocabulary::NAO::userVisible()), > + > Soprano::Node::literalToN3(Soprano::LiteralValue(true))); > } > > > diff --git a/services/storage/ontologyloader.cpp > b/services/storage/ontologyloader.cpp > index c06a297..a09665b 100644 > --- a/services/storage/ontologyloader.cpp > +++ b/services/storage/ontologyloader.cpp > @@ -27,6 +27,7 @@ > #include > #include > #include > +#include > > #include > #include > @@ -42,14 +43,23 @@ > #include > #include > > +#include > +#include > +#include > > using namespace Soprano; > +using namespace Soprano::Vocabulary; > + > +namespace { > + const char* s_typeVisibilityGraph = "nepomuk:/ctx/typevisibility"; > +} > > class Nepomuk::OntologyLoader::Private > { > public: > Private( OntologyLoader* p ) > : forceOntologyUpdate( false ), > + someOntologyUpdated( false ), > q( p ) { > } > > @@ -59,6 +69,9 @@ public: > bool forceOntologyUpdate; > QStringList desktopFilesToUpdate; > > + // true if at least one ontology has been updated > + bool someOntologyUpdated; > + > void updateOntology( const QString& filename ); > > private: > @@ -110,6 +123,8 @@ void Nepomuk::OntologyLoader::Private::updateOntology( > const QString& filename ) > } > > if( update ) { > + someOntologyUpdated = true; > + > QString mimeType = df.readEntry( "MimeType", QString() ); > > const Soprano::Parser* parser > @@ -179,6 +194,7 @@ Nepomuk::OntologyLoader::~OntologyLoader() > > void Nepomuk::OntologyLoader::updateLocalOntologies() > { > + d->someOntologyUpdated = false; > d->desktopFilesToUpdate = KGlobal::dirs()->findAllResources( > "xdgdata-ontology", "*.ontology", > KStandardDirs::Recursive|KStandardDirs::NoDuplicates ); > if(d->desktopFilesToUpdate.isEmpty()) > kError() << "No ontology files found! Make sure the > shared-desktop-ontologies project is installed and XDG_DATA_DIRS is set > properly."; > @@ -201,6 +217,16 @@ void Nepomuk::OntologyLoader::updateNextOntology() > else { > d->forceOntologyUpdate = false; > d->updateTimer.stop(); > + > + // update graph visibility if something has changed or if we > never did it > + const QUrl visibilityGraph = > QUrl::fromEncoded(s_typeVisibilityGraph); > + if(d->someOntologyUpdated || > + !d->model->executeQuery(QString::fromLatin1("ask where { graph > %1 { ?s ?p ?o . } }") > + > .arg(Soprano::Node::resourceToN3(visibilityGraph)), > + > Soprano::Query::QueryLanguageSparql).boolValue()) { > + updateTypeVisibility(); > + } > + > emit ontologyLoadingFinished(this); > } > } > @@ -229,6 +255,7 @@ void > Nepomuk::OntologyLoader::slotGraphRetrieverResult( KJob* job ) > // TODO: find a way to check if the imported version of the > ontology > // is newer than the already installed one > if ( d->model->updateOntology( graphRetriever->statements(), > QUrl()/*graphRetriever->url()*/ ) ) { > + updateTypeVisibility(); > emit ontologyUpdated( QString::fromAscii( > graphRetriever->url().toEncoded() ) ); > } > else { > @@ -237,4 +264,29 @@ void > Nepomuk::OntologyLoader::slotGraphRetrieverResult( KJob* job ) > } > } > > +void Nepomuk::OntologyLoader::updateTypeVisibility() > +{ > + const QUrl visibilityGraph = QUrl::fromEncoded(s_typeVisibilityGraph); > + > + // 1. remove all visibility values we added ourselves > + d->model->removeContext(visibilityGraph); > + > + // 2. make rdfs:Resource non-visible (this is required since with KDE > 4.9 we introduced a new > + // way of visibility handling which relies on types alone rather > than visibility values on > + // resources. Any visible type will make all sub-types visible, > too. If rdfs:Resource were > + // visible everything would be. > + d->model->removeAllStatements(RDFS::Resource(), NAO::userVisible(), > Soprano::Node()); > + > + // 3. Set each type visible which is not rdfs:Resource and does not > have a non-visible parent > + d->model->executeQuery(QString::fromLatin1("insert into %1 { " > + "?t %2 'true'^^%3 . " > + "} where { " > + "?t a rdfs:Class . " > + "filter not exists { ?tt > %2 'false'^^%3 . ?t rdfs:subClassOf ?tt . } }") > + > .arg(Soprano::Node::resourceToN3(visibilityGraph), > + > Soprano::Node::resourceToN3(NAO::userVisible()), > + > Soprano::Node::resourceToN3(XMLSchema::boolean())), > + Soprano::Query::QueryLanguageSparql); > +} > + > #include "ontologyloader.moc" > diff --git a/services/storage/ontologyloader.h > b/services/storage/ontologyloader.h > index 4e0ce26..d714249 100644 > --- a/services/storage/ontologyloader.h > +++ b/services/storage/ontologyloader.h > @@ -1,5 +1,5 @@ > /* This file is part of the KDE Project > - Copyright (c) 2007-2010 Sebastian Trueg > + Copyright (c) 2007-2012 Sebastian Trueg > > This library is free software; you can redistribute it and/or > modify it under the terms of the GNU Library General Public > @@ -91,6 +91,7 @@ namespace Nepomuk { > // a little async updating > void updateNextOntology(); > void slotGraphRetrieverResult( KJob* job ); > + void updateTypeVisibility(); > > private: > class Private; > > --f46d04426bd6dea2b704bc8ed81f Content-Type: text/html; charset=ISO-8859-1 Content-Transfer-Encoding: quoted-printable

On Sun, Apr 1, 2012 at 12:30 AM, Sebasti= an Trueg <trueg@kde.o= rg> wrote:
Git commit 9476df60687ecb133ebc9dd2f9c893edc1ab097a by Sebastian Trueg.
Committed on 31/03/2012 at 20:46.
Pushed by trueg into branch 'feature/virtuosoInference'.

New resource visibility handling.

Until now the query api would use the nao:userVisible values stored for
each resource to determine if a resource should be returned.
This requires a lot of nao:userVisible value maintenance and makes for
slower queries.
The new solution is much simpler: only types have nao:userVisible values and the query api simply checks if the result candidate has a type
which is visible.
In contrast to the old one the new solution does not allow to hide a
sub-class of a visible class. But since we never did that anyway and
the resulting queries are faster and it allows us to remove a lot
maintenance code (which eats CPU cycles and disk space) the change is
worth it.

I'm not sure about this. I kinda lik= ed being able to hide certain resources from "normal" view. I alw= ays thought it could be used to hold Artem's decision resources. But th= en it wasn't really used.

M =A0+8 =A0 =A0-13 =A0 libnepomukcore/query/query.cpp
M =A0+52 =A0 -0 =A0 =A0services/storage/ontologyloader.cpp
M =A0+2 =A0 =A0-1 =A0 =A0services/storage/ontologyloader.h

http://commits.kde.org/nepomuk-core/9476df6= 0687ecb133ebc9dd2f9c893edc1ab097a

diff --git a/libnepomukcore/query/query.cpp b/libnepomukcore/query/query.cp= p
index 943705d..6e242f4 100644
--- a/libnepomukcore/query/query.cpp
+++ b/libnepomukcore/query/query.cpp
@@ -1,6 +1,6 @@
=A0/*
=A0 =A0This file is part of the Nepomuk KDE project.
- =A0 Copyright (C) 2008-2010 Sebastian Trueg <trueg@kde.org>
+ =A0 Copyright (C) 2008-2012 Sebastian Trueg <trueg@kde.org>

=A0 =A0This library is free software; you can redistribute it and/or
=A0 =A0modify it under the terms of the GNU Library General Public
@@ -450,22 +450,17 @@ QString Nepomuk::Query::Query::toSparqlQuery( SparqlF= lags sparqlFlags ) const
=A0 =A0 QueryBuilderData qbd( d.constData(), sparqlFlags );


- =A0 =A0// We restrict results to user visible types only. There is no nee= d for this with file queries as normally all files are visible.
=A0 =A0 //
- =A0 =A0// Here we use an optimizations:
- =A0 =A0// The storage service creates "nao:userVisible 1" entri= es for all visible resources. This contains two optimizations:
- =A0 =A0// - We use an integer instead of a boolean because Virtuoso does = not support booleans and, thus, Soprano converts
- =A0 =A0// =A0 booleans into a fake type which is stored as a string. This= makes comparision much slower.
- =A0 =A0// - Instead of using crappy inference via "?r a ?t . ?t nao:= userVisible true" we can directly check the resources.
- =A0 =A0// - Instead of restricting the visibility to 1 we trick the Virtu= oso query optimizer so it won't try to use the visibility
- =A0 =A0// =A0 as priority. We do this with a filter that does not test fo= r equality. Using "?r nao:userVisible 1" has a drastic
- =A0 =A0// =A0 performance impact which the filter has not.
+ =A0 =A0// We restrict results to user visible types only. There is no nee= d for this with file queries as normally all files are visible.
+ =A0 =A0// We need the additional type pattern to ensure that the variable= ?r is actually available to the filter (in case there is only
+ =A0 =A0// UNIONs before.
=A0 =A0 //
=A0 =A0 QString userVisibilityRestriction;
=A0 =A0 if( !(queryFlags()&NoResultRestrictions) && !d->m_i= sFileQuery ) {
- =A0 =A0 =A0 =A0userVisibilityRestriction =3D QString::fromLatin1("?r= %1 %2 . FILTER(%2>0) . ")
- =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0.arg(Soprano::Node::resourceToN3(Soprano::= Vocabulary::NAO::userVisible()),
- =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 qbd.uniqueVarName());
+ =A0 =A0 =A0 =A0userVisibilityRestriction =3D QString::fromLatin1("?r= a %1 . FILTER EXISTS { ?r a [ %2 %3 ] . } . ")
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0.a= rg(qbd.uniqueVarName(),
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 = =A0 =A0 Soprano::Node::resourceToN3(Soprano::Vocabulary::NAO::userVisible()= ),
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 = =A0 =A0 Soprano::Node::literalToN3(Soprano::LiteralValue(true)));
=A0 =A0 }


diff --git a/services/storage/ontologyloader.cpp b/services/storage/ontolog= yloader.cpp
index c06a297..a09665b 100644
--- a/services/storage/ontologyloader.cpp
+++ b/services/storage/ontologyloader.cpp
@@ -27,6 +27,7 @@
=A0#include <Soprano/PluginManager>
=A0#include <Soprano/StatementIterator>
=A0#include <Soprano/Parser>
+#include <Soprano/QueryResultIterator>

=A0#include <KConfig>
=A0#include <KConfigGroup>
@@ -42,14 +43,23 @@
=A0#include <kpluginfactory.h>
=A0#include <kpluginloader.h>

+#include <Soprano/Vocabulary/RDFS>
+#include <Soprano/Vocabulary/NAO>
+#include <Soprano/Vocabulary/XMLSchema>

=A0using namespace Soprano;
+using namespace Soprano::Vocabulary;
+
+namespace {
+ =A0 =A0const char* s_typeVisibilityGraph =3D "nepomuk:/ctx/typevisib= ility";
+}

=A0class Nepomuk::OntologyLoader::Private
=A0{
=A0public:
=A0 =A0 Private( OntologyLoader* p )
=A0 =A0 =A0 =A0 : forceOntologyUpdate( false ),
+ =A0 =A0 =A0 =A0 =A0someOntologyUpdated( false ),
=A0 =A0 =A0 =A0 =A0 q( p ) {
=A0 =A0 }

@@ -59,6 +69,9 @@ public:
=A0 =A0 bool forceOntologyUpdate;
=A0 =A0 QStringList desktopFilesToUpdate;

+ =A0 =A0// true if at least one ontology has been updated
+ =A0 =A0bool someOntologyUpdated;
+
=A0 =A0 void updateOntology( const QString& filename );

=A0private:
@@ -110,6 +123,8 @@ void Nepomuk::OntologyLoader::Private::updateOntology( = const QString& filename )
=A0 =A0 }

=A0 =A0 if( update ) {
+ =A0 =A0 =A0 =A0someOntologyUpdated =3D true;
+
=A0 =A0 =A0 =A0 QString mimeType =3D df.readEntry( "MimeType", Q= String() );

=A0 =A0 =A0 =A0 const Soprano::Parser* parser
@@ -179,6 +194,7 @@ Nepomuk::OntologyLoader::~OntologyLoader()

=A0void Nepomuk::OntologyLoader::updateLocalOntologies()
=A0{
+ =A0 =A0d->someOntologyUpdated =3D false;
=A0 =A0 d->desktopFilesToUpdate =3D KGlobal::dirs()->findAllResource= s( "xdgdata-ontology", "*.ontology", KStandardDirs::Rec= ursive|KStandardDirs::NoDuplicates );
=A0 =A0 if(d->desktopFilesToUpdate.isEmpty())
=A0 =A0 =A0 =A0 kError() << "No ontology files found! Make sure= the shared-desktop-ontologies project is installed and XDG_DATA_DIRS is se= t properly.";
@@ -201,6 +217,16 @@ void Nepomuk::OntologyLoader::updateNextOntology()
=A0 =A0 else {
=A0 =A0 =A0 =A0 d->forceOntologyUpdate =3D false;
=A0 =A0 =A0 =A0 d->updateTimer.stop();
+
+ =A0 =A0 =A0 =A0// update graph visibility if something has changed or if = we never did it
+ =A0 =A0 =A0 =A0const QUrl visibilityGraph =3D QUrl::fromEncoded(s_typeVis= ibilityGraph);
+ =A0 =A0 =A0 =A0if(d->someOntologyUpdated ||
+ =A0 =A0 =A0 =A0 =A0 !d->model->executeQuery(QString::fromLatin1(&qu= ot;ask where { graph %1 { ?s ?p ?o . } }")
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 .arg(= Soprano::Node::resourceToN3(visibilityGraph)),
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 Sopra= no::Query::QueryLanguageSparql).boolValue()) {
+ =A0 =A0 =A0 =A0 =A0 =A0updateTypeVisibility();
+ =A0 =A0 =A0 =A0}
+
=A0 =A0 =A0 =A0 emit ontologyLoadingFinished(this);
=A0 =A0 }
=A0}
@@ -229,6 +255,7 @@ void Nepomuk::OntologyLoader::slotGraphRetrieverResult(= KJob* job )
=A0 =A0 =A0 =A0 // TODO: find a way to check if the imported version of th= e ontology
=A0 =A0 =A0 =A0 // is newer than the already installed one
=A0 =A0 =A0 =A0 if ( d->model->updateOntology( graphRetriever->st= atements(), QUrl()/*graphRetriever->url()*/ ) ) {
+ =A0 =A0 =A0 =A0 =A0 =A0updateTypeVisibility();
=A0 =A0 =A0 =A0 =A0 =A0 emit ontologyUpdated( QString::fromAscii( graphRet= riever->url().toEncoded() ) );
=A0 =A0 =A0 =A0 }
=A0 =A0 =A0 =A0 else {
@@ -237,4 +264,29 @@ void Nepomuk::OntologyLoader::slotGraphRetrieverResult= ( KJob* job )
=A0 =A0 }
=A0}

+void Nepomuk::OntologyLoader::updateTypeVisibility()
+{
+ =A0 =A0const QUrl visibilityGraph =3D QUrl::fromEncoded(s_typeVisibilityG= raph);
+
+ =A0 =A0// 1. remove all visibility values we added ourselves
+ =A0 =A0d->model->removeContext(visibilityGraph);
+
+ =A0 =A0// 2. make rdfs:Resource non-visible (this is required since with = KDE 4.9 we introduced a new
+ =A0 =A0// =A0 =A0way of visibility handling which relies on types alone r= ather than visibility values on
+ =A0 =A0// =A0 =A0resources. Any visible type will make all sub-types visi= ble, too. If rdfs:Resource were
+ =A0 =A0// =A0 =A0visible everything would be.
+ =A0 =A0d->model->removeAllStatements(RDFS::Resource(), NAO::userVis= ible(), Soprano::Node());
+
+ =A0 =A0// 3. Set each type visible which is not rdfs:Resource and does no= t have a non-visible parent
+ =A0 =A0d->model->executeQuery(QString::fromLatin1("insert into= %1 { "
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 = =A0 =A0 =A0 =A0 =A0 "?t %2 'true'^^%3 . "
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 = =A0 =A0 =A0 =A0 =A0 "} where { "
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 = =A0 =A0 =A0 =A0 =A0 "?t a rdfs:Class . "
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 = =A0 =A0 =A0 =A0 =A0 "filter not exists { ?tt %2 'false'^^%3 . = =A0?t rdfs:subClassOf ?tt . } }")
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 .arg(Soprano::Node::r= esourceToN3(visibilityGraph),
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0Soprano::N= ode::resourceToN3(NAO::userVisible()),
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0Soprano::N= ode::resourceToN3(XMLSchema::boolean())),
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 Soprano::Query::Query= LanguageSparql);
+}
+
=A0#include "ontologyloader.moc"
diff --git a/services/storage/ontologyloader.h b/services/storage/ontologyl= oader.h
index 4e0ce26..d714249 100644
--- a/services/storage/ontologyloader.h
+++ b/services/storage/ontologyloader.h
@@ -1,5 +1,5 @@
=A0/* This file is part of the KDE Project
- =A0 Copyright (c) 2007-2010 Sebastian Trueg <trueg@kde.org>
+ =A0 Copyright (c) 2007-2012 Sebastian Trueg <trueg@kde.org>

=A0 =A0This library is free software; you can redistribute it and/or
=A0 =A0modify it under the terms of the GNU Library General Public
@@ -91,6 +91,7 @@ namespace Nepomuk {
=A0 =A0 =A0 =A0 // a little async updating
=A0 =A0 =A0 =A0 void updateNextOntology();
=A0 =A0 =A0 =A0 void slotGraphRetrieverResult( KJob* job );
+ =A0 =A0 =A0 =A0void updateTypeVisibility();

=A0 =A0 private:
=A0 =A0 =A0 =A0 class Private;


--f46d04426bd6dea2b704bc8ed81f--