[prev in list] [next in list] [prev in thread] [next in thread]
List: kde-commits
Subject: [marble] src/lib/marble: FileStorageWatcher: honor disk cache limit
From: Thibaut Gridel <tgridel () free ! fr>
Date: 2014-09-08 18:13:10
Message-ID: E1XR3R8-0007qV-2b () scm ! kde ! org
[Download RAW message or body]
Git commit 8b54dcdc1002b26c18761ca4292cce7e1857f459 by Thibaut Gridel.
Committed on 08/09/2014 at 17:59.
Pushed by tgridel into branch 'master'.
FileStorageWatcher: honor disk cache limit
REVIEW: 119982
M +83 -215 src/lib/marble/FileStorageWatcher.cpp
M +7 -34 src/lib/marble/FileStorageWatcher.h
M +0 -4 src/lib/marble/MarbleModel.cpp
http://commits.kde.org/marble/8b54dcdc1002b26c18761ca4292cce7e1857f459
diff --git a/src/lib/marble/FileStorageWatcher.cpp b/src/lib/marble/FileStorageWatcher.cpp
index 8706e71..7793590 100644
--- a/src/lib/marble/FileStorageWatcher.cpp
+++ b/src/lib/marble/FileStorageWatcher.cpp
@@ -12,7 +12,6 @@
#include "FileStorageWatcher.h"
// Qt
-#include <QDateTime>
#include <QDir>
#include <QDirIterator>
#include <QFileInfo>
@@ -84,15 +83,6 @@ void FileStorageWatcherThread::resetCurrentSize()
emit variableChanged();
}
-void FileStorageWatcherThread::updateTheme( const QString &mapTheme )
-{
- mDebug() << "Theme changed to " << mapTheme;
- m_themeMutex.lock();
- m_mapThemeId = mapTheme;
- m_themeMutex.unlock();
- emit variableChanged();
-}
-
void FileStorageWatcherThread::prepareQuit()
{
m_willQuit = true;
@@ -102,13 +92,31 @@ void FileStorageWatcherThread::getCurrentCacheSize()
{
mDebug() << "FileStorageWatcher: Creating cache size";
quint64 dataSize = 0;
- QDirIterator it( m_dataDirectory, QDir::Files, QDirIterator::Subdirectories );
+ QString basePath = m_dataDirectory + "/maps";
+ QDirIterator it( basePath,
+ QDir::Files | QDir::Writable,
+ QDirIterator::Subdirectories );
- while( it.hasNext() && !m_willQuit )
- {
- it.next();
- QFileInfo file = it.fileInfo();
- dataSize += file.size();
+ int basePathDepth = basePath.split("/").size();
+ while( it.hasNext() && !m_willQuit ) {
+ it.next();
+ QFileInfo file = it.fileInfo();
+ // We try to be very careful and just delete images
+ // FIXME, when vectortiling I suppose also vector tiles will have
+ // to be deleted
+ QString suffix = file.suffix().toLower();
+ QStringList path = file.path().split("/");
+
+ // planet/theme/tilelevel should be deeper than 4
+ if ( ( path.size() > basePathDepth + 3 ) &&
+ ( path[basePathDepth + 2].toInt() >= maxBaseTileLevel ) &&
+ ( ( suffix == "jpg"
+ || suffix == "png"
+ || suffix == "gif"
+ || suffix == "svg" ) ) ) {
+ dataSize += file.size();
+ m_filesCache.insert(file.lastModified(), file.absoluteFilePath());
+ }
}
m_currentCacheSize = dataSize;
}
@@ -124,172 +132,44 @@ void FileStorageWatcherThread::ensureCacheSize()
|| ( m_deleting && ( m_currentCacheSize > m_cacheSoftLimit ) ) )
&& ( m_cacheLimit != 0 )
&& ( m_cacheSoftLimit != 0 )
- && !( m_mapThemeId.isEmpty() )
- && !m_willQuit )
- {
- // The counter for deleted files
- m_filesDeleted = 0;
- // We have not reached our soft limit, yet.
- m_deleting = true;
-
- // Make sure that we are in the right directory
- if ( m_dataDirectory.isEmpty() ||
- !m_dataDirectory.endsWith(QLatin1String( "data" )) )
- {
- mDebug()
- << "Error: Refusing to erase files under"
- << "unknown conditions for safety reasons!";
- return;
- }
-
- // Which planet do we show now.
- // We have to delete files for this planet at last
- QStringList currentList = m_mapThemeId.split( '/' );
- QString shownPlanet( currentList.first() );
- currentList.removeFirst();
- // The QString to save the path of the directory to the current planet
- QString lastPlanet;
-
- QString cachedMapsDirectory = m_dataDirectory + "/maps";
-
- // Iterator over planet directories
- QDirIterator it( cachedMapsDirectory,
- QDir::NoDotAndDotDot | QDir::Dirs );
-
- while ( it.hasNext() &&
- keepDeleting() ) {
- it.next();
-
- // Our current planet directory
- QString planetDirectory = it.filePath();
- QFileInfo fileInfo = it.fileInfo();
-
- // We have found the currently shown planet.
- // Please delete here at last.
- if( fileInfo.fileName() == shownPlanet ) {
- lastPlanet = planetDirectory;
- continue;
- }
-
- ensureSizePerPlanet( planetDirectory );
- }
-
- if( keepDeleting() ) {
- ensureSizePerPlanet( lastPlanet, currentList.first() );
- }
-
- // We have deleted enough files.
- // Perhaps there are changes.
- if( m_filesDeleted > maxFilesDelete ) {
- QTimer::singleShot( 100, this, SLOT(ensureCacheSize()) );
- return;
- }
- else {
- // We haven't stopped because of to many files
- m_deleting = false;
- }
-
- if( m_currentCacheSize > m_cacheSoftLimit ) {
- mDebug() << "FileStorageWatcher: Could not set cache size.";
- // Set the cache limit to a higher value, so we won't start
- // trying to delete something next time. Softlimit is now exactly
- // on the current cache size.
- setCacheLimit( m_currentCacheSize / ( 100 - softLimitPercent ) * 100 );
- }
- }
-}
-
-void FileStorageWatcherThread::ensureSizePerPlanet( const QString &planetDirectory,
- const QString ¤tTheme )
-{
- mDebug() << "Deleting from folder: " << planetDirectory;
-
- // lastTheme will store the path to our currentTheme
- QString lastTheme;
-
- QDirIterator itPlanet( planetDirectory,
- QDir::NoDotAndDotDot | QDir::Dirs );
- while( itPlanet.hasNext() &&
- keepDeleting() ) {
- itPlanet.next();
- QString themeDirectory = itPlanet.filePath();
- QFileInfo fileInfo = itPlanet.fileInfo();
-
- // We have found the currently shown theme.
- // Please delete here at last.
- if( !currentTheme.isEmpty() && fileInfo.fileName() == currentTheme ) {
- mDebug() << "FileStorageWatcher: Skipping " << themeDirectory
- << " for now";
- lastTheme = themeDirectory;
- continue;
- }
-
- ensureSizePerTheme( themeDirectory );
- }
-
- if( keepDeleting() ) {
- mDebug() << "Removing files of: "
- << lastTheme;
- ensureSizePerTheme( lastTheme );
- }
-}
-
-static bool greaterThanByNumber( const QString &s1, const QString &s2)
-{
- return s1.toInt() > s2.toInt();
-}
-
-void FileStorageWatcherThread::ensureSizePerTheme( const QString &themeDirectory )
-{
- mDebug() << "Deleting from folder: " << themeDirectory;
-
- // Delete from folders with high numbers first
- QStringList folders =
- QDir( themeDirectory ).entryList( QDir::Dirs
- | QDir::NoDotAndDotDot );
- qSort( folders.begin(), folders.end(), greaterThanByNumber );
-
- QStringListIterator itTheme( folders );
- while ( itTheme.hasNext() &&
- keepDeleting() ) {
- QString subDirectory = itTheme.next();
-
- // Do not delete base tiles
- if ( subDirectory.toInt() <= maxBaseTileLevel ) {
- continue;
- }
-
- QString tileDirectory = themeDirectory + '/' + subDirectory;
-
- QDirIterator itTile( tileDirectory,
- QDir::Files | QDir::NoSymLinks,
- QDirIterator::Subdirectories );
- while ( itTile.hasNext() &&
- keepDeleting() ) {
- itTile.next();
- QString filePath = itTile.filePath();
- QString lowerCase = filePath.toLower();
-
- QFileInfo info( filePath );
-
- // We try to be very careful and just delete images
- // Do not delete files younger than two minutes.
- // FIXME, when vectortiling I suppose also vector tiles will have
- // to be deleted
- if ( ( lowerCase.endsWith( QLatin1String( ".jpg" ) )
- || lowerCase.endsWith( QLatin1String( ".png" ) )
- || lowerCase.endsWith( QLatin1String( ".gif" ) )
- || lowerCase.endsWith( QLatin1String( ".svg" ) ) )
- && ( info.lastModified().secsTo( QDateTime::currentDateTime() )
- > deleteOnlyFilesOlderThan ) )
- {
- mDebug() << "FileStorageWatcher: Delete "
- << filePath;
- m_filesDeleted++;
- m_currentCacheSize -= info.size();
- QFile::remove( filePath );
- }
- }
+ && !m_willQuit ) {
+
+ mDebug() << "Deleting extra cached tiles";
+ // The counter for deleted files
+ m_filesDeleted = 0;
+ // We have not reached our soft limit, yet.
+ m_deleting = true;
+
+ QMultiMap<QDateTime, QString>::iterator it= m_filesCache.begin();
+ while ( it != m_filesCache.end() &&
+ keepDeleting() ) {
+ QString filePath = it.value();
+ QFileInfo info( filePath );
+
+ m_filesDeleted++;
+ m_currentCacheSize -= info.size();
+ it = m_filesCache.erase(it);
+ QFile::remove( filePath );
+ }
+
+ // We have deleted enough files.
+ // Perhaps there are changes.
+ if( m_filesDeleted > maxFilesDelete ) {
+ QTimer::singleShot( 1000, this, SLOT(ensureCacheSize()) );
+ return;
+ }
+ else {
+ // We haven't stopped because of too many files
+ m_deleting = false;
+ }
+
+ if( m_currentCacheSize > m_cacheSoftLimit ) {
+ mDebug() << "FileStorageWatcher: Could not set cache size.";
+ // Set the cache limit to a higher value, so we won't start
+ // trying to delete something next time. Softlimit is now exactly
+ // on the current cache size.
+ setCacheLimit( m_currentCacheSize / ( 100 - softLimitPercent ) * 100 );
+ }
}
}
@@ -314,7 +194,7 @@ FileStorageWatcher::FileStorageWatcher( const QString &dataDirectory, QObject *
QDir::root().mkpath( m_dataDirectory );
m_started = false;
- m_themeLimitMutex = new QMutex();
+ m_limitMutex = new QMutex();
m_thread = 0;
m_quitting = false;
@@ -337,12 +217,12 @@ FileStorageWatcher::~FileStorageWatcher()
delete m_thread;
- delete m_themeLimitMutex;
+ delete m_limitMutex;
}
void FileStorageWatcher::setCacheLimit( quint64 bytes )
{
- QMutexLocker locker( m_themeLimitMutex );
+ QMutexLocker locker( m_limitMutex );
if( m_started )
// This is done directly to ensure that a running ensureCacheSize()
// recognizes the new size.
@@ -369,40 +249,28 @@ void FileStorageWatcher::resetCurrentSize()
emit cleared();
}
-void FileStorageWatcher::updateTheme( const QString &mapTheme )
-{
- QMutexLocker locker( m_themeLimitMutex );
- if( m_started )
- // This is done directly
- m_thread->updateTheme( mapTheme );
- // Save the theme, thread has to initialized with the right one.
- m_theme = mapTheme;
-}
-
void FileStorageWatcher::run()
{
m_thread = new FileStorageWatcherThread( m_dataDirectory );
if( !m_quitting ) {
- m_themeLimitMutex->lock();
- m_thread->setCacheLimit( m_limit );
- m_thread->updateTheme( m_theme );
- m_started = true;
- mDebug() << m_started;
- m_themeLimitMutex->unlock();
-
- m_thread->getCurrentCacheSize();
-
- connect( this, SIGNAL(sizeChanged(qint64)),
- m_thread, SLOT(addToCurrentSize(qint64)) );
- connect( this, SIGNAL(cleared()),
- m_thread, SLOT(resetCurrentSize()) );
-
- // Make sure that we don't want to stop process.
- // The thread wouldn't exit from event loop.
- if( !m_quitting )
- exec();
-
- m_started = false;
+ m_limitMutex->lock();
+ m_thread->setCacheLimit( m_limit );
+ m_started = true;
+ m_limitMutex->unlock();
+
+ m_thread->getCurrentCacheSize();
+
+ connect( this, SIGNAL(sizeChanged(qint64)),
+ m_thread, SLOT(addToCurrentSize(qint64)) );
+ connect( this, SIGNAL(cleared()),
+ m_thread, SLOT(resetCurrentSize()) );
+
+ // Make sure that we don't want to stop process.
+ // The thread wouldn't exit from event loop.
+ if( !m_quitting )
+ exec();
+
+ m_started = false;
}
delete m_thread;
m_thread = 0;
diff --git a/src/lib/marble/FileStorageWatcher.h b/src/lib/marble/FileStorageWatcher.h
index cf1ae64..39b9a8f 100644
--- a/src/lib/marble/FileStorageWatcher.h
+++ b/src/lib/marble/FileStorageWatcher.h
@@ -13,7 +13,8 @@
#include <QThread>
#include <QMutex>
-#include <QSet>
+#include <QMultiMap>
+#include <QDateTime>
namespace Marble
{
@@ -54,14 +55,6 @@ class FileStorageWatcherThread : public QObject
void resetCurrentSize();
/**
- * Updates the name of the theme.
- * Important for deleting behavior.
- *
- * @param mapTheme The identifier of the new theme.
- */
- void updateTheme( const QString &mapTheme );
-
- /**
* Stop doing things that take a long time to quit.
*/
void prepareQuit();
@@ -81,30 +74,18 @@ class FileStorageWatcherThread : public QObject
Q_DISABLE_COPY( FileStorageWatcherThread )
/**
- * Deletes files of a planet if needed
- */
- void ensureSizePerPlanet( const QString &planetDirectory, const QString ¤tTheme = QString() );
-
- /**
- * Deletes files of a theme if needed
- */
- void ensureSizePerTheme( const QString &themeDirectory );
-
- /**
* Returns true if it is necessary to delete files.
*/
bool keepDeleting() const;
QString m_dataDirectory;
-
- quint64 m_cacheLimit;
+ QMultiMap<QDateTime,QString> m_filesCache;
+ quint64 m_cacheLimit;
quint64 m_cacheSoftLimit;
- quint64 m_currentCacheSize;
+ quint64 m_currentCacheSize;
int m_filesDeleted;
bool m_deleting;
- QString m_mapThemeId;
QMutex m_limitMutex;
- QMutex m_themeMutex;
bool m_willQuit;
};
@@ -148,14 +129,7 @@ class FileStorageWatcher : public QThread
*/
void resetCurrentSize();
- /**
- * Updates the name of the theme.
- * Important for deleting behavior.
- *
- * @param mapTheme The identifier of the new theme.
- */
- void updateTheme( const QString &mapTheme );
-
+
Q_SIGNALS:
void sizeChanged( qint64 bytes );
void cleared();
@@ -172,8 +146,7 @@ class FileStorageWatcher : public QThread
QString m_dataDirectory;
FileStorageWatcherThread *m_thread;
- QMutex *m_themeLimitMutex;
- QString m_theme;
+ QMutex *m_limitMutex;
quint64 m_limit;
bool m_started;
bool m_quitting;
diff --git a/src/lib/marble/MarbleModel.cpp b/src/lib/marble/MarbleModel.cpp
index 6f1ef64..17126fb 100644
--- a/src/lib/marble/MarbleModel.cpp
+++ b/src/lib/marble/MarbleModel.cpp
@@ -182,10 +182,6 @@ MarbleModel::MarbleModel( QObject *parent )
new QNetworkConfigurationManager( this );
#endif
- // The thread will be started at setting persistent tile cache size.
- connect( this, SIGNAL(themeChanged(QString)),
- &d->m_storageWatcher, SLOT(updateTheme(QString)) );
-
// connect the StoragePolicy used by the download manager to the FileStorageWatcher
connect( &d->m_storagePolicy, SIGNAL(cleared()),
&d->m_storageWatcher, SLOT(resetCurrentSize()) );
[prev in list] [next in list] [prev in thread] [next in thread]
Configure |
About |
News |
Add a list |
Sponsored by KoreLogic