[prev in list] [next in list] [prev in thread] [next in thread]
List: kde-commits
Subject: [messagelib] webengineviewer/src/checkphishingurl: continue to improve safe browsing
From: Montel Laurent <montel () kde ! org>
Date: 2016-12-09 19:30:50
Message-ID: E1cFQsc-0006l6-LF () code ! kde ! org
[Download RAW message or body]
Git commit f4569586fc00c2e05886da6fdef264db2e47200a by Montel Laurent.
Committed on 09/12/2016 at 19:30.
Pushed by mlaurent into branch 'master'.
continue to improve safe browsing
M +13 -5 webengineviewer/src/checkphishingurl/autotests/searchfullhashjobtest.cpp
M +2 -1 webengineviewer/src/checkphishingurl/localdatabasefile.cpp
M +20 -8 webengineviewer/src/checkphishingurl/localdatabasemanager.cpp
M +1 -1 webengineviewer/src/checkphishingurl/localdatabasemanager.h
M +26 -16 webengineviewer/src/checkphishingurl/searchfullhashjob.cpp
M +4 -3 webengineviewer/src/checkphishingurl/searchfullhashjob.h
M +1 -1 webengineviewer/src/checkphishingurl/tests/searchfullhashgui.cpp
M +6 -3 webengineviewer/src/checkphishingurl/urlhashing.cpp
https://commits.kde.org/messagelib/f4569586fc00c2e05886da6fdef264db2e47200a
diff --git a/webengineviewer/src/checkphishingurl/autotests/searchfullhashjobtest.cpp \
b/webengineviewer/src/checkphishingurl/autotests/searchfullhashjobtest.cpp index \
b5d42fe4..3bfaba17 100644
--- a/webengineviewer/src/checkphishingurl/autotests/searchfullhashjobtest.cpp
+++ b/webengineviewer/src/checkphishingurl/autotests/searchfullhashjobtest.cpp
@@ -44,13 +44,17 @@ void SearchFullHashJobTest::shouldCreateRequest_data()
QTest::addColumn<QByteArray>("hash");
QTest::addColumn<QStringList>("databaseHash");
QTest::addColumn<QString>("request");
+ QTest::addColumn<QUrl>("url");
QTest::addColumn<bool>("canStart");
- QTest::newRow("no hash") << QByteArray() << QStringList() << QString() << false;
- QTest::newRow("database hash but not hash") << QByteArray() << \
QStringList{QStringLiteral("boo")} << QString() << false; + QTest::newRow("no \
hash") << QByteArray() << QStringList() << QString() << QUrl() << false; + \
QTest::newRow("database hash but not hash and not url") << QByteArray() << \
QStringList{QStringLiteral("boo")} << QString() << QUrl() << false; + \
QTest::newRow("database hash but hash and not url") << QByteArrayLiteral("bla") << \
QStringList{QStringLiteral("boo")} << QString() << QUrl() << false;
QTest::newRow("database hash and hash") << QByteArrayLiteral("bla") << \
QStringList{QStringLiteral("boo")}
- << \
QStringLiteral("{\"client\":{\"clientId\":\"KDE\",\"clientVersion\":\"%1\"},\"clientSt \
ates\":[\"boo\"],\"threatInfo\":{\"platformTypes\":[\"WINDOWS\"],\"threatEntries\":[{\ \
"hash\":\"bla\"}],\"threatEntryTypes\":[\"URL\"],\"threatTypes\":[\"MALWARE\"]}}").arg(WebEngineViewer::CheckPhishingUrlUtil::versionApps()) \
<< true; + << \
QStringLiteral("{\"client\":{\"clientId\":\"KDE\",\"clientVersion\":\"%1\"},\"clientSt \
ates\":[\"boo\"],\"threatInfo\":{\"platformTypes\":[\"WINDOWS\"],\"threatEntries\":[{\ \
"hash\":\"bla\"}],\"threatEntryTypes\":[\"URL\"],\"threatTypes\":[\"MALWARE\"]}}").arg(WebEngineViewer::CheckPhishingUrlUtil::versionApps())
+ << \
QUrl(QStringLiteral("http://www.kde.org")) << true;
QTest::newRow("multi database hash and hash") << QByteArrayLiteral("bla") << \
(QStringList() << QStringLiteral("boo") << QStringLiteral("bli"))
- << QStringLiteral("{\"client\":{\"clientId\":\"KDE\",\"clientVersion\":\" \
%1\"},\"clientStates\":[\"boo\",\"bli\"],\"threatInfo\":{\"platformTypes\":[\"WINDOWS\ \
"],\"threatEntries\":[{\"hash\":\"bla\"}],\"threatEntryTypes\":[\"URL\"],\"threatTypes\":[\"MALWARE\"]}}").arg(WebEngineViewer::CheckPhishingUrlUtil::versionApps()) \
<< true; + << \
QStringLiteral("{\"client\":{\"clientId\":\"KDE\",\"clientVersion\":\"%1\"},\"clientSt \
ates\":[\"boo\",\"bli\"],\"threatInfo\":{\"platformTypes\":[\"WINDOWS\"],\"threatEntri \
es\":[{\"hash\":\"bla\"}],\"threatEntryTypes\":[\"URL\"],\"threatTypes\":[\"MALWARE\"]}}").arg(WebEngineViewer::CheckPhishingUrlUtil::versionApps())
+ << QUrl(QStringLiteral("http://www.kde.org")) << true;
}
void SearchFullHashJobTest::shouldCreateRequest()
@@ -58,10 +62,14 @@ void SearchFullHashJobTest::shouldCreateRequest()
QFETCH(QByteArray, hash);
QFETCH(QStringList, databaseHash);
QFETCH(QString, request);
+ QFETCH(QUrl, url);
QFETCH(bool, canStart);
WebEngineViewer::SearchFullHashJob job;
job.setDatabaseState(databaseHash);
- job.setSearchHash(hash);
+ job.setSearchFullHashForUrl(url);
+ if (!hash.isEmpty()) {
+ job.setSearchHash(QList<QByteArray>() << hash);
+ }
QCOMPARE(job.canStart(), canStart);
if (canStart) {
QCOMPARE(job.jsonRequest(), request.toLatin1());
diff --git a/webengineviewer/src/checkphishingurl/localdatabasefile.cpp \
b/webengineviewer/src/checkphishingurl/localdatabasefile.cpp index 37312a71..8140dabc \
100644
--- a/webengineviewer/src/checkphishingurl/localdatabasefile.cpp
+++ b/webengineviewer/src/checkphishingurl/localdatabasefile.cpp
@@ -80,6 +80,7 @@ void LocalDataBaseFilePrivate::close()
bool LocalDataBaseFilePrivate::reload()
{
+ qCDebug(WEBENGINEVIEWER_LOG) << "Reload File";
close();
return load();
}
@@ -153,7 +154,7 @@ QByteArray LocalDataBaseFile::searchHash(const QByteArray \
&hashToSearch) const char *hashCharStar = getCharStar(hashOffset);
const int cmp = qstrcmp(hashCharStar, hashToSearch.constData());
currentValue = QByteArray(hashCharStar);
- qCWarning(WEBENGINEVIEWER_LOG) << "search " << hashToSearch << " begin " \
<< begin << " end " << end << " hashCharStar" << hashCharStar; + \
//qCWarning(WEBENGINEVIEWER_LOG) << "search " << hashToSearch << " begin " << begin \
<< " end " << end << " hashCharStar" << hashCharStar; if (end == begin) {
return currentValue;
}
diff --git a/webengineviewer/src/checkphishingurl/localdatabasemanager.cpp \
b/webengineviewer/src/checkphishingurl/localdatabasemanager.cpp index \
e9db7abc..c2d92af4 100644
--- a/webengineviewer/src/checkphishingurl/localdatabasemanager.cpp
+++ b/webengineviewer/src/checkphishingurl/localdatabasemanager.cpp
@@ -18,7 +18,6 @@
*/
#include "localdatabasemanager.h"
#include "webengineviewer_debug.h"
-#include "checkphishingurlfromlocaldatabasejob.h"
#include "createphishingurldatabasejob.h"
#include "createdatabasefilejob.h"
#include "checkphishingurlutil.h"
@@ -139,6 +138,7 @@ void LocalDataBaseManager::initialize()
return;
}
if (!d->mDataBaseOk) {
+ qCDebug(WEBENGINEVIEWER_LOG) << "Start to create database";
if (!QFile(databaseFullPath()).exists()) {
downloadFullDataBase();
} else {
@@ -193,6 +193,7 @@ void LocalDataBaseManager::slotDownloadDataBaseFinished(const \
WebEngineViewer::U }
}
d->mDownloadProgress = false;
+ qCDebug(WEBENGINEVIEWER_LOG) << "Download done";
}
void LocalDataBaseManager::slotCreateDataBaseFileNameFinished(bool success, const \
QString &newClientState) @@ -220,20 +221,30 @@ void \
LocalDataBaseManager::setDownloadProgress(bool downloadProgress) void \
LocalDataBaseManager::checkUrl(const QUrl &url) {
if (d->mDataBaseOk) {
- QByteArray hash;
- QByteArray result = d->mFile.searchHash(hash);
- if (hash.contains(result)) {
+ //TODO fixme short hash! we don't need to use it.
+ WebEngineViewer::UrlHashing urlHashing(url);
+ QList<QByteArray> hashList = urlHashing.hashList();
+ QList<QByteArray> conflictHashs;
+ Q_FOREACH(const QByteArray &ba, hashList) {
+ QByteArray result = d->mFile.searchHash(ba);
+ if (ba == result) {
+ conflictHashs << ba.toBase64();
+ }
+ }
+ if (conflictHashs.isEmpty()) {
+ Q_EMIT checkUrlFinished(url, \
WebEngineViewer::LocalDataBaseManager::UrlOk); + } else {
+ qCWarning(WEBENGINEVIEWER_LOG) << " We need to Check Server Database";
if (d->mNewClientState.isEmpty()) {
qCWarning(WEBENGINEVIEWER_LOG) << "Database client state is \
unknown";
Q_EMIT checkUrlFinished(url, \
WebEngineViewer::LocalDataBaseManager::Unknown); } else {
WebEngineViewer::SearchFullHashJob *job = new \
WebEngineViewer::SearchFullHashJob(this);
job->setDatabaseState(QStringList() << d->mNewClientState);
+ job->setSearchHash(conflictHashs);
connect(job, &SearchFullHashJob::result, this, \
&LocalDataBaseManager::slotSearchOnServerResult); job->start();
}
- } else {
- Q_EMIT checkUrlFinished(url, \
WebEngineViewer::LocalDataBaseManager::UrlOk); }
} else {
qCWarning(WEBENGINEVIEWER_LOG) << "Database not ok";
@@ -244,8 +255,9 @@ void LocalDataBaseManager::checkUrl(const QUrl &url)
}
}
-void LocalDataBaseManager::slotSearchOnServerResult(WebEngineViewer::SearchFullHashJob::UrlStatus \
status, const QByteArray &hash, const QStringList &listHash) +void \
LocalDataBaseManager::slotSearchOnServerResult(WebEngineViewer::SearchFullHashJob::UrlStatus \
status, const QUrl &url) {
- qCDebug(WEBENGINEVIEWER_LOG) << "hash " << hash << " listHash " << listHash;
+ qDebug() <<" void \
LocalDataBaseManager::slotSearchOnServerResult(WebEngineViewer::SearchFullHashJob::UrlStatus \
status, const QList<QByteArray> &hashs, const QStringList &listHash)"; + \
qCWarning(WEBENGINEVIEWER_LOG) << " Url " << url << " status "<< status; //TODO
}
diff --git a/webengineviewer/src/checkphishingurl/localdatabasemanager.h \
b/webengineviewer/src/checkphishingurl/localdatabasemanager.h index \
dcd86c08..233b93df 100644
--- a/webengineviewer/src/checkphishingurl/localdatabasemanager.h
+++ b/webengineviewer/src/checkphishingurl/localdatabasemanager.h
@@ -58,7 +58,7 @@ protected:
virtual void downloadPartialDataBase();
private:
- void slotSearchOnServerResult(WebEngineViewer::SearchFullHashJob::UrlStatus \
status, const QByteArray &hash, const QStringList &listHash); + void \
slotSearchOnServerResult(WebEngineViewer::SearchFullHashJob::UrlStatus status, const \
QUrl &url); void slotDownloadDataBaseFinished(const \
WebEngineViewer::UpdateDataBaseInfo &infoDataBase, \
WebEngineViewer::CreatePhishingUrlDataBaseJob::DataBaseDownloadResult status); void \
slotCheckDataBase(); void downloadDataBase(const QString &clientState);
diff --git a/webengineviewer/src/checkphishingurl/searchfullhashjob.cpp \
b/webengineviewer/src/checkphishingurl/searchfullhashjob.cpp index 86876848..1a1f058a \
100644
--- a/webengineviewer/src/checkphishingurl/searchfullhashjob.cpp
+++ b/webengineviewer/src/checkphishingurl/searchfullhashjob.cpp
@@ -38,7 +38,8 @@ public:
{
}
- QByteArray mHash;
+ QList<QByteArray> mHashs;
+ QUrl mUrl;
QStringList mDatabaseHashes;
QNetworkAccessManager *mNetworkAccessManager;
};
@@ -100,18 +101,18 @@ void SearchFullHashJob::parse(const QByteArray &replyStr)
*/
QJsonDocument document = QJsonDocument::fromJson(replyStr);
if (document.isNull()) {
- Q_EMIT result(WebEngineViewer::SearchFullHashJob::Unknown, d->mHash);
+ Q_EMIT result(WebEngineViewer::SearchFullHashJob::Unknown, d->mUrl);
} else {
+ qDebug()<<" document" << document.toJson(QJsonDocument::Indented);
const QVariantMap answer = document.toVariant().toMap();
if (answer.isEmpty()) {
- Q_EMIT result(WebEngineViewer::SearchFullHashJob::Ok, d->mHash);
+ Q_EMIT result(WebEngineViewer::SearchFullHashJob::Ok, d->mUrl);
return;
} else {
const QVariantList info = \
answer.value(QStringLiteral("matches")).toList(); //TODO
const QString minimumWaitDuration = \
answer.value(QStringLiteral("minimumWaitDuration")).toString();
const QString negativeCacheDuration = \
answer.value(QStringLiteral("negativeCacheDuration")).toString();
-
//Implement multi match ?
if (info.count() == 1) {
const QVariantMap map = info.at(0).toMap();
@@ -122,14 +123,17 @@ void SearchFullHashJob::parse(const QByteArray &replyStr)
if (threatTypeStr == QStringLiteral("MALWARE")) {
const QVariantMap urlMap = \
map[QStringLiteral("threat")].toMap(); QMapIterator<QString, QVariant> \
urlMapIt(urlMap);
- QStringList hashList;
+ QList<QByteArray> hashList;
while (urlMapIt.hasNext()) {
urlMapIt.next();
- const QString hashStr = urlMapIt.value().toString();
+ const QByteArray hashStr = urlMapIt.value().toByteArray();
hashList << hashStr;
}
+ //TODO check if it's the malware url
if (!hashList.isEmpty()) {
- Q_EMIT result(WebEngineViewer::SearchFullHashJob::MalWare, \
d->mHash, hashList); + Q_EMIT \
result(WebEngineViewer::SearchFullHashJob::MalWare, d->mUrl); + } \
else { + Q_EMIT \
result(WebEngineViewer::SearchFullHashJob::Unknown, d->mUrl); }
const QVariantMap threatEntryMetadataMap = \
map[QStringLiteral("threatEntryMetadata")].toMap(); if \
(!threatEntryMetadataMap.isEmpty()) { @@ -140,22 +144,22 @@ void \
SearchFullHashJob::parse(const QByteArray &replyStr) }
} else {
qCWarning(WEBENGINEVIEWER_LOG) << " SearchFullHashJob::parse matches \
multi element : " << info.count(); + Q_EMIT \
result(WebEngineViewer::SearchFullHashJob::Unknown, d->mUrl); }
- Q_EMIT result(WebEngineViewer::SearchFullHashJob::Unknown, d->mHash);
}
}
+ deleteLater();
}
void SearchFullHashJob::slotCheckUrlFinished(QNetworkReply *reply)
{
parse(reply->readAll());
reply->deleteLater();
- deleteLater();
}
-void SearchFullHashJob::setSearchHash(const QByteArray &hash)
+void SearchFullHashJob::setSearchHash(const QList<QByteArray> &hash)
{
- d->mHash = hash;
+ d->mHashs = hash;
}
QByteArray SearchFullHashJob::jsonRequest() const
@@ -211,8 +215,9 @@ QByteArray SearchFullHashJob::jsonRequest() const
QVariantList threatEntriesList;
QVariantMap hashUrlMap;
- //We can have multi hash
- hashUrlMap.insert(QStringLiteral("hash"), d->mHash);
+ Q_FOREACH(const QByteArray &hash, d->mHashs) {
+ hashUrlMap.insert(QStringLiteral("hash"), hash);
+ }
threatEntriesList.append(hashUrlMap);
threatMap.insert(QStringLiteral("threatEntries"), threatEntriesList);
@@ -227,7 +232,7 @@ QByteArray SearchFullHashJob::jsonRequest() const
void SearchFullHashJob::start()
{
if (!PimCommon::NetworkManager::self()->networkConfigureManager()->isOnline()) {
- Q_EMIT result(WebEngineViewer::SearchFullHashJob::BrokenNetwork, d->mHash);
+ Q_EMIT result(WebEngineViewer::SearchFullHashJob::BrokenNetwork, d->mUrl);
deleteLater();
} else if (canStart()) {
QUrl safeUrl = \
QUrl(QStringLiteral("https://safebrowsing.googleapis.com/v4/fullHashes:find")); @@ \
-241,7 +246,7 @@ void SearchFullHashJob::start()
QNetworkReply *reply = d->mNetworkAccessManager->post(request, baPostData);
connect(reply, static_cast<void \
(QNetworkReply::*)(QNetworkReply::NetworkError)>(&QNetworkReply::error), this, \
&SearchFullHashJob::slotError); } else {
- Q_EMIT result(WebEngineViewer::SearchFullHashJob::InvalidUrl, d->mHash);
+ Q_EMIT result(WebEngineViewer::SearchFullHashJob::InvalidUrl, d->mUrl);
deleteLater();
}
}
@@ -256,10 +261,15 @@ void SearchFullHashJob::slotError(QNetworkReply::NetworkError \
error)
bool SearchFullHashJob::canStart() const
{
- return !d->mHash.isEmpty() && !d->mDatabaseHashes.isEmpty();
+ return !d->mHashs.isEmpty() && !d->mDatabaseHashes.isEmpty() && \
!d->mUrl.isEmpty(); }
void SearchFullHashJob::setDatabaseState(const QStringList &hash)
{
d->mDatabaseHashes = hash;
}
+
+void SearchFullHashJob::setSearchFullHashForUrl(const QUrl &url)
+{
+ d->mUrl = url;
+}
diff --git a/webengineviewer/src/checkphishingurl/searchfullhashjob.h \
b/webengineviewer/src/checkphishingurl/searchfullhashjob.h index 54d46b5f..3b97a4ba \
100644
--- a/webengineviewer/src/checkphishingurl/searchfullhashjob.h
+++ b/webengineviewer/src/checkphishingurl/searchfullhashjob.h
@@ -28,7 +28,7 @@ class QNetworkAccessManager;
namespace WebEngineViewer
{
class SearchFullHashJobPrivate;
-/* https://developers.google.com/safe-browsing/v4/lookup-api */
+/* https://developers.google.com/safe-browsing/v4/update-api */
class WEBENGINEVIEWER_EXPORT SearchFullHashJob : public QObject
{
Q_OBJECT
@@ -48,13 +48,14 @@ public:
bool canStart() const;
void setDatabaseState(const QStringList &hash);
+ void setSearchFullHashForUrl(const QUrl &url);
QByteArray jsonRequest() const;
void parse(const QByteArray &replyStr);
- void setSearchHash(const QByteArray &hash);
+ void setSearchHash(const QList<QByteArray> &hash);
Q_SIGNALS:
- void result(WebEngineViewer::SearchFullHashJob::UrlStatus status, const \
QByteArray &hash, const QStringList &listHash = QStringList()); + void \
result(WebEngineViewer::SearchFullHashJob::UrlStatus status, const QUrl &url); void \
debugJson(const QByteArray &ba);
private Q_SLOTS:
diff --git a/webengineviewer/src/checkphishingurl/tests/searchfullhashgui.cpp \
b/webengineviewer/src/checkphishingurl/tests/searchfullhashgui.cpp index \
564f3bcc..74bee60f 100644
--- a/webengineviewer/src/checkphishingurl/tests/searchfullhashgui.cpp
+++ b/webengineviewer/src/checkphishingurl/tests/searchfullhashgui.cpp
@@ -84,7 +84,7 @@ void SearchFullHashGui::slotCheckUrl()
connect(job, &WebEngineViewer::SearchFullHashJob::result, this, \
&SearchFullHashGui::slotGetResult);
connect(job, &WebEngineViewer::SearchFullHashJob::debugJson, this, \
&SearchFullHashGui::slotJSonDebug); job->setDatabaseState(QStringList() << \
databaseHashStr);
- job->setSearchHash(hashStr.toLatin1());
+ job->setSearchHash(QList<QByteArray>() << hashStr.toLatin1());
job->start();
}
diff --git a/webengineviewer/src/checkphishingurl/urlhashing.cpp \
b/webengineviewer/src/checkphishingurl/urlhashing.cpp index a0fb8f9d..e5a97015 100644
--- a/webengineviewer/src/checkphishingurl/urlhashing.cpp
+++ b/webengineviewer/src/checkphishingurl/urlhashing.cpp
@@ -137,15 +137,18 @@ QList<QByteArray> UrlHashing::hashList()
QList<QByteArray> lst;
if (mUrl.isValid()) {
const QString result = WebEngineViewer::UrlHashing::canonicalizeUrl(mUrl);
- QUrl url(result);
+ const QUrl url(result);
const QStringList hosts = \
WebEngineViewer::UrlHashing::generateHostsToCheck(url.host());
const QStringList paths = \
WebEngineViewer::UrlHashing::generatePathsToCheck(url.path(), url.query());
Q_FOREACH(const QString &host, hosts) {
Q_FOREACH(const QString &path, paths) {
const QString str = host + path;
- //TODO truncated it.
- lst << QCryptographicHash::hash(str.toLatin1(), \
QCryptographicHash::Sha256); + QByteArray ba = \
QCryptographicHash::hash(str.toLatin1(), QCryptographicHash::Sha256); + \
qDebug() << " ba " << ba.toBase64(); + //We need to keep 4
+ ba.truncate(4);
+ lst << ba;
}
}
}
[prev in list] [next in list] [prev in thread] [next in thread]
Configure |
About |
News |
Add a list |
Sponsored by KoreLogic