From kde-commits Fri Mar 31 21:56:22 2017 From: Thomas Fischer Date: Fri, 31 Mar 2017 21:56:22 +0000 To: kde-commits Subject: [kbibtex] src/networking/zotero: Improving handling Zotero's 'Backoff'/'Retry-After' headers Message-Id: X-MARC-Message: https://marc.info/?l=kde-commits&m=149099740315503 Git commit f94fb1453d1cb4c745446fe41808b3debfa4d0f4 by Thomas Fischer. Committed on 31/03/2017 at 19:29. Pushed by thomasfischer into branch 'master'. Improving handling Zotero's 'Backoff'/'Retry-After' headers ... by checking the current back-off time before a new rquest and correctly parsing HTTP headers for 'Backoff' and 'Retry-After', respectively. Due to lack of test cases, the current code can only be assumed to work, but has not been tested or verified. M +1 -1 src/networking/zotero/api.cpp M +26 -6 src/networking/zotero/collection.cpp M +22 -8 src/networking/zotero/groups.cpp M +20 -5 src/networking/zotero/items.cpp M +22 -8 src/networking/zotero/tags.cpp https://commits.kde.org/kbibtex/f94fb1453d1cb4c745446fe41808b3debfa4d0f4 diff --git a/src/networking/zotero/api.cpp b/src/networking/zotero/api.cpp index b57ba025..e1ddb40d 100644 --- a/src/networking/zotero/api.cpp +++ b/src/networking/zotero/api.cpp @@ -87,7 +87,7 @@ QNetworkRequest API::request(const QUrl &url) const } = void API::startBackoff(int duration) { - if (duration > 0) { + if (duration > 0 && !inBackoffMode()) { d->backoffElapseTime =3D QDateTime::currentDateTime().addSecs(dura= tion + 1); emit backoffModeStart(); /// Use single-shot timer and functor to emit signal diff --git a/src/networking/zotero/collection.cpp b/src/networking/zotero/c= ollection.cpp index e61aadaa..d2aec073 100644 --- a/src/networking/zotero/collection.cpp +++ b/src/networking/zotero/collection.cpp @@ -72,7 +72,13 @@ public: QUrl url =3D api->baseUrl(); url =3D url.adjusted(QUrl::StripTrailingSlash); url.setPath(url.path() + QString(QStringLiteral("/collections/= %1/collections")).arg(head)); - requestZoteroUrl(url); + if (api->inBackoffMode()) + /// If Zotero asked to 'back off', wait until this period = is over before issuing the next request + QTimer::singleShot((api->backoffSecondsLeft() + 1) * 1000,= [ =3D ]() { + requestZoteroUrl(url); + }); + else + requestZoteroUrl(url); } else { initialized =3D true; p->emitFinishedLoading(); @@ -91,6 +97,7 @@ Collection::Collection(QSharedPointer api, Q= Object *parent) url =3D url.adjusted(QUrl::StripTrailingSlash); url.setPath(url.path() + QStringLiteral("/collections/top")); if (api->inBackoffMode()) + /// If Zotero asked to 'back off', wait until this period is over = before issuing the next request QTimer::singleShot((api->backoffSecondsLeft() + 1) * 1000, [ =3D ]= () { d->requestZoteroUrl(url); }); @@ -163,10 +170,17 @@ void Collection::finishedFetchingCollection() QNetworkReply *reply =3D static_cast(sender()); QString parentId =3D Private::top; = - if (reply->hasRawHeader("Backoff")) - d->api->startBackoff(QString::fromLatin1(reply->rawHeader("Backoff= ").constData()).toInt()); - else if (reply->hasRawHeader("Retry-After")) - d->api->startBackoff(QString::fromLatin1(reply->rawHeader("Retry-A= fter").constData()).toInt()); + if (reply->hasRawHeader("Backoff")) { + bool ok =3D false; + int time =3D QString::fromLatin1(reply->rawHeader("Backoff").const= Data()).toInt(&ok); + if (!ok) time =3D 10; ///< parsing argument of raw header 'Backoff= ' failed? 10 seconds is fallback + d->api->startBackoff(time); + } else if (reply->hasRawHeader("Retry-After")) { + bool ok =3D false; + int time =3D QString::fromLatin1(reply->rawHeader("Retry-After").c= onstData()).toInt(&ok); + if (!ok) time =3D 10; ///< parsing argument of raw header 'Retry-A= fter' failed? 10 seconds is fallback + d->api->startBackoff(time); + } = if (reply->error() =3D=3D QNetworkReply::NoError) { QString nextPage; @@ -214,7 +228,13 @@ void Collection::finishedFetchingCollection() } = if (!nextPage.isEmpty()) { - d->requestZoteroUrl(nextPage); + if (d->api->inBackoffMode()) + /// If Zotero asked to 'back off', wait until this period = is over before issuing the next request + QTimer::singleShot((d->api->backoffSecondsLeft() + 1) * 10= 00, [ =3D ]() { + d->requestZoteroUrl(nextPage); + }); + else + d->requestZoteroUrl(nextPage); } else d->runNextInDownloadQueue(); } else { diff --git a/src/networking/zotero/groups.cpp b/src/networking/zotero/group= s.cpp index d81d8ad5..f3a1576e 100644 --- a/src/networking/zotero/groups.cpp +++ b/src/networking/zotero/groups.cpp @@ -1,5 +1,5 @@ /*************************************************************************= ** - * Copyright (C) 2004-2014 by Thomas Fischer = * + * Copyright (C) 2004-2017 by Thomas Fischer = * * = * * This program is free software; you can redistribute it and/or modify = * * it under the terms of the GNU General Public License as published by = * @@ -66,6 +66,7 @@ Groups::Groups(QSharedPointer api, QObject *= parent) url.setPath(url.path() + QStringLiteral("/groups")); = if (d->api->inBackoffMode()) + /// If Zotero asked to 'back off', wait until this period is over = before issuing the next request QTimer::singleShot((d->api->backoffSecondsLeft() + 1) * 1000, [ = =3D ]() { d->requestZoteroUrl(url); }); @@ -97,10 +98,17 @@ void Groups::finishedFetchingGroups() { QNetworkReply *reply =3D static_cast(sender()); = - if (reply->hasRawHeader("Backoff")) - d->api->startBackoff(QString::fromLatin1(reply->rawHeader("Backoff= ").constData()).toInt()); - else if (reply->hasRawHeader("Retry-After")) - d->api->startBackoff(QString::fromLatin1(reply->rawHeader("Retry-A= fter").constData()).toInt()); + if (reply->hasRawHeader("Backoff")) { + bool ok =3D false; + int time =3D QString::fromLatin1(reply->rawHeader("Backoff").const= Data()).toInt(&ok); + if (!ok) time =3D 10; ///< parsing argument of raw header 'Backoff= ' failed? 10 seconds is fallback + d->api->startBackoff(time); + } else if (reply->hasRawHeader("Retry-After")) { + bool ok =3D false; + int time =3D QString::fromLatin1(reply->rawHeader("Retry-After").c= onstData()).toInt(&ok); + if (!ok) time =3D 10; ///< parsing argument of raw header 'Retry-A= fter' failed? 10 seconds is fallback + d->api->startBackoff(time); + } = if (reply->error() =3D=3D QNetworkReply::NoError) { QString nextPage; @@ -132,9 +140,15 @@ void Groups::finishedFetchingGroups() break; } = - if (!nextPage.isEmpty()) - d->requestZoteroUrl(nextPage); - else { + if (!nextPage.isEmpty()) { + if (d->api->inBackoffMode()) + /// If Zotero asked to 'back off', wait until this period = is over before issuing the next request + QTimer::singleShot((d->api->backoffSecondsLeft() + 1) * 10= 00, [ =3D ]() { + d->requestZoteroUrl(nextPage); + }); + else + d->requestZoteroUrl(nextPage); + } else { d->busy =3D false; d->initialized =3D true; emit finishedLoading(); diff --git a/src/networking/zotero/items.cpp b/src/networking/zotero/items.= cpp index 47e9c3e8..13287954 100644 --- a/src/networking/zotero/items.cpp +++ b/src/networking/zotero/items.cpp @@ -62,7 +62,13 @@ public: query.addQueryItem(queryItemStart, QString::number(start)); internalUrl.setQuery(query); = - requestZoteroUrl(internalUrl); + if (api->inBackoffMode()) + /// If Zotero asked to 'back off', wait until this period is o= ver before issuing the next request + QTimer::singleShot((api->backoffSecondsLeft() + 1) * 1000, [ = =3D ]() { + requestZoteroUrl(internalUrl); + }); + else + requestZoteroUrl(internalUrl); } }; = @@ -89,6 +95,7 @@ void Items::retrieveItemsByCollection(const QString &coll= ection) url.setQuery(query); = if (d->api->inBackoffMode()) + /// If Zotero asked to 'back off', wait until this period is over = before issuing the next request QTimer::singleShot((d->api->backoffSecondsLeft() + 1) * 1000, [ = =3D ]() { d->retrieveItems(url, 0); }); @@ -107,6 +114,7 @@ void Items::retrieveItemsByTag(const QString &tag) url.setQuery(query); = if (d->api->inBackoffMode()) + /// If Zotero asked to 'back off', wait until this period is over = before issuing the next request QTimer::singleShot((d->api->backoffSecondsLeft() + 1) * 1000, [ = =3D ]() { d->retrieveItems(url, 0); }); @@ -121,10 +129,17 @@ void Items::finishedFetchingItems() bool ok =3D false; const int start =3D QUrlQuery(reply->url()).queryItemValue(queryItemSt= art).toInt(&ok); = - if (reply->hasRawHeader("Backoff")) - d->api->startBackoff(QString::fromLatin1(reply->rawHeader("Backoff= ").constData()).toInt()); - else if (reply->hasRawHeader("Retry-After")) - d->api->startBackoff(QString::fromLatin1(reply->rawHeader("Retry-A= fter").constData()).toInt()); + if (reply->hasRawHeader("Backoff")) { + bool ok =3D false; + int time =3D QString::fromLatin1(reply->rawHeader("Backoff").const= Data()).toInt(&ok); + if (!ok) time =3D 10; ///< parsing argument of raw header 'Backoff= ' failed? 10 seconds is fallback + d->api->startBackoff(time); + } else if (reply->hasRawHeader("Retry-After")) { + bool ok =3D false; + int time =3D QString::fromLatin1(reply->rawHeader("Retry-After").c= onstData()).toInt(&ok); + if (!ok) time =3D 10; ///< parsing argument of raw header 'Retry-A= fter' failed? 10 seconds is fallback + d->api->startBackoff(time); + } = if (reply->error() =3D=3D QNetworkReply::NoError && ok) { const QString bibTeXcode =3D QString::fromUtf8(reply->readAll().co= nstData()); diff --git a/src/networking/zotero/tags.cpp b/src/networking/zotero/tags.cpp index f5ce1d7d..83556fc1 100644 --- a/src/networking/zotero/tags.cpp +++ b/src/networking/zotero/tags.cpp @@ -1,5 +1,5 @@ /*************************************************************************= ** - * Copyright (C) 2004-2014 by Thomas Fischer = * + * Copyright (C) 2004-2017 by Thomas Fischer = * * = * * This program is free software; you can redistribute it and/or modify = * * it under the terms of the GNU General Public License as published by = * @@ -65,6 +65,7 @@ Tags::Tags(QSharedPointer api, QObject *pare= nt) url.setPath(url.path() + QStringLiteral("/tags")); = if (api->inBackoffMode()) + /// If Zotero asked to 'back off', wait until this period is over = before issuing the next request QTimer::singleShot((d->api->backoffSecondsLeft() + 1) * 1000, [ = =3D ]() { d->requestZoteroUrl(url); }); @@ -96,10 +97,17 @@ void Tags::finishedFetchingTags() { QNetworkReply *reply =3D static_cast(sender()); = - if (reply->hasRawHeader("Backoff")) - d->api->startBackoff(QString::fromLatin1(reply->rawHeader("Backoff= ").constData()).toInt()); - else if (reply->hasRawHeader("Retry-After")) - d->api->startBackoff(QString::fromLatin1(reply->rawHeader("Retry-A= fter").constData()).toInt()); + if (reply->hasRawHeader("Backoff")) { + bool ok =3D false; + int time =3D QString::fromLatin1(reply->rawHeader("Backoff").const= Data()).toInt(&ok); + if (!ok) time =3D 10; ///< parsing argument of raw header 'Backoff= ' failed? 10 seconds is fallback + d->api->startBackoff(time); + } else if (reply->hasRawHeader("Retry-After")) { + bool ok =3D false; + int time =3D QString::fromLatin1(reply->rawHeader("Retry-After").c= onstData()).toInt(&ok); + if (!ok) time =3D 10; ///< parsing argument of raw header 'Retry-A= fter' failed? 10 seconds is fallback + d->api->startBackoff(time); + } = if (reply->error() =3D=3D QNetworkReply::NoError) { QString nextPage; @@ -131,9 +139,15 @@ void Tags::finishedFetchingTags() break; } = - if (!nextPage.isEmpty()) - d->requestZoteroUrl(nextPage); - else { + if (!nextPage.isEmpty()) { + if (d->api->inBackoffMode()) + /// If Zotero asked to 'back off', wait until this period = is over before issuing the next request + QTimer::singleShot((d->api->backoffSecondsLeft() + 1) * 10= 00, [ =3D ]() { + d->requestZoteroUrl(nextPage); + }); + else + d->requestZoteroUrl(nextPage); + } else { d->busy =3D false; d->initialized =3D true; emit finishedLoading();