Rewrite HttpGet based on QNetworkAccessManager.

HttpGet used to use QHttp which has been deprecated since a while and has been
removed from Qt5. Rewrite the class based on QNetworkAccessManager which is the
recommended way these days.

Change-Id: I4902309c433a85ec18e157ef3a9f5e60fd0f4b1f

rbutil/rbutilqt/base/httpget.cpp

@@ -6,7 +6,7 @@
  *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
  *                     \/            \/     \/    \/            \/
  *
- *   Copyright (C) 2007 by Dominik Riebeling
+ *   Copyright (C) 2013 by Dominik Riebeling
  *
  * All files in this archive are subject to the GNU General Public License.
  * See the file COPYING in the source tree root for full license agreement.
@@ -16,52 +16,31 @@
  *
  ****************************************************************************/
 
-#include <QtCore>
 #include <QtNetwork>
 #include <QtDebug>
 
+#include <QNetworkAccessManager>
+#include <QNetworkRequest>
+
 #include "httpget.h"
 
-QDir HttpGet::m_globalCache; //< global cach path value for new objects
-QUrl HttpGet::m_globalProxy; //< global proxy value for new objects
 QString HttpGet::m_globalUserAgent; //< globally set user agent for requests
+QDir HttpGet::m_globalCache; //< global cach path value for new objects
+QNetworkProxy HttpGet::m_globalProxy;
 
 HttpGet::HttpGet(QObject *parent)
     : QObject(parent)
 {
-    outputToBuffer = true;
+    m_mgr = new QNetworkAccessManager(this);
-    m_cached = false;
+    m_cache = NULL;
-    getRequest = -1;
-    headRequest = -1;
-    // if a request is cancelled before a reponse is available return some
-    // hint about this in the http response instead of nonsense.
-    m_response = -1;
-    m_useproxy = false;
-
-    // default to global proxy / cache if not empty.
-    // proxy is automatically enabled, disable it by setting an empty proxy
-    // cache is enabled to be in line, can get disabled with setCache(bool)
-    if(!m_globalProxy.isEmpty())
-        setProxy(m_globalProxy);
-    m_usecache = false;
     m_cachedir = m_globalCache;
-
+    setCache(true);
-    m_serverTimestamp = QDateTime();
+    connect(m_mgr, SIGNAL(finished(QNetworkReply*)),
-
+            this, SLOT(requestFinished(QNetworkReply*)));
-    connect(&http, SIGNAL(done(bool)), this, SLOT(httpDone(bool)));
+    m_outputFile = NULL;
-    connect(&http, SIGNAL(dataReadProgress(int, int)),
+    m_lastServerTimestamp = QDateTime();
-            this, SIGNAL(dataReadProgress(int, int)));
+    m_proxy = QNetworkProxy::NoProxy;
-    connect(&http, SIGNAL(requestFinished(int, bool)),
+    m_reply = NULL;
-            this, SLOT(httpFinished(int, bool)));
-    connect(&http, SIGNAL(responseHeaderReceived(const QHttpResponseHeader&)),
-            this, SLOT(httpResponseHeader(const QHttpResponseHeader&)));
-//    connect(&http, SIGNAL(stateChanged(int)), this, SLOT(httpState(int)));
-    connect(&http, SIGNAL(requestStarted(int)), this, SLOT(httpStarted(int)));
-
-    connect(&http, SIGNAL(readyRead(const QHttpResponseHeader&)),
-            this, SLOT(httpResponseHeader(const QHttpResponseHeader&)));
-    connect(this, SIGNAL(headerFinished()), this, SLOT(getFileFinish()));
-
 }
 
 
@@ -69,10 +48,10 @@ HttpGet::HttpGet(QObject *parent)
 //  @param d new directory to use as cache path
 void HttpGet::setCache(const QDir& d)
 {
-    m_cachedir = d;
+    if(m_cache && m_cachedir == d.absolutePath())
-    bool result;
+        return;
-    result = initializeCache(d);
+    m_cachedir = d.absolutePath();
-    m_usecache = result;
+    setCache(true);
 }
 
 
@@ -81,29 +60,29 @@ void HttpGet::setCache(const QDir& d)
  */
 void HttpGet::setCache(bool c)
 {
-    m_usecache = c;
+    // don't change cache if it's already (un)set.
-    // make sure cache is initialized
+    if(c && m_cache) return;
-    if(c)
+    if(!c && !m_cache) return;
-        m_usecache = initializeCache(m_cachedir);
+    // don't delete the old cache directly, it might still be in use. Just
-}
+    // instruct it to delete itself later.
+    if(m_cache) m_cache->deleteLater();
+    m_cache = NULL;
 
+    QString path = m_cachedir.absolutePath();
 
-bool HttpGet::initializeCache(const QDir& d)
+    if(!c || m_cachedir.absolutePath().isEmpty()) {
-{
+        qDebug() << "[HttpGet] disabling download cache";
-    bool result;
-    QString p = d.absolutePath() + "/rbutil-cache";
-    if(QFileInfo(d.absolutePath()).isDir())
-    {
-        if(!QFileInfo(p).isDir())
-            result = d.mkdir("rbutil-cache");
-        else
-            result = true;
     }
-    else
+    else {
-        result = false;
+        // append the cache path to make it unique in case the path points to
-
+        // the system temporary path. In that case using it directly might
-    return result;
+        // cause problems. Extra path also used in configure dialog.
-
+        path += "/rbutil-cache";
+        qDebug() << "[HttpGet] setting cache folder to" << path;
+        m_cache = new QNetworkDiskCache(this);
+        m_cache->setCacheDirectory(path);
+    }
+    m_mgr->setCache(m_cache);
 }
 
 
@@ -112,324 +91,133 @@ bool HttpGet::initializeCache(const QDir& d)
  */
 QByteArray HttpGet::readAll()
 {
-    return dataBuffer;
+    return m_data;
 }
 
 
 void HttpGet::setProxy(const QUrl &proxy)
 {
-    m_proxy = proxy;
+    qDebug() << "[HttpGet] Proxy set to" << proxy;
-    m_useproxy = true;
+    m_proxy.setType(QNetworkProxy::HttpProxy);
-    http.setProxy(m_proxy.host(), m_proxy.port(),
+    m_proxy.setHostName(proxy.host());
-                  m_proxy.userName(), m_proxy.password());
+    m_proxy.setPort(proxy.port());
+    m_proxy.setUser(proxy.userName());
+    m_proxy.setPassword(proxy.password());
+    m_mgr->setProxy(m_proxy);
 }
 
 
 void HttpGet::setProxy(bool enable)
 {
-    if(enable) {
+    if(enable) m_mgr->setProxy(m_proxy);
-        m_useproxy = true;
+    else m_mgr->setProxy(QNetworkProxy::NoProxy);
-        http.setProxy(m_proxy.host(), m_proxy.port(),
-                      m_proxy.userName(), m_proxy.password());
-    }
-    else {
-        m_useproxy = false;
-        http.setProxy("", 0);
-    }
 }
 
 
 void HttpGet::setFile(QFile *file)
 {
-    outputFile = file;
+    m_outputFile = file;
-    outputToBuffer = false;
 }
 
 
 void HttpGet::abort()
 {
-    qDebug() << "[HTTP] Aborting requests, pending:" << http.hasPendingRequests();
+    if(m_reply) m_reply->abort();
-    http.abort();
+}
-    if(!outputToBuffer)
+
-        outputFile->close();
+
+void HttpGet::requestFinished(QNetworkReply* reply)
+{
+    m_lastStatusCode
+        = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
+    qDebug() << "[HttpGet] Request finished, status code:" << m_lastStatusCode;
+    m_lastServerTimestamp
+        = reply->header(QNetworkRequest::LastModifiedHeader).toDateTime().toLocalTime();
+    qDebug() << "[HttpGet] Data from cache:"
+             << reply->attribute(QNetworkRequest::SourceIsFromCacheAttribute).toBool();
+    m_lastRequestCached =
+        reply->attribute(QNetworkRequest::SourceIsFromCacheAttribute).toBool();
+    if(reply->attribute(QNetworkRequest::RedirectionTargetAttribute).isValid()) {
+        // handle relative URLs using QUrl::resolved()
+        QUrl org = reply->request().url();
+        QUrl url = QUrl(org).resolved(
+                reply->attribute(QNetworkRequest::RedirectionTargetAttribute).toUrl());
+        // reconstruct query
+#if QT_VERSION < 0x050000
+        QList<QPair<QByteArray, QByteArray> > qitms = org.encodedQueryItems();
+        for(int i = 0; i < qitms.size(); ++i)
+            url.addEncodedQueryItem(qitms.at(i).first, qitms.at(i).second);
+#else
+        url.setQuery(org.query());
+#endif
+        qDebug() << "[HttpGet] Redirected to" << url;
+        startRequest(url);
+        return;
+    }
+    else if(m_lastStatusCode == 200) {
+        m_data = reply->readAll();
+        if(m_outputFile && m_outputFile->open(QIODevice::WriteOnly)) {
+            m_outputFile->write(m_data);
+            m_outputFile->close();
+        }
+        emit done(false);
+    }
+    else {
+        m_data.clear();
+        emit done(true);
+    }
+    reply->deleteLater();
+    m_reply = NULL;
+}
+
+
+void HttpGet::downloadProgress(qint64 received, qint64 total)
+{
+    emit dataReadProgress((int)received, (int)total);
+}
+
+
+void HttpGet::startRequest(QUrl url)
+{
+    qDebug() << "[HttpGet] Request started";
+    QNetworkRequest req(url);
+    if(!m_globalUserAgent.isEmpty())
+        req.setRawHeader("User-Agent", m_globalUserAgent.toLatin1());
+
+    m_reply = m_mgr->get(req);
+    connect(m_reply, SIGNAL(error(QNetworkReply::NetworkError)),
+            this, SLOT(networkError(QNetworkReply::NetworkError)));
+    connect(m_reply, SIGNAL(downloadProgress(qint64, qint64)),
+            this, SLOT(downloadProgress(qint64, qint64)));
+}
+
+
+void HttpGet::networkError(QNetworkReply::NetworkError error)
+{
+    qDebug() << "[HttpGet] NetworkError occured:"
+             << error << m_reply->errorString();
+    m_lastErrorString = m_reply->errorString();
 }
 
 
 bool HttpGet::getFile(const QUrl &url)
 {
-    if (!url.isValid()) {
+    qDebug() << "[HttpGet] Get URI" << url.toString();
-        qDebug() << "[HTTP] Error: Invalid URL" << endl;
+    m_data.clear();
-        return false;
+    startRequest(url);
-    }
 
-    if (url.scheme() != "http") {
+    return false;
-        qDebug() << "[HTTP] Error: URL must start with 'http:'" << endl;
-        return false;
-    }
-
-    if (url.path().isEmpty()) {
-        qDebug() << "[HTTP] Error: URL has no path" << endl;
-        return false;
-    }
-    m_serverTimestamp = QDateTime();
-    // if no output file was set write to buffer
-    if(!outputToBuffer) {
-        if (!outputFile->open(QIODevice::ReadWrite)) {
-            qDebug() << "[HTTP] Error: Cannot open " << qPrintable(outputFile->fileName())
-                     << " for writing: " << qPrintable(outputFile->errorString());
-            return false;
-        }
-    }
-    else {
-        // output to buffer. Make sure buffer is empty so no old data gets
-        // returned in case the object is reused.
-        dataBuffer.clear();
-    }
-    qDebug() << "[HTTP] GET URI" << url.toEncoded();
-    // create request
-    http.setHost(url.host(), url.port(80));
-    // construct query (if any)
-    QList<QPair<QString, QString> > qitems = url.queryItems();
-    if(url.hasQuery()) {
-        m_query = "?";
-        for(int i = 0; i < qitems.size(); i++)
-            m_query += QUrl::toPercentEncoding(qitems.at(i).first, "/") + "="
-                  + QUrl::toPercentEncoding(qitems.at(i).second, "/") + "&";
-    }
-
-    // create hash used for caching
-    m_hash = QCryptographicHash::hash(url.toEncoded(), QCryptographicHash::Md5).toHex();
-    m_cachefile = m_cachedir.absolutePath() + "/rbutil-cache/" + m_hash;
-    // RFC2616: the absoluteURI form must get used when the request is being
-    // sent to a proxy.
-    m_path.clear();
-    if(m_useproxy)
-        m_path = url.scheme() + "://" + url.host();
-    m_path += QString(QUrl::toPercentEncoding(url.path(), "/"));
-
-    // construct request header
-    m_header.setValue("Host", url.host());
-    m_header.setValue("User-Agent", m_globalUserAgent);
-    m_header.setValue("Connection", "Keep-Alive");
-
-    if(!m_usecache || !QFileInfo(m_cachefile).exists()) {
-        getFileFinish();
-    }
-    else {
-        // schedule HTTP header request
-        m_header.setRequest("HEAD", m_path + m_query);
-        headRequest = http.request(m_header);
-        qDebug() << "[HTTP] HEAD scheduled:  " << headRequest;
-    }
-
-    return true;
 }
 
 
-void HttpGet::getFileFinish()
+QString HttpGet::errorString(void)
 {
-    QString indexFile = m_cachedir.absolutePath() + "/rbutil-cache/cache.txt";
+    return m_lastErrorString;
-    if(m_usecache) {
-        // check if the file is present in cache
-        QFileInfo cachefile = QFileInfo(m_cachefile);
-        if(cachefile.isReadable()
-            && cachefile.size() > 0
-            && cachefile.lastModified() > m_serverTimestamp) {
-
-            qDebug() << "[HTTP] Cache: up-to-date file found:" << m_cachefile;
-
-            getRequest = -1;
-            QFile c(m_cachefile);
-            if(!outputToBuffer) {
-                qDebug() << "[HTTP] Cache: copying file to output"
-                         << outputFile->fileName();
-                c.open(QIODevice::ReadOnly);
-                outputFile->open(QIODevice::ReadWrite);
-                outputFile->write(c.readAll());
-                outputFile->close();
-                c.close();
-            }
-            else {
-                qDebug() << "[HTTP] Cache: reading file into buffer";
-                c.open(QIODevice::ReadOnly);
-                dataBuffer = c.readAll();
-                c.close();
-            }
-            m_response = 200; // fake "200 OK" HTTP response
-            m_cached = true;
-            httpDone(false); // we're done now. Handle http "done" signal.
-            return;
-        }
-        else {
-            // unlink old cache file
-            if(cachefile.isReadable()) {
-                QFile(m_cachefile).remove();
-                qDebug() << "[HTTP] Cache: outdated, timestamp:"
-                         << cachefile.lastModified();
-            }
-            qDebug() << "[HTTP] Cache: caching as" << m_cachefile;
-            // update cache index file
-            QFile idxFile(indexFile);
-            idxFile.open(QIODevice::ReadOnly);
-            QByteArray currLine;
-            do {
-                QByteArray currLine = idxFile.readLine(1000);
-                if(currLine.startsWith(m_hash.toUtf8()))
-                    break;
-            } while(!currLine.isEmpty());
-            idxFile.close();
-            if(currLine.isEmpty()) {
-                idxFile.open(QIODevice::Append);
-                QString outline = m_hash + "\t" + m_header.value("Host") + "\t"
-                    + m_path + "\t" + m_query + "\n";
-                idxFile.write(outline.toUtf8());
-                idxFile.close();
-            }
-        }
-
-    }
-    else {
-        qDebug() << "[HTTP] cache DISABLED";
-    }
-    // schedule GET request
-
-    m_header.setRequest("GET", m_path + m_query);
-    if(outputToBuffer) {
-        qDebug() << "[HTTP] downloading to buffer.";
-        getRequest = http.request(m_header);
-    }
-    else {
-        qDebug() << "[HTTP] downloading to file:"
-            << qPrintable(outputFile->fileName());
-       getRequest = http.request(m_header, 0, outputFile);
-    }
-    qDebug() << "[HTTP] GET scheduled:   " << getRequest;
-
-    return;
 }
 
 
-void HttpGet::httpDone(bool error)
+int HttpGet::httpResponse(void)
 {
-    if (error) {
+    return m_lastStatusCode;
-        qDebug() << "[HTTP] Error:" << qPrintable(http.errorString()) << httpResponse();
-    }
-    if(!outputToBuffer)
-        outputFile->close();
-
-    if(m_usecache && !m_cached && !error) {
-        qDebug() << "[HTTP] creating cache file" << m_cachefile;
-        QFile c(m_cachefile);
-        c.open(QIODevice::ReadWrite);
-        if(!outputToBuffer) {
-            outputFile->open(QIODevice::ReadOnly | QIODevice::Truncate);
-            c.write(outputFile->readAll());
-            outputFile->close();
-        }
-        else
-            c.write(dataBuffer);
-
-        c.close();
-    }
-    // take care of concurring requests. If there is still one running,
-    // don't emit done(). That request will call this slot again.
-    if(http.currentId() == 0 && !http.hasPendingRequests())
-        emit done(error);
-}
-
-
-void HttpGet::httpFinished(int id, bool error)
-{
-    qDebug() << "[HTTP] Request finished:" << id << "Error:" << error
-             << "pending requests:" << http.hasPendingRequests();
-    if(id == getRequest) {
-        dataBuffer = http.readAll();
-        emit requestFinished(id, error);
-    }
-
-    QHttpResponseHeader h = http.lastResponse();
-    QString date = h.value("Last-Modified").simplified();
-    if(date.isEmpty()) {
-        m_serverTimestamp = QDateTime(); // no value = invalid
-        if(id == headRequest)
-            emit headerFinished();
-        else
-            emit requestFinished(id, error);
-        return;
-    }
-    // to successfully parse the date strip weekday and timezone
-    date.remove(0, date.indexOf(" ") + 1);
-    if(date.endsWith("GMT")) date.truncate(date.indexOf(" GMT"));
-    // distinguish input formats (see RFC1945)
-    if(date.at(0).isLetter()) // asctime format
-        m_serverTimestamp = QLocale::c().toDateTime(date, "MMM d hh:mm:ss yyyy");
-    else // RFC 822
-        m_serverTimestamp = QLocale::c().toDateTime(date, "dd MMM yyyy hh:mm:ss");
-
-    qDebug() << "[HTTP] file server date:" << date
-             << "parsed:" << m_serverTimestamp;
-
-    if(id == headRequest)
-        emit headerFinished();
-    else
-        emit requestFinished(id, error);
-    return;
-}
-
-void HttpGet::httpStarted(int id)
-{
-    qDebug() << "[HTTP] Request started: " << id << "Header req:"
-             << headRequest << "Get req:" << getRequest;
-}
-
-
-QString HttpGet::errorString()
-{
-    return http.errorString();
-}
-
-
-void HttpGet::httpResponseHeader(const QHttpResponseHeader &resp)
-{
-    // if there is a network error abort all scheduled requests for
-    // this download
-    m_response = resp.statusCode();
-
-    // 301 -- moved permanently
-    // 302 -- found
-    // 303 -- see other
-    // 307 -- moved temporarily
-    // in all cases, header: location has the correct address so we can follow.
-    if(m_response == 301 || m_response == 302 || m_response == 303 || m_response == 307) {
-        //abort without sending any signals
-        http.blockSignals(true);
-        http.abort();
-        http.blockSignals(false);
-        // start new request with new url
-        qDebug() << "[HTTP] response =" << m_response << "- following";
-        getFile(resp.value("location") + m_query);
-    }
-    else if(m_response != 200) {
-        // all other errors are fatal.
-        http.abort();
-        qDebug() << "[HTTP] Response error:" << m_response << resp.reasonPhrase();
-    }
-
-}
-
-
-int HttpGet::httpResponse()
-{
-    return m_response;
-}
-
-
-void HttpGet::httpState(int state)
-{
-    QString s[] = {"Unconnected", "HostLookup", "Connecting", "Sending",
-        "Reading", "Connected", "Closing"};
-    if(state <= 6)
-        qDebug() << "[HTTP] State:" << s[state];
-    else qDebug() << "[HTTP] State:" << state;
 }
 

rbutil/rbutilqt/base/httpget.h

@@ -24,6 +24,7 @@
 
 #include <QtCore>
 #include <QtNetwork>
+#include <QNetworkAccessManager>
 
 class HttpGet : public QObject
 {
@@ -42,14 +43,34 @@ class HttpGet : public QObject
         int httpResponse(void);
         QByteArray readAll(void);
         bool isCached()
-            { return m_cached; }
+            { return m_lastRequestCached; }
         QDateTime timestamp(void)
-            { return m_serverTimestamp; }
+            { return m_lastServerTimestamp; }
-        static void setGlobalCache(const QDir& d) //< set global cache path
+        //< set global cache path
-            { m_globalCache = d; }
+        static void setGlobalCache(const QDir& d)
-        static void setGlobalProxy(const QUrl& p) //< set global proxy value
+        {
-            { m_globalProxy = p; }
+            qDebug() << "[HttpGet] Global cache set to" << d.absolutePath();
-        static void setGlobalUserAgent(const QString& u) //< set global user agent string
+            m_globalCache = d;
+        }
+        //< set global proxy value
+        static void setGlobalProxy(const QUrl& p)
+        {
+            qDebug() << "[HttpGet] setting global proxy" << p;
+            if(!p.isValid() || p.isEmpty()) {
+                HttpGet::m_globalProxy.setType(QNetworkProxy::NoProxy);
+            }
+            else {
+                HttpGet::m_globalProxy.setType(QNetworkProxy::HttpProxy);
+                HttpGet::m_globalProxy.setHostName(p.host());
+                HttpGet::m_globalProxy.setPort(p.port());
+                HttpGet::m_globalProxy.setUser(p.userName());
+                HttpGet::m_globalProxy.setPassword(p.password());
+            }
+            QNetworkProxy::setApplicationProxy(QNetworkProxy::NoProxy);
+            QNetworkProxy::setApplicationProxy(HttpGet::m_globalProxy);
+        }
+        //< set global user agent string
+        static void setGlobalUserAgent(const QString& u)
             { m_globalUserAgent = u; }
 
     public slots:
@@ -62,37 +83,28 @@ class HttpGet : public QObject
         void headerFinished(void);
 
     private slots:
-        void httpDone(bool error);
+        void requestFinished(QNetworkReply* reply);
-        void httpFinished(int, bool);
+        void startRequest(QUrl url);
-        void httpResponseHeader(const QHttpResponseHeader&);
+        void downloadProgress(qint64 received, qint64 total);
-        void httpState(int);
+        void networkError(QNetworkReply::NetworkError error);
-        void httpStarted(int);
-        void getFileFinish(void);
 
     private:
-        bool initializeCache(const QDir&);
+        static QString m_globalUserAgent;
-        QHttp http; //< download object
+        static QNetworkProxy m_globalProxy;
-        QFile *outputFile;
+        QNetworkAccessManager *m_mgr;
-        int m_response; //< http response
+        QNetworkReply *m_reply;
-        int getRequest;  //! get file http request id
+        QNetworkDiskCache *m_cache;
-        int headRequest; //! get http header request id
-        QByteArray dataBuffer;
-        bool outputToBuffer;
-        bool m_usecache;
         QDir m_cachedir;
-        QString m_cachefile; // cached filename
-        bool m_cached;
-        QUrl m_proxy;
-        bool m_useproxy;
-        QDateTime m_serverTimestamp; //< timestamp of file on server
-        QString m_query; //< constructed query to pass http getter
-        QString m_path; //< constructed path to pass http getter
-        QString m_hash; //< caching hash
-        QHttpRequestHeader m_header;
-
         static QDir m_globalCache; //< global cache path value
-        static QUrl m_globalProxy; //< global proxy value
+        QByteArray m_data;
-        static QString m_globalUserAgent; //< global user agent string
+        QFile *m_outputFile;
+        int m_lastStatusCode;
+        QString m_lastErrorString;
+        QDateTime m_lastServerTimestamp;
+        bool m_lastRequestCached;
+        QNetworkProxy m_proxy;
 };
 
+
 #endif
+