Nyaaori/rockbox
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Rewrite HttpGet based on QNetworkAccessManager.

Browse source 
HttpGet used to use QHttp which has been deprecated since a while and has been
removed from Qt5. Rewrite the class based on QNetworkAccessManager which is the
recommended way these days.

Change-Id: I4902309c433a85ec18e157ef3a9f5e60fd0f4b1f
This commit is contained in:
Dominik Riebeling 2013-01-25 22:57:19 +01:00
parent 13c5c463d0
commit 3144c2c74c
2 changed files with 179 additions and 379 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

478
rbutil/rbutilqt/base/httpget.cpp
View file

@ -6,7 +6,7 @@
* Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \
* \/ \/ \/ \/ \/
*
* Copyright (C) 2007 by Dominik Riebeling
* Copyright (C) 2013 by Dominik Riebeling
*
* All files in this archive are subject to the GNU General Public License.
* See the file COPYING in the source tree root for full license agreement.
@ -16,52 +16,31 @@
*
****************************************************************************/
#include <QtCore>
#include <QtNetwork>
#include <QtDebug>
#include <QNetworkAccessManager>
#include <QNetworkRequest>
#include "httpget.h"
QDir HttpGet::m_globalCache; //< global cach path value for new objects
QUrl HttpGet::m_globalProxy; //< global proxy value for new objects
QString HttpGet::m_globalUserAgent; //< globally set user agent for requests
QDir HttpGet::m_globalCache; //< global cach path value for new objects
QNetworkProxy HttpGet::m_globalProxy;
HttpGet::HttpGet(QObject *parent)
: QObject(parent)
{
outputToBuffer = true;
m_cached = false;
getRequest = -1;
headRequest = -1;
// if a request is cancelled before a reponse is available return some
// hint about this in the http response instead of nonsense.
m_response = -1;
m_useproxy = false;
// default to global proxy / cache if not empty.
// proxy is automatically enabled, disable it by setting an empty proxy
// cache is enabled to be in line, can get disabled with setCache(bool)
if(!m_globalProxy.isEmpty())
setProxy(m_globalProxy);
m_usecache = false;
m_mgr = new QNetworkAccessManager(this);
m_cache = NULL;
m_cachedir = m_globalCache;
m_serverTimestamp = QDateTime();
connect(&http, SIGNAL(done(bool)), this, SLOT(httpDone(bool)));
connect(&http, SIGNAL(dataReadProgress(int, int)),
this, SIGNAL(dataReadProgress(int, int)));
connect(&http, SIGNAL(requestFinished(int, bool)),
this, SLOT(httpFinished(int, bool)));
connect(&http, SIGNAL(responseHeaderReceived(const QHttpResponseHeader&)),
this, SLOT(httpResponseHeader(const QHttpResponseHeader&)));
// connect(&http, SIGNAL(stateChanged(int)), this, SLOT(httpState(int)));
connect(&http, SIGNAL(requestStarted(int)), this, SLOT(httpStarted(int)));
connect(&http, SIGNAL(readyRead(const QHttpResponseHeader&)),
this, SLOT(httpResponseHeader(const QHttpResponseHeader&)));
connect(this, SIGNAL(headerFinished()), this, SLOT(getFileFinish()));
setCache(true);
connect(m_mgr, SIGNAL(finished(QNetworkReply*)),
this, SLOT(requestFinished(QNetworkReply*)));
m_outputFile = NULL;
m_lastServerTimestamp = QDateTime();
m_proxy = QNetworkProxy::NoProxy;
m_reply = NULL;
}
@ -69,10 +48,10 @@ HttpGet::HttpGet(QObject *parent)
// @param d new directory to use as cache path
void HttpGet::setCache(const QDir& d)
{
m_cachedir = d;
bool result;
result = initializeCache(d);
m_usecache = result;
if(m_cache && m_cachedir == d.absolutePath())
return;
m_cachedir = d.absolutePath();
setCache(true);
}
@ -81,29 +60,29 @@ void HttpGet::setCache(const QDir& d)
*/
void HttpGet::setCache(bool c)
{
m_usecache = c;
// make sure cache is initialized
if(c)
m_usecache = initializeCache(m_cachedir);
}
// don't change cache if it's already (un)set.
if(c && m_cache) return;
if(!c && !m_cache) return;
// don't delete the old cache directly, it might still be in use. Just
// instruct it to delete itself later.
if(m_cache) m_cache->deleteLater();
m_cache = NULL;
QString path = m_cachedir.absolutePath();
bool HttpGet::initializeCache(const QDir& d)
{
bool result;
QString p = d.absolutePath() + "/rbutil-cache";
if(QFileInfo(d.absolutePath()).isDir())
{
if(!QFileInfo(p).isDir())
result = d.mkdir("rbutil-cache");
else
result = true;
if(!c || m_cachedir.absolutePath().isEmpty()) {
qDebug() << "[HttpGet] disabling download cache";
}
else
result = false;
return result;
else {
// append the cache path to make it unique in case the path points to
// the system temporary path. In that case using it directly might
// cause problems. Extra path also used in configure dialog.
path += "/rbutil-cache";
qDebug() << "[HttpGet] setting cache folder to" << path;
m_cache = new QNetworkDiskCache(this);
m_cache->setCacheDirectory(path);
}
m_mgr->setCache(m_cache);
}
@ -112,324 +91,133 @@ bool HttpGet::initializeCache(const QDir& d)
*/
QByteArray HttpGet::readAll()
{
return dataBuffer;
return m_data;
}
void HttpGet::setProxy(const QUrl &proxy)
{
m_proxy = proxy;
m_useproxy = true;
http.setProxy(m_proxy.host(), m_proxy.port(),
m_proxy.userName(), m_proxy.password());
qDebug() << "[HttpGet] Proxy set to" << proxy;
m_proxy.setType(QNetworkProxy::HttpProxy);
m_proxy.setHostName(proxy.host());
m_proxy.setPort(proxy.port());
m_proxy.setUser(proxy.userName());
m_proxy.setPassword(proxy.password());
m_mgr->setProxy(m_proxy);
}
void HttpGet::setProxy(bool enable)
{
if(enable) {
m_useproxy = true;
http.setProxy(m_proxy.host(), m_proxy.port(),
m_proxy.userName(), m_proxy.password());
}
else {
m_useproxy = false;
http.setProxy("", 0);
}
if(enable) m_mgr->setProxy(m_proxy);
else m_mgr->setProxy(QNetworkProxy::NoProxy);
}
void HttpGet::setFile(QFile *file)
{
outputFile = file;
outputToBuffer = false;
m_outputFile = file;
}
void HttpGet::abort()
{
qDebug() << "[HTTP] Aborting requests, pending:" << http.hasPendingRequests();
http.abort();
if(!outputToBuffer)
outputFile->close();
if(m_reply) m_reply->abort();
}
void HttpGet::requestFinished(QNetworkReply* reply)
{
m_lastStatusCode
= reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
qDebug() << "[HttpGet] Request finished, status code:" << m_lastStatusCode;
m_lastServerTimestamp
= reply->header(QNetworkRequest::LastModifiedHeader).toDateTime().toLocalTime();
qDebug() << "[HttpGet] Data from cache:"
<< reply->attribute(QNetworkRequest::SourceIsFromCacheAttribute).toBool();
m_lastRequestCached =
reply->attribute(QNetworkRequest::SourceIsFromCacheAttribute).toBool();
if(reply->attribute(QNetworkRequest::RedirectionTargetAttribute).isValid()) {
// handle relative URLs using QUrl::resolved()
QUrl org = reply->request().url();
QUrl url = QUrl(org).resolved(
reply->attribute(QNetworkRequest::RedirectionTargetAttribute).toUrl());
// reconstruct query
#if QT_VERSION < 0x050000
QList<QPair<QByteArray, QByteArray> > qitms = org.encodedQueryItems();
for(int i = 0; i < qitms.size(); ++i)
url.addEncodedQueryItem(qitms.at(i).first, qitms.at(i).second);
#else
url.setQuery(org.query());
#endif
qDebug() << "[HttpGet] Redirected to" << url;
startRequest(url);
return;
}
else if(m_lastStatusCode == 200) {
m_data = reply->readAll();
if(m_outputFile && m_outputFile->open(QIODevice::WriteOnly)) {
m_outputFile->write(m_data);
m_outputFile->close();
}
emit done(false);
}
else {
m_data.clear();
emit done(true);
}
reply->deleteLater();
m_reply = NULL;
}
void HttpGet::downloadProgress(qint64 received, qint64 total)
{
emit dataReadProgress((int)received, (int)total);
}
void HttpGet::startRequest(QUrl url)
{
qDebug() << "[HttpGet] Request started";
QNetworkRequest req(url);
if(!m_globalUserAgent.isEmpty())
req.setRawHeader("User-Agent", m_globalUserAgent.toLatin1());
m_reply = m_mgr->get(req);
connect(m_reply, SIGNAL(error(QNetworkReply::NetworkError)),
this, SLOT(networkError(QNetworkReply::NetworkError)));
connect(m_reply, SIGNAL(downloadProgress(qint64, qint64)),
this, SLOT(downloadProgress(qint64, qint64)));
}
void HttpGet::networkError(QNetworkReply::NetworkError error)
{
qDebug() << "[HttpGet] NetworkError occured:"
<< error << m_reply->errorString();
m_lastErrorString = m_reply->errorString();
}
bool HttpGet::getFile(const QUrl &url)
{
if (!url.isValid()) {
qDebug() << "[HTTP] Error: Invalid URL" << endl;
return false;
}
qDebug() << "[HttpGet] Get URI" << url.toString();
m_data.clear();
startRequest(url);
if (url.scheme() != "http") {
qDebug() << "[HTTP] Error: URL must start with 'http:'" << endl;
return false;
}
if (url.path().isEmpty()) {
qDebug() << "[HTTP] Error: URL has no path" << endl;
return false;
}
m_serverTimestamp = QDateTime();
// if no output file was set write to buffer
if(!outputToBuffer) {
if (!outputFile->open(QIODevice::ReadWrite)) {
qDebug() << "[HTTP] Error: Cannot open " << qPrintable(outputFile->fileName())
<< " for writing: " << qPrintable(outputFile->errorString());
return false;
}
}
else {
// output to buffer. Make sure buffer is empty so no old data gets
// returned in case the object is reused.
dataBuffer.clear();
}
qDebug() << "[HTTP] GET URI" << url.toEncoded();
// create request
http.setHost(url.host(), url.port(80));
// construct query (if any)
QList<QPair<QString, QString> > qitems = url.queryItems();
if(url.hasQuery()) {
m_query = "?";
for(int i = 0; i < qitems.size(); i++)
m_query += QUrl::toPercentEncoding(qitems.at(i).first, "/") + "="
+ QUrl::toPercentEncoding(qitems.at(i).second, "/") + "&";
}
// create hash used for caching
m_hash = QCryptographicHash::hash(url.toEncoded(), QCryptographicHash::Md5).toHex();
m_cachefile = m_cachedir.absolutePath() + "/rbutil-cache/" + m_hash;
// RFC2616: the absoluteURI form must get used when the request is being
// sent to a proxy.
m_path.clear();
if(m_useproxy)
m_path = url.scheme() + "://" + url.host();
m_path += QString(QUrl::toPercentEncoding(url.path(), "/"));
// construct request header
m_header.setValue("Host", url.host());
m_header.setValue("User-Agent", m_globalUserAgent);
m_header.setValue("Connection", "Keep-Alive");
if(!m_usecache || !QFileInfo(m_cachefile).exists()) {
getFileFinish();
}
else {
// schedule HTTP header request
m_header.setRequest("HEAD", m_path + m_query);
headRequest = http.request(m_header);
qDebug() << "[HTTP] HEAD scheduled: " << headRequest;
}
return true;
return false;
}
void HttpGet::getFileFinish()
QString HttpGet::errorString(void)
{
QString indexFile = m_cachedir.absolutePath() + "/rbutil-cache/cache.txt";
if(m_usecache) {
// check if the file is present in cache
QFileInfo cachefile = QFileInfo(m_cachefile);
if(cachefile.isReadable()
&& cachefile.size() > 0
&& cachefile.lastModified() > m_serverTimestamp) {
qDebug() << "[HTTP] Cache: up-to-date file found:" << m_cachefile;
getRequest = -1;
QFile c(m_cachefile);
if(!outputToBuffer) {
qDebug() << "[HTTP] Cache: copying file to output"
<< outputFile->fileName();
c.open(QIODevice::ReadOnly);
outputFile->open(QIODevice::ReadWrite);
outputFile->write(c.readAll());
outputFile->close();
c.close();
}
else {
qDebug() << "[HTTP] Cache: reading file into buffer";
c.open(QIODevice::ReadOnly);
dataBuffer = c.readAll();
c.close();
}
m_response = 200; // fake "200 OK" HTTP response
m_cached = true;
httpDone(false); // we're done now. Handle http "done" signal.
return;
}
else {
// unlink old cache file
if(cachefile.isReadable()) {
QFile(m_cachefile).remove();
qDebug() << "[HTTP] Cache: outdated, timestamp:"
<< cachefile.lastModified();
}
qDebug() << "[HTTP] Cache: caching as" << m_cachefile;
// update cache index file
QFile idxFile(indexFile);
idxFile.open(QIODevice::ReadOnly);
QByteArray currLine;
do {
QByteArray currLine = idxFile.readLine(1000);
if(currLine.startsWith(m_hash.toUtf8()))
break;
} while(!currLine.isEmpty());
idxFile.close();
if(currLine.isEmpty()) {
idxFile.open(QIODevice::Append);
QString outline = m_hash + "\t" + m_header.value("Host") + "\t"
+ m_path + "\t" + m_query + "\n";
idxFile.write(outline.toUtf8());
idxFile.close();
}
}
}
else {
qDebug() << "[HTTP] cache DISABLED";
}
// schedule GET request
m_header.setRequest("GET", m_path + m_query);
if(outputToBuffer) {
qDebug() << "[HTTP] downloading to buffer.";
getRequest = http.request(m_header);
}
else {
qDebug() << "[HTTP] downloading to file:"
<< qPrintable(outputFile->fileName());
getRequest = http.request(m_header, 0, outputFile);
}
qDebug() << "[HTTP] GET scheduled: " << getRequest;
return;
return m_lastErrorString;
}
void HttpGet::httpDone(bool error)
int HttpGet::httpResponse(void)
{
if (error) {
qDebug() << "[HTTP] Error:" << qPrintable(http.errorString()) << httpResponse();
}
if(!outputToBuffer)
outputFile->close();
if(m_usecache && !m_cached && !error) {
qDebug() << "[HTTP] creating cache file" << m_cachefile;
QFile c(m_cachefile);
c.open(QIODevice::ReadWrite);
if(!outputToBuffer) {
outputFile->open(QIODevice::ReadOnly | QIODevice::Truncate);
c.write(outputFile->readAll());
outputFile->close();
}
else
c.write(dataBuffer);
c.close();
}
// take care of concurring requests. If there is still one running,
// don't emit done(). That request will call this slot again.
if(http.currentId() == 0 && !http.hasPendingRequests())
emit done(error);
}
void HttpGet::httpFinished(int id, bool error)
{
qDebug() << "[HTTP] Request finished:" << id << "Error:" << error
<< "pending requests:" << http.hasPendingRequests();
if(id == getRequest) {
dataBuffer = http.readAll();
emit requestFinished(id, error);
}
QHttpResponseHeader h = http.lastResponse();
QString date = h.value("Last-Modified").simplified();
if(date.isEmpty()) {
m_serverTimestamp = QDateTime(); // no value = invalid
if(id == headRequest)
emit headerFinished();
else
emit requestFinished(id, error);
return;
}
// to successfully parse the date strip weekday and timezone
date.remove(0, date.indexOf(" ") + 1);
if(date.endsWith("GMT")) date.truncate(date.indexOf(" GMT"));
// distinguish input formats (see RFC1945)
if(date.at(0).isLetter()) // asctime format
m_serverTimestamp = QLocale::c().toDateTime(date, "MMM d hh:mm:ss yyyy");
else // RFC 822
m_serverTimestamp = QLocale::c().toDateTime(date, "dd MMM yyyy hh:mm:ss");
qDebug() << "[HTTP] file server date:" << date
<< "parsed:" << m_serverTimestamp;
if(id == headRequest)
emit headerFinished();
else
emit requestFinished(id, error);
return;
}
void HttpGet::httpStarted(int id)
{
qDebug() << "[HTTP] Request started: " << id << "Header req:"
<< headRequest << "Get req:" << getRequest;
}
QString HttpGet::errorString()
{
return http.errorString();
}
void HttpGet::httpResponseHeader(const QHttpResponseHeader &resp)
{
// if there is a network error abort all scheduled requests for
// this download
m_response = resp.statusCode();
// 301 -- moved permanently
// 302 -- found
// 303 -- see other
// 307 -- moved temporarily
// in all cases, header: location has the correct address so we can follow.
if(m_response == 301 || m_response == 302 || m_response == 303 || m_response == 307) {
//abort without sending any signals
http.blockSignals(true);
http.abort();
http.blockSignals(false);
// start new request with new url
qDebug() << "[HTTP] response =" << m_response << "- following";
getFile(resp.value("location") + m_query);
}
else if(m_response != 200) {
// all other errors are fatal.
http.abort();
qDebug() << "[HTTP] Response error:" << m_response << resp.reasonPhrase();
}
}
int HttpGet::httpResponse()
{
return m_response;
}
void HttpGet::httpState(int state)
{
QString s[] = {"Unconnected", "HostLookup", "Connecting", "Sending",
"Reading", "Connected", "Closing"};
if(state <= 6)
qDebug() << "[HTTP] State:" << s[state];
else qDebug() << "[HTTP] State:" << state;
return m_lastStatusCode;
}

80
rbutil/rbutilqt/base/httpget.h
View file

@ -24,6 +24,7 @@
#include <QtCore>
#include <QtNetwork>
#include <QNetworkAccessManager>
class HttpGet : public QObject
{
@ -42,14 +43,34 @@ class HttpGet : public QObject
int httpResponse(void);
QByteArray readAll(void);
bool isCached()
{ return m_cached; }
{ return m_lastRequestCached; }
QDateTime timestamp(void)
{ return m_serverTimestamp; }
static void setGlobalCache(const QDir& d) //< set global cache path
{ m_globalCache = d; }
static void setGlobalProxy(const QUrl& p) //< set global proxy value
{ m_globalProxy = p; }
static void setGlobalUserAgent(const QString& u) //< set global user agent string
{ return m_lastServerTimestamp; }
//< set global cache path
static void setGlobalCache(const QDir& d)
{
qDebug() << "[HttpGet] Global cache set to" << d.absolutePath();
m_globalCache = d;
}
//< set global proxy value
static void setGlobalProxy(const QUrl& p)
{
qDebug() << "[HttpGet] setting global proxy" << p;
if(!p.isValid() || p.isEmpty()) {
HttpGet::m_globalProxy.setType(QNetworkProxy::NoProxy);
}
else {
HttpGet::m_globalProxy.setType(QNetworkProxy::HttpProxy);
HttpGet::m_globalProxy.setHostName(p.host());
HttpGet::m_globalProxy.setPort(p.port());
HttpGet::m_globalProxy.setUser(p.userName());
HttpGet::m_globalProxy.setPassword(p.password());
}
QNetworkProxy::setApplicationProxy(QNetworkProxy::NoProxy);
QNetworkProxy::setApplicationProxy(HttpGet::m_globalProxy);
}
//< set global user agent string
static void setGlobalUserAgent(const QString& u)
{ m_globalUserAgent = u; }
public slots:
@ -62,37 +83,28 @@ class HttpGet : public QObject
void headerFinished(void);
private slots:
void httpDone(bool error);
void httpFinished(int, bool);
void httpResponseHeader(const QHttpResponseHeader&);
void httpState(int);
void httpStarted(int);
void getFileFinish(void);
void requestFinished(QNetworkReply* reply);
void startRequest(QUrl url);
void downloadProgress(qint64 received, qint64 total);
void networkError(QNetworkReply::NetworkError error);
private:
bool initializeCache(const QDir&);
QHttp http; //< download object
QFile *outputFile;
int m_response; //< http response
int getRequest; //! get file http request id
int headRequest; //! get http header request id
QByteArray dataBuffer;
bool outputToBuffer;
bool m_usecache;
static QString m_globalUserAgent;
static QNetworkProxy m_globalProxy;
QNetworkAccessManager *m_mgr;
QNetworkReply *m_reply;
QNetworkDiskCache *m_cache;
QDir m_cachedir;
QString m_cachefile; // cached filename
bool m_cached;
QUrl m_proxy;
bool m_useproxy;
QDateTime m_serverTimestamp; //< timestamp of file on server
QString m_query; //< constructed query to pass http getter
QString m_path; //< constructed path to pass http getter
QString m_hash; //< caching hash
QHttpRequestHeader m_header;
static QDir m_globalCache; //< global cache path value
static QUrl m_globalProxy; //< global proxy value
static QString m_globalUserAgent; //< global user agent string
QByteArray m_data;
QFile *m_outputFile;
int m_lastStatusCode;
QString m_lastErrorString;
QDateTime m_lastServerTimestamp;
bool m_lastRequestCached;
QNetworkProxy m_proxy;
};
#endif