Files
@ 372ee69043f1
Branch filter:
Location: libtransport.git/backends/libyahoo2/httpfetch.cpp
372ee69043f1
4.0 KiB
text/x-c++hdr
Create directories with proper owner/group/permissions. Moved StorageBackend related methods from Util into StorageBackend
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 |
#include "httpfetch.h"
#include "transport/logging.h"
#if WIN32
#define snprintf sprintf_s
#endif
DEFINE_LOGGER(logger, "HTTPFetch");
static int url_to_host_port_path(const char *url,
char *host, int *port, char *path, int *ssl)
{
char *urlcopy = NULL;
char *slash = NULL;
char *colon = NULL;
/*
* http://hostname
* http://hostname/
* http://hostname/path
* http://hostname/path:foo
* http://hostname:port
* http://hostname:port/
* http://hostname:port/path
* http://hostname:port/path:foo
* and https:// variants of the above
*/
if (strstr(url, "http://") == url) {
urlcopy = strdup(url + 7);
} else if (strstr(url, "https://") == url) {
urlcopy = strdup(url + 8);
*ssl = 1;
} else {
return 0;
}
slash = strchr(urlcopy, '/');
colon = strchr(urlcopy, ':');
if (!colon || (slash && slash < colon)) {
if (*ssl)
*port = 443;
else
*port = 80;
} else {
*colon = 0;
*port = atoi(colon + 1);
}
if (!slash) {
strcpy(path, "/");
} else {
strcpy(path, slash);
*slash = 0;
}
strcpy(host, urlcopy);
free(urlcopy);
return 1;
}
HTTPFetch::HTTPFetch(Swift::BoostIOServiceThread *ioService, Swift::ConnectionFactory *factory) : m_ioService(ioService), m_factory(factory) {
m_afterHeader = false;
}
HTTPFetch::~HTTPFetch() {
}
void HTTPFetch::_connected(boost::shared_ptr<Swift::Connection> conn, const std::string url, bool error) {
if (error) {
_disconnected(conn);
}
else {
char host[255];
int port = 80;
char path[255];
int ssl = 0;
if (!url_to_host_port_path(url.c_str(), host, &port, path, &ssl))
return;
static char buff[2048];
snprintf(buff, sizeof(buff),
"GET %s HTTP/1.1\r\n"
"Host: %s\r\n"
"User-Agent: Mozilla/4.5 [en] (1/1)\r\n"
"Accept: */*\r\n"
"%s" "\r\n", path, host,
"Connection: close\r\n");
LOG4CXX_INFO(logger, "Sending " << buff << "\n");
conn->write(Swift::createSafeByteArray(buff));
}
}
void HTTPFetch::_disconnected(boost::shared_ptr<Swift::Connection> conn) {
conn->onConnectFinished.disconnect_all_slots();
conn->onDisconnected.disconnect_all_slots();
conn->onDataRead.disconnect_all_slots();
if (m_buffer.size() == 0) {
onURLFetched("");
}
else {
std::string img = m_buffer.substr(m_buffer.find("\r\n\r\n") + 4);
onURLFetched(img);
}
}
void HTTPFetch::_read(boost::shared_ptr<Swift::Connection> conn, boost::shared_ptr<Swift::SafeByteArray> data) {
std::string d(data->begin(), data->end());
// std::cout << d << "\n";
std::string img = d.substr(d.find("\r\n\r\n") + 4);
if (d.find("Location: ") == std::string::npos) {
m_buffer += d;
}
else {
d = d.substr(d.find("Location: ") + 10);
if (d.find("\r") == std::string::npos) {
d = d.substr(0, d.find("\n"));
}
else {
d = d.substr(0, d.find("\r"));
}
LOG4CXX_INFO(logger, "Next url is '" << d << "'");
fetchURL(d);
conn->onConnectFinished.disconnect_all_slots();
conn->onDisconnected.disconnect_all_slots();
conn->onDataRead.disconnect_all_slots();
}
}
bool HTTPFetch::fetchURL(const std::string &url) {
char host[255];
int port = 80;
char path[255];
char buff[1024];
int ssl = 0;
if (!url_to_host_port_path(url.c_str(), host, &port, path, &ssl)) {
LOG4CXX_ERROR(logger, "Invalid URL " << url);
return false;
}
LOG4CXX_INFO(logger, "Connecting to " << host << ":" << port);
boost::asio::ip::tcp::resolver resolver(*m_ioService->getIOService());
boost::asio::ip::tcp::resolver::query query(host, "");
boost::asio::ip::address address;
for(boost::asio::ip::tcp::resolver::iterator i = resolver.resolve(query); i != boost::asio::ip::tcp::resolver::iterator(); ++i) {
boost::asio::ip::tcp::endpoint end = *i;
address = end.address();
break;
}
boost::shared_ptr<Swift::Connection> conn = m_factory->createConnection();
conn->onConnectFinished.connect(boost::bind(&HTTPFetch::_connected, this, conn, url, _1));
conn->onDisconnected.connect(boost::bind(&HTTPFetch::_disconnected, this, conn));
conn->onDataRead.connect(boost::bind(&HTTPFetch::_read, this, conn, _1));
conn->connect(Swift::HostAddressPort(Swift::HostAddress(address), port));
return true;
}
|