// BlockyFroggy
// Copyright © 2017 John Ryland.
// All rights reserved.
#include "HttpClient.h"
#include "Log.h"
#include <cstdlib>
#include <cstring>
#include <cstdio>
#include <sstream>
#include <netdb.h>
#include <arpa/inet.h>
#include <unistd.h>
#include "Utilities.h"
// Based on code from: https://github.com/reagent/http
#define debug(Msg, ...) Log(LL_Debug, Msg, ##__VA_ARGS__)
#define jump_unless(cond) if (!(cond)) { goto error; }
#define error_unless(cond, Msg, ...) if (!(cond)) { Log(LL_Error, Msg, ##__VA_ARGS__); goto error; }
#define RECV_SIZE 4096
std::vector<std::string> splitHeaderLine(const std::string &s)
{
std::vector<std::string> elems;
std::stringstream ss(s);
std::string item;
std::getline(ss, item, ':');
elems.push_back(item);
if (ss.peek() == ' ')
ss.get();
std::getline(ss, item, '\r');
elems.push_back(item);
return elems;
}
int makeSyncHttpRequest(std::vector<uint8_t>& response, const char* a_method, const char* a_url, const std::map<std::string,std::string>& args)
{
response.reserve(10000);
struct addrinfo *res = NULL;
int sockfd = 0;
int status = 0;
std::vector<std::string> url_parts = Utilities::split(a_url, '/');
std::string url_scheme, url_hostname, url_port, url_path;
if (url_parts.size() >= 3)
{
for (size_t i = 3; i < url_parts.size(); i++)
url_path += "/" + url_parts[i];
for (std::map<std::string, std::string>::const_iterator i = args.begin(); i != args.end(); ++i)
url_path += (i==args.begin()?"?":"&") + i->first + "=" + i->second;
url_scheme = Utilities::split(url_parts[0], ':')[0];
if (url_parts[2].size())
{
url_parts = Utilities::split(url_parts[2], ':');
url_hostname = url_parts[0]; // "192.168.0.15";
if (url_parts.size() >= 2)
url_port = url_parts[1]; // "8000";
}
// std::string url_fragment = split(url_path, '#')[1]; // fragment is html document location
status = 1;
//debug("Scheme: '%s', Hostname: '%s', Port: '%s', Path: '%s'",
// url_scheme.c_str(), url_hostname.c_str(), url_port.c_str(), url_path.c_str());
}
error_unless(status > 0, "Invalid URL supplied: '%s'", a_url);
#if 1
{
struct addrinfo hints;
memset(&hints, 0, sizeof(hints));
hints.ai_family = AF_UNSPEC;
hints.ai_socktype = SOCK_STREAM;
status = getaddrinfo(url_hostname.c_str(), url_port.c_str(), &hints, &res);
error_unless(status == 0, "Could not resolve host: %s\n", gai_strerror(status));
sockfd = socket(res->ai_family, res->ai_socktype, res->ai_protocol);
if (sockfd > 0) {
if (connect(sockfd, res->ai_addr, res->ai_addrlen) != 0) {
close(sockfd);
sockfd = -1;
}
}
}
#else
{
// Shortcut for when hostname is just an IP address
sockfd = socket(AF_INET, SOCK_STREAM, 0);
if (sockfd > 0) {
struct sockaddr_in addr;
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = inet_addr(url_hostname.c_str()); // assumes IP address
addr.sin_port = htons(atoi(url_port.c_str()));
//error_unless(status == 0, "Could not resolve host: %s\n", gai_strerror(status));
if (connect(sockfd, (struct sockaddr*)&addr, sizeof(addr)) != 0) {
close(sockfd);
sockfd = -1;
}
}
}
#endif
error_unless(sockfd > 0, "Could not make connection to '%s' on port '%s'", url_hostname.c_str(), url_port.c_str());
{
std::string str;
str += std::string(a_method) + " " + url_path + " HTTP/1.0\r\n";
str += "Host: " + url_hostname + "\r\n";
str += "Connection: close\r\n\r\n";
const char *request = str.c_str();
size_t bytes_sent = 0;
size_t total_bytes_sent = 0;
size_t bytes_to_send = str.size();
while (1) {
bytes_sent = send(sockfd, request, strlen(request), 0);
total_bytes_sent += bytes_sent;
if (total_bytes_sent >= bytes_to_send) {
break;
}
}
status = (int)total_bytes_sent;
}
error_unless(status > 0, "Sending request failed");
{
bool parsedHeaders = false;
while (1) {
char data[RECV_SIZE];
int bytes_received = (int)recv(sockfd, data, RECV_SIZE, 0);
if (bytes_received == -1) {
status = -1;
break;
} else if (bytes_received == 0) {
status = 0;
break;
}
// Parsing the headers here is part of an optimization to get the content-size
// so we can resize the vector to make this more efficient
// We perhaps could properly parse the headers here if we detect we have got all the
// headers. This is duplicated in the wrapper currently
if (!parsedHeaders && response.size() >= 5000) {
parsedHeaders = true;
std::map<std::string,std::string> headerStrs;
int c = 0;
int h = 0;
for (size_t i = 0; i < response.size(); i++) {
if (((c&1)==0) && response[i] == '\r') c++;
else if (((c&1)==1) && response[i] == '\n') c++;
else {
if (c==2) {
std::string header(&response[h], &response[i]-2);
std::vector<std::string> hdrNameValue = splitHeaderLine(header);
if (hdrNameValue.size() == 2)
headerStrs[hdrNameValue[0]] = hdrNameValue[1];
h = int(i);
}
c = 0;
}
if (c==4) {
std::string header(&response[h], &response[i]-3);
std::vector<std::string> hdrNameValue = splitHeaderLine(header);
if (hdrNameValue.size() == 2)
headerStrs[hdrNameValue[0]] = hdrNameValue[1];
break;
}
}
if (headerStrs.count("Content-Length"))
{
int size = atoi(headerStrs["Content-Length"].c_str());
if (size > 5000) {
response.reserve(size + 10000);
}
}
}
if (bytes_received > 0 && bytes_received <= RECV_SIZE) {
// More efficient way to append an array of data to the vector
response.insert(std::end(response), (uint8_t*)data, (uint8_t*)(data + bytes_received));
}
}
}
error_unless(status >= 0, "Fetching response failed");
close(sockfd);
freeaddrinfo(res);
return 0;
error:
Log(LL_Error, "error case");
if (sockfd > 0) { close(sockfd); }
if (res != NULL) { freeaddrinfo(res); }
return 1;
}
void makeHttpRequestWrapper(const HttpRequest& request, HttpResponse& result, std::vector<uint8_t>& body)
{
std::vector<uint8_t> response;
std::vector<std::string> headerStrs;
result.result = makeSyncHttpRequest(response, request.method.c_str(), request.url.c_str(), request.args);
if (response.data())
{
bool inHeaders = true;
int c = 0;
int h = 0;
for (size_t i = 0; i < response.size(); i++) {
if (inHeaders) {
if (((c&1)==0) && response[i] == '\r') c++;
else if (((c&1)==1) && response[i] == '\n') c++;
else {
if (c==2) {
headerStrs.push_back(std::string(&response[h], &response[i]-2));
h = int(i);
}
c = 0;
}
if (c==4) {
headerStrs.push_back(std::string(&response[h], &response[i]-3));
inHeaders = false;
}
} else {
body.push_back(response[i]);
}
}
}
body.push_back(0); // Null terminate incase it is used as a string
result.status = "Connection error\n";
if (headerStrs.size()) {
result.status = headerStrs[0];
for (size_t i = 1; i < headerStrs.size(); i++)
result.headers[splitHeaderLine(headerStrs[i])[0]] = splitHeaderLine(headerStrs[i])[1];
}
}
#include <functional>
#include <errno.h>
#include <fcntl.h>
#include <sys/types.h>
#include <sys/socket.h>
/*
struct EventNotifier
{
EventNotifier() {
FD_ZERO(&rfds);
FD_ZERO(&wfds);
FD_ZERO(&xfds);
}
fd_set rfds, wfds, xfds;
struct timeval tv = { 0, 0 };
};
*/
struct FileDescriptorSet
{
FileDescriptorSet() {
FD_ZERO(&m_set);
}
FileDescriptorSet(int a_socket) {
FD_ZERO(&m_set);
addSocket(a_socket);
}
void addSocket(int a_socket) {
FD_SET(a_socket, &m_set);
}
fd_set* operation() { return &m_set; }
fd_set m_set;
};
int selectWrapper(int fd, int set)
{
struct timeval tv = { 0, 0 };
fd_set fds;
FD_ZERO(&fds);
FD_SET(fd, &fds);
int res = select(fd+1, (set==0)?&fds:nullptr, (set==1)?&fds:nullptr, (set==2)?&fds:nullptr, &tv);
if (res == -1)
return -1;
return (res == 0 || !FD_ISSET(fd, &fds)) ? 0 : 1;
}
struct AsyncHttpRequest
{
// Request
std::string method;
std::string url;
std::map<std::string,std::string> args;
std::map<std::string,std::string> req_headers;
// Response
int result;
std::string status;
std::map<std::string,std::string> headers;
std::vector<uint8_t> body;
// Callback
int state = 0; // state machine
int sockfd = 0;
std::string url_path;
std::string url_hostname;
std::string url_port;
std::string request_str;
int bytes_sent = 0;
bool pendingConnect = false;
bool pendingSend = false;
bool pendingRead = false;
std::vector<uint8_t> response;
std::function<void(const AsyncHttpRequest&)> callback;
int ParseURL(const char* a_url) {
std::vector<std::string> url_parts = Utilities::split(a_url, '/');
std::string url_scheme;
url_port = "80";
if (url_parts.size() >= 3)
{
for (size_t i = 3; i < url_parts.size(); i++)
url_path += "/" + url_parts[i];
for (std::map<std::string, std::string>::const_iterator i = args.begin(); i != args.end(); ++i)
url_path += (i==args.begin()?"?":"&") + i->first + "=" + i->second;
url_scheme = Utilities::split(url_parts[0], ':')[0];
if (url_parts[2].size())
{
url_parts = Utilities::split(url_parts[2], ':');
url_hostname = url_parts[0]; // "192.168.0.15";
if (url_parts.size() >= 2)
url_port = url_parts[1]; // "8000";
return 1;
}
// std::string url_fragment = split(url_path, '#')[1]; // fragment is html document location
//debug("Scheme: '%s', Hostname: '%s', Port: '%s', Path: '%s'",
// url_scheme.c_str(), url_hostname.c_str(), url_port.c_str(), url_path.c_str());
}
return 0;
}
void OnError() {
if (sockfd)
close(sockfd);
sockfd = 0;
state = -1; // Done
result = 1; // Error
callback(*this);
}
/*
void HandleError(const char* a_str) {
puts(a_str);
OnError();
}
*/
template<class ...Ts>
void HandleError(const char* a_str, Ts... args) {
Log(LL_Error, a_str, std::forward<Ts>(args)...);
OnError();
}
};
void updateASyncHttpRequest(AsyncHttpRequest& request)
{
switch (request.state)
{
case 0: // Start
{
int status = request.ParseURL(request.url.c_str());
if (status == 0)
return request.HandleError("Invalid URL supplied: '%s'\n", request.url.c_str());
struct addrinfo *res = NULL;
struct addrinfo hints;
memset(&hints, 0, sizeof(hints));
hints.ai_family = AF_UNSPEC;
hints.ai_socktype = SOCK_STREAM;
//! @todo this is not actually async - this bit could block
status = getaddrinfo(request.url_hostname.c_str(), request.url_port.c_str(), &hints, &res);
if (status != 0 || !res)
return request.HandleError("Could not resolve host: %s\n", gai_strerror(status));
request.sockfd = socket(res->ai_family, res->ai_socktype, res->ai_protocol);
if (request.sockfd <= 0)
return request.HandleError("Could not create socket\n");
// Make non-blocking
int flags = fcntl(request.sockfd, F_GETFL, 0);
flags = (flags == -1) ? 0 : flags;
if (fcntl(request.sockfd, F_SETFL, flags | O_NONBLOCK) == -1)
return request.HandleError("Could not make the socket non-blocking\n");
// Connect to server
request.state = 1;
if (connect(request.sockfd, res->ai_addr, res->ai_addrlen) != 0)
if (errno != EINPROGRESS)
return request.HandleError("Immediate failure when connecting\n");
freeaddrinfo(res);
std::string str;
str += request.method + " " + request.url_path + " HTTP/1.0\r\n";
str += "Host: " + request.url_hostname + "\r\n";
str += "Connection: close\r\n\r\n";
request.request_str = str;
// Fall through to next state
}
case 1: // Waiting for connect / ready to send
{
int res = selectWrapper(request.sockfd, 1);
if (res == -1)
return request.HandleError("Error waiting for socket to be ready to send\n");
else if (res == 0)
// Exit the switch and test again next time update called
//debug("waiting still for connect");
return;
int err = 0;
socklen_t len = sizeof(int);
if (getsockopt(request.sockfd, SOL_SOCKET, SO_ERROR, &err, &len) != 0 || err != 0)
return request.HandleError("Could not make connection to '%s'\n", request.url.c_str());
const std::string str = request.request_str;
int bytes_sent = request.bytes_sent;
bytes_sent = (int)send(request.sockfd, str.c_str() + bytes_sent, str.size() - bytes_sent, 0);
if (bytes_sent == -1)
{
if (errno != EAGAIN && errno != EWOULDBLOCK)
return request.HandleError("Sending request failed\n");
return; // try the send again later
}
request.bytes_sent += bytes_sent;
if (size_t(request.bytes_sent) < request.request_str.size())
return; // try to send the rest later
// All of the request should have been sent
request.state = 3; // Skip straight to reading
}
case 3:
{
// First check we are ready to read more
int res = selectWrapper(request.sockfd, 0);
if (res == -1)
return request.HandleError("Error waiting for data to read\n");
if (res == 0)
//debug("waiting still for reading request");
return; // try reading again later
// else we can call read some data without blocking
//int status = 0;
while (1) { // drain the recv buffer (we read in RECV_SIZE chunks, so we may need to loop a few times)
char buffer[RECV_SIZE];
int bytes_received = (int)recv(request.sockfd, buffer, RECV_SIZE, 0);
if (bytes_received == -1) {
if (errno == EAGAIN || errno == EWOULDBLOCK)
return; // Go back to waiting for more data to read
else
return request.HandleError("Fetching response failed\n");
} else if (bytes_received == 0) {
request.state = 4;
break;
// Fall out of the while loop, and then through to state 4
}
if (bytes_received > 0 && bytes_received <= RECV_SIZE) {
for (int i = 0; i < bytes_received; i++)
request.response.push_back((uint8_t)buffer[i]);
}
}
// Falls through to the state 4 in the switch statement
}
case 4: // Cleanup and process responses
{
close(request.sockfd);
std::vector<uint8_t>& response = request.response;
std::vector<std::string> headerStrs;
if (response.size() && response.data())
{
bool inHeaders = true;
int c = 0;
int h = 0;
for (size_t i = 0; i < response.size(); i++) {
if (inHeaders) {
if (((c&1)==0) && response[i] == '\r') c++;
else if (((c&1)==1) && response[i] == '\n') c++;
else {
if (c==2) {
headerStrs.push_back(std::string(&response[h], &response[i]-2));
h = int(i);
}
c = 0;
}
if (c==4) {
headerStrs.push_back(std::string(&response[h], &response[i]-3));
inHeaders = false;
}
} else {
request.body.push_back(response[i]);
}
}
}
request.body.push_back(0); // Null terminate incase it is used as a string
request.status = headerStrs[0];
for (size_t i = 1; i < headerStrs.size(); i++)
request.headers[splitHeaderLine(headerStrs[i])[0]] = splitHeaderLine(headerStrs[i])[1];
request.state = -1; // Done
request.result = 0; // No error
request.callback(request);
return;
}
case -1: // Done/Error state
default:
{
debug("Update called on a completed Http request");
break;
}
}
}
void makeAsyncHttpRequestWrapper(const HttpRequest& request)
{
AsyncHttpRequest req;
req.method = request.method;
req.url = request.url;
req.args = request.args;
req.callback = [](const AsyncHttpRequest& req) {
debug("\n");
if (req.result != 0) {
debug("callback: got error in async request");
} else {
debug("callback: success: %s", req.status.c_str());
debug("callback: body: %s", (const char*)req.body.data());
//std::map<std::string,std::string> headers;
}
};
debug("Making ASYNC http request!");
while (req.state != -1)
{
usleep(100); // 0.1ms
printf(".");
updateASyncHttpRequest(req);
}
}
// Next level will be handling lists of requests
// And then generalizing to files or sockets or other
// And to make it cross platform
// Potential code to help extend to support auth
/*
void HTTPClient::basicAuth(const char* user, const char* password) //Basic Authentification
{
#if 1
if (m_basicAuthUser)
swFree(m_basicAuthUser);
m_basicAuthUser = (char *)swMalloc(strlen(user)+1);
strcpy(m_basicAuthUser, user);
if (m_basicAuthPassword)
swFree(m_basicAuthPassword);
m_basicAuthPassword = (char *)swMalloc(strlen(password)+1);
strcpy(m_basicAuthPassword, password);
#else
m_basicAuthUser = user;
m_basicAuthPassword = password;
#endif
}
void HTTPClient::createauth (const char *user, const char *pwd, char *buf, int len)
{
char tmp[80];
snprintf(tmp, sizeof(tmp), "%s:%s", user, pwd);
base64enc(tmp, strlen(tmp), &buf[strlen(buf)], len - strlen(buf));
}
// Copyright (c) 2010 Donatien Garnier (donatiengar [at] gmail [dot] com)
int base64enc(const char *input, unsigned int length, char *output, int len)
{
static const char base64[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
unsigned int c, c1, c2, c3;
if ((uint16_t)len < ((((length-1)/3)+1)<<2)) return -1;
for(unsigned int i = 0, j = 0; i<length; i+=3,j+=4) {
c1 = ((((unsigned char)*((unsigned char *)&input[i]))));
c2 = (length>i+1)?((((unsigned char)*((unsigned char *)&input[i+1])))):0;
c3 = (length>i+2)?((((unsigned char)*((unsigned char *)&input[i+2])))):0;
c = ((c1 & 0xFC) >> 2);
output[j+0] = base64[c];
c = ((c1 & 0x03) << 4) | ((c2 & 0xF0) >> 4);
output[j+1] = base64[c];
c = ((c2 & 0x0F) << 2) | ((c3 & 0xC0) >> 6);
output[j+2] = (length>i+1)?base64[c]:'=';
c = (c3 & 0x3F);
output[j+3] = (length>i+2)?base64[c]:'=';
}
output[(((length-1)/3)+1)<<2] = '\0';
return 0;
}
*/