#include #include #include #include #include "HttpClientHelpers.h" #include "HttpClient.h" namespace PresenceDetection { namespace Util { HttpClient::HttpClient() { curl_global_init(CURL_GLOBAL_ALL); m_userAgents.push_back("Opera/9.80 (X11; Linux i686; U; en) Presto/2.7.62 Version/11.00"); InitializeLocks(); } HttpClient::~HttpClient() { FreeLocks(); curl_global_cleanup(); } std::string HttpClient::GetUrlContents(const std::string& url) const { std::string buffer; CURL* curl = curl_easy_init(); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1); curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0); curl_easy_setopt(curl, CURLOPT_USERAGENT, m_userAgents[rand() % m_userAgents.size()].c_str()); curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &WriteCallback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buffer); int code = 0; curl_easy_perform(curl); curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); curl_easy_cleanup(curl); if (code != 200) { std::stringstream error; error << "Error (" << code << ") encountered while retrieving " << url << "\n"; error << "Data: " << buffer; throw std::runtime_error(error.str()); } return buffer; } void HttpClient::GetUrlSilent(const std::string& url) const { CURL* curl = curl_easy_init(); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1); curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0); curl_easy_setopt(curl, CURLOPT_USERAGENT, m_userAgents[rand() % m_userAgents.size()].c_str()); curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &WriteCallback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, nullptr); int code = 0; curl_easy_perform(curl); curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); curl_easy_cleanup(curl); if (code != 200) { std::stringstream error; error << "Error (" << code << ") encountered while retrieving " << url << "\n"; throw std::runtime_error(error.str()); } } std::string HttpClient::GetUrlContents(const std::string& url, const std::string& cookieFile) const { std::string buffer; CURL* curl = curl_easy_init(); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1); curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0); curl_easy_setopt(curl, CURLOPT_USERAGENT, m_userAgents[rand() % m_userAgents.size()].c_str()); curl_easy_setopt(curl, CURLOPT_COOKIEFILE, cookieFile.c_str()); curl_easy_setopt(curl, CURLOPT_COOKIEJAR, cookieFile.c_str()); curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &WriteCallback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buffer); int code = 0; curl_easy_perform(curl); curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); curl_easy_cleanup(curl); if (code != 200) { std::stringstream error; error << "Error (" << code << ") encountered while retrieving " << url << "\n"; error << "Data: " << buffer; throw std::runtime_error(error.str()); } return buffer; } void HttpClient::GetUrlSilent(const std::string& url, const std::string& cookieFile) const { CURL* curl = curl_easy_init(); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1); curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0); curl_easy_setopt(curl, CURLOPT_USERAGENT, m_userAgents[rand() % m_userAgents.size()].c_str()); curl_easy_setopt(curl, CURLOPT_COOKIEFILE, cookieFile.c_str()); curl_easy_setopt(curl, CURLOPT_COOKIEJAR, cookieFile.c_str()); curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &WriteCallback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, nullptr); int code = 0; curl_easy_perform(curl); curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); curl_easy_cleanup(curl); if (code != 200) { std::stringstream error; error << "Error (" << code << ") encountered while retrieving " << url << "\n"; throw std::runtime_error(error.str()); } } std::string HttpClient::GetUrlPostContents(const std::string& url, const std::string& postData, const std::string& contentType) const { std::string buffer; CURL* curl = curl_easy_init(); std::stringstream contentTypeString; contentTypeString << "Content-Type: " << contentType; struct curl_slist *list = NULL; list = curl_slist_append(list, contentTypeString.str().c_str()); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1); curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0); curl_easy_setopt(curl, CURLOPT_USERAGENT, m_userAgents[rand() % m_userAgents.size()].c_str()); curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list); curl_easy_setopt(curl, CURLOPT_POST, 1); curl_easy_setopt(curl, CURLOPT_POSTFIELDS, postData.c_str()); curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, postData.size()); curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &WriteCallback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buffer); int code = 0; curl_easy_perform(curl); curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); curl_easy_cleanup(curl); if (code != 200) { std::stringstream error; error << "Error (" << code << ") encountered while retrieving " << url << "\n"; error << "Data: " << buffer; throw std::runtime_error(error.str()); } return buffer; } void HttpClient::GetUrlPostSilent(const std::string& url, const std::string& postData, const std::string& contentType) const { CURL* curl = curl_easy_init(); std::stringstream contentTypeString; contentTypeString << "Content-Type: " << contentType; struct curl_slist *list = NULL; list = curl_slist_append(list, contentTypeString.str().c_str()); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1); curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0); curl_easy_setopt(curl, CURLOPT_USERAGENT, m_userAgents[rand() % m_userAgents.size()].c_str()); curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list); curl_easy_setopt(curl, CURLOPT_POST, 1); curl_easy_setopt(curl, CURLOPT_POSTFIELDS, postData.c_str()); curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, postData.size()); curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &WriteCallback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, nullptr); int code = 0; curl_easy_perform(curl); curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); curl_easy_cleanup(curl); if (code != 200) { std::stringstream error; error << "Error (" << code << ") encountered while retrieving " << url << "\n"; throw std::runtime_error(error.str()); } } std::string HttpClient::GetUrlPostContents(const std::string& url, const std::string& cookieFile, const std::string& postData, const std::string& contentType) const { std::string buffer; CURL* curl = curl_easy_init(); std::stringstream contentTypeString; contentTypeString << "Content-Type: " << contentType; struct curl_slist *list = NULL; list = curl_slist_append(list, contentTypeString.str().c_str()); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1); curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0); curl_easy_setopt(curl, CURLOPT_USERAGENT, m_userAgents[rand() % m_userAgents.size()].c_str()); curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list); curl_easy_setopt(curl, CURLOPT_POST, 1); curl_easy_setopt(curl, CURLOPT_POSTFIELDS, postData.c_str()); curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, postData.size()); curl_easy_setopt(curl, CURLOPT_COOKIEFILE, cookieFile.c_str()); curl_easy_setopt(curl, CURLOPT_COOKIEJAR, cookieFile.c_str()); curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &WriteCallback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buffer); int code = 0; curl_easy_perform(curl); curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); curl_easy_cleanup(curl); if (code != 200) { std::stringstream error; error << "Error (" << code << ") encountered while retrieving " << url << "\n"; error << "Data: " << buffer; throw std::runtime_error(error.str()); } return buffer; } void HttpClient::GetUrlPostSilent(const std::string& url, const std::string& cookieFile, const std::string& postData, const std::string& contentType) const { CURL* curl = curl_easy_init(); std::stringstream contentTypeString; contentTypeString << "Content-Type: " << contentType; struct curl_slist *list = NULL; list = curl_slist_append(list, contentTypeString.str().c_str()); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1); curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0); curl_easy_setopt(curl, CURLOPT_USERAGENT, m_userAgents[rand() % m_userAgents.size()].c_str()); curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list); curl_easy_setopt(curl, CURLOPT_POST, 1); curl_easy_setopt(curl, CURLOPT_POSTFIELDS, postData.c_str()); curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, postData.size()); curl_easy_setopt(curl, CURLOPT_COOKIEFILE, cookieFile.c_str()); curl_easy_setopt(curl, CURLOPT_COOKIEJAR, cookieFile.c_str()); curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &WriteCallback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, nullptr); int code = 0; curl_easy_perform(curl); curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); curl_easy_cleanup(curl); if (code != 200) { std::stringstream error; error << "Error (" << code << ") encountered while retrieving " << url << "\n"; throw std::runtime_error(error.str()); } } std::string HttpClient::GetUrlPostAttachmentContents(const std::string& url, const std::string& postData, const std::string& filename, const std::string& fileFieldname) const { std::string buffer; std::string contents; std::ifstream fileStream(filename, std::ios::in | std::ios::binary); if (fileStream) { fileStream.seekg(0, std::ios::end); contents.resize(fileStream.tellg()); fileStream.seekg(0, std::ios::beg); fileStream.read(&contents[0], contents.size()); fileStream.close(); } CURL* curl = curl_easy_init(); CURLcode result; struct curl_httppost *formpost = nullptr; struct curl_httppost *lastptr = nullptr; struct curl_slist *headerlist = nullptr; static const char headerBuffer[] = "Expect:"; curl_global_init(CURL_GLOBAL_ALL); curl_formadd(&formpost, &lastptr, CURLFORM_COPYNAME, "cache-control:", CURLFORM_COPYCONTENTS, "no-cache", CURLFORM_END); curl_formadd(&formpost, &lastptr, CURLFORM_COPYNAME, "content-type:", CURLFORM_COPYCONTENTS, "multipart/form-data", CURLFORM_END); std::vector postTokens; boost::split(postTokens, postData, boost::is_any_of("&")); for (std::vector::iterator it = postTokens.begin(); it != postTokens.end(); ++it) { std::vector tokens; boost::split(tokens, *it, boost::is_any_of("=")); curl_formadd(&formpost, &lastptr, CURLFORM_COPYNAME, tokens[0].c_str(), CURLFORM_COPYCONTENTS, tokens[1].c_str(), CURLFORM_END); } curl_formadd(&formpost, &lastptr, CURLFORM_COPYNAME, fileFieldname.c_str(), CURLFORM_BUFFER, "data", CURLFORM_BUFFERPTR, contents.data(), CURLFORM_BUFFERLENGTH, contents.size(), CURLFORM_END); headerlist = curl_slist_append(headerlist, headerBuffer); curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &WriteCallback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buffer); int code = 0; curl_easy_perform(curl); curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); curl_easy_cleanup(curl); curl_formfree(formpost); curl_slist_free_all(headerlist); if (code != 200) { std::stringstream error; error << "Error (" << code << ") encountered while retrieving " << url << "\n"; error << "Data: " << buffer; throw std::runtime_error(error.str()); } return buffer; } void HttpClient::GetUrlPostAttachmentSilent(const std::string& url, const std::string& postData, const std::string& filename, const std::string& fileFieldname) const { std::string contents; std::ifstream fileStream(filename, std::ios::in | std::ios::binary); if (fileStream) { fileStream.seekg(0, std::ios::end); contents.resize(fileStream.tellg()); fileStream.seekg(0, std::ios::beg); fileStream.read(&contents[0], contents.size()); fileStream.close(); } CURL* curl = curl_easy_init(); CURLcode result; struct curl_httppost *formpost = nullptr; struct curl_httppost *lastptr = nullptr; struct curl_slist *headerlist = nullptr; static const char headerBuffer[] = "Expect:"; curl_global_init(CURL_GLOBAL_ALL); curl_formadd(&formpost, &lastptr, CURLFORM_COPYNAME, "cache-control:", CURLFORM_COPYCONTENTS, "no-cache", CURLFORM_END); curl_formadd(&formpost, &lastptr, CURLFORM_COPYNAME, "content-type:", CURLFORM_COPYCONTENTS, "multipart/form-data", CURLFORM_END); std::vector postTokens; boost::split(postTokens, postData, boost::is_any_of("&")); for (std::vector::iterator it = postTokens.begin(); it != postTokens.end(); ++it) { std::vector tokens; boost::split(tokens, *it, boost::is_any_of("=")); curl_formadd(&formpost, &lastptr, CURLFORM_COPYNAME, tokens[0].c_str(), CURLFORM_COPYCONTENTS, tokens[1].c_str(), CURLFORM_END); } curl_formadd(&formpost, &lastptr, CURLFORM_COPYNAME, fileFieldname.c_str(), CURLFORM_BUFFER, "data", CURLFORM_BUFFERPTR, contents.data(), CURLFORM_BUFFERLENGTH, contents.size(), CURLFORM_END); headerlist = curl_slist_append(headerlist, headerBuffer); curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost); int code = 0; curl_easy_perform(curl); curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); curl_easy_cleanup(curl); curl_formfree(formpost); curl_slist_free_all(headerlist); if (code != 200) { std::stringstream error; error << "Error (" << code << ") encountered while retrieving " << url << "\n"; throw std::runtime_error(error.str()); } } std::string HttpClient::GetUrlPostAttachmentContents(const std::string& url, const std::string& cookieFile, const std::string& postData, const std::string& filename, const std::string& fileFieldname) const { std::string buffer; std::string contents; std::ifstream fileStream(filename, std::ios::in | std::ios::binary); if (fileStream) { fileStream.seekg(0, std::ios::end); contents.resize(fileStream.tellg()); fileStream.seekg(0, std::ios::beg); fileStream.read(&contents[0], contents.size()); fileStream.close(); } CURL* curl = curl_easy_init(); CURLcode result; struct curl_httppost *formpost = nullptr; struct curl_httppost *lastptr = nullptr; struct curl_slist *headerlist = nullptr; static const char headerBuffer[] = "Expect:"; curl_global_init(CURL_GLOBAL_ALL); curl_formadd(&formpost, &lastptr, CURLFORM_COPYNAME, "cache-control:", CURLFORM_COPYCONTENTS, "no-cache", CURLFORM_END); curl_formadd(&formpost, &lastptr, CURLFORM_COPYNAME, "content-type:", CURLFORM_COPYCONTENTS, "multipart/form-data", CURLFORM_END); std::vector postTokens; boost::split(postTokens, postData, boost::is_any_of("&")); for (std::vector::iterator it = postTokens.begin(); it != postTokens.end(); ++it) { std::vector tokens; boost::split(tokens, *it, boost::is_any_of("=")); curl_formadd(&formpost, &lastptr, CURLFORM_COPYNAME, tokens[0].c_str(), CURLFORM_COPYCONTENTS, tokens[1].c_str(), CURLFORM_END); } curl_formadd(&formpost, &lastptr, CURLFORM_COPYNAME, fileFieldname.c_str(), CURLFORM_BUFFER, "data", CURLFORM_BUFFERPTR, contents.data(), CURLFORM_BUFFERLENGTH, contents.size(), CURLFORM_END); headerlist = curl_slist_append(headerlist, headerBuffer); curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost); curl_easy_setopt(curl, CURLOPT_COOKIEFILE, cookieFile.c_str()); curl_easy_setopt(curl, CURLOPT_COOKIEJAR, cookieFile.c_str()); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &WriteCallback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buffer); int code = 0; curl_easy_perform(curl); curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); curl_easy_cleanup(curl); curl_formfree(formpost); curl_slist_free_all(headerlist); if (code != 200) { std::stringstream error; error << "Error (" << code << ") encountered while retrieving " << url << "\n"; error << "Data: " << buffer; throw std::runtime_error(error.str()); } return buffer; } void HttpClient::GetUrlPostAttachmentSilent(const std::string& url, const std::string& cookieFile, const std::string& postData, const std::string& filename, const std::string& fileFieldname) const { std::string contents; std::ifstream fileStream(filename, std::ios::in | std::ios::binary); if (fileStream) { fileStream.seekg(0, std::ios::end); contents.resize(fileStream.tellg()); fileStream.seekg(0, std::ios::beg); fileStream.read(&contents[0], contents.size()); fileStream.close(); } CURL* curl = curl_easy_init(); CURLcode result; struct curl_httppost *formpost = nullptr; struct curl_httppost *lastptr = nullptr; struct curl_slist *headerlist = nullptr; static const char headerBuffer[] = "Expect:"; curl_global_init(CURL_GLOBAL_ALL); curl_formadd(&formpost, &lastptr, CURLFORM_COPYNAME, "cache-control:", CURLFORM_COPYCONTENTS, "no-cache", CURLFORM_END); curl_formadd(&formpost, &lastptr, CURLFORM_COPYNAME, "content-type:", CURLFORM_COPYCONTENTS, "multipart/form-data", CURLFORM_END); std::vector postTokens; boost::split(postTokens, postData, boost::is_any_of("&")); for (std::vector::iterator it = postTokens.begin(); it != postTokens.end(); ++it) { std::vector tokens; boost::split(tokens, *it, boost::is_any_of("=")); curl_formadd(&formpost, &lastptr, CURLFORM_COPYNAME, tokens[0].c_str(), CURLFORM_COPYCONTENTS, tokens[1].c_str(), CURLFORM_END); } curl_formadd(&formpost, &lastptr, CURLFORM_COPYNAME, fileFieldname.c_str(), CURLFORM_BUFFER, "data", CURLFORM_BUFFERPTR, contents.data(), CURLFORM_BUFFERLENGTH, contents.size(), CURLFORM_END); headerlist = curl_slist_append(headerlist, headerBuffer); curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost); curl_easy_setopt(curl, CURLOPT_COOKIEFILE, cookieFile.c_str()); curl_easy_setopt(curl, CURLOPT_COOKIEJAR, cookieFile.c_str()); int code = 0; curl_easy_perform(curl); curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); curl_easy_cleanup(curl); curl_formfree(formpost); curl_slist_free_all(headerlist); if (code != 200) { std::stringstream error; error << "Error (" << code << ") encountered while retrieving " << url << "\n"; throw std::runtime_error(error.str()); } } size_t HttpClient::WriteCallback(char* data, size_t size, size_t nmemb, std::string* writerData) { if (writerData == nullptr) return size * nmemb; writerData->append(data, size * nmemb); return size * nmemb; } } // namespace Util } // namespace PresenceDetection