Save cURL content result into a string in C++

int main(void)
{
  CURL *curl;
  CURLcode res;

  curl = curl_easy_init();
  if(curl) {
    curl_easy_setopt(curl, CURLOPT_URL, "http://www.google.com");
    curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
    curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
    res = curl_easy_perform(curl);
    curl_easy_cleanup(curl);
  }
  _getch();
  return 0;
}

string contents = “”;

I would like to save the result of the curl html content in a string, how do I do this? It’s a silly question but unfortunately, I couldn’t find anywhere in the cURL examples for C++ thanks!

Getting cURL result in MFC C++

I’m trying to messagebox the cURL result: CURL *curl; CURLcode res; curl = curl_easy_init(); if (curl) { curl_easy_setopt(curl, CURLOPT_URL, http://example.com); curl_easy_setopt(curl, CURLOPT_FOLL



Retrieving full request string using Ruby curl

I intend to send a request like the following: c = Curl::Easy.http_post(https://example.com, json_string ) do |curl| curl.headers[‘Accept’] = ‘application/json’ curl.headers[‘Content-Type’] = ‘appli

Save cURL Display Output String in Variable PHP

is their an option to save the outpout of a curl request in a php variable? Because if i only save the $result i get a 1 or nothing <?php $url=’http://icanhazip.com’; $proxy=file (‘proxy.txt’); $us

C# Save a textures content path as string

I’m wondering if there is a simple way to store the path of a texture in a string. I’m wondering this because I’m using a save system for a game where I need to store the path of a texture in a file t

Extract a specific string from a curl’d result

Given this curl command: curl –user-agent fogent –silent -o page.html http://www.google.com/search?q=insansiate * Spelling is intentionally incorrect. I want to grab the suggestion as my result.

cURL returns different result

The below function returns different result than that actually we render the page directly. What would be the issue? function file_get_contents_curl($url) { $ch = curl_init(); $useragent=Mozilla/5.0

putting CURL result in a string and not STDOUT?

I have the following curl code, which make a request to website and retrieve data from it, it works well, but I want to store my data in a string and not in the output window. Any idea? #include <s

save the output of a curl perform to a vector in c++

as my title says, i would like to save the output of a curl perform to a vector.. can any1 please give me a sample code? i was able to save it into a structure in c. but i want to save it to a vector

PHP – Download cURL result

I wrote function below: function download_xfs($url) { $ch = curl_init(); curl_setopt($ch, CURLOPT_URL, $url); curl_setopt($ch, CURLOPT_COOKIE, ‘login=michael; xfss=08ruiweu4tuhb5xqs8’); curl_setopt($c

C++ cURL – how to save a full webpage to a file?

I’m trying to save a full webpage to a .txt file with C++ (Visual Studio 2013). I’m using cURL. Everything works fine, but the website I’m trying to save – uses a lot of javascript to generate the pag

Answers

You will have to use CURLOPT_WRITEFUNCTION to set a callback for writing. I can’t test to compile this right now, but the function should look something close to;

static std::string readBuffer;

static size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp)
{ 
    size_t realsize = size * nmemb;
    readBuffer.append(contents, realsize);
    return realsize;
}

Then call it by doing;

readBuffer.clear();
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
// ...other curl options
res = curl_easy_perform(curl);

After the call, readBuffershould have your contents.

Edit: You can use CURLOPT_WRITEDATA to pass the buffer string instead of making it static. In this case I just made it static for simplicity. A good page to look (besides the linked example above) is here for an explanation of the options.

Edit2: As requested, here’s a complete working example without the static string buffer;

#include <iostream>
#include <string>
#include <curl/curl.h>


static size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp)
{
    ((std::string*)userp)->append((char*)contents, size * nmemb);
    return size * nmemb;
}

int main(void)
{
  CURL *curl;
  CURLcode res;
  std::string readBuffer;

  curl = curl_easy_init();
  if(curl) {
    curl_easy_setopt(curl, CURLOPT_URL, "http://www.google.com");
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer);
    res = curl_easy_perform(curl);
    curl_easy_cleanup(curl);

    std::cout << readBuffer << std::endl;
  }
  return 0;
}

This might not work right away but should give you an idea:

#include <string>
#include <curl.h>
#include <stdio.h>
size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) {
    size_t written;
    written = fwrite(ptr, size, nmemb, stream);
    return written;
}

int main() {
    std::string tempname = "temp";
    CURL *curl;
    CURLcode res;
    curl = curl_easy_init();
    if(curl) {
      FILE *fp = fopen(tempname.c_str(),"wb");
      curl_easy_setopt(curl, CURLOPT_URL, "http://www.google.com");
      curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data); 
      curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
      res = curl_easy_perform(curl);
      curl_easy_cleanup(curl);
      fclose(fp);
      fp = fopen(tempname.c_str(),"rb");
      fseek (fp , 0 , SEEK_END);
      long lSize = ftell (fp);
      rewind(fp);
      char *buffer = new char[lSize+1];
      fread (buffer, 1, lSize, fp);
      buffer[lSize] = 0;
      fclose(fp);
      std::string content(buffer);
      delete [] buffer;
    }
}

Based on @JoachimIsaksson answer, here is a more verbose output that handles out-of-memory and has a limit for the maximum output from curl (as CURLOPT_MAXFILESIZE limits only based on header information and not on the actual size transferred ).

#DEFINE MAX_FILE_SIZE = 10485760 //10 MiB

size_t curl_to_string(void *ptr, size_t size, size_t count, void *stream)
{
    if(((string*)stream)->size() + (size * count) > MAX_FILE_SIZE)
    {
        cerr<<endl<<"Could not allocate curl to string, output size (current_size:"<<((string*)stream)->size()<<"bytes + buffer:"<<(size * count) << "bytes) would exceed the MAX_FILE_SIZE ("<<MAX_FILE_SIZE<<"bytes)";
        return 0;
    }
    int retry=0;
    while(true)
    {
        try{
            ((string*)stream)->append((char*)ptr, 0, size*count);
            break;// successful
        }catch (const std::bad_alloc&) {
            retry++;
            if(retry>100)
            {
                cerr<<endl<<"Could not allocate curl to string, probably not enough memory, aborting after : "<<retry<<" tries at 10s apart";
                return 0;
            }
            cerr<<endl<<"Could not allocate curl to string, probably not enough memory, sleeping 10s, try:"<<retry;
            sleep(10);
        }
    }
  return size*count;
}

On my blog I have published a simple wrapper class to perform this task.

Usage example:

#include "HTTPDownloader.hpp"

int main(int argc, char** argv) {
    HTTPDownloader downloader;
    std::string content = downloader.download("https://stackoverflow.com");
    std::cout << content << std::endl;
}

Here’s the header file:

/**
 * HTTPDownloader.hpp
 *
 * A simple C++ wrapper for the libcurl easy API.
 *
 * Written by Uli Köhler (techoverflow.net)
 * Published under CC0 1.0 Universal (public domain)
 */
#ifndef HTTPDOWNLOADER_HPP
#define HTTPDOWNLOADER_HPP

#include <string>

/**
 * A non-threadsafe simple libcURL-easy based HTTP downloader
 */
class HTTPDownloader {
public:
    HTTPDownloader();
    ~HTTPDownloader();
    /**
     * Download a file using HTTP GET and store in in a std::string
     * @param url The URL to download
     * @return The download result
     */
    std::string download(const std::string& url);
private:
    void* curl;
};

#endif  /* HTTPDOWNLOADER_HPP */

Here’s the source code:

/**
 * HTTPDownloader.cpp
 *
 * A simple C++ wrapper for the libcurl easy API.
 *
 * Written by Uli Köhler (techoverflow.net)
 * Published under CC0 1.0 Universal (public domain)
 */
#include "HTTPDownloader.hpp"
#include <curl/curl.h>
#include <curl/easy.h>
#include <curl/curlbuild.h>
#include <sstream>
#include <iostream>
using namespace std;

size_t write_data(void *ptr, size_t size, size_t nmemb, void *stream) {
    string data((const char*) ptr, (size_t) size * nmemb);
    *((stringstream*) stream) << data << endl;
    return size * nmemb;
}

HTTPDownloader::HTTPDownloader() {
    curl = curl_easy_init();
}

HTTPDownloader::~HTTPDownloader() {
    curl_easy_cleanup(curl);
}

string HTTPDownloader::download(const std::string& url) {
    curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
    /* example.com is redirected, so we tell libcurl to follow redirection */
    curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
    curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); //Prevent "longjmp causes uninitialized stack frame" bug
    curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, "deflate");
    std::stringstream out;
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, &out);
    /* Perform the request, res will get the return code */
    CURLcode res = curl_easy_perform(curl);
    /* Check for errors */
    if (res != CURLE_OK) {
        fprintf(stderr, "curl_easy_perform() failed: %s/n",
                curl_easy_strerror(res));
    }
    return out.str();
}

Came out with useful, yet simple solution, which overloads std::ostream::operator<<

#include <ostream>

#include <curl/curl.h>

size_t curlCbToStream (
    char * buffer,
    size_t nitems,
    size_t size,
    std::ostream * sout
)
{
    *sout << buffer;

    return nitems * size;
}

std::ostream & operator<< (
    std::ostream & sout,
    CURL * request
)
{
    ::curl_easy_setopt(request, CURLOPT_WRITEDATA, & sout);
    ::curl_easy_setopt(request, CURLOPT_WRITEFUNCTION, curlCbToStream);
    ::curl_easy_perform(request);

    return sout;
}

Possible drawback of taken approach could be:

typedef void CURL;

That means it covers all known pointer types.