diff --git a/binding.gyp b/binding.gyp index 10ca1d18fe..d5db880780 100644 --- a/binding.gyp +++ b/binding.gyp @@ -13,6 +13,7 @@ "src/mapnik_geometry.cpp", "src/mapnik_feature.cpp", "src/mapnik_image.cpp", + "src/mapnik_image_encode_chunked.cpp", "src/mapnik_image_view.cpp", "src/mapnik_grid.cpp", "src/mapnik_grid_view.cpp", diff --git a/src/callback_streambuf.hpp b/src/callback_streambuf.hpp new file mode 100644 index 0000000000..f919e9fd22 --- /dev/null +++ b/src/callback_streambuf.hpp @@ -0,0 +1,47 @@ +#ifndef CALLBACK_STREAMBUF_H +#define CALLBACK_STREAMBUF_H + +#include +#include + +template +class callback_streambuf : public std::basic_streambuf +{ +public: + using base_type = std::streambuf; + using char_type = typename base_type::char_type; + using int_type = typename base_type::int_type; + + callback_streambuf(Callback callback, std::size_t buffer_size) + : callback_(callback), + buffer_(buffer_size) + { + base_type::setp(buffer_.data(), buffer_.data() + buffer_.size()); + } + +protected: + int sync() + { + bool ok = callback_(base_type::pbase(), + base_type::pptr() - base_type::pbase()); + base_type::setp(buffer_.data(), buffer_.data() + buffer_.size()); + return ok ? 0 : -1; + } + + int_type overflow(int_type ch) + { + int ret = sync(); + if (ch == base_type::traits_type::eof()) + { + return ch; + } + base_type::sputc(ch); + return ret == 0 ? 0 : base_type::traits_type::eof(); + } + +private: + Callback callback_; + std::vector buffer_; +}; + +#endif diff --git a/src/mapnik_image.cpp b/src/mapnik_image.cpp index 19c256098a..9541bd8969 100644 --- a/src/mapnik_image.cpp +++ b/src/mapnik_image.cpp @@ -22,12 +22,12 @@ #include #include -#include "mapnik_image.hpp" +#include "mapnik_image_encode.hpp" #include "mapnik_image_view.hpp" -#include "mapnik_palette.hpp" #include "mapnik_color.hpp" #include "utils.hpp" +#include "callback_streambuf.hpp" #include "agg_rasterizer_scanline_aa.h" #include "agg_basics.h" @@ -82,6 +82,7 @@ void Image::Initialize(v8::Local target) { Nan::SetPrototypeMethod(lcons, "setPixel", setPixel); Nan::SetPrototypeMethod(lcons, "encodeSync", encodeSync); Nan::SetPrototypeMethod(lcons, "encode", encode); + Nan::SetPrototypeMethod(lcons, "encodeChunked", encodeChunked); Nan::SetPrototypeMethod(lcons, "view", view); Nan::SetPrototypeMethod(lcons, "saveSync", saveSync); Nan::SetPrototypeMethod(lcons, "save", save); @@ -3609,17 +3610,6 @@ NAN_METHOD(Image::encodeSync) } } -typedef struct { - uv_work_t request; - Image* im; - std::string format; - palette_ptr palette; - bool error; - std::string error_name; - Nan::Persistent cb; - std::string result; -} encode_image_baton_t; - /** * Encode this image into a buffer of encoded data * diff --git a/src/mapnik_image.hpp b/src/mapnik_image.hpp index 454b58418b..04fa9812b2 100644 --- a/src/mapnik_image.hpp +++ b/src/mapnik_image.hpp @@ -29,8 +29,11 @@ class Image: public Nan::ObjectWrap { static NAN_METHOD(setPixel); static NAN_METHOD(encodeSync); static NAN_METHOD(encode); + static NAN_METHOD(encodeChunked); static void EIO_Encode(uv_work_t* req); static void EIO_AfterEncode(uv_work_t* req); + static void EIO_EncodeChunked(uv_work_t* req); + static void EIO_AfterEncodeChunked(uv_work_t* req, int status); static NAN_METHOD(setGrayScaleToAlpha); static NAN_METHOD(width); diff --git a/src/mapnik_image_encode.hpp b/src/mapnik_image_encode.hpp new file mode 100644 index 0000000000..229401055a --- /dev/null +++ b/src/mapnik_image_encode.hpp @@ -0,0 +1,18 @@ +#ifndef __NODE_MAPNIK_IMAGE_ENCODE_H__ +#define __NODE_MAPNIK_IMAGE_ENCODE_H__ + +#include "mapnik_image.hpp" +#include "mapnik_palette.hpp" + +typedef struct { + uv_work_t request; + Image* im; + std::string format; + palette_ptr palette; + bool error; + std::string error_name; + Nan::Persistent cb; + std::string result; +} encode_image_baton_t; + +#endif diff --git a/src/mapnik_image_encode_chunked.cpp b/src/mapnik_image_encode_chunked.cpp new file mode 100644 index 0000000000..6e96e38b8c --- /dev/null +++ b/src/mapnik_image_encode_chunked.cpp @@ -0,0 +1,225 @@ +// mapnik +#include // for image types +#include // for save_to_stream + +#include "mapnik_image_encode.hpp" +#include "mapnik_color.hpp" + +#include "utils.hpp" +#include "callback_streambuf.hpp" + +struct chunked_encode_image_baton_t +{ + encode_image_baton_t image_baton; + + uv_async_t async; + uv_mutex_t mutex; + + using char_type = char; + using buffer_type = std::vector; + using buffer_list_type = std::vector; + buffer_list_type buffers; + + const std::size_t buffer_size; + + chunked_encode_image_baton_t(std::size_t buffer_size_) + : buffer_size(buffer_size_) + { + // The reinterpret_cast is for backward compatibility + // https://github.com/libuv/libuv/commit/db2a9072bce129630214904be5e2eedeaafc9835 + if (uv_async_init(uv_default_loop(), &async, + reinterpret_cast(yield_chunk))) + { + throw std::runtime_error("Cannot create async handler"); + } + + if (uv_mutex_init(&mutex)) + { + uv_close(reinterpret_cast(&async), NULL); + throw std::runtime_error("Cannot create mutex"); + } + + async.data = this; + } + + ~chunked_encode_image_baton_t() + { + uv_mutex_destroy(&mutex); + } + + template + bool operator()(const Char* buffer, Size size) + { + uv_mutex_lock(&mutex); + buffers.emplace_back(buffer, buffer + size); + uv_mutex_unlock(&mutex); + + return uv_async_send(&async) == 0; + } + + static void yield_chunk(uv_async_t* handle) + { + using closure_type = chunked_encode_image_baton_t; + closure_type & closure = *reinterpret_cast(handle->data); + + if (closure.image_baton.error) + { + uv_close(reinterpret_cast(handle), async_close_cb); + return; + } + + buffer_list_type local_buffers; + + uv_mutex_lock(&closure.mutex); + closure.buffers.swap(local_buffers); + uv_mutex_unlock(&closure.mutex); + + Nan::HandleScope scope; + bool done = false; + + for (auto const & buffer : local_buffers) + { + v8::Local argv[2] = { + Nan::Null(), Nan::CopyBuffer(buffer.data(), + buffer.size()).ToLocalChecked() }; + Nan::MakeCallback(Nan::GetCurrentContext()->Global(), + Nan::New(closure.image_baton.cb), 2, argv); + done = buffer.empty(); + } + + if (done) + { + uv_close(reinterpret_cast(handle), async_close_cb); + } + } + + static void async_close_cb(uv_handle_t* handle) + { + using closure_type = chunked_encode_image_baton_t; + closure_type & closure = *reinterpret_cast(handle->data); + + if (closure.image_baton.error) + { + Nan::HandleScope scope; + v8::Local argv[1] = { + Nan::Error(closure.image_baton.error_name.c_str()) }; + Nan::MakeCallback(Nan::GetCurrentContext()->Global(), + Nan::New(closure.image_baton.cb), 1, argv); + } + + closure.image_baton.im->_unref(); + closure.image_baton.cb.Reset(); + delete &closure; + } +}; + +void Image::EIO_EncodeChunked(uv_work_t* work) +{ + using closure_type = chunked_encode_image_baton_t; + closure_type & closure = *reinterpret_cast(work->data); + try + { + callback_streambuf streambuf(closure, closure.buffer_size); + std::ostream stream(&streambuf); + + if (closure.image_baton.palette) + { + mapnik::save_to_stream(*closure.image_baton.im->this_, + stream, + closure.image_baton.format, + *closure.image_baton.palette); + } + else + { + mapnik::save_to_stream(*closure.image_baton.im->this_, + stream, + closure.image_baton.format); + } + + stream.flush(); + } + catch (std::exception const& ex) + { + closure.image_baton.error = true; + closure.image_baton.error_name = ex.what(); + } + + // Signalize end of stream + closure(static_cast(NULL), 0); +} + +NAN_METHOD(Image::encodeChunked) +{ + Image* im = Nan::ObjectWrap::Unwrap(info.Holder()); + + std::string format = "png"; + palette_ptr palette; + + if (info.Length() != 4) + { + Nan::ThrowTypeError("Function requires four arguments"); + return; + } + + // accept custom format + if (!info[0]->IsString()) + { + Nan::ThrowTypeError("first arg, 'format' must be a string"); + return; + } + format = TOSTR(info[0]); + + // options hash + if (!info[1]->IsObject()) + { + Nan::ThrowTypeError("second arg must be an options object"); + return; + } + + v8::Local options = info[1].As(); + + if (options->Has(Nan::New("palette").ToLocalChecked())) + { + v8::Local format_opt = options->Get(Nan::New("palette").ToLocalChecked()); + if (!format_opt->IsObject()) + { + Nan::ThrowTypeError("'palette' must be an object"); + return; + } + + v8::Local obj = format_opt.As(); + if (obj->IsNull() || obj->IsUndefined() || !Nan::New(Palette::constructor)->HasInstance(obj)) + { + Nan::ThrowTypeError("mapnik.Palette expected as second arg"); + return; + } + + palette = Nan::ObjectWrap::Unwrap(obj)->palette(); + } + + int buffer_size; + if (!info[2]->IsNumber() || (buffer_size = info[2]->IntegerValue()) < 1) + { + Nan::ThrowTypeError("third arg must be a positive integer"); + return; + } + + // ensure callback is a function + v8::Local callback = info[info.Length() - 1]; + if (!callback->IsFunction()) + { + Nan::ThrowTypeError("last argument must be a callback function"); + return; + } + + chunked_encode_image_baton_t *closure = new chunked_encode_image_baton_t(buffer_size); + closure->image_baton.request.data = closure; + closure->image_baton.im = im; + closure->image_baton.format = format; + closure->image_baton.palette = palette; + closure->image_baton.error = false; + closure->image_baton.cb.Reset(callback.As()); + + uv_queue_work(uv_default_loop(), &closure->image_baton.request, EIO_EncodeChunked, NULL); + im->Ref(); +} diff --git a/test/image.test.js b/test/image.test.js index 92cf5ee363..fbad7990c5 100644 --- a/test/image.test.js +++ b/test/image.test.js @@ -55,6 +55,11 @@ describe('mapnik.Image ', function() { assert.throws(function() { im.encode('png', null, function(err, result) {}); }); assert.throws(function() { im.encode(1, {}, function(err, result) {}); }); assert.throws(function() { im.encode('png', {}, null); }); + assert.throws(function() { im.encodeChunked('png', {palette:{}}, function(err, result) {}); }); + assert.throws(function() { im.encodeChunked('png', {palette:null}, function(err, result) {}); }); + assert.throws(function() { im.encodeChunked('png', null, function(err, result) {}); }); + assert.throws(function() { im.encodeChunked(1, {}, function(err, result) {}); }); + assert.throws(function() { im.encodeChunked('png', {}, null); }); im.encode('foo', {}, function(err, result) { assert.throws(function() { if (err) throw err; }); done(); @@ -146,6 +151,117 @@ describe('mapnik.Image ', function() { assert.equal(im.encodeSync().length, im2.encodeSync().length); }); + it('should be able to encode by chunks', function(done) { + var im = new mapnik.Image(256, 256); + assert.ok(im instanceof mapnik.Image); + + assert.equal(im.width(), 256); + assert.equal(im.height(), 256); + + var actual_length = 0; + var chunk_count = 0; + + im.encodeChunked('png32', {}, 1024, function(err, result) { + if (err) throw err; + if (result.length == 0) { + assert.equal(1, chunk_count); + assert.equal(im.encodeSync('png32').length, actual_length); + done(); + } else { + chunk_count++; + actual_length += result.length; + } + }); + }); + + it('should be able to encode by chunks - multiple chunks', function(done) { + var im = new mapnik.Image.openSync('./test/data/images/sat_image.png'); + assert.ok(im instanceof mapnik.Image); + + assert.equal(im.width(), 75); + assert.equal(im.height(), 75); + + var actual_length = 0; + var chunk_count = 0; + + im.encodeChunked('png32', {}, 1024, function(err, result) { + if (err) throw err; + if (result.length == 0) { + assert.equal(16, chunk_count); + assert.equal(im.encodeSync('png32').length, actual_length); + done(); + } else { + chunk_count++; + actual_length += result.length; + } + }); + }); + + it('chunked encoding should throw on bad chunk size argument', function() { + var im = new mapnik.Image(256, 256); + assert.ok(im instanceof mapnik.Image); + + assert.equal(im.width(), 256); + assert.equal(im.height(), 256); + + assert.throws(function() { + im.encodeChunked('png32', {}, 0, function(err, result) { + }); + }); + }); + + it('should be able to encode by chunks, chunk size is 1', function(done) { + var im = new mapnik.Image(256, 256); + assert.ok(im instanceof mapnik.Image); + + assert.equal(im.width(), 256); + assert.equal(im.height(), 256); + + var actual_length = 0; + var reference_length = im.encodeSync('png32').length; + var chunk_count = 0; + + im.encodeChunked('png32', {}, 1, function(err, result) { + if (err) throw err; + if (result.length == 0) { + assert.equal(reference_length, chunk_count); + assert.equal(reference_length, actual_length); + done(); + } else { + chunk_count++; + actual_length += result.length; + } + }); + }); + + it('encode by chunks should encode with a palette', function(done) { + var im = new mapnik.Image.openSync('./test/data/images/sat_image.png'); + var pal = new mapnik.Palette(new Buffer('\xff\x00\xff\xff\xff\xff','ascii'), 'rgb'); + + var actual_length = 0; + var chunk_count = 0; + + im.encodeChunked('png', {palette:pal}, 1024, function(err, result) { + if (err) throw err; + if (result.length == 0) { + assert.equal(1, chunk_count); + assert.equal(im.encodeSync('png', {palette:pal}).length, actual_length); + done(); + } else { + chunk_count++; + actual_length += result.length; + } + }); + }); + + it('chunked encoding should throw with invalid encoding format', function(done) { + var im = new mapnik.Image(256, 256); + im.encodeChunked('foo', {}, 1024, function(err, result) { + assert.ok(err); + done(); + }); + }); + it('should be able to open via byte stream', function(done) { var im = new mapnik.Image(256, 256); // png