Open
Description
I'm attempting to use the library to run a few background threads in a node app, each processing an 8-32MB Uint8Array. Running with 4 workers and 4MB of data each, it currently takes more than 15 minutes to post the initial messages (before it can even start processing). The performance is rather bad, since typed arrays are supposed to be Transferable.
So far as I can tell, each array is being serialized repeatedly when being transferred between threads. This appears to be happening here (thanks to apsillers for finding that). Producing those strings peaks around 7GB of RES memory.
My test app consists of two files:
- index.js:
var Worker = require('webworker-threads').Worker;
var workerCount = 4, finishedCount = 0, bufferSize = (1024 * 1024 * 4);
var workers = {};
var buffers = {};
function messageReceiver(event) {
var data = event.data;
var id = data.id;
var buffer = data.buffer;
console.log('Received', buffer[0], 'from', id);
workers[id].terminate();
if (++finishedCount === workerCount) {
console.log('All workers finished.');
}
}
for (var i = 0; i < workerCount; ++i) {
console.log('Spawning worker', i);
var worker = new Worker('worker.js');
worker.onmessage = messageReceiver;
workers[i] = worker;
}
console.log('All workers spawned.');
for (var i = 0; i < workerCount; ++i) {
console.log('Allocating buffer', i);
buffers[i] = new Uint32Array(new Buffer(bufferSize));
}
console.log('All buffers allocated.');
for (var i = 0; i < workerCount; ++i) {
console.log('Launching worker', i);
var w = workers[i];
var a = buffers[i];
var data = {
id: i,
buffer: a.buffer
};
w.postMessage(data, [data.buffer]);
}
console.log('All workers launched.');
- worker.js:
function finishWorker(id, buffer) {
console.log('Thread ' + id + ' finished filling buffer.');
var data = {
id: id,
buffer: buffer
};
postMessage(data, [data.buffer]);
}
function fillBuffer(buffer) {
for (var i = 0; i < buffer.byteLength; ++i) {
if (i % (1024 * 1024) === 0) {
console.log('Thread ' + id + ' filled ' + i + ' bytes');
}
buffer[i] = ((Math.random() * 255) | 0) % 255;
}
}
this.onmessage = function (event) {
var id = event.data.id, buffer = event.data.buffer;
console.log('Thread ' + id + ' received buffer of ' + buffer.byteLength);
fillBuffer(buffer);
finishWorker(id, buffer);
};