Create a zip file on S3 from files on S3 using Lambda Node
<p>I need to create a Zip file that consists of a selection of files (videos and images) located in my s3 bucket. </p>

<p>The problem at the moment using my code below is that I quickly hit the memory limit on Lambda.</p>

<pre><code>async.eachLimit(files, 10, function(file, next) {
    var params = {
        Bucket: bucket, // bucket name
        Key: file.key
    };
    s3.getObject(params, function(err, data) {
        if (err) {
            console.log('file', file.key);
            console.log('get image files err',err, err.stack); // an error occurred
        } else {
            console.log('file', file.key);
            zip.file(file.key, data.Body);
            next();
        }
    });
}, 
function(err) {
    if (err) {
        console.log('err', err);
    } else {
        console.log('zip', zip);
        content = zip.generateNodeStream({
            type: 'nodebuffer',
            streamFiles:true
        });
        var params = {
            Bucket: bucket, // name of dest bucket
            Key: 'zipped/images.zip',
            Body: content
        };
        s3.upload(params, function(err, data) {
            if (err) {
                console.log('upload zip to s3 err',err, err.stack); // an error occurred
            } else {
                console.log(data); // successful response
            }
        });
    }
});
</code></pre>

<ul>
<li><p>Is this possible using Lambda, or should I look at a different
approach?</p></li>
<li><p>Is it possible to write to a compressed zip file on the fly, therefore eliminating the memory issue somewhat, or do I need to have the files collected before compression?</p></li>
</ul>

<p>Any help would be much appreciated.</p>