storage: add gzip option

Author: stephenplusplus

lib/storage/bucket.js

@@ -875,6 +875,8 @@ Bucket.prototype.setMetadata = function(metadata, callback) {
  *     will be uploaded to the File object's bucket and under the File object's
  *     name. Lastly, when this argument is omitted, the file is uploaded to your
  *     bucket using the name of the local file.
+ * @param {boolean} options.gzip - Automatically gzip the file. This will set
+ *     `options.metadata.contentEncoding` to `gzip`.
  * @param {object=} options.metadata - Metadata to set for your file.
  * @param {boolean=} options.resumable - Force a resumable upload. (default:
  *     true for files larger than 5MB). Read more about resumable uploads
@@ -927,6 +929,17 @@ Bucket.prototype.setMetadata = function(metadata, callback) {
  * });
  *
  * //-
+ * // You can also have a file gzip'd on the fly.
+ * //-
+ * bucket.upload('index.html', { gzip: true }, function(err, file) {
+ *   // Your bucket now contains:
+ *   // - "index.html" (automatically compressed with gzip)
+ *
+ *   // Downloading the file with `file.download` will automatically decode the
+ *   // file.
+ * });
+ *
+ * //-
  * // You may also re-use a File object, {module:storage/file}, that references
  * // the file you wish to create or overwrite.
  * //-
@@ -990,7 +1003,8 @@ Bucket.prototype.upload = function(localPath, options, callback) {
       .pipe(newFile.createWriteStream({
         validation: options.validation,
         resumable: resumable,
-        metadata: metadata
+        metadata: metadata,
+        gzip: options.gzip
       }))
       .on('error', function(err) {
         callback(err);

lib/storage/file.js

@@ -34,6 +34,7 @@ var request = require('request').defaults({
 });
 var streamEvents = require('stream-events');
 var through = require('through2');
+var zlib = require('zlib');
 
 /**
  * @type {module:storage/acl}
@@ -611,8 +612,10 @@ File.prototype.createReadStream = function(options) {
  * uploaded.
  *
  * @param {object=} options - Configuration object.
- * @param {object=} options.metadata - Set the metadata for this file.
- * @param {boolean=} options.resumable - Force a resumable upload. NOTE: When
+ * @param {boolean} options.gzip - Automatically gzip the file. This will set
+ *     `options.metadata.contentEncoding` to `gzip`.
+ * @param {object} options.metadata - Set the metadata for this file.
+ * @param {boolean} options.resumable - Force a resumable upload. NOTE: When
  *     working with streams, the file format and size is unknown until it's
  *     completely consumed. Because of this, it's best for you to be explicit
  *     for what makes sense given your input. Read more about resumable uploads
@@ -642,6 +645,24 @@ File.prototype.createReadStream = function(options) {
  *   });
  *
  * //-
+ * // <h4>Uploading a File with gzip compression</h4>
+ * //-
+ * var fs = require('fs');
+ * var htmlFile = myBucket.file('index.html');
+ *
+ * fs.createReadStream('/Users/stephen/site/index.html')
+ *   .pipe(htmlFile.createWriteStream({ gzip: true }))
+ *   .on('error', function(err) {})
+ *   .on('complete', function(metadata) {
+ *     // The file upload is complete.
+ *   });
+ *
+ * //-
+ * // Downloading the file with `createReadStream` will automatically decode the
+ * // file.
+ * //-
+ *
+ * //-
  * // <h4>Uploading a File with Metadata</h4>
  * //
  * // One last case you may run into is when you want to upload a file to your
@@ -661,13 +682,23 @@ File.prototype.createReadStream = function(options) {
  *       }
  *     }
  *   }))
- *   .on('error', function(err) {});
+ *   .on('error', function(err) {})
+ *   .on('complete', function(metadata) {
+ *     // The file upload is complete.
+ *   });
  */
 File.prototype.createWriteStream = function(options) {
   options = options || {};
 
   var that = this;
+
+  var gzip = options.gzip;
+
   var metadata = options.metadata || {};
+  if (gzip) {
+    metadata.contentEncoding = 'gzip';
+  }
+
   var validations = ['crc32c', 'md5'];
   var validation;
 
@@ -693,9 +724,11 @@ File.prototype.createWriteStream = function(options) {
   var localCrc32cHash;
   var localMd5Hash = crypto.createHash('md5');
 
-  var dup = streamEvents(duplexify());
+  var writableStream = streamEvents(duplexify());
 
-  var throughStream = through(function(chunk, enc, next) {
+  var throughStream = through();
+
+  var validationStream = through(function(chunk, enc, next) {
     if (crc32c) {
       localCrc32cHash = crc.calculate(chunk, localCrc32cHash);
     }
@@ -708,25 +741,30 @@ File.prototype.createWriteStream = function(options) {
     next();
   });
 
+  validationStream.on('end', function() {
+    if (crc32c) {
+      localCrc32cHash = new Buffer([localCrc32cHash]).toString('base64');
+    }
+
+    if (md5) {
+      localMd5Hash = localMd5Hash.digest('base64');
+    }
+  });
+
   throughStream
-    .on('end', function() {
-      if (crc32c) {
-        localCrc32cHash = new Buffer([localCrc32cHash]).toString('base64');
-      }
 
-      if (md5) {
-        localMd5Hash = localMd5Hash.digest('base64');
-      }
-    })
+    .pipe(gzip ? zlib.createGzip() : through())
+
+    .pipe(validationStream)
 
-    .pipe(dup)
+    .pipe(writableStream)
 
     // Wait until we've received data to determine what upload technique to use.
     .once('writing', function() {
-      if (util.is(options.resumable, 'boolean') && !options.resumable) {
-        that.startSimpleUpload_(dup, metadata);
+      if (options.resumable === false) {
+        that.startSimpleUpload_(writableStream, metadata);
       } else {
-        that.startResumableUpload_(dup, metadata);
+        that.startResumableUpload_(writableStream, metadata);
       }
     })
 

system-test/data/long-html-file.html

@@ -42,8 +42,11 @@ var files = {
   big: {
     path: 'system-test/data/three-mb-file.tif'
   },
-  gzip: {
+  html: {
     path: 'system-test/data/long-html-file.html'
+  },
+  gzip: {
+    path: 'system-test/data/long-html-file.html.gz'
   }
 };
 
@@ -535,16 +538,41 @@ describe('storage', function() {
       });
     });
 
+    it('should gzip a file on the fly and download it', function(done) {
+      var options = {
+        gzip: true
+      };
+
+      var expectedContents = fs.readFileSync(files.html.path, 'utf-8');
+
+      bucket.upload(files.html.path, options, function(err, file) {
+        assert.ifError(err);
+
+        file.download(function(err, contents) {
+          assert.ifError(err);
+          assert.strictEqual(contents.toString(), expectedContents);
+          file.delete(done);
+        });
+      });
+    });
+
     it('should upload a gzipped file and download it', function(done) {
       var options = {
         metadata: {
           contentEncoding: 'gzip'
         }
       };
 
+      var expectedContents = fs.readFileSync(files.html.path, 'utf-8');
+
       bucket.upload(files.gzip.path, options, function(err, file) {
         assert.ifError(err);
-        file.download(done);
+
+        file.download(function(err, contents) {
+          assert.ifError(err);
+          assert.strictEqual(contents.toString(), expectedContents);
+          file.delete(done);
+        });
       });
     });
 

system-test/data/long-html-file.html.gz

@@ -960,6 +960,21 @@ describe('Bucket', function() {
       bucket.upload(filepath, options, assert.ifError);
     });
 
+    it('should allow specifying options.gzip', function(done) {
+      var fakeFile = new FakeFile(bucket, 'file-name');
+      var options = { destination: fakeFile, gzip: true };
+      fakeFile.createWriteStream = function(options) {
+        var ws = new stream.Writable();
+        ws.write = util.noop;
+        setImmediate(function() {
+          assert.strictEqual(options.gzip, true);
+          done();
+        });
+        return ws;
+      };
+      bucket.upload(filepath, options, assert.ifError);
+    });
+
     it('should allow specifying options.resumable', function(done) {
       var fakeFile = new FakeFile(bucket, 'file-name');
       var options = { destination: fakeFile, resumable: false };

system-test/storage.js

@@ -920,6 +920,17 @@ describe('File', function() {
       writable.write('data');
     });
 
+    it('should set metadata.contentEncoding with gzip', function(done) {
+      var writable = file.createWriteStream({ gzip: true });
+
+      file.startResumableUpload_ = function(stream, metadata) {
+        assert.strictEqual(metadata.contentEncoding, 'gzip');
+        done();
+      };
+
+      writable.write('data');
+    });
+
     describe('validation', function() {
       var data = 'test';