add validation choices to createreadstream

Author: stephenplusplus

lib/storage/file.js

@@ -20,6 +20,7 @@
 
 'use strict';
 
+var crc = require('fast-crc32c');
 var crypto = require('crypto');
 var duplexify = require('duplexify');
 var request = require('request');
@@ -213,6 +214,13 @@ File.prototype.copy = function(destination, callback) {
  * 911. If you receive this error, the best recourse is to try downloading the
  * file again.
  *
+ * @param {object=} options - Configuration object.
+ * @param {string|boolean} options.validation - Possible values: `"md5"`,
+ *     `"crc32c"`, or `false`. By default, data integrity is validated with an
+ *     MD5 checksum for maximum reliability. CRC32c will provide better
+ *     performance with less reliability. You may also choose to skip validation
+ *     completely, however this is **not recommended**.
+ *
  * @example
  * //-
  * // <h4>Downloading a File</h4>
@@ -228,19 +236,45 @@ File.prototype.copy = function(destination, callback) {
  *   .pipe(fs.createWriteStream('/Users/stephen/Photos/image.png'))
  *   .on('error', function(err) {});
  */
-File.prototype.createReadStream = function() {
+File.prototype.createReadStream = function(options) {
+  options = options || {};
+
   var that = this;
   var throughStream = through();
 
-  this.getMetadata(function(err, metadata) {
-    if (err) {
-      throughStream.emit('error', err);
-      throughStream.end();
-      return;
+  var validations = ['crc32c', 'md5'];
+  var validation;
+
+  if (util.is(options.validation, 'string')) {
+    options.validation = options.validation.toLowerCase();
+
+    if (validations.indexOf(options.validation) > -1) {
+      validation = options.validation;
+    } else {
+      validation = 'all';
     }
+  }
 
-    createAuthorizedReq(metadata.mediaLink);
-  });
+  if (util.is(options.validation, 'undefined')) {
+    validation = 'all';
+  }
+
+  var crc32c = validation === 'crc32c' || validation === 'all';
+  var md5 = validation === 'md5' || validation === 'all';
+
+  if (this.metadata.mediaLink) {
+    createAuthorizedReq(this.metadata.mediaLink);
+  } else {
+    this.getMetadata(function(err, metadata) {
+      if (err) {
+        throughStream.emit('error', err);
+        throughStream.end();
+        return;
+      }
+
+      createAuthorizedReq(metadata.mediaLink);
+    });
+  }
 
   return throughStream;
 
@@ -261,6 +295,7 @@ File.prototype.createReadStream = function() {
 
         // For data integrity, hash the contents of the stream as we receive it
         // from the server.
+        var localCrc32cHash;
         var localMd5Hash = crypto.createHash('md5');
 
         request(authorizedReqOpts)
@@ -270,15 +305,46 @@ File.prototype.createReadStream = function() {
           })
 
           .on('data', function(chunk) {
-            localMd5Hash.update(chunk);
+            if (crc32c) {
+              localCrc32cHash = crc.calculate(chunk, localCrc32cHash);
+            }
+
+            if (md5) {
+              localMd5Hash.update(chunk);
+            }
           })
 
-          .on('complete', function() {
-            localMd5Hash = localMd5Hash.digest('base64');
+          .on('complete', function(res) {
+            var failed = false;
 
-            if (that.metadata.md5Hash === localMd5Hash) {
-              throughStream.emit('complete');
-            } else {
+            if (crc32c) {
+              localCrc32cHash = new Buffer([localCrc32cHash]);
+              localCrc32cHash = localCrc32cHash.toString('base64');
+            }
+
+            if (md5) {
+              localMd5Hash = localMd5Hash.digest('base64');
+            }
+
+            var hashes = {};
+            res.headers['x-goog-hash'].split(',').forEach(function (hash) {
+              var hashType = hash.split('=')[0];
+              hashes[hashType] = hash.substr(hash.indexOf('=') + 1);
+            });
+
+            if (validation === 'all') {
+              if (hashes.md5) {
+                failed = localMd5Hash !== hashes.md5;
+              } else if (hashes.crc32) {
+                failed = localCrc32cHash !== hashes.crc32c.substr(4);
+              }
+            } else if (md5) {
+              failed = localMd5Hash !== hashes.md5;
+            } else if (crc32c) {
+              failed = localCrc32cHash !== hashes.crc32c.substr(4);
+            }
+
+            if (failed) {
               var error = new Error({
                 code: 911,
                 message: [
@@ -289,6 +355,8 @@ File.prototype.createReadStream = function() {
               });
 
               throughStream.emit('error', error);
+            } else {
+              throughStream.emit('complete');
             }
 
             throughStream.end();

package.json

@@ -46,6 +46,7 @@
   "dependencies": {
     "duplexify": "^3.1.2",
     "extend": "^1.3.0",
+    "fast-crc32c": "^0.1.3",
     "google-service-account": "^1.0.3",
     "mime": "^1.2.11",
     "node-uuid": "^1.4.1",