Skip to content

To save stream file directly to GoogleCloudStorage via GoogleCloudFunction #513

Description

@geoseong

I want to save stream data directly to GoogleCloudStorage via GoogleCloudFunction Node.js.

I tried some ways but works nothing.
Now i'm trying at localhost server first.

so once I tried to test, this error shown :
req.file.cloudStorageError = err;
TypeError: Cannot set property 'cloudStorageError' of undefined

I want to know how this works and get some clue.. plz help me with do this.

client side code

HTML

    <form data-brackets-id='37' action="/api/browser/proxy?url=http%3A%2F%2Flocalhost%3A8989%2Ffileupload" method="post" enctype="multipart/form-data" name="">
        <input data-brackets-id='38' type="file" name="file" id="file" value="">
        <input data-brackets-id='39' type="button" data-inline="true" value="Input" id="filesubmit">
    </form>

Script

    $('form #filesubmit').on('click', (e)=>{
        var uploadfile = $("input[name=file]")[0].files[0];
        var formData = new FormData(); 
        formData.append("myfile", $("input[name=file]")[0].files[0]); 
        
        console.log('uploadfile', uploadfile, uploadfile.type);
       
        var googleCloud = 'https://us-central1-xxxx-xxxxx.cloudfunctions.net/functionNm';
        var localurl = 'http://localhost:8989/fileupload'
        $.ajax({ 
            url: localurl,  // googleCloud
            data: formData, 
            processData: false, 
            contentType: false, 
            type: 'POST', 
            success: function(data){ 
                console.log('response data', data);
            } 
        });

    });

server side code

// start form
var Storage = require('@google-cloud/storage');
var multiparty = require('multiparty');
var fs = require('fs');
var form = new multiparty.Form();

const storage = Storage({
    projectId: [My Project ID]
});
const bucket = storage.bucket('multilang_service');

function getPublicUrl (filename) {
  return `https://storage.googleapis.com/${CLOUD_BUCKET}/${filename}`;
}

// get field name & value
form.on('field',function(name,value){
  console.log('[?] normal field / name = '+name+' , value = '+value);
});

// file upload handling
form.on('part',function(part){
    var filename;
    var size;
    if (part.filename) {
        filename = part.filename;
        size = part.byteCount;
    }else{
        part.resume();
    }   
    
    const gcsname = Date.now() + filename;
    const file = bucket.file(gcsname);
    const stream = file.createWriteStream({
        metadata: {
          contentType: part.headers['content-type']
        }
    });
    stream.on('error', (err) => {
        req.file.cloudStorageError = err;
        next(err);
    });

    stream.on('finish', () => {
        req.file.cloudStorageObject = gcsname;
        file.makePublic().then(() => {
          req.file.cloudStoragePublicUrl = getPublicUrl(gcsname);
          next();
        });
    });
    stream.end(form);
    
    part.on('data',function(chunk){
        console.log('[2] '+filename+' read '+chunk.length + 'bytes');
    });

    part.on('end',function(){
        console.log('[4] '+filename+' Part read complete');
    });
});

// track progress
form.on('progress',function(byteRead,byteExpected){
    console.log('[3] Reading total  '+byteRead+'/'+byteExpected);
});

// all uploads are completed
form.on('close',function(){
    res.send('done for test');
});

form.parse(req);
// end form

Activity

  1. geoseong commented on Nov 3, 2017

    @geoseong
    Author

    I think I solved problem using 'multer.js'.

    and
    Here's my sample.
    and how to use multer between client and server.

    var multer  = require('multer');
    var storage = multer.memoryStorage()
    var upload = multer({ storage: storage })
    
    app.post('/fileupload', upload.single('streamfile'), function (req, res, next) {
    //app.post('/fileupload', (req, res) => {
        console.log('fileupload req.file', req.file);
        sendUploadToGCS(req, res, next);
    });
    
    function sendUploadToGCS(req, res, next) {
    
      // start form
        var Storage = require('@google-cloud/storage');
      	var multiparty = require('multiparty');
        var fs = require('fs');
      	var form = new multiparty.Form();
        
        const CLOUD_BUCKET = '[my bucket name]';
        var filename = req.file.originalname;
        
        const storage = Storage({
          projectId: '[my gcp project id]'
        });
        const bucket = storage.bucket('[my bucket name]'');
        function getPublicUrl (filename) {
          return `https://storage.googleapis.com/${CLOUD_BUCKET}/${filename}`;
        }
        const gcsname = Date.now() + filename;
        const file = bucket.file(gcsname);
        const stream = file.createWriteStream({
            metadata: {
              contentType: req.file.mimetype
            }
        });
    
        console.log('gcsname', gcsname);
        console.log('file', file);
        console.log('stream', stream);
    
        stream.on('error', (err) => {
            req.file.cloudStorageError = err;
                console.log('req.file.cloudStorageError',err);
            next(err);
        });
    
        stream.on('finish', () => {
            req.file.cloudStorageObject = gcsname;
            file.makePublic().then(() => {
              req.file.cloudStoragePublicUrl = getPublicUrl(gcsname);
                    console.log('req.file.cloudStoragePublicUrl', req.file.cloudStoragePublicUrl);
                    console.log('req.file.buffer', req.file);
              next();
            });
        });
    
        stream.end(req.file.buffer);
        
    };
    

    I hope this will be helpful.
    Thanks.

  2. added a commit that references this issue on Nov 8, 2022
  3. added a commit that references this issue on Nov 17, 2022
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

      Milestone

      No milestone

      Relationships

      None yet

      Development

      No branches or pull requests

      Issue actions