s3-client

Installation

Features

Retries get pushed to the end of the parallelization queue.

or downloading files unnecessarily.

Synopsis

Create a client

var s3 = require('s3-client');

var client = s3.createClient({
  maxAsyncS3: 20,     // this is the default
  s3RetryCount: 3,    // this is the default
  s3RetryDelay: 1000, // this is the default
  multipartUploadThreshold: 20971520, // this is the default (20 MB)
  multipartUploadSize: 15728640, // this is the default (15 MB)
  s3Options: {
    accessKeyId: "your s3 key",
    secretAccessKey: "your s3 secret",
    region: "your region",
    // endpoint: 's3.yourdomain.com',
    // sslEnabled: false
    // any other options are passed to new AWS.S3()
    // See: http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Config.html#constructor-property
  },
});

Create a client from existing AWS.S3 object

var s3 = require('s3-client');
var awsS3Client = new AWS.S3(s3Options);
var options = {
  s3Client: awsS3Client,
  // more options available. See API docs below.
};
var client = s3.createClient(options);

Upload a file to S3

var params = {
  localFile: "some/local/file",

  s3Params: {
    Bucket: "s3 bucket name",
    Key: "some/remote/file",
    // other options supported by putObject, except Body and ContentLength.
    // See: http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObject-property
  },
};
var uploader = client.uploadFile(params);
uploader.on('error', function(err) {
  console.error("unable to upload:", err.stack);
});
uploader.on('progress', function() {
  console.log("progress", uploader.progressMd5Amount,
            uploader.progressAmount, uploader.progressTotal);
});
uploader.on('end', function() {
  console.log("done uploading");
});

Download a file from S3

var params = {
  localFile: "some/local/file",

  s3Params: {
    Bucket: "s3 bucket name",
    Key: "some/remote/file",
    // other options supported by getObject
    // See: http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#getObject-property
  },
};
var downloader = client.downloadFile(params);
downloader.on('error', function(err) {
  console.error("unable to download:", err.stack);
});
downloader.on('progress', function() {
  console.log("progress", downloader.progressAmount, downloader.progressTotal);
});
downloader.on('end', function() {
  console.log("done downloading");
});

Sync a directory to S3

var params = {
  localDir: "some/local/dir",
  deleteRemoved: true, // default false, whether to remove s3 objects
                       // that have no corresponding local file.

  s3Params: {
    Bucket: "s3 bucket name",
    Prefix: "some/remote/dir/",
    // other options supported by putObject, except Body and ContentLength.
    // See: http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObject-property
  },
};
var uploader = client.uploadDir(params);
uploader.on('error', function(err) {
  console.error("unable to sync:", err.stack);
});
uploader.on('progress', function() {
  console.log("progress", uploader.progressAmount, uploader.progressTotal);
});
uploader.on('end', function() {
  console.log("done uploading");
});

Tips

agents. This will improve bandwidth when using `uploadDir` and `downloadDir`
functions. For example:

```js
http.globalAgent.maxSockets = https.globalAgent.maxSockets = 20;
```

- See AWS SDK documentation for available options which are passed to `new AWS.S3()`:
  http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Config.html#constructor-property

ever have open to S3. defaults to `20`.

Default 3.

operation. Default 1000.

will be uploaded via a multipart request. Default is 20MB. Minimum is 5MB.
Maximum is 5GB.

The minimum size is 5MB. The maximum size is 5GB. Default is 15MB. Note that
S3 has a maximum of 10000 parts for a multipart upload, so if this value is
too small, it will be ignored in favor of the minimum necessary value
required to upload the file.

s3.getPublicUrl(bucket, key, bucketLocation)

- "" (default) - US Standard
- "eu-west-1"
- "us-west-1"
- "us-west-2"
- "ap-southeast-1"
- "ap-southeast-2"
- "ap-northeast-1"
- "sa-east-1"

parameter in `s3Params`, it will be automatically set for you based on the
file extension of `localFile`. If the extension is unrecognized,
`defaultContentType` will be used instead. Defaults to
`application/octet-stream`.

parallel.

- `data` is the same object that you get from `putObject` in AWS SDK

`progressTotal` properties change. Note that it is possible for progress to
go backwards when an upload fails and must be retried.

is opened with the [fd-slicer](https://github.com/andrewrk/node-fd-slicer)
module because we might need to read from multiple locations in the file at
the same time. `fdSlicer` is an object for which you can call
`createReadStream(options)`. See the fd-slicer README for more information.

properties change.

client.downloadBuffer(s3Params)

`buffer` is a `Buffer` containing the object data.

properties change.

client.downloadStream(s3Params)

headers and status code.

into directories. Default `false`.

the same as the `data` object in AWS SDK.

`dirsFound` properties change.

client.deleteObjects(s3Params)

greater than 1000.

client.uploadDir(params)

- `Prefix` (required)
- `Bucket` (required)

default false

needs to be uploaded. You can use this to skip some files. See below.

parameter in `s3Params`, it will be automatically set for you based on the
file extension of `localFile`. If the extension is unrecognized,
`defaultContentType` will be used instead. Defaults to
`application/octet-stream`.

Defaults to `true`.

function getS3Params(localFile, stat, callback) {
  // call callback like this:
  var err = new Error(...); // only if there is an error
  var s3Params = { // if there is no error
    ContentType: getMimeType(localFile), // just an example
  };
  // pass `null` for `s3Params` if you want to skip uploading this file.
  callback(err, s3Params);
}

uploading.

finishes uploading.

returned objects to be in sorted order.

MD5 sums.

each operation progresses we compare both sorted lists side-by-side,
iterating over them one at a time, uploading files whose MD5 sums don't
match the remote object (or the remote object is missing), and, if
`deleteRemoved` is set, deleting remote objects whose corresponding local
files are missing.

client.downloadDir(params)

- `Prefix` (required)
- `Bucket` (required)

needs to be downloaded. You can use this to skip downloading some objects.
See below.

Defaults to `true`.

function getS3Params(localFile, s3Object, callback) {
  // localFile is the destination path where the object will be written to
  // s3Object is same as one element in the `Contents` array from here:
  // http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#listObjects-property

  // call callback like this:
  var err = new Error(...); // only if there is an error
  var s3Params = { // if there is no error
    VersionId: "abcd", // just an example
  };
  // pass `null` for `s3Params` if you want to skip downloading this object.
  callback(err, s3Params);
}

downloading.

finishes downloading.

returned objects to be in sorted order.

MD5 sums.

each operation progresses we compare both sorted lists side-by-side,
iterating over them one at a time, downloading objects whose MD5 sums don't
match the local file (or the local file is missing), and, if
`deleteRemoved` is set, deleting local files whose corresponding objects
are missing.

per response.

immediately send a delete request for all of them.

client.moveObject(s3Params)

var client = require('s3-client').createClient({ /* options */ });
client.s3.headObject({
  Bucket: 's3 bucket name',
  Key: 'some/remote/file'
}, function(err, data) {
  if (err) {
    // file does not exist (err.statusCode == 404)
    return;
  }
  // file exists
});