I have a Lambda in a VPC using NodeJS 20. I created the EFS file system using the default settings - so General purpose, Elastic. The lambda and EFS drive are in the same region. I can successfully write files.
When I write 1000 small (4Kb) files in parallel on my averagely-speced windows laptop the files are written in under 300ms. However when I run this same code in Lambda the files are written in approx 17 seconds!.
Is this really as fast as EFS goes in this configuration? Is there some way I can troubleshoot this issue?
My code - for reference
'use strict'
import fs from 'fs'
import path from 'path'
import * as url from 'node:url'
import { randomUUID } from 'crypto'
import _ from 'lodash'
import prettyMilliseconds from 'pretty-ms'
import prettyBytes from 'pretty-bytes'
import casual from 'casual'
const DIRNAME = url.fileURLToPath(new URL('.', import.meta.url))
const NUMBER_CONTACTS = 1000
const REPEAT_NOTES = 100
async function fulfill(AWS, event, config, context) {
let filePathStub
if (process.env.AWS_LAMBDA_FUNCTION_NAME) {
filePathStub = path.resolve('/mnt/mounted')
} else {
filePathStub = path.join(DIRNAME, 'mounted')
}
// create some sample data
const contactsToPut = []
for (let i = 0; i < NUMBER_CONTACTS; i++) {
const contact = {}
contact.Id = randomUUID()
contact.RecordType = 'PERSON'
contact.PersonName = {
FirstName: casual.first_name,
LastName: casual.last_name
}
contact.WorkDetails = {
JobTitle: 'branch manager',
Company: casual.company_name
}
contact.Email = [
{ Label: 'default', 'Value': casual.email.toLowerCase() },
{ Label: 'default', 'Value': casual.email.toLowerCase() },
{ Label: 'default', 'Value': casual.email.toLowerCase() },
{ Label: 'default', 'Value': casual.email.toLowerCase() },
{ Label: 'default', 'Value': casual.email.toLowerCase() }
]
contact.Notes = randomUUID().repeat(REPEAT_NOTES)
contactsToPut.push(JSON.stringify(contact))
}
const now = new Date()
// break the contacts into chunks and execute each chunk in sequence
// so we can control parallelism.
// for this example the chunk size will be the number of contacts
// meaning all contacts are written in parallel
const chunks = _.chunk(contactsToPut, NUMBER_CONTACTS)
for (let i = 0; i < chunks.length; i++) {
console.log (`writing chunk ${i+1} of ${chunks.length}`)
const chunk = chunks[i]
// write the files in parallel
await Promise.allSettled(chunk.map(async (contact) => {
fs.writeFileSync(path.join(filePathStub, contact.Id + '.json'), contact)
}))
}
const duration = prettyMilliseconds(new Date() - now)
console.log(`${contactsToPut.length} files of ${prettyBytes(Buffer.byteLength(chunks[0][0]))} totalling ${prettyBytes(Buffer.byteLength(JSON.stringify(contactsToPut)))} written in ${duration}`)
}
export {
fulfill
}