I am attempting to use S3 MultipartUpload to concat files in an S3 bucket. If you have several files >5MB (the last file can be smaller), you can concatenate them in S3 into a larger file. It's basically the equivalent of using cat
to merge files together. When I attempt to do this in Go, I get:
An error occurred (AccessDenied) when calling the UploadPartCopy operation: Access Denied
The code looks kind of like this:
mpuOut, err := s3CreateMultipartUpload(&S3.CreateMultipartUploadInput{
Bucket: aws.String(bucket),
Key: aws.String(concatenatedFile),
})
if err != nil {
return err
}
var ps []*S3.CompletedPart
for i, part := range parts { // parts is a list of paths to things in s3
partNumber := int64(i) + 1
upOut, err := s3UploadPartCopy(&S3.UploadPartCopyInput{
Bucket: aws.String(bucket),
CopySource: aws.String(part),
Key: aws.String(concatenatedFile),
UploadId: aws.String(*mpuOut.UploadId),
PartNumber: aws.Int64(partNumber),
})
if err != nil {
return err // <- fails here
}
ps = append(ps, &S3.CompletedPart{
ETag: s3Out.CopyPartResult.ETag,
PartNumber: aws.Int64(int64(i)),
})
}
_, err = s3CompleteMultipartUpload(&S3.CompleteMultipartUploadInput{
Bucket: aws.String(bucket),
Key: aws.String(concatenatedFile),
MultipartUpload: &S3.CompletedMultipartUpload{Parts: ps},
UploadId: aws.String(*mpuOut.UploadId),
})
if err != nil {
return err
}
_, err = s3CompleteMultipartUpload(&S3.CompleteMultipartUploadInput{
Bucket: aws.String(bucket),
Key: aws.String(concatenatedFile),
MultipartUpload: &S3.CompletedMultipartUpload{Parts: ps},
UploadId: aws.String(*mpuOut.UploadId),
})
if err != nil {
return err
}
When it runs, it blows up with the error above. The permissions on the bucket are wide open. Any ideas?