14 kms "github.com/aliyun/alibaba-cloud-sdk-go/services/kms"
15 "github.com/aliyun/aliyun-oss-go-sdk/oss"
18 // MasterCipherManager is interface for getting master key with MatDesc(material desc)
19 // If you may use different master keys for encrypting and decrypting objects,each master
20 // key must have a unique, non-emtpy, unalterable MatDesc(json string format) and you must provide this interface
21 // If you always use the same master key for encrypting and decrypting objects, MatDesc
22 // can be empty and you don't need to provide this interface
24 // matDesc map[string]string:is converted by matDesc json string
25 // return: []string the secret key information,such as {"rsa-public-key","rsa-private-key"} or {"non-rsa-key"}
26 type MasterCipherManager interface {
27 GetMasterKey(matDesc map[string]string) ([]string, error)
30 // ExtraCipherBuilder is interface for creating a decrypt ContentCipher with Envelope
31 // If the objects you need to decrypt are neither encrypted with ContentCipherBuilder
32 // you provided, nor encrypted with rsa and ali kms master keys, you must provide this interface
34 // ContentCipher the interface used to decrypt objects
35 type ExtraCipherBuilder interface {
36 GetDecryptCipher(envelope Envelope, cm MasterCipherManager) (ContentCipher, error)
39 // CryptoBucketOption CryptoBucket option such as SetAliKmsClient, SetMasterCipherManager, SetDecryptCipherManager.
40 type CryptoBucketOption func(*CryptoBucket)
42 // SetAliKmsClient set field AliKmsClient of CryptoBucket
43 // If the objects you need to decrypt are encrypted with ali kms master key,but not with ContentCipherBuilder
44 // you provided, you must provide this interface
45 func SetAliKmsClient(client *kms.Client) CryptoBucketOption {
46 return func(bucket *CryptoBucket) {
47 bucket.AliKmsClient = client
51 // SetMasterCipherManager set field MasterCipherManager of CryptoBucket
52 func SetMasterCipherManager(manager MasterCipherManager) CryptoBucketOption {
53 return func(bucket *CryptoBucket) {
54 bucket.MasterCipherManager = manager
58 // SetExtraCipherBuilder set field ExtraCipherBuilder of CryptoBucket
59 func SetExtraCipherBuilder(extraBuilder ExtraCipherBuilder) CryptoBucketOption {
60 return func(bucket *CryptoBucket) {
61 bucket.ExtraCipherBuilder = extraBuilder
65 // DefaultExtraCipherBuilder is Default implementation of the ExtraCipherBuilder for rsa and kms master keys
66 type DefaultExtraCipherBuilder struct {
67 AliKmsClient *kms.Client
70 // GetDecryptCipher is used to get ContentCipher for decrypt object
71 func (decb *DefaultExtraCipherBuilder) GetDecryptCipher(envelope Envelope, cm MasterCipherManager) (ContentCipher, error) {
73 return nil, fmt.Errorf("DefaultExtraCipherBuilder GetDecryptCipher error,MasterCipherManager is nil")
76 if envelope.CEKAlg != AesCtrAlgorithm {
77 return nil, fmt.Errorf("DefaultExtraCipherBuilder GetDecryptCipher error,not supported content algorithm %s", envelope.CEKAlg)
80 if envelope.WrapAlg != RsaCryptoWrap && envelope.WrapAlg != KmsAliCryptoWrap {
81 return nil, fmt.Errorf("DefaultExtraCipherBuilder GetDecryptCipher error,not supported envelope wrap algorithm %s", envelope.WrapAlg)
84 matDesc := make(map[string]string)
85 if envelope.MatDesc != "" {
86 err := json.Unmarshal([]byte(envelope.MatDesc), &matDesc)
92 masterKeys, err := cm.GetMasterKey(matDesc)
97 var contentCipher ContentCipher
98 if envelope.WrapAlg == RsaCryptoWrap {
100 if len(masterKeys) != 2 {
101 return nil, fmt.Errorf("rsa keys count must be 2,now is %d", len(masterKeys))
103 rsaCipher, err := CreateMasterRsa(matDesc, masterKeys[0], masterKeys[1])
107 aesCtrBuilder := CreateAesCtrCipher(rsaCipher)
108 contentCipher, err = aesCtrBuilder.ContentCipherEnv(envelope)
110 } else if envelope.WrapAlg == KmsAliCryptoWrap {
111 // for kms master key
112 if len(masterKeys) != 1 {
113 return nil, fmt.Errorf("non-rsa keys count must be 1,now is %d", len(masterKeys))
116 if decb.AliKmsClient == nil {
117 return nil, fmt.Errorf("aliyun kms client is nil")
120 kmsCipher, err := CreateMasterAliKms(matDesc, masterKeys[0], decb.AliKmsClient)
124 aesCtrBuilder := CreateAesCtrCipher(kmsCipher)
125 contentCipher, err = aesCtrBuilder.ContentCipherEnv(envelope)
128 // for master keys which are neither rsa nor kms
131 return contentCipher, err
134 // CryptoBucket implements the operations for encrypting and decrypting objects
135 // ContentCipherBuilder is used to encrypt and decrypt objects by default
136 // when the object's MatDesc which you want to decrypt is emtpy or same to the
137 // master key's MatDesc you provided in ContentCipherBuilder, sdk try to
138 // use ContentCipherBuilder to decrypt
139 type CryptoBucket struct {
141 ContentCipherBuilder ContentCipherBuilder
142 ExtraCipherBuilder ExtraCipherBuilder
143 MasterCipherManager MasterCipherManager
144 AliKmsClient *kms.Client
147 // GetCryptoBucket create a client encyrption bucket
148 func GetCryptoBucket(client *oss.Client, bucketName string, builder ContentCipherBuilder,
149 options ...CryptoBucketOption) (*CryptoBucket, error) {
150 var cryptoBucket CryptoBucket
151 cryptoBucket.Client = *client
152 cryptoBucket.BucketName = bucketName
153 cryptoBucket.ContentCipherBuilder = builder
155 for _, option := range options {
156 option(&cryptoBucket)
159 if cryptoBucket.ExtraCipherBuilder == nil {
160 cryptoBucket.ExtraCipherBuilder = &DefaultExtraCipherBuilder{AliKmsClient: cryptoBucket.AliKmsClient}
163 return &cryptoBucket, nil
166 // PutObject creates a new object and encyrpt it on client side when uploading to oss
167 func (bucket CryptoBucket) PutObject(objectKey string, reader io.Reader, options ...oss.Option) error {
168 options = bucket.AddEncryptionUaSuffix(options)
169 cc, err := bucket.ContentCipherBuilder.ContentCipher()
174 cryptoReader, err := cc.EncryptContent(reader)
179 var request *oss.PutObjectRequest
180 srcLen, err := oss.GetReaderLen(reader)
182 request = &oss.PutObjectRequest{
183 ObjectKey: objectKey,
184 Reader: cryptoReader,
187 encryptedLen := cc.GetEncryptedLen(srcLen)
188 request = &oss.PutObjectRequest{
189 ObjectKey: objectKey,
190 Reader: oss.LimitReadCloser(cryptoReader, encryptedLen),
194 opts := addCryptoHeaders(options, cc.GetCipherData())
195 resp, err := bucket.DoPutObject(request, opts)
199 defer resp.Body.Close()
204 // GetObject downloads the object from oss
205 // If the object is encrypted, sdk decrypt it automaticly
206 func (bucket CryptoBucket) GetObject(objectKey string, options ...oss.Option) (io.ReadCloser, error) {
207 options = bucket.AddEncryptionUaSuffix(options)
208 result, err := bucket.DoGetObject(&oss.GetObjectRequest{ObjectKey: objectKey}, options)
212 return result.Response, nil
215 // GetObjectToFile downloads the object from oss to local file
216 // If the object is encrypted, sdk decrypt it automaticly
217 func (bucket CryptoBucket) GetObjectToFile(objectKey, filePath string, options ...oss.Option) error {
218 options = bucket.AddEncryptionUaSuffix(options)
219 tempFilePath := filePath + oss.TempFileSuffix
221 // Calls the API to actually download the object. Returns the result instance.
222 result, err := bucket.DoGetObject(&oss.GetObjectRequest{ObjectKey: objectKey}, options)
226 defer result.Response.Close()
228 // If the local file does not exist, create a new one. If it exists, overwrite it.
229 fd, err := os.OpenFile(tempFilePath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, oss.FilePermMode)
234 // Copy the data to the local file path.
235 _, err = io.Copy(fd, result.Response.Body)
241 // Compares the CRC value
242 hasRange, _, _ := oss.IsOptionSet(options, oss.HTTPHeaderRange)
243 encodeOpt, _ := oss.FindOption(options, oss.HTTPHeaderAcceptEncoding, nil)
245 if encodeOpt != nil {
246 acceptEncoding = encodeOpt.(string)
248 if bucket.GetConfig().IsEnableCRC && !hasRange && acceptEncoding != "gzip" {
249 result.Response.ClientCRC = result.ClientCRC.Sum64()
250 err = oss.CheckCRC(result.Response, "GetObjectToFile")
252 os.Remove(tempFilePath)
257 return os.Rename(tempFilePath, filePath)
260 // DoGetObject is the actual API that gets the encrypted or not encrypted object.
261 // It's the internal function called by other public APIs.
262 func (bucket CryptoBucket) DoGetObject(request *oss.GetObjectRequest, options []oss.Option) (*oss.GetObjectResult, error) {
263 options = bucket.AddEncryptionUaSuffix(options)
265 // first,we must head object
266 metaInfo, err := bucket.GetObjectDetailedMeta(request.ObjectKey)
271 isEncryptedObj := isEncryptedObject(metaInfo)
273 return bucket.Bucket.DoGetObject(request, options)
276 envelope, err := getEnvelopeFromHeader(metaInfo)
281 if !isValidContentAlg(envelope.CEKAlg) {
282 return nil, fmt.Errorf("not supported content algorithm %s,object:%s", envelope.CEKAlg, request.ObjectKey)
285 if !envelope.IsValid() {
286 return nil, fmt.Errorf("getEnvelopeFromHeader error,object:%s", request.ObjectKey)
289 // use ContentCipherBuilder to decrpt object by default
290 encryptMatDesc := bucket.ContentCipherBuilder.GetMatDesc()
293 if envelope.MatDesc == encryptMatDesc {
294 cc, err = bucket.ContentCipherBuilder.ContentCipherEnv(envelope)
296 cc, err = bucket.ExtraCipherBuilder.GetDecryptCipher(envelope, bucket.MasterCipherManager)
300 return nil, fmt.Errorf("%s,object:%s", err.Error(), request.ObjectKey)
303 discardFrontAlignLen := int64(0)
304 uRange, err := oss.GetRangeConfig(options)
309 if uRange != nil && uRange.HasStart {
310 // process range to align key size
311 adjustStart := adjustRangeStart(uRange.Start, cc)
312 discardFrontAlignLen = uRange.Start - adjustStart
313 if discardFrontAlignLen > 0 {
314 uRange.Start = adjustStart
315 options = oss.DeleteOption(options, oss.HTTPHeaderRange)
316 options = append(options, oss.NormalizedRange(oss.GetRangeString(*uRange)))
320 cipherData := cc.GetCipherData().Clone()
321 cipherData.SeekIV(uint64(adjustStart))
322 cc, _ = cc.Clone(cipherData)
325 params, _ := oss.GetRawParams(options)
326 resp, err := bucket.Do("GET", request.ObjectKey, params, options, nil, nil)
331 result := &oss.GetObjectResult{
336 var crcCalc hash.Hash64
337 hasRange, _, _ := oss.IsOptionSet(options, oss.HTTPHeaderRange)
338 if bucket.GetConfig().IsEnableCRC && !hasRange {
339 crcCalc = crc64.New(oss.CrcTable())
340 result.ServerCRC = resp.ServerCRC
341 result.ClientCRC = crcCalc
345 listener := oss.GetProgressListener(options)
346 contentLen, _ := strconv.ParseInt(resp.Headers.Get(oss.HTTPHeaderContentLength), 10, 64)
347 resp.Body = oss.TeeReader(resp.Body, crcCalc, contentLen, listener, nil)
348 resp.Body, err = cc.DecryptContent(resp.Body)
349 if err == nil && discardFrontAlignLen > 0 {
350 resp.Body = &oss.DiscardReadCloser{
352 Discard: int(discardFrontAlignLen)}
357 // PutObjectFromFile creates a new object from the local file
358 // the object will be encrypted automaticly on client side when uploaded to oss
359 func (bucket CryptoBucket) PutObjectFromFile(objectKey, filePath string, options ...oss.Option) error {
360 options = bucket.AddEncryptionUaSuffix(options)
361 fd, err := os.Open(filePath)
367 opts := oss.AddContentType(options, filePath, objectKey)
368 cc, err := bucket.ContentCipherBuilder.ContentCipher()
373 cryptoReader, err := cc.EncryptContent(fd)
378 var request *oss.PutObjectRequest
379 srcLen, err := oss.GetReaderLen(fd)
381 request = &oss.PutObjectRequest{
382 ObjectKey: objectKey,
383 Reader: cryptoReader,
386 encryptedLen := cc.GetEncryptedLen(srcLen)
387 request = &oss.PutObjectRequest{
388 ObjectKey: objectKey,
389 Reader: oss.LimitReadCloser(cryptoReader, encryptedLen),
393 opts = addCryptoHeaders(opts, cc.GetCipherData())
394 resp, err := bucket.DoPutObject(request, opts)
398 defer resp.Body.Close()
402 // AppendObject please refer to Bucket.AppendObject
403 func (bucket CryptoBucket) AppendObject(objectKey string, reader io.Reader, appendPosition int64, options ...oss.Option) (int64, error) {
404 return 0, fmt.Errorf("CryptoBucket doesn't support AppendObject")
407 // DoAppendObject please refer to Bucket.DoAppendObject
408 func (bucket CryptoBucket) DoAppendObject(request *oss.AppendObjectRequest, options []oss.Option) (*oss.AppendObjectResult, error) {
409 return nil, fmt.Errorf("CryptoBucket doesn't support DoAppendObject")
412 // PutObjectWithURL please refer to Bucket.PutObjectWithURL
413 func (bucket CryptoBucket) PutObjectWithURL(signedURL string, reader io.Reader, options ...oss.Option) error {
414 return fmt.Errorf("CryptoBucket doesn't support PutObjectWithURL")
417 // PutObjectFromFileWithURL please refer to Bucket.PutObjectFromFileWithURL
418 func (bucket CryptoBucket) PutObjectFromFileWithURL(signedURL, filePath string, options ...oss.Option) error {
419 return fmt.Errorf("CryptoBucket doesn't support PutObjectFromFileWithURL")
422 // DoPutObjectWithURL please refer to Bucket.DoPutObjectWithURL
423 func (bucket CryptoBucket) DoPutObjectWithURL(signedURL string, reader io.Reader, options []oss.Option) (*oss.Response, error) {
424 return nil, fmt.Errorf("CryptoBucket doesn't support DoPutObjectWithURL")
427 // GetObjectWithURL please refer to Bucket.GetObjectWithURL
428 func (bucket CryptoBucket) GetObjectWithURL(signedURL string, options ...oss.Option) (io.ReadCloser, error) {
429 return nil, fmt.Errorf("CryptoBucket doesn't support GetObjectWithURL")
432 // GetObjectToFileWithURL please refer to Bucket.GetObjectToFileWithURL
433 func (bucket CryptoBucket) GetObjectToFileWithURL(signedURL, filePath string, options ...oss.Option) error {
434 return fmt.Errorf("CryptoBucket doesn't support GetObjectToFileWithURL")
437 // DoGetObjectWithURL please refer to Bucket.DoGetObjectWithURL
438 func (bucket CryptoBucket) DoGetObjectWithURL(signedURL string, options []oss.Option) (*oss.GetObjectResult, error) {
439 return nil, fmt.Errorf("CryptoBucket doesn't support DoGetObjectWithURL")
442 // ProcessObject please refer to Bucket.ProcessObject
443 func (bucket CryptoBucket) ProcessObject(objectKey string, process string, options ...oss.Option) (oss.ProcessObjectResult, error) {
444 var out oss.ProcessObjectResult
445 return out, fmt.Errorf("CryptoBucket doesn't support ProcessObject")
448 func (bucket CryptoBucket) AddEncryptionUaSuffix(options []oss.Option) []oss.Option {
449 var outOption []oss.Option
450 bSet, _, _ := oss.IsOptionSet(options, oss.HTTPHeaderUserAgent)
451 if bSet || bucket.Client.Config.UserSetUa {
455 outOption = append(options, oss.UserAgentHeader(bucket.Client.Config.UserAgent+"/"+EncryptionUaSuffix))
459 // isEncryptedObject judge the object is encrypted or not
460 func isEncryptedObject(headers http.Header) bool {
461 encrptedKey := headers.Get(oss.HTTPHeaderOssMetaPrefix + OssClientSideEncryptionKey)
462 return len(encrptedKey) > 0
465 // addCryptoHeaders save Envelope information in oss meta
466 func addCryptoHeaders(options []oss.Option, cd *CipherData) []oss.Option {
467 opts := []oss.Option{}
469 // convert content-md5
470 md5Option, _ := oss.FindOption(options, oss.HTTPHeaderContentMD5, nil)
471 if md5Option != nil {
472 opts = append(opts, oss.Meta(OssClientSideEncryptionUnencryptedContentMD5, md5Option.(string)))
473 options = oss.DeleteOption(options, oss.HTTPHeaderContentMD5)
476 // convert content-length
477 lenOption, _ := oss.FindOption(options, oss.HTTPHeaderContentLength, nil)
478 if lenOption != nil {
479 opts = append(opts, oss.Meta(OssClientSideEncryptionUnencryptedContentLength, lenOption.(string)))
480 options = oss.DeleteOption(options, oss.HTTPHeaderContentLength)
483 opts = append(opts, options...)
486 if cd.MatDesc != "" {
487 opts = append(opts, oss.Meta(OssClientSideEncryptionMatDesc, cd.MatDesc))
491 strEncryptedKey := base64.StdEncoding.EncodeToString(cd.EncryptedKey)
492 opts = append(opts, oss.Meta(OssClientSideEncryptionKey, strEncryptedKey))
495 strEncryptedIV := base64.StdEncoding.EncodeToString(cd.EncryptedIV)
496 opts = append(opts, oss.Meta(OssClientSideEncryptionStart, strEncryptedIV))
499 opts = append(opts, oss.Meta(OssClientSideEncryptionWrapAlg, cd.WrapAlgorithm))
502 opts = append(opts, oss.Meta(OssClientSideEncryptionCekAlg, cd.CEKAlgorithm))
507 func getEnvelopeFromHeader(header http.Header) (Envelope, error) {
508 var envelope Envelope
509 envelope.IV = header.Get(oss.HTTPHeaderOssMetaPrefix + OssClientSideEncryptionStart)
510 decodedIV, err := base64.StdEncoding.DecodeString(envelope.IV)
514 envelope.IV = string(decodedIV)
516 envelope.CipherKey = header.Get(oss.HTTPHeaderOssMetaPrefix + OssClientSideEncryptionKey)
517 decodedKey, err := base64.StdEncoding.DecodeString(envelope.CipherKey)
521 envelope.CipherKey = string(decodedKey)
523 envelope.MatDesc = header.Get(oss.HTTPHeaderOssMetaPrefix + OssClientSideEncryptionMatDesc)
524 envelope.WrapAlg = header.Get(oss.HTTPHeaderOssMetaPrefix + OssClientSideEncryptionWrapAlg)
525 envelope.CEKAlg = header.Get(oss.HTTPHeaderOssMetaPrefix + OssClientSideEncryptionCekAlg)
526 envelope.UnencryptedMD5 = header.Get(oss.HTTPHeaderOssMetaPrefix + OssClientSideEncryptionUnencryptedContentMD5)
527 envelope.UnencryptedContentLen = header.Get(oss.HTTPHeaderOssMetaPrefix + OssClientSideEncryptionUnencryptedContentLength)
531 func isValidContentAlg(algName string) bool {
532 // now content encyrption only support aec/ctr algorithm
533 return algName == AesCtrAlgorithm
536 func adjustRangeStart(start int64, cc ContentCipher) int64 {
537 alignLen := int64(cc.GetAlignLen())
538 return (start / alignLen) * alignLen