vendor/github.com/aliyun/aliyun-oss-go-sdk/oss/download_test.go

   1 package oss
   2
   3 import (
   4         "bytes"
   5         "fmt"
   6         "net/http"
   7         "os"
   8         "strings"
   9         "time"
  10
  11         . "gopkg.in/check.v1"
  12 )
  13
  14 type OssDownloadSuite struct {
  15         client *Client
  16         bucket *Bucket
  17 }
  18
  19 var _ = Suite(&OssDownloadSuite{})
  20
  21 // SetUpSuite runs once when the suite starts running
  22 func (s *OssDownloadSuite) SetUpSuite(c *C) {
  23         client, err := New(endpoint, accessID, accessKey)
  24         c.Assert(err, IsNil)
  25         s.client = client
  26
  27         s.client.CreateBucket(bucketName)
  28
  29         bucket, err := s.client.Bucket(bucketName)
  30         c.Assert(err, IsNil)
  31         s.bucket = bucket
  32
  33         testLogger.Println("test download started")
  34 }
  35
  36 // TearDownSuite runs before each test or benchmark starts running
  37 func (s *OssDownloadSuite) TearDownSuite(c *C) {
  38         // Delete part
  39         keyMarker := KeyMarker("")
  40         uploadIDMarker := UploadIDMarker("")
  41         for {
  42                 lmur, err := s.bucket.ListMultipartUploads(keyMarker, uploadIDMarker)
  43                 c.Assert(err, IsNil)
  44                 for _, upload := range lmur.Uploads {
  45                         var imur = InitiateMultipartUploadResult{Bucket: s.bucket.BucketName,
  46                                 Key: upload.Key, UploadID: upload.UploadID}
  47                         err = s.bucket.AbortMultipartUpload(imur)
  48                         c.Assert(err, IsNil)
  49                 }
  50                 keyMarker = KeyMarker(lmur.NextKeyMarker)
  51                 uploadIDMarker = UploadIDMarker(lmur.NextUploadIDMarker)
  52                 if !lmur.IsTruncated {
  53                         break
  54                 }
  55         }
  56
  57         // Delete objects
  58         marker := Marker("")
  59         for {
  60                 lor, err := s.bucket.ListObjects(marker)
  61                 c.Assert(err, IsNil)
  62                 for _, object := range lor.Objects {
  63                         err = s.bucket.DeleteObject(object.Key)
  64                         c.Assert(err, IsNil)
  65                 }
  66                 marker = Marker(lor.NextMarker)
  67                 if !lor.IsTruncated {
  68                         break
  69                 }
  70         }
  71
  72         // Delete bucket
  73         err := s.client.DeleteBucket(s.bucket.BucketName)
  74         c.Assert(err, IsNil)
  75
  76         testLogger.Println("test download completed")
  77 }
  78
  79 // SetUpTest runs after each test or benchmark runs
  80 func (s *OssDownloadSuite) SetUpTest(c *C) {
  81         err := removeTempFiles("../oss", ".jpg")
  82         c.Assert(err, IsNil)
  83 }
  84
  85 // TearDownTest runs once after all tests or benchmarks have finished running
  86 func (s *OssDownloadSuite) TearDownTest(c *C) {
  87         err := removeTempFiles("../oss", ".jpg")
  88         c.Assert(err, IsNil)
  89
  90         err = removeTempFiles("../oss", ".temp")
  91         c.Assert(err, IsNil)
  92 }
  93
  94 // TestDownloadRoutineWithoutRecovery multipart downloads without checkpoint
  95 func (s *OssDownloadSuite) TestDownloadRoutineWithoutRecovery(c *C) {
  96         objectName := objectNamePrefix + RandStr(8)
  97         fileName := "../sample/BingWallpaper-2015-11-07.jpg"
  98         newFile := RandStr(8) + ".jpg"
  99
 100         // Upload a file
 101         err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3))
 102         c.Assert(err, IsNil)
 103
 104         // Download the file by default
 105         err = s.bucket.DownloadFile(objectName, newFile, 100*1024)
 106         c.Assert(err, IsNil)
 107
 108         // Check
 109         eq, err := compareFiles(fileName, newFile)
 110         c.Assert(err, IsNil)
 111         c.Assert(eq, Equals, true)
 112
 113         // Use 2 coroutines to download the file and total parts count is 5
 114         os.Remove(newFile)
 115         err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(2))
 116         c.Assert(err, IsNil)
 117
 118         // Check
 119         eq, err = compareFiles(fileName, newFile)
 120         c.Assert(err, IsNil)
 121         c.Assert(eq, Equals, true)
 122
 123         // Use 5 coroutines to download the file and the total parts count is 5.
 124         os.Remove(newFile)
 125         err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(5))
 126         c.Assert(err, IsNil)
 127
 128         // Check
 129         eq, err = compareFiles(fileName, newFile)
 130         c.Assert(err, IsNil)
 131         c.Assert(eq, Equals, true)
 132
 133         // Use 10 coroutines to download the file and the total parts count is 5.
 134         os.Remove(newFile)
 135         err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(10))
 136         c.Assert(err, IsNil)
 137
 138         // Check
 139         eq, err = compareFiles(fileName, newFile)
 140         c.Assert(err, IsNil)
 141         c.Assert(eq, Equals, true)
 142
 143         err = s.bucket.DeleteObject(objectName)
 144         c.Assert(err, IsNil)
 145 }
 146
 147 // DownErrorHooker requests hook by downloadPart
 148 func DownErrorHooker(part downloadPart) error {
 149         if part.Index == 4 {
 150                 time.Sleep(time.Second)
 151                 return fmt.Errorf("ErrorHooker")
 152         }
 153         return nil
 154 }
 155
 156 // TestDownloadRoutineWithRecovery multi-routine resumable download
 157 func (s *OssDownloadSuite) TestDownloadRoutineWithRecovery(c *C) {
 158         objectName := objectNamePrefix + RandStr(8)
 159         fileName := "../sample/BingWallpaper-2015-11-07.jpg"
 160         newFile := RandStr(8) + ".jpg"
 161
 162         // Upload a file
 163         err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3))
 164         c.Assert(err, IsNil)
 165
 166         // Download a file with default checkpoint
 167         downloadPartHooker = DownErrorHooker
 168         err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Checkpoint(true, newFile+".cp"))
 169         c.Assert(err, NotNil)
 170         c.Assert(err.Error(), Equals, "ErrorHooker")
 171         downloadPartHooker = defaultDownloadPartHook
 172
 173         // Check
 174         dcp := downloadCheckpoint{}
 175         err = dcp.load(newFile + ".cp")
 176         c.Assert(err, IsNil)
 177         c.Assert(dcp.Magic, Equals, downloadCpMagic)
 178         c.Assert(len(dcp.MD5), Equals, len("LC34jZU5xK4hlxi3Qn3XGQ=="))
 179         c.Assert(dcp.FilePath, Equals, newFile)
 180         c.Assert(dcp.ObjStat.Size, Equals, int64(482048))
 181         c.Assert(len(dcp.ObjStat.LastModified) > 0, Equals, true)
 182         c.Assert(dcp.ObjStat.Etag, Equals, "\"2351E662233817A7AE974D8C5B0876DD-5\"")
 183         c.Assert(dcp.Object, Equals, objectName)
 184         c.Assert(len(dcp.Parts), Equals, 5)
 185         c.Assert(len(dcp.todoParts()), Equals, 1)
 186
 187         err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Checkpoint(true, newFile+".cp"))
 188         c.Assert(err, IsNil)
 189         //download success, checkpoint file has been deleted
 190         err = dcp.load(newFile + ".cp")
 191         c.Assert(err, NotNil)
 192
 193         eq, err := compareFiles(fileName, newFile)
 194         c.Assert(err, IsNil)
 195         c.Assert(eq, Equals, true)
 196
 197         // Resumable download with empty checkpoint file path
 198         downloadPartHooker = DownErrorHooker
 199         err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Checkpoint(true, ""))
 200         c.Assert(err, NotNil)
 201         c.Assert(err.Error(), Equals, "ErrorHooker")
 202         downloadPartHooker = defaultDownloadPartHook
 203
 204         dcp = downloadCheckpoint{}
 205         err = dcp.load(newFile + ".cp")
 206         c.Assert(err, NotNil)
 207
 208         // Resumable download with checkpoint dir
 209         os.Remove(newFile)
 210         downloadPartHooker = DownErrorHooker
 211         err = s.bucket.DownloadFile(objectName, newFile, 100*1024, CheckpointDir(true, "./"))
 212         c.Assert(err, NotNil)
 213         c.Assert(err.Error(), Equals, "ErrorHooker")
 214         downloadPartHooker = defaultDownloadPartHook
 215
 216         // Check
 217         dcp = downloadCheckpoint{}
 218         cpConf := cpConfig{IsEnable: true, DirPath: "./"}
 219         cpFilePath := getDownloadCpFilePath(&cpConf, s.bucket.BucketName, objectName, "",newFile)
 220         err = dcp.load(cpFilePath)
 221         c.Assert(err, IsNil)
 222         c.Assert(dcp.Magic, Equals, downloadCpMagic)
 223         c.Assert(len(dcp.MD5), Equals, len("LC34jZU5xK4hlxi3Qn3XGQ=="))
 224         c.Assert(dcp.FilePath, Equals, newFile)
 225         c.Assert(dcp.ObjStat.Size, Equals, int64(482048))
 226         c.Assert(len(dcp.ObjStat.LastModified) > 0, Equals, true)
 227         c.Assert(dcp.ObjStat.Etag, Equals, "\"2351E662233817A7AE974D8C5B0876DD-5\"")
 228         c.Assert(dcp.Object, Equals, objectName)
 229         c.Assert(len(dcp.Parts), Equals, 5)
 230         c.Assert(len(dcp.todoParts()), Equals, 1)
 231
 232         err = s.bucket.DownloadFile(objectName, newFile, 100*1024, CheckpointDir(true, "./"))
 233         c.Assert(err, IsNil)
 234         //download success, checkpoint file has been deleted
 235         err = dcp.load(cpFilePath)
 236         c.Assert(err, NotNil)
 237
 238         eq, err = compareFiles(fileName, newFile)
 239         c.Assert(err, IsNil)
 240         c.Assert(eq, Equals, true)
 241
 242         // Resumable download with checkpoint at a time. No error is expected in the download procedure.
 243         os.Remove(newFile)
 244         err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Checkpoint(true, newFile+".cp"))
 245         c.Assert(err, IsNil)
 246
 247         err = dcp.load(newFile + ".cp")
 248         c.Assert(err, NotNil)
 249
 250         eq, err = compareFiles(fileName, newFile)
 251         c.Assert(err, IsNil)
 252         c.Assert(eq, Equals, true)
 253
 254         // Resumable download with checkpoint at a time. No error is expected in the download procedure.
 255         os.Remove(newFile)
 256         err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(10), Checkpoint(true, newFile+".cp"))
 257         c.Assert(err, IsNil)
 258
 259         err = dcp.load(newFile + ".cp")
 260         c.Assert(err, NotNil)
 261
 262         eq, err = compareFiles(fileName, newFile)
 263         c.Assert(err, IsNil)
 264         c.Assert(eq, Equals, true)
 265
 266         err = s.bucket.DeleteObject(objectName)
 267         c.Assert(err, IsNil)
 268 }
 269
 270 // TestDownloadOption options
 271 func (s *OssDownloadSuite) TestDownloadOption(c *C) {
 272         objectName := objectNamePrefix + RandStr(8)
 273         fileName := "../sample/BingWallpaper-2015-11-07.jpg"
 274         newFile := RandStr(8) + ".jpg"
 275
 276         // Upload the file
 277         err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3))
 278         c.Assert(err, IsNil)
 279
 280         meta, err := s.bucket.GetObjectDetailedMeta(objectName)
 281         c.Assert(err, IsNil)
 282
 283         // IfMatch
 284         os.Remove(newFile)
 285         err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(3), IfMatch(meta.Get("Etag")))
 286         c.Assert(err, IsNil)
 287
 288         eq, err := compareFiles(fileName, newFile)
 289         c.Assert(err, IsNil)
 290         c.Assert(eq, Equals, true)
 291
 292         // IfNoneMatch
 293         os.Remove(newFile)
 294         err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(3), IfNoneMatch(meta.Get("Etag")))
 295         c.Assert(err, NotNil)
 296
 297         // IfMatch
 298         err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(3), IfMatch(meta.Get("Etag")))
 299         c.Assert(err, IsNil)
 300
 301         eq, err = compareFiles(fileName, newFile)
 302         c.Assert(err, IsNil)
 303         c.Assert(eq, Equals, true)
 304
 305         // IfNoneMatch
 306         err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(3), IfNoneMatch(meta.Get("Etag")))
 307         c.Assert(err, NotNil)
 308 }
 309
 310 // TestDownloadObjectChange tests the file is updated during the upload
 311 func (s *OssDownloadSuite) TestDownloadObjectChange(c *C) {
 312         objectName := objectNamePrefix + RandStr(8)
 313         fileName := "../sample/BingWallpaper-2015-11-07.jpg"
 314         newFile := RandStr(8) + ".jpg"
 315
 316         // Upload a file
 317         err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3))
 318         c.Assert(err, IsNil)
 319
 320         // Download with default checkpoint
 321         downloadPartHooker = DownErrorHooker
 322         err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Checkpoint(true, newFile+".cp"))
 323         c.Assert(err, NotNil)
 324         c.Assert(err.Error(), Equals, "ErrorHooker")
 325         downloadPartHooker = defaultDownloadPartHook
 326
 327         err = s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3))
 328         c.Assert(err, IsNil)
 329
 330         err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Checkpoint(true, newFile+".cp"))
 331         c.Assert(err, IsNil)
 332
 333         eq, err := compareFiles(fileName, newFile)
 334         c.Assert(err, IsNil)
 335         c.Assert(eq, Equals, true)
 336 }
 337
 338 // TestDownloadNegative tests downloading negative
 339 func (s *OssDownloadSuite) TestDownloadNegative(c *C) {
 340         objectName := objectNamePrefix + RandStr(8)
 341         fileName := "../sample/BingWallpaper-2015-11-07.jpg"
 342         newFile := RandStr(8) + ".jpg"
 343
 344         // Upload a file
 345         err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3))
 346         c.Assert(err, IsNil)
 347
 348         // Worker routine error
 349         downloadPartHooker = DownErrorHooker
 350         err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(2))
 351         c.Assert(err, NotNil)
 352         c.Assert(err.Error(), Equals, "ErrorHooker")
 353         downloadPartHooker = defaultDownloadPartHook
 354
 355         // Local file does not exist
 356         err = s.bucket.DownloadFile(objectName, "/tmp/", 100*1024, Routines(2))
 357         c.Assert(err, NotNil)
 358
 359         // Invalid part size
 360         err = s.bucket.DownloadFile(objectName, newFile, 0, Routines(2))
 361         c.Assert(err, NotNil)
 362
 363         err = s.bucket.DownloadFile(objectName, newFile, 1024*1024*1024*100, Routines(2))
 364         c.Assert(err, IsNil)
 365
 366         err = s.bucket.DeleteObject(objectName)
 367         c.Assert(err, IsNil)
 368
 369         // Local file does not exist
 370         err = s.bucket.DownloadFile(objectName, "/tmp/", 100*1024)
 371         c.Assert(err, NotNil)
 372
 373         err = s.bucket.DownloadFile(objectName, "/tmp/", 100*1024, Routines(2))
 374         c.Assert(err, NotNil)
 375
 376         // Invalid part size
 377         err = s.bucket.DownloadFile(objectName, newFile, -1)
 378         c.Assert(err, NotNil)
 379
 380         err = s.bucket.DownloadFile(objectName, newFile, 0, Routines(2))
 381         c.Assert(err, NotNil)
 382
 383         err = s.bucket.DownloadFile(objectName, newFile, 1024*1024*1024*100)
 384         c.Assert(err, NotNil)
 385
 386         err = s.bucket.DownloadFile(objectName, newFile, 1024*1024*1024*100, Routines(2))
 387         c.Assert(err, NotNil)
 388 }
 389
 390 // TestDownloadWithRange tests concurrent downloading with range specified and checkpoint enabled
 391 func (s *OssDownloadSuite) TestDownloadWithRange(c *C) {
 392         objectName := objectNamePrefix + RandStr(8)
 393         fileName := "../sample/BingWallpaper-2015-11-07.jpg"
 394         newFile := RandStr(8) + ".jpg"
 395         newFileGet := RandStr(8) + "-.jpg"
 396
 397         // Upload a file
 398         err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3))
 399         c.Assert(err, IsNil)
 400
 401         fileSize, err := getFileSize(fileName)
 402         c.Assert(err, IsNil)
 403
 404         // Download with range, from 1024 to 4096
 405         os.Remove(newFile)
 406         err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(3), Range(1024, 4095))
 407         c.Assert(err, IsNil)
 408
 409         // Check
 410         eq, err := compareFilesWithRange(fileName, 1024, newFile, 0, 3072)
 411         c.Assert(err, IsNil)
 412         c.Assert(eq, Equals, true)
 413
 414         os.Remove(newFileGet)
 415         err = s.bucket.GetObjectToFile(objectName, newFileGet, Range(1024, 4095))
 416         c.Assert(err, IsNil)
 417
 418         // Compare get and download
 419         eq, err = compareFiles(newFile, newFileGet)
 420         c.Assert(err, IsNil)
 421         c.Assert(eq, Equals, true)
 422
 423         // Download with range, from 1024 to 4096
 424         os.Remove(newFile)
 425         err = s.bucket.DownloadFile(objectName, newFile, 1024, Routines(3), NormalizedRange("1024-4095"))
 426         c.Assert(err, IsNil)
 427
 428         // Check
 429         eq, err = compareFilesWithRange(fileName, 1024, newFile, 0, 3072)
 430         c.Assert(err, IsNil)
 431         c.Assert(eq, Equals, true)
 432
 433         os.Remove(newFileGet)
 434         err = s.bucket.GetObjectToFile(objectName, newFileGet, NormalizedRange("1024-4095"))
 435         c.Assert(err, IsNil)
 436
 437         // Compare get and download
 438         eq, err = compareFiles(newFile, newFileGet)
 439         c.Assert(err, IsNil)
 440         c.Assert(eq, Equals, true)
 441
 442         // Download with range, from 2048 to the end
 443         os.Remove(newFile)
 444         err = s.bucket.DownloadFile(objectName, newFile, 1024*1024, Routines(3), NormalizedRange("2048-"))
 445         c.Assert(err, IsNil)
 446
 447         // Check
 448         eq, err = compareFilesWithRange(fileName, 2048, newFile, 0, fileSize-2048)
 449         c.Assert(err, IsNil)
 450         c.Assert(eq, Equals, true)
 451
 452         os.Remove(newFileGet)
 453         err = s.bucket.GetObjectToFile(objectName, newFileGet, NormalizedRange("2048-"))
 454         c.Assert(err, IsNil)
 455
 456         // Compare get and download
 457         eq, err = compareFiles(newFile, newFileGet)
 458         c.Assert(err, IsNil)
 459         c.Assert(eq, Equals, true)
 460
 461         // Download with range, the last 4096
 462         os.Remove(newFile)
 463         err = s.bucket.DownloadFile(objectName, newFile, 1024, Routines(3), NormalizedRange("-4096"))
 464         c.Assert(err, IsNil)
 465
 466         // Check
 467         eq, err = compareFilesWithRange(fileName, fileSize-4096, newFile, 0, 4096)
 468         c.Assert(err, IsNil)
 469         c.Assert(eq, Equals, true)
 470
 471         os.Remove(newFileGet)
 472         err = s.bucket.GetObjectToFile(objectName, newFileGet, NormalizedRange("-4096"))
 473         c.Assert(err, IsNil)
 474
 475         // Compare get and download
 476         eq, err = compareFiles(newFile, newFileGet)
 477         c.Assert(err, IsNil)
 478         c.Assert(eq, Equals, true)
 479
 480         err = s.bucket.DeleteObject(objectName)
 481         c.Assert(err, IsNil)
 482 }
 483
 484 // TestDownloadWithCheckoutAndRange tests concurrent downloading with range specified and checkpoint enabled
 485 func (s *OssDownloadSuite) TestDownloadWithCheckoutAndRange(c *C) {
 486         objectName := objectNamePrefix + RandStr(8)
 487         fileName := "../sample/BingWallpaper-2015-11-07.jpg"
 488         newFile := RandStr(8) + ".jpg"
 489         newFileGet := RandStr(8) + "-get.jpg"
 490
 491         // Upload a file
 492         err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3), Checkpoint(true, fileName+".cp"))
 493         c.Assert(err, IsNil)
 494
 495         fileSize, err := getFileSize(fileName)
 496         c.Assert(err, IsNil)
 497
 498         // Download with range, from 1024 to 4096
 499         os.Remove(newFile)
 500         err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(3), Checkpoint(true, newFile+".cp"), Range(1024, 4095))
 501         c.Assert(err, IsNil)
 502
 503         // Check
 504         eq, err := compareFilesWithRange(fileName, 1024, newFile, 0, 3072)
 505         c.Assert(err, IsNil)
 506         c.Assert(eq, Equals, true)
 507
 508         os.Remove(newFileGet)
 509         err = s.bucket.GetObjectToFile(objectName, newFileGet, Range(1024, 4095))
 510         c.Assert(err, IsNil)
 511
 512         // Compare get and download
 513         eq, err = compareFiles(newFile, newFileGet)
 514         c.Assert(err, IsNil)
 515         c.Assert(eq, Equals, true)
 516
 517         // Download with range, from 1024 to 4096
 518         os.Remove(newFile)
 519         err = s.bucket.DownloadFile(objectName, newFile, 1024, Routines(3), Checkpoint(true, newFile+".cp"), NormalizedRange("1024-4095"))
 520         c.Assert(err, IsNil)
 521
 522         // Check
 523         eq, err = compareFilesWithRange(fileName, 1024, newFile, 0, 3072)
 524         c.Assert(err, IsNil)
 525         c.Assert(eq, Equals, true)
 526
 527         os.Remove(newFileGet)
 528         err = s.bucket.GetObjectToFile(objectName, newFileGet, NormalizedRange("1024-4095"))
 529         c.Assert(err, IsNil)
 530
 531         // Compare get and download
 532         eq, err = compareFiles(newFile, newFileGet)
 533         c.Assert(err, IsNil)
 534         c.Assert(eq, Equals, true)
 535
 536         // Download with range, from 2048 to the end
 537         os.Remove(newFile)
 538         err = s.bucket.DownloadFile(objectName, newFile, 1024*1024, Routines(3), Checkpoint(true, newFile+".cp"), NormalizedRange("2048-"))
 539         c.Assert(err, IsNil)
 540
 541         // Check
 542         eq, err = compareFilesWithRange(fileName, 2048, newFile, 0, fileSize-2048)
 543         c.Assert(err, IsNil)
 544         c.Assert(eq, Equals, true)
 545
 546         os.Remove(newFileGet)
 547         err = s.bucket.GetObjectToFile(objectName, newFileGet, NormalizedRange("2048-"))
 548         c.Assert(err, IsNil)
 549
 550         // Compare get and download
 551         eq, err = compareFiles(newFile, newFileGet)
 552         c.Assert(err, IsNil)
 553         c.Assert(eq, Equals, true)
 554
 555         // Download with range, the last 4096 bytes
 556         os.Remove(newFile)
 557         err = s.bucket.DownloadFile(objectName, newFile, 1024, Routines(3), Checkpoint(true, newFile+".cp"), NormalizedRange("-4096"))
 558         c.Assert(err, IsNil)
 559
 560         // Check
 561         eq, err = compareFilesWithRange(fileName, fileSize-4096, newFile, 0, 4096)
 562         c.Assert(err, IsNil)
 563         c.Assert(eq, Equals, true)
 564
 565         os.Remove(newFileGet)
 566         err = s.bucket.GetObjectToFile(objectName, newFileGet, NormalizedRange("-4096"))
 567         c.Assert(err, IsNil)
 568
 569         // Compare get and download
 570         eq, err = compareFiles(newFile, newFileGet)
 571         c.Assert(err, IsNil)
 572         c.Assert(eq, Equals, true)
 573
 574         err = s.bucket.DeleteObject(objectName)
 575         c.Assert(err, IsNil)
 576 }
 577
 578 // TestCombineCRCInDownloadParts tests combineCRCInParts
 579 func (s *OssDownloadSuite) TestCombineCRCInDownloadParts(c *C) {
 580         crc := combineCRCInParts(nil)
 581         c.Assert(crc == 0, Equals, true)
 582
 583         crc = combineCRCInParts(make([]downloadPart, 0))
 584         c.Assert(crc == 0, Equals, true)
 585
 586         parts := make([]downloadPart, 1)
 587         parts[0].CRC64 = 10278880121275185425
 588         crc = combineCRCInParts(parts)
 589         c.Assert(crc == 10278880121275185425, Equals, true)
 590
 591         parts = make([]downloadPart, 2)
 592         parts[0].CRC64 = 6748440630437108969
 593         parts[0].Start = 0
 594         parts[0].End = 4
 595         parts[1].CRC64 = 10278880121275185425
 596         parts[1].Start = 5
 597         parts[1].End = 8
 598         crc = combineCRCInParts(parts)
 599         c.Assert(crc == 11051210869376104954, Equals, true)
 600 }
 601
 602 func getFileSize(fileName string) (int64, error) {
 603         file, err := os.Open(fileName)
 604         if err != nil {
 605                 return 0, err
 606         }
 607         defer file.Close()
 608
 609         stat, err := file.Stat()
 610         if err != nil {
 611                 return 0, err
 612         }
 613
 614         return stat.Size(), nil
 615 }
 616
 617 // compareFilesWithRange compares the content between fileL and fileR with specified range
 618 func compareFilesWithRange(fileL string, offsetL int64, fileR string, offsetR int64, size int64) (bool, error) {
 619         finL, err := os.Open(fileL)
 620         if err != nil {
 621                 return false, err
 622         }
 623         defer finL.Close()
 624         finL.Seek(offsetL, os.SEEK_SET)
 625
 626         finR, err := os.Open(fileR)
 627         if err != nil {
 628                 return false, err
 629         }
 630         defer finR.Close()
 631         finR.Seek(offsetR, os.SEEK_SET)
 632
 633         statL, err := finL.Stat()
 634         if err != nil {
 635                 return false, err
 636         }
 637
 638         statR, err := finR.Stat()
 639         if err != nil {
 640                 return false, err
 641         }
 642
 643         if (offsetL+size > statL.Size()) || (offsetR+size > statR.Size()) {
 644                 return false, nil
 645         }
 646
 647         part := statL.Size() - offsetL
 648         if part > 16*1024 {
 649                 part = 16 * 1024
 650         }
 651
 652         bufL := make([]byte, part)
 653         bufR := make([]byte, part)
 654         for readN := int64(0); readN < size; {
 655                 n, _ := finL.Read(bufL)
 656                 if 0 == n {
 657                         break
 658                 }
 659
 660                 n, _ = finR.Read(bufR)
 661                 if 0 == n {
 662                         break
 663                 }
 664
 665                 tailer := part
 666                 if tailer > size-readN {
 667                         tailer = size - readN
 668                 }
 669                 readN += tailer
 670
 671                 if !bytes.Equal(bufL[0:tailer], bufR[0:tailer]) {
 672                         return false, nil
 673                 }
 674         }
 675
 676         return true, nil
 677 }
 678
 679 func (s *OssDownloadSuite) TestVersioningDownloadWithoutCheckPoint(c *C) {
 680         // create a bucket with default proprety
 681         client, err := New(endpoint, accessID, accessKey)
 682         c.Assert(err, IsNil)
 683
 684         bucketName := bucketNamePrefix + RandLowStr(6)
 685         err = client.CreateBucket(bucketName)
 686         c.Assert(err, IsNil)
 687
 688         bucket, err := client.Bucket(bucketName)
 689
 690         // put bucket version:enabled
 691         var versioningConfig VersioningConfig
 692         versioningConfig.Status = string(VersionEnabled)
 693         err = client.SetBucketVersioning(bucketName, versioningConfig)
 694         c.Assert(err, IsNil)
 695
 696         // begin test
 697         objectName := objectNamePrefix + RandStr(8)
 698         fileName := "test-file-" + RandStr(8)
 699         fileData := RandStr(500 * 1024)
 700         CreateFile(fileName, fileData, c)
 701
 702         newFile := RandStr(8) + ".jpg"
 703         newFileGet := RandStr(8) + "-.jpg"
 704
 705         // Upload a file
 706         var respHeader http.Header
 707         options := []Option{Routines(3), GetResponseHeader(&respHeader)}
 708         err = bucket.UploadFile(objectName, fileName, 100*1024, options...)
 709         c.Assert(err, IsNil)
 710         versionId := GetVersionId(respHeader)
 711         c.Assert(len(versionId) > 0, Equals, true)
 712
 713         fileSize, err := getFileSize(fileName)
 714         c.Assert(err, IsNil)
 715
 716         // overwrite emtpy object
 717         err = bucket.PutObject(objectName, strings.NewReader(""))
 718         c.Assert(err, IsNil)
 719
 720         // Download with range, from 1024 to 4096
 721         os.Remove(newFile)
 722         options = []Option{Routines(3), Range(1024, 4095), VersionId(versionId)}
 723         err = bucket.DownloadFile(objectName, newFile, 100*1024, options...)
 724         c.Assert(err, IsNil)
 725
 726         // Check
 727         eq, err := compareFilesWithRange(fileName, 1024, newFile, 0, 3072)
 728         c.Assert(err, IsNil)
 729         c.Assert(eq, Equals, true)
 730
 731         os.Remove(newFileGet)
 732         options = []Option{Range(1024, 4095), VersionId(versionId)}
 733         err = bucket.GetObjectToFile(objectName, newFileGet, options...)
 734         c.Assert(err, IsNil)
 735
 736         // Compare get and download
 737         eq, err = compareFiles(newFile, newFileGet)
 738         c.Assert(err, IsNil)
 739         c.Assert(eq, Equals, true)
 740
 741         // Download with range, from 1024 to 4096
 742         os.Remove(newFile)
 743         options = []Option{Routines(3), NormalizedRange("1024-4095"), VersionId(versionId)}
 744         err = bucket.DownloadFile(objectName, newFile, 1024, options...)
 745         c.Assert(err, IsNil)
 746
 747         // Check
 748         eq, err = compareFilesWithRange(fileName, 1024, newFile, 0, 3072)
 749         c.Assert(err, IsNil)
 750         c.Assert(eq, Equals, true)
 751
 752         os.Remove(newFileGet)
 753         options = []Option{NormalizedRange("1024-4095"), VersionId(versionId)}
 754         err = bucket.GetObjectToFile(objectName, newFileGet, options...)
 755         c.Assert(err, IsNil)
 756
 757         // Compare get and download
 758         eq, err = compareFiles(newFile, newFileGet)
 759         c.Assert(err, IsNil)
 760         c.Assert(eq, Equals, true)
 761
 762         // Download with range, from 2048 to the end
 763         os.Remove(newFile)
 764         options = []Option{NormalizedRange("2048-"), VersionId(versionId)}
 765         err = bucket.DownloadFile(objectName, newFile, 1024*1024, options...)
 766         c.Assert(err, IsNil)
 767
 768         // Check
 769         eq, err = compareFilesWithRange(fileName, 2048, newFile, 0, fileSize-2048)
 770         c.Assert(err, IsNil)
 771         c.Assert(eq, Equals, true)
 772
 773         os.Remove(newFileGet)
 774         options = []Option{NormalizedRange("2048-"), VersionId(versionId)}
 775         err = bucket.GetObjectToFile(objectName, newFileGet, options...)
 776         c.Assert(err, IsNil)
 777
 778         // Compare get and download
 779         eq, err = compareFiles(newFile, newFileGet)
 780         c.Assert(err, IsNil)
 781         c.Assert(eq, Equals, true)
 782
 783         // Download with range, the last 4096
 784         os.Remove(newFile)
 785         options = []Option{Routines(3), NormalizedRange("-4096"), VersionId(versionId)}
 786         err = bucket.DownloadFile(objectName, newFile, 1024, options...)
 787         c.Assert(err, IsNil)
 788
 789         // Check
 790         eq, err = compareFilesWithRange(fileName, fileSize-4096, newFile, 0, 4096)
 791         c.Assert(err, IsNil)
 792         c.Assert(eq, Equals, true)
 793
 794         os.Remove(newFileGet)
 795         options = []Option{NormalizedRange("-4096"), VersionId(versionId)}
 796         err = bucket.GetObjectToFile(objectName, newFileGet, options...)
 797         c.Assert(err, IsNil)
 798
 799         // Compare get and download
 800         eq, err = compareFiles(newFile, newFileGet)
 801         c.Assert(err, IsNil)
 802         c.Assert(eq, Equals, true)
 803
 804         // download whole file
 805         os.Remove(newFileGet)
 806         options = []Option{Routines(3), VersionId(versionId)}
 807         err = bucket.GetObjectToFile(objectName, newFileGet, options...)
 808         c.Assert(err, IsNil)
 809
 810         // Compare get and download
 811         eq, err = compareFiles(fileName, newFileGet)
 812         c.Assert(err, IsNil)
 813         c.Assert(eq, Equals, true)
 814
 815         os.Remove(fileName)
 816         os.Remove(newFileGet)
 817         err = bucket.DeleteObject(objectName)
 818         c.Assert(err, IsNil)
 819         ForceDeleteBucket(client, bucketName, c)
 820 }
 821
 822 func (s *OssDownloadSuite) TestVersioningDownloadWithCheckPoint(c *C) {
 823         // create a bucket with default proprety
 824         client, err := New(endpoint, accessID, accessKey)
 825         c.Assert(err, IsNil)
 826
 827         bucketName := bucketNamePrefix + RandLowStr(6)
 828         err = client.CreateBucket(bucketName)
 829         c.Assert(err, IsNil)
 830
 831         bucket, err := client.Bucket(bucketName)
 832
 833         // put bucket version:enabled
 834         var versioningConfig VersioningConfig
 835         versioningConfig.Status = string(VersionEnabled)
 836         err = client.SetBucketVersioning(bucketName, versioningConfig)
 837         c.Assert(err, IsNil)
 838
 839         // begin test
 840         objectName := objectNamePrefix + RandStr(8)
 841         fileName := "test-file-" + RandStr(8)
 842         fileData := RandStr(500 * 1024)
 843         CreateFile(fileName, fileData, c)
 844         newFile := RandStr(8) + ".jpg"
 845
 846         // Upload a file
 847         var respHeader http.Header
 848         options := []Option{Routines(3), GetResponseHeader(&respHeader)}
 849         err = bucket.UploadFile(objectName, fileName, 100*1024, options...)
 850         c.Assert(err, IsNil)
 851         versionId := GetVersionId(respHeader)
 852         c.Assert(len(versionId) > 0, Equals, true)
 853
 854         // Resumable download with checkpoint dir
 855         os.Remove(newFile)
 856         downloadPartHooker = DownErrorHooker
 857         options = []Option{CheckpointDir(true, "./"), VersionId(versionId)}
 858
 859         strPayer := getPayer(options)
 860         c.Assert(strPayer, Equals, "")
 861
 862         err = bucket.DownloadFile(objectName, newFile, 100*1024, options...)
 863         c.Assert(err, NotNil)
 864         c.Assert(err.Error(), Equals, "ErrorHooker")
 865
 866         // download again
 867         downloadPartHooker = defaultDownloadPartHook
 868         options = []Option{CheckpointDir(true, "./"), VersionId(versionId), GetResponseHeader(&respHeader)}
 869         err = bucket.DownloadFile(objectName, newFile, 100*1024, options...)
 870         c.Assert(err, IsNil)
 871         c.Assert(GetVersionId(respHeader), Equals, versionId)
 872
 873         eq, err := compareFiles(fileName, newFile)
 874         c.Assert(err, IsNil)
 875         c.Assert(eq, Equals, true)
 876
 877         os.Remove(fileName)
 878         os.Remove(newFile)
 879         err = bucket.DeleteObject(objectName)
 880         c.Assert(err, IsNil)
 881         ForceDeleteBucket(client, bucketName, c)
 882 }
 883
 884 func (s *OssDownloadSuite) TestdownloadFileChoiceOptions(c *C) {
 885         // create a bucket with default proprety
 886         client, err := New(endpoint, accessID, accessKey)
 887         c.Assert(err, IsNil)
 888
 889         bucketName := bucketNamePrefix + RandLowStr(6)
 890         err = client.CreateBucket(bucketName)
 891         c.Assert(err, IsNil)
 892
 893         bucket, err := client.Bucket(bucketName)
 894
 895         // begin test
 896         objectName := objectNamePrefix + RandStr(8)
 897         fileName := "test-file-" + RandStr(8)
 898         fileData := RandStr(500 * 1024)
 899         CreateFile(fileName, fileData, c)
 900         newFile := RandStr(8) + ".jpg"
 901
 902         // Upload a file
 903         var respHeader http.Header
 904         options := []Option{Routines(3), GetResponseHeader(&respHeader)}
 905         err = bucket.UploadFile(objectName, fileName, 100*1024, options...)
 906         c.Assert(err, IsNil)
 907
 908         // Resumable download with checkpoint dir
 909         os.Remove(newFile)
 910
 911         // downloadFile with properties
 912         options = []Option{
 913                 ObjectACL(ACLPublicRead),
 914                 RequestPayer(Requester),
 915                 TrafficLimitHeader(1024 * 1024 * 8),
 916         }
 917
 918         err = bucket.DownloadFile(objectName, newFile, 100*1024, options...)
 919         c.Assert(err, IsNil)
 920
 921         eq, err := compareFiles(fileName, newFile)
 922         c.Assert(err, IsNil)
 923         c.Assert(eq, Equals, true)
 924
 925         os.Remove(fileName)
 926         os.Remove(newFile)
 927         err = bucket.DeleteObject(objectName)
 928         c.Assert(err, IsNil)
 929         ForceDeleteBucket(client, bucketName, c)
 930 }
 931
 932 func (s *OssDownloadSuite) TestdownloadFileWithCpChoiceOptions(c *C) {
 933         // create a bucket with default proprety
 934         client, err := New(endpoint, accessID, accessKey)
 935         c.Assert(err, IsNil)
 936
 937         bucketName := bucketNamePrefix + RandLowStr(6)
 938         err = client.CreateBucket(bucketName)
 939         c.Assert(err, IsNil)
 940
 941         bucket, err := client.Bucket(bucketName)
 942
 943         // begin test
 944         objectName := objectNamePrefix + RandStr(8)
 945         fileName := "test-file-" + RandStr(8)
 946         fileData := RandStr(500 * 1024)
 947         CreateFile(fileName, fileData, c)
 948         newFile := RandStr(8) + ".jpg"
 949
 950         // Upload a file
 951         var respHeader http.Header
 952         options := []Option{Routines(3), GetResponseHeader(&respHeader)}
 953         err = bucket.UploadFile(objectName, fileName, 100*1024, options...)
 954         c.Assert(err, IsNil)
 955
 956         // Resumable download with checkpoint dir
 957         os.Remove(newFile)
 958
 959         // DownloadFile with properties
 960         options = []Option{
 961                 ObjectACL(ACLPublicRead),
 962                 RequestPayer(Requester),
 963                 TrafficLimitHeader(1024 * 1024 * 8),
 964                 CheckpointDir(true, "./"),
 965         }
 966
 967         err = bucket.DownloadFile(objectName, newFile, 100*1024, options...)
 968         c.Assert(err, IsNil)
 969
 970         eq, err := compareFiles(fileName, newFile)
 971         c.Assert(err, IsNil)
 972         c.Assert(eq, Equals, true)
 973
 974         os.Remove(fileName)
 975         os.Remove(newFile)
 976         err = bucket.DeleteObject(objectName)
 977         c.Assert(err, IsNil)
 978         ForceDeleteBucket(client, bucketName, c)
 979 }