OSDN Git Service

Create ossClient.go (#574)
[bytom/vapor.git] / vendor / github.com / aliyun / aliyun-oss-go-sdk / oss / progress_test.go
1 // bucket test
2
3 package oss
4
5 import (
6         "bytes"
7         "io/ioutil"
8         "math/rand"
9         "os"
10         "strings"
11         "sync/atomic"
12
13         . "gopkg.in/check.v1"
14 )
15
16 type OssProgressSuite struct {
17         client *Client
18         bucket *Bucket
19 }
20
21 var _ = Suite(&OssProgressSuite{})
22
23 // SetUpSuite runs once when the suite starts running
24 func (s *OssProgressSuite) SetUpSuite(c *C) {
25         client, err := New(endpoint, accessID, accessKey)
26         c.Assert(err, IsNil)
27         s.client = client
28
29         s.client.CreateBucket(bucketName)
30
31         bucket, err := s.client.Bucket(bucketName)
32         c.Assert(err, IsNil)
33         s.bucket = bucket
34
35         testLogger.Println("test progress started")
36 }
37
38 // TearDownSuite runs before each test or benchmark starts running
39 func (s *OssProgressSuite) TearDownSuite(c *C) {
40         // Abort multipart uploads
41         keyMarker := KeyMarker("")
42         uploadIDMarker := UploadIDMarker("")
43         for {
44                 lmu, err := s.bucket.ListMultipartUploads(keyMarker, uploadIDMarker)
45                 c.Assert(err, IsNil)
46                 for _, upload := range lmu.Uploads {
47                         imur := InitiateMultipartUploadResult{Bucket: bucketName, Key: upload.Key, UploadID: upload.UploadID}
48                         err = s.bucket.AbortMultipartUpload(imur)
49                         c.Assert(err, IsNil)
50                 }
51                 keyMarker = KeyMarker(lmu.NextKeyMarker)
52                 uploadIDMarker = UploadIDMarker(lmu.NextUploadIDMarker)
53                 if !lmu.IsTruncated {
54                         break
55                 }
56         }
57
58         // Delete objects
59         marker := Marker("")
60         for {
61                 lor, err := s.bucket.ListObjects(marker)
62                 c.Assert(err, IsNil)
63                 for _, object := range lor.Objects {
64                         err = s.bucket.DeleteObject(object.Key)
65                         c.Assert(err, IsNil)
66                 }
67                 marker = Marker(lor.NextMarker)
68                 if !lor.IsTruncated {
69                         break
70                 }
71         }
72
73         // Delete bucket
74         err := s.client.DeleteBucket(s.bucket.BucketName)
75         c.Assert(err, IsNil)
76
77         testLogger.Println("test progress completed")
78 }
79
80 // SetUpTest runs after each test or benchmark runs
81 func (s *OssProgressSuite) SetUpTest(c *C) {
82         err := removeTempFiles("../oss", ".jpg")
83         c.Assert(err, IsNil)
84
85         err = removeTempFiles("../oss", ".txt")
86         c.Assert(err, IsNil)
87
88         err = removeTempFiles("../oss", ".html")
89         c.Assert(err, IsNil)
90 }
91
92 // TearDownTest runs once after all tests or benchmarks have finished running
93 func (s *OssProgressSuite) TearDownTest(c *C) {
94         err := removeTempFiles("../oss", ".jpg")
95         c.Assert(err, IsNil)
96
97         err = removeTempFiles("../oss", ".txt")
98         c.Assert(err, IsNil)
99
100         err = removeTempFiles("../oss", ".html")
101         c.Assert(err, IsNil)
102 }
103
104 // OssProgressListener is the progress listener
105 type OssProgressListener struct {
106         TotalRwBytes int64
107 }
108
109 // ProgressChanged handles progress event
110 func (listener *OssProgressListener) ProgressChanged(event *ProgressEvent) {
111         switch event.EventType {
112         case TransferStartedEvent:
113                 testLogger.Printf("Transfer Started, ConsumedBytes: %d, TotalBytes %d.\n",
114                         event.ConsumedBytes, event.TotalBytes)
115         case TransferDataEvent:
116                 atomic.AddInt64(&listener.TotalRwBytes, event.RwBytes)
117                 testLogger.Printf("Transfer Data, ConsumedBytes: %d, TotalBytes %d, %d%%.\n",
118                         event.ConsumedBytes, event.TotalBytes, event.ConsumedBytes*100/event.TotalBytes)
119         case TransferCompletedEvent:
120                 testLogger.Printf("Transfer Completed, ConsumedBytes: %d, TotalBytes %d.\n",
121                         event.ConsumedBytes, event.TotalBytes)
122         case TransferFailedEvent:
123                 testLogger.Printf("Transfer Failed, ConsumedBytes: %d, TotalBytes %d.\n",
124                         event.ConsumedBytes, event.TotalBytes)
125         default:
126         }
127 }
128
129 // TestPutObject
130 func (s *OssProgressSuite) TestPutObject(c *C) {
131         objectName := RandStr(8) + ".jpg"
132         localFile := "../sample/The Go Programming Language.html"
133
134         fileInfo, err := os.Stat(localFile)
135         c.Assert(err, IsNil)
136
137         // PutObject
138         fd, err := os.Open(localFile)
139         c.Assert(err, IsNil)
140         defer fd.Close()
141
142         progressListener := OssProgressListener{}
143         err = s.bucket.PutObject(objectName, fd, Progress(&progressListener))
144         c.Assert(err, IsNil)
145         c.Assert(progressListener.TotalRwBytes, Equals, fileInfo.Size())
146
147         // PutObjectFromFile
148         progressListener.TotalRwBytes = 0
149         err = s.bucket.PutObjectFromFile(objectName, localFile, Progress(&progressListener))
150         c.Assert(err, IsNil)
151         c.Assert(progressListener.TotalRwBytes, Equals, fileInfo.Size())
152
153         // DoPutObject
154         fd, err = os.Open(localFile)
155         c.Assert(err, IsNil)
156         defer fd.Close()
157
158         request := &PutObjectRequest{
159                 ObjectKey: objectName,
160                 Reader:    fd,
161         }
162
163         progressListener.TotalRwBytes = 0
164         options := []Option{Progress(&progressListener)}
165         _, err = s.bucket.DoPutObject(request, options)
166         c.Assert(err, IsNil)
167         c.Assert(progressListener.TotalRwBytes, Equals, fileInfo.Size())
168
169         // PutObject size is 0
170         progressListener.TotalRwBytes = 0
171         err = s.bucket.PutObject(objectName, strings.NewReader(""), Progress(&progressListener))
172         c.Assert(err, IsNil)
173         c.Assert(progressListener.TotalRwBytes, Equals, int64(0))
174
175         testLogger.Println("OssProgressSuite.TestPutObject")
176 }
177
178 // TestSignURL
179 func (s *OssProgressSuite) SignURLTestFunc(c *C, authVersion AuthVersionType, extraHeaders []string) {
180         objectName := objectNamePrefix + RandStr(8)
181         filePath := RandLowStr(10)
182         content := RandStr(20)
183         CreateFile(filePath, content, c)
184
185         oldType := s.bucket.Client.Config.AuthVersion
186         oldHeaders := s.bucket.Client.Config.AdditionalHeaders
187         s.bucket.Client.Config.AuthVersion = authVersion
188         s.bucket.Client.Config.AdditionalHeaders = extraHeaders
189
190         // Sign URL for put
191         progressListener := OssProgressListener{}
192         str, err := s.bucket.SignURL(objectName, HTTPPut, 60, Progress(&progressListener))
193         c.Assert(err, IsNil)
194         if s.bucket.Client.Config.AuthVersion == AuthV1 {
195                 c.Assert(strings.Contains(str, HTTPParamExpires+"="), Equals, true)
196                 c.Assert(strings.Contains(str, HTTPParamAccessKeyID+"="), Equals, true)
197                 c.Assert(strings.Contains(str, HTTPParamSignature+"="), Equals, true)
198         } else {
199                 c.Assert(strings.Contains(str, HTTPParamSignatureVersion+"=OSS2"), Equals, true)
200                 c.Assert(strings.Contains(str, HTTPParamExpiresV2+"="), Equals, true)
201                 c.Assert(strings.Contains(str, HTTPParamAccessKeyIDV2+"="), Equals, true)
202                 c.Assert(strings.Contains(str, HTTPParamSignatureV2+"="), Equals, true)
203         }
204
205         // Put object with URL
206         fd, err := os.Open(filePath)
207         c.Assert(err, IsNil)
208         defer fd.Close()
209
210         err = s.bucket.PutObjectWithURL(str, fd, Progress(&progressListener))
211         c.Assert(err, IsNil)
212         c.Assert(progressListener.TotalRwBytes, Equals, int64(len(content)))
213
214         // Put object from file with URL
215         progressListener.TotalRwBytes = 0
216         err = s.bucket.PutObjectFromFileWithURL(str, filePath, Progress(&progressListener))
217         c.Assert(err, IsNil)
218         c.Assert(progressListener.TotalRwBytes, Equals, int64(len(content)))
219
220         // DoPutObject
221         fd, err = os.Open(filePath)
222         c.Assert(err, IsNil)
223         defer fd.Close()
224
225         progressListener.TotalRwBytes = 0
226         options := []Option{Progress(&progressListener)}
227         _, err = s.bucket.DoPutObjectWithURL(str, fd, options)
228         c.Assert(err, IsNil)
229         c.Assert(progressListener.TotalRwBytes, Equals, int64(len(content)))
230
231         // Sign URL for get
232         str, err = s.bucket.SignURL(objectName, HTTPGet, 60, Progress(&progressListener))
233         c.Assert(err, IsNil)
234         if s.bucket.Client.Config.AuthVersion == AuthV1 {
235                 c.Assert(strings.Contains(str, HTTPParamExpires+"="), Equals, true)
236                 c.Assert(strings.Contains(str, HTTPParamAccessKeyID+"="), Equals, true)
237                 c.Assert(strings.Contains(str, HTTPParamSignature+"="), Equals, true)
238         } else {
239                 c.Assert(strings.Contains(str, HTTPParamSignatureVersion+"=OSS2"), Equals, true)
240                 c.Assert(strings.Contains(str, HTTPParamExpiresV2+"="), Equals, true)
241                 c.Assert(strings.Contains(str, HTTPParamAccessKeyIDV2+"="), Equals, true)
242                 c.Assert(strings.Contains(str, HTTPParamSignatureV2+"="), Equals, true)
243         }
244
245         // Get object with URL
246         progressListener.TotalRwBytes = 0
247         body, err := s.bucket.GetObjectWithURL(str, Progress(&progressListener))
248         c.Assert(err, IsNil)
249         str, err = readBody(body)
250         c.Assert(err, IsNil)
251         c.Assert(str, Equals, content)
252         c.Assert(progressListener.TotalRwBytes, Equals, int64(len(content)))
253
254         // Get object to file with URL
255         progressListener.TotalRwBytes = 0
256         str, err = s.bucket.SignURL(objectName, HTTPGet, 10, Progress(&progressListener))
257         c.Assert(err, IsNil)
258
259         newFile := RandStr(10)
260         progressListener.TotalRwBytes = 0
261         err = s.bucket.GetObjectToFileWithURL(str, newFile, Progress(&progressListener))
262         c.Assert(progressListener.TotalRwBytes, Equals, int64(len(content)))
263         c.Assert(err, IsNil)
264         eq, err := compareFiles(filePath, newFile)
265         c.Assert(err, IsNil)
266         c.Assert(eq, Equals, true)
267
268         os.Remove(filePath)
269         os.Remove(newFile)
270
271         err = s.bucket.DeleteObject(objectName)
272         c.Assert(err, IsNil)
273
274         testLogger.Println("OssProgressSuite.TestSignURL")
275
276         s.bucket.Client.Config.AuthVersion = oldType
277         s.bucket.Client.Config.AdditionalHeaders = oldHeaders
278 }
279
280 func (s *OssProgressSuite) TestSignURL(c *C) {
281         s.SignURLTestFunc(c, AuthV1, []string{})
282         s.SignURLTestFunc(c, AuthV2, []string{})
283         s.SignURLTestFunc(c, AuthV2, []string{"host", "range", "user-agent"})
284 }
285
286 func (s *OssProgressSuite) TestPutObjectNegative(c *C) {
287         objectName := objectNamePrefix + RandStr(8)
288         localFile := "../sample/The Go Programming Language.html"
289
290         // Invalid endpoint
291         client, err := New("http://oss-cn-taikang.aliyuncs.com", accessID, accessKey)
292         c.Assert(err, IsNil)
293
294         bucket, err := client.Bucket(bucketName)
295         c.Assert(err, IsNil)
296
297         err = bucket.PutObjectFromFile(objectName, localFile, Progress(&OssProgressListener{}))
298         testLogger.Println(err)
299         c.Assert(err, NotNil)
300
301         testLogger.Println("OssProgressSuite.TestPutObjectNegative")
302 }
303
304 // TestAppendObject
305 func (s *OssProgressSuite) TestAppendObject(c *C) {
306         objectName := objectNamePrefix + RandStr(8)
307         objectValue := RandStr(100)
308         var val = []byte(objectValue)
309         var nextPos int64
310         var midPos = 1 + rand.Intn(len(val)-1)
311
312         // AppendObject
313         progressListener := OssProgressListener{}
314         nextPos, err := s.bucket.AppendObject(objectName, bytes.NewReader(val[0:midPos]), nextPos, Progress(&progressListener))
315         c.Assert(err, IsNil)
316         c.Assert(progressListener.TotalRwBytes, Equals, nextPos)
317
318         // DoAppendObject
319         request := &AppendObjectRequest{
320                 ObjectKey: objectName,
321                 Reader:    bytes.NewReader(val[midPos:]),
322                 Position:  nextPos,
323         }
324         options := []Option{Progress(&OssProgressListener{})}
325         _, err = s.bucket.DoAppendObject(request, options)
326         c.Assert(err, IsNil)
327
328         testLogger.Println("OssProgressSuite.TestAppendObject")
329 }
330
331 // TestMultipartUpload
332 func (s *OssProgressSuite) TestMultipartUpload(c *C) {
333         objectName := objectNamePrefix + RandStr(8)
334         var fileName = "../sample/BingWallpaper-2015-11-07.jpg"
335
336         fileInfo, err := os.Stat(fileName)
337         c.Assert(err, IsNil)
338
339         chunks, err := SplitFileByPartNum(fileName, 3)
340         c.Assert(err, IsNil)
341         testLogger.Println("chunks:", chunks)
342
343         fd, err := os.Open(fileName)
344         c.Assert(err, IsNil)
345         defer fd.Close()
346
347         // Initiate
348         progressListener := OssProgressListener{}
349         imur, err := s.bucket.InitiateMultipartUpload(objectName)
350         c.Assert(err, IsNil)
351
352         // UploadPart
353         var parts []UploadPart
354         for _, chunk := range chunks {
355                 fd.Seek(chunk.Offset, os.SEEK_SET)
356                 part, err := s.bucket.UploadPart(imur, fd, chunk.Size, chunk.Number, Progress(&progressListener))
357                 c.Assert(err, IsNil)
358                 parts = append(parts, part)
359         }
360
361         // Complete
362         _, err = s.bucket.CompleteMultipartUpload(imur, parts)
363         c.Assert(err, IsNil)
364         c.Assert(progressListener.TotalRwBytes, Equals, fileInfo.Size())
365
366         err = s.bucket.DeleteObject(objectName)
367         c.Assert(err, IsNil)
368
369         testLogger.Println("OssProgressSuite.TestMultipartUpload")
370 }
371
372 // TestMultipartUploadFromFile
373 func (s *OssProgressSuite) TestMultipartUploadFromFile(c *C) {
374         objectName := objectNamePrefix + RandStr(8)
375         var fileName = "../sample/BingWallpaper-2015-11-07.jpg"
376         fileInfo, err := os.Stat(fileName)
377         c.Assert(err, IsNil)
378
379         chunks, err := SplitFileByPartNum(fileName, 3)
380         c.Assert(err, IsNil)
381
382         // Initiate
383         imur, err := s.bucket.InitiateMultipartUpload(objectName)
384         c.Assert(err, IsNil)
385
386         // UploadPart
387         progressListener := OssProgressListener{}
388         var parts []UploadPart
389         for _, chunk := range chunks {
390                 part, err := s.bucket.UploadPartFromFile(imur, fileName, chunk.Offset, chunk.Size, chunk.Number, Progress(&progressListener))
391                 c.Assert(err, IsNil)
392                 parts = append(parts, part)
393         }
394
395         // Complete
396         _, err = s.bucket.CompleteMultipartUpload(imur, parts)
397         c.Assert(err, IsNil)
398         c.Assert(progressListener.TotalRwBytes, Equals, fileInfo.Size())
399
400         err = s.bucket.DeleteObject(objectName)
401         c.Assert(err, IsNil)
402
403         testLogger.Println("OssProgressSuite.TestMultipartUploadFromFile")
404 }
405
406 // TestGetObject
407 func (s *OssProgressSuite) TestGetObject(c *C) {
408         objectName := objectNamePrefix + RandStr(8)
409         localFile := "../sample/BingWallpaper-2015-11-07.jpg"
410         newFile := "newpic-progress-1.jpg"
411
412         fileInfo, err := os.Stat(localFile)
413         c.Assert(err, IsNil)
414
415         progressListener := OssProgressListener{}
416         // PutObject
417         err = s.bucket.PutObjectFromFile(objectName, localFile, Progress(&progressListener))
418         c.Assert(err, IsNil)
419         c.Assert(progressListener.TotalRwBytes, Equals, fileInfo.Size())
420
421         // GetObject
422         progressListener.TotalRwBytes = 0
423         body, err := s.bucket.GetObject(objectName, Progress(&progressListener))
424         c.Assert(err, IsNil)
425         _, err = ioutil.ReadAll(body)
426         c.Assert(err, IsNil)
427         body.Close()
428         c.Assert(progressListener.TotalRwBytes, Equals, fileInfo.Size())
429
430         // GetObjectToFile
431         progressListener.TotalRwBytes = 0
432         err = s.bucket.GetObjectToFile(objectName, newFile, Progress(&progressListener))
433         c.Assert(err, IsNil)
434         c.Assert(progressListener.TotalRwBytes, Equals, fileInfo.Size())
435
436         // DoGetObject
437         progressListener.TotalRwBytes = 0
438         request := &GetObjectRequest{objectName}
439         options := []Option{Progress(&progressListener)}
440         result, err := s.bucket.DoGetObject(request, options)
441         c.Assert(err, IsNil)
442         _, err = ioutil.ReadAll(result.Response.Body)
443         c.Assert(err, IsNil)
444         result.Response.Body.Close()
445         c.Assert(progressListener.TotalRwBytes, Equals, fileInfo.Size())
446
447         // GetObject with range
448         progressListener.TotalRwBytes = 0
449         body, err = s.bucket.GetObject(objectName, Range(1024, 4*1024), Progress(&progressListener))
450         c.Assert(err, IsNil)
451         text, err := ioutil.ReadAll(body)
452         c.Assert(err, IsNil)
453         body.Close()
454         c.Assert(progressListener.TotalRwBytes, Equals, int64(len(text)))
455
456         // PutObject size is 0
457         progressListener.TotalRwBytes = 0
458         err = s.bucket.PutObject(objectName, strings.NewReader(""), Progress(&progressListener))
459         c.Assert(err, IsNil)
460         c.Assert(progressListener.TotalRwBytes, Equals, int64(0))
461
462         // GetObject size is 0
463         progressListener.TotalRwBytes = 0
464         body, err = s.bucket.GetObject(objectName, Progress(&progressListener))
465         c.Assert(err, IsNil)
466         _, err = ioutil.ReadAll(body)
467         c.Assert(err, IsNil)
468         body.Close()
469         c.Assert(progressListener.TotalRwBytes, Equals, int64(0))
470
471         testLogger.Println("OssProgressSuite.TestGetObject")
472 }
473
474 // TestGetObjectNegative
475 func (s *OssProgressSuite) TestGetObjectNegative(c *C) {
476         objectName := objectNamePrefix + RandStr(8)
477         localFile := "../sample/BingWallpaper-2015-11-07.jpg"
478
479         // PutObject
480         err := s.bucket.PutObjectFromFile(objectName, localFile)
481         c.Assert(err, IsNil)
482
483         // GetObject
484         body, err := s.bucket.GetObject(objectName, Progress(&OssProgressListener{}))
485         c.Assert(err, IsNil)
486
487         buf := make([]byte, 4*1024)
488         n, err := body.Read(buf)
489         c.Assert(err, IsNil)
490
491         //time.Sleep(70 * time.Second) TODO
492
493         // Read should fail
494         for err == nil {
495                 n, err = body.Read(buf)
496                 n += n
497         }
498         c.Assert(err, NotNil)
499         body.Close()
500
501         testLogger.Println("OssProgressSuite.TestGetObjectNegative")
502 }
503
504 // TestUploadFile
505 func (s *OssProgressSuite) TestUploadFile(c *C) {
506         objectName := objectNamePrefix + RandStr(8)
507         fileName := "../sample/BingWallpaper-2015-11-07.jpg"
508
509         fileInfo, err := os.Stat(fileName)
510         c.Assert(err, IsNil)
511
512         progressListener := OssProgressListener{}
513         err = s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(5), Progress(&progressListener))
514         c.Assert(err, IsNil)
515         c.Assert(progressListener.TotalRwBytes, Equals, fileInfo.Size())
516
517         progressListener.TotalRwBytes = 0
518         err = s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3), Checkpoint(true, objectName+".cp"), Progress(&progressListener))
519         c.Assert(err, IsNil)
520         c.Assert(progressListener.TotalRwBytes, Equals, fileInfo.Size())
521
522         testLogger.Println("OssProgressSuite.TestUploadFile")
523 }
524
525 // TestDownloadFile
526 func (s *OssProgressSuite) TestDownloadFile(c *C) {
527         objectName := objectNamePrefix + RandStr(8)
528         fileName := "../sample/BingWallpaper-2015-11-07.jpg"
529         newFile := "down-new-file-progress-2.jpg"
530
531         fileInfo, err := os.Stat(fileName)
532         c.Assert(err, IsNil)
533
534         // Upload
535         err = s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3))
536         c.Assert(err, IsNil)
537
538         progressListener := OssProgressListener{}
539         err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(5), Progress(&progressListener))
540         c.Assert(err, IsNil)
541         c.Assert(progressListener.TotalRwBytes, Equals, fileInfo.Size())
542
543         progressListener.TotalRwBytes = 0
544         err = s.bucket.DownloadFile(objectName, newFile, 1024*1024, Routines(3), Progress(&progressListener))
545         c.Assert(err, IsNil)
546         c.Assert(progressListener.TotalRwBytes, Equals, fileInfo.Size())
547
548         progressListener.TotalRwBytes = 0
549         err = s.bucket.DownloadFile(objectName, newFile, 50*1024, Routines(3), Checkpoint(true, ""), Progress(&progressListener))
550         c.Assert(err, IsNil)
551         c.Assert(progressListener.TotalRwBytes, Equals, fileInfo.Size())
552
553         testLogger.Println("OssProgressSuite.TestDownloadFile")
554 }
555
556 // TestCopyFile
557 func (s *OssProgressSuite) TestCopyFile(c *C) {
558         srcObjectName := objectNamePrefix + RandStr(8)
559         destObjectName := srcObjectName + "-copy"
560         fileName := "../sample/BingWallpaper-2015-11-07.jpg"
561
562         fileInfo, err := os.Stat(fileName)
563         c.Assert(err, IsNil)
564
565         // Upload
566         progressListener := OssProgressListener{}
567         err = s.bucket.UploadFile(srcObjectName, fileName, 100*1024, Routines(3), Progress(&progressListener))
568         c.Assert(err, IsNil)
569         c.Assert(progressListener.TotalRwBytes, Equals, fileInfo.Size())
570
571         progressListener.TotalRwBytes = 0
572         err = s.bucket.CopyFile(bucketName, srcObjectName, destObjectName, 100*1024, Routines(5), Progress(&progressListener))
573         c.Assert(err, IsNil)
574         c.Assert(progressListener.TotalRwBytes, Equals, fileInfo.Size())
575
576         progressListener.TotalRwBytes = 0
577         err = s.bucket.CopyFile(bucketName, srcObjectName, destObjectName, 1024*100, Routines(3), Checkpoint(true, ""), Progress(&progressListener))
578         c.Assert(err, IsNil)
579         c.Assert(progressListener.TotalRwBytes, Equals, fileInfo.Size())
580
581         testLogger.Println("OssProgressSuite.TestCopyFile")
582 }