vendor/github.com/aliyun/aliyun-oss-go-sdk/oss/select_object_read_file_test.go

   1 package oss
   2
   3 import (
   4         "bufio"
   5         "io"
   6         "os"
   7         "strings"
   8         "strconv"
   9         "regexp"
  10         "encoding/json"
  11         "encoding/csv"
  12 )
  13
  14 func handleError(err error) error {
  15         if err == nil {
  16                 return nil
  17         }
  18         return err
  19 }
  20
  21 func readCsvLine(fileName string) (int, error) {
  22         file, err := os.Open(fileName)
  23         if err != nil {
  24                 return 0,err
  25         }
  26         defer file.Close()
  27         rd := csv.NewReader(file)
  28         rc, err := rd.ReadAll()
  29         return len(rc), err
  30 }
  31 func readCsvIsEmpty(fileName string) (string, error) {
  32         file, err := os.Open(fileName)
  33         if err != nil {
  34                 return "",err
  35         }
  36         defer file.Close()
  37         var out string
  38         var i, index int
  39         var indexYear, indexStateAbbr, indexCityName, indexPopulationCount int
  40
  41         rd := bufio.NewReader(file)
  42         for {
  43                 line, err := rd.ReadString('\n') // read a line
  44                 if io.EOF == err {
  45                         break
  46                 }
  47                 if err != nil {
  48                         return "",err
  49                 }
  50
  51                 sptLint := strings.Split(line, ",")
  52                 if i == 0 {
  53                         i = 1
  54                         for _, val := range sptLint {
  55                                 switch val {
  56                                 case "Year":
  57                                         indexYear = index
  58                                 case "StateAbbr":
  59                                         indexStateAbbr = index
  60                                 case "CityName":
  61                                         indexCityName = index
  62                                 case "PopulationCount":
  63                                         indexPopulationCount = index
  64                                 }
  65                                 index++
  66                         }
  67                 } else {
  68                         if sptLint[indexCityName] != "" {
  69                                 outLine := sptLint[indexYear] + "," + sptLint[indexStateAbbr] + "," + sptLint[indexCityName] + "," + sptLint[indexPopulationCount] + "\n"
  70                                 out += outLine
  71                         }
  72                 }
  73         }
  74
  75         return out, nil
  76 }
  77
  78 func readCsvLike(fileName string) (string, error) {
  79         file, err := os.Open(fileName)
  80         if err != nil {
  81                 return "",err
  82         }
  83         defer file.Close()
  84         var out string
  85         var i, index int
  86         var indexYear,indexStateAbbr,indexCityName,indexPopulationCount,indexMeasure int
  87
  88         rd := bufio.NewReader(file)
  89         for {
  90                 line, err := rd.ReadString('\n') // read a line
  91                 if io.EOF == err {
  92                         break
  93                 }
  94                 if err != nil {
  95                         return "",err
  96                 }
  97
  98                 //utf8Lint := ConvertToString(line,"gbk", "utf-8")
  99                 sptLint := strings.Split(line[:(len(line)-1)], ",")
 100                 if i == 0 {
 101                         i = 1
 102                         for _, val := range sptLint {
 103                                 switch val {
 104                                 case "Year":indexYear = index
 105                                 case "StateAbbr":indexStateAbbr = index
 106                                 case "CityName":indexCityName = index
 107                                 case "Short_Question_Text":indexPopulationCount = index
 108                                 case "Measure":indexMeasure = index
 109                                 }
 110                                 index++
 111                         }
 112                 } else {
 113                         if sptLint[indexMeasure] != "" {
 114                                 reg := regexp.MustCompile("^.*blood pressure.*Years$")
 115                                 res := reg.FindAllString(sptLint[indexMeasure], -1)
 116                                 if len(res) > 0 {
 117                                         outLine := sptLint[indexYear] + "," +sptLint[indexStateAbbr] + "," +sptLint[indexCityName] + "," + sptLint[indexPopulationCount] + "\n"
 118                                         out += outLine
 119                                 }
 120                         }
 121                 }
 122         }
 123
 124         return out, nil
 125 }
 126
 127 func readCsvRange(fileName string, l int, r int) (string, error) {
 128         file, err := os.Open(fileName)
 129         if err != nil {
 130                 return "", err
 131         }
 132         defer file.Close()
 133         var out string
 134         var i, index int
 135         var indexYear,indexStateAbbr,indexCityName,indexPopulationCount int
 136
 137         rd := bufio.NewReader(file)
 138         for j := 0; j < r + 1; j++ {
 139                 if j < l {
 140                         continue
 141                 }
 142                 line, err := rd.ReadString('\n') // read a line
 143                 if io.EOF == err {
 144                         break
 145                 }
 146                 if err != nil {
 147                         return "", err
 148                 }
 149
 150                 sptLint := strings.Split(line[:(len(line)-1)], ",")
 151                 if i == 0 {
 152                         i = 1
 153                         for _, val := range sptLint {
 154                                 switch val {
 155                                 case "Year":indexYear = index
 156                                 case "StateAbbr":indexStateAbbr = index
 157                                 case "CityName":indexCityName = index
 158                                 case "Short_Question_Text":indexPopulationCount = index
 159                                 }
 160                                 index++
 161                         }
 162                 } else {
 163                         outLine := sptLint[indexYear] + "," +sptLint[indexStateAbbr] + "," +sptLint[indexCityName] + "," + sptLint[indexPopulationCount] + "\n"
 164                         out += outLine
 165                 }
 166         }
 167
 168         return out, nil
 169 }
 170
 171 func readCsvFloatAgg(fileName string) (avg, max, sum float64,  er error) {
 172         file, err := os.Open(fileName)
 173         if err != nil {
 174                 er = err
 175                 return
 176         }
 177         defer file.Close()
 178         var i, index int
 179         var indexDataValue int
 180
 181         rd := csv.NewReader(file)
 182
 183         for {
 184                 rc, err := rd.Read()
 185                 if io.EOF == err {
 186                         break
 187                 }
 188                 if err != nil {
 189                         er = err
 190                         return
 191                 }
 192                 if i == 0 {
 193                         i=1
 194                         for index = 0; index < len(rc); index++ {
 195                                 if rc[index] == "Data_Value" {
 196                                         indexDataValue = index
 197                                 }
 198                         }
 199                 } else {
 200                         if rc[indexDataValue] != "" {
 201                                 s1, err := strconv.ParseFloat(rc[indexDataValue], 64)
 202                                 if err != nil {
 203                                         er = err
 204                                         return
 205                                 }
 206                                 sum +=s1
 207                                 if s1 > max {
 208                                         max = s1
 209                                 }
 210                                 i++
 211                         }
 212                 }
 213         }
 214         avg = sum / float64(i-1)
 215         return
 216 }
 217 func readCsvConcat(fileName string) (string, error) {
 218         var out string
 219         file, err := os.Open(fileName)
 220         if err != nil {
 221                 return out, err
 222         }
 223         defer file.Close()
 224         var i int
 225         var indexDataValue int
 226         var indexYear,indexStateAbbr,indexCityName,indexShortQuestionText, indexDataValueUnit int
 227
 228         rd := csv.NewReader(file)
 229
 230         for {
 231                 rc, err := rd.Read()
 232                 if io.EOF == err {
 233                         break
 234                 }
 235                 if err != nil {
 236                         return out, err
 237                 }
 238                 if i == 0 {
 239                         for j, v := range rc {
 240                                 switch v {
 241                                         case "Year":indexYear = j
 242                                         case "StateAbbr":indexStateAbbr = j
 243                                         case "CityName":indexCityName = j
 244                                         case "Short_Question_Text":indexShortQuestionText = j
 245                                         case "Data_Value_Unit":indexDataValueUnit = j
 246                                         case "Data_Value":indexDataValue = j
 247                                 }
 248                         }
 249                 } else {
 250                         i++
 251                         if rc[indexDataValue] != "" || rc[indexDataValueUnit] != "" {
 252                                 reg := regexp.MustCompile("^14.8.*$")
 253                                 reD := reg.FindAllString(rc[indexDataValue], -1)
 254                                 reDU := reg.FindAllString(rc[indexDataValueUnit], -1)
 255                                 if len(reD) > 0 || len(reDU) > 0 {
 256                                         outLine := rc[indexYear] + "," +rc[indexStateAbbr] + "," +rc[indexCityName] + "," + rc[indexShortQuestionText] + "\n"
 257                                         out += outLine
 258                                 }
 259                         }
 260                 }
 261                 i++
 262         }
 263         return out, nil
 264 }
 265 func readCsvComplicateCondition(fileName string)(string, error) {
 266         var out string
 267         file, err := os.Open(fileName)
 268         if err != nil {
 269                 return out, err
 270         }
 271         defer file.Close()
 272         var i int
 273         var indexDataValue, indexCategory, indexHighConfidenceLimit, indexMeasure int
 274         var indexYear,indexStateAbbr,indexCityName,indexShortQuestionText, indexDataValueUnit int
 275
 276         rd := csv.NewReader(file)
 277
 278         for {
 279                 rc, err := rd.Read()
 280                 if io.EOF == err {
 281                         break
 282                 }
 283                 if err != nil {
 284                         return out, err
 285                 }
 286                 if i == 0 {
 287                         for j, v := range rc {
 288                                 switch v {
 289                                         case "Year":indexYear = j
 290                                         case "StateAbbr":indexStateAbbr = j
 291                                         case "CityName":indexCityName = j
 292                                         case "Short_Question_Text":indexShortQuestionText = j
 293                                         case "Data_Value_Unit":indexDataValueUnit = j
 294                                         case "Data_Value":indexDataValue = j
 295                                         case "Measure":indexMeasure = j
 296                                         case "Category":indexCategory = j
 297                                         case "High_Confidence_Limit":indexHighConfidenceLimit = j
 298                                 }
 299                         }
 300                 } else {
 301                         reg := regexp.MustCompile("^.*18 Years$")
 302                         reM := reg.FindAllString(rc[indexMeasure], -1)
 303                         var dataV, limitV float64
 304                         if rc[indexDataValue] != "" {
 305                                 dataV, err = strconv.ParseFloat(rc[indexDataValue], 64)
 306                                 if err != nil {
 307                                         return out, err
 308                                 }
 309                         }
 310                         if rc[indexHighConfidenceLimit] != "" {
 311                                 limitV, err = strconv.ParseFloat(rc[indexHighConfidenceLimit], 64)
 312                                 if err != nil {
 313                                         return out, err
 314                                 }
 315                         }
 316                         if dataV > 14.8 && rc[indexDataValueUnit] == "%" || len(reM) > 0  &&
 317                                 rc[indexCategory] == "Unhealthy Behaviors" || limitV > 70.0 {
 318                                         outLine := rc[indexYear] + "," +rc[indexStateAbbr] + "," +rc[indexCityName] + "," + rc[indexShortQuestionText] + "," + rc[indexDataValue] + "," + rc[indexDataValueUnit] + "," + rc[indexCategory] + "," + rc[indexHighConfidenceLimit] + "\n"
 319                                         out += outLine
 320                         }
 321                 }
 322                 i++
 323         }
 324         return out, nil
 325 }
 326
 327
 328 type Extra struct {
 329         Address string `json:"address"`
 330         ContactForm string `json:"contact_form"`
 331         Fax string `json:"fax,omitempty"`
 332         How string `json:"how,omitempty"`
 333         Office string `json:"office"`
 334         RssUrl string `json:"rss_url,omitempty"`
 335 }
 336
 337 type Person struct {
 338         Bioguideid string `json:"bioguideid"`
 339         Birthday string `json:"birthday"`
 340         Cspanid int `json:"cspanid"`
 341         Firstname string `json:"firstname"`
 342         Gender string `json:"gender"`
 343         GenderLabel string `json:"gender_label"`
 344         Lastname string `json:"lastname"`
 345         Link string `json:"link"`
 346         Middlename string `json:"middlename"`
 347         Name string `json:"name"`
 348         Namemod string `json:"namemod"`
 349         Nickname string `json:"nickname"`
 350         Osid string `json:"osid"`
 351         Pvsid *string `json:"pvsid"`
 352         Sortname string `json:"sortname"`
 353         Twitterid *string `json:"twitterid"`
 354         Youtubeid *string `json:"youtubeid"`
 355 }
 356
 357 type JsonLineSt struct {
 358         Caucus *string `json:"caucus"`
 359         CongressNumbers []int `json:"congress_numbers"`
 360         Current  bool `json:"current"`
 361         Description string `json:"description"`
 362         District *string `json:"district"`
 363         Enddate string `json:"enddate"`
 364         Extra Extra `json:"extra"`
 365         LeadershipTitle *string `json:"leadership_title"`
 366         Party string `json:"party"`
 367         Person Person `json:"person"`
 368         Phone string `json:"phone"`
 369         RoleType string `json:"role_type"`
 370         RoleTypeLabel string `json:"role_type_label"`
 371         SenatorClass string `json:"senator_class"`
 372         SenatorClassLabel string `json:"senator_class_label"`
 373         SenatorRank string `json:"senator_rank"`
 374         SenatorRankLabel string `json:"senator_rank_label"`
 375         Startdate string `json:"startdate"`
 376         State string `json:"state"`
 377         Title string `json:"title"`
 378         TitleLong string `json:"title_long"`
 379         Website string `json:"website"`
 380 }
 381 type Metast struct {
 382         limit int
 383         Offset int
 384         TotalCount int
 385 }
 386
 387 type JsonSt struct {
 388         Meta Metast
 389         Objects []JsonLineSt `json:"objects"`
 390 }
 391
 392 func readJsonDocument(fileName string) (string, error){
 393         var out string
 394         var data JsonSt
 395     file, err := os.Open(fileName)
 396     if err != nil {
 397             return "",err
 398     }
 399     decoder := json.NewDecoder(file)
 400     err = decoder.Decode(&data)
 401     for _, v := range data.Objects {
 402             if v.Party == "Democrat"{
 403                    lint, err := json.Marshal(v)
 404                    if err != nil {
 405                            return "",err
 406                    }
 407                    lints := strings.Replace(string(lint), "\\u0026", "&", -1)
 408                    out += lints + ","
 409             }
 410     }
 411
 412     return out, err
 413 }
 414 func readJsonLinesLike(fileName string) (string, error){
 415         var out string
 416         var data JsonSt
 417         file, err := os.Open(fileName)
 418         if err != nil {
 419                 return "",err
 420         }
 421         decoder := json.NewDecoder(file)
 422         err = decoder.Decode(&data)
 423         reg := regexp.MustCompile("^1959.*")
 424         for _, v := range data.Objects {
 425                 reB := reg.FindAllString(v.Person.Birthday, -1)
 426                 if len(reB) > 0 {
 427                         lints := "{\"firstname\":\"" + v.Person.Firstname + "\",\"lastname\":\"" + v.Person.Lastname + "\"}"
 428                         out += lints + ","
 429                 }
 430         }
 431
 432         return out, err
 433 }
 434
 435 func readJsonLinesRange(fileName string, l, r int) (string, error){
 436         var out string
 437         var data JsonSt
 438         var i int
 439         file, err := os.Open(fileName)
 440         if err != nil {
 441                 return "",err
 442         }
 443         decoder := json.NewDecoder(file)
 444         err = decoder.Decode(&data)
 445         for _, v := range data.Objects {
 446                 if i < l {
 447                         continue
 448                 }
 449                 if i >= r {
 450                         break
 451                 }
 452                 extrb, err := json.Marshal(v.Extra)
 453                 if err != nil {
 454                         return "",err
 455                 }
 456                 extr := strings.Replace(string(extrb), "\\u0026", "&", -1)
 457
 458                 lints := "{\"firstname\":\"" + v.Person.Firstname + "\",\"lastname\":\"" + v.Person.Lastname +
 459                         "\",\"extra\":" + extr + "}"
 460                 out += lints + ","
 461                 i++
 462         }
 463
 464         return out, err
 465 }
 466
 467 func readJsonFloatAggregation(fileName string) (float64, float64, float64, error){
 468         var avg, max, min, sum float64
 469         var data JsonSt
 470         var i int
 471         file, err := os.Open(fileName)
 472         if err != nil {
 473                 return  avg, max, min, err
 474         }
 475         decoder := json.NewDecoder(file)
 476         err = decoder.Decode(&data)
 477         for _, v := range data.Objects {
 478                 if i == 0 {
 479                         min = float64(v.Person.Cspanid)
 480                 }
 481                 if max < float64(v.Person.Cspanid) {
 482                         max = float64(v.Person.Cspanid)
 483                 }
 484                 if min > float64(v.Person.Cspanid) {
 485                         min = float64(v.Person.Cspanid)
 486                 }
 487                 sum += float64(v.Person.Cspanid)
 488                 i++
 489         }
 490         avg = sum / float64(i)
 491         return avg, max, min, err
 492 }
 493
 494 func readJsonDocumentConcat(fileName string) (string, error){
 495         var out string
 496         var data JsonSt
 497         file, err := os.Open(fileName)
 498         if err != nil {
 499                 return "",err
 500         }
 501         decoder := json.NewDecoder(file)
 502         err = decoder.Decode(&data)
 503
 504         for _, v := range data.Objects {
 505                 if v.Person.Firstname + v.Person.Lastname == "JohnKennedy" {
 506                         extrb, err := json.Marshal(v.Person)
 507                         if err != nil {
 508                                 return "",err
 509                         }
 510                         extr := "{\"person\":" + strings.Replace(string(extrb), "\\u0026", "&", -1) + "}"
 511                         out += extr + ","
 512                 }
 513         }
 514
 515         return out, err
 516 }
 517
 518 func readJsonComplicateConcat(fileName string) (string, error){
 519         var out string
 520         var data JsonSt
 521         file, err := os.Open(fileName)
 522         if err != nil {
 523                 return "",err
 524         }
 525         decoder := json.NewDecoder(file)
 526         err = decoder.Decode(&data)
 527
 528         for _, v := range data.Objects {
 529                 if v.Startdate > "2017-01-01" && v.SenatorRank == "junior" ||
 530                         v.State == "CA" && v.Party == "Repulican" {
 531                         cn := "["
 532                         for _,vv := range v.CongressNumbers {
 533                                 cn += strconv.Itoa(vv) + ","
 534                         }
 535                         cn = cn[:len(cn)-1] + "]"
 536                         lints := "{\"firstname\":\"" + v.Person.Firstname + "\",\"lastname\":\"" + v.Person.Lastname + "\",\"congress_numbers\":" + cn + "}"
 537                         out += lints + ","
 538                 }
 539         }
 540
 541         return out, err
 542 }