src/com/ranfa/lib/Scraping.java

   1 package com.ranfa.lib;
   2
   3 import java.io.File;
   4 import java.io.IOException;
   5 import java.nio.file.Files;
   6 import java.nio.file.Path;
   7 import java.nio.file.Paths;
   8 import java.util.ArrayList;
   9 import java.util.concurrent.CompletableFuture;
  10 import java.util.concurrent.ExecutionException;
  11 import java.util.concurrent.ExecutorService;
  12 import java.util.concurrent.Executors;
  13
  14 import javax.swing.JOptionPane;
  15
  16 import org.jsoup.Jsoup;
  17 import org.jsoup.nodes.Document;
  18 import org.jsoup.select.Elements;
  19
  20 import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
  21 import com.fasterxml.jackson.databind.ObjectMapper;
  22 import com.fasterxml.jackson.databind.ObjectWriter;
  23 import com.ranfa.lib.AlbumTypeEstimate.MASTERPLUS_TYPE;
  24
  25 public class Scraping {
  26
  27         private final static String URI = "https://imascg-slstage-wiki.gamerch.com/%E6%A5%BD%E6%9B%B2%E8%A9%B3%E7%B4%B0%E4%B8%80%E8%A6%A7";
  28         private final static String DBPATH = "generated/database.json";
  29         public final static String NONSELECTED = "指定なし";
  30         public final static String ALL = "全タイプ";
  31         public final static String CUTE = "キュート";
  32         public final static String COOL = "クール";
  33         public final static String PASSION = "パッション";
  34         public final static String DEBUT = "DEBUT";
  35         public final static String REGULAR = "REGULAR";
  36         public final static String PRO = "PRO";
  37         public final static String MASTER = "MASTER";
  38         public final static String MASTERPLUS = "MASTER+";
  39         public final static String LEGACYMASTERPLUS = "ⓁMASTER+";
  40         public final static String LIGHT = "LIGHT";
  41         public final static String TRICK = "TRICK";
  42         public final static String PIANO = "PIANO";
  43         public final static String FORTE = "FORTE";
  44         public final static String WITCH = "WITCH";
  45
  46         public static boolean databaseExists() {
  47                 Path path = Paths.get(DBPATH);
  48                 return Files.exists(path);
  49         }
  50
  51         public static String getDBPath() {
  52                 return DBPATH;
  53         }
  54
  55         public static ArrayList<Song> getWholeData() {
  56                 long time = System.currentTimeMillis();
  57                 ExecutorService es = Executors.newWorkStealingPool();
  58                 CompletableFuture<ArrayList<ArrayList<Album>>> typeFetchFuture = CompletableFuture.supplyAsync(() -> AlbumTypeEstimate.getAlbumType(), es);
  59                 // if(databaseExists())
  60                 //      return null;
  61                 ArrayList<Song> res = new ArrayList<>();
  62                 try {
  63
  64                         Document document = Jsoup.connect(URI)
  65                                         .userAgent("Java/DeresteRandomSelector  More information is available at https://github.com/hizumiaoba/DeresteRandomSelector/")
  66                                         .maxBodySize(0)
  67                                         .timeout(0)
  68                                         .get();
  69                         Elements rows = document.getElementsByTag("tbody").get(0).select("tr");
  70                         ArrayList<ArrayList<Album>> typeLists = typeFetchFuture.get();
  71                         for(int i = 0; i < rows.size(); i++) {
  72                                 String attribute = rows.get(i).select("td").get(0).text();
  73                                 String name = rows.get(i).select("td").get(1).text();
  74                                 String difficulty = rows.get(i).select("td").get(2).text();
  75                                 int level = Integer.parseInt(rows.get(i).select("td").get(3).text());
  76                                 int notes = 0;
  77                                 if(rows.get(i).select("td").get(5).text().indexOf(",") == -1) {
  78                                         notes = Integer.parseInt(rows.get(i).select("td").get(5).text());
  79                                 } else {
  80                                         String temp = rows.get(i).select("td").get(5).text();
  81                                         String first = temp.substring(0, temp.indexOf(","));
  82                                         String end = temp.substring(temp.indexOf(",") + 1);
  83                                         notes = Integer.parseInt(first + end);
  84                                 }
  85                                 Song tmp = new Song();
  86                                 tmp.setAttribute(attribute);
  87                                 tmp.setName(name);
  88                                 tmp.setDifficulty(difficulty);
  89                                 tmp.setLevel(level);
  90                                 tmp.setNotes(notes);
  91                                 if(difficulty.equals(LEGACYMASTERPLUS)) {
  92                                         ArrayList<Album> newTypeList = typeLists.get(MASTERPLUS_TYPE.LEGACYMASTERPLUS.ordinal());
  93                                         for(int j = 0; j < newTypeList.size(); j++) {
  94                                                 if(newTypeList.get(j).getSongName().equals(name))
  95                                                         tmp.setAlbumType(newTypeList.get(j).getAlbumType());
  96                                         }
  97                                 } else if(difficulty.equals(MASTERPLUS)) {
  98                                         ArrayList<Album> legacyTypeList = typeLists.get(MASTERPLUS_TYPE.NEWMASTERPLUS.ordinal());
  99                                         for(int j = 0; j < legacyTypeList.size(); j++) {
 100                                                 if(legacyTypeList.get(j).getSongName().equals(name))
 101                                                         tmp.setAlbumType(legacyTypeList.get(j).getAlbumType());
 102                                         }
 103                                 } else {
 104                                         tmp.setAlbumType("Not-Implemented");
 105                                 }
 106                                 res.add(tmp);
 107                         }
 108                 } catch (IOException | InterruptedException | ExecutionException e) {
 109                         e.printStackTrace();
 110                 }
 111                 LimitedLog.println(Scraping.class + ":[INFO]: scraping compeleted in " + (System.currentTimeMillis() - time)+ "ms");
 112                 return res;
 113         }
 114
 115         public static ArrayList<Song> getSpecificAttributeSongs(ArrayList<Song> data, String attribute) {
 116                 if(!attribute.equals(ALL)
 117                                 && !attribute.equals(CUTE)
 118                                 && !attribute.equals(COOL)
 119                                 && !attribute.equals(PASSION)
 120                                 && !attribute.equals(NONSELECTED))
 121                         throw new IllegalArgumentException("Illegal attribute value: " + attribute);
 122                 if(data.isEmpty()) {
 123                         JOptionPane.showMessageDialog(null, "指定された属性の曲は存在しません。\n条件を変えてみてください");
 124                         throw new IllegalArgumentException("ArrayList must not empty.");
 125                 }
 126                 ArrayList<Song> res = new ArrayList<Song>();
 127                 if(attribute.equals(NONSELECTED)) {
 128                         res = data;
 129                 } else {
 130                         data.stream()
 131                         .filter(element -> element.getAttribute().equals(attribute))
 132                         .forEach(res::add);
 133                 }
 134                 return res;
 135         }
 136
 137         public static  ArrayList<Song> getSpecificDifficultySongs(ArrayList<Song> data, String difficulty) {
 138                 if(!difficulty.equals(DEBUT)
 139                                 && !difficulty.equals(REGULAR)
 140                                 && !difficulty.equals(PRO)
 141                                 && !difficulty.equals(MASTER)
 142                                 && !difficulty.equals(MASTERPLUS)
 143                                 && !difficulty.equals(LEGACYMASTERPLUS)
 144                                 && !difficulty.equals(LIGHT)
 145                                 && !difficulty.equals(TRICK)
 146                                 && !difficulty.equals(PIANO)
 147                                 && !difficulty.equals(FORTE)
 148                                 && !difficulty.equals(WITCH)
 149                                 && !difficulty.equals(NONSELECTED))
 150                         throw new IllegalArgumentException("Illegal difficulty value.");
 151                 if(data.isEmpty())
 152                         throw new IllegalArgumentException("ArrayList must not empty.");
 153                 ArrayList<Song> res = new ArrayList<Song>();
 154                 if(difficulty.equals(NONSELECTED)) {
 155                         res = data;
 156                 } else {
 157                         data.stream()
 158                         .filter(element -> element.getDifficulty().equals(difficulty))
 159                         .forEach(res::add);
 160                 }
 161                 return res;
 162         }
 163
 164         public static  ArrayList<Song> getSpecificLevelSongs(ArrayList<Song> data, int level, boolean isLess, boolean isMore) {
 165                 if(level <= 0)
 166                         throw new IllegalArgumentException("Level must not negative.");
 167                 if(data.isEmpty())
 168                         throw new IllegalArgumentException("ArrayList must not empty.");
 169                 if(!(isLess || isMore))
 170                         throw new IllegalArgumentException("Illegal boolean value.");
 171                 if(isLess && isMore)
 172                         return getOnlyLevelSongs(data, level);
 173                 ArrayList<Song> res = new ArrayList<Song>();
 174                 if(isLess) {
 175                         data.stream()
 176                         .filter(element -> element.getLevel() < level)
 177                         .forEach(res::add);
 178                 } else if (isMore) {
 179                         data.stream()
 180                         .filter(element -> element.getLevel() > level)
 181                         .forEach(res::add);
 182                 }
 183                 return res;
 184         }
 185
 186         private static ArrayList<Song> getOnlyLevelSongs(ArrayList<Song> data, int level) {
 187                 ArrayList<Song> res = new ArrayList<Song>();
 188                 data.stream()
 189                 .filter(element -> element.getLevel() == level)
 190                 .forEach(res::add);
 191                 return res;
 192
 193         }
 194
 195         public static ArrayList<Song> getFromJson() {
 196                 long time = System.currentTimeMillis();
 197                 SongJSONProperty property = null;
 198                 try {
 199                         property = new ObjectMapper().readValue(new File(DBPATH), SongJSONProperty.class);
 200                 } catch (IOException e) {
 201                         // TODO 自動生成された catch ブロック
 202                         e.printStackTrace();
 203                 }
 204                 ArrayList<Song> res = new ArrayList<Song>();
 205                 res.addAll(property.getList());
 206                 LimitedLog.println(Scraping.class + ":[INFO]: JSON reading compeleted in " + (System.currentTimeMillis() - time) + "ms");
 207                 return res;
 208         }
 209
 210         public static boolean writeToJson(ArrayList<Song> list) {
 211                 boolean res = true;
 212                 SongJSONProperty property = new SongJSONProperty();
 213                 property.setList(list);
 214                 ObjectWriter writer = new ObjectMapper().writer(new DefaultPrettyPrinter());
 215                 try {
 216                         writer.writeValue(Paths.get(DBPATH).toFile(), property);
 217                 } catch (IOException e) {
 218                         res = false;
 219                 }
 220                 return res;
 221         }
 222 }