src/com/ranfa/lib/Scraping.java

   1 package com.ranfa.lib;
   2
   3 import java.io.File;
   4 import java.io.IOException;
   5 import java.nio.file.Files;
   6 import java.nio.file.Path;
   7 import java.nio.file.Paths;
   8 import java.util.ArrayList;
   9 import java.util.concurrent.CompletableFuture;
  10 import java.util.concurrent.ExecutionException;
  11 import java.util.concurrent.ExecutorService;
  12 import java.util.concurrent.Executors;
  13
  14 import javax.swing.JOptionPane;
  15
  16 import org.jsoup.Jsoup;
  17 import org.jsoup.nodes.Document;
  18 import org.jsoup.select.Elements;
  19 import org.slf4j.Logger;
  20 import org.slf4j.LoggerFactory;
  21
  22 import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
  23 import com.fasterxml.jackson.databind.ObjectMapper;
  24 import com.fasterxml.jackson.databind.ObjectWriter;
  25 import com.ranfa.lib.AlbumTypeEstimate.MASTERPLUS_TYPE;
  26 import com.ranfa.main.Messages;
  27
  28 public class Scraping {
  29
  30         private final static String URI = "https://imascg-slstage-wiki.gamerch.com/%E6%A5%BD%E6%9B%B2%E8%A9%B3%E7%B4%B0%E4%B8%80%E8%A6%A7";
  31         private final static String DBPATH = "generated/database.json";
  32         private static Logger logger = LoggerFactory.getLogger(Scraping.class);
  33         public final static String NONSELECTED = "指定なし";
  34         public final static String ALL = "全タイプ";
  35         public final static String CUTE = "キュート";
  36         public final static String COOL = "クール";
  37         public final static String PASSION = "パッション";
  38         public final static String DEBUT = "DEBUT";
  39         public final static String REGULAR = "REGULAR";
  40         public final static String PRO = "PRO";
  41         public final static String MASTER = "MASTER";
  42         public final static String MASTERPLUS = "MASTER+";
  43         public final static String LEGACYMASTERPLUS = "ⓁMASTER+";
  44         public final static String LIGHT = "LIGHT";
  45         public final static String TRICK = "TRICK";
  46         public final static String PIANO = "PIANO";
  47         public final static String FORTE = "FORTE";
  48         public final static String WITCH = "WITCH";
  49
  50         public static boolean databaseExists() {
  51                 Path path = Paths.get(DBPATH);
  52                 return Files.exists(path);
  53         }
  54
  55         public static String getDBPath() {
  56                 return DBPATH;
  57         }
  58
  59         public static ArrayList<Song> getWholeData() {
  60                 long time = System.currentTimeMillis();
  61                 ExecutorService es = Executors.newWorkStealingPool();
  62                 CompletableFuture<ArrayList<ArrayList<Album>>> typeFetchFuture = CompletableFuture.supplyAsync(() -> AlbumTypeEstimate.getAlbumType(), es);
  63                 // if(databaseExists())
  64                 //      return null;
  65                 ArrayList<Song> res = new ArrayList<>();
  66                 try {
  67
  68                         Document document = Jsoup.connect(URI)
  69                                         .userAgent("Java/DeresteRandomSelector  More information is available at https://github.com/hizumiaoba/DeresteRandomSelector/")
  70                                         .maxBodySize(0)
  71                                         .timeout(0)
  72                                         .get();
  73                         Elements rows = document.getElementsByTag("tbody").get(0).select("tr");
  74                         ArrayList<ArrayList<Album>> typeLists = typeFetchFuture.get();
  75                         for(int i = 0; i < rows.size(); i++) {
  76                                 String attribute = rows.get(i).select("td").get(0).text();
  77                                 String name = rows.get(i).select("td").get(1).text();
  78                                 String difficulty = rows.get(i).select("td").get(2).text();
  79                                 int level = Integer.parseInt(rows.get(i).select("td").get(3).text());
  80                                 int notes = 0;
  81                                 if(rows.get(i).select("td").get(5).text().indexOf(",") == -1) {
  82                                         notes = Integer.parseInt(rows.get(i).select("td").get(5).text());
  83                                 } else {
  84                                         String temp = rows.get(i).select("td").get(5).text();
  85                                         String first = temp.substring(0, temp.indexOf(","));
  86                                         String end = temp.substring(temp.indexOf(",") + 1);
  87                                         notes = Integer.parseInt(first + end);
  88                                 }
  89                                 Song tmp = new Song();
  90                                 tmp.setAttribute(attribute);
  91                                 tmp.setName(name);
  92                                 tmp.setDifficulty(difficulty);
  93                                 tmp.setLevel(level);
  94                                 tmp.setNotes(notes);
  95                                 if(difficulty.equals(LEGACYMASTERPLUS)) {
  96                                         ArrayList<Album> newTypeList = typeLists.get(MASTERPLUS_TYPE.LEGACYMASTERPLUS.ordinal());
  97                                         newTypeList.stream()
  98                                         .filter(element -> element.getSongName().equals(name))
  99                                         .forEach(element -> {
 100                                                 tmp.setAlbumType(element.getAlbumType());
 101                                         });
 102                                 } else if(difficulty.equals(MASTERPLUS)) {
 103                                         ArrayList<Album> legacyTypeList = typeLists.get(MASTERPLUS_TYPE.NEWMASTERPLUS.ordinal());
 104                                         legacyTypeList.stream()
 105                                         .filter(element -> element.getSongName().equals(name))
 106                                         .forEach(element -> {
 107                                                 tmp.setAlbumType(element.getAlbumType());
 108                                         });
 109                                 } else {
 110                                         tmp.setAlbumType("Not-Implemented");
 111                                 }
 112                                 res.add(tmp);
 113                         }
 114                 } catch (IOException | InterruptedException | ExecutionException e) {
 115                         logger.error("Exception was thrown during web scraping", e);
 116                 }
 117                 logger.info("scraping compeleted in {} ms", (System.currentTimeMillis() - time));
 118                 return res;
 119         }
 120
 121         public static ArrayList<Song> getSpecificAttributeSongs(ArrayList<Song> data, String attribute) {
 122                 if(!attribute.equals(ALL)
 123                                 && !attribute.equals(CUTE)
 124                                 && !attribute.equals(COOL)
 125                                 && !attribute.equals(PASSION)
 126                                 && !attribute.equals(Messages.MSGNonSelected.toString())) {
 127                         throw new IllegalArgumentException("Illegal attribute value: " + attribute);
 128                 }
 129                 if(data.isEmpty()) {
 130                         JOptionPane.showMessageDialog(null, "指定された属性の曲は存在しません。\n条件を変えてみてください");
 131                         throw new IllegalArgumentException("ArrayList must not empty.");
 132                 }
 133                 ArrayList<Song> res = new ArrayList<>();
 134                 if(attribute.equals(Messages.MSGNonSelected.toString())) {
 135                         res = data;
 136                 } else {
 137                         data.stream()
 138                         .filter(element -> element.getAttribute().equals(attribute))
 139                         .forEach(res::add);
 140                 }
 141                 return res;
 142         }
 143
 144         public static  ArrayList<Song> getSpecificDifficultySongs(ArrayList<Song> data, String difficulty) {
 145                 if(!difficulty.equals(DEBUT)
 146                                 && !difficulty.equals(REGULAR)
 147                                 && !difficulty.equals(PRO)
 148                                 && !difficulty.equals(MASTER)
 149                                 && !difficulty.equals(MASTERPLUS)
 150                                 && !difficulty.equals(LEGACYMASTERPLUS)
 151                                 && !difficulty.equals(LIGHT)
 152                                 && !difficulty.equals(TRICK)
 153                                 && !difficulty.equals(PIANO)
 154                                 && !difficulty.equals(FORTE)
 155                                 && !difficulty.equals(WITCH)
 156                                 && !difficulty.equals(Messages.MSGNonSelected.toString())) {
 157                         throw new IllegalArgumentException("Illegal difficulty value.");
 158                 }
 159                 if(data.isEmpty()) {
 160                         throw new IllegalArgumentException("ArrayList must not empty.");
 161                 }
 162                 ArrayList<Song> res = new ArrayList<>();
 163                 if(difficulty.equals(Messages.MSGNonSelected.toString())) {
 164                         res = data;
 165                 } else {
 166                         data.stream()
 167                         .filter(element -> element.getDifficulty().equals(difficulty))
 168                         .forEach(res::add);
 169                 }
 170                 return res;
 171         }
 172
 173         public static  ArrayList<Song> getSpecificLevelSongs(ArrayList<Song> data, int level, boolean isLess, boolean isMore) {
 174                 if(level <= 0) {
 175                         throw new IllegalArgumentException("Level must not negative.");
 176                 }
 177                 if(data.isEmpty()) {
 178                         throw new IllegalArgumentException("ArrayList must not empty.");
 179                 }
 180                 if(!(isLess || isMore)) {
 181                         throw new IllegalArgumentException("Illegal boolean value.");
 182                 }
 183                 if(isLess && isMore) {
 184                         return getOnlyLevelSongs(data, level);
 185                 }
 186                 ArrayList<Song> res = new ArrayList<>();
 187                 if(isLess) {
 188                         data.stream()
 189                         .filter(element -> element.getLevel() < level)
 190                         .forEach(res::add);
 191                 } else if (isMore) {
 192                         data.stream()
 193                         .filter(element -> element.getLevel() > level)
 194                         .forEach(res::add);
 195                 }
 196                 return res;
 197         }
 198
 199         public static ArrayList<Song> getSpecificAlbumTypeSongs(ArrayList<Song> data, String type) {
 200                 if(type == null) {
 201                         throw new IllegalArgumentException("type must not null.");
 202                 }
 203                 if(data.isEmpty()) {
 204                         throw new IllegalArgumentException("ArrayList must not empty");
 205                 }
 206                 ArrayList<Song> res = new ArrayList<>();
 207                 data.stream()
 208                 .filter(element -> element.getAlbumType().equals(type))
 209                 .forEach(res::add);
 210                 return res;
 211         }
 212
 213         private static ArrayList<Song> getOnlyLevelSongs(ArrayList<Song> data, int level) {
 214                 ArrayList<Song> res = new ArrayList<>();
 215                 data.stream()
 216                 .filter(element -> element.getLevel() == level)
 217                 .forEach(res::add);
 218                 return res;
 219
 220         }
 221
 222         public static ArrayList<Song> getFromJson() {
 223                 long time = System.currentTimeMillis();
 224                 SongJSONProperty property = null;
 225                 try {
 226                         property = new ObjectMapper().readValue(new File(DBPATH), SongJSONProperty.class);
 227                 } catch (IOException e) {
 228                         logger.error("IOException was thrown during reading From JSON Database file.", e);
 229                 }
 230                 ArrayList<Song> res = new ArrayList<>();
 231                 if(property != null) {
 232                         res.addAll(property.getList());
 233                 } else {
 234                         throw new NullPointerException("json is null.");
 235                 }
 236                 logger.info("JSON reading compeleted in {} ms", (System.currentTimeMillis() - time));
 237                 return res;
 238         }
 239
 240         public static boolean writeToJson(ArrayList<Song> list) {
 241                 boolean res = true;
 242                 SongJSONProperty property = new SongJSONProperty();
 243                 property.setList(list);
 244                 ObjectWriter writer = new ObjectMapper().writer(new DefaultPrettyPrinter());
 245                 try {
 246                         if(Files.notExists(Paths.get("generated"))) {
 247                                 Files.createDirectory(Paths.get("generated"));
 248                         }
 249                         writer.writeValue(Paths.get(DBPATH).toFile(), property);
 250                 } catch (IOException e) {
 251                         logger.error("IOException was thrown during writing to JSON database file.", e);
 252                         res = false;
 253                 }
 254                 return res;
 255         }
 256 }