From 19bd0d192ddae6dfc879fb0c139e2d56c994ec77 Mon Sep 17 00:00:00 2001 From: moriq Date: Thu, 10 Jun 2010 03:35:14 +0900 Subject: [PATCH] added sn_uploader --- bin/3ch-idx.rb | 32 ++++++++++ bin/hnt-idx.rb | 28 +++++++++ bin/mmd-idx.rb | 28 +++++++++ bin/mmdacc1-idx.rb | 28 +++++++++ bin/mmdacc2-idx.rb | 28 +++++++++ bin/mmdfile-idx.rb | 25 ++++++++ bin/mod-idx.rb | 28 +++++++++ bin/pnt-idx.rb | 28 +++++++++ bin/tap-idx.rb | 28 +++++++++ bin/tim-idx.rb | 28 +++++++++ bin/wpc-idx.rb | 30 ++++++++++ bin/xpc-idx.rb | 30 ++++++++++ bin/xpp-idx.rb | 27 +++++++++ lib/sn_uploader.rb | 170 +++++++++++++++++++++++++++++++++++++++++++++++++++++ lib/user_agent.rb | 11 ++++ 15 files changed, 549 insertions(+) create mode 100755 bin/3ch-idx.rb create mode 100755 bin/hnt-idx.rb create mode 100755 bin/mmd-idx.rb create mode 100755 bin/mmdacc1-idx.rb create mode 100755 bin/mmdacc2-idx.rb create mode 100755 bin/mmdfile-idx.rb create mode 100755 bin/mod-idx.rb create mode 100755 bin/pnt-idx.rb create mode 100755 bin/tap-idx.rb create mode 100755 bin/tim-idx.rb create mode 100755 bin/wpc-idx.rb create mode 100755 bin/xpc-idx.rb create mode 100755 bin/xpp-idx.rb create mode 100755 lib/sn_uploader.rb create mode 100755 lib/user_agent.rb diff --git a/bin/3ch-idx.rb b/bin/3ch-idx.rb new file mode 100755 index 0000000..e0524b2 --- /dev/null +++ b/bin/3ch-idx.rb @@ -0,0 +1,32 @@ +#!/usr/bin/ruby +# download from 3ch +# http://www.esc-j.net/tech-arts/ta3dc/t1a931d9c1s9.html + +$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib') +require 'sn_uploader' + +def each_fileno(text) + file_href = 'http://www.esc-j.net/tech-arts/ta3dc/upload.cgi?mode=dl&file=' + file_re = Regexp.new(Regexp.escape(file_href) + '(\d+)') + text.scan(/href="(.+?)"/) do |href, | + if md = file_re.match(href) + yield md[1] + end + end +end + +uploader = SnUploader.new +uploader.host = "www.esc-j.net" +uploader.root_path = "/tech-arts/ta3dc" +uploader.base_html = "t1a931d9c1s9.html" +uploader.kcode = 'U' +uploader.authorization = "Basic " + ["tech:mybride"].pack('m').chomp +uploader.local_dir = File.dirname(__FILE__) + '/../arc/3ch' +body = uploader.get_base +each_fileno(body) do |fileno| + basename_without_extension = "TA3CH%04d" % fileno.to_i + filename_re = Regexp.new('\A' + Regexp.escape(basename_without_extension) + '\.') + exist = uploader.local_file_match?(filename_re) + puts [ fileno, exist ? 'exist' : 'download' ].join("\t") + uploader.download(fileno) unless exist +end diff --git a/bin/hnt-idx.rb b/bin/hnt-idx.rb new file mode 100755 index 0000000..caf089a --- /dev/null +++ b/bin/hnt-idx.rb @@ -0,0 +1,28 @@ +#!/usr/bin/ruby +# download from futabacustom save +# http://www.nijibox5.com/futabacustom/save/ + +$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib') +require 'sn_uploader' + +def each_filename(text) + file_href = './src/' + file_re = Regexp.new(Regexp.escape(file_href) + '(.+?)\.html') + text.scan(/href="(.+?)"/) do |href, | + if md = file_re.match(href) + yield md[1] + end + end +end + +uploader = SnUploader.new +uploader.host = "www.nijibox5.com" +uploader.root_path = "/futabacustom/save" +uploader.base_html = "" +uploader.local_dir = File.dirname(__FILE__) + '/../arc/hnt' +body = uploader.get_base +each_filename(body) do |filename| + exist = uploader.local_file_exist?(filename) + puts [ filename, exist ? 'exist' : 'download' ].join("\t") + uploader.download_file(filename) unless exist +end diff --git a/bin/mmd-idx.rb b/bin/mmd-idx.rb new file mode 100755 index 0000000..bf054d9 --- /dev/null +++ b/bin/mmd-idx.rb @@ -0,0 +1,28 @@ +#!/usr/bin/ruby +# download from mikumikudance +# http://bytatsu.net/uploader/mikumikudance/index.html + +$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib') +require 'sn_uploader' + +def each_filename(text) + file_href = './src/' + file_re = Regexp.new(Regexp.escape(file_href) + '(.+)') + text.scan(/href="(.+?)"/) do |href, | + if md = file_re.match(href) + yield md[1] + end + end +end + +uploader = SnUploader.new +uploader.host = "bytatsu.net" +uploader.root_path = "/uploader/mikumikudance" +uploader.base_html = "index.html" +uploader.local_dir = File.dirname(__FILE__) + '/../arc/mmd' +body = uploader.get_base +each_filename(body) do |filename| + exist = uploader.local_file_exist?(filename) + puts [ filename, exist ? 'exist' : 'download' ].join("\t") + uploader.download_file_directly(filename) unless exist +end diff --git a/bin/mmdacc1-idx.rb b/bin/mmdacc1-idx.rb new file mode 100755 index 0000000..0fda001 --- /dev/null +++ b/bin/mmdacc1-idx.rb @@ -0,0 +1,28 @@ +#!/usr/bin/ruby +# download from mikumikudance +# http://www9.atpages.jp/~mmdaccessory/uploader/index.php + +$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib') +require 'sn_uploader' + +def each_filename(text) + file_href = './log/' + file_re = Regexp.new(Regexp.escape(file_href) + '(.+)') + text.scan(/href='(.+?)'/) do |href, | + if md = file_re.match(href) + yield md[1] + end + end +end + +uploader = SnUploader.new +uploader.host = "www9.atpages.jp" +uploader.root_path = "/~mmdaccessory/uploader" +uploader.local_dir = File.dirname(__FILE__) + '/../arc/mmdacc1' + uploader.base_html = "index.php" + body = uploader.get_base + each_filename(body) do |filename| + exist = uploader.local_file_exist?(filename) + puts [ filename, exist ? 'exist' : 'download' ].join("\t") + uploader.download_file_directly(filename, '/log') unless exist + end diff --git a/bin/mmdacc2-idx.rb b/bin/mmdacc2-idx.rb new file mode 100755 index 0000000..7117149 --- /dev/null +++ b/bin/mmdacc2-idx.rb @@ -0,0 +1,28 @@ +#!/usr/bin/ruby +# download from mikumikudance +# http://www9.atpages.jp/~mmdaccessory/uploader2/index.html + +$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib') +require 'sn_uploader' + +def each_filename(text) + file_href = 'http://www9.atpages.jp/~mmdaccessory/uploader2/src/' + file_re = Regexp.new(Regexp.escape(file_href) + '(.+)\.html') + text.scan(/href="(.+?)"/) do |href, | + if md = file_re.match(href) + yield md[1] + end + end +end + +uploader = SnUploader.new +uploader.host = "www9.atpages.jp" +uploader.root_path = "/~mmdaccessory/uploader2" +uploader.local_dir = File.dirname(__FILE__) + '/../arc/mmdacc2' + uploader.base_html = "index.html" + body = uploader.get_base + each_filename(body) do |filename| + exist = uploader.local_file_exist?(filename) + puts [ filename, exist ? 'exist' : 'download' ].join("\t") + uploader.download_file_through_counter(filename) unless exist + end diff --git a/bin/mmdfile-idx.rb b/bin/mmdfile-idx.rb new file mode 100755 index 0000000..a3f57fc --- /dev/null +++ b/bin/mmdfile-idx.rb @@ -0,0 +1,25 @@ +#!/usr/bin/ruby +# download from mikumikudance +# http://loda.jp/mmdfile/ + +$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib') +require 'sn_uploader' + +def each_filename(text) + text.scan(%r((.+?))) do |href, name | + yield name + end +end + +uploader = SnUploader.new +uploader.host = "loda.jp" +uploader.root_path = "/mmdfile" +uploader.local_dir = '/Volumes/uploader/arc/mmdfile' + uploader.base_html = "/?page=1" + body = uploader.get_base + open('/Volumes/uploader/src/mmdfile/1.html', 'w') { |o| o.puts body } + each_filename(body) do |filename| + exist = uploader.local_file_exist?(filename) + puts [ filename, exist ? 'exist' : 'download' ].join("\t") + uploader.download_file_directly_id(filename) + end diff --git a/bin/mod-idx.rb b/bin/mod-idx.rb new file mode 100755 index 0000000..a736f96 --- /dev/null +++ b/bin/mod-idx.rb @@ -0,0 +1,28 @@ +#!/usr/bin/ruby +# download from futabacustom mod +# http://www.nijibox5.com/futabacustom/mod/ + +$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib') +require 'sn_uploader' + +def each_filename(text) + file_href = './src/' + file_re = Regexp.new(Regexp.escape(file_href) + '(.+?)\.html') + text.scan(/href="(.+?)"/) do |href, | + if md = file_re.match(href) + yield md[1] + end + end +end + +uploader = SnUploader.new +uploader.host = "www.nijibox5.com" +uploader.root_path = "/futabacustom/mod" +uploader.base_html = "" +uploader.local_dir = File.dirname(__FILE__) + '/../arc/mod' +body = uploader.get_base +each_filename(body) do |filename| + exist = uploader.local_file_exist?(filename) + puts [ filename, exist ? 'exist' : 'download' ].join("\t") + uploader.download_file(filename) unless exist +end diff --git a/bin/pnt-idx.rb b/bin/pnt-idx.rb new file mode 100755 index 0000000..f8a39b2 --- /dev/null +++ b/bin/pnt-idx.rb @@ -0,0 +1,28 @@ +#!/usr/bin/ruby +# download from futabacustom pose +# http://www.nijibox5.com/futabacustom/pose/ + +$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib') +require 'sn_uploader' + +def each_filename(text) + file_href = './src/' + file_re = Regexp.new(Regexp.escape(file_href) + '(.+?)\.html') + text.scan(/href="(.+?)"/) do |href, | + if md = file_re.match(href) + yield md[1] + end + end +end + +uploader = SnUploader.new +uploader.host = "www.nijibox5.com" +uploader.root_path = "/futabacustom/pose" +uploader.base_html = "" +uploader.local_dir = File.dirname(__FILE__) + '/../arc/pnt' +body = uploader.get_base +each_filename(body) do |filename| + exist = uploader.local_file_exist?(filename) + puts [ filename, exist ? 'exist' : 'download' ].join("\t") + uploader.download_file(filename) unless exist +end diff --git a/bin/tap-idx.rb b/bin/tap-idx.rb new file mode 100755 index 0000000..0a88e26 --- /dev/null +++ b/bin/tap-idx.rb @@ -0,0 +1,28 @@ +#!/usr/bin/ruby +# download from 3ch pose +# http://www.esc-j.net/tech-arts/pose/ + +$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib') +require 'sn_uploader' + +def each_filename(text) + file_href = 'http://www.esc-j.net/tech-arts/pose/src/' + file_re = Regexp.new(Regexp.escape(file_href) + '(.+?)\.html') + text.scan(/href="(.+?)"/) do |href, | + if md = file_re.match(href) + yield md[1] + end + end +end + +uploader = SnUploader.new +uploader.host = "www.esc-j.net" +uploader.root_path = "/tech-arts/pose" +uploader.base_html = "" +uploader.local_dir = File.dirname(__FILE__) + '/../arc/tap' +body = uploader.get_base +each_filename(body) do |filename| + exist = uploader.local_file_exist?(filename) + puts [ filename, exist ? 'exist' : 'download' ].join("\t") + uploader.download_file(filename) unless exist +end diff --git a/bin/tim-idx.rb b/bin/tim-idx.rb new file mode 100755 index 0000000..a75f065 --- /dev/null +++ b/bin/tim-idx.rb @@ -0,0 +1,28 @@ +#!/usr/bin/ruby +# download from futabacustom time +# http://www.nijibox5.com/futabacustom/time/ + +$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib') +require 'sn_uploader' + +def each_filename(text) + file_href = './src/' + file_re = Regexp.new(Regexp.escape(file_href) + '(.+?)\.html') + text.scan(/href="(.+?)"/) do |href, | + if md = file_re.match(href) + yield md[1] + end + end +end + +uploader = SnUploader.new +uploader.host = "www.nijibox5.com" +uploader.root_path = "/futabacustom/time" +uploader.base_html = "" +uploader.local_dir = File.dirname(__FILE__) + '/../arc/tim' +body = uploader.get_base +each_filename(body) do |filename| + exist = uploader.local_file_exist?(filename) + puts [ filename, exist ? 'exist' : 'download' ].join("\t") + uploader.download_file(filename) unless exist +end diff --git a/bin/wpc-idx.rb b/bin/wpc-idx.rb new file mode 100755 index 0000000..8a2f6f2 --- /dev/null +++ b/bin/wpc-idx.rb @@ -0,0 +1,30 @@ +#!/usr/bin/ruby +# download from 3DCG Craftsmen's Guild +# http://3dcustom.ath.cx/wordpress/wp-content/uploader/upload.html + +$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib') +require 'sn_uploader' + +def each_fileno(text) + file_href = './upload.cgi?mode=dl&file=' + file_re = Regexp.new(Regexp.escape(file_href) + '(\d+)') + text.scan(/href="(.+?)"/) do |href, | + if md = file_re.match(href) + yield md[1] + end + end +end + +uploader = SnUploader.new +uploader.host = "3dcustom.ath.cx" +uploader.root_path = "/uploader/mod" +uploader.base_html = "upload.html" +uploader.local_dir = File.dirname(__FILE__) + '/../arc/wpc' +body = uploader.get_base +each_fileno(body) do |fileno| + basename_without_extension = "MODS%04d" % fileno.to_i + filename_re = Regexp.new('\A' + Regexp.escape(basename_without_extension) + '\.') + exist = uploader.local_file_match?(filename_re) + puts [ fileno, exist ? 'exist' : 'download' ].join("\t") + uploader.download(fileno) unless exist +end diff --git a/bin/xpc-idx.rb b/bin/xpc-idx.rb new file mode 100755 index 0000000..ba68b1a --- /dev/null +++ b/bin/xpc-idx.rb @@ -0,0 +1,30 @@ +#!/usr/bin/ruby +# download from 3dcustom.net +# http://3dcustom.net/TACuploader/upload.html + +$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib') +require 'sn_uploader' + +def each_fileno(text) + file_href = './upload.cgi?mode=dl&file=' + file_re = Regexp.new(Regexp.escape(file_href) + '(\d+)') + text.scan(/href="(.+?)"/) do |href, | + if md = file_re.match(href) + yield md[1] + end + end +end + +uploader = SnUploader.new +uploader.host = "up1.3dcustom.net" +uploader.root_path = "/TACuploader" +uploader.base_html = "upload.html" +uploader.local_dir = File.dirname(__FILE__) + '/../arc/xpc' +body = uploader.get_base +each_fileno(body) do |fileno| + basename_without_extension = "XPC%05d" % fileno.to_i + filename_re = Regexp.new('\A' + Regexp.escape(basename_without_extension) + '\.') + exist = uploader.local_file_match?(filename_re) + puts [ fileno, exist ? 'exist' : 'download' ].join("\t") + uploader.download(fileno) unless exist +end diff --git a/bin/xpp-idx.rb b/bin/xpp-idx.rb new file mode 100755 index 0000000..546763f --- /dev/null +++ b/bin/xpp-idx.rb @@ -0,0 +1,27 @@ +#!/usr/bin/ruby +# download from 3dcustom.net pose +# +$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib') +require 'sn_uploader' + +def each_filename(text) + file_href = './src/' + file_re = Regexp.new(Regexp.escape(file_href) + '(.+?)\.html') + text.scan(/href="(.+?)"/) do |href, | + if md = file_re.match(href) + yield md[1] + end + end +end + +uploader = SnUploader.new +uploader.host = "3dcustom.net" +uploader.root_path = "/TAPuploader" +uploader.base_html = "upload.html" +uploader.local_dir = File.dirname(__FILE__) + '/../arc/xpp' +body = uploader.get_base +each_filename(body) do |filename| + exist = uploader.local_file_exist?(filename) + puts [ filename, exist ? 'exist' : 'download' ].join("\t") + uploader.download_file(filename) unless exist +end diff --git a/lib/sn_uploader.rb b/lib/sn_uploader.rb new file mode 100755 index 0000000..34e303b --- /dev/null +++ b/lib/sn_uploader.rb @@ -0,0 +1,170 @@ +require 'net/http' +require 'erb' +require 'nkf' +require 'user_agent' + +class SnUploader + attr_accessor :host, :root_path, :jcode, :dlkey, :authorization, :local_dir, :base_html + + def initialize + @jcode = ERB::Util.u("Š¿Žš") + @dlkey = "custom" + @authorization = nil + @local_dir = "." + @base_html = "upload.html" + end + + def kcode=(kcode) + case kcode + when /^U/i + @jcode = ERB::Util.u(NKF.nkf('-Sw', "Š¿Žš")) + end + end + + def header + header = UserAgent.default_http_header.dup + header["Authorization"] = authorization if authorization + header + end + + def local_path(path) + File.join(local_dir, File.basename(path)) + end + + def local_file_exist?(filename) + File.exist?(local_path(filename)) + end + + def local_file_match?(filename_re) + Dir.foreach(local_dir) do |ent| + if md = filename_re.match(ent) + return md[0] + end + end + false + end + + def download(fileno, filename_re = nil) + if filename_re && local_file_exist?(filename_re) + return nil + end + + Net::HTTP.new(host).start do |http| + response = http.post("#{root_path}/upload.cgi", "file=#{fileno}&jcode=#{jcode}&mode=dl&dlkey=#{dlkey}", header) + body = response.body + + if md = /"1;URL=(.+?)"/.match(body) + path = md[1] + path.sub!(/\A\./, root_path) + end + puts path + if path + open(local_path(path), "wb") do |file| + response = http.get(path, header) do |data| + file.write data + end + end + end + end + end + + def download_file_through_counter(filename) + if local_file_exist?(filename) + return nil + end + + Net::HTTP.new(host).start do |http| + response = http.get("#{root_path}/src/#{filename}.html", header) + file_re = Regexp.new(Regexp.escape('count.cgi?') + '(.+)') + text = response.body + text.scan(/href="(.+?)"/) do |href, | + if md = file_re.match(href) + path = root_path + '/src/' + md[1] + open(local_path(filename), "wb") do |file| + response = http.get(path, header) do |data| + file.write data + end + end + end + end + end + end + + def download_file(filename) + if local_file_exist?(filename) + return nil + end + + Net::HTTP.new(host).start do |http| + response = http.get("#{root_path}/src/#{filename}.html", header) + body = response.body + if md = /href="(.+?)"/.match(body) + path = md[1] + path.sub!(/\A\./, root_path + "/src") + end + puts path + if path + open(local_path(path), "wb") do |file| + response = http.get(path, header) do |data| + file.write data + end + end + end + end + end + + def download_file_directly(filename, dir = '/src') + if local_file_exist?(filename) + return nil + end + + path = "#{root_path}#{dir}/#{filename}" + puts path + Net::HTTP.new(host).start do |http| + open(local_path(path), "wb") do |file| + response = http.get(path, header) do |data| + file.write data + end + end + end + end + + def download_file_directly_id(filename) + if local_file_exist?(filename) + return nil + end + + path = "#{root_path}/?id=#{filename}" + puts path + http = Net::HTTP.new(host) + http.open_timeout = 10 + http.read_timeout = 20 + begin + http.start do + open(local_path(filename), "wb") do |file| + response = http.get(path, header) do |data| + file.write data + end + end + end + rescue Timeout::Error + puts "timeout..." + sleep(20) + puts "retry" + retry + end + end + + def get_base + body = nil + Net::HTTP.new(host).start do |http| + response = http.get("#{root_path}/#{base_html}", header) + # response.each do |name, value| + # puts [name, value].join(': ') + # end + # puts + body = response.body + end + body + end +end diff --git a/lib/user_agent.rb b/lib/user_agent.rb new file mode 100755 index 0000000..b3cf55a --- /dev/null +++ b/lib/user_agent.rb @@ -0,0 +1,11 @@ +module UserAgent +module_function + def default_http_header + @_default_http_header ||= { + "User-Agent" => "Mozilla/5.0 (Windows; U; Windows NT 6.0; ja; rv:1.9.0.10) Gecko/2009042316 Firefox/3.0.10 (.NET CLR 3.5.30729)", + "Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language" => "ja,en-us;q=0.7,en;q=0.3", + "Accept-Charset" => "Shift_JIS,utf-8;q=0.7,*;q=0.7", + } + end +end -- 2.11.0