From: nomeu Date: Sun, 25 Sep 2011 16:50:45 +0000 (+0900) Subject: removed 3dcustom X-Git-Url: http://git.osdn.net/view?p=tdcgexplorer%2Fnimono-crawlers.git;a=commitdiff_plain;h=2bc2be5108633018ae601caf0e99d897546771f9 removed 3dcustom --- diff --git a/bin/3ch-idx.rb b/bin/3ch-idx.rb deleted file mode 100644 index 642859a..0000000 --- a/bin/3ch-idx.rb +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/ruby -# download from 3ch -# http://www.esc-j.net/tech-arts/ta3dc/t1a931d9c1s9.html - -$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib') -require 'sn_uploader' - -def each_fileno(text) - file_href = 'http://www.esc-j.net/tech-arts/ta3dc/upload.cgi?mode=dl&file=' - file_re = Regexp.new(Regexp.escape(file_href) + '(\d+)') - text.scan(/href="(.+?)"/) do |href, | - if md = file_re.match(href) - yield md[1] - end - end -end - -uploader = SnUploader.new -uploader.host = "www.esc-j.net" -uploader.root_path = "/tech-arts/ta3dc" -uploader.base_html = "t1a931d9c1s9.html" -uploader.kcode = 'U' -uploader.authorization = "Basic " + ["tech:mybride"].pack('m').chomp -uploader.local_dir = '/Volumes/uploader/arc/3ch' -body = uploader.get_base -each_fileno(body) do |fileno| - prefix = (fileno.to_i > 735) ? "3DCH" : "TA3CH" - basename_without_extension = "#{prefix}%04d" % fileno.to_i - filename_re = Regexp.new('\A' + Regexp.escape(basename_without_extension) + '\.') - exist = uploader.local_file_match?(filename_re) - puts [ fileno, exist ? 'exist' : 'download' ].join("\t") - uploader.download(fileno) unless exist -end diff --git a/bin/crawl.sh b/bin/crawl.sh index 0f09428..21cb985 100755 --- a/bin/crawl.sh +++ b/bin/crawl.sh @@ -1,17 +1,7 @@ #!/bin/sh -ruby bin/3ch-idx.rb -ruby bin/wpc-idx.rb -ruby bin/xpc-idx.rb -ruby bin/xpp-idx.rb - ruby bin/mmd-idx.rb ruby bin/mmdfile-idx.rb # ruby bin/mmdacc1-idx.rb # ruby bin/mmdacc2-idx.rb ruby bin/mmd3dcu-idx.rb ruby bin/mmdbowl-idx.rb - -ruby bin/hnt-idx.rb -ruby bin/mod-idx.rb -ruby bin/pnt-idx.rb -ruby bin/tim-idx.rb diff --git a/bin/hnt-idx.rb b/bin/hnt-idx.rb deleted file mode 100644 index 3c26e97..0000000 --- a/bin/hnt-idx.rb +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/ruby -# download from futabacustom save -# http://www.nijibox5.com/futabacustom/save/ - -$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib') -require 'sn_uploader' - -def each_filename(text) - file_href = './src/' - file_re = Regexp.new(Regexp.escape(file_href) + '(.+?)\.html') - text.scan(/href="(.+?)"/) do |href, | - if md = file_re.match(href) - yield md[1] - end - end -end - -uploader = SnUploader.new -uploader.host = "www.nijibox5.com" -uploader.root_path = "/futabacustom/save" -uploader.base_html = "" -uploader.local_dir = '/Volumes/uploader/arc/hnt' -body = uploader.get_base -each_filename(body) do |filename| - exist = uploader.local_file_exist?(filename) - puts [ filename, exist ? 'exist' : 'download' ].join("\t") - uploader.download_file(filename) unless exist -end diff --git a/bin/mod-idx.rb b/bin/mod-idx.rb deleted file mode 100644 index 074c3b2..0000000 --- a/bin/mod-idx.rb +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/ruby -# download from futabacustom mod -# http://www.nijibox5.com/futabacustom/mod/ - -$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib') -require 'sn_uploader' - -def each_filename(text) - file_href = './src/' - file_re = Regexp.new(Regexp.escape(file_href) + '(.+?)\.html') - text.scan(/href="(.+?)"/) do |href, | - if md = file_re.match(href) - yield md[1] - end - end -end - -uploader = SnUploader.new -uploader.host = "www.nijibox5.com" -uploader.root_path = "/futabacustom/mod" -uploader.base_html = "" -uploader.local_dir = '/Volumes/uploader/arc/mod' -body = uploader.get_base -each_filename(body) do |filename| - exist = uploader.local_file_exist?(filename) - puts [ filename, exist ? 'exist' : 'download' ].join("\t") - uploader.download_file(filename) unless exist -end diff --git a/bin/pnt-idx.rb b/bin/pnt-idx.rb deleted file mode 100644 index 4bf4b2f..0000000 --- a/bin/pnt-idx.rb +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/ruby -# download from futabacustom pose -# http://www.nijibox5.com/futabacustom/pose/ - -$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib') -require 'sn_uploader' - -def each_filename(text) - file_href = './src/' - file_re = Regexp.new(Regexp.escape(file_href) + '(.+?)\.html') - text.scan(/href="(.+?)"/) do |href, | - if md = file_re.match(href) - yield md[1] - end - end -end - -uploader = SnUploader.new -uploader.host = "www.nijibox5.com" -uploader.root_path = "/futabacustom/pose" -uploader.base_html = "" -uploader.local_dir = '/Volumes/uploader/arc/pnt' -body = uploader.get_base -each_filename(body) do |filename| - exist = uploader.local_file_exist?(filename) - puts [ filename, exist ? 'exist' : 'download' ].join("\t") - uploader.download_file(filename) unless exist -end diff --git a/bin/tap-idx.rb.removed b/bin/tap-idx.rb.removed deleted file mode 100644 index 87ae479..0000000 --- a/bin/tap-idx.rb.removed +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/ruby -# download from 3ch pose -# http://www.esc-j.net/tech-arts/pose/ - -$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib') -require 'sn_uploader' - -def each_filename(text) - file_href = 'http://www.esc-j.net/tech-arts/pose/src/' - file_re = Regexp.new(Regexp.escape(file_href) + '(.+?)\.html') - text.scan(/href="(.+?)"/) do |href, | - if md = file_re.match(href) - yield md[1] - end - end -end - -uploader = SnUploader.new -uploader.host = "www.esc-j.net" -uploader.root_path = "/tech-arts/pose" -uploader.base_html = "" -uploader.local_dir = '/Volumes/uploader/arc/tap' -body = uploader.get_base -each_filename(body) do |filename| - exist = uploader.local_file_exist?(filename) - puts [ filename, exist ? 'exist' : 'download' ].join("\t") - uploader.download_file(filename) unless exist -end diff --git a/bin/tim-idx.rb b/bin/tim-idx.rb deleted file mode 100644 index fe5d7fa..0000000 --- a/bin/tim-idx.rb +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/ruby -# download from futabacustom time -# http://www.nijibox5.com/futabacustom/time/ - -$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib') -require 'sn_uploader' - -def each_filename(text) - file_href = './src/' - file_re = Regexp.new(Regexp.escape(file_href) + '(.+?)\.html') - text.scan(/href="(.+?)"/) do |href, | - if md = file_re.match(href) - yield md[1] - end - end -end - -uploader = SnUploader.new -uploader.host = "www.nijibox5.com" -uploader.root_path = "/futabacustom/time" -uploader.base_html = "" -uploader.local_dir = '/Volumes/uploader/arc/tim' -body = uploader.get_base -each_filename(body) do |filename| - exist = uploader.local_file_exist?(filename) - puts [ filename, exist ? 'exist' : 'download' ].join("\t") - uploader.download_file(filename) unless exist -end diff --git a/bin/uppervolta-3d.net-all.rb b/bin/uppervolta-3d.net-all.rb deleted file mode 100644 index fda39c1..0000000 --- a/bin/uppervolta-3d.net-all.rb +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/ruby -# download from uppervolta-3d.net -# -$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib') -require 'sn_uploader' - -def each_filename(text) - file_href = './src/' - file_re = Regexp.new(Regexp.escape(file_href) + '(.+?)\.html') - text.scan(/href="(.+?)"/) do |href, | - if md = file_re.match(href) - yield md[1] - end - end -end - -name = ARGV.shift || 'kiss' - -uploader = SnUploader.new -uploader.host = "uppervolta-3d.net" -uploader.root_path = "/" + name -uploader.base_html = "all.html" -uploader.local_dir = "/Volumes/uploader/arc/uppervolta-3d.net/" + name - -src_path = "/Volumes/uploader/src/uppervolta-3d.net/" + name + "/all.html" - -if false - # read index.html from remote site - body = uploader.get_base - # save index.html as local cache - open(src_path, 'wb') { |f| f.write body } -end - -# read index.html from local cache -body = IO.read(src_path) - -each_filename(body) do |filename| - exist = uploader.local_file_exist?(filename) - puts [ filename, exist ? 'exist' : 'download' ].join("\t") - unless exist - sleep(5) - uploader.download_file_through_clicker(filename) - end -end diff --git a/bin/uppervolta-3d.net.rb b/bin/uppervolta-3d.net.rb deleted file mode 100644 index ec3b201..0000000 --- a/bin/uppervolta-3d.net.rb +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/ruby -# download from uppervolta-3d.net -# -$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib') -require 'sn_uploader' - -def each_filename(text) - file_href = './src/' - file_re = Regexp.new(Regexp.escape(file_href) + '(.+?)\.html') - text.scan(/href="(.+?)"/) do |href, | - if md = file_re.match(href) - yield md[1] - end - end -end - -name = ARGV.shift || 'kiss' - -uploader = SnUploader.new -uploader.host = "uppervolta-3d.net" -uploader.root_path = "/" + name -uploader.base_html = "" -uploader.local_dir = "/Volumes/uploader/arc/uppervolta-3d.net/" + name -body = uploader.get_base -each_filename(body) do |filename| - exist = uploader.local_file_exist?(filename) - puts [ filename, exist ? 'exist' : 'download' ].join("\t") - unless exist - sleep(1) - uploader.download_file_through_clicker(filename) - end -end diff --git a/bin/wpc-idx.rb b/bin/wpc-idx.rb deleted file mode 100644 index 90746e1..0000000 --- a/bin/wpc-idx.rb +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/ruby -# download from 3DCG Craftsmen's Guild -# http://3dcustom.ath.cx/wordpress/wp-content/uploader/upload.html - -$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib') -require 'sn_uploader' - -def each_fileno(text) - file_href = './upload.cgi?mode=dl&file=' - file_re = Regexp.new(Regexp.escape(file_href) + '(\d+)') - text.scan(/href="(.+?)"/) do |href, | - if md = file_re.match(href) - yield md[1] - end - end -end - -uploader = SnUploader.new -uploader.host = "3dcustom.ath.cx" -uploader.root_path = "/uploader/mod" -uploader.base_html = "upload.html" -uploader.local_dir = '/Volumes/uploader/arc/wpc' -body = uploader.get_base -each_fileno(body) do |fileno| - basename_without_extension = "MODS%04d" % fileno.to_i - filename_re = Regexp.new('\A' + Regexp.escape(basename_without_extension) + '\.') - exist = uploader.local_file_match?(filename_re) - puts [ fileno, exist ? 'exist' : 'download' ].join("\t") - uploader.download(fileno) unless exist -end diff --git a/bin/xpc-idx.rb b/bin/xpc-idx.rb deleted file mode 100644 index 311f6f1..0000000 --- a/bin/xpc-idx.rb +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/ruby -# download from 3dcustom.net -# http://3dcustom.net/TACuploader/upload.html - -$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib') -require 'sn_uploader' - -def each_fileno(text) - file_href = './upload.cgi?mode=dl&file=' - file_re = Regexp.new(Regexp.escape(file_href) + '(\d+)') - text.scan(/href="(.+?)"/) do |href, | - if md = file_re.match(href) - yield md[1] - end - end -end - -uploader = SnUploader.new -uploader.host = "cdn.3dcustom.net" -uploader.root_path = "/TACuploader" -uploader.base_html = "upload.html" -uploader.local_dir = '/Volumes/uploader/arc/xpc' -body = uploader.get_base -each_fileno(body) do |fileno| - basename_without_extension = "XPC%05d" % fileno.to_i - filename_re = Regexp.new('\A' + Regexp.escape(basename_without_extension) + '\.') - exist = uploader.local_file_match?(filename_re) - puts [ fileno, exist ? 'exist' : 'download' ].join("\t") - uploader.download(fileno) unless exist -end diff --git a/bin/xpp-idx.rb b/bin/xpp-idx.rb deleted file mode 100644 index 5a478f2..0000000 --- a/bin/xpp-idx.rb +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/ruby -# download from 3dcustom.net pose -# -$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib') -require 'sn_uploader' - -def each_filename(text) - file_href = './src/' - file_re = Regexp.new(Regexp.escape(file_href) + '(.+?)\.html') - text.scan(/href="(.+?)"/) do |href, | - if md = file_re.match(href) - yield md[1] - end - end -end - -uploader = SnUploader.new -uploader.host = "cdn.3dcustom.net" -uploader.root_path = "/TAPuploader" -uploader.base_html = "upload.html" -uploader.local_dir = '/Volumes/uploader/arc/xpp' -body = uploader.get_base -each_filename(body) do |filename| - exist = uploader.local_file_exist?(filename) - puts [ filename, exist ? 'exist' : 'download' ].join("\t") - uploader.download_file(filename) unless exist -end