ウェブ-HACKS 「とりあえずやってみよう！」ブログでタグ「mechanize」が付けられているもの

    page = @agent.get('http://example.com')
    search_form = page.forms.with.action('http://example.com/search.html').first
    search_form.keyword-field = keyword

    page = @agent.get('http://example.com')
    search_form = page.forms.with.action('http://example.com/search.html').first
    search_form['keyword-field'] = keyword

#!/usr/bin/ruby -Ks
require 'hpricot'
require 'mechanize'
require 'kconv'
$KCODE = 's'

class Crawler
  def initialize(wait_time, internal, ngwords)
    @agent = WWW::Mechanize.new
    @wait_time = wait_time
    @internal = internal
    @ngwords = ngwords
    @links = []
  end

  def http_connect(url)
    print "OK\n"
    @page = @agent.get(url)
  end

  def crawl(root)
    @page = @agent.get(root)
    @page.links.with.href(%r{}).each{|link|
      if link.href =~ /^http/
        url = link.href
      else
        %r|http://([^/]*)/| =~ root
        url = 'http://' + $1 + '/' + link.href
      end
      @links << url.sub(/#.*$/,'')
    }
    @links.uniq!.delete(root)
    puts @links
    @links.each{|link|
      begin
        print link if $DEBUG
        next unless access_test?(link)
        http_connect(link)
        sleep @wait_time
      rescue => ex
        puts ex.message
      end
    }
  end

 private
  def access_test?(link)
    unless %r|#{@internal}| =~ link
      print "External Link.\n"
      return false
    end
    @ngwords.each{|ng|
      if /#{ng}/ =~ link
        print "NG Word\n"
        return false
      end
    }
    true
  end
end

• Rubyでスクレーピング(mechanizeとhpricot)
• Perlでクロールする（LWP::UserAgent）
  
• RubyScraping - FrontPage
• Greenbear Diary - RubyでHTMLとWebを操作するためのライブラリ、HpricotとWWW::Mechanize , HikiReload

$ gem install hpricot
$ gem install mechanize

#!/usr/bin/ruby -Ks
require 'hpricot'
require 'mechanize'
require 'kconv'
require 'jcode'

$KCODE = 's'

# 初期設定
@user_agent = 'Mac Safari'
@wait_time = 5
@url = 'taget_url'

def http_connect(url, keyword)
  puts url if $DEBUG
  @page = nil;@price = nil;@content = ""
  begin
    @agent = WWW::Mechanize.new
    @agent.user_agent_alias = @user_agent
    @page = @agent.get(url)
  rescue
    puts "#{keyword}#{@drim}CONNECTION ERROR"
    return
  end
  link = @page.links.with.href(%r{http://target_url/\w+?/}).last
  puts link.href if $DEBUG
  if(link == nil || link.href.empty?)
    puts "#{keyword}#{@drim}LINKERROR"
    return
  end
  @page = @agent.click(link)
  doc = @page.root
  @name = (doc/"span.item_name").inner_html.tosjis
  (doc/"span.item").each {|item|
    item_str = item.inner_html.tosjis
    item_str.scan(/
(.*?)<\/div>
(.*?)<\/div>/){|title, content|
      @content += ":#{title}:#{content}" # if $DEBUG
    }
    item_str.scan(/([\d+] 円)/){|price|
      @price = price
    }
    puts "#{keyword}#{@drim}#{@price}#{@drim}#{@name}#{@drim}#{@content}#{@drim}#{link.href}"
  }
end

def generate_url
  puts ARGV[0] if $DEBUG

  File.open(ARGV[0]){|f|
    f.each{|line|
      keyword = line.chomp!
      puts keyword if $DEBUG
      target="#{@targeturl}?keyword=#{keyword}"
      http_connect(target, keyword)
      sleep @wait_time
    }
  }
end

puts @header
generate_url

require 'jcode'

• Rubyリファレンスマニュアル - FAQ::日本語の取り扱い
• Rubyリファレンスマニュアル - jcode
  
• http://blade.nagaokaut.ac.jp/cgi-bin/vframe.rb/ruby/ruby-list/31741?31634-32001

#!/usr/bin/ruby
require 'mechanize'
require 'net/smtp'
require 'kconv'

url = "http://localhost/"
wait_time = 15
@from_address = "from@mail_domain"
@send_address = "send@mail_domain"
agent = WWW::Mechanize.new

def send_alert( msg )
  body = [ "Subject: HTTP Access Error\n", "\n", msg ]
  Net::SMTP.start('mail_server') do |smtp|
    smtp.sendmail( body, @from_address, @send_address )
  end
  exit
end

begin
  while 1
    page = agent.get(url)
    s = page.title.tosjis
    if s =~ /match_word/
      puts "#{url}: #{s}\n" if $DEBUG
    else
      puts "#{url}: #{s}=#{match_str.tosjis}\n" if $DEBUG
      send_alert("#{url}: #{s}\n")
    end
    sleep wait_time
  end
rescue => ex
  puts "#{url}: #{ex.message}\n" if $DEBUG
  send_alert("#{url}:#{ex.message}\n")
end

• http://mono.kmc.gr.jp/~yhara/d/?date=20070205
• http://www.ibm.com/developerworks/jp/linux/library/l-spider/