From 00cd6678b35e63ceea628b417b430b667e7ed23d Mon Sep 17 00:00:00 2001 From: Zhang Jun Date: Tue, 3 Sep 2013 13:28:52 +0800 Subject: [PATCH] remember the URI might be a good idea --- examples/redmine.rb | 10 ++++++++++ lib/harvestman/crawler/parser.rb | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 examples/redmine.rb diff --git a/examples/redmine.rb b/examples/redmine.rb new file mode 100644 index 0000000..815777e --- /dev/null +++ b/examples/redmine.rb @@ -0,0 +1,10 @@ +require 'harvestman' + +Harvestman.crawl 'http://www.redmine.org/issues/*',['14822','14820'] do + puts @uri + + description = css 'div.subject' + + puts "description: #{description}" + +end diff --git a/lib/harvestman/crawler/parser.rb b/lib/harvestman/crawler/parser.rb index 7129313..14ed891 100644 --- a/lib/harvestman/crawler/parser.rb +++ b/lib/harvestman/crawler/parser.rb @@ -2,7 +2,8 @@ module Harvestman module Crawler class Parser def initialize(url) - @document = Nokogiri::HTML(open(url)) + @uri = URI.parse(url) + @document = Nokogiri::HTML(open(@uri)) end def css(path, &block)