使用Ruby程序实现web信息抓取的教程

#!/usr/bin/env ruby
require 'rubygems'
require 'nokogiri'
require 'open-uri'

# Define the URL with the argument passed by the user
uri = "http://www.crunchbase.com/company/#{ARGV[0]}"

# Use Nokogiri to get the document
doc = Nokogiri::HTML(open(uri))

# Find the link of interest
link = doc.search('tr span[1]')

# Emit the content associated with that link
puts link[0].content

#!/usr/bin/env ruby
require 'rubygems'
require 'json'
require 'net/http'

# Define the URL with the argument passed by the user
uri = "http://api.crunchbase.com/v/1/company/#{ARGV[0]}.js"

# Perform the HTTP GET request, and return the response
resp = Net::HTTP.get_response(URI.parse(uri))

# Parse the JSON from the response body
jresp = JSON.parse(resp.body)

# Emit the content of interest
puts jresp['number_of_employees']

$ ./parse.rb ibm
388,000
$ ./api.rb ibm
388000
$ ./parse.rb cisco
63,000
$ ./api.rb cisco
63000
$ ./parse.rb paypal
300,000
$ ./api.rb paypal
300000
$

#!/usr/bin/ruby
require 'rubygems'
require 'oauth'
require 'json'

pquery = "http://api.linkedin.com/v1/people/~?format=json"
cquery='http://api.linkedin.com/v1/people/~/suggestions/to-follow/companies?format=json'
jquery='http://api.linkedin.com/v1/people/~/suggestions/job-suggestions?format=json'
 
# Fill the keys and secrets you retrieved after registering your app
api_key = 'api key'
api_secret = 'api secret'
user_token = 'user token'
user_secret = 'user secret'
 
# Specify LinkedIn API endpoint
configuration = { :site => 'https://api.linkedin.com' }
 
# Use the API key and secret to instantiate consumer object
consumer = OAuth::Consumer.new(api_key, api_secret, configuration)
 
# Use the developer token and secret to instantiate access token object
access_token = OAuth::AccessToken.new(consumer, user_token, user_secret)

# Get the username for this profile
response = access_token.get(pquery)
jresp = JSON.parse(response.body)
myName = "#{jresp['firstName']} #{jresp['lastName']}"
puts "\nSuggested companies to follow for #{myName}"

# Get the suggested companies to follow
response = access_token.get(cquery)
jresp = JSON.parse(response.body)

# Iterate through each and display the company name
jresp['values'].each do | company |
  puts " #{company['name']}"
end

# Get the job suggestions
response = access_token.get(jquery)
jresp = JSON.parse(response.body)
puts "\nSuggested jobs for #{myName}"

# Iterate through each suggested job and print the company name
jresp['jobs']['values'].each do | job |
  puts " #{job['company']['name']} in #{job['locationDescription']}"
end

puts "\n"

$ ./lkdin.rb

Suggested companies to follow for M. Tim Jones
 Open Kernel Labs, Inc.
 Linaro
 Wind River
 DDC-I
 Linsyssoft Technologies
 Kalray
 American Megatrends
 JetHead Development
 Evidence Srl
 Aizyc Technology

Suggested jobs for M. Tim Jones
 Kozio in Greater Denver Area
 Samsung Semiconductor Inc in San Jose, CA
 Terran Systems in Sunnyvale, CA
 Magnum Semiconductor in San Francisco Bay Area
 RGB Spectrum in Alameda, CA
 Aptina in San Francisco Bay Area
 CyberCoders in San Francisco, CA
 CyberCoders in Alameda, CA
 SanDisk in Longmont, CO
 SanDisk in Longmont, CO

$

#!/usr/bin/ruby
require 'rubygems'
require 'oauth'
require 'json'

consumer_key = 'your consumer key'
consumer_secret = 'your consumer secret'
token = 'your token'
token_secret = 'your token secret'
api_host = 'http://api.yelp.com'

consumer = OAuth::Consumer.new(consumer_key, consumer_secret, {:site => api_host})
access_token = OAuth::AccessToken.new(consumer, token, token_secret)

path = "/v2/search?term=restaurants&location=Boulder,CO"

jresp = JSON.parse(access_token.get(path).body)

jresp['businesses'].each do | business |
  if business['is_closed'] == false
   printf("%-32s %10s %3d %1.1f\n", 
        business['name'], business['phone'], 
        business['review_count'], business['rating'])
  end
end

$ ./yelp.rb
Frasca Food and Wine       3034426966 189 4.5
John's Restaurant         3034445232  51 4.5
Leaf Vegetarian Restaurant    3034421485 144 4.0
Nepal Cuisine           3035545828  65 4.5
Black Cat Bistro         3034445500  72 4.0
The Mediterranean Restaurant   3034445335 306 4.0
Arugula Bar E Ristorante     3034435100  48 4.0
Ras Kassa's Ethiopia Restaurant  3034472919 101 4.0
L'Atelier             3034427233  58 4.0
Bombay Bistro           3034444721  87 4.0
Brasserie Ten Ten         3039981010 200 4.0
Flagstaff House          3034424640  86 4.5
Pearl Street Mall         3034493774  77 4.0
Gurkhas on the Hill        3034431355  19 4.0
The Kitchen            3035445973 274 4.0
Chez Thuy Restaurant       3034421700  99 3.5
Il Pastaio            3034479572 113 4.5
3 Margaritas           3039981234  11 3.5
Q's Restaurant          3034424880  65 4.0
Julia's Kitchen                 8 5.0

$

#!/usr/bin/env ruby
require 'net/http'

aggr = ""
key = 'your api key here'

# Get the IP address for the domain using the 'host' command
IO.popen("host #{ARGV[0]}") { | line |
 until line.eof?
  aggr += line.gets
 end
}

# Find the IP address in the response from the 'host' command
pattern = /\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/
if m = pattern.match(aggr)
  uri = "http://api.opencrypt.com/ip/?IP=#{m[0]}&key=#{key}"
  resp = Net::HTTP.get_response(URI.parse(uri))
  puts resp.body
end

$ ./where.rb www.baynet.ne.jp
IP=111.68.239.125
CC=JP
CN=Japan
$ ./where.rb www.pravda.ru
IP=212.76.137.2
CC=RU
CN=Russian Federation
$

#!/usr/bin/ruby
require 'rubygems'
require 'net/https'
require 'json'

url = 'https://www.googleapis.com/discovery/v1/apis'

uri = URI.parse(url)

# Set up a connection to the Google API Service
http = Net::HTTP.new( uri.host, 443 )
http.use_ssl = true
http.verify_mode = OpenSSL::SSL::VERIFY_NONE

# Connect to the service
req = Net::HTTP::Get.new(uri.request_uri)
resp = http.request(req)

# Get the JSON representation
jresp = JSON.parse(resp.body)

# Iterate through the API List
jresp['items'].each do | item |
 if item['preferred'] == true
  name = item['name']
  title = item['title']
  link = item['discoveryLink']
  printf("%-17s %-34s %-20s\n", name, title, link)
 end
end

$ ./gdir.rb
adexchangebuyer  Ad Exchange Buyer API       ./apis/adexchangebuyer/v1.1/rest
adsense      AdSense Management API       ./apis/adsense/v1.1/rest
adsensehost    AdSense Host API          ./apis/adsensehost/v4.1/rest
analytics     Google Analytics API        ./apis/analytics/v3/rest
androidpublisher Google Play Android Developer API ./apis/androidpublisher/v1/rest
audit       Enterprise Audit API        ./apis/audit/v1/rest
bigquery     BigQuery API            ./apis/bigquery/v2/rest
blogger      Blogger API            ./apis/blogger/v3/rest
books       Books API             ./apis/books/v1/rest
calendar     Calendar API            ./apis/calendar/v3/rest
compute      Compute Engine API         ./apis/compute/v1beta12/rest
coordinate    Google Maps Coordinate API     ./apis/coordinate/v1/rest
customsearch   CustomSearch API          ./apis/customsearch/v1/rest
dfareporting   DFA Reporting API         ./apis/dfareporting/v1/rest
discovery     APIs Discovery Service       ./apis/discovery/v1/rest
drive       Drive API             ./apis/drive/v2/rest
...
storage      Cloud Storage API         ./apis/storage/v1beta1/rest
taskqueue     TaskQueue API           ./apis/taskqueue/v1beta2/rest
tasks       Tasks API             ./apis/tasks/v1/rest
translate     Translate API           ./apis/translate/v2/rest
urlshortener   URL Shortener API         ./apis/urlshortener/v1/rest
webfonts     Google Web Fonts Developer API   ./apis/webfonts/v1/rest
youtube      YouTube API            ./apis/youtube/v3alpha/rest
youtubeAnalytics YouTube Analytics API       ./apis/youtubeAnalytics/v1/rest
$