Switch to the JSON output format for easier parsing

lib/wayback_machine_downloader.rb

@@ -84,7 +84,7 @@ def get_all_snapshots_to_consider
  # Note: Passing a page index parameter allow us to get more snapshots,
  # but from a less fresh index
  print "Getting snapshot pages"
- snapshot_list_to_consider = ""
+ snapshot_list_to_consider = []
  snapshot_list_to_consider += get_raw_list_from_api(@base_url, nil)
  print "."
  unless @exact_url
@@ -95,17 +95,15 @@ def get_all_snapshots_to_consider
  print "."
  end
  end
- puts " found #{snapshot_list_to_consider.lines.count} snaphots to consider."
+ puts " found #{snapshot_list_to_consider.length} snaphots to consider."
  puts
  snapshot_list_to_consider
  end
 
  def get_file_list_curated
  file_list_curated = Hash.new
- get_all_snapshots_to_consider.each_line do |line|
- next unless line.include?('/')
- file_timestamp = line[0..13].to_i
- file_url = line[15..-2]
+ get_all_snapshots_to_consider.each do |file_timestamp, file_url|
+ next unless file_url.include?('/')
  file_id = file_url.split('/')[3..-1].join('/')
  file_id = CGI::unescape file_id 
  file_id = file_id.tidy_bytes unless file_id == ""
@@ -130,10 +128,8 @@ def get_file_list_curated
 
  def get_file_list_all_timestamps
  file_list_curated = Hash.new
- get_all_snapshots_to_consider.each_line do |line|
- next unless line.include?('/')
- file_timestamp = line[0..13].to_i
- file_url = line[15..-2]
+ get_all_snapshots_to_consider.each do |file_timestamp, file_url|
+ next unless file_url.include?('/')
  file_id = file_url.split('/')[3..-1].join('/')
  file_id_and_timestamp = [file_timestamp, file_id].join('/')
  file_id_and_timestamp = CGI::unescape file_id_and_timestamp 

lib/wayback_machine_downloader/archive_api.rb

@@ -1,14 +1,23 @@
+require 'json'
 require 'uri'
 
 module ArchiveAPI
 
  def get_raw_list_from_api url, page_index
  request_url = URI("https://web.archive.org/cdx/search/xd")
- params = [["url", url]]
+ params = [["output", "json"], ["url", url]]
  params += parameters_for_api page_index
  request_url.query = URI.encode_www_form(params)
 
- URI.open(request_url).read
+ begin
+ json = JSON.parse(URI.open(request_url).read)
+ if (json[0] <=> ["timestamp","original"]) == 0
+ json.shift
+ end
+ json
+ rescue JSON::ParserError
+ []
+ end
  end
 
  def parameters_for_api page_index