Added changes by @knu

app/models/agents/website_agent.rb

@@ -155,13 +155,11 @@ def check_urls(in_url)
  return unless in_url.present?
 
  Array(in_url).each do |url|
- check_url(url).map do |doc|
- create_event payload: doc
- end
+ check_url(url)
  end
  end
 
- def check_url(url)
+ def check_url(url, payload = {})
  log "Fetching #{url}"
  response = faraday.get(url)
  raise "Failed: #{response.inspect}" unless response.success?
@@ -174,13 +172,12 @@ def check_url(url)
  end
  doc = parse(body)
 
- results = []
  if extract_full_json?
  if store_payload!(previous_payloads(1), doc)
  log "Storing new result for '#{name}': #{doc.inspect}"
- results << doc
+ create_event payload: payload.merge(doc)
  end
- return results
+ return
  end
 
  output =
@@ -211,28 +208,21 @@ def check_url(url)
 
  if store_payload!(old_events, result)
  log "Storing new parsed result for '#{name}': #{result.inspect}"
- results << result
+ create_event payload: payload.merge(result)
  end
  end
-
- results
  }
  rescue => e
  error "Error when fetching url: #{e.message}\n#{e.backtrace.join("\n")}"
- return []
  end
 
  def receive(incoming_events)
  incoming_events.each do |event|
  interpolate_with(event) do
  url_to_scrape = event.payload['url']
- docs = []
- docs = check_url(url_to_scrape) if url_to_scrape =~ /^https?:\/\//i
- docs.each do |doc|
- new_payload = interpolated['mode'].to_s == "merge" ? event.payload.dup : {}
- new_payload.merge! doc
- create_event payload: new_payload
- end
+ next unless url_to_scrape =~ /^https?:\/\//i
+ check_url(url_to_scrape,
+ interpolated['mode'].to_s == "merge" ? event.payload : {})
  end
  end
  end