Merge pull request #943 from cantino/rss_agent_logs_exact_feed_on_error
RSSAgent should record which feed errored and keep going on the other ones
Andrew Cantino
2 arquivos alterados com 34 adições e 18 exclusões
+ 24
- 18
app/models/agents/rss_agent.rb
@@ -87,29 +87,35 @@ module Agents |
||
| 87 | 87 |
|
| 88 | 88 |
def check |
| 89 | 89 |
Array(interpolated['url']).each do |url| |
| 90 |
- response = faraday.get(url) |
|
| 91 |
- if response.success? |
|
| 92 |
- feed = FeedNormalizer::FeedNormalizer.parse(response.body) |
|
| 93 |
- feed.clean! if boolify(interpolated['clean']) |
|
| 94 |
- max_events = (interpolated['max_events_per_run'].presence || 0).to_i |
|
| 95 |
- created_event_count = 0 |
|
| 96 |
- sort_events(feed_to_events(feed)).each.with_index do |event, index| |
|
| 97 |
- break if max_events && max_events > 0 && index >= max_events |
|
| 98 |
- entry_id = event.payload[:id] |
|
| 99 |
- if check_and_track(entry_id) |
|
| 100 |
- created_event_count += 1 |
|
| 101 |
- create_event(event) |
|
| 102 |
- end |
|
| 103 |
- end |
|
| 104 |
- log "Fetched #{url} and created #{created_event_count} event(s)."
|
|
| 105 |
- else |
|
| 106 |
- error "Failed to fetch #{url}: #{response.inspect}"
|
|
| 107 |
- end |
|
| 90 |
+ check_url(url) |
|
| 108 | 91 |
end |
| 109 | 92 |
end |
| 110 | 93 |
|
| 111 | 94 |
protected |
| 112 | 95 |
|
| 96 |
+ def check_url(url) |
|
| 97 |
+ response = faraday.get(url) |
|
| 98 |
+ if response.success? |
|
| 99 |
+ feed = FeedNormalizer::FeedNormalizer.parse(response.body) |
|
| 100 |
+ feed.clean! if boolify(interpolated['clean']) |
|
| 101 |
+ max_events = (interpolated['max_events_per_run'].presence || 0).to_i |
|
| 102 |
+ created_event_count = 0 |
|
| 103 |
+ sort_events(feed_to_events(feed)).each.with_index do |event, index| |
|
| 104 |
+ break if max_events && max_events > 0 && index >= max_events |
|
| 105 |
+ entry_id = event.payload[:id] |
|
| 106 |
+ if check_and_track(entry_id) |
|
| 107 |
+ created_event_count += 1 |
|
| 108 |
+ create_event(event) |
|
| 109 |
+ end |
|
| 110 |
+ end |
|
| 111 |
+ log "Fetched #{url} and created #{created_event_count} event(s)."
|
|
| 112 |
+ else |
|
| 113 |
+ error "Failed to fetch #{url}: #{response.inspect}"
|
|
| 114 |
+ end |
|
| 115 |
+ rescue => e |
|
| 116 |
+ error "Failed to fetch #{url} with message '#{e.message}': #{e.backtrace}"
|
|
| 117 |
+ end |
|
| 118 |
+ |
|
| 113 | 119 |
def get_entry_id(entry) |
| 114 | 120 |
entry.id.presence || Digest::MD5.hexdigest(entry.content) |
| 115 | 121 |
end |
+ 10
- 0
spec/models/agents/rss_agent_spec.rb
@@ -148,4 +148,14 @@ describe Agents::RssAgent do |
||
| 148 | 148 |
expect(agent.memory['seen_ids']).to eq(agent.events.map {|e| Digest::MD5.hexdigest(e.payload['content']) })
|
| 149 | 149 |
end |
| 150 | 150 |
end |
| 151 |
+ |
|
| 152 |
+ describe 'logging errors with the feed url' do |
|
| 153 |
+ it 'includes the feed URL when an exception is raised' do |
|
| 154 |
+ mock(FeedNormalizer::FeedNormalizer).parse(anything) { raise StandardError.new("Some error!") }
|
|
| 155 |
+ expect(lambda {
|
|
| 156 |
+ agent.check |
|
| 157 |
+ }).not_to raise_error |
|
| 158 |
+ expect(agent.logs.last.message).to match(%r[Failed to fetch https://github.com]) |
|
| 159 |
+ end |
|
| 160 |
+ end |
|
| 151 | 161 |
end |