Home Explore Help
Register Sign In
J1X
/
huginn
Watch 2
Star 0
Fork 0

Browse Source
the RSSAgent should record which feed errored and keep going on the other ones

Andrew Cantino 10 years ago
parent
6285e30952
commit
438d09edf1
2 changed files with 45 additions and 29 deletions
Show Diff Stats
  1. 35 29
      app/models/agents/rss_agent.rb
  2. 10 0
      spec/models/agents/rss_agent_spec.rb

+ 35 - 29
app/models/agents/rss_agent.rb
View File 

@@ -74,40 +74,46 @@ module Agents
74 74
75 75 
     def check
76 76 
       Array(interpolated['url']).each do |url|
77 
-        response = faraday.get(url)
78 
-        if response.success?
79 
-          feed = FeedNormalizer::FeedNormalizer.parse(response.body)
80 
-          feed.clean! if interpolated['clean'] == 'true'
81 
-          max_events = (interpolated['max_events_per_run'].presence || 0).to_i
82 
-          created_event_count = 0
83 
-          feed.entries.sort_by { |entry| [entry.date_published, entry.last_updated] }.each.with_index do |entry, index|
84 
-            break if max_events && max_events > 0 && index >= max_events
85 
-            entry_id = get_entry_id(entry)
86 
-            if check_and_track(entry_id)
87 
-              created_event_count += 1
88 
-              create_event(payload: {
89 
-                id: entry_id,
90 
-                date_published: entry.date_published,
91 
-                last_updated: entry.last_updated,
92 
-                url: entry.url,
93 
-                urls: entry.urls,
94 
-                description: entry.description,
95 
-                content: entry.content,
96 
-                title: entry.title,
97 
-                authors: entry.authors,
98 
-                categories: entry.categories
99 
-              })
100 
-            end
101 
-          end
102 
-          log "Fetched #{url} and created #{created_event_count} event(s)."
103 
-        else
104 
-          error "Failed to fetch #{url}: #{response.inspect}"
105 
-        end
77 
+        check_url(url)
106 78 
       end
107 79 
     end
108 80
109 81 
     protected
110 82
83 
+    def check_url(url)
84 
+      response = faraday.get(url)
85 
+      if response.success?
86 
+        feed = FeedNormalizer::FeedNormalizer.parse(response.body)
87 
+        feed.clean! if interpolated['clean'] == 'true'
88 
+        max_events = (interpolated['max_events_per_run'].presence || 0).to_i
89 
+        created_event_count = 0
90 
+        feed.entries.sort_by { |entry| [entry.date_published, entry.last_updated] }.each.with_index do |entry, index|
91 
+          break if max_events && max_events > 0 && index >= max_events
92 
+          entry_id = get_entry_id(entry)
93 
+          if check_and_track(entry_id)
94 
+            created_event_count += 1
95 
+            create_event(payload: {
96 
+                           id: entry_id,
97 
+                           date_published: entry.date_published,
98 
+                           last_updated: entry.last_updated,
99 
+                           url: entry.url,
100 
+                           urls: entry.urls,
101 
+                           description: entry.description,
102 
+                           content: entry.content,
103 
+                           title: entry.title,
104 
+                           authors: entry.authors,
105 
+                           categories: entry.categories
106 
+                         })
107 
+          end
108 
+        end
109 
+        log "Fetched #{url} and created #{created_event_count} event(s)."
110 
+      else
111 
+        error "Failed to fetch #{url}: #{response.inspect}"
112 
+      end
113 
+    rescue => e
114 
+      error "Failed to fetch #{url} with message '#{e.message}': #{e.backtrace}"
115 
+    end
116 
+
111 117 
     def get_entry_id(entry)
112 118 
       entry.id.presence || Digest::MD5.hexdigest(entry.content)
113 119 
     end

+ 10 - 0
spec/models/agents/rss_agent_spec.rb
View File 

@@ -133,4 +133,14 @@ describe Agents::RssAgent do
133 133 
       expect(agent.memory['seen_ids']).to eq(agent.events.map {|e| Digest::MD5.hexdigest(e.payload['content']) })
134 134 
     end
135 135 
   end
136 
+
137 
+  describe 'logging errors with the feed url' do
138 
+    it 'includes the feed URL when an exception is raised' do
139 
+      mock(FeedNormalizer::FeedNormalizer).parse(anything) { raise StandardError.new("Some error!") }
140 
+      expect(lambda {
141 
+        agent.check
142 
+      }).not_to raise_error
143 
+      expect(agent.logs.last.message).to match(%r[Failed to fetch https://github.com])
144 
+    end
145 
+  end
136 146 
 end