RssAgent: Add limit on feed items fetched
Irfan Charania
1 changed files with 5 additions and 0 deletions
+ 5
- 0
app/models/agents/rss_agent.rb
@@ -27,6 +27,7 @@ module Agents |
||
| 27 | 27 |
* `basic_auth` - Specify HTTP basic auth parameters: `"username:password"`, or `["username", "password"]`. |
| 28 | 28 |
* `disable_ssl_verification` - Set to `true` to disable ssl verification. |
| 29 | 29 |
* `user_agent` - A custom User-Agent name (default: "Faraday v#{Faraday::VERSION}").
|
| 30 |
+ * `max_items_per_feed` - Limit number of items parsed (events created) per feed. |
|
| 30 | 31 |
MD |
| 31 | 32 |
end |
| 32 | 33 |
|
@@ -76,8 +77,12 @@ module Agents |
||
| 76 | 77 |
if response.success? |
| 77 | 78 |
feed = FeedNormalizer::FeedNormalizer.parse(response.body) |
| 78 | 79 |
feed.clean! if interpolated['clean'] == 'true' |
| 80 |
+ |
|
| 81 |
+ max_events = Integer(interpolated['max_items_per_feed']) if options['max_items_per_feed'].present? |
|
| 82 |
+ |
|
| 79 | 83 |
created_event_count = 0 |
| 80 | 84 |
feed.entries.sort_by { |entry| [entry.date_published, entry.last_updated] }.each do |entry|
|
| 85 |
+ break if (!max_events.nil?) && (max_events >= 0) && (created_event_count >= max_events) |
|
| 81 | 86 |
entry_id = get_entry_id(entry) |
| 82 | 87 |
if check_and_track(entry_id) |
| 83 | 88 |
created_event_count += 1 |