Merge pull request #1254 from dsander/add-no-bulk-receive
Add no_bulk_receive! option to ensure each event is processed individually
Dominik Sander
20 changed files with 57 additions and 9 deletions
+ 22
- 6
app/models/agent.rb
@@ -208,6 +208,10 @@ class Agent < ActiveRecord::Base |
||
| 208 | 208 |
self.class.can_dry_run? |
| 209 | 209 |
end |
| 210 | 210 |
|
| 211 |
+ def no_bulk_receive? |
|
| 212 |
+ self.class.no_bulk_receive? |
|
| 213 |
+ end |
|
| 214 |
+ |
|
| 211 | 215 |
def log(message, options = {})
|
| 212 | 216 |
AgentLog.log_for_agent(self, message, options) |
| 213 | 217 |
end |
@@ -350,6 +354,14 @@ class Agent < ActiveRecord::Base |
||
| 350 | 354 |
!!@can_dry_run |
| 351 | 355 |
end |
| 352 | 356 |
|
| 357 |
+ def no_bulk_receive! |
|
| 358 |
+ @no_bulk_receive = true |
|
| 359 |
+ end |
|
| 360 |
+ |
|
| 361 |
+ def no_bulk_receive? |
|
| 362 |
+ !!@no_bulk_receive |
|
| 363 |
+ end |
|
| 364 |
+ |
|
| 353 | 365 |
def gem_dependency_check |
| 354 | 366 |
@gem_dependencies_checked = true |
| 355 | 367 |
@gem_dependencies_met = yield |
@@ -365,7 +377,7 @@ class Agent < ActiveRecord::Base |
||
| 365 | 377 |
def receive!(options={})
|
| 366 | 378 |
Agent.transaction do |
| 367 | 379 |
scope = Agent. |
| 368 |
- select("agents.id AS receiver_agent_id, sources.id AS source_agent_id, events.id AS event_id").
|
|
| 380 |
+ select("agents.id AS receiver_agent_id, events.id AS event_id").
|
|
| 369 | 381 |
joins("JOIN links ON (links.receiver_id = agents.id)").
|
| 370 | 382 |
joins("JOIN agents AS sources ON (links.source_id = sources.id)").
|
| 371 | 383 |
joins("JOIN events ON (events.agent_id = sources.id AND events.id > links.event_id_at_creation)").
|
@@ -377,21 +389,25 @@ class Agent < ActiveRecord::Base |
||
| 377 | 389 |
sql = scope.to_sql() |
| 378 | 390 |
|
| 379 | 391 |
agents_to_events = {}
|
| 380 |
- Agent.connection.select_rows(sql).each do |receiver_agent_id, source_agent_id, event_id| |
|
| 392 |
+ Agent.connection.select_rows(sql).each do |receiver_agent_id, event_id| |
|
| 381 | 393 |
agents_to_events[receiver_agent_id.to_i] ||= [] |
| 382 | 394 |
agents_to_events[receiver_agent_id.to_i] << event_id |
| 383 | 395 |
end |
| 384 | 396 |
|
| 385 |
- event_ids = agents_to_events.values.flatten.uniq.compact |
|
| 386 |
- |
|
| 387 | 397 |
Agent.where(:id => agents_to_events.keys).each do |agent| |
| 398 |
+ event_ids = agents_to_events[agent.id].uniq |
|
| 388 | 399 |
agent.update_attribute :last_checked_event_id, event_ids.max |
| 389 |
- Agent.async_receive(agent.id, agents_to_events[agent.id].uniq) |
|
| 400 |
+ |
|
| 401 |
+ if agent.no_bulk_receive? |
|
| 402 |
+ event_ids.each { |event_id| Agent.async_receive(agent.id, [event_id]) }
|
|
| 403 |
+ else |
|
| 404 |
+ Agent.async_receive(agent.id, event_ids) |
|
| 405 |
+ end |
|
| 390 | 406 |
end |
| 391 | 407 |
|
| 392 | 408 |
{
|
| 393 | 409 |
:agent_count => agents_to_events.keys.length, |
| 394 |
- :event_count => event_ids.length |
|
| 410 |
+ :event_count => agents_to_events.values.flatten.uniq.compact.length |
|
| 395 | 411 |
} |
| 396 | 412 |
end |
| 397 | 413 |
end |
+ 1
- 0
app/models/agents/beeper_agent.rb
@@ -2,6 +2,7 @@ module Agents |
||
| 2 | 2 |
class BeeperAgent < Agent |
| 3 | 3 |
cannot_be_scheduled! |
| 4 | 4 |
cannot_create_events! |
| 5 |
+ no_bulk_receive! |
|
| 5 | 6 |
|
| 6 | 7 |
description <<-MD |
| 7 | 8 |
Beeper agent sends messages to Beeper app on your mobile device via Push notifications. |
+ 1
- 0
app/models/agents/dropbox_file_url_agent.rb
@@ -3,6 +3,7 @@ module Agents |
||
| 3 | 3 |
include DropboxConcern |
| 4 | 4 |
|
| 5 | 5 |
cannot_be_scheduled! |
| 6 |
+ no_bulk_receive! |
|
| 6 | 7 |
|
| 7 | 8 |
description <<-MD |
| 8 | 9 |
The Dropbox File Url Agent is used to work with Dropbox. It takes a file path (or multiple file paths) and emits events with [temporary links](https://www.dropbox.com/developers/core/docs#media). |
+ 1
- 0
app/models/agents/email_agent.rb
@@ -4,6 +4,7 @@ module Agents |
||
| 4 | 4 |
|
| 5 | 5 |
cannot_be_scheduled! |
| 6 | 6 |
cannot_create_events! |
| 7 |
+ no_bulk_receive! |
|
| 7 | 8 |
|
| 8 | 9 |
description <<-MD |
| 9 | 10 |
The Email Agent sends any events it receives via email immediately. |
+ 1
- 0
app/models/agents/google_calendar_publish_agent.rb
@@ -3,6 +3,7 @@ require 'json' |
||
| 3 | 3 |
module Agents |
| 4 | 4 |
class GoogleCalendarPublishAgent < Agent |
| 5 | 5 |
cannot_be_scheduled! |
| 6 |
+ no_bulk_receive! |
|
| 6 | 7 |
|
| 7 | 8 |
gem_dependency_check { defined?(Google) && defined?(Google::APIClient) }
|
| 8 | 9 |
|
+ 1
- 0
app/models/agents/hipchat_agent.rb
@@ -4,6 +4,7 @@ module Agents |
||
| 4 | 4 |
|
| 5 | 5 |
cannot_be_scheduled! |
| 6 | 6 |
cannot_create_events! |
| 7 |
+ no_bulk_receive! |
|
| 7 | 8 |
|
| 8 | 9 |
gem_dependency_check { defined?(HipChat) }
|
| 9 | 10 |
|
+ 1
- 0
app/models/agents/pdf_info_agent.rb
@@ -7,6 +7,7 @@ module Agents |
||
| 7 | 7 |
gem_dependency_check { defined?(HyPDF) }
|
| 8 | 8 |
|
| 9 | 9 |
cannot_be_scheduled! |
| 10 |
+ no_bulk_receive! |
|
| 10 | 11 |
|
| 11 | 12 |
description <<-MD |
| 12 | 13 |
The PDF Info Agent returns the metadata contained within a given PDF file, using HyPDF. |
+ 1
- 0
app/models/agents/post_agent.rb
@@ -3,6 +3,7 @@ module Agents |
||
| 3 | 3 |
include WebRequestConcern |
| 4 | 4 |
|
| 5 | 5 |
can_dry_run! |
| 6 |
+ no_bulk_receive! |
|
| 6 | 7 |
default_schedule "never" |
| 7 | 8 |
|
| 8 | 9 |
description <<-MD |
+ 1
- 0
app/models/agents/pushbullet_agent.rb
@@ -4,6 +4,7 @@ module Agents |
||
| 4 | 4 |
|
| 5 | 5 |
cannot_be_scheduled! |
| 6 | 6 |
cannot_create_events! |
| 7 |
+ no_bulk_receive! |
|
| 7 | 8 |
|
| 8 | 9 |
before_validation :create_device, on: :create |
| 9 | 10 |
|
+ 2
- 0
app/models/agents/pushover_agent.rb
@@ -2,6 +2,8 @@ module Agents |
||
| 2 | 2 |
class PushoverAgent < Agent |
| 3 | 3 |
cannot_be_scheduled! |
| 4 | 4 |
cannot_create_events! |
| 5 |
+ no_bulk_receive! |
|
| 6 |
+ |
|
| 5 | 7 |
|
| 6 | 8 |
API_URL = 'https://api.pushover.net/1/messages.json' |
| 7 | 9 |
|
+ 2
- 0
app/models/agents/shell_command_agent.rb
@@ -3,6 +3,8 @@ module Agents |
||
| 3 | 3 |
default_schedule "never" |
| 4 | 4 |
|
| 5 | 5 |
can_dry_run! |
| 6 |
+ no_bulk_receive! |
|
| 7 |
+ |
|
| 6 | 8 |
|
| 7 | 9 |
def self.should_run? |
| 8 | 10 |
ENV['ENABLE_INSECURE_AGENTS'] == "true" |
+ 1
- 0
app/models/agents/slack_agent.rb
@@ -5,6 +5,7 @@ module Agents |
||
| 5 | 5 |
|
| 6 | 6 |
cannot_be_scheduled! |
| 7 | 7 |
cannot_create_events! |
| 8 |
+ no_bulk_receive! |
|
| 8 | 9 |
|
| 9 | 10 |
gem_dependency_check { defined?(Slack) }
|
| 10 | 11 |
|
+ 1
- 0
app/models/agents/twilio_agent.rb
@@ -4,6 +4,7 @@ module Agents |
||
| 4 | 4 |
class TwilioAgent < Agent |
| 5 | 5 |
cannot_be_scheduled! |
| 6 | 6 |
cannot_create_events! |
| 7 |
+ no_bulk_receive! |
|
| 7 | 8 |
|
| 8 | 9 |
gem_dependency_check { defined?(Twilio) }
|
| 9 | 10 |
|
+ 1
- 0
app/models/agents/website_agent.rb
@@ -7,6 +7,7 @@ module Agents |
||
| 7 | 7 |
|
| 8 | 8 |
can_dry_run! |
| 9 | 9 |
can_order_created_events! |
| 10 |
+ no_bulk_receive! |
|
| 10 | 11 |
|
| 11 | 12 |
default_schedule "every_12h" |
| 12 | 13 |
|
+ 1
- 0
app/models/agents/witai_agent.rb
@@ -1,6 +1,7 @@ |
||
| 1 | 1 |
module Agents |
| 2 | 2 |
class WitaiAgent < Agent |
| 3 | 3 |
cannot_be_scheduled! |
| 4 |
+ no_bulk_receive! |
|
| 4 | 5 |
|
| 5 | 6 |
description <<-MD |
| 6 | 7 |
The `wit.ai` agent receives events, sends a text query to your `wit.ai` instance and generates outcome events. |
+ 1
- 0
app/models/agents/wunderlist_agent.rb
@@ -5,6 +5,7 @@ module Agents |
||
| 5 | 5 |
valid_oauth_providers :wunderlist |
| 6 | 6 |
|
| 7 | 7 |
cannot_be_scheduled! |
| 8 |
+ no_bulk_receive! |
|
| 8 | 9 |
|
| 9 | 10 |
gem_dependency_check { Devise.omniauth_providers.include?(:wunderlist) }
|
| 10 | 11 |
|
+ 2
- 2
spec/lib/huginn_scheduler_spec.rb
@@ -56,7 +56,7 @@ describe HuginnScheduler do |
||
| 56 | 56 |
end |
| 57 | 57 |
end |
| 58 | 58 |
|
| 59 |
- describe "cleanup_failed_jobs!", focus: true do |
|
| 59 |
+ describe "cleanup_failed_jobs!" do |
|
| 60 | 60 |
before do |
| 61 | 61 |
3.times do |i| |
| 62 | 62 |
Delayed::Job.create(failed_at: Time.now - i.minutes) |
@@ -64,7 +64,7 @@ describe HuginnScheduler do |
||
| 64 | 64 |
@keep = Delayed::Job.order(:failed_at)[1] |
| 65 | 65 |
end |
| 66 | 66 |
|
| 67 |
- it "work with set FAILED_JOBS_TO_KEEP env variable", focus: true do |
|
| 67 |
+ it "work with set FAILED_JOBS_TO_KEEP env variable" do |
|
| 68 | 68 |
expect { @scheduler.send(:cleanup_failed_jobs!) }.to change(Delayed::Job, :count).by(-1)
|
| 69 | 69 |
expect { @scheduler.send(:cleanup_failed_jobs!) }.to change(Delayed::Job, :count).by(0)
|
| 70 | 70 |
expect(@keep.id).to eq(Delayed::Job.order(:failed_at)[0].id) |
+ 8
- 0
spec/models/agent_spec.rb
@@ -295,6 +295,14 @@ describe Agent do |
||
| 295 | 295 |
Agent.receive! |
| 296 | 296 |
end |
| 297 | 297 |
|
| 298 |
+ it "should call receive for each event when no_bulk_receive! is used" do |
|
| 299 |
+ mock.any_instance_of(Agents::TriggerAgent).receive(anything).twice |
|
| 300 |
+ stub(Agents::TriggerAgent).no_bulk_receive? { true }
|
|
| 301 |
+ Agent.async_check(agents(:bob_weather_agent).id) |
|
| 302 |
+ Agent.async_check(agents(:bob_weather_agent).id) |
|
| 303 |
+ Agent.receive! |
|
| 304 |
+ end |
|
| 305 |
+ |
|
| 298 | 306 |
it "should ignore events that were created before a particular Link" do |
| 299 | 307 |
agent2 = Agents::SomethingSource.new(:name => "something") |
| 300 | 308 |
agent2.user = users(:bob) |
+ 7
- 0
spec/rails_helper.rb
@@ -50,6 +50,13 @@ RSpec.configure do |config| |
||
| 50 | 50 |
# rspec-rails. |
| 51 | 51 |
config.infer_base_class_for_anonymous_controllers = false |
| 52 | 52 |
|
| 53 |
+ # These two settings work together to allow you to limit a spec run |
|
| 54 |
+ # to individual examples or groups you care about by tagging them with |
|
| 55 |
+ # `:focus` metadata. When nothing is tagged with `:focus`, all examples |
|
| 56 |
+ # get run. |
|
| 57 |
+ config.filter_run :focus |
|
| 58 |
+ config.run_all_when_everything_filtered = true |
|
| 59 |
+ |
|
| 53 | 60 |
# Run specs in random order to surface order dependencies. If you find an |
| 54 | 61 |
# order dependency and want to debug it, you can fix the order by providing |
| 55 | 62 |
# the seed, which is printed after each run. |
+ 1
- 1
spec/support/shared_examples/liquid_interpolatable.rb
@@ -30,7 +30,7 @@ shared_examples_for LiquidInterpolatable do |
||
| 30 | 30 |
}) |
| 31 | 31 |
end |
| 32 | 32 |
|
| 33 |
- it "should work with arrays", focus: true do |
|
| 33 |
+ it "should work with arrays" do |
|
| 34 | 34 |
@checker.options = {"value" => ["{{variable}}", "Much array", "Hey, {{hello_world}}"]}
|
| 35 | 35 |
expect(@checker.interpolate_options(@checker.options, @event)).to eq({
|
| 36 | 36 |
"value" => ["hello", "Much array", "Hey, Hello world"] |