Home Explore Help
Register Sign In
J1X
/
huginn
Watch 2
Star 0
Fork 0

Browse Source
Merge pull request #118 from cantino/add_keep_events_for_to_agents

Allow agents to determine how long to keep events for

Andrew Cantino 11 years ago
parent
bdc1f3f038 276d5936ec
commit
3bf1961ef6
16 changed files with 314 additions and 110 deletions
Show Diff Stats
  1. 1 0
      CHANGES.md
  2. 17 0
      app/assets/javascripts/application.js.coffee.erb
  3. 1 0
      app/controllers/agents_controller.rb
  4. 49 26
      app/models/agent.rb
  5. 2 0
      app/models/event.rb
  6. 10 0
      app/views/agents/_form.html.erb
  7. 5 0
      app/views/agents/show.html.erb
  8. 5 0
      app/views/events/show.html.erb
  9. 3 2
      app/views/home/_signed_out_index.html.erb
  10. 5 0
      db/migrate/20131222211558_add_keep_events_for_to_agents.rb
  11. 6 3
      db/schema.rb
  12. 1 1
      spec/controllers/events_controller_spec.rb
  13. 4 0
      spec/fixtures/agents.yml
  14. 104 2
      spec/models/agent_spec.rb
  15. 68 68
      spec/models/agents/website_agent_spec.rb
  16. 33 8
      spec/models/event_spec.rb

+ 1 - 0
CHANGES.md
View File 

@@ -1,5 +1,6 @@
1 1 
 # Changes
2 2
3 
+* 0.31 (Jan 2, 2014)   - Agents now have an optional keep\_events\_for option that is propagated to created events' expires\_at field, and they update their events' expires\_at fields on change.
3 4 
 * 0.3 (Jan 1, 2014)    - Remove symbolization of memory, options, and payloads; convert memory, options, and payloads to JSON from YAML.  Migration will perform conversion and adjust tables to be UTF-8.  Recommend making a DB backup before migrating.
4 5 
 * 0.2 (Nov 6, 2013)    - PeakDetectorAgent now uses `window_duration_in_days` and `min_peak_spacing_in_days`.  Additionally, peaks trigger when the time series rises over the standard deviation multiple, not after it starts to fall.
5 6 
 * June 29, 2013        - Removed rails\_admin because it was causing deployment issues. Better to have people install their favorite admin tool if they want one.

+ 17 - 0
app/assets/javascripts/application.js.coffee.erb
View File 

@@ -34,6 +34,12 @@ showLinks = ->
34 34 
   $(".link-region .cannot-receive-events").hide()
35 35 
   showEventDescriptions()
36 36
37 
+hideEventCreation = ->
38 
+  $(".event-related-region").hide()
39 
+
40 
+showEventCreation = ->
41 
+  $(".event-related-region").show()
42 
+
37 43 
 showEventDescriptions = ->
38 44 
   if $("#agent_source_ids").val()
39 45 
     $.getJSON "/agents/event_descriptions", { ids: $("#agent_source_ids").val().join(",") }, (json) =>
 
@@ -132,6 +138,11 @@ $(document).ready ->
132 138 
         else
133 139 
           hideLinks()
134 140
141 
+        if json.can_create_events
142 
+          showEventCreation()
143 
+        else
144 
+          hideEventCreation()
145 
+
135 146 
         $(".description").html(json.description_html) if json.description_html?
136 147
137 148 
         if $("#agent_options").hasClass("showing-default") || $("#agent_options").val().match(/\A\s*(\{\s*\}|)\s*\Z/g)
 
@@ -153,3 +164,9 @@ $(document).ready ->
153 164 
       showLinks()
154 165 
     else
155 166 
       hideLinks()
167 
+
168 
+  if $(".event-related-region")
169 
+    if $(".event-related-region").data("can-create-events") == true
170 
+      showEventCreation()
171 
+    else
172 
+      hideEventCreation()

+ 1 - 0
app/controllers/agents_controller.rb
View File 

@@ -33,6 +33,7 @@ class AgentsController < ApplicationController
33 33 
     render :json => {
34 34 
         :can_be_scheduled => agent.can_be_scheduled?,
35 35 
         :can_receive_events => agent.can_receive_events?,
36 
+        :can_create_events => agent.can_create_events?,
36 37 
         :options => agent.default_options,
37 38 
         :description_html => agent.html_description
38 39 
     }

+ 49 - 26
app/models/agent.rb
View File 

@@ -15,11 +15,14 @@ class Agent < ActiveRecord::Base
15 15 
   SCHEDULES = %w[every_2m every_5m every_10m every_30m every_1h every_2h every_5h every_12h every_1d every_2d every_7d
16 16 
                  midnight 1am 2am 3am 4am 5am 6am 7am 8am 9am 10am 11am noon 1pm 2pm 3pm 4pm 5pm 6pm 7pm 8pm 9pm 10pm 11pm]
17 17
18 
-  attr_accessible :options, :memory, :name, :type, :schedule, :source_ids
18 
+  EVENT_RETENTION_SCHEDULES = [["Forever", 0], ["1 day", 1], *([2, 3, 4, 5, 7, 14, 21, 30, 45, 90, 180, 365].map {|n| ["#{n} days", n] })]
19 
+
20 
+  attr_accessible :options, :memory, :name, :type, :schedule, :source_ids, :keep_events_for
19 21
20 22 
   json_serialize :options, :memory
21 23
22 24 
   validates_presence_of :name, :user
25 
+  validates_inclusion_of :keep_events_for, :in => EVENT_RETENTION_SCHEDULES.map(&:last)
23 26 
   validate :sources_are_owned
24 27 
   validate :validate_schedule
25 28 
   validate :validate_options
 
@@ -29,6 +32,7 @@ class Agent < ActiveRecord::Base
29 32 
   before_validation :unschedule_if_cannot_schedule
30 33 
   before_save :unschedule_if_cannot_schedule
31 34 
   before_create :set_last_checked_event_id
35 
+  after_save :possibly_update_event_expirations
32 36
33 37 
   belongs_to :user, :inverse_of => :agents
34 38 
   has_many :events, :dependent => :delete_all, :inverse_of => :agent, :order => "events.id desc"
 
@@ -87,21 +91,23 @@ class Agent < ActiveRecord::Base
87 91 
     last_event_at && last_error_log_at && last_error_log_at > (last_event_at - 2.minutes)
88 92 
   end
89 93
90 
-  def sources_are_owned
91 
-    errors.add(:sources, "must be owned by you") unless sources.all? {|s| s.user == user }
92 
-  end
93 
-
94 94 
   def create_event(attrs)
95 95 
     if can_create_events?
96 
-      events.create!({ :user => user }.merge(attrs))
96 
+      events.create!({ :user => user, :expires_at => new_event_expiration_date }.merge(attrs))
97 97 
     else
98 98 
       error "This Agent cannot create events!"
99 99 
     end
100 100 
   end
101 101
102 
-  def validate_schedule
103 
-    unless cannot_be_scheduled?
104 
-      errors.add(:schedule, "is not a valid schedule") unless SCHEDULES.include?(schedule.to_s)
102 
+  def new_event_expiration_date
103 
+    keep_events_for > 0 ? keep_events_for.days.from_now : nil
104 
+  end
105 
+
106 
+  def update_event_expirations!
107 
+    if keep_events_for == 0
108 
+      events.update_all :expires_at => nil
109 
+    else
110 
+      events.update_all "expires_at = DATE_ADD(`created_at`, INTERVAL #{keep_events_for.to_i} DAY)"
105 111 
     end
106 112 
   end
107 113 
 
@@ -116,14 +122,6 @@ class Agent < ActiveRecord::Base
116 122 
     end
117 123 
   end
118 124
119 
-  def set_default_schedule
120 
-    self.schedule = default_schedule unless schedule.present? || cannot_be_scheduled?
121 
-  end
122 
-
123 
-  def unschedule_if_cannot_schedule
124 
-    self.schedule = nil if cannot_be_scheduled?
125 
-  end
126 
-
127 125 
   def default_schedule
128 126 
     self.class.default_schedule
129 127 
   end
 
@@ -152,10 +150,13 @@ class Agent < ActiveRecord::Base
152 150 
     !cannot_create_events?
153 151 
   end
154 152
155 
-  def set_last_checked_event_id
156 
-    if newest_event_id = Event.order("id desc").limit(1).pluck(:id).first
157 
-      self.last_checked_event_id = newest_event_id
158 
-    end
153 
+  def log(message, options = {})
154 
+    puts "Agent##{id}: #{message}" unless Rails.env.test?
155 
+    AgentLog.log_for_agent(self, message, options)
156 
+  end
157 
+
158 
+  def error(message, options = {})
159 
+    log(message, options.merge(:level => 4))
159 160 
   end
160 161
161 162 
   def delete_logs!
 
@@ -163,16 +164,38 @@ class Agent < ActiveRecord::Base
163 164 
     update_column :last_error_log_at, nil
164 165 
   end
165 166
166 
-  def log(message, options = {})
167 
-    puts "Agent##{id}: #{message}" unless Rails.env.test?
168 
-    AgentLog.log_for_agent(self, message, options)
167 
+  # Validations and Callbacks
168 
+
169 
+  def sources_are_owned
170 
+    errors.add(:sources, "must be owned by you") unless sources.all? {|s| s.user == user }
169 171 
   end
170 172
171 
-  def error(message, options = {})
172 
-    log(message, options.merge(:level => 4))
173 
+  def validate_schedule
174 
+    unless cannot_be_scheduled?
175 
+      errors.add(:schedule, "is not a valid schedule") unless SCHEDULES.include?(schedule.to_s)
176 
+    end
177 
+  end
178 
+
179 
+  def set_default_schedule
180 
+    self.schedule = default_schedule unless schedule.present? || cannot_be_scheduled?
181 
+  end
182 
+
183 
+  def unschedule_if_cannot_schedule
184 
+    self.schedule = nil if cannot_be_scheduled?
185 
+  end
186 
+
187 
+  def set_last_checked_event_id
188 
+    if newest_event_id = Event.order("id desc").limit(1).pluck(:id).first
189 
+      self.last_checked_event_id = newest_event_id
190 
+    end
191 
+  end
192 
+
193 
+  def possibly_update_event_expirations
194 
+    update_event_expirations! if keep_events_for_changed?
173 195 
   end
174 196
175 197 
   # Class Methods
198 
+
176 199 
   class << self
177 200 
     def cannot_be_scheduled!
178 201 
       @cannot_be_scheduled = true

+ 2 - 0
app/models/event.rb
View File 

@@ -21,6 +21,8 @@ class Event < ActiveRecord::Base
21 21 
   end
22 22
23 23 
   def self.cleanup_expired!
24 
+    affected_agents = Event.where("expires_at IS NOT NULL AND expires_at < ?", Time.now).group("agent_id").pluck(:agent_id)
24 25 
     Event.where("expires_at IS NOT NULL AND expires_at < ?", Time.now).delete_all
26 
+    Agent.where(:id => affected_agents).update_all "events_count = (select count(*) from events where agent_id = agents.id)"
25 27 
   end
26 28 
 end

+ 10 - 0
app/views/agents/_form.html.erb
View File 

@@ -44,6 +44,16 @@
44 44 
     </div>
45 45 
   </div>
46 46
47 
+
48 
+  <div class='event-related-region' data-can-create-events="<%= @agent.can_create_events? %>">
49 
+    <div class="control-group">
50 
+      <%= f.label :keep_events_for, "Keep events", :class => 'control-label' %>
51 
+      <div class="controls">
52 
+        <%= f.select :keep_events_for, options_for_select(Agent::EVENT_RETENTION_SCHEDULES, @agent.keep_events_for), {}, :class => 'span4' %>
53 
+      </div>
54 
+    </div>
55 
+  </div>
56 
+
47 57 
   <div class="control-group">
48 58 
     <%= f.label :sources, :class => 'control-label' %>
49 59 
     <div class="controls link-region" data-can-receive-events="<%= @agent.can_receive_events? %>">

+ 5 - 0
app/views/agents/show.html.erb
View File 

@@ -84,6 +84,11 @@
84 84
85 85 
             <% if @agent.can_create_events? %>
86 86 
               <p>
87 
+                <b>Keep events:</b>
88 
+                <%= (Agent::EVENT_RETENTION_SCHEDULES.detect {|s| s.last == @agent.keep_events_for } || [@agent.keep_events_for]).first %>
89 
+              </p>
90 
+
91 
+              <p>
87 92 
                 <b>Last event created:</b>
88 93 
                 <%= @agent.last_event_at ? time_ago_in_words(@agent.last_event_at) + " ago" : "never" %>
89 94 
               </p>

+ 5 - 0
app/views/events/show.html.erb
View File 

@@ -6,6 +6,11 @@
6 6 
       </div>
7 7
8 8 
       <p>
9 
+        <b>Expires in:</b>
10 
+        <%= @event.expires_at ? time_ago_in_words(@event.expires_at) : 'never' %>
11 
+      </p>
12 
+
13 
+      <p>
9 14 
         <b>Payload:</b>
10 15 
         <pre><%= Utils.pretty_jsonify @event.payload || {} %></pre>
11 16 
       </p>

+ 3 - 2
app/views/home/_signed_out_index.html.erb
View File 

@@ -2,9 +2,10 @@
2 2 
   <div class='row'>
3 3 
     <div class="span5 offset2">
4 4 
       <h1>Your agents are standing by</h1>
5 
-      <p>Know the world around you</p>
5 
+      <p>Huginn monitors the world and acts on your behalf.</p>
6 6
7 
-      <%= link_to "Signup", new_user_registration_path, :class => "btn btn-primary btn-large center" %>
7 
+      <%= link_to "Login", new_user_session_path, :class => "btn btn-large" %>
8 
+      <%= link_to "Signup", new_user_registration_path, :class => "btn btn-primary btn-large" %>
8 9 
     </div>
9 10 
     <div class="span3">
10 11 
       <%= image_tag 'odin.jpg', :class => 'img-rounded', :title => "Wägner, Wilhelm. 1882. Nordisch-germanische Götter und Helden. Otto Spamer, Leipzig & Berlin. Page 7." %>

+ 5 - 0
db/migrate/20131222211558_add_keep_events_for_to_agents.rb
View File 

@@ -0,0 +1,5 @@
1 
+class AddKeepEventsForToAgents < ActiveRecord::Migration
2 
+  def change
3 
+    add_column :agents, :keep_events_for, :integer, :null => false, :default => 0
4 
+  end
5 
+end

+ 6 - 3
db/schema.rb
View File 

@@ -11,7 +11,7 @@
11 11 
 #
12 12 
 # It's strongly recommended to check this file into your version control system.
13 13
14 
-ActiveRecord::Schema.define(:version => 20131105063248) do
14 
+ActiveRecord::Schema.define(:version => 20131227000021) do
15 15
16 16 
   create_table "agent_logs", :force => true do |t|
17 17 
     t.integer  "agent_id",                         :null => false
 
@@ -33,10 +33,13 @@ ActiveRecord::Schema.define(:version => 20131105063248) do
33 33 
     t.datetime "last_check_at"
34 34 
     t.datetime "last_receive_at"
35 35 
     t.integer  "last_checked_event_id"
36 
-    t.datetime "created_at",                                  :null => false
37 
-    t.datetime "updated_at",                                  :null => false
36 
+    t.datetime "created_at",                                                 :null => false
37 
+    t.datetime "updated_at",                                                 :null => false
38 38 
     t.text     "memory",                :limit => 2147483647
39 39 
     t.datetime "last_webhook_at"
40 
+    t.datetime "last_event_at"
41 
+    t.datetime "last_error_log_at"
42 
+    t.integer  "keep_events_for",                             :default => 0, :null => false
40 43 
   end
41 44
42 45 
   add_index "agents", ["schedule"], :name => "index_agents_on_schedule"

+ 1 - 1
spec/controllers/events_controller_spec.rb
View File 

@@ -49,7 +49,7 @@ describe EventsController do
49 49 
       }.should change { Event.count }.by(1)
50 50 
       Event.last.payload.should == events(:bob_website_agent_event).payload
51 51 
       Event.last.agent.should == events(:bob_website_agent_event).agent
52 
-      Event.last.created_at.should be_within(1).of(Time.now)
52 
+      Event.last.created_at.to_i.should be_within(2).of(Time.now.to_i)
53 53 
     end
54 54
55 55 
     it "can only re-emit Events for the current user" do

+ 4 - 0
spec/fixtures/agents.yml
View File 

@@ -1,6 +1,7 @@
1 1 
 jane_website_agent:
2 2 
   type: Agents::WebsiteAgent
3 3 
   user: jane
4 
+  events_count: 1
4 5 
   schedule: "5pm"
5 6 
   name: "ZKCD"
6 7 
   options: <%= {
 
@@ -16,6 +17,7 @@ jane_website_agent:
16 17 
 bob_website_agent:
17 18 
   type: Agents::WebsiteAgent
18 19 
   user: bob
20 
+  events_count: 1
19 21 
   schedule: "midnight"
20 22 
   name: "ZKCD"
21 23 
   options: <%= {
 
@@ -33,6 +35,7 @@ bob_weather_agent:
33 35 
   user: bob
34 36 
   schedule: "midnight"
35 37 
   name: "SF Weather"
38 
+  keep_events_for: 45
36 39 
   options: <%= {
37 40 
                  :location => 94102,
38 41 
                  :lat => 37.779329,
 
@@ -45,6 +48,7 @@ jane_weather_agent:
45 48 
   user: jane
46 49 
   schedule: "midnight"
47 50 
   name: "SF Weather"
51 
+  keep_events_for: 30
48 52 
   options: <%= {
49 53 
                  :location => 94103,
50 54 
                  :lat => 37.779329,

+ 104 - 2
spec/models/agent_spec.rb
View File 

@@ -249,7 +249,7 @@ describe Agent do
249 249 
         agent.should have(0).errors_on(:base)
250 250 
       end
251 251
252 
-      it "symbolizes options before validating" do
252 
+      it "makes options symbol-indifferent before validating" do
253 253 
         agent = Agents::SomethingSource.new(:name => "something")
254 254 
         agent.user = users(:bob)
255 255 
         agent.options["bad"] = true
 
@@ -258,7 +258,7 @@ describe Agent do
258 258 
         agent.should have(0).errors_on(:base)
259 259 
       end
260 260
261 
-      it "symbolizes memory before validating" do
261 
+      it "makes memory symbol-indifferent before validating" do
262 262 
         agent = Agents::SomethingSource.new(:name => "something")
263 263 
         agent.user = users(:bob)
264 264 
         agent.memory["bad"] = 2
 
@@ -318,7 +318,85 @@ describe Agent do
318 318 
         agent.user = users(:jane)
319 319 
         agent.should have(0).errors_on(:sources)
320 320 
       end
321 
+
322 
+      it "validates keep_events_for" do
323 
+        agent = Agents::SomethingSource.new(:name => "something")
324 
+        agent.user = users(:bob)
325 
+        agent.should be_valid
326 
+        agent.keep_events_for = nil
327 
+        agent.should have(1).errors_on(:keep_events_for)
328 
+        agent.keep_events_for = 1000
329 
+        agent.should have(1).errors_on(:keep_events_for)
330 
+        agent.keep_events_for = ""
331 
+        agent.should have(1).errors_on(:keep_events_for)
332 
+        agent.keep_events_for = 5
333 
+        agent.should be_valid
334 
+        agent.keep_events_for = 0
335 
+        agent.should be_valid
336 
+        agent.keep_events_for = 365
337 
+        agent.should be_valid
338 
+
339 
+        # Rails seems to call to_i on the input. This guards against future changes to that behavior.
340 
+        agent.keep_events_for = "drop table;"
341 
+        agent.keep_events_for.should == 0
342 
+      end
321 343 
     end
344 
+
345 
+    describe "cleaning up now-expired events" do
346 
+      before do
347 
+        @agent = Agents::SomethingSource.new(:name => "something")
348 
+        @agent.keep_events_for = 5
349 
+        @agent.user = users(:bob)
350 
+        @agent.save!
351 
+        @event = @agent.create_event :payload => { "hello" => "world" }
352 
+        @event.expires_at.to_i.should be_within(2).of(5.days.from_now.to_i)
353 
+      end
354 
+
355 
+      describe "when keep_events_for has not changed" do
356 
+        it "does nothing" do
357 
+          mock(@agent).update_event_expirations!.times(0)
358 
+
359 
+          @agent.options[:foo] = "bar1"
360 
+          @agent.save!
361 
+
362 
+          @agent.options[:foo] = "bar1"
363 
+          @agent.keep_events_for = 5
364 
+          @agent.save!
365 
+        end
366 
+      end
367 
+
368 
+      describe "when keep_events_for is changed" do
369 
+        it "updates events' expires_at" do
370 
+          lambda {
371 
+            @agent.options[:foo] = "bar1"
372 
+            @agent.keep_events_for = 3
373 
+            @agent.save!
374 
+          }.should change { @event.reload.expires_at }
375 
+          @event.expires_at.to_i.should be_within(2).of(3.days.from_now.to_i)
376 
+        end
377 
+
378 
+        it "updates events relative to their created_at" do
379 
+          @event.update_attribute :created_at, 2.days.ago
380 
+          @event.reload.created_at.to_i.should be_within(2).of(2.days.ago.to_i)
381 
+
382 
+          lambda {
383 
+            @agent.options[:foo] = "bar2"
384 
+            @agent.keep_events_for = 3
385 
+            @agent.save!
386 
+          }.should change { @event.reload.expires_at }
387 
+          @event.expires_at.to_i.should be_within(2).of(1.days.from_now.to_i)
388 
+        end
389 
+
390 
+        it "nulls out expires_at when keep_events_for is set to 0" do
391 
+          lambda {
392 
+            @agent.options[:foo] = "bar"
393 
+            @agent.keep_events_for = 0
394 
+            @agent.save!
395 
+          }.should change { @event.reload.expires_at }.to(nil)
396 
+        end
397 
+      end
398 
+    end
399 
+
322 400 
   end
323 401
324 402 
   describe "recent_error_logs?" do
 
@@ -371,4 +449,28 @@ describe Agent do
371 449 
       end
372 450 
     end
373 451 
   end
452 
+
453 
+  describe "#create_event" do
454 
+    describe "when the agent has keep_events_for set" do
455 
+      before do
456 
+        agents(:jane_weather_agent).keep_events_for.should > 0
457 
+      end
458 
+
459 
+      it "sets expires_at on created events" do
460 
+        event = agents(:jane_weather_agent).create_event :payload => { 'hi' => 'there' }
461 
+        event.expires_at.to_i.should be_within(5).of(agents(:jane_weather_agent).keep_events_for.days.from_now.to_i)
462 
+      end
463 
+    end
464 
+
465 
+    describe "when the agent does not have keep_events_for set" do
466 
+      before do
467 
+        agents(:jane_website_agent).keep_events_for.should == 0
468 
+      end
469 
+
470 
+      it "does not set expires_at on created events" do
471 
+        event = agents(:jane_website_agent).create_event :payload => { 'hi' => 'there' }
472 
+        event.expires_at.should be_nil
473 
+      end
474 
+    end
475 
+  end
374 476 
 end

+ 68 - 68
spec/models/agents/website_agent_spec.rb
View File 

@@ -4,15 +4,15 @@ describe Agents::WebsiteAgent do
4 4 
   before do
5 5 
     stub_request(:any, /xkcd/).to_return(:body => File.read(Rails.root.join("spec/data_fixtures/xkcd.html")), :status => 200)
6 6 
     @site = {
7 
-        :name => "XKCD",
8 
-        :expected_update_period_in_days => 2,
9 
-        :type => "html",
10 
-        :url => "http://xkcd.com",
11 
-        :mode => :on_change,
12 
-        :extract => {
13 
-            :url => {:css => "#comic img", :attr => "src"},
14 
-            :title => {:css => "#comic img", :attr => "title"}
15 
-        }
7 
+      'name' => "XKCD",
8 
+      'expected_update_period_in_days' => 2,
9 
+      'type' => "html",
10 
+      'url' => "http://xkcd.com",
11 
+      'mode' => 'on_change',
12 
+      'extract' => {
13 
+        'url' => {'css' => "#comic img", 'attr' => "src"},
14 
+        'title' => {'css' => "#comic img", 'attr' => "title"}
15 
+      }
16 16 
     }
17 17 
     @checker = Agents::WebsiteAgent.new(:name => "xkcd", :options => @site)
18 18 
     @checker.user = users(:bob)
 
@@ -27,7 +27,7 @@ describe Agents::WebsiteAgent do
27 27
28 28 
     it "should always save events when in :all mode" do
29 29 
       lambda {
30 
-        @site[:mode] = :all
30 
+        @site['mode'] = 'all'
31 31 
         @checker.options = @site
32 32 
         @checker.check
33 33 
         @checker.check
 
@@ -35,7 +35,7 @@ describe Agents::WebsiteAgent do
35 35 
     end
36 36
37 37 
     it "should log an error if the number of results for a set of extraction patterns differs" do
38 
-      @site[:extract][:url][:css] = "div"
38 
+      @site['extract']['url']['css'] = "div"
39 39 
       @checker.options = @site
40 40 
       @checker.check
41 41 
       @checker.logs.first.message.should =~ /Got an uneven number of matches/
 
@@ -68,20 +68,20 @@ describe Agents::WebsiteAgent do
68 68 
     it "parses CSS" do
69 69 
       @checker.check
70 70 
       event = Event.last
71 
-      event.payload[:url].should == "http://imgs.xkcd.com/comics/evolving.png"
72 
-      event.payload[:title].should =~ /^Biologists play reverse/
71 
+      event.payload['url'].should == "http://imgs.xkcd.com/comics/evolving.png"
72 
+      event.payload['title'].should =~ /^Biologists play reverse/
73 73 
     end
74 74
75 75 
     it "should turn relative urls to absolute" do
76 76 
       rel_site = {
77 
-        :name => "XKCD",
78 
-        :expected_update_period_in_days => 2,
79 
-        :type => "html",
80 
-        :url => "http://xkcd.com",
81 
-        :mode => :on_change,
82 
-        :extract => {
83 
-            :url => {:css => "#topLeft a", :attr => "href"},
84 
-            :title => {:css => "#topLeft a", :text => "true"}
77 
+        'name' => "XKCD",
78 
+        'expected_update_period_in_days' => 2,
79 
+        'type' => "html",
80 
+        'url' => "http://xkcd.com",
81 
+        'mode' => :on_change,
82 
+        'extract' => {
83 
+          'url' => {'css' => "#topLeft a", 'attr' => "href"},
84 
+          'title' => {'css' => "#topLeft a", 'text' => "true"}
85 85 
         }
86 86 
       }
87 87 
       rel = Agents::WebsiteAgent.new(:name => "xkcd", :options => rel_site)
 
@@ -89,28 +89,28 @@ describe Agents::WebsiteAgent do
89 89 
       rel.save!
90 90 
       rel.check
91 91 
       event = Event.last
92 
-      event.payload[:url].should == "http://xkcd.com/about"
92 
+      event.payload['url'].should == "http://xkcd.com/about"
93 93 
     end
94 
-
94 
+
95 95 
     describe "JSON" do
96 96 
       it "works with paths" do
97 97 
         json = {
98 
-            :response => {
99 
-                :version => 2,
100 
-                :title => "hello!"
101 
-            }
98 
+          'response' => {
99 
+            'version' => 2,
100 
+            'title' => "hello!"
101 
+          }
102 102 
         }
103 103 
         stub_request(:any, /json-site/).to_return(:body => json.to_json, :status => 200)
104 104 
         site = {
105 
-            :name => "Some JSON Response",
106 
-            :expected_update_period_in_days => 2,
107 
-            :type => "json",
108 
-            :url => "http://json-site.com",
109 
-            :mode => :on_change,
110 
-            :extract => {
111 
-                :version => { :path => "response.version" },
112 
-                :title => { :path => "response.title" }
113 
-            }
105 
+          'name' => "Some JSON Response",
106 
+          'expected_update_period_in_days' => 2,
107 
+          'type' => "json",
108 
+          'url' => "http://json-site.com",
109 
+          'mode' => 'on_change',
110 
+          'extract' => {
111 
+            'version' => {'path' => "response.version"},
112 
+            'title' => {'path' => "response.title"}
113 
+          }
114 114 
         }
115 115 
         checker = Agents::WebsiteAgent.new(:name => "Weather Site", :options => site)
116 116 
         checker.user = users(:bob)
 
@@ -118,30 +118,30 @@ describe Agents::WebsiteAgent do
118 118
119 119 
         checker.check
120 120 
         event = Event.last
121 
-        event.payload[:version].should == 2
122 
-        event.payload[:title].should == "hello!"
121 
+        event.payload['version'].should == 2
122 
+        event.payload['title'].should == "hello!"
123 123 
       end
124 124
125 125 
       it "can handle arrays" do
126 126 
         json = {
127 
-            :response => {
128 
-                :data => [
129 
-                    { :title => "first", :version => 2 },
130 
-                    { :title => "second", :version => 2.5 }
131 
-                ]
132 
-            }
127 
+          'response' => {
128 
+            'data' => [
129 
+              {'title' => "first", 'version' => 2},
130 
+              {'title' => "second", 'version' => 2.5}
131 
+            ]
132 
+          }
133 133 
         }
134 134 
         stub_request(:any, /json-site/).to_return(:body => json.to_json, :status => 200)
135 135 
         site = {
136 
-            :name => "Some JSON Response",
137 
-            :expected_update_period_in_days => 2,
138 
-            :type => "json",
139 
-            :url => "http://json-site.com",
140 
-            :mode => :on_change,
141 
-            :extract => {
142 
-                :title => { :path => "response.data[*].title" },
143 
-                :version => { :path => "response.data[*].version" }
144 
-            }
136 
+          'name' => "Some JSON Response",
137 
+          'expected_update_period_in_days' => 2,
138 
+          'type' => "json",
139 
+          'url' => "http://json-site.com",
140 
+          'mode' => 'on_change',
141 
+          'extract' => {
142 
+            :title => {'path' => "response.data[*].title"},
143 
+            :version => {'path' => "response.data[*].version"}
144 
+          }
145 145 
         }
146 146 
         checker = Agents::WebsiteAgent.new(:name => "Weather Site", :options => site)
147 147 
         checker.user = users(:bob)
 
@@ -152,28 +152,28 @@ describe Agents::WebsiteAgent do
152 152 
         }.should change { Event.count }.by(2)
153 153
154 154 
         event = Event.all[-1]
155 
-        event.payload[:version].should == 2.5
156 
-        event.payload[:title].should == "second"
155 
+        event.payload['version'].should == 2.5
156 
+        event.payload['title'].should == "second"
157 157
158 158 
         event = Event.all[-2]
159 
-        event.payload[:version].should == 2
160 
-        event.payload[:title].should == "first"
159 
+        event.payload['version'].should == 2
160 
+        event.payload['title'].should == "first"
161 161 
       end
162 162
163 163 
       it "stores the whole object if :extract is not specified" do
164 164 
         json = {
165 
-            :response => {
166 
-                :version => 2,
167 
-                :title => "hello!"
168 
-            }
165 
+          'response' => {
166 
+            'version' => 2,
167 
+            'title' => "hello!"
168 
+          }
169 169 
         }
170 170 
         stub_request(:any, /json-site/).to_return(:body => json.to_json, :status => 200)
171 171 
         site = {
172 
-            :name => "Some JSON Response",
173 
-            :expected_update_period_in_days => 2,
174 
-            :type => "json",
175 
-            :url => "http://json-site.com",
176 
-            :mode => :on_change
172 
+          'name' => "Some JSON Response",
173 
+          'expected_update_period_in_days' => 2,
174 
+          'type' => "json",
175 
+          'url' => "http://json-site.com",
176 
+          'mode' => 'on_change'
177 177 
         }
178 178 
         checker = Agents::WebsiteAgent.new(:name => "Weather Site", :options => site)
179 179 
         checker.user = users(:bob)
 
@@ -181,8 +181,8 @@ describe Agents::WebsiteAgent do
181 181
182 182 
         checker.check
183 183 
         event = Event.last
184 
-        event.payload[:response][:version].should == 2
185 
-        event.payload[:response][:title].should == "hello!"
184 
+        event.payload['response']['version'].should == 2
185 
+        event.payload['response']['title'].should == "hello!"
186 186 
       end
187 187 
     end
188 188 
   end

+ 33 - 8
spec/models/event_spec.rb
View File 

@@ -13,25 +13,50 @@ describe Event do
13 13 
       Event.last.agent.should == events(:bob_website_agent_event).agent
14 14 
       Event.last.lat.should == 2
15 15 
       Event.last.lng.should == 3
16 
-      Event.last.created_at.should be_within(1).of(Time.now)
16 
+      Event.last.created_at.to_i.should be_within(2).of(Time.now.to_i)
17 17 
     end
18 18 
   end
19 19
20 20 
   describe ".cleanup_expired!" do
21 
-    it "removes any Events whose expired_at date is non-null and in the past" do
22 
-      event = agents(:jane_weather_agent).create_event :expires_at => 2.hours.from_now
21 
+    it "removes any Events whose expired_at date is non-null and in the past, updating Agent counter caches" do
22 
+      half_hour_event = agents(:jane_weather_agent).create_event :expires_at => 20.minutes.from_now
23 
+      one_hour_event = agents(:bob_weather_agent).create_event :expires_at => 1.hours.from_now
24 
+      two_hour_event = agents(:jane_weather_agent).create_event :expires_at => 2.hours.from_now
25 
+      three_hour_event = agents(:jane_weather_agent).create_event :expires_at => 3.hours.from_now
26 
+      non_expiring_event = agents(:bob_weather_agent).create_event({})
27 
+
28 
+      initial_bob_count = agents(:bob_weather_agent).reload.events_count
29 
+      initial_jane_count = agents(:jane_weather_agent).reload.events_count
23 30
24 31 
       current_time = Time.now
25 32 
       stub(Time).now { current_time }
26 33
27 34 
       Event.cleanup_expired!
28 
-      Event.find_by_id(event.id).should_not be_nil
29 
-      current_time = 119.minutes.from_now
35 
+      Event.find_by_id(half_hour_event.id).should_not be_nil
36 
+      Event.find_by_id(one_hour_event.id).should_not be_nil
37 
+      Event.find_by_id(two_hour_event.id).should_not be_nil
38 
+      Event.find_by_id(three_hour_event.id).should_not be_nil
39 
+      Event.find_by_id(non_expiring_event.id).should_not be_nil
40 
+      agents(:bob_weather_agent).reload.events_count.should == initial_bob_count
41 
+      agents(:jane_weather_agent).reload.events_count.should == initial_jane_count
42 
+
43 
+      current_time = 119.minutes.from_now # move almost 2 hours into the future
30 44 
       Event.cleanup_expired!
31 
-      Event.find_by_id(event.id).should_not be_nil
32 
-      current_time = 2.minutes.from_now
45 
+      Event.find_by_id(half_hour_event.id).should be_nil
46 
+      Event.find_by_id(one_hour_event.id).should be_nil
47 
+      Event.find_by_id(two_hour_event.id).should_not be_nil
48 
+      Event.find_by_id(three_hour_event.id).should_not be_nil
49 
+      Event.find_by_id(non_expiring_event.id).should_not be_nil
50 
+      agents(:bob_weather_agent).reload.events_count.should == initial_bob_count - 1
51 
+      agents(:jane_weather_agent).reload.events_count.should == initial_jane_count - 1
52 
+
53 
+      current_time = 2.minutes.from_now # move 2 minutes further into the future
33 54 
       Event.cleanup_expired!
34 
-      Event.find_by_id(event.id).should be_nil
55 
+      Event.find_by_id(two_hour_event.id).should be_nil
56 
+      Event.find_by_id(three_hour_event.id).should_not be_nil
57 
+      Event.find_by_id(non_expiring_event.id).should_not be_nil
58 
+      agents(:bob_weather_agent).reload.events_count.should == initial_bob_count - 1
59 
+      agents(:jane_weather_agent).reload.events_count.should == initial_jane_count - 2
35 60 
     end
36 61
37 62 
     it "doesn't touch Events with no expired_at" do