Home Explore Help
Register Sign In
J1X
/
huginn
Watch 2
Star 0
Fork 0

Browse Source
Fixed conflict in docker/scripts/init with bind to 0.0.0.0

Ian Blenke 9 years ago
parent
0764e95b53 671e0105fb
commit
16b4c6cad1
58 changed files with 1404 additions and 382 deletions
Show Diff Stats
  1. 8 1
      .env.example
  2. 9 0
      CHANGES.md
  3. 4 1
      Gemfile
  4. 7 4
      Gemfile.lock
  5. 4 2
      README.md
  6. 1 0
      app.json
  7. 61 4
      app/assets/javascripts/components/utils.js.coffee
  8. 31 15
      app/concerns/dry_runnable.rb
  9. 72 7
      app/concerns/liquid_interpolatable.rb
  10. 161 0
      app/concerns/sortable_events.rb
  11. 79 0
      app/concerns/web_request_concern.rb
  12. 8 2
      app/controllers/agents_controller.rb
  13. 12 0
      app/helpers/agent_helper.rb
  14. 1 0
      app/helpers/application_helper.rb
  15. 16 8
      app/models/agent.rb
  16. 6 2
      app/models/agents/data_output_agent.rb
  17. 1 1
      app/models/agents/email_agent.rb
  18. 1 1
      app/models/agents/email_digest_agent.rb
  19. 56 29
      app/models/agents/rss_agent.rb
  20. 6 20
      app/models/agents/website_agent.rb
  21. 30 18
      app/models/scenario_import.rb
  22. 5 1
      app/models/user.rb
  23. 1 1
      app/views/agents/_action_menu.html.erb
  24. 1 1
      app/views/agents/_options.erb
  25. 8 6
      app/views/devise/registrations/new.html.erb
  26. 1 1
      app/views/scenario_imports/_step_two.html.erb
  27. 2 12
      config/initializers/action_mailer.rb
  28. 1 1
      config/initializers/delayed_job.rb
  29. 8 0
      config/initializers/liquid.rb
  30. 1 1
      config/routes.rb
  31. 13 0
      db/migrate/20150507153436_update_keep_events_for_to_be_in_seconds.rb
  32. 5 0
      db/migrate/20150808115436_remove_requirement_from_users_invitation_code.rb
  33. 77 77
      db/schema.rb
  34. 0 5
      deployment/heroku/unicorn.rb
  35. 3 0
      deployment/site-cookbooks/huginn_production/files/default/env.example
  36. 7 2
      docker/scripts/init
  37. 17 3
      lib/agents_exporter.rb
  38. 0 3
      lib/ar_mysql_column_charset.rb
  39. 1 1
      lib/huginn_scheduler.rb
  40. 0 85
      lib/prepend.rb
  41. 39 0
      lib/utils.rb
  42. 46 17
      spec/concerns/dry_runnable_spec.rb
  43. 39 15
      spec/concerns/liquid_interpolatable_spec.rb
  44. 264 0
      spec/concerns/sortable_events_spec.rb
  45. 18 1
      spec/controllers/agents_controller_spec.rb
  46. 2 2
      spec/fixtures/agents.yml
  47. 9 0
      spec/lib/agents_exporter_spec.rb
  48. 58 0
      spec/lib/utils_spec.rb
  49. 11 11
      spec/models/agent_spec.rb
  50. 16 0
      spec/models/agents/data_output_agent_spec.rb
  51. 1 1
      spec/models/agents/ftpsite_agent_spec.rb
  52. 1 1
      spec/models/agents/imap_folder_agent_spec.rb
  53. 25 0
      spec/models/agents/rss_agent_spec.rb
  54. 2 2
      spec/models/agents/website_agent_spec.rb
  55. 89 9
      spec/models/scenario_import_spec.rb
  56. 25 7
      spec/models/users_spec.rb
  57. 33 0
      spec/support/shared_examples/web_request_concern.rb
  58. 1 1
      spec/support/vcr_support.rb

+ 8 - 1
.env.example
View File 

@@ -47,6 +47,9 @@ FORCE_SSL=false
47 47 
 # You can see its use in user.rb.  PLEASE CHANGE THIS!
48 48 
 INVITATION_CODE=try-huginn
49 49
50 
+# If you don't want to require new users to have an invitation code in order to sign up, set this to true.
51 
+SKIP_INVITATION_CODE=false
52 
+
50 53 
 #############################
51 54 
 #    Email Configuration    #
52 55 
 #############################
 
@@ -151,6 +154,10 @@ ENABLE_SECOND_PRECISION_SCHEDULE=false
151 154 
 # at the expense of time accuracy.
152 155 
 SCHEDULER_FREQUENCY=0.3
153 156
157 
+# Specify the frequency with which the scheduler checks for and cleans up expired events.
158 
+# You can use `m` for minutes, `h` for hours, and `d` for days.
159 
+EVENT_EXPIRATION_CHECK=6h
160 
+
154 161 
 # Use Graphviz for generating diagrams instead of using Google Chart
155 162 
 # Tools.  Specify a dot(1) command path built with SVG support
156 163 
 # enabled.
 
@@ -163,7 +170,7 @@ TIMEZONE="Pacific Time (US & Canada)"
163 170 
 FAILED_JOBS_TO_KEEP=100
164 171
165 172 
 # Maximum runtime of background jobs in minutes
166 
-DELAYED_JOB_MAX_RUNTIME=20
173 
+DELAYED_JOB_MAX_RUNTIME=2
167 174
168 175 
 # Amount of seconds for delayed_job to sleep before checking for new jobs
169 176 
 DELAYED_JOB_SLEEP_DELAY=10

+ 9 - 0
CHANGES.md
View File 

@@ -1,10 +1,19 @@
1 1 
 # Changes
2 2
3 
+* Jul 30, 2015   - RssAgent can configure the order of events created via `events_order`.
4 
+* Jul 29, 2015   - WebsiteAgent can configure the order of events created via `events_order`.
5 
+* Jul 29, 2015   - DataOutputAgent can configure the order of events in the output via `events_order`.
6 
+* Jul 20, 2015   - Control Links (used by the SchedularAgent) are correctly exported in Scenarios.
7 
+* Jul 20, 2015   - keep\_events\_for was moved from days to seconds; Scenarios have a schema verison.
8 
+* Jul 8, 2015    - DataOutputAgent supports feed icon, and a new template variable `events`.
3 9 
 * Jul 1, 2015    - DeDuplicationAgent properly handles destruction of memory.
4 10 
 * Jun 26, 2015   - Add `max_events_per_run` to RssAgent.
5 11 
 * Jun 19, 2015   - Add `url_from_event` to WebsiteAgent.
6 12 
 * Jun 17, 2015   - RssAgent emits events for new feed items in chronological order.
13 
+* Jun 17, 2015   - Liquid filter `unescape` added.
14 
+* Jun 17, 2015   - Liquid filter `regex_replace` and `regex_replace_first` added, with escape sequence support.
7 15 
 * Jun 15, 2015   - Liquid filter `uri_expand` added.
16 
+* Jun 13, 2015   - Liquid templating engine is upgraded to version 3.
8 17 
 * Jun 12, 2015   - RSSAgent can now accept an array of URLs.
9 18 
 * Jun 8, 2015    - WebsiteAgent includes a `use_namespaces` option to enable XML namespaces.
10 19 
 * May 27, 2015   - Validation warns user if they have not provided a `path` when using JSONPath in WebsiteAgent.

+ 4 - 1
Gemfile
View File 

@@ -1,5 +1,8 @@
1 1 
 source 'https://rubygems.org'
2 2
3 
+# Ruby 2.0 is the minimum requirement
4 
+ruby ['2.0.0', RUBY_VERSION].max
5 
+
3 6 
 # Optional libraries.  To conserve RAM, comment out any that you don't need,
4 7 
 # then run `bundle` and commit the updated Gemfile and Gemfile.lock.
5 8 
 gem 'twilio-ruby', '~> 3.11.5'    # TwilioAgent
 
@@ -75,7 +78,7 @@ gem 'json', '~> 1.8.1'
75 78 
 gem 'jsonpath', '~> 0.5.6'
76 79 
 gem 'kaminari', '~> 0.16.1'
77 80 
 gem 'kramdown', '~> 1.3.3'
78 
-gem 'liquid', '~> 2.6.1'
81 
+gem 'liquid', '~> 3.0.3'
79 82 
 gem 'mini_magick'
80 83 
 gem 'mysql2', '~> 0.3.16'
81 84 
 gem 'multi_xml'

+ 7 - 4
Gemfile.lock
View File 

@@ -224,7 +224,7 @@ GEM
224 224 
       railties (>= 3.0, < 5.0)
225 225 
       thor (>= 0.14, < 2.0)
226 226 
     json (1.8.3)
227 
-    jsonpath (0.5.6)
227 
+    jsonpath (0.5.7)
228 228 
       multi_json
229 229 
     jwt (1.4.1)
230 230 
     kaminari (0.16.1)
 
@@ -235,7 +235,7 @@ GEM
235 235 
     launchy (2.4.2)
236 236 
       addressable (~> 2.3)
237 237 
     libv8 (3.16.14.7)
238 
-    liquid (2.6.2)
238 
+    liquid (3.0.6)
239 239 
     listen (2.7.9)
240 240 
       celluloid (>= 0.15.2)
241 241 
       rb-fsevent (>= 0.9.3)
 
@@ -255,7 +255,7 @@ GEM
255 255 
     mini_portile (0.6.2)
256 256 
     minitest (5.7.0)
257 257 
     mqtt (0.3.1)
258 
-    multi_json (1.11.1)
258 
+    multi_json (1.11.2)
259 259 
     multi_xml (0.5.5)
260 260 
     multipart-post (2.0.0)
261 261 
     mysql2 (0.3.16)
 
@@ -529,7 +529,7 @@ DEPENDENCIES
529 529 
   jsonpath (~> 0.5.6)
530 530 
   kaminari (~> 0.16.1)
531 531 
   kramdown (~> 1.3.3)
532 
-  liquid (~> 2.6.1)
532 
+  liquid (~> 3.0.3)
533 533 
   mini_magick
534 534 
   mqtt
535 535 
   multi_xml
 
@@ -580,3 +580,6 @@ DEPENDENCIES
580 580 
   weibo_2!
581 581 
   wunderground (~> 1.2.0)
582 582 
   xmpp4r (~> 0.5.6)
583 
+
584 
+BUNDLED WITH
585 
+   1.10.5

+ 4 - 2
README.md
View File 

@@ -80,9 +80,11 @@ All agents have specs! Test all specs with `bundle exec rspec`, or test a specif
80 80
81 81 
 ## Deployment
82 82
83 
-[![Deploy](https://www.herokucdn.com/deploy/button.png)](https://heroku.com/deploy) (Takes a few minutes to setup.  Be sure to click 'View it' after launch!)
83 
+Try Huginn on Heroku: [![Deploy](https://www.herokucdn.com/deploy/button.png)](https://heroku.com/deploy) (Takes a few minutes to setup.  Be sure to click 'View it' after launch!)
84 84
85 
-Huginn can run on Heroku for free!  Please see [the Huginn Wiki](https://github.com/cantino/huginn/wiki#deploying-huginn) for detailed deployment strategies for different providers.
85 
+Huginn works on the free version of Heroku [with limitations](https://github.com/cantino/huginn/wiki/Run-Huginn-for-free-on-Heroku). For non-experimental use, we recommend Heroku's cheapest paid plan or our Docker container.
86 
+
87 
+Please see [the Huginn Wiki](https://github.com/cantino/huginn/wiki#deploying-huginn) for detailed deployment strategies for different providers.
86 88
87 89 
 ### Optional Setup
88 90

+ 1 - 0
app.json
View File 

@@ -3,6 +3,7 @@
3 3 
     "description": "Build agents that monitor and act on your behalf.  Your agents are standing by!",
4 4 
     "website": "https://github.com/cantino/huginn",
5 5 
     "repository": "https://github.com/cantino/huginn",
6 
+    "logo": "https://raw.githubusercontent.com/cantino/huginn/master/media/huginn-icon-64.png",
6 7 
     "env": {
7 8 
         "BUILDPACK_URL": "https://github.com/heroku/heroku-buildpack-multi.git",
8 9 
         "APP_SECRET_TOKEN": {

+ 61 - 4
app/assets/javascripts/components/utils.js.coffee
View File 

@@ -34,10 +34,67 @@ class @Utils
34 34 
     body?(modal.querySelector('.modal-body'))
35 35 
     $(modal).modal('show')
36 36
37 
-  @handleDryRunButton: (button, data = $(button.form).serialize()) ->
37 
+  @handleDryRunButton: (button, data = if button.form then $(':input[name!="_method"]', button.form).serialize() else '') ->
38 38 
     $(button).prop('disabled', true)
39 
+    cleanup = -> $(button).prop('disabled', false)
40 
+
41 
+    url = $(button).data('action-url')
42 
+    with_event_mode = $(button).data('with-event-mode')
43 
+
44 
+    if with_event_mode is 'no'
45 
+      return @invokeDryRun(url, data, cleanup)
46 
+
47 
+    Utils.showDynamicModal """
48 
+      <h5>Event to send#{if with_event_mode is 'maybe' then ' (Optional)' else ''}</h5>
49 
+      <form class="dry-run-form" method="post">
50 
+        <div class="form-group">
51 
+          <textarea rows="10" name="event" class="payload-editor" data-height="200">
52 
+            {}
53 
+          </textarea>
54 
+        </div>
55 
+        <div class="form-group">
56 
+          <input value="Dry Run" class="btn btn-primary" type="submit" />
57 
+        </div>
58 
+      </form>
59 
+      """,
60 
+      body: (body) =>
61 
+        form = $(body).find('.dry-run-form')
62 
+        payload_editor = form.find('.payload-editor')
63 
+        if previous = $(button).data('payload')
64 
+          payload_editor.text(previous)
65 
+        window.setupJsonEditor(payload_editor)
66 
+        form.submit (e) =>
67 
+          e.preventDefault()
68 
+          json = $(e.target).find('.payload-editor').val()
69 
+          json = '{}' if json == ''
70 
+          try
71 
+            payload = JSON.parse(json)
72 
+            throw true unless payload.constructor is Object
73 
+            if Object.keys(payload).length == 0
74 
+              json = ''
75 
+            else
76 
+              json = JSON.stringify(payload)
77 
+          catch
78 
+            alert 'Invalid JSON object.'
79 
+            return
80 
+          if json == ''
81 
+            if with_event_mode is 'yes'
82 
+              alert 'Event is required for this agent to run.'
83 
+              return
84 
+            dry_run_data = data
85 
+            $(button).data('payload', null)
86 
+          else
87 
+            dry_run_data = "event=#{encodeURIComponent(json)}&#{data}"
88 
+            $(button).data('payload', json)
89 
+          $(body).closest('[role=dialog]').on 'hidden.bs.modal', =>
90 
+            @invokeDryRun(url, dry_run_data, cleanup)
91 
+          .modal('hide')
92 
+      title: 'Dry Run'
93 
+      onHide: cleanup
94 
+
95 
+  @invokeDryRun: (url, data, callback) ->
39 96 
     $('body').css(cursor: 'progress')
40 
-    $.ajax type: 'POST', url: $(button).data('action-url'), dataType: 'json', data: data
97 
+    $.ajax type: 'POST', url: url, dataType: 'json', data: data
41 98 
       .always =>
42 99 
         $('body').css(cursor: 'auto')
43 100 
       .done (json) =>
 
@@ -55,7 +112,7 @@ class @Utils
55 112 
               find('.agent-dry-run-events').text(json.events).end().
56 113 
               find('.agent-dry-run-memory').text(json.memory)
57 114 
           title: 'Dry Run Results',
58 
-          onHide: -> $(button).prop('disabled', false)
115 
+          onHide: callback
59 116 
       .fail (xhr, status, error) ->
60 117 
         alert('Error: ' + error)
61 
-        $(button).prop('disabled', false)
118 
+        callback()

+ 31 - 15
app/concerns/dry_runnable.rb
View File 

@@ -1,10 +1,8 @@
1 1 
 module DryRunnable
2 
-  def dry_run!
3 
-    readonly!
2 
+  extend ActiveSupport::Concern
4 3
5 
-    class << self
6 
-      prepend Sandbox
7 
-    end
4 
+  def dry_run!(event = nil)
5 
+    @dry_run = true
8 6
9 7 
     log = StringIO.new
10 8 
     @dry_run_logger = Logger.new(log)
 
@@ -14,7 +12,13 @@ module DryRunnable
14 12
15 13 
     begin
16 14 
       raise "#{short_type} does not support dry-run" unless can_dry_run?
17 
-      check
15 
+      readonly!
16 
+      if event
17 
+        raise "This agent cannot receive an event!" unless can_receive_events?
18 
+        receive([event])
19 
+      else
20 
+        check
21 
+      end
18 22 
     rescue => e
19 23 
       error "Exception during dry-run. #{e.message}: #{e.backtrace.join("\n")}"
20 24 
     end
 
@@ -23,28 +27,38 @@ module DryRunnable
23 27 
       memory: memory,
24 28 
       log: log.string,
25 29 
     )
30 
+  ensure
31 
+    @dry_run = false
26 32 
   end
27 33
28 34 
   def dry_run?
29 
-    is_a? Sandbox
35 
+    !!@dry_run
36 
+  end
37 
+
38 
+  included do
39 
+    prepend Wrapper
30 40 
   end
31 41
32 
-  module Sandbox
42 
+  module Wrapper
33 43 
     attr_accessor :results
34 44
35 45 
     def logger
46 
+      return super unless dry_run?
36 47 
       @dry_run_logger
37 48 
     end
38 49
39 
-    def save
40 
-      valid?
50 
+    def save(options = {})
51 
+      return super unless dry_run?
52 
+      perform_validations(options)
41 53 
     end
42 54
43 
-    def save!
44 
-      save or raise ActiveRecord::RecordNotSaved
55 
+    def save!(options = {})
56 
+      return super unless dry_run?
57 
+      save(options) or raise_record_invalid
45 58 
     end
46 59
47 60 
     def log(message, options = {})
61 
+      return super unless dry_run?
48 62 
       case options[:level] || 3
49 63 
       when 0..2
50 64 
         sev = Logger::DEBUG
 
@@ -57,10 +71,12 @@ module DryRunnable
57 71 
       logger.log(sev, message)
58 72 
     end
59 73
60 
-    def create_event(event_hash)
74 
+    def create_event(event)
75 
+      return super unless dry_run?
61 76 
       if can_create_events?
62 
-        @dry_run_results[:events] << event_hash[:payload]
63 
-        events.build({ user: user, expires_at: new_event_expiration_date }.merge(event_hash))
77 
+        event = build_event(event)
78 
+        @dry_run_results[:events] << event.payload
79 
+        event
64 80 
       else
65 81 
         error "This Agent cannot create events!"
66 82 
       end

+ 72 - 7
app/concerns/liquid_interpolatable.rb
View File 

@@ -185,6 +185,14 @@ module LiquidInterpolatable
185 185 
       url
186 186 
     end
187 187
188 
+    # Unescape (basic) HTML entities in a string
189 
+    #
190 
+    # This currently decodes the following entities only: "&apos;",
191 
+    # "&quot;", "&lt;", "&gt;", "&amp;", "&#dd;" and "&#xhh;".
192 
+    def unescape(input)
193 
+      CGI.unescapeHTML(input) rescue input
194 
+    end
195 
+
188 196 
     # Escape a string for use in XPath expression
189 197 
     def to_xpath(string)
190 198 
       subs = string.to_s.scan(/\G(?:\A\z|[^"]+|[^']+)/).map { |x|
 
@@ -201,15 +209,15 @@ module LiquidInterpolatable
201 209 
         'concat(' << subs.join(', ') << ')'
202 210 
       end
203 211 
     end
204 
-
205 
-    def regex_replace(input, regex, replacement = ''.freeze)
206 
-      input.to_s.gsub(Regexp.new(regex), replacement.to_s)
212 
+
213 
+    def regex_replace(input, regex, replacement = nil)
214 
+      input.to_s.gsub(Regexp.new(regex), unescape_replacement(replacement.to_s))
207 215 
     end
208 
-
209 
-    def regex_replace_first(input, regex, replacement = ''.freeze)
210 
-      input.to_s.sub(Regexp.new(regex), replacement.to_s)
216 
+
217 
+    def regex_replace_first(input, regex, replacement = nil)
218 
+      input.to_s.sub(Regexp.new(regex), unescape_replacement(replacement.to_s))
211 219 
     end
212 
-
220 
+
213 221 
     private
214 222
215 223 
     def logger
 
@@ -221,6 +229,63 @@ module LiquidInterpolatable
221 229 
           Logger.new(STDERR)
222 230 
         end
223 231 
     end
232 
+
233 
+    BACKSLASH = "\\".freeze
234 
+
235 
+    UNESCAPE = {
236 
+      "a" => "\a",
237 
+      "b" => "\b",
238 
+      "e" => "\e",
239 
+      "f" => "\f",
240 
+      "n" => "\n",
241 
+      "r" => "\r",
242 
+      "s" => " ",
243 
+      "t" => "\t",
244 
+      "v" => "\v",
245 
+    }
246 
+
247 
+    # Unescape a replacement text for use in the second argument of
248 
+    # gsub/sub.  The following escape sequences are recognized:
249 
+    #
250 
+    # - "\\" (backslash itself)
251 
+    # - "\a" (alert)
252 
+    # - "\b" (backspace)
253 
+    # - "\e" (escape)
254 
+    # - "\f" (form feed)
255 
+    # - "\n" (new line)
256 
+    # - "\r" (carriage return)
257 
+    # - "\s" (space)
258 
+    # - "\t" (horizontal tab)
259 
+    # - "\u{XXXX}" (unicode codepoint)
260 
+    # - "\v" (vertical tab)
261 
+    # - "\xXX" (hexadecimal character)
262 
+    # - "\1".."\9" (numbered capture groups)
263 
+    # - "\+" (last capture group)
264 
+    # - "\k<name>" (named capture group)
265 
+    # - "\&" or "\0" (complete matched text)
266 
+    # - "\`" (string before match)
267 
+    # - "\'" (string after match)
268 
+    #
269 
+    # Octal escape sequences are deliberately unsupported to avoid
270 
+    # conflict with numbered capture groups.  Rather obscure Emacs
271 
+    # style character codes ("\C-x", "\M-\C-x" etc.) are also omitted
272 
+    # from this implementation.
273 
+    def unescape_replacement(s)
274 
+      s.gsub(/\\(?:([\d+&`'\\]|k<\w+>)|u\{([[:xdigit:]]+)\}|x([[:xdigit:]]{2})|(.))/) {
275 
+        if c = $1
276 
+          BACKSLASH + c
277 
+        elsif c = ($2 && [$2.to_i(16)].pack('U')) ||
278 
+                  ($3 && [$3.to_i(16)].pack('C'))
279 
+          if c == BACKSLASH
280 
+            BACKSLASH + c
281 
+          else
282 
+            c
283 
+          end
284 
+        else
285 
+          UNESCAPE[$4] || $4
286 
+        end
287 
+      }
288 
+    end
224 289 
   end
225 290 
   Liquid::Template.register_filter(LiquidInterpolatable::Filters)
226 291

+ 161 - 0
app/concerns/sortable_events.rb
View File 

@@ -0,0 +1,161 @@
1 
+module SortableEvents
2 
+  extend ActiveSupport::Concern
3 
+
4 
+  included do
5 
+    validate :validate_events_order
6 
+  end
7 
+
8 
+  def description_events_order(*args)
9 
+    self.class.description_events_order(*args)
10 
+  end
11 
+
12 
+  module ClassMethods
13 
+    def can_order_created_events!
14 
+      raise if cannot_create_events?
15 
+      prepend AutomaticSorter
16 
+    end
17 
+
18 
+    def can_order_created_events?
19 
+      include? AutomaticSorter
20 
+    end
21 
+
22 
+    def cannot_order_created_events?
23 
+      !can_order_created_events?
24 
+    end
25 
+
26 
+    def description_events_order(events = 'events created in each run')
27 
+      <<-MD.lstrip
28 
+        To specify the order of #{events}, set `events_order` to an array of sort keys, each of which looks like either `expression` or `[expression, type, descending]`, as described as follows:
29 
+
30 
+        * _expression_ is a Liquid template to generate a string to be used as sort key.
31 
+
32 
+        * _type_ (optional) is one of `string` (default), `number` and `time`, which specifies how to evaluate _expression_ for comparison.
33 
+
34 
+        * _descending_ (optional) is a boolean value to determine if comparison should be done in descending (reverse) order, which defaults to `false`.
35 
+
36 
+        Sort keys listed earlier take precedence over ones listed later.  For example, if you want to sort articles by the date and then by the author, specify `[["{{date}}", "time"], "{{author}}"]`.
37 
+
38 
+        Sorting is done stably, so even if all events have the same set of sort key values the original order is retained.  Also, a special Liquid variable `_index_` is provided, which contains the zero-based index number of each event, which means you can exactly reverse the order of events by specifying `[["{{_index_}}", "number", true]]`.
39 
+      MD
40 
+    end
41 
+  end
42 
+
43 
+  def can_order_created_events?
44 
+    self.class.can_order_created_events?
45 
+  end
46 
+
47 
+  def cannot_order_created_events?
48 
+    self.class.cannot_order_created_events?
49 
+  end
50 
+
51 
+  def events_order
52 
+    options['events_order']
53 
+  end
54 
+
55 
+  module AutomaticSorter
56 
+    def check
57 
+      return super unless events_order
58 
+      sorting_events do
59 
+        super
60 
+      end
61 
+    end
62 
+
63 
+    def receive(incoming_events)
64 
+      return super unless events_order
65 
+      # incoming events should be processed sequentially
66 
+      incoming_events.each do |event|
67 
+        sorting_events do
68 
+          super([event])
69 
+        end
70 
+      end
71 
+    end
72 
+
73 
+    def create_event(event)
74 
+      if @sortable_events
75 
+        event = build_event(event)
76 
+        @sortable_events << event
77 
+        event
78 
+      else
79 
+        super
80 
+      end
81 
+    end
82 
+
83 
+    private
84 
+
85 
+    def sorting_events(&block)
86 
+      @sortable_events = []
87 
+      yield
88 
+    ensure
89 
+      events, @sortable_events = @sortable_events, nil
90 
+      sort_events(events).each do |event|
91 
+        create_event(event)
92 
+      end
93 
+    end
94 
+  end
95 
+
96 
+  private
97 
+
98 
+  EXPRESSION_PARSER = {
99 
+    'string' => ->string { string },
100 
+    'number' => ->string { string.to_f },
101 
+    'time'   => ->string { Time.zone.parse(string) },
102 
+  }
103 
+  EXPRESSION_TYPES = EXPRESSION_PARSER.keys.freeze
104 
+
105 
+  def validate_events_order
106 
+    case order_by = events_order
107 
+    when nil
108 
+    when Array
109 
+      # Each tuple may be either [expression, type, desc] or just
110 
+      # expression.
111 
+      order_by.each do |expression, type, desc|
112 
+        case expression
113 
+        when String
114 
+          # ok
115 
+        else
116 
+          errors.add(:base, "first element of each events_order tuple must be a Liquid template")
117 
+          break
118 
+        end
119 
+        case type
120 
+        when nil, *EXPRESSION_TYPES
121 
+          # ok
122 
+        else
123 
+          errors.add(:base, "second element of each events_order tuple must be #{EXPRESSION_TYPES.to_sentence(last_word_connector: ' or ')}")
124 
+          break
125 
+        end
126 
+        if !desc.nil? && boolify(desc).nil?
127 
+          errors.add(:base, "third element of each events_order tuple must be a boolean value")
128 
+          break
129 
+        end
130 
+      end
131 
+    else
132 
+      errors.add(:base, "events_order must be an array of arrays")
133 
+    end
134 
+  end
135 
+
136 
+  # Sort given events in order specified by the "events_order" option
137 
+  def sort_events(events)
138 
+    order_by = events_order.presence or
139 
+      return events
140 
+
141 
+    orders = order_by.map { |_, _, desc = false| boolify(desc) }
142 
+
143 
+    Utils.sort_tuples!(
144 
+      events.map.with_index { |event, index|
145 
+        interpolate_with(event) {
146 
+          interpolation_context['_index_'] = index
147 
+          order_by.map { |expression, type, _|
148 
+            string = interpolate_string(expression)
149 
+            begin
150 
+              EXPRESSION_PARSER[type || 'string'.freeze][string]
151 
+            rescue
152 
+              error "Cannot parse #{string.inspect} as #{type}; treating it as string"
153 
+              string
154 
+            end
155 
+          }
156 
+        } << index << event  # index is to make sorting stable
157 
+      },
158 
+      orders
159 
+    ).collect!(&:last)
160 
+  end
161 
+end

+ 79 - 0
app/concerns/web_request_concern.rb
View File 

@@ -2,6 +2,58 @@ require 'faraday'
2 2 
 require 'faraday_middleware'
3 3
4 4 
 module WebRequestConcern
5 
+  module DoNotEncoder
6 
+    def self.encode(params)
7 
+      params.map do |key, value|
8 
+        value.nil? ? "#{key}" : "#{key}=#{value}"
9 
+      end.join('&')
10 
+    end
11 
+
12 
+    def self.decode(val)
13 
+      [val]
14 
+    end
15 
+  end
16 
+
17 
+  class CharacterEncoding < Faraday::Middleware
18 
+    def initialize(app, force_encoding: nil, default_encoding: nil, unzip: nil)
19 
+      super(app)
20 
+      @force_encoding   = force_encoding
21 
+      @default_encoding = default_encoding
22 
+      @unzip            = unzip
23 
+    end
24 
+
25 
+    def call(env)
26 
+      @app.call(env).on_complete do |env|
27 
+        body = env[:body]
28 
+
29 
+        case @unzip
30 
+        when 'gzip'.freeze
31 
+          body.replace(ActiveSupport::Gzip.decompress(body))
32 
+        end
33 
+
34 
+        case
35 
+        when @force_encoding
36 
+          encoding = @force_encoding
37 
+        when body.encoding == Encoding::ASCII_8BIT
38 
+          # Not all Faraday adapters support automatic charset
39 
+          # detection, so we do that.
40 
+          case env[:response_headers][:content_type]
41 
+          when /;\s*charset\s*=\s*([^()<>@,;:\\\"\/\[\]?={}\s]+)/i
42 
+            encoding = Encoding.find($1) rescue nil
43 
+          when /\A\s*(?:text\/[^\s;]+|application\/(?:[^\s;]+\+)?(?:xml|json))\s*(?:;|\z)/i
44 
+            encoding = @default_encoding
45 
+          else
46 
+            # Never try to transcode a binary content
47 
+            return
48 
+          end
49 
+        end
50 
+        body.encode!(Encoding::UTF_8, encoding) unless body.encoding == Encoding::UTF_8
51 
+      end
52 
+    end
53 
+  end
54 
+
55 
+  Faraday::Response.register_middleware character_encoding: CharacterEncoding
56 
+
5 57 
   extend ActiveSupport::Concern
6 58
7 59 
   def validate_web_request_options!
 
@@ -22,6 +74,23 @@ module WebRequestConcern
22 74 
     rescue ArgumentError => e
23 75 
       errors.add(:base, e.message)
24 76 
     end
77 
+
78 
+    if (encoding = options['force_encoding']).present?
79 
+      case encoding
80 
+      when String
81 
+        begin
82 
+          Encoding.find(encoding)
83 
+        rescue ArgumentError
84 
+          errors.add(:base, "Unknown encoding: #{encoding.inspect}")
85 
+        end
86 
+      else
87 
+        errors.add(:base, "force_encoding must be a string")
88 
+      end
89 
+    end
90 
+  end
91 
+
92 
+  def default_encoding
93 
+    Encoding::UTF_8
25 94 
   end
26 95
27 96 
   def faraday
 
@@ -32,12 +101,22 @@ module WebRequestConcern
32 101 
     }
33 102
34 103 
     @faraday ||= Faraday.new(faraday_options) { |builder|
104 
+      builder.response :character_encoding,
105 
+                       force_encoding: interpolated['force_encoding'].presence,
106 
+                       default_encoding: default_encoding,
107 
+                       unzip: interpolated['unzip'].presence
108 
+
35 109 
       builder.headers = headers if headers.length > 0
36 110
37 111 
       builder.headers[:user_agent] = user_agent
38 112
39 113 
       builder.use FaradayMiddleware::FollowRedirects
40 114 
       builder.request :url_encoded
115 
+
116 
+      if boolify(interpolated['disable_url_encoding'])
117 
+        builder.options.params_encoder = DoNotEncoder
118 
+      end
119 
+
41 120 
       if userinfo = basic_auth_credentials
42 121 
         builder.request :basic_auth, *userinfo
43 122 
       end

+ 8 - 2
app/controllers/agents_controller.rb
View File 

@@ -37,7 +37,7 @@ class AgentsController < ApplicationController
37 37 
   def dry_run
38 38 
     attrs = params[:agent] || {}
39 39 
     if agent = current_user.agents.find_by(id: params[:id])
40 
-      # PUT /agents/:id/dry_run
40 
+      # POST /agents/:id/dry_run
41 41 
       if attrs.present?
42 42 
         type = agent.type
43 43 
         agent = Agent.build_for_type(type, current_user, attrs)
 
@@ -50,7 +50,13 @@ class AgentsController < ApplicationController
50 50 
     agent.name ||= '(Untitled)'
51 51
52 52 
     if agent.valid?
53 
-      results = agent.dry_run!
53 
+      if event_payload = params[:event]
54 
+        dummy_agent = Agent.build_for_type('ManualEventAgent', current_user, name: 'Dry-Runner')
55 
+        dummy_agent.readonly!
56 
+        event = dummy_agent.events.build(user: current_user, payload: event_payload)
57 
+      end
58 
+
59 
+      results = agent.dry_run!(event)
54 60
55 61 
       render json: {
56 62 
         log: results[:log],

+ 12 - 0
app/helpers/agent_helper.rb
View File 

@@ -37,4 +37,16 @@ module AgentHelper
37 37 
       }.join(delimiter).html_safe
38 38 
     end
39 39 
   end
40 
+
41 
+  def agent_dry_run_with_event_mode(agent)
42 
+    case
43 
+    when agent.cannot_receive_events?
44 
+      'no'.freeze
45 
+    when agent.cannot_be_scheduled?
46 
+      # incoming event is the only trigger for the agent
47 
+      'yes'.freeze
48 
+    else
49 
+      'maybe'.freeze
50 
+    end
51 
+  end
40 52 
 end

+ 1 - 0
app/helpers/application_helper.rb
View File 

@@ -80,6 +80,7 @@ module ApplicationHelper
80 80 
   end
81 81
82 82 
   def service_label(service)
83 
+    return if service.nil?
83 84 
     content_tag :span, [
84 85 
       omniauth_provider_icon(service.provider),
85 86 
       service_label_text(service)

+ 16 - 8
app/models/agent.rb
View File 

@@ -13,6 +13,7 @@ class Agent < ActiveRecord::Base
13 13 
   include HasGuid
14 14 
   include LiquidDroppable
15 15 
   include DryRunnable
16 
+  include SortableEvents
16 17
17 18 
   markdown_class_attributes :description, :event_description
18 19 
 
@@ -21,7 +22,7 @@ class Agent < ActiveRecord::Base
21 22 
   SCHEDULES = %w[every_1m every_2m every_5m every_10m every_30m every_1h every_2h every_5h every_12h every_1d every_2d every_7d
22 23 
                  midnight 1am 2am 3am 4am 5am 6am 7am 8am 9am 10am 11am noon 1pm 2pm 3pm 4pm 5pm 6pm 7pm 8pm 9pm 10pm 11pm never]
23 24
24 
-  EVENT_RETENTION_SCHEDULES = [["Forever", 0], ["1 day", 1], *([2, 3, 4, 5, 7, 14, 21, 30, 45, 90, 180, 365].map {|n| ["#{n} days", n] })]
25 
+  EVENT_RETENTION_SCHEDULES = [["Forever", 0], ['1 hour', 1.hour], ['6 hours', 6.hours], ["1 day", 1.day], *([2, 3, 4, 5, 7, 14, 21, 30, 45, 90, 180, 365].map {|n| ["#{n} days", n.days] })]
25 26
26 27 
   attr_accessible :options, :memory, :name, :type, :schedule, :controller_ids, :control_target_ids, :disabled, :source_ids, :scenario_ids, :keep_events_for, :propagate_immediately, :drop_pending_events
27 28 
 
@@ -104,12 +105,19 @@ class Agent < ActiveRecord::Base
104 105 
     raise "Implement me in your subclass"
105 106 
   end
106 107
107 
-  def create_event(attrs)
108 
+  def build_event(event)
109 
+    event = events.build(event) if event.is_a?(Hash)
110 
+    event.agent = self
111 
+    event.user = user
112 
+    event.expires_at ||= new_event_expiration_date
113 
+    event
114 
+  end
115 
+
116 
+  def create_event(event)
108 117 
     if can_create_events?
109 
-      events.create!({
110 
-         :user => user,
111 
-         :expires_at => new_event_expiration_date
112 
-      }.merge(attrs))
118 
+      event = build_event(event)
119 
+      event.save!
120 
+      event
113 121 
     else
114 122 
       error "This Agent cannot create events!"
115 123 
     end
 
@@ -130,14 +138,14 @@ class Agent < ActiveRecord::Base
130 138 
   end
131 139
132 140 
   def new_event_expiration_date
133 
-    keep_events_for > 0 ? keep_events_for.days.from_now : nil
141 
+    keep_events_for > 0 ? keep_events_for.seconds.from_now : nil
134 142 
   end
135 143
136 144 
   def update_event_expirations!
137 145 
     if keep_events_for == 0
138 146 
       events.update_all :expires_at => nil
139 147 
     else
140 
-      events.update_all "expires_at = " + rdbms_date_add("created_at", "DAY", keep_events_for.to_i)
148 
+      events.update_all "expires_at = " + rdbms_date_add("created_at", "SECOND", keep_events_for.to_i)
141 149 
     end
142 150 
   end
143 151

+ 6 - 2
app/models/agents/data_output_agent.rb
View File 

@@ -40,11 +40,15 @@ module Agents
40 40 
               "_contents": "tag contents (can be an object for nesting)"
41 41 
             }
42 42
43 
+        # Ordering events in the output
44 
+
45 
+        #{description_events_order('events in the output')}
46 
+
43 47 
         # Liquid Templating
44 48
45 49 
         In Liquid templating, the following variable is available:
46 50
47 
-        * `events`: An array of events being output, sorted in descending order up to `events_to_show` in number.  For example, if source events contain a site title in the `site_title` key, you can refer to it in `template.title` by putting `{{events.first.site_title}}`.
51 
+        * `events`: An array of events being output, sorted in the given order, up to `events_to_show` in number.  For example, if source events contain a site title in the `site_title` key, you can refer to it in `template.title` by putting `{{events.first.site_title}}`.
48 52
49 53 
       MD
50 54 
     end
 
@@ -134,7 +138,7 @@ module Agents
134 138 
         end
135 139 
       end
136 140
137 
-      source_events = received_events.order(id: :desc).limit(events_to_show).to_a
141 
+      source_events = sort_events(received_events.order(id: :desc).limit(events_to_show).to_a)
138 142
139 143 
       interpolation_context.stack do
140 144 
         interpolation_context['events'] = source_events

+ 1 - 1
app/models/agents/email_agent.rb
View File 

@@ -33,8 +33,8 @@ module Agents
33 33
34 34 
     def receive(incoming_events)
35 35 
       incoming_events.each do |event|
36 
-        log "Sending digest mail to #{user.email} with event #{event.id}"
37 36 
         recipients(event.payload).each do |recipient|
37 
+          log "Sending digest mail to #{recipient} with event #{event.id}"
38 38 
           SystemMailer.send_message(:to => recipient, :subject => interpolated(event)['subject'], :headline => interpolated(event)['headline'], :body => interpolated(event)['body'], :groups => [present(event.payload)]).deliver_later
39 39 
         end
40 40 
       end

+ 1 - 1
app/models/agents/email_digest_agent.rb
View File 

@@ -40,8 +40,8 @@ module Agents
40 40 
       if self.memory['queue'] && self.memory['queue'].length > 0
41 41 
         ids = self.memory['events'].join(",")
42 42 
         groups = self.memory['queue'].map { |payload| present(payload) }
43 
-        log "Sending digest mail to #{user.email} with events [#{ids}]"
44 43 
         recipients.each do |recipient|
44 
+          log "Sending digest mail to #{recipient} with events [#{ids}]"
45 45 
           SystemMailer.send_message(:to => recipient, :subject => interpolated['subject'], :headline => interpolated['headline'], :groups => groups).deliver_later
46 46 
         end
47 47 
         self.memory['queue'] = []

+ 56 - 29
app/models/agents/rss_agent.rb
View File 

@@ -9,6 +9,8 @@ module Agents
9 9 
     can_dry_run!
10 10 
     default_schedule "every_1d"
11 11
12 
+    DEFAULT_EVENTS_ORDER = [['{{date_published}}', 'time'], ['{{last_updated}}', 'time']]
13 
+
12 14 
     description do
13 15 
       <<-MD
14 16 
         This Agent consumes RSS feeds and emits events when they change.
 
@@ -26,8 +28,16 @@ module Agents
26 28 
           * `headers` - When present, it should be a hash of headers to send with the request.
27 29 
           * `basic_auth` - Specify HTTP basic auth parameters: `"username:password"`, or `["username", "password"]`.
28 30 
           * `disable_ssl_verification` - Set to `true` to disable ssl verification.
31 
+          * `disable_url_encoding` - Set to `true` to disable url encoding.
32 
+          * `force_encoding` - Set `force_encoding` to an encoding name if the website is known to respond with a missing, invalid or wrong charset in the Content-Type header.  Note that a text content without a charset is taken as encoded in UTF-8 (not ISO-8859-1).
29 33 
           * `user_agent` - A custom User-Agent name (default: "Faraday v#{Faraday::VERSION}").
30 34 
           * `max_events_per_run` - Limit number of events created (items parsed) per run for feed.
35 
+
36 
+        # Ordering Events
37 
+
38 
+        #{description_events_order}
39 
+
40 
+        In this Agent, the default value for `events_order` is `#{DEFAULT_EVENTS_ORDER.to_json}`.
31 41 
       MD
32 42 
     end
33 43 
 
@@ -69,44 +79,44 @@ module Agents
69 79 
       end
70 80
71 81 
       validate_web_request_options!
82 
+      validate_events_order
83 
+    end
84 
+
85 
+    def events_order
86 
+      super.presence || DEFAULT_EVENTS_ORDER
72 87 
     end
73 88
74 89 
     def check
75 90 
       Array(interpolated['url']).each do |url|
76 
-        response = faraday.get(url)
77 
-        if response.success?
78 
-          feed = FeedNormalizer::FeedNormalizer.parse(response.body)
79 
-          feed.clean! if interpolated['clean'] == 'true'
80 
-          max_events = (interpolated['max_events_per_run'].presence || 0).to_i
81 
-          created_event_count = 0
82 
-          feed.entries.sort_by { |entry| [entry.date_published, entry.last_updated] }.each.with_index do |entry, index|
83 
-            break if max_events && max_events > 0 && index >= max_events
84 
-            entry_id = get_entry_id(entry)
85 
-            if check_and_track(entry_id)
86 
-              created_event_count += 1
87 
-              create_event(payload: {
88 
-                id: entry_id,
89 
-                date_published: entry.date_published,
90 
-                last_updated: entry.last_updated,
91 
-                url: entry.url,
92 
-                urls: entry.urls,
93 
-                description: entry.description,
94 
-                content: entry.content,
95 
-                title: entry.title,
96 
-                authors: entry.authors,
97 
-                categories: entry.categories
98 
-              })
99 
-            end
100 
-          end
101 
-          log "Fetched #{url} and created #{created_event_count} event(s)."
102 
-        else
103 
-          error "Failed to fetch #{url}: #{response.inspect}"
104 
-        end
91 
+        check_url(url)
105 92 
       end
106 93 
     end
107 94
108 95 
     protected
109 96
97 
+    def check_url(url)
98 
+      response = faraday.get(url)
99 
+      if response.success?
100 
+        feed = FeedNormalizer::FeedNormalizer.parse(response.body)
101 
+        feed.clean! if boolify(interpolated['clean'])
102 
+        max_events = (interpolated['max_events_per_run'].presence || 0).to_i
103 
+        created_event_count = 0
104 
+        sort_events(feed_to_events(feed)).each.with_index do |event, index|
105 
+          break if max_events && max_events > 0 && index >= max_events
106 
+          entry_id = event.payload[:id]
107 
+          if check_and_track(entry_id)
108 
+            created_event_count += 1
109 
+            create_event(event)
110 
+          end
111 
+        end
112 
+        log "Fetched #{url} and created #{created_event_count} event(s)."
113 
+      else
114 
+        error "Failed to fetch #{url}: #{response.inspect}"
115 
+      end
116 
+    rescue => e
117 
+      error "Failed to fetch #{url} with message '#{e.message}': #{e.backtrace}"
118 
+    end
119 
+
110 120 
     def get_entry_id(entry)
111 121 
       entry.id.presence || Digest::MD5.hexdigest(entry.content)
112 122 
     end
 
@@ -121,5 +131,22 @@ module Agents
121 131 
         true
122 132 
       end
123 133 
     end
134 
+
135 
+    def feed_to_events(feed)
136 
+      feed.entries.map { |entry|
137 
+        Event.new(payload: {
138 
+                    id: get_entry_id(entry),
139 
+                    date_published: entry.date_published,
140 
+                    last_updated: entry.last_updated,
141 
+                    url: entry.url,
142 
+                    urls: entry.urls,
143 
+                    description: entry.description,
144 
+                    content: entry.content,
145 
+                    title: entry.title,
146 
+                    authors: entry.authors,
147 
+                    categories: entry.categories
148 
+                  })
149 
+      }
150 
+    end
124 151 
   end
125 152 
 end

+ 6 - 20
app/models/agents/website_agent.rb
View File 

@@ -6,6 +6,7 @@ module Agents
6 6 
     include WebRequestConcern
7 7
8 8 
     can_dry_run!
9 
+    can_order_created_events!
9 10
10 11 
     default_schedule "every_12h"
11 12 
 
@@ -86,7 +87,7 @@ module Agents
86 87
87 88 
       Set `uniqueness_look_back` to limit the number of events checked for uniqueness (typically for performance).  This defaults to the larger of #{UNIQUENESS_LOOK_BACK} or #{UNIQUENESS_FACTOR}x the number of detected received results.
88 89
89 
-      Set `force_encoding` to an encoding name if the website does not return a Content-Type header with a proper charset.
90 
+      Set `force_encoding` to an encoding name if the website is known to respond with a missing, invalid or wrong charset in the Content-Type header.  Note that a text content without a charset is taken as encoded in UTF-8 (not ISO-8859-1).
90 91
91 92 
       Set `user_agent` to a custom User-Agent name if the website does not like the default value (`#{default_user_agent}`).
92 93 
 
@@ -105,6 +106,10 @@ module Agents
105 106 
           * `status`: HTTP status as integer. (Almost always 200)
106 107
107 108 
           * `headers`: Response headers; for example, `{{ _response_.headers.Content-Type }}` expands to the value of the Content-Type header.  Keys are insensitive to cases and -/_.
109 
+
110 
+      # Ordering Events
111 
+
112 
+      #{description_events_order}
108 113 
     MD
109 114
110 115 
     event_description do
 
@@ -152,19 +157,6 @@ module Agents
152 157 
         errors.add(:base, "Invalid uniqueness_look_back format") unless is_positive_integer?(options['uniqueness_look_back'])
153 158 
       end
154 159
155 
-      if (encoding = options['force_encoding']).present?
156 
-        case encoding
157 
-        when String
158 
-          begin
159 
-            Encoding.find(encoding)
160 
-          rescue ArgumentError
161 
-            errors.add(:base, "Unknown encoding: #{encoding.inspect}")
162 
-          end
163 
-        else
164 
-          errors.add(:base, "force_encoding must be a string")
165 
-        end
166 
-      end
167 
-
168 160 
       validate_web_request_options!
169 161 
     end
170 162 
 
@@ -279,12 +271,6 @@ module Agents
279 271 
       interpolation_context.stack {
280 272 
         interpolation_context['_response_'] = ResponseDrop.new(response)
281 273 
         body = response.body
282 
-        if (encoding = interpolated['force_encoding']).present?
283 
-          body = body.encode(Encoding::UTF_8, encoding)
284 
-        end
285 
-        if interpolated['unzip'] == "gzip"
286 
-          body = ActiveSupport::Gzip.decompress(body)
287 
-        end
288 274 
         doc = parse(body)
289 275
290 276 
         if extract_full_json?

+ 30 - 18
app/models/scenario_import.rb
View File 

@@ -60,6 +60,7 @@ class ScenarioImport
60 60 
     description = parsed_data['description']
61 61 
     name = parsed_data['name']
62 62 
     links = parsed_data['links']
63 
+    control_links = parsed_data['control_links'] || []
63 64 
     tag_fg_color = parsed_data['tag_fg_color']
64 65 
     tag_bg_color = parsed_data['tag_bg_color']
65 66 
     source_url = parsed_data['source_url'].presence || nil
 
@@ -87,12 +88,19 @@ class ScenarioImport
87 88 
         end
88 89 
         agent
89 90 
       end
91 
+
90 92 
       if success
91 93 
         links.each do |link|
92 94 
           receiver = created_agents[link['receiver']]
93 95 
           source = created_agents[link['source']]
94 96 
           receiver.sources << source unless receiver.sources.include?(source)
95 97 
         end
98 
+
99 
+        control_links.each do |control_link|
100 
+          controller = created_agents[control_link['controller']]
101 
+          control_target = created_agents[control_link['control_target']]
102 
+          controller.control_targets << control_target unless controller.control_targets.include?(control_target)
103 
+        end
96 104 
       end
97 105 
     end
98 106 
 
@@ -142,7 +150,7 @@ class ScenarioImport
142 150 
   def generate_diff
143 151 
     @agent_diffs = (parsed_data['agents'] || []).map.with_index do |agent_data, index|
144 152 
       # AgentDiff is defined at the end of this file.
145 
-      agent_diff = AgentDiff.new(agent_data)
153 
+      agent_diff = AgentDiff.new(agent_data, parsed_data['schema_version'])
146 154 
       if existing_scenario
147 155 
         # If this Agent exists already, update the AgentDiff with the local version's information.
148 156 
         agent_diff.diff_with! existing_scenario.agents.find_by(:guid => agent_data['guid'])
 
@@ -184,14 +192,16 @@ class ScenarioImport
184 192 
       end
185 193 
     end
186 194
187 
-    def initialize(agent_data)
195 
+    def initialize(agent_data, schema_version)
188 196 
       super()
197 
+      @schema_version = schema_version
189 198 
       @requires_merge = false
190 199 
       self.agent = nil
191 200 
       store! agent_data
192 201 
     end
193 202
194 203 
     BASE_FIELDS = %w[name schedule keep_events_for propagate_immediately disabled guid]
204 
+    FIELDS_REQUIRING_TRANSLATION = %w[keep_events_for]
195 205
196 206 
     def agent_exists?
197 207 
       !!agent
 
@@ -209,10 +219,27 @@ class ScenarioImport
209 219 
       self.type = FieldDiff.new(agent_data["type"].split("::").pop)
210 220 
       self.options = FieldDiff.new(agent_data['options'] || {})
211 221 
       BASE_FIELDS.each do |option|
212 
-        self[option] = FieldDiff.new(agent_data[option]) if agent_data.has_key?(option)
222 
+        if agent_data.has_key?(option)
223 
+          value = agent_data[option]
224 
+          value = send(:"translate_#{option}", value) if option.in?(FIELDS_REQUIRING_TRANSLATION)
225 
+          self[option] = FieldDiff.new(value)
226 
+        end
213 227 
       end
214 228 
     end
215 229
230 
+    def translate_keep_events_for(old_value)
231 
+      if schema_version < 1
232 
+        # Was stored in days, now is stored in seconds.
233 
+        old_value.to_i.days
234 
+      else
235 
+        old_value
236 
+      end
237 
+    end
238 
+
239 
+    def schema_version
240 
+      (@schema_version || 0).to_i
241 
+    end
242 
+
216 243 
     def diff_with!(agent)
217 244 
       return unless agent.present?
218 245 
 
@@ -251,21 +278,6 @@ class ScenarioImport
251 278 
       yield 'disabled', disabled, boolean if disabled.requires_merge?
252 279 
     end
253 280
254 
-    # Unfortunately Ruby 1.9's OpenStruct doesn't expose [] and []=.
255 
-    unless instance_methods.include?(:[]=)
256 
-      def [](key)
257 
-        self.send(sanitize key)
258 
-      end
259 
-
260 
-      def []=(key, val)
261 
-        self.send("#{sanitize key}=", val)
262 
-      end
263 
-
264 
-      def sanitize(key)
265 
-        key.gsub(/[^a-zA-Z0-9_-]/, '')
266 
-      end
267 
-    end
268 
-
269 281 
     def agent_instance
270 282 
       "Agents::#{self.type.updated}".constantize.new
271 283 
     end

+ 5 - 1
app/models/user.rb
View File 

@@ -18,7 +18,7 @@ class User < ActiveRecord::Base
18 18 
   validates_presence_of :username
19 19 
   validates_uniqueness_of :username
20 20 
   validates_format_of :username, :with => /\A[a-zA-Z0-9_-]{3,15}\Z/, :message => "can only contain letters, numbers, underscores, and dashes, and must be between 3 and 15 characters in length."
21 
-  validates_inclusion_of :invitation_code, :on => :create, :in => INVITATION_CODES, :message => "is not valid"
21 
+  validates_inclusion_of :invitation_code, :on => :create, :in => INVITATION_CODES, :message => "is not valid", if: ->{ User.using_invitation_code? }
22 22
23 23 
   has_many :user_credentials, :dependent => :destroy, :inverse_of => :user
24 24 
   has_many :events, -> { order("events.created_at desc") }, :dependent => :delete_all, :inverse_of => :user
 
@@ -40,4 +40,8 @@ class User < ActiveRecord::Base
40 40 
       where(conditions).first
41 41 
     end
42 42 
   end
43 
+
44 
+  def self.using_invitation_code?
45 
+    ENV['SKIP_INVITATION_CODE'] != 'true'
46 
+  end
43 47 
 end

+ 1 - 1
app/views/agents/_action_menu.html.erb
View File 

@@ -7,7 +7,7 @@
7 7
8 8 
   <% if agent.can_dry_run? %>
9 9 
     <li>
10 
-      <%= link_to icon_tag('glyphicon-refresh') + ' Dry Run', '#', 'data-action-url' => dry_run_agent_path(agent), tabindex: "-1", onclick: "Utils.handleDryRunButton(this, '_method=PUT')" %>
10 
+      <%= link_to icon_tag('glyphicon-refresh') + ' Dry Run', '#', 'data-action-url' => dry_run_agent_path(agent), 'data-with-event-mode' => agent_dry_run_with_event_mode(agent), tabindex: "-1", onclick: "Utils.handleDryRunButton(this)" %>
11 11 
     </li>
12 12 
   <% end %>
13 13

+ 1 - 1
app/views/agents/_options.erb
View File 

@@ -25,6 +25,6 @@
25 25 
 <div class="form-group">
26 26 
   <%= submit_tag "Save", :class => "btn btn-primary" %>
27 27 
   <% if agent.can_dry_run? %>
28 
-    <%= button_tag class: 'btn btn-default agent-dry-run-button', type: 'button', 'data-action-url' => agent.persisted? ? dry_run_agent_path(agent) : dry_run_agents_path do %><%= icon_tag('glyphicon-refresh') %> Dry Run<% end %>
28 
+    <%= button_tag class: 'btn btn-default agent-dry-run-button', type: 'button', 'data-action-url' => agent.persisted? ? dry_run_agent_path(agent) : dry_run_agents_path, 'data-with-event-mode' => agent_dry_run_with_event_mode(agent) do %><%= icon_tag('glyphicon-refresh') %> Dry Run<% end %>
29 29 
   <% end %>
30 30 
 </div>

+ 8 - 6
app/views/devise/registrations/new.html.erb
View File 

@@ -30,13 +30,15 @@ bin/setup_heroku
30 30 
           </div>
31 31 
           <% end %>
32 32
33 
-          <div class="form-group">
34 
-            <%= f.label :invitation_code, class: 'col-md-4 control-label' %>
35 
-            <div class="col-md-6">
36 
-              <%= f.text_field :invitation_code, class: 'form-control' %>
37 
-              <span class="help-inline">We are not yet open to the public.  If you have an invitation code, please enter it here.</span>
33 
+          <% if User.using_invitation_code? %>
34 
+            <div class="form-group">
35 
+              <%= f.label :invitation_code, class: 'col-md-4 control-label' %>
36 
+              <div class="col-md-6">
37 
+                <%= f.text_field :invitation_code, class: 'form-control' %>
38 
+                <span class="help-inline">We are not yet open to the public.  If you have an invitation code, please enter it here.</span>
39 
+              </div>
38 40 
             </div>
39 
-          </div>
41 
+          <% end %>
40 42
41 43 
           <div class="form-group">
42 44 
             <%= f.label :email, class: 'col-md-4 control-label' %>

+ 1 - 1
app/views/scenario_imports/_step_two.html.erb
View File 

@@ -14,7 +14,7 @@
14 14 
         <span class='glyphicon glyphicon-warning-sign'></span>
15 15 
         This Scenario already exists in your system. The import will update your existing
16 16 
         <%= scenario_label(@scenario_import.existing_scenario) %> Scenario's title,
17 
-        description and tag colors. Below you can customize how the individual agents get updated.
17 
+        description, and tag colors. Below you can customize how the individual agents get updated.
18 18 
       </div>
19 19 
     <% end %>
20 20

+ 2 - 12
config/initializers/action_mailer.rb
View File 

@@ -2,15 +2,5 @@
2 2
3 3 
 smtp_config = YAML::load(ERB.new(File.read(Rails.root.join('config', 'smtp.yml'))).result)
4 4 
 if smtp_config.keys.include? Rails.env
5 
-  Huginn::Application.config.action_mailer.smtp_settings = smtp_config[Rails.env].symbolize_keys
6 
-end
7 
-
8 
-# Huginn::Application.config.action_mailer.smtp_settings = {
9 
-#   address: ENV['SMTP_SERVER'] || 'smtp.gmail.com',
10 
-#   port: ENV['SMTP_PORT'] || 587,
11 
-#   domain: ENV['SMTP_DOMAIN'],
12 
-#   authentication: ENV['SMTP_AUTHENTICATION'] || 'plain',
13 
-#   enable_starttls_auto: ENV['SMTP_ENABLE_STARTTLS_AUTO'] == 'true' ? true : false,
14 
-#   user_name: ENV['SMTP_USER_NAME'],
15 
-#   password: ENV['SMTP_PASSWORD']
16 
-# }
5 
+  ActionMailer::Base.smtp_settings = smtp_config[Rails.env].symbolize_keys
6 
+end

+ 1 - 1
config/initializers/delayed_job.rb
View File 

@@ -1,6 +1,6 @@
1 1 
 Delayed::Worker.destroy_failed_jobs = false
2 2 
 Delayed::Worker.max_attempts = 5
3 
-Delayed::Worker.max_run_time = (ENV['DELAYED_JOB_MAX_RUNTIME'].presence || 20).to_i.minutes
3 
+Delayed::Worker.max_run_time = (ENV['DELAYED_JOB_MAX_RUNTIME'].presence || 2).to_i.minutes
4 4 
 Delayed::Worker.read_ahead = 5
5 5 
 Delayed::Worker.default_priority = 10
6 6 
 Delayed::Worker.delay_jobs = !Rails.env.test?

+ 8 - 0
config/initializers/liquid.rb
View File 

@@ -0,0 +1,8 @@
1 
+module Liquid
2 
+  # https://github.com/Shopify/liquid/pull/623
3 
+  remove_const :PartialTemplateParser
4 
+  remove_const :TemplateParser
5 
+
6 
+  PartialTemplateParser       = /#{TagStart}.*?#{TagEnd}|#{VariableStart}(?:(?:[^'"{}]+|#{QuotedString})*?|.*?)#{VariableIncompleteEnd}/m
7 
+  TemplateParser              = /(#{PartialTemplateParser}|#{AnyStartingTag})/m
8 
+end

+ 1 - 1
config/routes.rb
View File 

@@ -2,7 +2,7 @@ Huginn::Application.routes.draw do
2 2 
   resources :agents do
3 3 
     member do
4 4 
       post :run
5 
-      put :dry_run
5 
+      post :dry_run
6 6 
       post :handle_details_post
7 7 
       put :leave_scenario
8 8 
       delete :remove_events

+ 13 - 0
db/migrate/20150507153436_update_keep_events_for_to_be_in_seconds.rb
View File 

@@ -0,0 +1,13 @@
1 
+class UpdateKeepEventsForToBeInSeconds < ActiveRecord::Migration
2 
+  class Agent < ActiveRecord::Base; end
3 
+
4 
+  SECONDS_IN_DAY = 60 * 60 * 24
5 
+
6 
+  def up
7 
+    Agent.update_all ['keep_events_for = keep_events_for * ?', SECONDS_IN_DAY]
8 
+  end
9 
+
10 
+  def down
11 
+    Agent.update_all ['keep_events_for = keep_events_for / ?', SECONDS_IN_DAY]
12 
+  end
13 
+end

+ 5 - 0
db/migrate/20150808115436_remove_requirement_from_users_invitation_code.rb
View File 

@@ -0,0 +1,5 @@
1 
+class RemoveRequirementFromUsersInvitationCode < ActiveRecord::Migration
2 
+  def change
3 
+    change_column_null :users, :invitation_code, true, ENV['INVITATION_CODE'].presence || 'try-huginn'
4 
+  end
5 
+end

+ 77 - 77
db/schema.rb
View File 

@@ -11,39 +11,39 @@
11 11 
 #
12 12 
 # It's strongly recommended that you check this file into your version control system.
13 13
14 
-ActiveRecord::Schema.define(version: 20140906030139) do
15 
-
16 
-  create_table "agent_logs", force: true do |t|
17 
-    t.integer  "agent_id",                      null: false
18 
-    t.text     "message",                       null: false, charset: "utf8mb4", collation: "utf8mb4_bin"
19 
-    t.integer  "level",             default: 3, null: false
20 
-    t.integer  "inbound_event_id"
21 
-    t.integer  "outbound_event_id"
14 
+ActiveRecord::Schema.define(version: 20150808115436) do
15 
+
16 
+  create_table "agent_logs", force: :cascade do |t|
17 
+    t.integer  "agent_id",          limit: 4,                 null: false
18 
+    t.text     "message",           limit: 65535,             null: false, charset: "utf8mb4", collation: "utf8mb4_bin"
19 
+    t.integer  "level",             limit: 4,     default: 3, null: false
20 
+    t.integer  "inbound_event_id",  limit: 4
21 
+    t.integer  "outbound_event_id", limit: 4
22 22 
     t.datetime "created_at"
23 23 
     t.datetime "updated_at"
24 24 
   end
25 25
26 
-  create_table "agents", force: true do |t|
27 
-    t.integer  "user_id"
28 
-    t.text     "options",                                                               charset: "utf8mb4", collation: "utf8mb4_bin"
29 
-    t.string   "type",                                                                                      collation: "utf8_bin"
30 
-    t.string   "name",                                                                  charset: "utf8mb4", collation: "utf8mb4_bin"
31 
-    t.string   "schedule",                                                                                  collation: "utf8_bin"
32 
-    t.integer  "events_count",                             default: 0,     null: false
26 
+  create_table "agents", force: :cascade do |t|
27 
+    t.integer  "user_id",               limit: 4
28 
+    t.text     "options",               limit: 65535,                                   charset: "utf8mb4", collation: "utf8mb4_bin"
29 
+    t.string   "type",                  limit: 255,                                                         collation: "utf8_bin"
30 
+    t.string   "name",                  limit: 255,                                     charset: "utf8mb4", collation: "utf8mb4_bin"
31 
+    t.string   "schedule",              limit: 255,                                                         collation: "utf8_bin"
32 
+    t.integer  "events_count",          limit: 4,          default: 0,     null: false
33 33 
     t.datetime "last_check_at"
34 34 
     t.datetime "last_receive_at"
35 
-    t.integer  "last_checked_event_id"
35 
+    t.integer  "last_checked_event_id", limit: 4
36 36 
     t.datetime "created_at"
37 37 
     t.datetime "updated_at"
38 
-    t.text     "memory",                limit: 2147483647,                              charset: "utf8mb4", collation: "utf8mb4_bin"
38 
+    t.text     "memory",                limit: 4294967295,                              charset: "utf8mb4", collation: "utf8mb4_bin"
39 39 
     t.datetime "last_web_request_at"
40 
-    t.integer  "keep_events_for",                          default: 0,     null: false
40 
+    t.integer  "keep_events_for",       limit: 4,          default: 0,     null: false
41 41 
     t.datetime "last_event_at"
42 42 
     t.datetime "last_error_log_at"
43 
-    t.boolean  "propagate_immediately",                    default: false, null: false
44 
-    t.boolean  "disabled",                                 default: false, null: false
45 
-    t.integer  "service_id"
46 
-    t.string   "guid",                                                     null: false
43 
+    t.boolean  "propagate_immediately", limit: 1,          default: false, null: false
44 
+    t.boolean  "disabled",              limit: 1,          default: false, null: false
45 
+    t.string   "guid",                  limit: 255,                        null: false, charset: "ascii",   collation: "ascii_bin"
46 
+    t.integer  "service_id",            limit: 4
47 47 
   end
48 48
49 49 
   add_index "agents", ["guid"], name: "index_agents_on_guid", using: :btree
 
@@ -51,9 +51,9 @@ ActiveRecord::Schema.define(version: 20140906030139) do
51 51 
   add_index "agents", ["type"], name: "index_agents_on_type", using: :btree
52 52 
   add_index "agents", ["user_id", "created_at"], name: "index_agents_on_user_id_and_created_at", using: :btree
53 53
54 
-  create_table "control_links", force: true do |t|
55 
-    t.integer  "controller_id",     null: false
56 
-    t.integer  "control_target_id", null: false
54 
+  create_table "control_links", force: :cascade do |t|
55 
+    t.integer  "controller_id",     limit: 4, null: false
56 
+    t.integer  "control_target_id", limit: 4, null: false
57 57 
     t.datetime "created_at"
58 58 
     t.datetime "updated_at"
59 59 
   end
 
@@ -61,25 +61,25 @@ ActiveRecord::Schema.define(version: 20140906030139) do
61 61 
   add_index "control_links", ["control_target_id"], name: "index_control_links_on_control_target_id", using: :btree
62 62 
   add_index "control_links", ["controller_id", "control_target_id"], name: "index_control_links_on_controller_id_and_control_target_id", unique: true, using: :btree
63 63
64 
-  create_table "delayed_jobs", force: true do |t|
65 
-    t.integer  "priority",                    default: 0
66 
-    t.integer  "attempts",                    default: 0
64 
+  create_table "delayed_jobs", force: :cascade do |t|
65 
+    t.integer  "priority",   limit: 4,        default: 0
66 
+    t.integer  "attempts",   limit: 4,        default: 0
67 67 
     t.text     "handler",    limit: 16777215,             charset: "utf8mb4", collation: "utf8mb4_bin"
68 
-    t.text     "last_error",                              charset: "utf8mb4", collation: "utf8mb4_bin"
68 
+    t.text     "last_error", limit: 65535,                charset: "utf8mb4", collation: "utf8mb4_bin"
69 69 
     t.datetime "run_at"
70 70 
     t.datetime "locked_at"
71 71 
     t.datetime "failed_at"
72 
-    t.string   "locked_by"
73 
-    t.string   "queue"
72 
+    t.string   "locked_by",  limit: 255
73 
+    t.string   "queue",      limit: 255
74 74 
     t.datetime "created_at"
75 75 
     t.datetime "updated_at"
76 76 
   end
77 77
78 78 
   add_index "delayed_jobs", ["priority", "run_at"], name: "delayed_jobs_priority", using: :btree
79 79
80 
-  create_table "events", force: true do |t|
81 
-    t.integer  "user_id"
82 
-    t.integer  "agent_id"
80 
+  create_table "events", force: :cascade do |t|
81 
+    t.integer  "user_id",    limit: 4
82 
+    t.integer  "agent_id",   limit: 4
83 83 
     t.decimal  "lat",                         precision: 15, scale: 10
84 84 
     t.decimal  "lng",                         precision: 15, scale: 10
85 85 
     t.text     "payload",    limit: 16777215,                           charset: "utf8mb4", collation: "utf8mb4_bin"
 
@@ -92,20 +92,20 @@ ActiveRecord::Schema.define(version: 20140906030139) do
92 92 
   add_index "events", ["expires_at"], name: "index_events_on_expires_at", using: :btree
93 93 
   add_index "events", ["user_id", "created_at"], name: "index_events_on_user_id_and_created_at", using: :btree
94 94
95 
-  create_table "links", force: true do |t|
96 
-    t.integer  "source_id"
97 
-    t.integer  "receiver_id"
95 
+  create_table "links", force: :cascade do |t|
96 
+    t.integer  "source_id",            limit: 4
97 
+    t.integer  "receiver_id",          limit: 4
98 98 
     t.datetime "created_at"
99 99 
     t.datetime "updated_at"
100 
-    t.integer  "event_id_at_creation", default: 0, null: false
100 
+    t.integer  "event_id_at_creation", limit: 4, default: 0, null: false
101 101 
   end
102 102
103 103 
   add_index "links", ["receiver_id", "source_id"], name: "index_links_on_receiver_id_and_source_id", using: :btree
104 104 
   add_index "links", ["source_id", "receiver_id"], name: "index_links_on_source_id_and_receiver_id", using: :btree
105 105
106 
-  create_table "scenario_memberships", force: true do |t|
107 
-    t.integer  "agent_id",    null: false
108 
-    t.integer  "scenario_id", null: false
106 
+  create_table "scenario_memberships", force: :cascade do |t|
107 
+    t.integer  "agent_id",    limit: 4, null: false
108 
+    t.integer  "scenario_id", limit: 4, null: false
109 109 
     t.datetime "created_at"
110 110 
     t.datetime "updated_at"
111 111 
   end
 
@@ -113,71 +113,71 @@ ActiveRecord::Schema.define(version: 20140906030139) do
113 113 
   add_index "scenario_memberships", ["agent_id"], name: "index_scenario_memberships_on_agent_id", using: :btree
114 114 
   add_index "scenario_memberships", ["scenario_id"], name: "index_scenario_memberships_on_scenario_id", using: :btree
115 115
116 
-  create_table "scenarios", force: true do |t|
117 
-    t.string   "name",                         null: false, charset: "utf8mb4", collation: "utf8mb4_bin"
118 
-    t.integer  "user_id",                      null: false
116 
+  create_table "scenarios", force: :cascade do |t|
117 
+    t.string   "name",         limit: 255,                   null: false, charset: "utf8mb4", collation: "utf8mb4_bin"
118 
+    t.integer  "user_id",      limit: 4,                     null: false
119 119 
     t.datetime "created_at"
120 120 
     t.datetime "updated_at"
121 
-    t.text     "description",                               charset: "utf8mb4", collation: "utf8mb4_bin"
122 
-    t.boolean  "public",       default: false, null: false
123 
-    t.string   "guid",                         null: false, charset: "ascii",   collation: "ascii_bin"
124 
-    t.string   "source_url"
125 
-    t.string   "tag_bg_color"
126 
-    t.string   "tag_fg_color"
121 
+    t.text     "description",  limit: 65535,                              charset: "utf8mb4", collation: "utf8mb4_bin"
122 
+    t.boolean  "public",       limit: 1,     default: false, null: false
123 
+    t.string   "guid",         limit: 255,                   null: false, charset: "ascii",   collation: "ascii_bin"
124 
+    t.string   "source_url",   limit: 255
125 
+    t.string   "tag_bg_color", limit: 255
126 
+    t.string   "tag_fg_color", limit: 255
127 127 
   end
128 128
129 129 
   add_index "scenarios", ["user_id", "guid"], name: "index_scenarios_on_user_id_and_guid", unique: true, using: :btree
130 130
131 
-  create_table "services", force: true do |t|
132 
-    t.integer  "user_id",                       null: false
133 
-    t.string   "provider",                      null: false
134 
-    t.string   "name",                          null: false
135 
-    t.text     "token",                         null: false
136 
-    t.text     "secret"
137 
-    t.text     "refresh_token"
131 
+  create_table "services", force: :cascade do |t|
132 
+    t.integer  "user_id",       limit: 4,                     null: false
133 
+    t.string   "provider",      limit: 255,                   null: false, collation: "utf8_general_ci"
134 
+    t.string   "name",          limit: 255,                   null: false, collation: "utf8_general_ci"
135 
+    t.text     "token",         limit: 65535,                 null: false, collation: "utf8_general_ci"
136 
+    t.text     "secret",        limit: 65535,                              collation: "utf8_general_ci"
137 
+    t.text     "refresh_token", limit: 65535,                              collation: "utf8_general_ci"
138 138 
     t.datetime "expires_at"
139 
-    t.boolean  "global",        default: false
140 
-    t.text     "options"
139 
+    t.boolean  "global",        limit: 1,     default: false
140 
+    t.text     "options",       limit: 65535,                              collation: "utf8_general_ci"
141 141 
     t.datetime "created_at"
142 142 
     t.datetime "updated_at"
143 
-    t.string   "uid"
143 
+    t.string   "uid",           limit: 255,                                collation: "utf8_general_ci"
144 144 
   end
145 145
146 146 
   add_index "services", ["provider"], name: "index_services_on_provider", using: :btree
147 147 
   add_index "services", ["uid"], name: "index_services_on_uid", using: :btree
148 148 
   add_index "services", ["user_id", "global"], name: "index_services_on_user_id_and_global", using: :btree
149 149
150 
-  create_table "user_credentials", force: true do |t|
151 
-    t.integer  "user_id",                           null: false
152 
-    t.string   "credential_name",                   null: false
153 
-    t.text     "credential_value",                  null: false
150 
+  create_table "user_credentials", force: :cascade do |t|
151 
+    t.integer  "user_id",          limit: 4,                      null: false
152 
+    t.string   "credential_name",  limit: 255,                    null: false
153 
+    t.text     "credential_value", limit: 65535,                  null: false
154 154 
     t.datetime "created_at"
155 155 
     t.datetime "updated_at"
156 
-    t.string   "mode",             default: "text", null: false, collation: "utf8_bin"
156 
+    t.string   "mode",             limit: 255,   default: "text", null: false, collation: "utf8_bin"
157 157 
   end
158 158
159 159 
   add_index "user_credentials", ["user_id", "credential_name"], name: "index_user_credentials_on_user_id_and_credential_name", unique: true, using: :btree
160 160
161 
-  create_table "users", force: true do |t|
162 
-    t.string   "email",                              default: "",    null: false,                     collation: "utf8_bin"
163 
-    t.string   "encrypted_password",                 default: "",    null: false, charset: "ascii",   collation: "ascii_bin"
164 
-    t.string   "reset_password_token",                                                                collation: "utf8_bin"
161 
+  create_table "users", force: :cascade do |t|
162 
+    t.string   "email",                  limit: 255, default: "",    null: false,                     collation: "utf8_bin"
163 
+    t.string   "encrypted_password",     limit: 255, default: "",    null: false, charset: "ascii",   collation: "ascii_bin"
164 
+    t.string   "reset_password_token",   limit: 255,                                                  collation: "utf8_bin"
165 165 
     t.datetime "reset_password_sent_at"
166 166 
     t.datetime "remember_created_at"
167 
-    t.integer  "sign_in_count",                      default: 0
167 
+    t.integer  "sign_in_count",          limit: 4,   default: 0
168 168 
     t.datetime "current_sign_in_at"
169 169 
     t.datetime "last_sign_in_at"
170 
-    t.string   "current_sign_in_ip"
171 
-    t.string   "last_sign_in_ip"
170 
+    t.string   "current_sign_in_ip",     limit: 255
171 
+    t.string   "last_sign_in_ip",        limit: 255
172 172 
     t.datetime "created_at"
173 173 
     t.datetime "updated_at"
174 
-    t.boolean  "admin",                              default: false, null: false
175 
-    t.integer  "failed_attempts",                    default: 0
176 
-    t.string   "unlock_token"
174 
+    t.boolean  "admin",                  limit: 1,   default: false, null: false
175 
+    t.integer  "failed_attempts",        limit: 4,   default: 0
176 
+    t.string   "unlock_token",           limit: 255
177 177 
     t.datetime "locked_at"
178 178 
     t.string   "username",               limit: 191,                 null: false, charset: "utf8mb4", collation: "utf8mb4_unicode_ci"
179 
-    t.string   "invitation_code",                                    null: false,                     collation: "utf8_bin"
180 
-    t.integer  "scenario_count",                     default: 0,     null: false
179 
+    t.string   "invitation_code",        limit: 255,                                                  collation: "utf8_bin"
180 
+    t.integer  "scenario_count",         limit: 4,   default: 0,     null: false
181 181 
   end
182 182
183 183 
   add_index "users", ["email"], name: "index_users_on_email", unique: true, using: :btree

+ 0 - 5
deployment/heroku/unicorn.rb
View File 

@@ -17,11 +17,6 @@ Thread.new do
17 17
18 18 
     sleep 45
19 19
20 
-    if ENV['DOMAIN']
21 
-      force_ssl = ENV['FORCE_SSL'] == 'true'
22 
-      Net::HTTP.get_response(URI((force_ssl ? "https://" : "http://") + ENV['DOMAIN']))
23 
-    end
24 
-
25 20 
     begin
26 21 
       Process.getpgid worker_pid
27 22 
     rescue Errno::ESRCH

+ 3 - 0
deployment/site-cookbooks/huginn_production/files/default/env.example
View File 

@@ -39,6 +39,9 @@ FORCE_SSL=false
39 39 
 # You can see its use in user.rb.  PLEASE CHANGE THIS!
40 40 
 INVITATION_CODE=try-huginn
41 41
42 
+# If you don't want to require users to have an invitation code, set this to true
43 
+SKIP_INVITATION_CODE=false
44 
+
42 45 
 #############################
43 46 
 #    Email Configuration    #
44 47 
 #############################

+ 7 - 2
docker/scripts/init
View File 

@@ -6,6 +6,11 @@ cd /app
6 6 
 # Default to the environment variable values set in .env.example
7 7 
 source /app/.env.example
8 8
9 
+# Cleanup any leftover pid file
10 
+if [ -f /app/tmp/pids/server.pid ]; then
11 
+  rm /app/tmp/pids/server.pid
12 
+fi
13 
+
9 14 
 # is a mysql or postgresql database linked?
10 15 
 # requires that the mysql or postgresql containers have exposed
11 16 
 # port 3306 and 5432 respectively.
 
@@ -177,8 +182,8 @@ if [ -n "\${DO_NOT_RUN_JOBS}" ]; then
177 182 
   perl -pi -e 's/^jobs:/#jobs:/' /app/Procfile
178 183 
 fi
179 184
180 
-perl -pi -e 's/rails server\$/rails server -b 0.0.0.0 -p \\\$PORT/' /app/Procfile
181 
-export PORT
185 
+perl -pi -e 's/rails server -b0.0.0.0\$/rails server -b 0.0.0.0 -p \\\$PORT/' /app/Procfile
186 
+export PORT="\${PORT:-3000}"
182 187
183 188 
 # Start huginn
184 189 
 exec sudo -u huginn -EH bash -xc 'exec bundle exec foreman start'

+ 17 - 3
lib/agents_exporter.rb
View File 

@@ -12,6 +12,7 @@ class AgentsExporter
12 12
13 13 
   def as_json(opts = {})
14 14 
     {
15 
+      :schema_version => 1,
15 16 
       :name => options[:name].presence || 'No name provided',
16 17 
       :description => options[:description].presence || 'No description provided',
17 18 
       :source_url => options[:source_url],
 
@@ -20,7 +21,8 @@ class AgentsExporter
20 21 
       :tag_bg_color => options[:tag_bg_color],
21 22 
       :exported_at => Time.now.utc.iso8601,
22 23 
       :agents => agents.map { |agent| agent_as_json(agent) },
23 
-      :links => links
24 
+      :links => links,
25 
+      :control_links => control_links
24 26 
     }
25 27 
   end
26 28 
 
@@ -32,14 +34,26 @@ class AgentsExporter
32 34 
     agent_ids = agents.map(&:id)
33 35
34 36 
     contained_links = agents.map.with_index do |agent, index|
35 
-      agent.links_as_source.where(:receiver_id => agent_ids).map do |link|
36 
-        { :source => index, :receiver => agent_ids.index(link.receiver_id) }
37 
+      agent.links_as_source.where(receiver_id: agent_ids).map do |link|
38 
+        { source: index, receiver: agent_ids.index(link.receiver_id) }
37 39 
       end
38 40 
     end
39 41
40 42 
     contained_links.flatten.compact
41 43 
   end
42 44
45 
+  def control_links
46 
+    agent_ids = agents.map(&:id)
47 
+
48 
+    contained_controller_links = agents.map.with_index do |agent, index|
49 
+      agent.control_links_as_controller.where(control_target_id: agent_ids).map do |control_link|
50 
+        { controller: index, control_target: agent_ids.index(control_link.control_target_id) }
51 
+      end
52 
+    end
53 
+
54 
+    contained_controller_links.flatten.compact
55 
+  end
56 
+
43 57 
   def agent_as_json(agent)
44 58 
     {
45 59 
       :type => agent.type,

+ 0 - 3
lib/ar_mysql_column_charset.rb
View File 

@@ -1,6 +1,3 @@
1 
-# Module#prepend support for Ruby 1.9
2 
-require 'prepend' unless Module.method_defined?(:prepend)
3 
-
4 1 
 require 'active_support'
5 2
6 3 
 ActiveSupport.on_load :active_record do

+ 1 - 1
lib/huginn_scheduler.rb
View File 

@@ -114,7 +114,7 @@ class HuginnScheduler
114 114 
     end
115 115
116 116 
     # Schedule event cleanup.
117 
-    @rufus_scheduler.cron "0 0 * * * " + tzinfo_friendly_timezone do
117 
+    @rufus_scheduler.every ENV['EVENT_EXPIRATION_CHECK'].presence || '6h' do
118 118 
       cleanup_expired_events!
119 119 
     end
120 120

+ 0 - 85
lib/prepend.rb
View File 

@@ -1,85 +0,0 @@
1 
-# Fake implementation of prepend(), which does not support overriding
2 
-# inherited methods nor methods that are formerly overridden by
3 
-# another invocation of prepend().
4 
-#
5 
-# Here's what <Original>.prepend(<Wrapper>) does:
6 
-#
7 
-# - Create an anonymous stub module (hereinafter <Stub>) and define
8 
-#   <Stub>#<method> that calls #<method>_without_<Wrapper> for each
9 
-#   instance method of <Wrapper>.
10 
-#
11 
-# - Rename <Original>#<method> to #<method>_without_<Wrapper> for each
12 
-#   instance method of <Wrapper>.
13 
-#
14 
-# - Include <Stub> and <Wrapper> into <Original> in that order.
15 
-#
16 
-# This way, a call of <Original>#<method> is dispatched to
17 
-# <Wrapper><method>, which may call super which is dispatched to
18 
-# <Stub>#<method>, which finally calls
19 
-# <Original>#<method>_without_<Wrapper> which is used to be called
20 
-# <Original>#<method>.
21 
-#
22 
-# Usage:
23 
-#
24 
-#     class Mechanize
25 
-#       # module with methods that overrides those of X
26 
-#       module Y
27 
-#       end
28 
-#
29 
-#       unless X.respond_to?(:prepend, true)
30 
-#         require 'mechanize/prependable'
31 
-#         X.extend(Prependable)
32 
-#       end
33 
-#
34 
-#       class X
35 
-#         prepend Y
36 
-#       end
37 
-#     end
38 
-class Module
39 
-  def prepend(mod)
40 
-    stub = Module.new
41 
-
42 
-    mod_id = (mod.name || 'Module__%d' % mod.object_id).gsub(/::/, '__')
43 
-
44 
-    mod.instance_methods.each { |name|
45 
-      method_defined?(name) or next
46 
-
47 
-      original = instance_method(name)
48 
-      next if original.owner != self
49 
-
50 
-      name = name.to_s
51 
-      name_without = name.sub(/(?=[?!=]?\z)/) { '_without_%s' % mod_id }
52 
-
53 
-      arity = original.arity
54 
-      arglist = (
55 
-        if arity >= 0
56 
-          (1..arity).map { |i| 'x%d' % i }
57 
-        else
58 
-          (1..(-arity - 1)).map { |i| 'x%d' % i } << '*a'
59 
-        end << '&b'
60 
-      ).join(', ')
61 
-
62 
-      if name.end_with?('=')
63 
-        stub.module_eval %{
64 
-          def #{name}(#{arglist})
65 
-            __send__(:#{name_without}, #{arglist})
66 
-          end
67 
-        }
68 
-      else
69 
-        stub.module_eval %{
70 
-          def #{name}(#{arglist})
71 
-            #{name_without}(#{arglist})
72 
-          end
73 
-        }
74 
-      end
75 
-      module_eval {
76 
-        alias_method name_without, name
77 
-        remove_method name
78 
-      }
79 
-    }
80 
-
81 
-    include stub
82 
-    include mod
83 
-  end
84 
-  private :prepend
85 
-end unless Module.method_defined?(:prepend)

+ 39 - 0
lib/utils.rb
View File 

@@ -79,4 +79,43 @@ module Utils
79 79 
   def self.pretty_jsonify(thing)
80 80 
     JSON.pretty_generate(thing).gsub('</', '<\/')
81 81 
   end
82 
+
83 
+  class TupleSorter
84 
+    class SortableTuple
85 
+      attr_reader :array
86 
+
87 
+      # The <=> method will call orders[n] to determine if the nth element
88 
+      # should be compared in descending order.
89 
+      def initialize(array, orders = [])
90 
+        @array = array
91 
+        @orders = orders
92 
+      end
93 
+
94 
+      def <=> other
95 
+        other = other.array
96 
+        @array.each_with_index do |e, i|
97 
+          o = other[i]
98 
+          case cmp = e <=> o || e.to_s <=> o.to_s
99 
+          when 0
100 
+            next
101 
+          else
102 
+            return @orders[i] ? -cmp : cmp
103 
+          end
104 
+        end
105 
+        0
106 
+      end
107 
+    end
108 
+
109 
+    class << self
110 
+      def sort!(array, orders = [])
111 
+        array.sort_by! do |e|
112 
+          SortableTuple.new(e, orders)
113 
+        end
114 
+      end
115 
+    end
116 
+  end
117 
+
118 
+  def self.sort_tuples!(array, orders = [])
119 
+    TupleSorter.sort!(array, orders)
120 
+  end
82 121 
 end

+ 46 - 17
spec/concerns/dry_runnable_spec.rb
View File 

@@ -7,10 +7,22 @@ describe DryRunnable do
7 7 
     can_dry_run!
8 8
9 9 
     def check
10 
+      perform
11 
+    end
12 
+
13 
+    def receive(events)
14 
+      events.each do |event|
15 
+        perform(event.payload['prefix'])
16 
+      end
17 
+    end
18 
+
19 
+    private
20 
+
21 
+    def perform(prefix = nil)
10 22 
       log "Logging"
11 
-      create_event payload: { 'test' => 'foo' }
23 
+      create_event payload: { 'test' => "#{prefix}foo" }
12 24 
       error "Recording error"
13 
-      create_event payload: { 'test' => 'bar' }
25 
+      create_event payload: { 'test' => "#{prefix}bar" }
14 26 
       self.memory = { 'last_status' => 'ok', 'dry_run' => dry_run? }
15 27 
       save!
16 28 
     end
 
@@ -46,21 +58,6 @@ describe DryRunnable do
46 58 
     expect(messages).to eq(['Logging', 'Recording error'])
47 59 
   end
48 60
49 
-  it "traps logging, event emission and memory updating, with dry_run? returning true" do
50 
-    results = nil
51 
-
52 
-    expect {
53 
-      results = @agent.dry_run!
54 
-      @agent.reload
55 
-    }.not_to change {
56 
-      [@agent.memory, counts]
57 
-    }
58 
-
59 
-    expect(results[:log]).to match(/\AI, .+ INFO -- : Logging\nE, .+ ERROR -- : Recording error\n/)
60 
-    expect(results[:events]).to eq([{ 'test' => 'foo' }, { 'test' => 'bar' }])
61 
-    expect(results[:memory]).to eq({ 'last_status' => 'ok', 'dry_run' => true })
62 
-  end
63 
-
64 61 
   it "does not perform dry-run if Agent does not support dry-run" do
65 62 
     stub(@agent).can_dry_run? { false }
66 63 
 
@@ -77,4 +74,36 @@ describe DryRunnable do
77 74 
     expect(results[:events]).to eq([])
78 75 
     expect(results[:memory]).to eq({})
79 76 
   end
77 
+
78 
+  describe "dry_run!" do
79 
+    it "traps any destructive operations during a run" do
80 
+      results = nil
81 
+
82 
+      expect {
83 
+        results = @agent.dry_run!
84 
+        @agent.reload
85 
+      }.not_to change {
86 
+        [@agent.memory, counts]
87 
+      }
88 
+
89 
+      expect(results[:log]).to match(/\AI, .+ INFO -- : Logging\nE, .+ ERROR -- : Recording error\n/)
90 
+      expect(results[:events]).to eq([{ 'test' => 'foo' }, { 'test' => 'bar' }])
91 
+      expect(results[:memory]).to eq({ 'last_status' => 'ok', 'dry_run' => true })
92 
+    end
93 
+
94 
+    it "traps any destructive operations during a run when an event is given" do
95 
+      results = nil
96 
+
97 
+      expect {
98 
+        results = @agent.dry_run!(Event.new(payload: { 'prefix' => 'super' }))
99 
+        @agent.reload
100 
+      }.not_to change {
101 
+        [@agent.memory, counts]
102 
+      }
103 
+
104 
+      expect(results[:log]).to match(/\AI, .+ INFO -- : Logging\nE, .+ ERROR -- : Recording error\n/)
105 
+      expect(results[:events]).to eq([{ 'test' => 'superfoo' }, { 'test' => 'superbar' }])
106 
+      expect(results[:memory]).to eq({ 'last_status' => 'ok', 'dry_run' => true })
107 
+    end
108 
+  end
80 109 
 end

+ 39 - 15
spec/concerns/liquid_interpolatable_spec.rb
View File 

@@ -38,6 +38,16 @@ describe LiquidInterpolatable::Filters do
38 38 
     end
39 39 
   end
40 40
41 
+  describe 'unescape' do
42 
+    let(:agent) { Agents::InterpolatableAgent.new(name: "test") }
43 
+
44 
+    it 'should unescape basic HTML entities' do
45 
+      agent.interpolation_context['something'] = '&#39;&lt;foo&gt; &amp; bar&#x27;'
46 
+      agent.options['cleaned'] = '{{ something | unescape }}'
47 
+      expect(agent.interpolated['cleaned']).to eq("'<foo> & bar'")
48 
+    end
49 
+  end
50 
+
41 51 
   describe 'to_xpath' do
42 52 
     before do
43 53 
       def @filter.to_xpath_roundtrip(string)
 
@@ -177,23 +187,37 @@ describe LiquidInterpolatable::Filters do
177 187 
         expect(@agent.interpolated['long_url']).to eq('http://2many.x/6')
178 188 
       end
179 189 
     end
180 
-
181 
-    describe 'regex replace' do
182 
-      let(:agent) { Agents::InterpolatableAgent.new(name: "test") }
190 
+  end
183 191
184 
-      it 'should replace the first occurrence of a string using regex' do
185 
-        agent.interpolation_context['something'] = 'foobar foobar'
186 
-        agent.options['cleaned'] = '{{ something | regex_replace_first: "\S+bar", "foobaz"  }}'
187 
-        expect(agent.interpolated['cleaned']).to eq('foobaz foobar')
188 
-      end
192 
+  describe 'regex_replace_first' do
193 
+    let(:agent) { Agents::InterpolatableAgent.new(name: "test") }
189 194
190 
-      it 'should replace the all occurrences of a string using regex' do
191 
-        agent.interpolation_context['something'] = 'foobar foobar'
192 
-        agent.options['cleaned'] = '{{ something | regex_replace: "\S+bar", "foobaz"  }}'
193 
-        expect(agent.interpolated['cleaned']).to eq('foobaz foobaz')
194 
-      end
195 
-
195 
+    it 'should replace the first occurrence of a string using regex' do
196 
+      agent.interpolation_context['something'] = 'foobar foobar'
197 
+      agent.options['cleaned'] = '{{ something | regex_replace_first: "\S+bar", "foobaz"  }}'
198 
+      expect(agent.interpolated['cleaned']).to eq('foobaz foobar')
199 
+    end
200 
+
201 
+    it 'should support escaped characters' do
202 
+      agent.interpolation_context['something'] = "foo\\1\n\nfoo\\bar\n\nfoo\\baz"
203 
+      agent.options['test'] = "{{ something | regex_replace_first: '\\\\(\\w{2,})', '\\1\\\\' | regex_replace_first: '\\n+', '\\n'  }}"
204 
+      expect(agent.interpolated['test']).to eq("foo\\1\nfoobar\\\n\nfoo\\baz")
205 
+    end
206 
+  end
207 
+
208 
+  describe 'regex_replace' do
209 
+    let(:agent) { Agents::InterpolatableAgent.new(name: "test") }
210 
+
211 
+    it 'should replace the all occurrences of a string using regex' do
212 
+      agent.interpolation_context['something'] = 'foobar foobar'
213 
+      agent.options['cleaned'] = '{{ something | regex_replace: "\S+bar", "foobaz"  }}'
214 
+      expect(agent.interpolated['cleaned']).to eq('foobaz foobaz')
215 
+    end
216 
+
217 
+    it 'should support escaped characters' do
218 
+      agent.interpolation_context['something'] = "foo\\1\n\nfoo\\bar\n\nfoo\\baz"
219 
+      agent.options['test'] = "{{ something | regex_replace: '\\\\(\\w{2,})', '\\1\\\\' | regex_replace: '\\n+', '\\n'  }}"
220 
+      expect(agent.interpolated['test']).to eq("foo\\1\nfoobar\\\nfoobaz\\")
196 221 
     end
197 
-
198 222 
   end
199 223 
 end

+ 264 - 0
spec/concerns/sortable_events_spec.rb
View File 

@@ -0,0 +1,264 @@
1 
+require 'spec_helper'
2 
+
3 
+describe SortableEvents do
4 
+  let(:agent_class) {
5 
+    Class.new(Agent) do
6 
+      include SortableEvents
7 
+
8 
+      default_schedule 'never'
9 
+
10 
+      def self.valid_type?(name)
11 
+        true
12 
+      end
13 
+    end
14 
+  }
15 
+
16 
+  def new_agent(events_order = nil)
17 
+    options = {}
18 
+    options['events_order'] = events_order if events_order
19 
+    agent_class.new(name: 'test', options: options) { |agent|
20 
+      agent.user = users(:bob)
21 
+    }
22 
+  end
23 
+
24 
+  describe 'validations' do
25 
+    let(:agent_class) {
26 
+      Class.new(Agent) do
27 
+        include SortableEvents
28 
+
29 
+        default_schedule 'never'
30 
+
31 
+        def self.valid_type?(name)
32 
+          true
33 
+        end
34 
+      end
35 
+    }
36 
+
37 
+    def new_agent(events_order = nil)
38 
+      options = {}
39 
+      options['events_order'] = events_order if events_order
40 
+      agent_class.new(name: 'test', options: options) { |agent|
41 
+        agent.user = users(:bob)
42 
+      }
43 
+    end
44 
+
45 
+    it 'should allow events_order to be unspecified, null or an empty array' do
46 
+      expect(new_agent()).to be_valid
47 
+      expect(new_agent(nil)).to be_valid
48 
+      expect(new_agent([])).to be_valid
49 
+    end
50 
+
51 
+    it 'should not allow events_order to be a non-array object' do
52 
+      agent = new_agent(0)
53 
+      expect(agent).not_to be_valid
54 
+      expect(agent.errors[:base]).to include(/events_order/)
55 
+
56 
+      agent = new_agent('')
57 
+      expect(agent).not_to be_valid
58 
+      expect(agent.errors[:base]).to include(/events_order/)
59 
+
60 
+      agent = new_agent({})
61 
+      expect(agent).not_to be_valid
62 
+      expect(agent.errors[:base]).to include(/events_order/)
63 
+    end
64 
+
65 
+    it 'should not allow events_order to be an array containing unexpected objects' do
66 
+      agent = new_agent(['{{key}}', 1])
67 
+      expect(agent).not_to be_valid
68 
+      expect(agent.errors[:base]).to include(/events_order/)
69 
+
70 
+      agent = new_agent(['{{key1}}', ['{{key2}}', 'unknown']])
71 
+      expect(agent).not_to be_valid
72 
+      expect(agent.errors[:base]).to include(/events_order/)
73 
+    end
74 
+
75 
+    it 'should allow events_order to be an array containing strings and valid tuples' do
76 
+      agent = new_agent(['{{key1}}', ['{{key2}}'], ['{{key3}}', 'number']])
77 
+      expect(agent).to be_valid
78 
+
79 
+      agent = new_agent(['{{key1}}', ['{{key2}}'], ['{{key3}}', 'number'], ['{{key4}}', 'time', true]])
80 
+      expect(agent).to be_valid
81 
+    end
82 
+  end
83 
+
84 
+  describe 'sort_events' do
85 
+    let(:payloads) {
86 
+      [
87 
+        { 'title' => 'TitleA', 'score' => 4,  'updated_on' => '7 Jul 2015' },
88 
+        { 'title' => 'TitleB', 'score' => 2,  'updated_on' => '25 Jun 2014' },
89 
+        { 'title' => 'TitleD', 'score' => 10, 'updated_on' => '10 Jan 2015' },
90 
+        { 'title' => 'TitleC', 'score' => 10, 'updated_on' => '9 Feb 2015' },
91 
+      ]
92 
+    }
93 
+
94 
+    let(:events) {
95 
+      payloads.map { |payload| Event.new(payload: payload) }
96 
+    }
97 
+
98 
+    it 'should sort events by a given key' do
99 
+      agent = new_agent(['{{title}}'])
100 
+      expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleA TitleB TitleC TitleD])
101 
+
102 
+      agent = new_agent([['{{title}}', 'string', true]])
103 
+      expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleD TitleC TitleB TitleA])
104 
+    end
105 
+
106 
+    it 'should sort events by multiple keys' do
107 
+      agent = new_agent([['{{score}}', 'number'], '{{title}}'])
108 
+      expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleB TitleA TitleC TitleD])
109 
+
110 
+      agent = new_agent([['{{score}}', 'number'], ['{{title}}', 'string', true]])
111 
+      expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleB TitleA TitleD TitleC])
112 
+    end
113 
+
114 
+    it 'should sort events by time' do
115 
+      agent = new_agent([['{{updated_on}}', 'time']])
116 
+      expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleB TitleD TitleC TitleA])
117 
+    end
118 
+
119 
+    it 'should sort events stably' do
120 
+      agent = new_agent(['<constant>'])
121 
+      expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleA TitleB TitleD TitleC])
122 
+
123 
+      agent = new_agent([['<constant>', 'string', true]])
124 
+      expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleA TitleB TitleD TitleC])
125 
+    end
126 
+
127 
+    it 'should support _index_' do
128 
+      agent = new_agent([['{{_index_}}', 'number', true]])
129 
+      expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleC TitleD TitleB TitleA])
130 
+    end
131 
+  end
132 
+
133 
+  describe 'automatic event sorter' do
134 
+    describe 'declaration' do
135 
+      let(:passive_agent_class) {
136 
+        Class.new(Agent) do
137 
+          include SortableEvents
138 
+
139 
+          cannot_create_events!
140 
+        end
141 
+      }
142 
+
143 
+      let(:active_agent_class) {
144 
+        Class.new(Agent) do
145 
+          include SortableEvents
146 
+        end
147 
+      }
148 
+
149 
+      describe 'can_order_created_events!' do
150 
+        it 'should refuse to work if called from an Agent that cannot create events' do
151 
+          expect {
152 
+            passive_agent_class.class_eval do
153 
+              can_order_created_events!
154 
+            end
155 
+          }.to raise_error
156 
+        end
157 
+
158 
+        it 'should work if called from an Agent that can create events' do
159 
+          expect {
160 
+            active_agent_class.class_eval do
161 
+              can_order_created_events!
162 
+            end
163 
+          }.not_to raise_error
164 
+        end
165 
+      end
166 
+
167 
+      describe 'can_order_created_events?' do
168 
+        it 'should return false unless an Agent declares can_order_created_events!' do
169 
+          expect(active_agent_class.can_order_created_events?).to eq(false)
170 
+          expect(active_agent_class.new.can_order_created_events?).to eq(false)
171 
+        end
172 
+
173 
+        it 'should return true if an Agent declares can_order_created_events!' do
174 
+          active_agent_class.class_eval do
175 
+            can_order_created_events!
176 
+          end
177 
+
178 
+          expect(active_agent_class.can_order_created_events?).to eq(true)
179 
+          expect(active_agent_class.new.can_order_created_events?).to eq(true)
180 
+        end
181 
+      end
182 
+    end
183 
+
184 
+    describe 'behavior' do
185 
+      class Agents::EventOrderableAgent < Agent
186 
+        include SortableEvents
187 
+
188 
+        default_schedule 'never'
189 
+
190 
+        can_order_created_events!
191 
+
192 
+        attr_accessor :payloads_to_emit
193 
+
194 
+        def self.valid_type?(name)
195 
+          true
196 
+        end
197 
+
198 
+        def check
199 
+          payloads_to_emit.each do |payload|
200 
+            create_event payload: payload
201 
+          end
202 
+        end
203 
+
204 
+        def receive(events)
205 
+          events.each do |event|
206 
+            payloads_to_emit.each do |payload|
207 
+              create_event payload: payload.merge('title' => payload['title'] + event.payload['title_suffix'])
208 
+            end
209 
+          end
210 
+        end
211 
+      end
212 
+
213 
+      def new_agent(events_order = nil)
214 
+        options = {}
215 
+        options['events_order'] = events_order if events_order
216 
+        Agents::EventOrderableAgent.new(name: 'test', options: options) { |agent|
217 
+          agent.user = users(:bob)
218 
+          agent.payloads_to_emit = payloads
219 
+        }
220 
+      end
221 
+
222 
+      let(:payloads) {
223 
+        [
224 
+          { 'title' => 'TitleA', 'score' => 4,  'updated_on' => '7 Jul 2015' },
225 
+          { 'title' => 'TitleB', 'score' => 2,  'updated_on' => '25 Jun 2014' },
226 
+          { 'title' => 'TitleD', 'score' => 10, 'updated_on' => '10 Jan 2015' },
227 
+          { 'title' => 'TitleC', 'score' => 10, 'updated_on' => '9 Feb 2015' },
228 
+        ]
229 
+      }
230 
+
231 
+      it 'should keep the order of created events unless events_order is specified' do
232 
+        [[], [nil], [[]]].each do |args|
233 
+          agent = new_agent(*args)
234 
+          agent.save!
235 
+          expect { agent.check }.to change { Event.count }.by(4)
236 
+          events = agent.events.last(4).sort_by(&:id)
237 
+          expect(events.map { |event| event.payload['title'] }).to eq(%w[TitleA TitleB TitleD TitleC])
238 
+        end
239 
+      end
240 
+
241 
+      it 'should sort events created in check() in the order specified in events_order' do
242 
+        agent = new_agent([['{{score}}', 'number'], ['{{title}}', 'string', true]])
243 
+        agent.save!
244 
+        expect { agent.check }.to change { Event.count }.by(4)
245 
+        events = agent.events.last(4).sort_by(&:id)
246 
+        expect(events.map { |event| event.payload['title'] }).to eq(%w[TitleB TitleA TitleD TitleC])
247 
+      end
248 
+
249 
+      it 'should sort events created in receive() in the order specified in events_order' do
250 
+        agent = new_agent([['{{score}}', 'number'], ['{{title}}', 'string', true]])
251 
+        agent.save!
252 
+        expect {
253 
+          agent.receive([Event.new(payload: { 'title_suffix' => ' [new]' }),
254 
+                         Event.new(payload: { 'title_suffix' => ' [popular]' })])
255 
+        }.to change { Event.count }.by(8)
256 
+        events = agent.events.last(8).sort_by(&:id)
257 
+        expect(events.map { |event| event.payload['title'] }).to eq([
258 
+          'TitleB [new]',     'TitleA [new]',     'TitleD [new]',     'TitleC [new]',
259 
+          'TitleB [popular]', 'TitleA [popular]', 'TitleD [popular]', 'TitleC [popular]',
260 
+        ])
261 
+      end
262 
+    end
263 
+  end
264 
+end

+ 18 - 1
spec/controllers/agents_controller_spec.rb
View File 

@@ -349,6 +349,10 @@ describe AgentsController do
349 349 
   end
350 350
351 351 
   describe "POST dry_run" do
352 
+    before do
353 
+      stub_request(:any, /xkcd/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/xkcd.html")), status: 200)
354 
+    end
355 
+
352 356 
     it "does not actually create any agent, event or log" do
353 357 
       sign_in users(:bob)
354 358 
       expect {
 
@@ -368,11 +372,24 @@ describe AgentsController do
368 372 
       sign_in users(:bob)
369 373 
       agent = agents(:bob_weather_agent)
370 374 
       expect {
371 
-        post :dry_run, id: agents(:bob_website_agent), agent: valid_attributes(name: 'New Name')
375 
+        post :dry_run, id: agent, agent: valid_attributes(name: 'New Name')
372 376 
       }.not_to change {
373 377 
         [users(:bob).agents.count, users(:bob).events.count, users(:bob).logs.count, agent.name, agent.updated_at]
374 378 
       }
375 379 
     end
380 
+
381 
+    it "accepts an event" do
382 
+      sign_in users(:bob)
383 
+      agent = agents(:bob_website_agent)
384 
+      url_from_event = "http://xkcd.com/?from_event=1".freeze
385 
+      expect {
386 
+        post :dry_run, id: agent, event: { url: url_from_event }
387 
+      }.not_to change {
388 
+        [users(:bob).agents.count, users(:bob).events.count, users(:bob).logs.count, agent.name, agent.updated_at]
389 
+      }
390 
+      json = JSON.parse(response.body)
391 
+      expect(json['log']).to match(/^I, .* : Fetching #{Regexp.quote(url_from_event)}$/)
392 
+    end
376 393 
   end
377 394
378 395 
   describe "DELETE memory" do

+ 2 - 2
spec/fixtures/agents.yml
View File 

@@ -38,7 +38,7 @@ bob_weather_agent:
38 38 
   schedule: "midnight"
39 39 
   name: "SF Weather"
40 40 
   guid: <%= SecureRandom.hex %>
41 
-  keep_events_for: 45
41 
+  keep_events_for: <%= 45.days %>
42 42 
   options: <%= { :location => 94102, :lat => 37.779329, :lng => -122.41915, :api_key => 'test' }.to_json.inspect %>
43 43
44 44 
 jane_weather_agent:
 
@@ -47,7 +47,7 @@ jane_weather_agent:
47 47 
   schedule: "midnight"
48 48 
   name: "SF Weather"
49 49 
   guid: <%= SecureRandom.hex %>
50 
-  keep_events_for: 30
50 
+  keep_events_for: <%= 30.days %>
51 51 
   options: <%= { :location => 94103, :lat => 37.779329, :lng => -122.41915, :api_key => 'test' }.to_json.inspect %>
52 52
53 53 
 jane_rain_notifier_agent:

+ 9 - 0
spec/lib/agents_exporter_spec.rb
View File 

@@ -21,10 +21,12 @@ describe AgentsExporter do
21 21 
       expect(data[:description]).to eq(description)
22 22 
       expect(data[:source_url]).to eq(source_url)
23 23 
       expect(data[:guid]).to eq(guid)
24 
+      expect(data[:schema_version]).to eq(1)
24 25 
       expect(data[:tag_fg_color]).to eq(tag_fg_color)
25 26 
       expect(data[:tag_bg_color]).to eq(tag_bg_color)
26 27 
       expect(Time.parse(data[:exported_at])).to be_within(2).of(Time.now.utc)
27 28 
       expect(data[:links]).to eq([{ :source => 0, :receiver => 1 }])
29 
+      expect(data[:control_links]).to eq([])
28 30 
       expect(data[:agents]).to eq(agent_list.map { |agent| exporter.agent_as_json(agent) })
29 31 
       expect(data[:agents].all? { |agent_json| agent_json[:guid].present? && agent_json[:type].present? && agent_json[:name].present? }).to be_truthy
30 32 
 
@@ -38,6 +40,13 @@ describe AgentsExporter do
38 40
39 41 
       expect(exporter.as_json[:links]).to eq([{ :source => 0, :receiver => 1 }])
40 42 
     end
43 
+
44 
+    it "outputs control links to agents within the incoming set, but not outside it" do
45 
+      agents(:jane_rain_notifier_agent).control_targets = [agents(:jane_weather_agent), agents(:jane_basecamp_agent)]
46 
+      agents(:jane_rain_notifier_agent).save!
47 
+
48 
+      expect(exporter.as_json[:control_links]).to eq([{ :controller => 1, :control_target => 0 }])
49 
+    end
41 50 
   end
42 51
43 52 
   describe "#filename" do

+ 58 - 0
spec/lib/utils_spec.rb
View File 

@@ -114,4 +114,62 @@ describe Utils do
114 114 
       expect(cleaned_json).to include("<\\/script>")
115 115 
     end
116 116 
   end
117 
+
118 
+  describe "#sort_tuples!" do
119 
+    let(:tuples) {
120 
+      time = Time.now
121 
+      [
122 
+        [2, "a", time - 1],  # 0
123 
+        [2, "b", time - 1],  # 1
124 
+        [1, "b", time - 1],  # 2
125 
+        [1, "b", time],      # 3
126 
+        [1, "a", time],      # 4
127 
+        [2, "a", time + 1],  # 5
128 
+        [2, "a", time],      # 6
129 
+      ]
130 
+    }
131 
+
132 
+    it "sorts tuples like arrays by default" do
133 
+      expected = tuples.values_at(4, 2, 3, 0, 6, 5, 1)
134 
+
135 
+      Utils.sort_tuples!(tuples)
136 
+      expect(tuples).to eq expected
137 
+    end
138 
+
139 
+    it "sorts tuples in order specified: case 1" do
140 
+      # order by x1 asc, x2 desc, c3 asc
141 
+      orders = [false, true, false]
142 
+      expected = tuples.values_at(2, 3, 4, 1, 0, 6, 5)
143 
+
144 
+      Utils.sort_tuples!(tuples, orders)
145 
+      expect(tuples).to eq expected
146 
+    end
147 
+
148 
+    it "sorts tuples in order specified: case 2" do
149 
+      # order by x1 desc, x2 asc, c3 desc
150 
+      orders = [true, false, true]
151 
+      expected = tuples.values_at(5, 6, 0, 1, 4, 3, 2)
152 
+
153 
+      Utils.sort_tuples!(tuples, orders)
154 
+      expect(tuples).to eq expected
155 
+    end
156 
+
157 
+    it "always succeeds in sorting even if it finds pairs of incomparable objects" do
158 
+      time = Time.now
159 
+      tuples = [
160 
+        [2,   "a", time - 1],  # 0
161 
+        [1,   "b", nil],       # 1
162 
+        [1,   "b", time],      # 2
163 
+        ["2", nil, time],      # 3
164 
+        [1,   nil, time],      # 4
165 
+        [nil, "a", time + 1],  # 5
166 
+        [2,   "a", time],      # 6
167 
+      ]
168 
+      orders = [true, false, true]
169 
+      expected = tuples.values_at(3, 6, 0, 4, 2, 1, 5)
170 
+
171 
+      Utils.sort_tuples!(tuples, orders)
172 
+      expect(tuples).to eq expected
173 
+    end
174 
+  end
117 175 
 end

+ 11 - 11
spec/models/agent_spec.rb
View File 

@@ -546,11 +546,11 @@ describe Agent do
546 546 
         expect(agent).to have(1).errors_on(:keep_events_for)
547 547 
         agent.keep_events_for = ""
548 548 
         expect(agent).to have(1).errors_on(:keep_events_for)
549 
-        agent.keep_events_for = 5
549 
+        agent.keep_events_for = 5.days.to_i
550 550 
         expect(agent).to be_valid
551 551 
         agent.keep_events_for = 0
552 552 
         expect(agent).to be_valid
553 
-        agent.keep_events_for = 365
553 
+        agent.keep_events_for = 365.days.to_i
554 554 
         expect(agent).to be_valid
555 555
556 556 
         # Rails seems to call to_i on the input. This guards against future changes to that behavior.
 
@@ -564,7 +564,7 @@ describe Agent do
564 564 
         @time = "2014-01-01 01:00:00 +00:00"
565 565 
         time_travel_to @time do
566 566 
           @agent = Agents::SomethingSource.new(:name => "something")
567 
-          @agent.keep_events_for = 5
567 
+          @agent.keep_events_for = 5.days
568 568 
           @agent.user = users(:bob)
569 569 
           @agent.save!
570 570 
           @event = @agent.create_event :payload => { "hello" => "world" }
 
@@ -580,7 +580,7 @@ describe Agent do
580 580 
           @agent.save!
581 581
582 582 
           @agent.options[:foo] = "bar1"
583 
-          @agent.keep_events_for = 5
583 
+          @agent.keep_events_for = 5.days
584 584 
           @agent.save!
585 585 
         end
586 586 
       end
 
@@ -590,7 +590,7 @@ describe Agent do
590 590 
           time_travel_to @time do
591 591 
             expect {
592 592 
                 @agent.options[:foo] = "bar1"
593 
-                @agent.keep_events_for = 3
593 
+                @agent.keep_events_for = 3.days
594 594 
                 @agent.save!
595 595 
             }.to change { @event.reload.expires_at }
596 596 
             expect(@event.expires_at.to_i).to be_within(2).of(3.days.from_now.to_i)
 
@@ -603,7 +603,7 @@ describe Agent do
603 603
604 604 
           expect {
605 605 
             @agent.options[:foo] = "bar2"
606 
-            @agent.keep_events_for = 3
606 
+            @agent.keep_events_for = 3.days
607 607 
             @agent.save!
608 608 
           }.to change { @event.reload.expires_at }
609 609 
           expect(@event.expires_at.to_i).to be_within(60 * 61).of(1.days.from_now.to_i) # The larger time is to deal with daylight savings
 
@@ -635,7 +635,7 @@ describe Agent do
635 635 
         @receiver = Agents::CannotBeScheduled.new(
636 636 
           name: 'Agent',
637 637 
           options: { foo: 'bar3' },
638 
-          keep_events_for: 3,
638 
+          keep_events_for: 3.days,
639 639 
           propagate_immediately: true)
640 640 
         @receiver.user = users(:bob)
641 641 
         @receiver.sources << @sender
 
@@ -747,7 +747,7 @@ describe Agent do
747 747
748 748 
       it "sets expires_at on created events" do
749 749 
         event = agents(:jane_weather_agent).create_event :payload => { 'hi' => 'there' }
750 
-        expect(event.expires_at.to_i).to be_within(5).of(agents(:jane_weather_agent).keep_events_for.days.from_now.to_i)
750 
+        expect(event.expires_at.to_i).to be_within(5).of(agents(:jane_weather_agent).keep_events_for.seconds.from_now.to_i)
751 751 
       end
752 752 
     end
753 753 
 
@@ -836,7 +836,7 @@ describe AgentDrop do
836 836 
         },
837 837 
       },
838 838 
       schedule: 'every_1h',
839 
-      keep_events_for: 2)
839 
+      keep_events_for: 2.days)
840 840 
     @wsa1.user = users(:bob)
841 841 
     @wsa1.save!
842 842 
 
@@ -853,7 +853,7 @@ describe AgentDrop do
853 853 
         },
854 854 
       },
855 855 
       schedule: 'every_12h',
856 
-      keep_events_for: 2)
856 
+      keep_events_for: 2.days)
857 857 
     @wsa2.user = users(:bob)
858 858 
     @wsa2.save!
859 859 
 
@@ -868,7 +868,7 @@ describe AgentDrop do
868 868 
         matchers: [],
869 869 
         skip_created_at: 'false',
870 870 
       },
871 
-      keep_events_for: 2,
871 
+      keep_events_for: 2.days,
872 872 
       propagate_immediately: true)
873 873 
     @efa.user = users(:bob)
874 874 
     @efa.sources << @wsa1 << @wsa2

+ 16 - 0
spec/models/agents/data_output_agent_spec.rb
View File 

@@ -209,6 +209,22 @@ describe Agents::DataOutputAgent do
209 209 
         })
210 210 
       end
211 211
212 
+      describe 'ordering' do
213 
+        before do
214 
+          agent.options['events_order'] = ['{{title}}']
215 
+        end
216 
+
217 
+        it 'can reorder the events_to_show last events based on a Liquid expression' do
218 
+          asc_content, _status, _content_type = agent.receive_web_request({ 'secret' => 'secret2' }, 'get', 'application/json')
219 
+          expect(asc_content['items'].map {|i| i["title"] }).to eq(["Evolving", "Evolving again", "Evolving yet again with a past date"])
220 
+
221 
+          agent.options['events_order'] = [['{{title}}', 'string', true]]
222 
+
223 
+          desc_content, _status, _content_type = agent.receive_web_request({ 'secret' => 'secret2' }, 'get', 'application/json')
224 
+          expect(desc_content['items']).to eq(asc_content['items'].reverse)
225 
+        end
226 
+      end
227 
+
212 228 
       describe "interpolating \"events\"" do
213 229 
         before do
214 230 
           agent.options['template']['title'] = "XKCD comics as a feed{% if events.first.site_title %} ({{events.first.site_title}}){% endif %}"

+ 1 - 1
spec/models/agents/ftpsite_agent_spec.rb
View File 

@@ -9,7 +9,7 @@ describe Agents::FtpsiteAgent do
9 9 
         'url' => "ftp://ftp.example.org/pub/releases/",
10 10 
         'patterns' => ["example*.tar.gz"],
11 11 
       }
12 
-      @checker = Agents::FtpsiteAgent.new(:name => "Example", :options => @site, :keep_events_for => 2)
12 
+      @checker = Agents::FtpsiteAgent.new(:name => "Example", :options => @site, :keep_events_for => 2.days)
13 13 
       @checker.user = users(:bob)
14 14 
       @checker.save!
15 15 
     end

+ 1 - 1
spec/models/agents/imap_folder_agent_spec.rb
View File 

@@ -14,7 +14,7 @@ describe Agents::ImapFolderAgent do
14 14 
         'conditions' => {
15 15 
         }
16 16 
       }
17 
-      @checker = Agents::ImapFolderAgent.new(:name => 'Example', :options => @site, :keep_events_for => 2)
17 
+      @checker = Agents::ImapFolderAgent.new(:name => 'Example', :options => @site, :keep_events_for => 2.days)
18 18 
       @checker.user = users(:bob)
19 19 
       @checker.save!
20 20

+ 25 - 0
spec/models/agents/rss_agent_spec.rb
View File 

@@ -66,6 +66,21 @@ describe Agents::RssAgent do
66 66 
       expect(last.payload['urls']).to eq(["https://github.com/cantino/huginn/commit/d465158f77dcd9078697e6167b50abbfdfa8b1af"])
67 67 
     end
68 68
69 
+    it "should emit items as events in the order specified in the events_order option" do
70 
+      expect {
71 
+        agent.options['events_order'] = ['{{title | replace_regex: "^[[:space:]]+", "" }}']
72 
+        agent.check
73 
+      }.to change { agent.events.count }.by(20)
74 
+
75 
+      first, *, last = agent.events.last(20)
76 
+      expect(first.payload['title'].strip).to eq('upgrade rails and gems')
77 
+      expect(first.payload['url']).to eq("https://github.com/cantino/huginn/commit/87a7abda23a82305d7050ac0bb400ce36c863d01")
78 
+      expect(first.payload['urls']).to eq(["https://github.com/cantino/huginn/commit/87a7abda23a82305d7050ac0bb400ce36c863d01"])
79 
+      expect(last.payload['title'].strip).to eq('Dashed line in a diagram indicates propagate_immediately being false.')
80 
+      expect(last.payload['url']).to eq("https://github.com/cantino/huginn/commit/0e80f5341587aace2c023b06eb9265b776ac4535")
81 
+      expect(last.payload['urls']).to eq(["https://github.com/cantino/huginn/commit/0e80f5341587aace2c023b06eb9265b776ac4535"])
82 
+    end
83 
+
69 84 
     it "should track ids and not re-emit the same item when seen again" do
70 85 
       agent.check
71 86 
       expect(agent.memory['seen_ids']).to eq(agent.events.map {|e| e.payload['id'] })
 
@@ -133,4 +148,14 @@ describe Agents::RssAgent do
133 148 
       expect(agent.memory['seen_ids']).to eq(agent.events.map {|e| Digest::MD5.hexdigest(e.payload['content']) })
134 149 
     end
135 150 
   end
151 
+
152 
+  describe 'logging errors with the feed url' do
153 
+    it 'includes the feed URL when an exception is raised' do
154 
+      mock(FeedNormalizer::FeedNormalizer).parse(anything) { raise StandardError.new("Some error!") }
155 
+      expect(lambda {
156 
+        agent.check
157 
+      }).not_to raise_error
158 
+      expect(agent.logs.last.message).to match(%r[Failed to fetch https://github.com])
159 
+    end
160 
+  end
136 161 
 end

+ 2 - 2
spec/models/agents/website_agent_spec.rb
View File 

@@ -20,7 +20,7 @@ describe Agents::WebsiteAgent do
20 20 
           'hovertext' => { 'css' => "#comic img", 'value' => "@title" }
21 21 
         }
22 22 
       }
23 
-      @checker = Agents::WebsiteAgent.new(:name => "xkcd", :options => @valid_options, :keep_events_for => 2)
23 
+      @checker = Agents::WebsiteAgent.new(:name => "xkcd", :options => @valid_options, :keep_events_for => 2.days)
24 24 
       @checker.user = users(:bob)
25 25 
       @checker.save!
26 26 
     end
 
@@ -386,7 +386,7 @@ describe Agents::WebsiteAgent do
386 386 
               'url' => { 'xpath' => '/feed/entry', 'value' => './link[1]/@href' },
387 387 
               'thumbnail' => { 'xpath' => '/feed/entry', 'value' => './thumbnail/@url' },
388 388 
             }
389 
-          }, keep_events_for: 2)
389 
+          }, keep_events_for: 2.days)
390 390 
           @checker.user = users(:bob)
391 391 
           @checker.save!
392 392 
         end

+ 89 - 9
spec/models/scenario_import_spec.rb
View File 

@@ -30,7 +30,7 @@ describe ScenarioImport do
30 30 
       :type => "Agents::WeatherAgent",
31 31 
       :name => "a weather agent",
32 32 
       :schedule => "5pm",
33 
-      :keep_events_for => 14,
33 
+      :keep_events_for => 14.days,
34 34 
       :disabled => true,
35 35 
       :guid => "a-weather-agent",
36 36 
       :options => weather_agent_options
 
@@ -61,6 +61,7 @@ describe ScenarioImport do
61 61 
   end
62 62 
   let(:valid_parsed_data) do
63 63 
     {
64 
+      :schema_version => 1,
64 65 
       :name => name,
65 66 
       :description => description,
66 67 
       :guid => guid,
 
@@ -74,7 +75,8 @@ describe ScenarioImport do
74 75 
       ],
75 76 
       :links => [
76 77 
         { :source => 0, :receiver => 1 }
77 
-      ]
78 
+      ],
79 
+      :control_links => []
78 80 
     }
79 81 
   end
80 82 
   let(:valid_data) { valid_parsed_data.to_json }
 
@@ -203,7 +205,7 @@ describe ScenarioImport do
203 205
204 206 
           expect(weather_agent.name).to eq("a weather agent")
205 207 
           expect(weather_agent.schedule).to eq("5pm")
206 
-          expect(weather_agent.keep_events_for).to eq(14)
208 
+          expect(weather_agent.keep_events_for).to eq(14.days)
207 209 
           expect(weather_agent.propagate_immediately).to be_falsey
208 210 
           expect(weather_agent).to be_disabled
209 211 
           expect(weather_agent.memory).to be_empty
 
@@ -226,6 +228,55 @@ describe ScenarioImport do
226 228 
             scenario_import.import
227 229 
           }.to change { users(:bob).agents.count }.by(2)
228 230 
         end
231 
+
232 
+        context "when the schema_version is less than 1" do
233 
+          before do
234 
+            valid_parsed_weather_agent_data[:keep_events_for] = 2
235 
+            valid_parsed_data.delete(:schema_version)
236 
+          end
237 
+
238 
+          it "translates keep_events_for from days to seconds" do
239 
+            scenario_import.import
240 
+            expect(scenario_import.errors).to be_empty
241 
+            weather_agent = scenario_import.scenario.agents.find_by(:guid => "a-weather-agent")
242 
+            trigger_agent = scenario_import.scenario.agents.find_by(:guid => "a-trigger-agent")
243 
+
244 
+            expect(weather_agent.keep_events_for).to eq(2.days)
245 
+            expect(trigger_agent.keep_events_for).to eq(0)
246 
+          end
247 
+        end
248 
+
249 
+        describe "with control links" do
250 
+          it 'creates the links' do
251 
+            valid_parsed_data[:control_links] = [
252 
+              { :controller => 1, :control_target => 0 }
253 
+            ]
254 
+
255 
+            expect {
256 
+              scenario_import.import
257 
+            }.to change { users(:bob).agents.count }.by(2)
258 
+
259 
+            weather_agent = scenario_import.scenario.agents.find_by(:guid => "a-weather-agent")
260 
+            trigger_agent = scenario_import.scenario.agents.find_by(:guid => "a-trigger-agent")
261 
+
262 
+            expect(trigger_agent.sources).to eq([weather_agent])
263 
+            expect(weather_agent.controllers.to_a).to eq([trigger_agent])
264 
+            expect(trigger_agent.control_targets.to_a).to eq([weather_agent])
265 
+          end
266 
+
267 
+          it "doesn't crash without any control links" do
268 
+            valid_parsed_data.delete(:control_links)
269 
+
270 
+            expect {
271 
+              scenario_import.import
272 
+            }.to change { users(:bob).agents.count }.by(2)
273 
+
274 
+            weather_agent = scenario_import.scenario.agents.find_by(:guid => "a-weather-agent")
275 
+            trigger_agent = scenario_import.scenario.agents.find_by(:guid => "a-trigger-agent")
276 
+
277 
+            expect(trigger_agent.sources).to eq([weather_agent])
278 
+          end
279 
+        end
229 280 
       end
230 281
231 282 
       describe "#generate_diff" do
 
@@ -309,7 +360,7 @@ describe ScenarioImport do
309 360
310 361 
           expect(weather_agent.name).to eq("a weather agent")
311 362 
           expect(weather_agent.schedule).to eq("5pm")
312 
-          expect(weather_agent.keep_events_for).to eq(14)
363 
+          expect(weather_agent.keep_events_for).to eq(14.days)
313 364 
           expect(weather_agent.propagate_immediately).to be_falsey
314 365 
           expect(weather_agent).to be_disabled
315 366 
           expect(weather_agent.memory).to be_empty
 
@@ -330,7 +381,7 @@ describe ScenarioImport do
330 381 
             "0" => {
331 382 
               "name" => "updated name",
332 383 
               "schedule" => "6pm",
333 
-              "keep_events_for" => "2",
384 
+              "keep_events_for" => 2.days.to_i.to_s,
334 385 
               "disabled" => "false",
335 386 
               "options" => weather_agent_options.merge("api_key" => "foo").to_json
336 387 
             }
 
@@ -343,7 +394,7 @@ describe ScenarioImport do
343 394 
           weather_agent = existing_scenario.agents.find_by(:guid => "a-weather-agent")
344 395 
           expect(weather_agent.name).to eq("updated name")
345 396 
           expect(weather_agent.schedule).to eq("6pm")
346 
-          expect(weather_agent.keep_events_for).to eq(2)
397 
+          expect(weather_agent.keep_events_for).to eq(2.days.to_i)
347 398 
           expect(weather_agent).not_to be_disabled
348 399 
           expect(weather_agent.options).to eq(weather_agent_options.merge("api_key" => "foo"))
349 400 
         end
 
@@ -353,7 +404,7 @@ describe ScenarioImport do
353 404 
             "0" => {
354 405 
               "name" => "",
355 406 
               "schedule" => "foo",
356 
-              "keep_events_for" => "2",
407 
+              "keep_events_for" => 2.days.to_i.to_s,
357 408 
               "options" => weather_agent_options.merge("api_key" => "").to_json
358 409 
             }
359 410 
           }
 
@@ -386,12 +437,40 @@ describe ScenarioImport do
386 437 
           end
387 438 
         end
388 439
440 
+        context "when the schema_version is less than 1" do
441 
+          it "translates keep_events_for from days to seconds" do
442 
+            valid_parsed_data.delete(:schema_version)
443 
+            valid_parsed_data[:agents] = [valid_parsed_weather_agent_data.merge(keep_events_for: 5)]
444 
+
445 
+            scenario_import.merges = {
446 
+              "0" => {
447 
+                "name" => "a new name",
448 
+                "schedule" => "6pm",
449 
+                "keep_events_for" => 2.days.to_i.to_s,
450 
+                "disabled" => "true",
451 
+                "options" => weather_agent_options.merge("api_key" => "foo").to_json
452 
+              }
453 
+            }
454 
+
455 
+            expect(scenario_import).to be_valid
456 
+
457 
+            weather_agent_diff = scenario_import.agent_diffs[0]
458 
+
459 
+            expect(weather_agent_diff.name.current).to eq(agents(:bob_weather_agent).name)
460 
+            expect(weather_agent_diff.name.incoming).to eq('a weather agent')
461 
+            expect(weather_agent_diff.name.updated).to eq('a new name')
462 
+            expect(weather_agent_diff.keep_events_for.current).to eq(45.days.to_i)
463 
+            expect(weather_agent_diff.keep_events_for.incoming).to eq(5.days.to_i)
464 
+            expect(weather_agent_diff.keep_events_for.updated).to eq(2.days.to_i.to_s)
465 
+          end
466 
+        end
467 
+
389 468 
         it "sets the 'updated' FieldDiff values based on any feedback from the user" do
390 469 
           scenario_import.merges = {
391 470 
             "0" => {
392 471 
               "name" => "a new name",
393 472 
               "schedule" => "6pm",
394 
-              "keep_events_for" => "2",
473 
+              "keep_events_for" => 2.days.to_s,
395 474 
               "disabled" => "true",
396 475 
               "options" => weather_agent_options.merge("api_key" => "foo").to_json
397 476 
             },
 
@@ -411,7 +490,8 @@ describe ScenarioImport do
411 490 
           expect(weather_agent_diff.name.updated).to eq("a new name")
412 491
413 492 
           expect(weather_agent_diff.schedule.updated).to eq("6pm")
414 
-          expect(weather_agent_diff.keep_events_for.updated).to eq("2")
493 
+          expect(weather_agent_diff.keep_events_for.current).to eq(45.days)
494 
+          expect(weather_agent_diff.keep_events_for.updated).to eq(2.days.to_s)
415 495 
           expect(weather_agent_diff.disabled.updated).to eq("true")
416 496 
           expect(weather_agent_diff.options.updated).to eq(weather_agent_options.merge("api_key" => "foo"))
417 497 
         end

+ 25 - 7
spec/models/users_spec.rb
View File 

@@ -3,15 +3,33 @@ require 'spec_helper'
3 3 
 describe User do
4 4 
   describe "validations" do
5 5 
     describe "invitation_code" do
6 
-      it "only accepts valid invitation codes" do
7 
-        User::INVITATION_CODES.each do |v|
8 
-          is_expected.to allow_value(v).for(:invitation_code)
6 
+      context "when configured to use invitation codes" do
7 
+        before do
8 
+          stub(User).using_invitation_code? {true}
9 
+        end
10 
+
11 
+        it "only accepts valid invitation codes" do
12 
+          User::INVITATION_CODES.each do |v|
13 
+            is_expected.to allow_value(v).for(:invitation_code)
14 
+          end
15 
+        end
16 
+
17 
+        it "can reject invalid invitation codes" do
18 
+          %w['foo', 'bar'].each do |v|
19 
+            is_expected.not_to allow_value(v).for(:invitation_code)
20 
+          end
9 21 
         end
10 22 
       end
11 
-
12 
-      it "can reject invalid invitation codes" do
13 
-        %w['foo', 'bar'].each do |v|
14 
-          is_expected.not_to allow_value(v).for(:invitation_code)
23 
+
24 
+      context "when configured not to use invitation codes" do
25 
+        before do
26 
+          stub(User).using_invitation_code? {false}
27 
+        end
28 
+
29 
+        it "skips this validation" do
30 
+          %w['foo', 'bar', nil, ''].each do |v|
31 
+            is_expected.to allow_value(v).for(:invitation_code)
32 
+          end
15 33 
         end
16 34 
       end
17 35 
     end

+ 33 - 0
spec/support/shared_examples/web_request_concern.rb
View File 

@@ -141,5 +141,38 @@ shared_examples_for WebRequestConcern do
141 141 
       agent.options['disable_ssl_verification'] = false
142 142 
       expect(agent.faraday.ssl.verify).to eq(true)
143 143 
     end
144 
+
145 
+    it "should use faradays default params_encoder" do
146 
+      expect(agent.faraday.options.params_encoder).to eq(nil)
147 
+      agent.options['disable_url_encoding'] = 'false'
148 
+      expect(agent.faraday.options.params_encoder).to eq(nil)
149 
+      agent.options['disable_url_encoding'] = false
150 
+      expect(agent.faraday.options.params_encoder).to eq(nil)
151 
+    end
152 
+
153 
+    it "should use WebRequestConcern::DoNotEncoder when disable_url_encoding is truthy" do
154 
+      agent.options['disable_url_encoding'] = true
155 
+      expect(agent.faraday.options.params_encoder).to eq(WebRequestConcern::DoNotEncoder)
156 
+      agent.options['disable_url_encoding'] = 'true'
157 
+      expect(agent.faraday.options.params_encoder).to eq(WebRequestConcern::DoNotEncoder)
158 
+    end
159 
+  end
160 
+
161 
+  describe WebRequestConcern::DoNotEncoder do
162 
+    it "should not encode special characters" do
163 
+      expect(WebRequestConcern::DoNotEncoder.encode('GetRss?CategoryNr=39207' => 'test')).to eq('GetRss?CategoryNr=39207=test')
164 
+    end
165 
+
166 
+    it "should work without a value present" do
167 
+      expect(WebRequestConcern::DoNotEncoder.encode('GetRss?CategoryNr=39207' => nil)).to eq('GetRss?CategoryNr=39207')
168 
+    end
169 
+
170 
+    it "should work without an empty value" do
171 
+      expect(WebRequestConcern::DoNotEncoder.encode('GetRss?CategoryNr=39207' => '')).to eq('GetRss?CategoryNr=39207=')
172 
+    end
173 
+
174 
+    it "should return the value when decoding" do
175 
+      expect(WebRequestConcern::DoNotEncoder.decode('val')).to eq(['val'])
176 
+    end
144 177 
   end
145 178 
 end

+ 1 - 1
spec/support/vcr_support.rb
View File 

@@ -2,7 +2,7 @@ require 'vcr'
2 2
3 3 
 VCR.configure do |c|
4 4 
   c.cassette_library_dir = 'spec/cassettes'
5 
-  c.allow_http_connections_when_no_cassette = true
5 
+  c.allow_http_connections_when_no_cassette = false
6 6 
   c.hook_into :webmock
7 7 
   c.default_cassette_options = { record: :new_episodes}
8 8 
   c.configure_rspec_metadata!