.env.example

@@ -47,6 +47,9 @@ FORCE_SSL=false

                 # You can see its use in user.rb.  PLEASE CHANGE THIS!
              

                 INVITATION_CODE=try-huginn
              

                 
              

                +# If you don't want to require new users to have an invitation code in order to sign up, set this to true.
              

                +SKIP_INVITATION_CODE=false
              

                +
              

                 #############################
              

                 #    Email Configuration    #
              

                 #############################
              
@@ -151,6 +154,10 @@ ENABLE_SECOND_PRECISION_SCHEDULE=false

                 # at the expense of time accuracy.
              

                 SCHEDULER_FREQUENCY=0.3
              

                 
              

                +# Specify the frequency with which the scheduler checks for and cleans up expired events.
              

                +# You can use `m` for minutes, `h` for hours, and `d` for days.
              

                +EVENT_EXPIRATION_CHECK=6h
              

                +
              

                 # Use Graphviz for generating diagrams instead of using Google Chart
              

                 # Tools.  Specify a dot(1) command path built with SVG support
              

                 # enabled.
              
@@ -163,7 +170,7 @@ TIMEZONE="Pacific Time (US & Canada)"

                 FAILED_JOBS_TO_KEEP=100
              

                 
              

                 # Maximum runtime of background jobs in minutes
              

                -DELAYED_JOB_MAX_RUNTIME=20
              

                +DELAYED_JOB_MAX_RUNTIME=2
              

                 
              

                 # Amount of seconds for delayed_job to sleep before checking for new jobs
              

                 DELAYED_JOB_SLEEP_DELAY=10
              

CHANGES.md

@@ -1,10 +1,19 @@

                 # Changes
              

                 
              

                +* Jul 30, 2015   - RssAgent can configure the order of events created via `events_order`.
              

                +* Jul 29, 2015   - WebsiteAgent can configure the order of events created via `events_order`.
              

                +* Jul 29, 2015   - DataOutputAgent can configure the order of events in the output via `events_order`.
              

                +* Jul 20, 2015   - Control Links (used by the SchedularAgent) are correctly exported in Scenarios.
              

                +* Jul 20, 2015   - keep\_events\_for was moved from days to seconds; Scenarios have a schema verison.
              

                +* Jul 8, 2015    - DataOutputAgent supports feed icon, and a new template variable `events`.
              

                 * Jul 1, 2015    - DeDuplicationAgent properly handles destruction of memory.
              

                 * Jun 26, 2015   - Add `max_events_per_run` to RssAgent.
              

                 * Jun 19, 2015   - Add `url_from_event` to WebsiteAgent.
              

                 * Jun 17, 2015   - RssAgent emits events for new feed items in chronological order.
              

                +* Jun 17, 2015   - Liquid filter `unescape` added.
              

                +* Jun 17, 2015   - Liquid filter `regex_replace` and `regex_replace_first` added, with escape sequence support.
              

                 * Jun 15, 2015   - Liquid filter `uri_expand` added.
              

                +* Jun 13, 2015   - Liquid templating engine is upgraded to version 3.
              

                 * Jun 12, 2015   - RSSAgent can now accept an array of URLs.
              

                 * Jun 8, 2015    - WebsiteAgent includes a `use_namespaces` option to enable XML namespaces.
              

                 * May 27, 2015   - Validation warns user if they have not provided a `path` when using JSONPath in WebsiteAgent.
              

Gemfile

@@ -1,5 +1,8 @@

                 source 'https://rubygems.org'
              

                 
              

                +# Ruby 2.0 is the minimum requirement
              

                +ruby ['2.0.0', RUBY_VERSION].max
              

                +
              

                 # Optional libraries.  To conserve RAM, comment out any that you don't need,
              

                 # then run `bundle` and commit the updated Gemfile and Gemfile.lock.
              

                 gem 'twilio-ruby', '~> 3.11.5'    # TwilioAgent
              
@@ -75,7 +78,7 @@ gem 'json', '~> 1.8.1'

                 gem 'jsonpath', '~> 0.5.6'
              

                 gem 'kaminari', '~> 0.16.1'
              

                 gem 'kramdown', '~> 1.3.3'
              

                -gem 'liquid', '~> 2.6.1'
              

                +gem 'liquid', '~> 3.0.3'
              

                 gem 'mini_magick'
              

                 gem 'mysql2', '~> 0.3.16'
              

                 gem 'multi_xml'
              

Gemfile.lock

@@ -224,7 +224,7 @@ GEM

                       railties (>= 3.0, < 5.0)
              

                       thor (>= 0.14, < 2.0)
              

                     json (1.8.3)
              

                -    jsonpath (0.5.6)
              

                +    jsonpath (0.5.7)
              

                       multi_json
              

                     jwt (1.4.1)
              

                     kaminari (0.16.1)
              
@@ -235,7 +235,7 @@ GEM

                     launchy (2.4.2)
              

                       addressable (~> 2.3)
              

                     libv8 (3.16.14.7)
              

                -    liquid (2.6.2)
              

                +    liquid (3.0.6)
              

                     listen (2.7.9)
              

                       celluloid (>= 0.15.2)
              

                       rb-fsevent (>= 0.9.3)
              
@@ -255,7 +255,7 @@ GEM

                     mini_portile (0.6.2)
              

                     minitest (5.7.0)
              

                     mqtt (0.3.1)
              

                -    multi_json (1.11.1)
              

                +    multi_json (1.11.2)
              

                     multi_xml (0.5.5)
              

                     multipart-post (2.0.0)
              

                     mysql2 (0.3.16)
              
@@ -529,7 +529,7 @@ DEPENDENCIES

                   jsonpath (~> 0.5.6)
              

                   kaminari (~> 0.16.1)
              

                   kramdown (~> 1.3.3)
              

                -  liquid (~> 2.6.1)
              

                +  liquid (~> 3.0.3)
              

                   mini_magick
              

                   mqtt
              

                   multi_xml
              
@@ -580,3 +580,6 @@ DEPENDENCIES

                   weibo_2!
              

                   wunderground (~> 1.2.0)
              

                   xmpp4r (~> 0.5.6)
              

                +
              

                +BUNDLED WITH
              

                +   1.10.5
              

README.md

@@ -80,9 +80,11 @@ All agents have specs! Test all specs with `bundle exec rspec`, or test a specif

                 
              

                 ## Deployment
              

                 
              

                -[![Deploy](https://www.herokucdn.com/deploy/button.png)](https://heroku.com/deploy) (Takes a few minutes to setup.  Be sure to click 'View it' after launch!)
              

                +Try Huginn on Heroku: [![Deploy](https://www.herokucdn.com/deploy/button.png)](https://heroku.com/deploy) (Takes a few minutes to setup.  Be sure to click 'View it' after launch!)
              

                 
              

                -Huginn can run on Heroku for free!  Please see [the Huginn Wiki](https://github.com/cantino/huginn/wiki#deploying-huginn) for detailed deployment strategies for different providers.
              

                +Huginn works on the free version of Heroku [with limitations](https://github.com/cantino/huginn/wiki/Run-Huginn-for-free-on-Heroku). For non-experimental use, we recommend Heroku's cheapest paid plan or our Docker container.
              

                +
              

                +Please see [the Huginn Wiki](https://github.com/cantino/huginn/wiki#deploying-huginn) for detailed deployment strategies for different providers.
              

                 
              

                 ### Optional Setup
              

                 
              

app.json

@@ -3,6 +3,7 @@

                     "description": "Build agents that monitor and act on your behalf.  Your agents are standing by!",
              

                     "website": "https://github.com/cantino/huginn",
              

                     "repository": "https://github.com/cantino/huginn",
              

                +    "logo": "https://raw.githubusercontent.com/cantino/huginn/master/media/huginn-icon-64.png",
              

                     "env": {
              

                         "BUILDPACK_URL": "https://github.com/heroku/heroku-buildpack-multi.git",
              

                         "APP_SECRET_TOKEN": {
              

app/assets/javascripts/components/utils.js.coffee

@@ -34,10 +34,67 @@ class @Utils

                     body?(modal.querySelector('.modal-body'))
              

                     $(modal).modal('show')
              

                 
              

                -  @handleDryRunButton: (button, data = $(button.form).serialize()) ->
              

                +  @handleDryRunButton: (button, data = if button.form then $(':input[name!="_method"]', button.form).serialize() else '') ->
              

                     $(button).prop('disabled', true)
              

                +    cleanup = -> $(button).prop('disabled', false)
              

                +
              

                +    url = $(button).data('action-url')
              

                +    with_event_mode = $(button).data('with-event-mode')
              

                +
              

                +    if with_event_mode is 'no'
              

                +      return @invokeDryRun(url, data, cleanup)
              

                +
              

                +    Utils.showDynamicModal """
              

                +      <h5>Event to send#{if with_event_mode is 'maybe' then ' (Optional)' else ''}</h5>
              

                +      <form class="dry-run-form" method="post">
              

                +        <div class="form-group">
              

                +          <textarea rows="10" name="event" class="payload-editor" data-height="200">
              

                +            {}
              

                +          </textarea>
              

                +        </div>
              

                +        <div class="form-group">
              

                +          <input value="Dry Run" class="btn btn-primary" type="submit" />
              

                +        </div>
              

                +      </form>
              

                +      """,
              

                +      body: (body) =>
              

                +        form = $(body).find('.dry-run-form')
              

                +        payload_editor = form.find('.payload-editor')
              

                +        if previous = $(button).data('payload')
              

                +          payload_editor.text(previous)
              

                +        window.setupJsonEditor(payload_editor)
              

                +        form.submit (e) =>
              

                +          e.preventDefault()
              

                +          json = $(e.target).find('.payload-editor').val()
              

                +          json = '{}' if json == ''
              

                +          try
              

                +            payload = JSON.parse(json)
              

                +            throw true unless payload.constructor is Object
              

                +            if Object.keys(payload).length == 0
              

                +              json = ''
              

                +            else
              

                +              json = JSON.stringify(payload)
              

                +          catch
              

                +            alert 'Invalid JSON object.'
              

                +            return
              

                +          if json == ''
              

                +            if with_event_mode is 'yes'
              

                +              alert 'Event is required for this agent to run.'
              

                +              return
              

                +            dry_run_data = data
              

                +            $(button).data('payload', null)
              

                +          else
              

                +            dry_run_data = "event=#{encodeURIComponent(json)}&#{data}"
              

                +            $(button).data('payload', json)
              

                +          $(body).closest('[role=dialog]').on 'hidden.bs.modal', =>
              

                +            @invokeDryRun(url, dry_run_data, cleanup)
              

                +          .modal('hide')
              

                +      title: 'Dry Run'
              

                +      onHide: cleanup
              

                +
              

                +  @invokeDryRun: (url, data, callback) ->
              

                     $('body').css(cursor: 'progress')
              

                -    $.ajax type: 'POST', url: $(button).data('action-url'), dataType: 'json', data: data
              

                +    $.ajax type: 'POST', url: url, dataType: 'json', data: data
              

                       .always =>
              

                         $('body').css(cursor: 'auto')
              

                       .done (json) =>
              
@@ -55,7 +112,7 @@ class @Utils

                               find('.agent-dry-run-events').text(json.events).end().
              

                               find('.agent-dry-run-memory').text(json.memory)
              

                           title: 'Dry Run Results',
              

                -          onHide: -> $(button).prop('disabled', false)
              

                +          onHide: callback
              

                       .fail (xhr, status, error) ->
              

                         alert('Error: ' + error)
              

                -        $(button).prop('disabled', false)
              

                +        callback()
              

app/concerns/dry_runnable.rb

@@ -1,10 +1,8 @@

                 module DryRunnable
              

                -  def dry_run!
              

                -    readonly!
              

                +  extend ActiveSupport::Concern
              

                 
              

                -    class << self
              

                -      prepend Sandbox
              

                -    end
              

                +  def dry_run!(event = nil)
              

                +    @dry_run = true
              

                 
              

                     log = StringIO.new
              

                     @dry_run_logger = Logger.new(log)
              
@@ -14,7 +12,13 @@ module DryRunnable

                 
              

                     begin
              

                       raise "#{short_type} does not support dry-run" unless can_dry_run?
              

                -      check
              

                +      readonly!
              

                +      if event
              

                +        raise "This agent cannot receive an event!" unless can_receive_events?
              

                +        receive([event])
              

                +      else
              

                +        check
              

                +      end
              

                     rescue => e
              

                       error "Exception during dry-run. #{e.message}: #{e.backtrace.join("\n")}"
              

                     end
              
@@ -23,28 +27,38 @@ module DryRunnable

                       memory: memory,
              

                       log: log.string,
              

                     )
              

                +  ensure
              

                +    @dry_run = false
              

                   end
              

                 
              

                   def dry_run?
              

                -    is_a? Sandbox
              

                +    !!@dry_run
              

                +  end
              

                +
              

                +  included do
              

                +    prepend Wrapper
              

                   end
              

                 
              

                -  module Sandbox
              

                +  module Wrapper
              

                     attr_accessor :results
              

                 
              

                     def logger
              

                +      return super unless dry_run?
              

                       @dry_run_logger
              

                     end
              

                 
              

                -    def save
              

                -      valid?
              

                +    def save(options = {})
              

                +      return super unless dry_run?
              

                +      perform_validations(options)
              

                     end
              

                 
              

                -    def save!
              

                -      save or raise ActiveRecord::RecordNotSaved
              

                +    def save!(options = {})
              

                +      return super unless dry_run?
              

                +      save(options) or raise_record_invalid
              

                     end
              

                 
              

                     def log(message, options = {})
              

                +      return super unless dry_run?
              

                       case options[:level] || 3
              

                       when 0..2
              

                         sev = Logger::DEBUG
              
@@ -57,10 +71,12 @@ module DryRunnable

                       logger.log(sev, message)
              

                     end
              

                 
              

                -    def create_event(event_hash)
              

                +    def create_event(event)
              

                +      return super unless dry_run?
              

                       if can_create_events?
              

                -        @dry_run_results[:events] << event_hash[:payload]
              

                -        events.build({ user: user, expires_at: new_event_expiration_date }.merge(event_hash))
              

                +        event = build_event(event)
              

                +        @dry_run_results[:events] << event.payload
              

                +        event
              

                       else
              

                         error "This Agent cannot create events!"
              

                       end
              

app/concerns/liquid_interpolatable.rb

@@ -185,6 +185,14 @@ module LiquidInterpolatable

                       url
              

                     end
              

                 
              

                +    # Unescape (basic) HTML entities in a string
              

                +    #
              

                +    # This currently decodes the following entities only: "'",
              

                +    # """, "<", ">", "&", "&#dd;" and "&#xhh;".
              

                +    def unescape(input)
              

                +      CGI.unescapeHTML(input) rescue input
              

                +    end
              

                +
              

                     # Escape a string for use in XPath expression
              

                     def to_xpath(string)
              

                       subs = string.to_s.scan(/\G(?:\A\z|[^"]+|[^']+)/).map { |x|
              
@@ -201,15 +209,15 @@ module LiquidInterpolatable

                         'concat(' << subs.join(', ') << ')'
              

                       end
              

                     end
              

                -    
              

                -    def regex_replace(input, regex, replacement = ''.freeze)
              

                -      input.to_s.gsub(Regexp.new(regex), replacement.to_s)
              

                +
              

                +    def regex_replace(input, regex, replacement = nil)
              

                +      input.to_s.gsub(Regexp.new(regex), unescape_replacement(replacement.to_s))
              

                     end
              

                -    
              

                -    def regex_replace_first(input, regex, replacement = ''.freeze)
              

                -      input.to_s.sub(Regexp.new(regex), replacement.to_s)
              

                +
              

                +    def regex_replace_first(input, regex, replacement = nil)
              

                +      input.to_s.sub(Regexp.new(regex), unescape_replacement(replacement.to_s))
              

                     end
              

                -    
              

                +
              

                     private
              

                 
              

                     def logger
              
@@ -221,6 +229,63 @@ module LiquidInterpolatable

                           Logger.new(STDERR)
              

                         end
              

                     end
              

                +
              

                +    BACKSLASH = "\\".freeze
              

                +
              

                +    UNESCAPE = {
              

                +      "a" => "\a",
              

                +      "b" => "\b",
              

                +      "e" => "\e",
              

                +      "f" => "\f",
              

                +      "n" => "\n",
              

                +      "r" => "\r",
              

                +      "s" => " ",
              

                +      "t" => "\t",
              

                +      "v" => "\v",
              

                +    }
              

                +
              

                +    # Unescape a replacement text for use in the second argument of
              

                +    # gsub/sub.  The following escape sequences are recognized:
              

                +    #
              

                +    # - "\\" (backslash itself)
              

                +    # - "\a" (alert)
              

                +    # - "\b" (backspace)
              

                +    # - "\e" (escape)
              

                +    # - "\f" (form feed)
              

                +    # - "\n" (new line)
              

                +    # - "\r" (carriage return)
              

                +    # - "\s" (space)
              

                +    # - "\t" (horizontal tab)
              

                +    # - "\u{XXXX}" (unicode codepoint)
              

                +    # - "\v" (vertical tab)
              

                +    # - "\xXX" (hexadecimal character)
              

                +    # - "\1".."\9" (numbered capture groups)
              

                +    # - "\+" (last capture group)
              

                +    # - "\k<name>" (named capture group)
              

                +    # - "\&" or "\0" (complete matched text)
              

                +    # - "\`" (string before match)
              

                +    # - "\'" (string after match)
              

                +    #
              

                +    # Octal escape sequences are deliberately unsupported to avoid
              

                +    # conflict with numbered capture groups.  Rather obscure Emacs
              

                +    # style character codes ("\C-x", "\M-\C-x" etc.) are also omitted
              

                +    # from this implementation.
              

                +    def unescape_replacement(s)
              

                +      s.gsub(/\\(?:([\d+&`'\\]|k<\w+>)|u\{([[:xdigit:]]+)\}|x([[:xdigit:]]{2})|(.))/) {
              

                +        if c = $1
              

                +          BACKSLASH + c
              

                +        elsif c = ($2 && [$2.to_i(16)].pack('U')) ||
              

                +                  ($3 && [$3.to_i(16)].pack('C'))
              

                +          if c == BACKSLASH
              

                +            BACKSLASH + c
              

                +          else
              

                +            c
              

                +          end
              

                +        else
              

                +          UNESCAPE[$4] || $4
              

                +        end
              

                +      }
              

                +    end
              

                   end
              

                   Liquid::Template.register_filter(LiquidInterpolatable::Filters)
              

                 
              

app/concerns/sortable_events.rb

@@ -0,0 +1,161 @@

                +module SortableEvents
              

                +  extend ActiveSupport::Concern
              

                +
              

                +  included do
              

                +    validate :validate_events_order
              

                +  end
              

                +
              

                +  def description_events_order(*args)
              

                +    self.class.description_events_order(*args)
              

                +  end
              

                +
              

                +  module ClassMethods
              

                +    def can_order_created_events!
              

                +      raise if cannot_create_events?
              

                +      prepend AutomaticSorter
              

                +    end
              

                +
              

                +    def can_order_created_events?
              

                +      include? AutomaticSorter
              

                +    end
              

                +
              

                +    def cannot_order_created_events?
              

                +      !can_order_created_events?
              

                +    end
              

                +
              

                +    def description_events_order(events = 'events created in each run')
              

                +      <<-MD.lstrip
              

                +        To specify the order of #{events}, set `events_order` to an array of sort keys, each of which looks like either `expression` or `[expression, type, descending]`, as described as follows:
              

                +
              

                +        * _expression_ is a Liquid template to generate a string to be used as sort key.
              

                +
              

                +        * _type_ (optional) is one of `string` (default), `number` and `time`, which specifies how to evaluate _expression_ for comparison.
              

                +
              

                +        * _descending_ (optional) is a boolean value to determine if comparison should be done in descending (reverse) order, which defaults to `false`.
              

                +
              

                +        Sort keys listed earlier take precedence over ones listed later.  For example, if you want to sort articles by the date and then by the author, specify `[["{{date}}", "time"], "{{author}}"]`.
              

                +
              

                +        Sorting is done stably, so even if all events have the same set of sort key values the original order is retained.  Also, a special Liquid variable `_index_` is provided, which contains the zero-based index number of each event, which means you can exactly reverse the order of events by specifying `[["{{_index_}}", "number", true]]`.
              

                +      MD
              

                +    end
              

                +  end
              

                +
              

                +  def can_order_created_events?
              

                +    self.class.can_order_created_events?
              

                +  end
              

                +
              

                +  def cannot_order_created_events?
              

                +    self.class.cannot_order_created_events?
              

                +  end
              

                +
              

                +  def events_order
              

                +    options['events_order']
              

                +  end
              

                +
              

                +  module AutomaticSorter
              

                +    def check
              

                +      return super unless events_order
              

                +      sorting_events do
              

                +        super
              

                +      end
              

                +    end
              

                +
              

                +    def receive(incoming_events)
              

                +      return super unless events_order
              

                +      # incoming events should be processed sequentially
              

                +      incoming_events.each do |event|
              

                +        sorting_events do
              

                +          super([event])
              

                +        end
              

                +      end
              

                +    end
              

                +
              

                +    def create_event(event)
              

                +      if @sortable_events
              

                +        event = build_event(event)
              

                +        @sortable_events << event
              

                +        event
              

                +      else
              

                +        super
              

                +      end
              

                +    end
              

                +
              

                +    private
              

                +
              

                +    def sorting_events(&block)
              

                +      @sortable_events = []
              

                +      yield
              

                +    ensure
              

                +      events, @sortable_events = @sortable_events, nil
              

                +      sort_events(events).each do |event|
              

                +        create_event(event)
              

                +      end
              

                +    end
              

                +  end
              

                +
              

                +  private
              

                +
              

                +  EXPRESSION_PARSER = {
              

                +    'string' => ->string { string },
              

                +    'number' => ->string { string.to_f },
              

                +    'time'   => ->string { Time.zone.parse(string) },
              

                +  }
              

                +  EXPRESSION_TYPES = EXPRESSION_PARSER.keys.freeze
              

                +
              

                +  def validate_events_order
              

                +    case order_by = events_order
              

                +    when nil
              

                +    when Array
              

                +      # Each tuple may be either [expression, type, desc] or just
              

                +      # expression.
              

                +      order_by.each do |expression, type, desc|
              

                +        case expression
              

                +        when String
              

                +          # ok
              

                +        else
              

                +          errors.add(:base, "first element of each events_order tuple must be a Liquid template")
              

                +          break
              

                +        end
              

                +        case type
              

                +        when nil, *EXPRESSION_TYPES
              

                +          # ok
              

                +        else
              

                +          errors.add(:base, "second element of each events_order tuple must be #{EXPRESSION_TYPES.to_sentence(last_word_connector: ' or ')}")
              

                +          break
              

                +        end
              

                +        if !desc.nil? && boolify(desc).nil?
              

                +          errors.add(:base, "third element of each events_order tuple must be a boolean value")
              

                +          break
              

                +        end
              

                +      end
              

                +    else
              

                +      errors.add(:base, "events_order must be an array of arrays")
              

                +    end
              

                +  end
              

                +
              

                +  # Sort given events in order specified by the "events_order" option
              

                +  def sort_events(events)
              

                +    order_by = events_order.presence or
              

                +      return events
              

                +
              

                +    orders = order_by.map { |_, _, desc = false| boolify(desc) }
              

                +
              

                +    Utils.sort_tuples!(
              

                +      events.map.with_index { |event, index|
              

                +        interpolate_with(event) {
              

                +          interpolation_context['_index_'] = index
              

                +          order_by.map { |expression, type, _|
              

                +            string = interpolate_string(expression)
              

                +            begin
              

                +              EXPRESSION_PARSER[type || 'string'.freeze][string]
              

                +            rescue
              

                +              error "Cannot parse #{string.inspect} as #{type}; treating it as string"
              

                +              string
              

                +            end
              

                +          }
              

                +        } << index << event  # index is to make sorting stable
              

                +      },
              

                +      orders
              

                +    ).collect!(&:last)
              

                +  end
              

                +end
              

app/concerns/web_request_concern.rb

@@ -2,6 +2,58 @@ require 'faraday'

                 require 'faraday_middleware'
              

                 
              

                 module WebRequestConcern
              

                +  module DoNotEncoder
              

                +    def self.encode(params)
              

                +      params.map do |key, value|
              

                +        value.nil? ? "#{key}" : "#{key}=#{value}"
              

                +      end.join('&')
              

                +    end
              

                +
              

                +    def self.decode(val)
              

                +      [val]
              

                +    end
              

                +  end
              

                +
              

                +  class CharacterEncoding < Faraday::Middleware
              

                +    def initialize(app, force_encoding: nil, default_encoding: nil, unzip: nil)
              

                +      super(app)
              

                +      @force_encoding   = force_encoding
              

                +      @default_encoding = default_encoding
              

                +      @unzip            = unzip
              

                +    end
              

                +
              

                +    def call(env)
              

                +      @app.call(env).on_complete do |env|
              

                +        body = env[:body]
              

                +
              

                +        case @unzip
              

                +        when 'gzip'.freeze
              

                +          body.replace(ActiveSupport::Gzip.decompress(body))
              

                +        end
              

                +
              

                +        case
              

                +        when @force_encoding
              

                +          encoding = @force_encoding
              

                +        when body.encoding == Encoding::ASCII_8BIT
              

                +          # Not all Faraday adapters support automatic charset
              

                +          # detection, so we do that.
              

                +          case env[:response_headers][:content_type]
              

                +          when /;\s*charset\s*=\s*([^()<>@,;:\\\"\/\[\]?={}\s]+)/i
              

                +            encoding = Encoding.find($1) rescue nil
              

                +          when /\A\s*(?:text\/[^\s;]+|application\/(?:[^\s;]+\+)?(?:xml|json))\s*(?:;|\z)/i
              

                +            encoding = @default_encoding
              

                +          else
              

                +            # Never try to transcode a binary content
              

                +            return
              

                +          end
              

                +        end
              

                +        body.encode!(Encoding::UTF_8, encoding) unless body.encoding == Encoding::UTF_8
              

                +      end
              

                +    end
              

                +  end
              

                +
              

                +  Faraday::Response.register_middleware character_encoding: CharacterEncoding
              

                +
              

                   extend ActiveSupport::Concern
              

                 
              

                   def validate_web_request_options!
              
@@ -22,6 +74,23 @@ module WebRequestConcern

                     rescue ArgumentError => e
              

                       errors.add(:base, e.message)
              

                     end
              

                +
              

                +    if (encoding = options['force_encoding']).present?
              

                +      case encoding
              

                +      when String
              

                +        begin
              

                +          Encoding.find(encoding)
              

                +        rescue ArgumentError
              

                +          errors.add(:base, "Unknown encoding: #{encoding.inspect}")
              

                +        end
              

                +      else
              

                +        errors.add(:base, "force_encoding must be a string")
              

                +      end
              

                +    end
              

                +  end
              

                +
              

                +  def default_encoding
              

                +    Encoding::UTF_8
              

                   end
              

                 
              

                   def faraday
              
@@ -32,12 +101,22 @@ module WebRequestConcern

                     }
              

                 
              

                     @faraday ||= Faraday.new(faraday_options) { |builder|
              

                +      builder.response :character_encoding,
              

                +                       force_encoding: interpolated['force_encoding'].presence,
              

                +                       default_encoding: default_encoding,
              

                +                       unzip: interpolated['unzip'].presence
              

                +
              

                       builder.headers = headers if headers.length > 0
              

                 
              

                       builder.headers[:user_agent] = user_agent
              

                 
              

                       builder.use FaradayMiddleware::FollowRedirects
              

                       builder.request :url_encoded
              

                +
              

                +      if boolify(interpolated['disable_url_encoding'])
              

                +        builder.options.params_encoder = DoNotEncoder
              

                +      end
              

                +
              

                       if userinfo = basic_auth_credentials
              

                         builder.request :basic_auth, *userinfo
              

                       end
              

app/controllers/agents_controller.rb

@@ -37,7 +37,7 @@ class AgentsController < ApplicationController

                   def dry_run
              

                     attrs = params[:agent] || {}
              

                     if agent = current_user.agents.find_by(id: params[:id])
              

                -      # PUT /agents/:id/dry_run
              

                +      # POST /agents/:id/dry_run
              

                       if attrs.present?
              

                         type = agent.type
              

                         agent = Agent.build_for_type(type, current_user, attrs)
              
@@ -50,7 +50,13 @@ class AgentsController < ApplicationController

                     agent.name ||= '(Untitled)'
              

                 
              

                     if agent.valid?
              

                -      results = agent.dry_run!
              

                +      if event_payload = params[:event]
              

                +        dummy_agent = Agent.build_for_type('ManualEventAgent', current_user, name: 'Dry-Runner')
              

                +        dummy_agent.readonly!
              

                +        event = dummy_agent.events.build(user: current_user, payload: event_payload)
              

                +      end
              

                +
              

                +      results = agent.dry_run!(event)
              

                 
              

                       render json: {
              

                         log: results[:log],
              

app/helpers/agent_helper.rb

@@ -37,4 +37,16 @@ module AgentHelper

                       }.join(delimiter).html_safe
              

                     end
              

                   end
              

                +
              

                +  def agent_dry_run_with_event_mode(agent)
              

                +    case
              

                +    when agent.cannot_receive_events?
              

                +      'no'.freeze
              

                +    when agent.cannot_be_scheduled?
              

                +      # incoming event is the only trigger for the agent
              

                +      'yes'.freeze
              

                +    else
              

                +      'maybe'.freeze
              

                +    end
              

                +  end
              

                 end
              

app/helpers/application_helper.rb

@@ -80,6 +80,7 @@ module ApplicationHelper

                   end
              

                 
              

                   def service_label(service)
              

                +    return if service.nil?
              

                     content_tag :span, [
              

                       omniauth_provider_icon(service.provider),
              

                       service_label_text(service)
              

app/models/agent.rb

@@ -13,6 +13,7 @@ class Agent < ActiveRecord::Base

                   include HasGuid
              

                   include LiquidDroppable
              

                   include DryRunnable
              

                +  include SortableEvents
              

                 
              

                   markdown_class_attributes :description, :event_description
              

                 
              
@@ -21,7 +22,7 @@ class Agent < ActiveRecord::Base

                   SCHEDULES = %w[every_1m every_2m every_5m every_10m every_30m every_1h every_2h every_5h every_12h every_1d every_2d every_7d
              

                                  midnight 1am 2am 3am 4am 5am 6am 7am 8am 9am 10am 11am noon 1pm 2pm 3pm 4pm 5pm 6pm 7pm 8pm 9pm 10pm 11pm never]
              

                 
              

                -  EVENT_RETENTION_SCHEDULES = [["Forever", 0], ["1 day", 1], *([2, 3, 4, 5, 7, 14, 21, 30, 45, 90, 180, 365].map {|n| ["#{n} days", n] })]
              

                +  EVENT_RETENTION_SCHEDULES = [["Forever", 0], ['1 hour', 1.hour], ['6 hours', 6.hours], ["1 day", 1.day], *([2, 3, 4, 5, 7, 14, 21, 30, 45, 90, 180, 365].map {|n| ["#{n} days", n.days] })]
              

                 
              

                   attr_accessible :options, :memory, :name, :type, :schedule, :controller_ids, :control_target_ids, :disabled, :source_ids, :scenario_ids, :keep_events_for, :propagate_immediately, :drop_pending_events
              

                 
              
@@ -104,12 +105,19 @@ class Agent < ActiveRecord::Base

                     raise "Implement me in your subclass"
              

                   end
              

                 
              

                -  def create_event(attrs)
              

                +  def build_event(event)
              

                +    event = events.build(event) if event.is_a?(Hash)
              

                +    event.agent = self
              

                +    event.user = user
              

                +    event.expires_at ||= new_event_expiration_date
              

                +    event
              

                +  end
              

                +
              

                +  def create_event(event)
              

                     if can_create_events?
              

                -      events.create!({
              

                -         :user => user,
              

                -         :expires_at => new_event_expiration_date
              

                -      }.merge(attrs))
              

                +      event = build_event(event)
              

                +      event.save!
              

                +      event
              

                     else
              

                       error "This Agent cannot create events!"
              

                     end
              
@@ -130,14 +138,14 @@ class Agent < ActiveRecord::Base

                   end
              

                 
              

                   def new_event_expiration_date
              

                -    keep_events_for > 0 ? keep_events_for.days.from_now : nil
              

                +    keep_events_for > 0 ? keep_events_for.seconds.from_now : nil
              

                   end
              

                 
              

                   def update_event_expirations!
              

                     if keep_events_for == 0
              

                       events.update_all :expires_at => nil
              

                     else
              

                -      events.update_all "expires_at = " + rdbms_date_add("created_at", "DAY", keep_events_for.to_i)
              

                +      events.update_all "expires_at = " + rdbms_date_add("created_at", "SECOND", keep_events_for.to_i)
              

                     end
              

                   end
              

                 
              

app/models/agents/data_output_agent.rb

@@ -40,11 +40,15 @@ module Agents

                               "_contents": "tag contents (can be an object for nesting)"
              

                             }
              

                 
              

                +        # Ordering events in the output
              

                +
              

                +        #{description_events_order('events in the output')}
              

                +
              

                         # Liquid Templating
              

                 
              

                         In Liquid templating, the following variable is available:
              

                 
              

                -        * `events`: An array of events being output, sorted in descending order up to `events_to_show` in number.  For example, if source events contain a site title in the `site_title` key, you can refer to it in `template.title` by putting `{{events.first.site_title}}`.
              

                +        * `events`: An array of events being output, sorted in the given order, up to `events_to_show` in number.  For example, if source events contain a site title in the `site_title` key, you can refer to it in `template.title` by putting `{{events.first.site_title}}`.
              

                 
              

                       MD
              

                     end
              
@@ -134,7 +138,7 @@ module Agents

                         end
              

                       end
              

                 
              

                -      source_events = received_events.order(id: :desc).limit(events_to_show).to_a
              

                +      source_events = sort_events(received_events.order(id: :desc).limit(events_to_show).to_a)
              

                 
              

                       interpolation_context.stack do
              

                         interpolation_context['events'] = source_events
              

app/models/agents/email_agent.rb

@@ -33,8 +33,8 @@ module Agents

                 
              

                     def receive(incoming_events)
              

                       incoming_events.each do |event|
              

                -        log "Sending digest mail to #{user.email} with event #{event.id}"
              

                         recipients(event.payload).each do |recipient|
              

                +          log "Sending digest mail to #{recipient} with event #{event.id}"
              

                           SystemMailer.send_message(:to => recipient, :subject => interpolated(event)['subject'], :headline => interpolated(event)['headline'], :body => interpolated(event)['body'], :groups => [present(event.payload)]).deliver_later
              

                         end
              

                       end
              

app/models/agents/email_digest_agent.rb

@@ -40,8 +40,8 @@ module Agents

                       if self.memory['queue'] && self.memory['queue'].length > 0
              

                         ids = self.memory['events'].join(",")
              

                         groups = self.memory['queue'].map { |payload| present(payload) }
              

                -        log "Sending digest mail to #{user.email} with events [#{ids}]"
              

                         recipients.each do |recipient|
              

                +          log "Sending digest mail to #{recipient} with events [#{ids}]"
              

                           SystemMailer.send_message(:to => recipient, :subject => interpolated['subject'], :headline => interpolated['headline'], :groups => groups).deliver_later
              

                         end
              

                         self.memory['queue'] = []
              

app/models/agents/rss_agent.rb

@@ -9,6 +9,8 @@ module Agents

                     can_dry_run!
              

                     default_schedule "every_1d"
              

                 
              

                +    DEFAULT_EVENTS_ORDER = [['{{date_published}}', 'time'], ['{{last_updated}}', 'time']]
              

                +
              

                     description do
              

                       <<-MD
              

                         This Agent consumes RSS feeds and emits events when they change.
              
@@ -26,8 +28,16 @@ module Agents

                           * `headers` - When present, it should be a hash of headers to send with the request.
              

                           * `basic_auth` - Specify HTTP basic auth parameters: `"username:password"`, or `["username", "password"]`.
              

                           * `disable_ssl_verification` - Set to `true` to disable ssl verification.
              

                +          * `disable_url_encoding` - Set to `true` to disable url encoding.
              

                +          * `force_encoding` - Set `force_encoding` to an encoding name if the website is known to respond with a missing, invalid or wrong charset in the Content-Type header.  Note that a text content without a charset is taken as encoded in UTF-8 (not ISO-8859-1).
              

                           * `user_agent` - A custom User-Agent name (default: "Faraday v#{Faraday::VERSION}").
              

                           * `max_events_per_run` - Limit number of events created (items parsed) per run for feed.
              

                +
              

                +        # Ordering Events
              

                +
              

                +        #{description_events_order}
              

                +
              

                +        In this Agent, the default value for `events_order` is `#{DEFAULT_EVENTS_ORDER.to_json}`.
              

                       MD
              

                     end
              

                 
              
@@ -69,44 +79,44 @@ module Agents

                       end
              

                 
              

                       validate_web_request_options!
              

                +      validate_events_order
              

                +    end
              

                +
              

                +    def events_order
              

                +      super.presence || DEFAULT_EVENTS_ORDER
              

                     end
              

                 
              

                     def check
              

                       Array(interpolated['url']).each do |url|
              

                -        response = faraday.get(url)
              

                -        if response.success?
              

                -          feed = FeedNormalizer::FeedNormalizer.parse(response.body)
              

                -          feed.clean! if interpolated['clean'] == 'true'
              

                -          max_events = (interpolated['max_events_per_run'].presence || 0).to_i
              

                -          created_event_count = 0
              

                -          feed.entries.sort_by { |entry| [entry.date_published, entry.last_updated] }.each.with_index do |entry, index|
              

                -            break if max_events && max_events > 0 && index >= max_events
              

                -            entry_id = get_entry_id(entry)
              

                -            if check_and_track(entry_id)
              

                -              created_event_count += 1
              

                -              create_event(payload: {
              

                -                id: entry_id,
              

                -                date_published: entry.date_published,
              

                -                last_updated: entry.last_updated,
              

                -                url: entry.url,
              

                -                urls: entry.urls,
              

                -                description: entry.description,
              

                -                content: entry.content,
              

                -                title: entry.title,
              

                -                authors: entry.authors,
              

                -                categories: entry.categories
              

                -              })
              

                -            end
              

                -          end
              

                -          log "Fetched #{url} and created #{created_event_count} event(s)."
              

                -        else
              

                -          error "Failed to fetch #{url}: #{response.inspect}"
              

                -        end
              

                +        check_url(url)
              

                       end
              

                     end
              

                 
              

                     protected
              

                 
              

                +    def check_url(url)
              

                +      response = faraday.get(url)
              

                +      if response.success?
              

                +        feed = FeedNormalizer::FeedNormalizer.parse(response.body)
              

                +        feed.clean! if boolify(interpolated['clean'])
              

                +        max_events = (interpolated['max_events_per_run'].presence || 0).to_i
              

                +        created_event_count = 0
              

                +        sort_events(feed_to_events(feed)).each.with_index do |event, index|
              

                +          break if max_events && max_events > 0 && index >= max_events
              

                +          entry_id = event.payload[:id]
              

                +          if check_and_track(entry_id)
              

                +            created_event_count += 1
              

                +            create_event(event)
              

                +          end
              

                +        end
              

                +        log "Fetched #{url} and created #{created_event_count} event(s)."
              

                +      else
              

                +        error "Failed to fetch #{url}: #{response.inspect}"
              

                +      end
              

                +    rescue => e
              

                +      error "Failed to fetch #{url} with message '#{e.message}': #{e.backtrace}"
              

                +    end
              

                +
              

                     def get_entry_id(entry)
              

                       entry.id.presence || Digest::MD5.hexdigest(entry.content)
              

                     end
              
@@ -121,5 +131,22 @@ module Agents

                         true
              

                       end
              

                     end
              

                +
              

                +    def feed_to_events(feed)
              

                +      feed.entries.map { |entry|
              

                +        Event.new(payload: {
              

                +                    id: get_entry_id(entry),
              

                +                    date_published: entry.date_published,
              

                +                    last_updated: entry.last_updated,
              

                +                    url: entry.url,
              

                +                    urls: entry.urls,
              

                +                    description: entry.description,
              

                +                    content: entry.content,
              

                +                    title: entry.title,
              

                +                    authors: entry.authors,
              

                +                    categories: entry.categories
              

                +                  })
              

                +      }
              

                +    end
              

                   end
              

                 end
              

app/models/agents/website_agent.rb

@@ -6,6 +6,7 @@ module Agents

                     include WebRequestConcern
              

                 
              

                     can_dry_run!
              

                +    can_order_created_events!
              

                 
              

                     default_schedule "every_12h"
              

                 
              
@@ -86,7 +87,7 @@ module Agents

                 
              

                       Set `uniqueness_look_back` to limit the number of events checked for uniqueness (typically for performance).  This defaults to the larger of #{UNIQUENESS_LOOK_BACK} or #{UNIQUENESS_FACTOR}x the number of detected received results.
              

                 
              

                -      Set `force_encoding` to an encoding name if the website does not return a Content-Type header with a proper charset.
              

                +      Set `force_encoding` to an encoding name if the website is known to respond with a missing, invalid or wrong charset in the Content-Type header.  Note that a text content without a charset is taken as encoded in UTF-8 (not ISO-8859-1).
              

                 
              

                       Set `user_agent` to a custom User-Agent name if the website does not like the default value (`#{default_user_agent}`).
              

                 
              
@@ -105,6 +106,10 @@ module Agents

                           * `status`: HTTP status as integer. (Almost always 200)
              

                 
              

                           * `headers`: Response headers; for example, `{{ _response_.headers.Content-Type }}` expands to the value of the Content-Type header.  Keys are insensitive to cases and -/_.
              

                +
              

                +      # Ordering Events
              

                +
              

                +      #{description_events_order}
              

                     MD
              

                 
              

                     event_description do
              
@@ -152,19 +157,6 @@ module Agents

                         errors.add(:base, "Invalid uniqueness_look_back format") unless is_positive_integer?(options['uniqueness_look_back'])
              

                       end
              

                 
              

                -      if (encoding = options['force_encoding']).present?
              

                -        case encoding
              

                -        when String
              

                -          begin
              

                -            Encoding.find(encoding)
              

                -          rescue ArgumentError
              

                -            errors.add(:base, "Unknown encoding: #{encoding.inspect}")
              

                -          end
              

                -        else
              

                -          errors.add(:base, "force_encoding must be a string")
              

                -        end
              

                -      end
              

                -
              

                       validate_web_request_options!
              

                     end
              

                 
              
@@ -279,12 +271,6 @@ module Agents

                       interpolation_context.stack {
              

                         interpolation_context['_response_'] = ResponseDrop.new(response)
              

                         body = response.body
              

                -        if (encoding = interpolated['force_encoding']).present?
              

                -          body = body.encode(Encoding::UTF_8, encoding)
              

                -        end
              

                -        if interpolated['unzip'] == "gzip"
              

                -          body = ActiveSupport::Gzip.decompress(body)
              

                -        end
              

                         doc = parse(body)
              

                 
              

                         if extract_full_json?
              

app/models/scenario_import.rb

@@ -60,6 +60,7 @@ class ScenarioImport

                     description = parsed_data['description']
              

                     name = parsed_data['name']
              

                     links = parsed_data['links']
              

                +    control_links = parsed_data['control_links'] || []
              

                     tag_fg_color = parsed_data['tag_fg_color']
              

                     tag_bg_color = parsed_data['tag_bg_color']
              

                     source_url = parsed_data['source_url'].presence || nil
              
@@ -87,12 +88,19 @@ class ScenarioImport

                         end
              

                         agent
              

                       end
              

                +
              

                       if success
              

                         links.each do |link|
              

                           receiver = created_agents[link['receiver']]
              

                           source = created_agents[link['source']]
              

                           receiver.sources << source unless receiver.sources.include?(source)
              

                         end
              

                +
              

                +        control_links.each do |control_link|
              

                +          controller = created_agents[control_link['controller']]
              

                +          control_target = created_agents[control_link['control_target']]
              

                +          controller.control_targets << control_target unless controller.control_targets.include?(control_target)
              

                +        end
              

                       end
              

                     end
              

                 
              
@@ -142,7 +150,7 @@ class ScenarioImport

                   def generate_diff
              

                     @agent_diffs = (parsed_data['agents'] || []).map.with_index do |agent_data, index|
              

                       # AgentDiff is defined at the end of this file.
              

                -      agent_diff = AgentDiff.new(agent_data)
              

                +      agent_diff = AgentDiff.new(agent_data, parsed_data['schema_version'])
              

                       if existing_scenario
              

                         # If this Agent exists already, update the AgentDiff with the local version's information.
              

                         agent_diff.diff_with! existing_scenario.agents.find_by(:guid => agent_data['guid'])
              
@@ -184,14 +192,16 @@ class ScenarioImport

                       end
              

                     end
              

                 
              

                -    def initialize(agent_data)
              

                +    def initialize(agent_data, schema_version)
              

                       super()
              

                +      @schema_version = schema_version
              

                       @requires_merge = false
              

                       self.agent = nil
              

                       store! agent_data
              

                     end
              

                 
              

                     BASE_FIELDS = %w[name schedule keep_events_for propagate_immediately disabled guid]
              

                +    FIELDS_REQUIRING_TRANSLATION = %w[keep_events_for]
              

                 
              

                     def agent_exists?
              

                       !!agent
              
@@ -209,10 +219,27 @@ class ScenarioImport

                       self.type = FieldDiff.new(agent_data["type"].split("::").pop)
              

                       self.options = FieldDiff.new(agent_data['options'] || {})
              

                       BASE_FIELDS.each do |option|
              

                -        self[option] = FieldDiff.new(agent_data[option]) if agent_data.has_key?(option)
              

                +        if agent_data.has_key?(option)
              

                +          value = agent_data[option]
              

                +          value = send(:"translate_#{option}", value) if option.in?(FIELDS_REQUIRING_TRANSLATION)
              

                +          self[option] = FieldDiff.new(value)
              

                +        end
              

                       end
              

                     end
              

                 
              

                +    def translate_keep_events_for(old_value)
              

                +      if schema_version < 1
              

                +        # Was stored in days, now is stored in seconds.
              

                +        old_value.to_i.days
              

                +      else
              

                +        old_value
              

                +      end
              

                +    end
              

                +
              

                +    def schema_version
              

                +      (@schema_version || 0).to_i
              

                +    end
              

                +
              

                     def diff_with!(agent)
              

                       return unless agent.present?
              

                 
              
@@ -251,21 +278,6 @@ class ScenarioImport

                       yield 'disabled', disabled, boolean if disabled.requires_merge?
              

                     end
              

                 
              

                -    # Unfortunately Ruby 1.9's OpenStruct doesn't expose [] and []=.
              

                -    unless instance_methods.include?(:[]=)
              

                -      def [](key)
              

                -        self.send(sanitize key)
              

                -      end
              

                -
              

                -      def []=(key, val)
              

                -        self.send("#{sanitize key}=", val)
              

                -      end
              

                -
              

                -      def sanitize(key)
              

                -        key.gsub(/[^a-zA-Z0-9_-]/, '')
              

                -      end
              

                -    end
              

                -
              

                     def agent_instance
              

                       "Agents::#{self.type.updated}".constantize.new
              

                     end
              

app/models/user.rb

@@ -18,7 +18,7 @@ class User < ActiveRecord::Base

                   validates_presence_of :username
              

                   validates_uniqueness_of :username
              

                   validates_format_of :username, :with => /\A[a-zA-Z0-9_-]{3,15}\Z/, :message => "can only contain letters, numbers, underscores, and dashes, and must be between 3 and 15 characters in length."
              

                -  validates_inclusion_of :invitation_code, :on => :create, :in => INVITATION_CODES, :message => "is not valid"
              

                +  validates_inclusion_of :invitation_code, :on => :create, :in => INVITATION_CODES, :message => "is not valid", if: ->{ User.using_invitation_code? }
              

                 
              

                   has_many :user_credentials, :dependent => :destroy, :inverse_of => :user
              

                   has_many :events, -> { order("events.created_at desc") }, :dependent => :delete_all, :inverse_of => :user
              
@@ -40,4 +40,8 @@ class User < ActiveRecord::Base

                       where(conditions).first
              

                     end
              

                   end
              

                +
              

                +  def self.using_invitation_code?
              

                +    ENV['SKIP_INVITATION_CODE'] != 'true'
              

                +  end
              

                 end
              

app/views/agents/_action_menu.html.erb

@@ -7,7 +7,7 @@

                 
              

                   <% if agent.can_dry_run? %>
              

                     <li>
              

                -      <%= link_to icon_tag('glyphicon-refresh') + ' Dry Run', '#', 'data-action-url' => dry_run_agent_path(agent), tabindex: "-1", onclick: "Utils.handleDryRunButton(this, '_method=PUT')" %>
              

                +      <%= link_to icon_tag('glyphicon-refresh') + ' Dry Run', '#', 'data-action-url' => dry_run_agent_path(agent), 'data-with-event-mode' => agent_dry_run_with_event_mode(agent), tabindex: "-1", onclick: "Utils.handleDryRunButton(this)" %>
              

                     </li>
              

                   <% end %>
              

                 
              

app/views/agents/_options.erb

@@ -25,6 +25,6 @@

                 <div class="form-group">
              

                   <%= submit_tag "Save", :class => "btn btn-primary" %>
              

                   <% if agent.can_dry_run? %>
              

                -    <%= button_tag class: 'btn btn-default agent-dry-run-button', type: 'button', 'data-action-url' => agent.persisted? ? dry_run_agent_path(agent) : dry_run_agents_path do %><%= icon_tag('glyphicon-refresh') %> Dry Run<% end %>
              

                +    <%= button_tag class: 'btn btn-default agent-dry-run-button', type: 'button', 'data-action-url' => agent.persisted? ? dry_run_agent_path(agent) : dry_run_agents_path, 'data-with-event-mode' => agent_dry_run_with_event_mode(agent) do %><%= icon_tag('glyphicon-refresh') %> Dry Run<% end %>
              

                   <% end %>
              

                 </div>
              

app/views/devise/registrations/new.html.erb

@@ -30,13 +30,15 @@ bin/setup_heroku

                           </div>
              

                           <% end %>
              

                 
              

                -          <div class="form-group">
              

                -            <%= f.label :invitation_code, class: 'col-md-4 control-label' %>
              

                -            <div class="col-md-6">
              

                -              <%= f.text_field :invitation_code, class: 'form-control' %>
              

                -              <span class="help-inline">We are not yet open to the public.  If you have an invitation code, please enter it here.</span>
              

                +          <% if User.using_invitation_code? %>
              

                +            <div class="form-group">
              

                +              <%= f.label :invitation_code, class: 'col-md-4 control-label' %>
              

                +              <div class="col-md-6">
              

                +                <%= f.text_field :invitation_code, class: 'form-control' %>
              

                +                <span class="help-inline">We are not yet open to the public.  If you have an invitation code, please enter it here.</span>
              

                +              </div>
              

                             </div>
              

                -          </div>
              

                +          <% end %>
              

                 
              

                           <div class="form-group">
              

                             <%= f.label :email, class: 'col-md-4 control-label' %>
              

app/views/scenario_imports/_step_two.html.erb

@@ -14,7 +14,7 @@

                         <span class='glyphicon glyphicon-warning-sign'></span>
              

                         This Scenario already exists in your system. The import will update your existing
              

                         <%= scenario_label(@scenario_import.existing_scenario) %> Scenario's title,
              

                -        description and tag colors. Below you can customize how the individual agents get updated.
              

                +        description, and tag colors. Below you can customize how the individual agents get updated.
              

                       </div>
              

                     <% end %>
              

                 
              

config/initializers/action_mailer.rb

@@ -2,15 +2,5 @@

                 
              

                 smtp_config = YAML::load(ERB.new(File.read(Rails.root.join('config', 'smtp.yml'))).result)
              

                 if smtp_config.keys.include? Rails.env
              

                -  Huginn::Application.config.action_mailer.smtp_settings = smtp_config[Rails.env].symbolize_keys
              

                -end
              

                -
              

                -# Huginn::Application.config.action_mailer.smtp_settings = {
              

                -#   address: ENV['SMTP_SERVER'] || 'smtp.gmail.com',
              

                -#   port: ENV['SMTP_PORT'] || 587,
              

                -#   domain: ENV['SMTP_DOMAIN'],
              

                -#   authentication: ENV['SMTP_AUTHENTICATION'] || 'plain',
              

                -#   enable_starttls_auto: ENV['SMTP_ENABLE_STARTTLS_AUTO'] == 'true' ? true : false,
              

                -#   user_name: ENV['SMTP_USER_NAME'],
              

                -#   password: ENV['SMTP_PASSWORD']
              

                -# }
              

                +  ActionMailer::Base.smtp_settings = smtp_config[Rails.env].symbolize_keys
              

                +end
              

config/initializers/delayed_job.rb

@@ -1,6 +1,6 @@

                 Delayed::Worker.destroy_failed_jobs = false
              

                 Delayed::Worker.max_attempts = 5
              

                -Delayed::Worker.max_run_time = (ENV['DELAYED_JOB_MAX_RUNTIME'].presence || 20).to_i.minutes
              

                +Delayed::Worker.max_run_time = (ENV['DELAYED_JOB_MAX_RUNTIME'].presence || 2).to_i.minutes
              

                 Delayed::Worker.read_ahead = 5
              

                 Delayed::Worker.default_priority = 10
              

                 Delayed::Worker.delay_jobs = !Rails.env.test?
              

config/initializers/liquid.rb

@@ -0,0 +1,8 @@

                +module Liquid
              

                +  # https://github.com/Shopify/liquid/pull/623
              

                +  remove_const :PartialTemplateParser
              

                +  remove_const :TemplateParser
              

                +
              

                +  PartialTemplateParser       = /#{TagStart}.*?#{TagEnd}|#{VariableStart}(?:(?:[^'"{}]+|#{QuotedString})*?|.*?)#{VariableIncompleteEnd}/m
              

                +  TemplateParser              = /(#{PartialTemplateParser}|#{AnyStartingTag})/m
              

                +end
              

config/routes.rb

@@ -2,7 +2,7 @@ Huginn::Application.routes.draw do

                   resources :agents do
              

                     member do
              

                       post :run
              

                -      put :dry_run
              

                +      post :dry_run
              

                       post :handle_details_post
              

                       put :leave_scenario
              

                       delete :remove_events
              

db/migrate/20150507153436_update_keep_events_for_to_be_in_seconds.rb

@@ -0,0 +1,13 @@

                +class UpdateKeepEventsForToBeInSeconds < ActiveRecord::Migration
              

                +  class Agent < ActiveRecord::Base; end
              

                +
              

                +  SECONDS_IN_DAY = 60 * 60 * 24
              

                +
              

                +  def up
              

                +    Agent.update_all ['keep_events_for = keep_events_for * ?', SECONDS_IN_DAY]
              

                +  end
              

                +
              

                +  def down
              

                +    Agent.update_all ['keep_events_for = keep_events_for / ?', SECONDS_IN_DAY]
              

                +  end
              

                +end
              

db/migrate/20150808115436_remove_requirement_from_users_invitation_code.rb

@@ -0,0 +1,5 @@

                +class RemoveRequirementFromUsersInvitationCode < ActiveRecord::Migration
              

                +  def change
              

                +    change_column_null :users, :invitation_code, true, ENV['INVITATION_CODE'].presence || 'try-huginn'
              

                +  end
              

                +end
              

db/schema.rb

@@ -11,39 +11,39 @@

                 #
              

                 # It's strongly recommended that you check this file into your version control system.
              

                 
              

                -ActiveRecord::Schema.define(version: 20140906030139) do
              

                -
              

                -  create_table "agent_logs", force: true do |t|
              

                -    t.integer  "agent_id",                      null: false
              

                -    t.text     "message",                       null: false, charset: "utf8mb4", collation: "utf8mb4_bin"
              

                -    t.integer  "level",             default: 3, null: false
              

                -    t.integer  "inbound_event_id"
              

                -    t.integer  "outbound_event_id"
              

                +ActiveRecord::Schema.define(version: 20150808115436) do
              

                +
              

                +  create_table "agent_logs", force: :cascade do |t|
              

                +    t.integer  "agent_id",          limit: 4,                 null: false
              

                +    t.text     "message",           limit: 65535,             null: false, charset: "utf8mb4", collation: "utf8mb4_bin"
              

                +    t.integer  "level",             limit: 4,     default: 3, null: false
              

                +    t.integer  "inbound_event_id",  limit: 4
              

                +    t.integer  "outbound_event_id", limit: 4
              

                     t.datetime "created_at"
              

                     t.datetime "updated_at"
              

                   end
              

                 
              

                -  create_table "agents", force: true do |t|
              

                -    t.integer  "user_id"
              

                -    t.text     "options",                                                               charset: "utf8mb4", collation: "utf8mb4_bin"
              

                -    t.string   "type",                                                                                      collation: "utf8_bin"
              

                -    t.string   "name",                                                                  charset: "utf8mb4", collation: "utf8mb4_bin"
              

                -    t.string   "schedule",                                                                                  collation: "utf8_bin"
              

                -    t.integer  "events_count",                             default: 0,     null: false
              

                +  create_table "agents", force: :cascade do |t|
              

                +    t.integer  "user_id",               limit: 4
              

                +    t.text     "options",               limit: 65535,                                   charset: "utf8mb4", collation: "utf8mb4_bin"
              

                +    t.string   "type",                  limit: 255,                                                         collation: "utf8_bin"
              

                +    t.string   "name",                  limit: 255,                                     charset: "utf8mb4", collation: "utf8mb4_bin"
              

                +    t.string   "schedule",              limit: 255,                                                         collation: "utf8_bin"
              

                +    t.integer  "events_count",          limit: 4,          default: 0,     null: false
              

                     t.datetime "last_check_at"
              

                     t.datetime "last_receive_at"
              

                -    t.integer  "last_checked_event_id"
              

                +    t.integer  "last_checked_event_id", limit: 4
              

                     t.datetime "created_at"
              

                     t.datetime "updated_at"
              

                -    t.text     "memory",                limit: 2147483647,                              charset: "utf8mb4", collation: "utf8mb4_bin"
              

                +    t.text     "memory",                limit: 4294967295,                              charset: "utf8mb4", collation: "utf8mb4_bin"
              

                     t.datetime "last_web_request_at"
              

                -    t.integer  "keep_events_for",                          default: 0,     null: false
              

                +    t.integer  "keep_events_for",       limit: 4,          default: 0,     null: false
              

                     t.datetime "last_event_at"
              

                     t.datetime "last_error_log_at"
              

                -    t.boolean  "propagate_immediately",                    default: false, null: false
              

                -    t.boolean  "disabled",                                 default: false, null: false
              

                -    t.integer  "service_id"
              

                -    t.string   "guid",                                                     null: false
              

                +    t.boolean  "propagate_immediately", limit: 1,          default: false, null: false
              

                +    t.boolean  "disabled",              limit: 1,          default: false, null: false
              

                +    t.string   "guid",                  limit: 255,                        null: false, charset: "ascii",   collation: "ascii_bin"
              

                +    t.integer  "service_id",            limit: 4
              

                   end
              

                 
              

                   add_index "agents", ["guid"], name: "index_agents_on_guid", using: :btree
              
@@ -51,9 +51,9 @@ ActiveRecord::Schema.define(version: 20140906030139) do

                   add_index "agents", ["type"], name: "index_agents_on_type", using: :btree
              

                   add_index "agents", ["user_id", "created_at"], name: "index_agents_on_user_id_and_created_at", using: :btree
              

                 
              

                -  create_table "control_links", force: true do |t|
              

                -    t.integer  "controller_id",     null: false
              

                -    t.integer  "control_target_id", null: false
              

                +  create_table "control_links", force: :cascade do |t|
              

                +    t.integer  "controller_id",     limit: 4, null: false
              

                +    t.integer  "control_target_id", limit: 4, null: false
              

                     t.datetime "created_at"
              

                     t.datetime "updated_at"
              

                   end
              
@@ -61,25 +61,25 @@ ActiveRecord::Schema.define(version: 20140906030139) do

                   add_index "control_links", ["control_target_id"], name: "index_control_links_on_control_target_id", using: :btree
              

                   add_index "control_links", ["controller_id", "control_target_id"], name: "index_control_links_on_controller_id_and_control_target_id", unique: true, using: :btree
              

                 
              

                -  create_table "delayed_jobs", force: true do |t|
              

                -    t.integer  "priority",                    default: 0
              

                -    t.integer  "attempts",                    default: 0
              

                +  create_table "delayed_jobs", force: :cascade do |t|
              

                +    t.integer  "priority",   limit: 4,        default: 0
              

                +    t.integer  "attempts",   limit: 4,        default: 0
              

                     t.text     "handler",    limit: 16777215,             charset: "utf8mb4", collation: "utf8mb4_bin"
              

                -    t.text     "last_error",                              charset: "utf8mb4", collation: "utf8mb4_bin"
              

                +    t.text     "last_error", limit: 65535,                charset: "utf8mb4", collation: "utf8mb4_bin"
              

                     t.datetime "run_at"
              

                     t.datetime "locked_at"
              

                     t.datetime "failed_at"
              

                -    t.string   "locked_by"
              

                -    t.string   "queue"
              

                +    t.string   "locked_by",  limit: 255
              

                +    t.string   "queue",      limit: 255
              

                     t.datetime "created_at"
              

                     t.datetime "updated_at"
              

                   end
              

                 
              

                   add_index "delayed_jobs", ["priority", "run_at"], name: "delayed_jobs_priority", using: :btree
              

                 
              

                -  create_table "events", force: true do |t|
              

                -    t.integer  "user_id"
              

                -    t.integer  "agent_id"
              

                +  create_table "events", force: :cascade do |t|
              

                +    t.integer  "user_id",    limit: 4
              

                +    t.integer  "agent_id",   limit: 4
              

                     t.decimal  "lat",                         precision: 15, scale: 10
              

                     t.decimal  "lng",                         precision: 15, scale: 10
              

                     t.text     "payload",    limit: 16777215,                           charset: "utf8mb4", collation: "utf8mb4_bin"
              
@@ -92,20 +92,20 @@ ActiveRecord::Schema.define(version: 20140906030139) do

                   add_index "events", ["expires_at"], name: "index_events_on_expires_at", using: :btree
              

                   add_index "events", ["user_id", "created_at"], name: "index_events_on_user_id_and_created_at", using: :btree
              

                 
              

                -  create_table "links", force: true do |t|
              

                -    t.integer  "source_id"
              

                -    t.integer  "receiver_id"
              

                +  create_table "links", force: :cascade do |t|
              

                +    t.integer  "source_id",            limit: 4
              

                +    t.integer  "receiver_id",          limit: 4
              

                     t.datetime "created_at"
              

                     t.datetime "updated_at"
              

                -    t.integer  "event_id_at_creation", default: 0, null: false
              

                +    t.integer  "event_id_at_creation", limit: 4, default: 0, null: false
              

                   end
              

                 
              

                   add_index "links", ["receiver_id", "source_id"], name: "index_links_on_receiver_id_and_source_id", using: :btree
              

                   add_index "links", ["source_id", "receiver_id"], name: "index_links_on_source_id_and_receiver_id", using: :btree
              

                 
              

                -  create_table "scenario_memberships", force: true do |t|
              

                -    t.integer  "agent_id",    null: false
              

                -    t.integer  "scenario_id", null: false
              

                +  create_table "scenario_memberships", force: :cascade do |t|
              

                +    t.integer  "agent_id",    limit: 4, null: false
              

                +    t.integer  "scenario_id", limit: 4, null: false
              

                     t.datetime "created_at"
              

                     t.datetime "updated_at"
              

                   end
              
@@ -113,71 +113,71 @@ ActiveRecord::Schema.define(version: 20140906030139) do

                   add_index "scenario_memberships", ["agent_id"], name: "index_scenario_memberships_on_agent_id", using: :btree
              

                   add_index "scenario_memberships", ["scenario_id"], name: "index_scenario_memberships_on_scenario_id", using: :btree
              

                 
              

                -  create_table "scenarios", force: true do |t|
              

                -    t.string   "name",                         null: false, charset: "utf8mb4", collation: "utf8mb4_bin"
              

                -    t.integer  "user_id",                      null: false
              

                +  create_table "scenarios", force: :cascade do |t|
              

                +    t.string   "name",         limit: 255,                   null: false, charset: "utf8mb4", collation: "utf8mb4_bin"
              

                +    t.integer  "user_id",      limit: 4,                     null: false
              

                     t.datetime "created_at"
              

                     t.datetime "updated_at"
              

                -    t.text     "description",                               charset: "utf8mb4", collation: "utf8mb4_bin"
              

                -    t.boolean  "public",       default: false, null: false
              

                -    t.string   "guid",                         null: false, charset: "ascii",   collation: "ascii_bin"
              

                -    t.string   "source_url"
              

                -    t.string   "tag_bg_color"
              

                -    t.string   "tag_fg_color"
              

                +    t.text     "description",  limit: 65535,                              charset: "utf8mb4", collation: "utf8mb4_bin"
              

                +    t.boolean  "public",       limit: 1,     default: false, null: false
              

                +    t.string   "guid",         limit: 255,                   null: false, charset: "ascii",   collation: "ascii_bin"
              

                +    t.string   "source_url",   limit: 255
              

                +    t.string   "tag_bg_color", limit: 255
              

                +    t.string   "tag_fg_color", limit: 255
              

                   end
              

                 
              

                   add_index "scenarios", ["user_id", "guid"], name: "index_scenarios_on_user_id_and_guid", unique: true, using: :btree
              

                 
              

                -  create_table "services", force: true do |t|
              

                -    t.integer  "user_id",                       null: false
              

                -    t.string   "provider",                      null: false
              

                -    t.string   "name",                          null: false
              

                -    t.text     "token",                         null: false
              

                -    t.text     "secret"
              

                -    t.text     "refresh_token"
              

                +  create_table "services", force: :cascade do |t|
              

                +    t.integer  "user_id",       limit: 4,                     null: false
              

                +    t.string   "provider",      limit: 255,                   null: false, collation: "utf8_general_ci"
              

                +    t.string   "name",          limit: 255,                   null: false, collation: "utf8_general_ci"
              

                +    t.text     "token",         limit: 65535,                 null: false, collation: "utf8_general_ci"
              

                +    t.text     "secret",        limit: 65535,                              collation: "utf8_general_ci"
              

                +    t.text     "refresh_token", limit: 65535,                              collation: "utf8_general_ci"
              

                     t.datetime "expires_at"
              

                -    t.boolean  "global",        default: false
              

                -    t.text     "options"
              

                +    t.boolean  "global",        limit: 1,     default: false
              

                +    t.text     "options",       limit: 65535,                              collation: "utf8_general_ci"
              

                     t.datetime "created_at"
              

                     t.datetime "updated_at"
              

                -    t.string   "uid"
              

                +    t.string   "uid",           limit: 255,                                collation: "utf8_general_ci"
              

                   end
              

                 
              

                   add_index "services", ["provider"], name: "index_services_on_provider", using: :btree
              

                   add_index "services", ["uid"], name: "index_services_on_uid", using: :btree
              

                   add_index "services", ["user_id", "global"], name: "index_services_on_user_id_and_global", using: :btree
              

                 
              

                -  create_table "user_credentials", force: true do |t|
              

                -    t.integer  "user_id",                           null: false
              

                -    t.string   "credential_name",                   null: false
              

                -    t.text     "credential_value",                  null: false
              

                +  create_table "user_credentials", force: :cascade do |t|
              

                +    t.integer  "user_id",          limit: 4,                      null: false
              

                +    t.string   "credential_name",  limit: 255,                    null: false
              

                +    t.text     "credential_value", limit: 65535,                  null: false
              

                     t.datetime "created_at"
              

                     t.datetime "updated_at"
              

                -    t.string   "mode",             default: "text", null: false, collation: "utf8_bin"
              

                +    t.string   "mode",             limit: 255,   default: "text", null: false, collation: "utf8_bin"
              

                   end
              

                 
              

                   add_index "user_credentials", ["user_id", "credential_name"], name: "index_user_credentials_on_user_id_and_credential_name", unique: true, using: :btree
              

                 
              

                -  create_table "users", force: true do |t|
              

                -    t.string   "email",                              default: "",    null: false,                     collation: "utf8_bin"
              

                -    t.string   "encrypted_password",                 default: "",    null: false, charset: "ascii",   collation: "ascii_bin"
              

                -    t.string   "reset_password_token",                                                                collation: "utf8_bin"
              

                +  create_table "users", force: :cascade do |t|
              

                +    t.string   "email",                  limit: 255, default: "",    null: false,                     collation: "utf8_bin"
              

                +    t.string   "encrypted_password",     limit: 255, default: "",    null: false, charset: "ascii",   collation: "ascii_bin"
              

                +    t.string   "reset_password_token",   limit: 255,                                                  collation: "utf8_bin"
              

                     t.datetime "reset_password_sent_at"
              

                     t.datetime "remember_created_at"
              

                -    t.integer  "sign_in_count",                      default: 0
              

                +    t.integer  "sign_in_count",          limit: 4,   default: 0
              

                     t.datetime "current_sign_in_at"
              

                     t.datetime "last_sign_in_at"
              

                -    t.string   "current_sign_in_ip"
              

                -    t.string   "last_sign_in_ip"
              

                +    t.string   "current_sign_in_ip",     limit: 255
              

                +    t.string   "last_sign_in_ip",        limit: 255
              

                     t.datetime "created_at"
              

                     t.datetime "updated_at"
              

                -    t.boolean  "admin",                              default: false, null: false
              

                -    t.integer  "failed_attempts",                    default: 0
              

                -    t.string   "unlock_token"
              

                +    t.boolean  "admin",                  limit: 1,   default: false, null: false
              

                +    t.integer  "failed_attempts",        limit: 4,   default: 0
              

                +    t.string   "unlock_token",           limit: 255
              

                     t.datetime "locked_at"
              

                     t.string   "username",               limit: 191,                 null: false, charset: "utf8mb4", collation: "utf8mb4_unicode_ci"
              

                -    t.string   "invitation_code",                                    null: false,                     collation: "utf8_bin"
              

                -    t.integer  "scenario_count",                     default: 0,     null: false
              

                +    t.string   "invitation_code",        limit: 255,                                                  collation: "utf8_bin"
              

                +    t.integer  "scenario_count",         limit: 4,   default: 0,     null: false
              

                   end
              

                 
              

                   add_index "users", ["email"], name: "index_users_on_email", unique: true, using: :btree
              

deployment/heroku/unicorn.rb

@@ -17,11 +17,6 @@ Thread.new do

                 
              

                     sleep 45
              

                 
              

                -    if ENV['DOMAIN']
              

                -      force_ssl = ENV['FORCE_SSL'] == 'true'
              

                -      Net::HTTP.get_response(URI((force_ssl ? "https://" : "http://") + ENV['DOMAIN']))
              

                -    end
              

                -
              

                     begin
              

                       Process.getpgid worker_pid
              

                     rescue Errno::ESRCH
              

deployment/site-cookbooks/huginn_production/files/default/env.example

@@ -39,6 +39,9 @@ FORCE_SSL=false

                 # You can see its use in user.rb.  PLEASE CHANGE THIS!
              

                 INVITATION_CODE=try-huginn
              

                 
              

                +# If you don't want to require users to have an invitation code, set this to true
              

                +SKIP_INVITATION_CODE=false
              

                +
              

                 #############################
              

                 #    Email Configuration    #
              

                 #############################
              

docker/scripts/init

@@ -6,6 +6,11 @@ cd /app

                 # Default to the environment variable values set in .env.example
              

                 source /app/.env.example
              

                 
              

                +# Cleanup any leftover pid file
              

                +if [ -f /app/tmp/pids/server.pid ]; then
              

                +  rm /app/tmp/pids/server.pid
              

                +fi
              

                +
              

                 # is a mysql or postgresql database linked?
              

                 # requires that the mysql or postgresql containers have exposed
              

                 # port 3306 and 5432 respectively.
              
@@ -177,8 +182,8 @@ if [ -n "\${DO_NOT_RUN_JOBS}" ]; then

                   perl -pi -e 's/^jobs:/#jobs:/' /app/Procfile
              

                 fi
              

                 
              

                -perl -pi -e 's/rails server\$/rails server -b 0.0.0.0 -p \\\$PORT/' /app/Procfile
              

                -export PORT
              

                +perl -pi -e 's/rails server -b0.0.0.0\$/rails server -b 0.0.0.0 -p \\\$PORT/' /app/Procfile
              

                +export PORT="\${PORT:-3000}"
              

                 
              

                 # Start huginn
              

                 exec sudo -u huginn -EH bash -xc 'exec bundle exec foreman start'
              

lib/agents_exporter.rb

@@ -12,6 +12,7 @@ class AgentsExporter

                 
              

                   def as_json(opts = {})
              

                     {
              

                +      :schema_version => 1,
              

                       :name => options[:name].presence || 'No name provided',
              

                       :description => options[:description].presence || 'No description provided',
              

                       :source_url => options[:source_url],
              
@@ -20,7 +21,8 @@ class AgentsExporter

                       :tag_bg_color => options[:tag_bg_color],
              

                       :exported_at => Time.now.utc.iso8601,
              

                       :agents => agents.map { |agent| agent_as_json(agent) },
              

                -      :links => links
              

                +      :links => links,
              

                +      :control_links => control_links
              

                     }
              

                   end
              

                 
              
@@ -32,14 +34,26 @@ class AgentsExporter

                     agent_ids = agents.map(&:id)
              

                 
              

                     contained_links = agents.map.with_index do |agent, index|
              

                -      agent.links_as_source.where(:receiver_id => agent_ids).map do |link|
              

                -        { :source => index, :receiver => agent_ids.index(link.receiver_id) }
              

                +      agent.links_as_source.where(receiver_id: agent_ids).map do |link|
              

                +        { source: index, receiver: agent_ids.index(link.receiver_id) }
              

                       end
              

                     end
              

                 
              

                     contained_links.flatten.compact
              

                   end
              

                 
              

                +  def control_links
              

                +    agent_ids = agents.map(&:id)
              

                +
              

                +    contained_controller_links = agents.map.with_index do |agent, index|
              

                +      agent.control_links_as_controller.where(control_target_id: agent_ids).map do |control_link|
              

                +        { controller: index, control_target: agent_ids.index(control_link.control_target_id) }
              

                +      end
              

                +    end
              

                +
              

                +    contained_controller_links.flatten.compact
              

                +  end
              

                +
              

                   def agent_as_json(agent)
              

                     {
              

                       :type => agent.type,
              

lib/ar_mysql_column_charset.rb

@@ -1,6 +1,3 @@

                -# Module#prepend support for Ruby 1.9
              

                -require 'prepend' unless Module.method_defined?(:prepend)
              

                -
              

                 require 'active_support'
              

                 
              

                 ActiveSupport.on_load :active_record do
              

lib/huginn_scheduler.rb

@@ -114,7 +114,7 @@ class HuginnScheduler

                     end
              

                 
              

                     # Schedule event cleanup.
              

                -    @rufus_scheduler.cron "0 0 * * * " + tzinfo_friendly_timezone do
              

                +    @rufus_scheduler.every ENV['EVENT_EXPIRATION_CHECK'].presence || '6h' do
              

                       cleanup_expired_events!
              

                     end
              

                 
              

lib/prepend.rb

@@ -1,85 +0,0 @@

                -# Fake implementation of prepend(), which does not support overriding
              

                -# inherited methods nor methods that are formerly overridden by
              

                -# another invocation of prepend().
              

                -#
              

                -# Here's what <Original>.prepend(<Wrapper>) does:
              

                -#
              

                -# - Create an anonymous stub module (hereinafter <Stub>) and define
              

                -#   <Stub>#<method> that calls #<method>_without_<Wrapper> for each
              

                -#   instance method of <Wrapper>.
              

                -#
              

                -# - Rename <Original>#<method> to #<method>_without_<Wrapper> for each
              

                -#   instance method of <Wrapper>.
              

                -#
              

                -# - Include <Stub> and <Wrapper> into <Original> in that order.
              

                -#
              

                -# This way, a call of <Original>#<method> is dispatched to
              

                -# <Wrapper><method>, which may call super which is dispatched to
              

                -# <Stub>#<method>, which finally calls
              

                -# <Original>#<method>_without_<Wrapper> which is used to be called
              

                -# <Original>#<method>.
              

                -#
              

                -# Usage:
              

                -#
              

                -#     class Mechanize
              

                -#       # module with methods that overrides those of X
              

                -#       module Y
              

                -#       end
              

                -#
              

                -#       unless X.respond_to?(:prepend, true)
              

                -#         require 'mechanize/prependable'
              

                -#         X.extend(Prependable)
              

                -#       end
              

                -#
              

                -#       class X
              

                -#         prepend Y
              

                -#       end
              

                -#     end
              

                -class Module
              

                -  def prepend(mod)
              

                -    stub = Module.new
              

                -
              

                -    mod_id = (mod.name || 'Module__%d' % mod.object_id).gsub(/::/, '__')
              

                -
              

                -    mod.instance_methods.each { |name|
              

                -      method_defined?(name) or next
              

                -
              

                -      original = instance_method(name)
              

                -      next if original.owner != self
              

                -
              

                -      name = name.to_s
              

                -      name_without = name.sub(/(?=[?!=]?\z)/) { '_without_%s' % mod_id }
              

                -
              

                -      arity = original.arity
              

                -      arglist = (
              

                -        if arity >= 0
              

                -          (1..arity).map { |i| 'x%d' % i }
              

                -        else
              

                -          (1..(-arity - 1)).map { |i| 'x%d' % i } << '*a'
              

                -        end << '&b'
              

                -      ).join(', ')
              

                -
              

                -      if name.end_with?('=')
              

                -        stub.module_eval %{
              

                -          def #{name}(#{arglist})
              

                -            __send__(:#{name_without}, #{arglist})
              

                -          end
              

                -        }
              

                -      else
              

                -        stub.module_eval %{
              

                -          def #{name}(#{arglist})
              

                -            #{name_without}(#{arglist})
              

                -          end
              

                -        }
              

                -      end
              

                -      module_eval {
              

                -        alias_method name_without, name
              

                -        remove_method name
              

                -      }
              

                -    }
              

                -
              

                -    include stub
              

                -    include mod
              

                -  end
              

                -  private :prepend
              

                -end unless Module.method_defined?(:prepend)
              

lib/utils.rb

@@ -79,4 +79,43 @@ module Utils

                   def self.pretty_jsonify(thing)
              

                     JSON.pretty_generate(thing).gsub('</', '<\/')
              

                   end
              

                +
              

                +  class TupleSorter
              

                +    class SortableTuple
              

                +      attr_reader :array
              

                +
              

                +      # The <=> method will call orders[n] to determine if the nth element
              

                +      # should be compared in descending order.
              

                +      def initialize(array, orders = [])
              

                +        @array = array
              

                +        @orders = orders
              

                +      end
              

                +
              

                +      def <=> other
              

                +        other = other.array
              

                +        @array.each_with_index do |e, i|
              

                +          o = other[i]
              

                +          case cmp = e <=> o || e.to_s <=> o.to_s
              

                +          when 0
              

                +            next
              

                +          else
              

                +            return @orders[i] ? -cmp : cmp
              

                +          end
              

                +        end
              

                +        0
              

                +      end
              

                +    end
              

                +
              

                +    class << self
              

                +      def sort!(array, orders = [])
              

                +        array.sort_by! do |e|
              

                +          SortableTuple.new(e, orders)
              

                +        end
              

                +      end
              

                +    end
              

                +  end
              

                +
              

                +  def self.sort_tuples!(array, orders = [])
              

                +    TupleSorter.sort!(array, orders)
              

                +  end
              

                 end
              

spec/concerns/dry_runnable_spec.rb

@@ -7,10 +7,22 @@ describe DryRunnable do

                     can_dry_run!
              

                 
              

                     def check
              

                +      perform
              

                +    end
              

                +
              

                +    def receive(events)
              

                +      events.each do |event|
              

                +        perform(event.payload['prefix'])
              

                +      end
              

                +    end
              

                +
              

                +    private
              

                +
              

                +    def perform(prefix = nil)
              

                       log "Logging"
              

                -      create_event payload: { 'test' => 'foo' }
              

                +      create_event payload: { 'test' => "#{prefix}foo" }
              

                       error "Recording error"
              

                -      create_event payload: { 'test' => 'bar' }
              

                +      create_event payload: { 'test' => "#{prefix}bar" }
              

                       self.memory = { 'last_status' => 'ok', 'dry_run' => dry_run? }
              

                       save!
              

                     end
              
@@ -46,21 +58,6 @@ describe DryRunnable do

                     expect(messages).to eq(['Logging', 'Recording error'])
              

                   end
              

                 
              

                -  it "traps logging, event emission and memory updating, with dry_run? returning true" do
              

                -    results = nil
              

                -
              

                -    expect {
              

                -      results = @agent.dry_run!
              

                -      @agent.reload
              

                -    }.not_to change {
              

                -      [@agent.memory, counts]
              

                -    }
              

                -
              

                -    expect(results[:log]).to match(/\AI, .+ INFO -- : Logging\nE, .+ ERROR -- : Recording error\n/)
              

                -    expect(results[:events]).to eq([{ 'test' => 'foo' }, { 'test' => 'bar' }])
              

                -    expect(results[:memory]).to eq({ 'last_status' => 'ok', 'dry_run' => true })
              

                -  end
              

                -
              

                   it "does not perform dry-run if Agent does not support dry-run" do
              

                     stub(@agent).can_dry_run? { false }
              

                 
              
@@ -77,4 +74,36 @@ describe DryRunnable do

                     expect(results[:events]).to eq([])
              

                     expect(results[:memory]).to eq({})
              

                   end
              

                +
              

                +  describe "dry_run!" do
              

                +    it "traps any destructive operations during a run" do
              

                +      results = nil
              

                +
              

                +      expect {
              

                +        results = @agent.dry_run!
              

                +        @agent.reload
              

                +      }.not_to change {
              

                +        [@agent.memory, counts]
              

                +      }
              

                +
              

                +      expect(results[:log]).to match(/\AI, .+ INFO -- : Logging\nE, .+ ERROR -- : Recording error\n/)
              

                +      expect(results[:events]).to eq([{ 'test' => 'foo' }, { 'test' => 'bar' }])
              

                +      expect(results[:memory]).to eq({ 'last_status' => 'ok', 'dry_run' => true })
              

                +    end
              

                +
              

                +    it "traps any destructive operations during a run when an event is given" do
              

                +      results = nil
              

                +
              

                +      expect {
              

                +        results = @agent.dry_run!(Event.new(payload: { 'prefix' => 'super' }))
              

                +        @agent.reload
              

                +      }.not_to change {
              

                +        [@agent.memory, counts]
              

                +      }
              

                +
              

                +      expect(results[:log]).to match(/\AI, .+ INFO -- : Logging\nE, .+ ERROR -- : Recording error\n/)
              

                +      expect(results[:events]).to eq([{ 'test' => 'superfoo' }, { 'test' => 'superbar' }])
              

                +      expect(results[:memory]).to eq({ 'last_status' => 'ok', 'dry_run' => true })
              

                +    end
              

                +  end
              

                 end
              

spec/concerns/liquid_interpolatable_spec.rb

@@ -38,6 +38,16 @@ describe LiquidInterpolatable::Filters do

                     end
              

                   end
              

                 
              

                +  describe 'unescape' do
              

                +    let(:agent) { Agents::InterpolatableAgent.new(name: "test") }
              

                +
              

                +    it 'should unescape basic HTML entities' do
              

                +      agent.interpolation_context['something'] = ''<foo> & bar''
              

                +      agent.options['cleaned'] = '{{ something | unescape }}'
              

                +      expect(agent.interpolated['cleaned']).to eq("'<foo> & bar'")
              

                +    end
              

                +  end
              

                +
              

                   describe 'to_xpath' do
              

                     before do
              

                       def @filter.to_xpath_roundtrip(string)
              
@@ -177,23 +187,37 @@ describe LiquidInterpolatable::Filters do

                         expect(@agent.interpolated['long_url']).to eq('http://2many.x/6')
              

                       end
              

                     end
              

                -    
              

                -    describe 'regex replace' do
              

                -      let(:agent) { Agents::InterpolatableAgent.new(name: "test") }
              

                +  end
              

                 
              

                -      it 'should replace the first occurrence of a string using regex' do
              

                -        agent.interpolation_context['something'] = 'foobar foobar'
              

                -        agent.options['cleaned'] = '{{ something | regex_replace_first: "\S+bar", "foobaz"  }}'
              

                -        expect(agent.interpolated['cleaned']).to eq('foobaz foobar')
              

                -      end
              

                +  describe 'regex_replace_first' do
              

                +    let(:agent) { Agents::InterpolatableAgent.new(name: "test") }
              

                 
              

                -      it 'should replace the all occurrences of a string using regex' do
              

                -        agent.interpolation_context['something'] = 'foobar foobar'
              

                -        agent.options['cleaned'] = '{{ something | regex_replace: "\S+bar", "foobaz"  }}'
              

                -        expect(agent.interpolated['cleaned']).to eq('foobaz foobaz') 
              

                -      end
              

                -    
              

                +    it 'should replace the first occurrence of a string using regex' do
              

                +      agent.interpolation_context['something'] = 'foobar foobar'
              

                +      agent.options['cleaned'] = '{{ something | regex_replace_first: "\S+bar", "foobaz"  }}'
              

                +      expect(agent.interpolated['cleaned']).to eq('foobaz foobar')
              

                +    end
              

                +
              

                +    it 'should support escaped characters' do
              

                +      agent.interpolation_context['something'] = "foo\\1\n\nfoo\\bar\n\nfoo\\baz"
              

                +      agent.options['test'] = "{{ something | regex_replace_first: '\\\\(\\w{2,})', '\\1\\\\' | regex_replace_first: '\\n+', '\\n'  }}"
              

                +      expect(agent.interpolated['test']).to eq("foo\\1\nfoobar\\\n\nfoo\\baz")
              

                +    end
              

                +  end
              

                +
              

                +  describe 'regex_replace' do
              

                +    let(:agent) { Agents::InterpolatableAgent.new(name: "test") }
              

                +
              

                +    it 'should replace the all occurrences of a string using regex' do
              

                +      agent.interpolation_context['something'] = 'foobar foobar'
              

                +      agent.options['cleaned'] = '{{ something | regex_replace: "\S+bar", "foobaz"  }}'
              

                +      expect(agent.interpolated['cleaned']).to eq('foobaz foobaz')
              

                +    end
              

                +
              

                +    it 'should support escaped characters' do
              

                +      agent.interpolation_context['something'] = "foo\\1\n\nfoo\\bar\n\nfoo\\baz"
              

                +      agent.options['test'] = "{{ something | regex_replace: '\\\\(\\w{2,})', '\\1\\\\' | regex_replace: '\\n+', '\\n'  }}"
              

                +      expect(agent.interpolated['test']).to eq("foo\\1\nfoobar\\\nfoobaz\\")
              

                     end
              

                -    
              

                   end
              

                 end
              

spec/concerns/sortable_events_spec.rb

@@ -0,0 +1,264 @@

                +require 'spec_helper'
              

                +
              

                +describe SortableEvents do
              

                +  let(:agent_class) {
              

                +    Class.new(Agent) do
              

                +      include SortableEvents
              

                +
              

                +      default_schedule 'never'
              

                +
              

                +      def self.valid_type?(name)
              

                +        true
              

                +      end
              

                +    end
              

                +  }
              

                +
              

                +  def new_agent(events_order = nil)
              

                +    options = {}
              

                +    options['events_order'] = events_order if events_order
              

                +    agent_class.new(name: 'test', options: options) { |agent|
              

                +      agent.user = users(:bob)
              

                +    }
              

                +  end
              

                +
              

                +  describe 'validations' do
              

                +    let(:agent_class) {
              

                +      Class.new(Agent) do
              

                +        include SortableEvents
              

                +
              

                +        default_schedule 'never'
              

                +
              

                +        def self.valid_type?(name)
              

                +          true
              

                +        end
              

                +      end
              

                +    }
              

                +
              

                +    def new_agent(events_order = nil)
              

                +      options = {}
              

                +      options['events_order'] = events_order if events_order
              

                +      agent_class.new(name: 'test', options: options) { |agent|
              

                +        agent.user = users(:bob)
              

                +      }
              

                +    end
              

                +
              

                +    it 'should allow events_order to be unspecified, null or an empty array' do
              

                +      expect(new_agent()).to be_valid
              

                +      expect(new_agent(nil)).to be_valid
              

                +      expect(new_agent([])).to be_valid
              

                +    end
              

                +
              

                +    it 'should not allow events_order to be a non-array object' do
              

                +      agent = new_agent(0)
              

                +      expect(agent).not_to be_valid
              

                +      expect(agent.errors[:base]).to include(/events_order/)
              

                +
              

                +      agent = new_agent('')
              

                +      expect(agent).not_to be_valid
              

                +      expect(agent.errors[:base]).to include(/events_order/)
              

                +
              

                +      agent = new_agent({})
              

                +      expect(agent).not_to be_valid
              

                +      expect(agent.errors[:base]).to include(/events_order/)
              

                +    end
              

                +
              

                +    it 'should not allow events_order to be an array containing unexpected objects' do
              

                +      agent = new_agent(['{{key}}', 1])
              

                +      expect(agent).not_to be_valid
              

                +      expect(agent.errors[:base]).to include(/events_order/)
              

                +
              

                +      agent = new_agent(['{{key1}}', ['{{key2}}', 'unknown']])
              

                +      expect(agent).not_to be_valid
              

                +      expect(agent.errors[:base]).to include(/events_order/)
              

                +    end
              

                +
              

                +    it 'should allow events_order to be an array containing strings and valid tuples' do
              

                +      agent = new_agent(['{{key1}}', ['{{key2}}'], ['{{key3}}', 'number']])
              

                +      expect(agent).to be_valid
              

                +
              

                +      agent = new_agent(['{{key1}}', ['{{key2}}'], ['{{key3}}', 'number'], ['{{key4}}', 'time', true]])
              

                +      expect(agent).to be_valid
              

                +    end
              

                +  end
              

                +
              

                +  describe 'sort_events' do
              

                +    let(:payloads) {
              

                +      [
              

                +        { 'title' => 'TitleA', 'score' => 4,  'updated_on' => '7 Jul 2015' },
              

                +        { 'title' => 'TitleB', 'score' => 2,  'updated_on' => '25 Jun 2014' },
              

                +        { 'title' => 'TitleD', 'score' => 10, 'updated_on' => '10 Jan 2015' },
              

                +        { 'title' => 'TitleC', 'score' => 10, 'updated_on' => '9 Feb 2015' },
              

                +      ]
              

                +    }
              

                +
              

                +    let(:events) {
              

                +      payloads.map { |payload| Event.new(payload: payload) }
              

                +    }
              

                +
              

                +    it 'should sort events by a given key' do
              

                +      agent = new_agent(['{{title}}'])
              

                +      expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleA TitleB TitleC TitleD])
              

                +
              

                +      agent = new_agent([['{{title}}', 'string', true]])
              

                +      expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleD TitleC TitleB TitleA])
              

                +    end
              

                +
              

                +    it 'should sort events by multiple keys' do
              

                +      agent = new_agent([['{{score}}', 'number'], '{{title}}'])
              

                +      expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleB TitleA TitleC TitleD])
              

                +
              

                +      agent = new_agent([['{{score}}', 'number'], ['{{title}}', 'string', true]])
              

                +      expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleB TitleA TitleD TitleC])
              

                +    end
              

                +
              

                +    it 'should sort events by time' do
              

                +      agent = new_agent([['{{updated_on}}', 'time']])
              

                +      expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleB TitleD TitleC TitleA])
              

                +    end
              

                +
              

                +    it 'should sort events stably' do
              

                +      agent = new_agent(['<constant>'])
              

                +      expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleA TitleB TitleD TitleC])
              

                +
              

                +      agent = new_agent([['<constant>', 'string', true]])
              

                +      expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleA TitleB TitleD TitleC])
              

                +    end
              

                +
              

                +    it 'should support _index_' do
              

                +      agent = new_agent([['{{_index_}}', 'number', true]])
              

                +      expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleC TitleD TitleB TitleA])
              

                +    end
              

                +  end
              

                +
              

                +  describe 'automatic event sorter' do
              

                +    describe 'declaration' do
              

                +      let(:passive_agent_class) {
              

                +        Class.new(Agent) do
              

                +          include SortableEvents
              

                +
              

                +          cannot_create_events!
              

                +        end
              

                +      }
              

                +
              

                +      let(:active_agent_class) {
              

                +        Class.new(Agent) do
              

                +          include SortableEvents
              

                +        end
              

                +      }
              

                +
              

                +      describe 'can_order_created_events!' do
              

                +        it 'should refuse to work if called from an Agent that cannot create events' do
              

                +          expect {
              

                +            passive_agent_class.class_eval do
              

                +              can_order_created_events!
              

                +            end
              

                +          }.to raise_error
              

                +        end
              

                +
              

                +        it 'should work if called from an Agent that can create events' do
              

                +          expect {
              

                +            active_agent_class.class_eval do
              

                +              can_order_created_events!
              

                +            end
              

                +          }.not_to raise_error
              

                +        end
              

                +      end
              

                +
              

                +      describe 'can_order_created_events?' do
              

                +        it 'should return false unless an Agent declares can_order_created_events!' do
              

                +          expect(active_agent_class.can_order_created_events?).to eq(false)
              

                +          expect(active_agent_class.new.can_order_created_events?).to eq(false)
              

                +        end
              

                +
              

                +        it 'should return true if an Agent declares can_order_created_events!' do
              

                +          active_agent_class.class_eval do
              

                +            can_order_created_events!
              

                +          end
              

                +
              

                +          expect(active_agent_class.can_order_created_events?).to eq(true)
              

                +          expect(active_agent_class.new.can_order_created_events?).to eq(true)
              

                +        end
              

                +      end
              

                +    end
              

                +
              

                +    describe 'behavior' do
              

                +      class Agents::EventOrderableAgent < Agent
              

                +        include SortableEvents
              

                +
              

                +        default_schedule 'never'
              

                +
              

                +        can_order_created_events!
              

                +
              

                +        attr_accessor :payloads_to_emit
              

                +
              

                +        def self.valid_type?(name)
              

                +          true
              

                +        end
              

                +
              

                +        def check
              

                +          payloads_to_emit.each do |payload|
              

                +            create_event payload: payload
              

                +          end
              

                +        end
              

                +
              

                +        def receive(events)
              

                +          events.each do |event|
              

                +            payloads_to_emit.each do |payload|
              

                +              create_event payload: payload.merge('title' => payload['title'] + event.payload['title_suffix'])
              

                +            end
              

                +          end
              

                +        end
              

                +      end
              

                +
              

                +      def new_agent(events_order = nil)
              

                +        options = {}
              

                +        options['events_order'] = events_order if events_order
              

                +        Agents::EventOrderableAgent.new(name: 'test', options: options) { |agent|
              

                +          agent.user = users(:bob)
              

                +          agent.payloads_to_emit = payloads
              

                +        }
              

                +      end
              

                +
              

                +      let(:payloads) {
              

                +        [
              

                +          { 'title' => 'TitleA', 'score' => 4,  'updated_on' => '7 Jul 2015' },
              

                +          { 'title' => 'TitleB', 'score' => 2,  'updated_on' => '25 Jun 2014' },
              

                +          { 'title' => 'TitleD', 'score' => 10, 'updated_on' => '10 Jan 2015' },
              

                +          { 'title' => 'TitleC', 'score' => 10, 'updated_on' => '9 Feb 2015' },
              

                +        ]
              

                +      }
              

                +
              

                +      it 'should keep the order of created events unless events_order is specified' do
              

                +        [[], [nil], [[]]].each do |args|
              

                +          agent = new_agent(*args)
              

                +          agent.save!
              

                +          expect { agent.check }.to change { Event.count }.by(4)
              

                +          events = agent.events.last(4).sort_by(&:id)
              

                +          expect(events.map { |event| event.payload['title'] }).to eq(%w[TitleA TitleB TitleD TitleC])
              

                +        end
              

                +      end
              

                +
              

                +      it 'should sort events created in check() in the order specified in events_order' do
              

                +        agent = new_agent([['{{score}}', 'number'], ['{{title}}', 'string', true]])
              

                +        agent.save!
              

                +        expect { agent.check }.to change { Event.count }.by(4)
              

                +        events = agent.events.last(4).sort_by(&:id)
              

                +        expect(events.map { |event| event.payload['title'] }).to eq(%w[TitleB TitleA TitleD TitleC])
              

                +      end
              

                +
              

                +      it 'should sort events created in receive() in the order specified in events_order' do
              

                +        agent = new_agent([['{{score}}', 'number'], ['{{title}}', 'string', true]])
              

                +        agent.save!
              

                +        expect {
              

                +          agent.receive([Event.new(payload: { 'title_suffix' => ' [new]' }),
              

                +                         Event.new(payload: { 'title_suffix' => ' [popular]' })])
              

                +        }.to change { Event.count }.by(8)
              

                +        events = agent.events.last(8).sort_by(&:id)
              

                +        expect(events.map { |event| event.payload['title'] }).to eq([
              

                +          'TitleB [new]',     'TitleA [new]',     'TitleD [new]',     'TitleC [new]',
              

                +          'TitleB [popular]', 'TitleA [popular]', 'TitleD [popular]', 'TitleC [popular]',
              

                +        ])
              

                +      end
              

                +    end
              

                +  end
              

                +end
              

spec/controllers/agents_controller_spec.rb

@@ -349,6 +349,10 @@ describe AgentsController do

                   end
              

                 
              

                   describe "POST dry_run" do
              

                +    before do
              

                +      stub_request(:any, /xkcd/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/xkcd.html")), status: 200)
              

                +    end
              

                +
              

                     it "does not actually create any agent, event or log" do
              

                       sign_in users(:bob)
              

                       expect {
              
@@ -368,11 +372,24 @@ describe AgentsController do

                       sign_in users(:bob)
              

                       agent = agents(:bob_weather_agent)
              

                       expect {
              

                -        post :dry_run, id: agents(:bob_website_agent), agent: valid_attributes(name: 'New Name')
              

                +        post :dry_run, id: agent, agent: valid_attributes(name: 'New Name')
              

                       }.not_to change {
              

                         [users(:bob).agents.count, users(:bob).events.count, users(:bob).logs.count, agent.name, agent.updated_at]
              

                       }
              

                     end
              

                +
              

                +    it "accepts an event" do
              

                +      sign_in users(:bob)
              

                +      agent = agents(:bob_website_agent)
              

                +      url_from_event = "http://xkcd.com/?from_event=1".freeze
              

                +      expect {
              

                +        post :dry_run, id: agent, event: { url: url_from_event }
              

                +      }.not_to change {
              

                +        [users(:bob).agents.count, users(:bob).events.count, users(:bob).logs.count, agent.name, agent.updated_at]
              

                +      }
              

                +      json = JSON.parse(response.body)
              

                +      expect(json['log']).to match(/^I, .* : Fetching #{Regexp.quote(url_from_event)}$/)
              

                +    end
              

                   end
              

                 
              

                   describe "DELETE memory" do
              

spec/fixtures/agents.yml

@@ -38,7 +38,7 @@ bob_weather_agent:

                   schedule: "midnight"
              

                   name: "SF Weather"
              

                   guid: <%= SecureRandom.hex %>
              

                -  keep_events_for: 45
              

                +  keep_events_for: <%= 45.days %>
              

                   options: <%= { :location => 94102, :lat => 37.779329, :lng => -122.41915, :api_key => 'test' }.to_json.inspect %>
              

                 
              

                 jane_weather_agent:
              
@@ -47,7 +47,7 @@ jane_weather_agent:

                   schedule: "midnight"
              

                   name: "SF Weather"
              

                   guid: <%= SecureRandom.hex %>
              

                -  keep_events_for: 30
              

                +  keep_events_for: <%= 30.days %>
              

                   options: <%= { :location => 94103, :lat => 37.779329, :lng => -122.41915, :api_key => 'test' }.to_json.inspect %>
              

                 
              

                 jane_rain_notifier_agent:
              

spec/lib/agents_exporter_spec.rb

@@ -21,10 +21,12 @@ describe AgentsExporter do

                       expect(data[:description]).to eq(description)
              

                       expect(data[:source_url]).to eq(source_url)
              

                       expect(data[:guid]).to eq(guid)
              

                +      expect(data[:schema_version]).to eq(1)
              

                       expect(data[:tag_fg_color]).to eq(tag_fg_color)
              

                       expect(data[:tag_bg_color]).to eq(tag_bg_color)
              

                       expect(Time.parse(data[:exported_at])).to be_within(2).of(Time.now.utc)
              

                       expect(data[:links]).to eq([{ :source => 0, :receiver => 1 }])
              

                +      expect(data[:control_links]).to eq([])
              

                       expect(data[:agents]).to eq(agent_list.map { |agent| exporter.agent_as_json(agent) })
              

                       expect(data[:agents].all? { |agent_json| agent_json[:guid].present? && agent_json[:type].present? && agent_json[:name].present? }).to be_truthy
              

                 
              
@@ -38,6 +40,13 @@ describe AgentsExporter do

                 
              

                       expect(exporter.as_json[:links]).to eq([{ :source => 0, :receiver => 1 }])
              

                     end
              

                +
              

                +    it "outputs control links to agents within the incoming set, but not outside it" do
              

                +      agents(:jane_rain_notifier_agent).control_targets = [agents(:jane_weather_agent), agents(:jane_basecamp_agent)]
              

                +      agents(:jane_rain_notifier_agent).save!
              

                +
              

                +      expect(exporter.as_json[:control_links]).to eq([{ :controller => 1, :control_target => 0 }])
              

                +    end
              

                   end
              

                 
              

                   describe "#filename" do
              

spec/lib/utils_spec.rb

@@ -114,4 +114,62 @@ describe Utils do

                       expect(cleaned_json).to include("<\\/script>")
              

                     end
              

                   end
              

                +
              

                +  describe "#sort_tuples!" do
              

                +    let(:tuples) {
              

                +      time = Time.now
              

                +      [
              

                +        [2, "a", time - 1],  # 0
              

                +        [2, "b", time - 1],  # 1
              

                +        [1, "b", time - 1],  # 2
              

                +        [1, "b", time],      # 3
              

                +        [1, "a", time],      # 4
              

                +        [2, "a", time + 1],  # 5
              

                +        [2, "a", time],      # 6
              

                +      ]
              

                +    }
              

                +
              

                +    it "sorts tuples like arrays by default" do
              

                +      expected = tuples.values_at(4, 2, 3, 0, 6, 5, 1)
              

                +
              

                +      Utils.sort_tuples!(tuples)
              

                +      expect(tuples).to eq expected
              

                +    end
              

                +
              

                +    it "sorts tuples in order specified: case 1" do
              

                +      # order by x1 asc, x2 desc, c3 asc
              

                +      orders = [false, true, false]
              

                +      expected = tuples.values_at(2, 3, 4, 1, 0, 6, 5)
              

                +
              

                +      Utils.sort_tuples!(tuples, orders)
              

                +      expect(tuples).to eq expected
              

                +    end
              

                +
              

                +    it "sorts tuples in order specified: case 2" do
              

                +      # order by x1 desc, x2 asc, c3 desc
              

                +      orders = [true, false, true]
              

                +      expected = tuples.values_at(5, 6, 0, 1, 4, 3, 2)
              

                +
              

                +      Utils.sort_tuples!(tuples, orders)
              

                +      expect(tuples).to eq expected
              

                +    end
              

                +
              

                +    it "always succeeds in sorting even if it finds pairs of incomparable objects" do
              

                +      time = Time.now
              

                +      tuples = [
              

                +        [2,   "a", time - 1],  # 0
              

                +        [1,   "b", nil],       # 1
              

                +        [1,   "b", time],      # 2
              

                +        ["2", nil, time],      # 3
              

                +        [1,   nil, time],      # 4
              

                +        [nil, "a", time + 1],  # 5
              

                +        [2,   "a", time],      # 6
              

                +      ]
              

                +      orders = [true, false, true]
              

                +      expected = tuples.values_at(3, 6, 0, 4, 2, 1, 5)
              

                +
              

                +      Utils.sort_tuples!(tuples, orders)
              

                +      expect(tuples).to eq expected
              

                +    end
              

                +  end
              

                 end
              

spec/models/agent_spec.rb

@@ -546,11 +546,11 @@ describe Agent do

                         expect(agent).to have(1).errors_on(:keep_events_for)
              

                         agent.keep_events_for = ""
              

                         expect(agent).to have(1).errors_on(:keep_events_for)
              

                -        agent.keep_events_for = 5
              

                +        agent.keep_events_for = 5.days.to_i
              

                         expect(agent).to be_valid
              

                         agent.keep_events_for = 0
              

                         expect(agent).to be_valid
              

                -        agent.keep_events_for = 365
              

                +        agent.keep_events_for = 365.days.to_i
              

                         expect(agent).to be_valid
              

                 
              

                         # Rails seems to call to_i on the input. This guards against future changes to that behavior.
              
@@ -564,7 +564,7 @@ describe Agent do

                         @time = "2014-01-01 01:00:00 +00:00"
              

                         time_travel_to @time do
              

                           @agent = Agents::SomethingSource.new(:name => "something")
              

                -          @agent.keep_events_for = 5
              

                +          @agent.keep_events_for = 5.days
              

                           @agent.user = users(:bob)
              

                           @agent.save!
              

                           @event = @agent.create_event :payload => { "hello" => "world" }
              
@@ -580,7 +580,7 @@ describe Agent do

                           @agent.save!
              

                 
              

                           @agent.options[:foo] = "bar1"
              

                -          @agent.keep_events_for = 5
              

                +          @agent.keep_events_for = 5.days
              

                           @agent.save!
              

                         end
              

                       end
              
@@ -590,7 +590,7 @@ describe Agent do

                           time_travel_to @time do
              

                             expect {
              

                                 @agent.options[:foo] = "bar1"
              

                -                @agent.keep_events_for = 3
              

                +                @agent.keep_events_for = 3.days
              

                                 @agent.save!
              

                             }.to change { @event.reload.expires_at }
              

                             expect(@event.expires_at.to_i).to be_within(2).of(3.days.from_now.to_i)
              
@@ -603,7 +603,7 @@ describe Agent do

                 
              

                           expect {
              

                             @agent.options[:foo] = "bar2"
              

                -            @agent.keep_events_for = 3
              

                +            @agent.keep_events_for = 3.days
              

                             @agent.save!
              

                           }.to change { @event.reload.expires_at }
              

                           expect(@event.expires_at.to_i).to be_within(60 * 61).of(1.days.from_now.to_i) # The larger time is to deal with daylight savings
              
@@ -635,7 +635,7 @@ describe Agent do

                         @receiver = Agents::CannotBeScheduled.new(
              

                           name: 'Agent',
              

                           options: { foo: 'bar3' },
              

                -          keep_events_for: 3,
              

                +          keep_events_for: 3.days,
              

                           propagate_immediately: true)
              

                         @receiver.user = users(:bob)
              

                         @receiver.sources << @sender
              
@@ -747,7 +747,7 @@ describe Agent do

                 
              

                       it "sets expires_at on created events" do
              

                         event = agents(:jane_weather_agent).create_event :payload => { 'hi' => 'there' }
              

                -        expect(event.expires_at.to_i).to be_within(5).of(agents(:jane_weather_agent).keep_events_for.days.from_now.to_i)
              

                +        expect(event.expires_at.to_i).to be_within(5).of(agents(:jane_weather_agent).keep_events_for.seconds.from_now.to_i)
              

                       end
              

                     end
              

                 
              
@@ -836,7 +836,7 @@ describe AgentDrop do

                         },
              

                       },
              

                       schedule: 'every_1h',
              

                -      keep_events_for: 2)
              

                +      keep_events_for: 2.days)
              

                     @wsa1.user = users(:bob)
              

                     @wsa1.save!
              

                 
              
@@ -853,7 +853,7 @@ describe AgentDrop do

                         },
              

                       },
              

                       schedule: 'every_12h',
              

                -      keep_events_for: 2)
              

                +      keep_events_for: 2.days)
              

                     @wsa2.user = users(:bob)
              

                     @wsa2.save!
              

                 
              
@@ -868,7 +868,7 @@ describe AgentDrop do

                         matchers: [],
              

                         skip_created_at: 'false',
              

                       },
              

                -      keep_events_for: 2,
              

                +      keep_events_for: 2.days,
              

                       propagate_immediately: true)
              

                     @efa.user = users(:bob)
              

                     @efa.sources << @wsa1 << @wsa2
              

spec/models/agents/data_output_agent_spec.rb

@@ -209,6 +209,22 @@ describe Agents::DataOutputAgent do

                         })
              

                       end
              

                 
              

                +      describe 'ordering' do
              

                +        before do
              

                +          agent.options['events_order'] = ['{{title}}']
              

                +        end
              

                +
              

                +        it 'can reorder the events_to_show last events based on a Liquid expression' do
              

                +          asc_content, _status, _content_type = agent.receive_web_request({ 'secret' => 'secret2' }, 'get', 'application/json')
              

                +          expect(asc_content['items'].map {|i| i["title"] }).to eq(["Evolving", "Evolving again", "Evolving yet again with a past date"])
              

                +
              

                +          agent.options['events_order'] = [['{{title}}', 'string', true]]
              

                +
              

                +          desc_content, _status, _content_type = agent.receive_web_request({ 'secret' => 'secret2' }, 'get', 'application/json')
              

                +          expect(desc_content['items']).to eq(asc_content['items'].reverse)
              

                +        end
              

                +      end
              

                +
              

                       describe "interpolating \"events\"" do
              

                         before do
              

                           agent.options['template']['title'] = "XKCD comics as a feed{% if events.first.site_title %} ({{events.first.site_title}}){% endif %}"
              

spec/models/agents/ftpsite_agent_spec.rb

@@ -9,7 +9,7 @@ describe Agents::FtpsiteAgent do

                         'url' => "ftp://ftp.example.org/pub/releases/",
              

                         'patterns' => ["example*.tar.gz"],
              

                       }
              

                -      @checker = Agents::FtpsiteAgent.new(:name => "Example", :options => @site, :keep_events_for => 2)
              

                +      @checker = Agents::FtpsiteAgent.new(:name => "Example", :options => @site, :keep_events_for => 2.days)
              

                       @checker.user = users(:bob)
              

                       @checker.save!
              

                     end
              

spec/models/agents/imap_folder_agent_spec.rb

@@ -14,7 +14,7 @@ describe Agents::ImapFolderAgent do

                         'conditions' => {
              

                         }
              

                       }
              

                -      @checker = Agents::ImapFolderAgent.new(:name => 'Example', :options => @site, :keep_events_for => 2)
              

                +      @checker = Agents::ImapFolderAgent.new(:name => 'Example', :options => @site, :keep_events_for => 2.days)
              

                       @checker.user = users(:bob)
              

                       @checker.save!
              

                 
              

spec/models/agents/rss_agent_spec.rb

@@ -66,6 +66,21 @@ describe Agents::RssAgent do

                       expect(last.payload['urls']).to eq(["https://github.com/cantino/huginn/commit/d465158f77dcd9078697e6167b50abbfdfa8b1af"])
              

                     end
              

                 
              

                +    it "should emit items as events in the order specified in the events_order option" do
              

                +      expect {
              

                +        agent.options['events_order'] = ['{{title | replace_regex: "^[[:space:]]+", "" }}']
              

                +        agent.check
              

                +      }.to change { agent.events.count }.by(20)
              

                +
              

                +      first, *, last = agent.events.last(20)
              

                +      expect(first.payload['title'].strip).to eq('upgrade rails and gems')
              

                +      expect(first.payload['url']).to eq("https://github.com/cantino/huginn/commit/87a7abda23a82305d7050ac0bb400ce36c863d01")
              

                +      expect(first.payload['urls']).to eq(["https://github.com/cantino/huginn/commit/87a7abda23a82305d7050ac0bb400ce36c863d01"])
              

                +      expect(last.payload['title'].strip).to eq('Dashed line in a diagram indicates propagate_immediately being false.')
              

                +      expect(last.payload['url']).to eq("https://github.com/cantino/huginn/commit/0e80f5341587aace2c023b06eb9265b776ac4535")
              

                +      expect(last.payload['urls']).to eq(["https://github.com/cantino/huginn/commit/0e80f5341587aace2c023b06eb9265b776ac4535"])
              

                +    end
              

                +
              

                     it "should track ids and not re-emit the same item when seen again" do
              

                       agent.check
              

                       expect(agent.memory['seen_ids']).to eq(agent.events.map {|e| e.payload['id'] })
              
@@ -133,4 +148,14 @@ describe Agents::RssAgent do

                       expect(agent.memory['seen_ids']).to eq(agent.events.map {|e| Digest::MD5.hexdigest(e.payload['content']) })
              

                     end
              

                   end
              

                +
              

                +  describe 'logging errors with the feed url' do
              

                +    it 'includes the feed URL when an exception is raised' do
              

                +      mock(FeedNormalizer::FeedNormalizer).parse(anything) { raise StandardError.new("Some error!") }
              

                +      expect(lambda {
              

                +        agent.check
              

                +      }).not_to raise_error
              

                +      expect(agent.logs.last.message).to match(%r[Failed to fetch https://github.com])
              

                +    end
              

                +  end
              

                 end
              

spec/models/agents/website_agent_spec.rb

@@ -20,7 +20,7 @@ describe Agents::WebsiteAgent do

                           'hovertext' => { 'css' => "#comic img", 'value' => "@title" }
              

                         }
              

                       }
              

                -      @checker = Agents::WebsiteAgent.new(:name => "xkcd", :options => @valid_options, :keep_events_for => 2)
              

                +      @checker = Agents::WebsiteAgent.new(:name => "xkcd", :options => @valid_options, :keep_events_for => 2.days)
              

                       @checker.user = users(:bob)
              

                       @checker.save!
              

                     end
              
@@ -386,7 +386,7 @@ describe Agents::WebsiteAgent do

                               'url' => { 'xpath' => '/feed/entry', 'value' => './link[1]/@href' },
              

                               'thumbnail' => { 'xpath' => '/feed/entry', 'value' => './thumbnail/@url' },
              

                             }
              

                -          }, keep_events_for: 2)
              

                +          }, keep_events_for: 2.days)
              

                           @checker.user = users(:bob)
              

                           @checker.save!
              

                         end
              

spec/models/scenario_import_spec.rb

@@ -30,7 +30,7 @@ describe ScenarioImport do

                       :type => "Agents::WeatherAgent",
              

                       :name => "a weather agent",
              

                       :schedule => "5pm",
              

                -      :keep_events_for => 14,
              

                +      :keep_events_for => 14.days,
              

                       :disabled => true,
              

                       :guid => "a-weather-agent",
              

                       :options => weather_agent_options
              
@@ -61,6 +61,7 @@ describe ScenarioImport do

                   end
              

                   let(:valid_parsed_data) do
              

                     {
              

                +      :schema_version => 1,
              

                       :name => name,
              

                       :description => description,
              

                       :guid => guid,
              
@@ -74,7 +75,8 @@ describe ScenarioImport do

                       ],
              

                       :links => [
              

                         { :source => 0, :receiver => 1 }
              

                -      ]
              

                +      ],
              

                +      :control_links => []
              

                     }
              

                   end
              

                   let(:valid_data) { valid_parsed_data.to_json }
              
@@ -203,7 +205,7 @@ describe ScenarioImport do

                 
              

                           expect(weather_agent.name).to eq("a weather agent")
              

                           expect(weather_agent.schedule).to eq("5pm")
              

                -          expect(weather_agent.keep_events_for).to eq(14)
              

                +          expect(weather_agent.keep_events_for).to eq(14.days)
              

                           expect(weather_agent.propagate_immediately).to be_falsey
              

                           expect(weather_agent).to be_disabled
              

                           expect(weather_agent.memory).to be_empty
              
@@ -226,6 +228,55 @@ describe ScenarioImport do

                             scenario_import.import
              

                           }.to change { users(:bob).agents.count }.by(2)
              

                         end
              

                +
              

                +        context "when the schema_version is less than 1" do
              

                +          before do
              

                +            valid_parsed_weather_agent_data[:keep_events_for] = 2
              

                +            valid_parsed_data.delete(:schema_version)
              

                +          end
              

                +
              

                +          it "translates keep_events_for from days to seconds" do
              

                +            scenario_import.import
              

                +            expect(scenario_import.errors).to be_empty
              

                +            weather_agent = scenario_import.scenario.agents.find_by(:guid => "a-weather-agent")
              

                +            trigger_agent = scenario_import.scenario.agents.find_by(:guid => "a-trigger-agent")
              

                +
              

                +            expect(weather_agent.keep_events_for).to eq(2.days)
              

                +            expect(trigger_agent.keep_events_for).to eq(0)
              

                +          end
              

                +        end
              

                +
              

                +        describe "with control links" do
              

                +          it 'creates the links' do
              

                +            valid_parsed_data[:control_links] = [
              

                +              { :controller => 1, :control_target => 0 }
              

                +            ]
              

                +
              

                +            expect {
              

                +              scenario_import.import
              

                +            }.to change { users(:bob).agents.count }.by(2)
              

                +
              

                +            weather_agent = scenario_import.scenario.agents.find_by(:guid => "a-weather-agent")
              

                +            trigger_agent = scenario_import.scenario.agents.find_by(:guid => "a-trigger-agent")
              

                +
              

                +            expect(trigger_agent.sources).to eq([weather_agent])
              

                +            expect(weather_agent.controllers.to_a).to eq([trigger_agent])
              

                +            expect(trigger_agent.control_targets.to_a).to eq([weather_agent])
              

                +          end
              

                +
              

                +          it "doesn't crash without any control links" do
              

                +            valid_parsed_data.delete(:control_links)
              

                +
              

                +            expect {
              

                +              scenario_import.import
              

                +            }.to change { users(:bob).agents.count }.by(2)
              

                +
              

                +            weather_agent = scenario_import.scenario.agents.find_by(:guid => "a-weather-agent")
              

                +            trigger_agent = scenario_import.scenario.agents.find_by(:guid => "a-trigger-agent")
              

                +
              

                +            expect(trigger_agent.sources).to eq([weather_agent])
              

                +          end
              

                +        end
              

                       end
              

                 
              

                       describe "#generate_diff" do
              
@@ -309,7 +360,7 @@ describe ScenarioImport do

                 
              

                           expect(weather_agent.name).to eq("a weather agent")
              

                           expect(weather_agent.schedule).to eq("5pm")
              

                -          expect(weather_agent.keep_events_for).to eq(14)
              

                +          expect(weather_agent.keep_events_for).to eq(14.days)
              

                           expect(weather_agent.propagate_immediately).to be_falsey
              

                           expect(weather_agent).to be_disabled
              

                           expect(weather_agent.memory).to be_empty
              
@@ -330,7 +381,7 @@ describe ScenarioImport do

                             "0" => {
              

                               "name" => "updated name",
              

                               "schedule" => "6pm",
              

                -              "keep_events_for" => "2",
              

                +              "keep_events_for" => 2.days.to_i.to_s,
              

                               "disabled" => "false",
              

                               "options" => weather_agent_options.merge("api_key" => "foo").to_json
              

                             }
              
@@ -343,7 +394,7 @@ describe ScenarioImport do

                           weather_agent = existing_scenario.agents.find_by(:guid => "a-weather-agent")
              

                           expect(weather_agent.name).to eq("updated name")
              

                           expect(weather_agent.schedule).to eq("6pm")
              

                -          expect(weather_agent.keep_events_for).to eq(2)
              

                +          expect(weather_agent.keep_events_for).to eq(2.days.to_i)
              

                           expect(weather_agent).not_to be_disabled
              

                           expect(weather_agent.options).to eq(weather_agent_options.merge("api_key" => "foo"))
              

                         end
              
@@ -353,7 +404,7 @@ describe ScenarioImport do

                             "0" => {
              

                               "name" => "",
              

                               "schedule" => "foo",
              

                -              "keep_events_for" => "2",
              

                +              "keep_events_for" => 2.days.to_i.to_s,
              

                               "options" => weather_agent_options.merge("api_key" => "").to_json
              

                             }
              

                           }
              
@@ -386,12 +437,40 @@ describe ScenarioImport do

                           end
              

                         end
              

                 
              

                +        context "when the schema_version is less than 1" do
              

                +          it "translates keep_events_for from days to seconds" do
              

                +            valid_parsed_data.delete(:schema_version)
              

                +            valid_parsed_data[:agents] = [valid_parsed_weather_agent_data.merge(keep_events_for: 5)]
              

                +
              

                +            scenario_import.merges = {
              

                +              "0" => {
              

                +                "name" => "a new name",
              

                +                "schedule" => "6pm",
              

                +                "keep_events_for" => 2.days.to_i.to_s,
              

                +                "disabled" => "true",
              

                +                "options" => weather_agent_options.merge("api_key" => "foo").to_json
              

                +              }
              

                +            }
              

                +
              

                +            expect(scenario_import).to be_valid
              

                +
              

                +            weather_agent_diff = scenario_import.agent_diffs[0]
              

                +
              

                +            expect(weather_agent_diff.name.current).to eq(agents(:bob_weather_agent).name)
              

                +            expect(weather_agent_diff.name.incoming).to eq('a weather agent')
              

                +            expect(weather_agent_diff.name.updated).to eq('a new name')
              

                +            expect(weather_agent_diff.keep_events_for.current).to eq(45.days.to_i)
              

                +            expect(weather_agent_diff.keep_events_for.incoming).to eq(5.days.to_i)
              

                +            expect(weather_agent_diff.keep_events_for.updated).to eq(2.days.to_i.to_s)
              

                +          end
              

                +        end
              

                +
              

                         it "sets the 'updated' FieldDiff values based on any feedback from the user" do
              

                           scenario_import.merges = {
              

                             "0" => {
              

                               "name" => "a new name",
              

                               "schedule" => "6pm",
              

                -              "keep_events_for" => "2",
              

                +              "keep_events_for" => 2.days.to_s,
              

                               "disabled" => "true",
              

                               "options" => weather_agent_options.merge("api_key" => "foo").to_json
              

                             },
              
@@ -411,7 +490,8 @@ describe ScenarioImport do

                           expect(weather_agent_diff.name.updated).to eq("a new name")
              

                 
              

                           expect(weather_agent_diff.schedule.updated).to eq("6pm")
              

                -          expect(weather_agent_diff.keep_events_for.updated).to eq("2")
              

                +          expect(weather_agent_diff.keep_events_for.current).to eq(45.days)
              

                +          expect(weather_agent_diff.keep_events_for.updated).to eq(2.days.to_s)
              

                           expect(weather_agent_diff.disabled.updated).to eq("true")
              

                           expect(weather_agent_diff.options.updated).to eq(weather_agent_options.merge("api_key" => "foo"))
              

                         end
              

spec/models/users_spec.rb

@@ -3,15 +3,33 @@ require 'spec_helper'

                 describe User do
              

                   describe "validations" do
              

                     describe "invitation_code" do
              

                -      it "only accepts valid invitation codes" do
              

                -        User::INVITATION_CODES.each do |v|
              

                -          is_expected.to allow_value(v).for(:invitation_code)
              

                +      context "when configured to use invitation codes" do
              

                +        before do
              

                +          stub(User).using_invitation_code? {true}
              

                +        end
              

                +        
              

                +        it "only accepts valid invitation codes" do
              

                +          User::INVITATION_CODES.each do |v|
              

                +            is_expected.to allow_value(v).for(:invitation_code)
              

                +          end
              

                +        end
              

                +  
              

                +        it "can reject invalid invitation codes" do
              

                +          %w['foo', 'bar'].each do |v|
              

                +            is_expected.not_to allow_value(v).for(:invitation_code)
              

                +          end
              

                         end
              

                       end
              

                -
              

                -      it "can reject invalid invitation codes" do
              

                -        %w['foo', 'bar'].each do |v|
              

                -          is_expected.not_to allow_value(v).for(:invitation_code)
              

                +      
              

                +      context "when configured not to use invitation codes" do
              

                +        before do
              

                +          stub(User).using_invitation_code? {false}
              

                +        end
              

                +        
              

                +        it "skips this validation" do
              

                +          %w['foo', 'bar', nil, ''].each do |v|
              

                +            is_expected.to allow_value(v).for(:invitation_code)
              

                +          end
              

                         end
              

                       end
              

                     end
              

spec/support/shared_examples/web_request_concern.rb

@@ -141,5 +141,38 @@ shared_examples_for WebRequestConcern do

                       agent.options['disable_ssl_verification'] = false
              

                       expect(agent.faraday.ssl.verify).to eq(true)
              

                     end
              

                +
              

                +    it "should use faradays default params_encoder" do
              

                +      expect(agent.faraday.options.params_encoder).to eq(nil)
              

                +      agent.options['disable_url_encoding'] = 'false'
              

                +      expect(agent.faraday.options.params_encoder).to eq(nil)
              

                +      agent.options['disable_url_encoding'] = false
              

                +      expect(agent.faraday.options.params_encoder).to eq(nil)
              

                +    end
              

                +
              

                +    it "should use WebRequestConcern::DoNotEncoder when disable_url_encoding is truthy" do
              

                +      agent.options['disable_url_encoding'] = true
              

                +      expect(agent.faraday.options.params_encoder).to eq(WebRequestConcern::DoNotEncoder)
              

                +      agent.options['disable_url_encoding'] = 'true'
              

                +      expect(agent.faraday.options.params_encoder).to eq(WebRequestConcern::DoNotEncoder)
              

                +    end
              

                +  end
              

                +
              

                +  describe WebRequestConcern::DoNotEncoder do
              

                +    it "should not encode special characters" do
              

                +      expect(WebRequestConcern::DoNotEncoder.encode('GetRss?CategoryNr=39207' => 'test')).to eq('GetRss?CategoryNr=39207=test')
              

                +    end
              

                +
              

                +    it "should work without a value present" do
              

                +      expect(WebRequestConcern::DoNotEncoder.encode('GetRss?CategoryNr=39207' => nil)).to eq('GetRss?CategoryNr=39207')
              

                +    end
              

                +
              

                +    it "should work without an empty value" do
              

                +      expect(WebRequestConcern::DoNotEncoder.encode('GetRss?CategoryNr=39207' => '')).to eq('GetRss?CategoryNr=39207=')
              

                +    end
              

                +
              

                +    it "should return the value when decoding" do
              

                +      expect(WebRequestConcern::DoNotEncoder.decode('val')).to eq(['val'])
              

                +    end
              

                   end
              

                 end
              

spec/support/vcr_support.rb

@@ -2,7 +2,7 @@ require 'vcr'

                 
              

                 VCR.configure do |c|
              

                   c.cassette_library_dir = 'spec/cassettes'
              

                -  c.allow_http_connections_when_no_cassette = true
              

                +  c.allow_http_connections_when_no_cassette = false
              

                   c.hook_into :webmock
              

                   c.default_cassette_options = { record: :new_episodes}
              

                   c.configure_rspec_metadata!