Home Erkunden Hilfe
Registrieren Anmelden
J1X
/
huginn
Beobachten 2
Markieren 0
Fork 0

Quellcode durchsuchen
Merge pull request #1301 from kreuzwerker/feature/file-handling

Introduce concept to handle files

Dominik Sander vor 9 Jahren
Ursprung
bf7c2feba4 9d1bfeb778
Commit
9a588e080b
20 geänderte Dateien mit 1799 neuen Zeilen und 31 gelöschten Zeilen
Diff-Statistik anzeigen
  1. 6 1
      Gemfile
  2. 5 0
      Gemfile.lock
  3. 58 0
      app/concerns/file_handling.rb
  4. 5 1
      app/concerns/working_helpers.rb
  5. 195 0
      app/models/agents/csv_agent.rb
  6. 96 28
      app/models/agents/ftpsite_agent.rb
  7. 190 0
      app/models/agents/local_file_agent.rb
  8. 50 0
      app/models/agents/read_file_agent.rb
  9. 206 0
      app/models/agents/s3_agent.rb
  10. 15 0
      db/migrate/20160224120316_add_mode_option_to_ftpsite_agents.rb
  11. 1 0
      lib/agent_runner.rb
  12. 1 1
      lib/utils.rb
  13. 1 0
      spec/env.test
  14. 244 0
      spec/models/agents/csv_agent_spec.rb
  15. 143 0
      spec/models/agents/ftpsite_agent_spec.rb
  16. 276 0
      spec/models/agents/local_file_agent_spec.rb
  17. 47 0
      spec/models/agents/read_file_agent_spec.rb
  18. 220 0
      spec/models/agents/s3_agent_spec.rb
  19. 16 0
      spec/support/shared_examples/file_handling_consumer.rb
  20. 24 0
      spec/support/shared_examples/working_helpers.rb

+ 6 - 1
Gemfile
Datei anzeigen 

@@ -63,6 +63,12 @@ gem 'haversine'
63 63 
 gem 'omniauth-evernote'
64 64 
 gem 'evernote_oauth'
65 65
66 
+# LocalFileAgent (watch functionality)
67 
+gem 'listen', '~> 3.0.5', require: false
68 
+
69 
+# S3Agent
70 
+gem 'aws-sdk-core', '~> 2.2.15'
71 
+
66 72 
 # Optional Services.
67 73 
 gem 'omniauth-37signals'          # BasecampAgent
68 74 
 gem 'omniauth-wunderlist', github: 'wunderlist/omniauth-wunderlist', ref: 'd0910d0396107b9302aa1bc50e74bb140990ccb8'
 
@@ -75,7 +81,6 @@ unless Gem::Version.new(Bundler::VERSION) >= Gem::Version.new('1.5.0')
75 81 
 end
76 82
77 83 
 gem 'protected_attributes', '~>1.0.8' # This must be loaded before some other gems, like delayed_job.
78 
-
79 84 
 gem 'ace-rails-ap', '~> 2.0.1'
80 85 
 gem 'bootstrap-kaminari-views', '~> 0.0.3'
81 86 
 gem 'bundler', '>= 1.5.0'

+ 5 - 0
Gemfile.lock
Datei anzeigen 

@@ -110,6 +110,8 @@ GEM
110 110 
       addressable (>= 2.3.1)
111 111 
       extlib (>= 0.9.15)
112 112 
       multi_json (>= 1.0.0)
113 
+    aws-sdk-core (2.2.15)
114 
+      jmespath (~> 1.0)
113 115 
     bcrypt (3.1.10)
114 116 
     better_errors (1.1.0)
115 117 
       coderay (>= 1.0.0)
 
@@ -279,6 +281,7 @@ GEM
279 281 
     hypdf (1.0.7)
280 282 
       httmultiparty (= 0.3.10)
281 283 
     i18n (0.7.0)
284 
+    jmespath (1.1.3)
282 285 
     jquery-rails (3.1.3)
283 286 
       railties (>= 3.0, < 5.0)
284 287 
       thor (>= 0.14, < 2.0)
 
@@ -577,6 +580,7 @@ PLATFORMS
577 580
578 581 
 DEPENDENCIES
579 582 
   ace-rails-ap (~> 2.0.1)
583 
+  aws-sdk-core (~> 2.2.15)
580 584 
   better_errors (~> 1.1)
581 585 
   binding_of_caller
582 586 
   bootstrap-kaminari-views (~> 0.0.3)
 
@@ -621,6 +625,7 @@ DEPENDENCIES
621 625 
   kramdown (~> 1.3.3)
622 626 
   letter_opener_web
623 627 
   liquid (~> 3.0.3)
628 
+  listen (~> 3.0.5)
624 629 
   mini_magick
625 630 
   mqtt
626 631 
   multi_xml

+ 58 - 0
app/concerns/file_handling.rb
Datei anzeigen 

@@ -0,0 +1,58 @@
1 
+module FileHandling
2 
+  extend ActiveSupport::Concern
3 
+
4 
+  def get_file_pointer(file)
5 
+    { file_pointer: { file: file, agent_id: id } }
6 
+  end
7 
+
8 
+  def get_io(event)
9 
+    return nil unless event.payload['file_pointer'] &&
10 
+                      event.payload['file_pointer']['file'] &&
11 
+                      event.payload['file_pointer']['agent_id']
12 
+    event.user.agents.find(event.payload['file_pointer']['agent_id']).get_io(event.payload['file_pointer']['file'])
13 
+  end
14 
+
15 
+  def emitting_file_handling_agent_description
16 
+    @emitting_file_handling_agent_description ||=
17 
+      "This agent only emits a 'file pointer', not the data inside the files, the following agents can consume the created events: `#{receiving_file_handling_agents.join('`, `')}`. Read more about the concept in the [wiki](https://github.com/cantino/huginn/wiki/How-Huginn-works-with-files)."
18 
+  end
19 
+
20 
+  def receiving_file_handling_agent_description
21 
+    @receiving_file_handling_agent_description ||=
22 
+      "This agent can consume a 'file pointer' event from the following agents with no additional configuration: `#{emitting_file_handling_agents.join('`, `')}`. Read more about the concept in the [wiki](https://github.com/cantino/huginn/wiki/How-Huginn-works-with-files)."
23 
+  end
24 
+
25 
+  private
26 
+
27 
+  def emitting_file_handling_agents
28 
+    emitting_file_handling_agents = file_handling_agents.select { |a| a.emits_file_pointer? }
29 
+    emitting_file_handling_agents.map { |a| a.to_s.demodulize }
30 
+  end
31 
+
32 
+  def receiving_file_handling_agents
33 
+    receiving_file_handling_agents = file_handling_agents.select { |a| a.consumes_file_pointer? }
34 
+    receiving_file_handling_agents.map { |a| a.to_s.demodulize }
35 
+  end
36 
+
37 
+  def file_handling_agents
38 
+    @file_handling_agents ||= Agent.types.select{ |c| c.included_modules.include?(FileHandling) }.map { |d| d.name.constantize }
39 
+  end
40 
+
41 
+  module ClassMethods
42 
+    def emits_file_pointer!
43 
+      @emits_file_pointer = true
44 
+    end
45 
+
46 
+    def emits_file_pointer?
47 
+      !!@emits_file_pointer
48 
+    end
49 
+
50 
+    def consumes_file_pointer!
51 
+      @consumes_file_pointer = true
52 
+    end
53 
+
54 
+    def consumes_file_pointer?
55 
+      !!@consumes_file_pointer
56 
+    end
57 
+  end
58 
+end

+ 5 - 1
app/concerns/working_helpers.rb
Datei anzeigen 

@@ -12,4 +12,8 @@ module WorkingHelpers
12 12 
   def received_event_without_error?
13 13 
     (last_receive_at.present? && last_error_log_at.blank?) || (last_receive_at.present? && last_error_log_at.present? && last_receive_at > last_error_log_at)
14 14 
   end
15 
-end
15 
+
16 
+  def checked_without_error?
17 
+    (last_check_at.present? && last_error_log_at.nil?) || (last_check_at.present? && last_error_log_at.present? && last_check_at > last_error_log_at)
18 
+  end
19 
+end

+ 195 - 0
app/models/agents/csv_agent.rb
Datei anzeigen 

@@ -0,0 +1,195 @@
1 
+module Agents
2 
+  class CsvAgent < Agent
3 
+    include FormConfigurable
4 
+    include FileHandling
5 
+
6 
+    cannot_be_scheduled!
7 
+    consumes_file_pointer!
8 
+
9 
+    def default_options
10 
+      {
11 
+        'mode' => 'parse',
12 
+        'separator' => ',',
13 
+        'use_fields' => '',
14 
+        'output' => 'event_per_row',
15 
+        'with_header' => 'true',
16 
+        'data_path' => '$.data',
17 
+        'data_key' => 'data'
18 
+      }
19 
+    end
20 
+
21 
+    description do
22 
+      <<-MD
23 
+        The `CsvAgent` parses or serializes CSV data. When parsing, events can either be emitted for the entire CSV, or one per row.
24 
+
25 
+        Set `mode` to `parse` to parse CSV from incoming event, when set to `serialize` the agent serilizes the data of events to CSV.
26 
+
27 
+        ### Universal options
28 
+
29 
+        Specify the `separator` which is used to seperate the fields from each other (default is `,`).
30 
+
31 
+        `data_key` sets the key which contains the serialized CSV or parsed CSV data in emitted events.
32 
+
33 
+        ### Parsing
34 
+
35 
+        If `use_fields` is set to a comma seperated string and the CSV file contains field headers the agent will only extract the specified fields.
36 
+
37 
+        `output` determines wheather one event per row is emitted or one event that includes all the rows.
38 
+
39 
+        Set `with_header` to `true` if first line of the CSV file are field names.
40 
+
41 
+        #{receiving_file_handling_agent_description}
42 
+
43 
+        When receiving the CSV data in a regular event use [JSONPath](http://goessner.net/articles/JsonPath/) to select the path in `data_path`. `data_path` is only used when the received event does not contain a 'file pointer'.
44 
+
45 
+        ### Serializing
46 
+
47 
+        If `use_fields` is set to a comma seperated string and the first received event has a object at the specified `data_path` the generated CSV will only include the given fields.
48 
+
49 
+        Set `with_header` to `true` to include a field header in the CSV.
50 
+
51 
+        Use [JSONPath](http://goessner.net/articles/JsonPath/) in `data_path` to select with part of the received events should be serialized.
52 
+      MD
53 
+    end
54 
+
55 
+    event_description do
56 
+      "Events will looks like this:\n\n    %s" % if interpolated['mode'] == 'parse'
57 
+        rows = if boolify(interpolated['with_header'])
58 
+          [{'column' => 'row1 value1', 'column2' => 'row1 value2'}, {'column' => 'row2 value3', 'column2' => 'row2 value4'}]
59 
+        else
60 
+          [['row1 value1', 'row1 value2'], ['row2 value1', 'row2 value2']]
61 
+        end
62 
+        if interpolated['output'] == 'event_per_row'
63 
+          Utils.pretty_print(interpolated['data_key'] => rows[0])
64 
+        else
65 
+          Utils.pretty_print(interpolated['data_key'] => rows)
66 
+        end
67 
+      else
68 
+        Utils.pretty_print(interpolated['data_key'] => '"generated","csv","data"' + "\n" + '"column1","column2","column3"')
69 
+      end
70 
+    end
71 
+
72 
+    form_configurable :mode, type: :array, values: %w(parse serialize)
73 
+    form_configurable :separator, type: :string
74 
+    form_configurable :data_key, type: :string
75 
+    form_configurable :with_header, type: :boolean
76 
+    form_configurable :use_fields, type: :string
77 
+    form_configurable :output, type: :array, values: %w(event_per_row event_per_file)
78 
+    form_configurable :data_path, type: :string
79 
+
80 
+    def validate_options
81 
+      if options['with_header'].blank? || ![true, false].include?(boolify(options['with_header']))
82 
+        errors.add(:base, "The 'with_header' options is required and must be set to 'true' or 'false'")
83 
+      end
84 
+      if options['mode'] == 'serialize' && options['data_path'].blank?
85 
+        errors.add(:base, "When mode is set to serialize data_path has to be present.")
86 
+      end
87 
+    end
88 
+
89 
+    def working?
90 
+      received_event_without_error?
91 
+    end
92 
+
93 
+    def receive(incoming_events)
94 
+      case options['mode']
95 
+      when 'parse'
96 
+        parse(incoming_events)
97 
+      when 'serialize'
98 
+        serialize(incoming_events)
99 
+      end
100 
+    end
101 
+
102 
+    private
103 
+    def serialize(incoming_events)
104 
+      mo = interpolated(incoming_events.first)
105 
+      rows = rows_from_events(incoming_events, mo)
106 
+      csv = CSV.generate(col_sep: separator(mo), force_quotes: true ) do |csv|
107 
+        if boolify(mo['with_header']) && rows.first.is_a?(Hash)
108 
+          if mo['use_fields'].present?
109 
+            csv << extract_options(mo)
110 
+          else
111 
+            csv << rows.first.keys
112 
+          end
113 
+        end
114 
+        rows.each do |data|
115 
+          if data.is_a?(Hash)
116 
+            if mo['use_fields'].present?
117 
+              csv << data.extract!(*extract_options(mo)).values
118 
+            else
119 
+              csv << data.values
120 
+            end
121 
+          else
122 
+            csv << data
123 
+          end
124 
+        end
125 
+      end
126 
+      create_event payload: { mo['data_key'] => csv }
127 
+    end
128 
+
129 
+    def rows_from_events(incoming_events, mo)
130 
+      [].tap do |rows|
131 
+        incoming_events.each do |event|
132 
+          data = Utils.value_at(event.payload, mo['data_path'])
133 
+          if data.is_a?(Array) && (data[0].is_a?(Array) || data[0].is_a?(Hash))
134 
+            data.each { |row| rows << row }
135 
+          else
136 
+            rows << data
137 
+          end
138 
+        end
139 
+      end
140 
+    end
141 
+
142 
+    def parse(incoming_events)
143 
+      incoming_events.each do |event|
144 
+        mo = interpolated(event)
145 
+        next unless io = local_get_io(event)
146 
+        if mo['output'] == 'event_per_row'
147 
+          parse_csv(io, mo) do |payload|
148 
+            create_event payload: { mo['data_key'] => payload }
149 
+          end
150 
+        else
151 
+          create_event payload: { mo['data_key'] => parse_csv(io, mo, []) }
152 
+        end
153 
+      end
154 
+    end
155 
+
156 
+    def local_get_io(event)
157 
+      if io = get_io(event)
158 
+        io
159 
+      else
160 
+        Utils.value_at(event.payload, interpolated['data_path'])
161 
+      end
162 
+    end
163 
+
164 
+    def parse_csv(io, mo, array = nil)
165 
+      CSV.new(io, col_sep: separator(mo), headers: boolify(mo['with_header'])).each do |row|
166 
+        if block_given?
167 
+          yield get_payload(row, mo)
168 
+        else
169 
+          array << get_payload(row, mo)
170 
+        end
171 
+      end
172 
+      array
173 
+    end
174 
+
175 
+    def separator(mo)
176 
+      mo['separator'] == '\\t' ? "\t" : mo['separator']
177 
+    end
178 
+
179 
+    def get_payload(row, mo)
180 
+      if boolify(mo['with_header'])
181 
+        if mo['use_fields'].present?
182 
+          row.to_hash.extract!(*extract_options(mo))
183 
+        else
184 
+          row.to_hash
185 
+        end
186 
+      else
187 
+        row
188 
+      end
189 
+    end
190 
+
191 
+    def extract_options(mo)
192 
+      mo['use_fields'].split(',').map(&:strip)
193 
+    end
194 
+  end
195 
+end

+ 96 - 28
app/models/agents/ftpsite_agent.rb
Datei anzeigen 

@@ -3,23 +3,42 @@ require 'time'
3 3
4 4 
 module Agents
5 5 
   class FtpsiteAgent < Agent
6 
-    cannot_receive_events!
6 
+    include FileHandling
7 7 
     default_schedule "every_12h"
8 8
9 9 
     gem_dependency_check { defined?(Net::FTP) && defined?(Net::FTP::List) }
10 10
11 
-    description <<-MD
12 
-      The FTP Site Agent checks an FTP site and creates Events based on newly uploaded files in a directory.
11 
+    emits_file_pointer!
13 12
14 
-      #{'## Include `net-ftp-list` in your Gemfile to use this Agent!' if dependencies_missing?}
13 
+    description do
14 
+      <<-MD
15 
+        The Ftp Site Agent checks an FTP site and creates Events based on newly uploaded files in a directory. When receiving events it creates files on the configured FTP server.
15 16
17 
+        #{'## Include `net-ftp-list` in your Gemfile to use this Agent!' if dependencies_missing?}
16 18
17 
-      Specify a `url` that represents a directory of an FTP site to watch, and a list of `patterns` to match against file names.
19 
+        `mode` must be present and either `read` or `write`, in `read` mode the agent checks the FTP site for changed files, with `write` it writes received events to a file on the server.
18 20
19 
-      Login credentials can be included in `url` if authentication is required.
21 
+        ### Universal options
20 22
21 
-      Only files with a last modification time later than the `after` value, if specifed, are notified.
22 
-    MD
23 
+        Specify a `url` that represents a directory of an FTP site to watch, and a list of `patterns` to match against file names.
24 
+
25 
+        Login credentials can be included in `url` if authentication is required: `ftp://username:password@ftp.example.com/path`. Liquid formatting is supported as well: `ftp://{% credential ftp_credentials %}@ftp.example.com/`
26 
+
27 
+        Optionally specify the encoding of the files you want to read/write in `force_encoding`, by default UTF-8 is used.
28 
+
29 
+        ### Reading
30 
+
31 
+        Only files with a last modification time later than the `after` value, if specifed, are emitted as event.
32 
+
33 
+        ### Writing
34 
+
35 
+        Specify the filename to use in `filename`, Liquid interpolation is possible to change the name per event.
36 
+
37 
+        Use [Liquid](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) templating in `data` to specify which part of the received event should be written.
38 
+
39 
+        #{emitting_file_handling_agent_description}
40 
+      MD
41 
+    end
23 42
24 43 
     event_description <<-MD
25 44 
       Events look like this:
 
@@ -32,42 +51,67 @@ module Agents
32 51 
     MD
33 52
34 53 
     def working?
35 
-      event_created_within?(interpolated['expected_update_period_in_days']) && !recent_error_logs?
54 
+      if interpolated['mode'] == 'read'
55 
+        event_created_within?(interpolated['expected_update_period_in_days']) && !recent_error_logs?
56 
+      else
57 
+        received_event_without_error?
58 
+      end
36 59 
     end
37 60
38 61 
     def default_options
39 62 
       {
63 
+        'mode' => 'read',
40 64 
         'expected_update_period_in_days' => "1",
41 65 
         'url' => "ftp://example.org/pub/releases/",
42 66 
         'patterns' => [
43 67 
           'foo-*.tar.gz',
44 68 
         ],
45 69 
         'after' => Time.now.iso8601,
70 
+        'force_encoding' => '',
71 
+        'filename' => '',
72 
+        'data' => '{{ data }}'
46 73 
       }
47 74 
     end
48 75
49 76 
     def validate_options
50 77 
       # Check for required fields
51 78 
       begin
52 
-        url = options['url']
53 
-        String === url or raise
54 
-        uri = URI(url)
55 
-        URI::FTP === uri or raise
56 
-        errors.add(:base, "url must end with a slash") unless uri.path.end_with?('/')
79 
+        if !options['url'].include?('{{')
80 
+          url = interpolated['url']
81 
+          String === url or raise
82 
+          uri = URI(url)
83 
+          URI::FTP === uri or raise
84 
+          errors.add(:base, "url must end with a slash") if uri.path.present? && !uri.path.end_with?('/')
85 
+        end
57 86 
       rescue
58 87 
         errors.add(:base, "url must be a valid FTP URL")
59 88 
       end
60 89
61 
-      patterns = options['patterns']
62 
-      case patterns
63 
-      when Array
64 
-        if patterns.empty?
65 
-          errors.add(:base, "patterns must not be empty")
90 
+      options['mode'] = 'read' if options['mode'].blank? && new_record?
91 
+      if options['mode'].blank? || !['read', 'write'].include?(options['mode'])
92 
+        errors.add(:base, "The 'mode' option is required and must be set to 'read' or 'write'")
93 
+      end
94 
+
95 
+      case interpolated['mode']
96 
+      when 'read'
97 
+        patterns = options['patterns']
98 
+        case patterns
99 
+        when Array
100 
+          if patterns.empty?
101 
+            errors.add(:base, "patterns must not be empty")
102 
+          end
103 
+        when nil, ''
104 
+          errors.add(:base, "patterns must be specified")
105 
+        else
106 
+          errors.add(:base, "patterns must be an array")
107 
+        end
108 
+      when 'write'
109 
+        if options['filename'].blank?
110 
+          errors.add(:base, "filename must be specified in 'write' mode")
111 
+        end
112 
+        if options['data'].blank?
113 
+          errors.add(:base, "data must be specified in 'write' mode")
66 114 
         end
67 
-      when nil, ''
68 
-        errors.add(:base, "patterns must be specified")
69 
-      else
70 
-        errors.add(:base, "patterns must be an array")
71 115 
       end
72 116
73 117 
       # Check for optional fields
 
@@ -85,6 +129,7 @@ module Agents
85 129 
     end
86 130
87 131 
     def check
132 
+      return if interpolated['mode'] != 'read'
88 133 
       saving_entries do |found|
89 134 
         each_entry { |filename, mtime|
90 135 
           found[filename, mtime]
 
@@ -92,6 +137,17 @@ module Agents
92 137 
       end
93 138 
     end
94 139
140 
+    def receive(incoming_events)
141 
+      return if interpolated['mode'] != 'write'
142 
+      incoming_events.each do |event|
143 
+        mo = interpolated(event)
144 
+        mo['data'].encode!(interpolated['force_encoding'], invalid: :replace, undef: :replace) if interpolated['force_encoding'].present?
145 
+        open_ftp(base_uri) do |ftp|
146 
+          ftp.storbinary("STOR #{mo['filename']}", StringIO.new(mo['data']), Net::FTP::DEFAULT_BLOCKSIZE)
147 
+        end
148 
+      end
149 
+    end
150 
+
95 151 
     def each_entry
96 152 
       patterns = interpolated['patterns']
97 153 
 
@@ -147,9 +203,10 @@ module Agents
147 203
148 204 
       ftp.passive = true
149 205
150 
-      path = uri.path.chomp('/')
151 
-      log "Changing directory to #{path}"
152 
-      ftp.chdir(path)
206 
+      if (path = uri.path.chomp('/')).present?
207 
+        log "Changing directory to #{path}"
208 
+        ftp.chdir(path)
209 
+      end
153 210
154 211 
       yield ftp
155 212 
     ensure
 
@@ -176,17 +233,28 @@ module Agents
176 233 
       new_files.sort_by { |filename|
177 234 
         found_entries[filename]
178 235 
       }.each { |filename|
179 
-        create_event payload: {
236 
+        create_event payload: get_file_pointer(filename).merge({
180 237 
           'url' => (base_uri + uri_path_escape(filename)).to_s,
181 238 
           'filename' => filename,
182 239 
           'timestamp' => found_entries[filename],
183 
-        }
240 
+        })
184 241 
       }
185 242
186 243 
       memory['known_entries'] = found_entries
187 244 
       save!
188 245 
     end
189 246
247 
+    def get_io(file)
248 
+      data = StringIO.new
249 
+      open_ftp(base_uri) do |ftp|
250 
+        ftp.getbinaryfile(file, nil) do |chunk|
251 
+          data.write chunk.force_encoding(options['force_encoding'].presence || 'UTF-8')
252 
+        end
253 
+      end
254 
+      data.rewind
255 
+      data
256 
+    end
257 
+
190 258 
     private
191 259
192 260 
     def is_positive_integer?(value)

+ 190 - 0
app/models/agents/local_file_agent.rb
Datei anzeigen 

@@ -0,0 +1,190 @@
1 
+module Agents
2 
+  class LocalFileAgent < Agent
3 
+    include LongRunnable
4 
+    include FormConfigurable
5 
+    include FileHandling
6 
+
7 
+    emits_file_pointer!
8 
+
9 
+    default_schedule 'every_1h'
10 
+
11 
+    def self.should_run?
12 
+      ENV['ENABLE_INSECURE_AGENTS'] == "true"
13 
+    end
14 
+
15 
+    description do
16 
+      <<-MD
17 
+        The LocalFileAgent can watch a file/directory for changes or emit an event for every file in that directory. When receiving an event it writes the received data into a file.
18 
+
19 
+        `mode` determines if the agent is emitting events for (changed) files or writing received event data to disk.
20 
+
21 
+        ### Reading
22 
+
23 
+        When `watch` is set to `true` the LocalFileAgent will watch the specified `path` for changes, the schedule is ignored and the file system is watched continuously. An event will be emitted for every detected change.
24 
+
25 
+        When `watch` is set to `false` the agent will emit an event for every file in the directory on each scheduled run.
26 
+
27 
+        #{emitting_file_handling_agent_description}
28 
+
29 
+        ### Writing
30 
+
31 
+        Every event will be writting into a file at `path`, Liquid interpolation is possible to change the path per event.
32 
+
33 
+        When `append` is true the received data will be appended to the file.
34 
+
35 
+        Use [Liquid](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) templating in `data` to specify which part of the received event should be written.
36 
+
37 
+        *Warning*: This type of Agent can read and write any file the user that runs the Huginn server has access to, and is #{Agents::LocalFileAgent.should_run? ? "**currently enabled**" : "**currently disabled**"}.
38 
+        Only enable this Agent if you trust everyone using your Huginn installation.
39 
+        You can enable this Agent in your .env file by setting `ENABLE_INSECURE_AGENTS` to `true`.
40 
+      MD
41 
+    end
42 
+
43 
+    event_description do
44 
+      "Events will looks like this:\n\n    %s" % if boolify(interpolated['watch'])
45 
+        Utils.pretty_print(
46 
+          "file_pointer" => {
47 
+            "file" => "/tmp/test/filename",
48 
+            "agent_id" => id
49 
+          },
50 
+          "event_type" => "modified/added/removed"
51 
+        )
52 
+      else
53 
+        Utils.pretty_print(
54 
+          "file_pointer" => {
55 
+            "file" => "/tmp/test/filename",
56 
+            "agent_id" => id
57 
+          }
58 
+        )
59 
+      end
60 
+    end
61 
+
62 
+    def default_options
63 
+      {
64 
+        'mode' => 'read',
65 
+        'watch' => 'true',
66 
+        'append' => 'false',
67 
+        'path' => "",
68 
+        'data' => '{{ data }}'
69 
+      }
70 
+    end
71 
+
72 
+    form_configurable :mode, type: :array, values: %w(read write)
73 
+    form_configurable :watch, type: :array, values: %w(true false)
74 
+    form_configurable :path, type: :string
75 
+    form_configurable :append, type: :boolean
76 
+    form_configurable :data, type: :string
77 
+
78 
+    def validate_options
79 
+      if options['mode'].blank? || !['read', 'write'].include?(options['mode'])
80 
+        errors.add(:base, "The 'mode' option is required and must be set to 'read' or 'write'")
81 
+      end
82 
+      if options['watch'].blank? || ![true, false].include?(boolify(options['watch']))
83 
+        errors.add(:base, "The 'watch' option is required and must be set to 'true' or 'false'")
84 
+      end
85 
+      if options['append'].blank? || ![true, false].include?(boolify(options['append']))
86 
+        errors.add(:base, "The 'append' option is required and must be set to 'true' or 'false'")
87 
+      end
88 
+      if options['path'].blank?
89 
+        errors.add(:base, "The 'path' option is required.")
90 
+      end
91 
+    end
92 
+
93 
+    def working?
94 
+      should_run?(false) && ((interpolated['mode'] == 'read' && check_path_existance && checked_without_error?) ||
95 
+                             (interpolated['mode'] == 'write' && received_event_without_error?))
96 
+    end
97 
+
98 
+    def check
99 
+      return if interpolated['mode'] != 'read' || boolify(interpolated['watch']) || !should_run?
100 
+      return unless check_path_existance(true)
101 
+      if File.directory?(expanded_path)
102 
+        Dir.glob(File.join(expanded_path, '*')).select { |f| File.file?(f) }
103 
+      else
104 
+        [expanded_path]
105 
+      end.each do |file|
106 
+        create_event payload: get_file_pointer(file)
107 
+      end
108 
+    end
109 
+
110 
+    def receive(incoming_events)
111 
+      return if interpolated['mode'] != 'write' || !should_run?
112 
+      incoming_events.each do |event|
113 
+        mo = interpolated(event)
114 
+        File.open(File.expand_path(mo['path']), boolify(mo['append']) ? 'a' : 'w') do |file|
115 
+          file.write(mo['data'])
116 
+        end
117 
+      end
118 
+    end
119 
+
120 
+    def start_worker?
121 
+      interpolated['mode'] == 'read' && boolify(interpolated['watch']) && should_run? && check_path_existance
122 
+    end
123 
+
124 
+    def check_path_existance(log = true)
125 
+      if !File.exist?(expanded_path)
126 
+        error("File or directory '#{expanded_path}' does not exist") if log
127 
+        return false
128 
+      end
129 
+      true
130 
+    end
131 
+
132 
+    def get_io(file)
133 
+      File.open(file, 'r')
134 
+    end
135 
+
136 
+    def expanded_path
137 
+      @expanded_path ||= File.expand_path(interpolated['path'])
138 
+    end
139 
+
140 
+    private
141 
+
142 
+    def should_run?(log = true)
143 
+      if self.class.should_run?
144 
+        true
145 
+      else
146 
+        error("Unable to run because insecure agents are not enabled. Set ENABLE_INSECURE_AGENTS to true in the Huginn .env configuration.") if log
147 
+        false
148 
+      end
149 
+    end
150 
+
151 
+    class Worker < LongRunnable::Worker
152 
+      def setup
153 
+        require 'listen'
154 
+        @listener = Listen.to(*listen_options, &method(:callback))
155 
+      end
156 
+
157 
+      def run
158 
+        sleep unless agent.check_path_existance(true)
159 
+
160 
+        @listener.start
161 
+        sleep
162 
+      end
163 
+
164 
+      def stop
165 
+        @listener.stop
166 
+      end
167 
+
168 
+      private
169 
+
170 
+      def callback(*changes)
171 
+        AgentRunner.with_connection do
172 
+          changes.zip([:modified, :added, :removed]).each do |files, event_type|
173 
+            files.each do |file|
174 
+              agent.create_event payload: agent.get_file_pointer(file).merge(event_type: event_type)
175 
+            end
176 
+          end
177 
+          agent.touch(:last_check_at)
178 
+        end
179 
+      end
180 
+
181 
+      def listen_options
182 
+        if File.directory?(agent.expanded_path)
183 
+          [agent.expanded_path, ignore!: [] ]
184 
+        else
185 
+          [File.dirname(agent.expanded_path), { ignore!: [], only: /\A#{Regexp.escape(File.basename(agent.expanded_path))}\z/ } ]
186 
+        end
187 
+      end
188 
+    end
189 
+  end
190 
+end

+ 50 - 0
app/models/agents/read_file_agent.rb
Datei anzeigen 

@@ -0,0 +1,50 @@
1 
+module Agents
2 
+  class ReadFileAgent < Agent
3 
+    include FormConfigurable
4 
+    include FileHandling
5 
+
6 
+    cannot_be_scheduled!
7 
+    consumes_file_pointer!
8 
+
9 
+    def default_options
10 
+      {
11 
+        'data_key' => 'data'
12 
+      }
13 
+    end
14 
+
15 
+    description do
16 
+      <<-MD
17 
+        The ReadFileAgent takes events from `FileHandling` agents, reads the file, and emits the contents as a string.
18 
+
19 
+        `data_key` specifies the key of the emitted event which contains the file contents.
20 
+
21 
+        #{receiving_file_handling_agent_description}
22 
+      MD
23 
+    end
24 
+
25 
+    event_description <<-MD
26 
+      {
27 
+        "data" => '...'
28 
+      }
29 
+    MD
30 
+
31 
+    form_configurable :data_key, type: :string
32 
+
33 
+    def validate_options
34 
+      if options['data_key'].blank?
35 
+        errors.add(:base, "The 'data_key' options is required.")
36 
+      end
37 
+    end
38 
+
39 
+    def working?
40 
+      received_event_without_error?
41 
+    end
42 
+
43 
+    def receive(incoming_events)
44 
+      incoming_events.each do |event|
45 
+        next unless io = get_io(event)
46 
+        create_event payload: { interpolated['data_key'] => io.read }
47 
+      end
48 
+    end
49 
+  end
50 
+end

+ 206 - 0
app/models/agents/s3_agent.rb
Datei anzeigen 

@@ -0,0 +1,206 @@
1 
+module Agents
2 
+  class S3Agent < Agent
3 
+    include FormConfigurable
4 
+    include FileHandling
5 
+
6 
+    emits_file_pointer!
7 
+    no_bulk_receive!
8 
+
9 
+    default_schedule 'every_1h'
10 
+
11 
+    gem_dependency_check { defined?(Aws::S3) }
12 
+
13 
+    description do
14 
+      <<-MD
15 
+        The S3Agent can watch a bucket for changes or emit an event for every file in that bucket. When receiving events, it writes the data into a file on S3.
16 
+
17 
+        #{'## Include `aws-sdk-core` in your Gemfile to use this Agent!' if dependencies_missing?}
18 
+
19 
+        `mode` must be present and either `read` or `write`, in `read` mode the agent checks the S3 bucket for changed files, with `write` it writes received events to a file in the bucket.
20 
+
21 
+        ### Universal options
22 
+
23 
+        To use credentials for the `access_key` and `access_key_secret` use the liquid `credential` tag like so `{% credential name-of-credential %}`
24 
+
25 
+        Select the `region` in which the bucket was created.
26 
+
27 
+        ### Reading
28 
+
29 
+        When `watch` is set to `true` the S3Agent will watch the specified `bucket` for changes. An event will be emitted for every detected change.
30 
+
31 
+        When `watch` is set to `false` the agent will emit an event for every file in the bucket on each sheduled run.
32 
+
33 
+        #{emitting_file_handling_agent_description}
34 
+
35 
+        ### Writing
36 
+
37 
+        Specify the filename to use in `filename`, Liquid interpolation is possible to change the name per event.
38 
+
39 
+        Use [Liquid](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) templating in `data` to specify which part of the received event should be written.
40 
+      MD
41 
+    end
42 
+
43 
+    event_description do
44 
+      "Events will looks like this:\n\n    %s" % if boolify(interpolated['watch'])
45 
+        Utils.pretty_print({
46 
+          "file_pointer" => {
47 
+            "file" => "filename",
48 
+            "agent_id" => id
49 
+          },
50 
+          "event_type" => "modified/added/removed"
51 
+        })
52 
+      else
53 
+        Utils.pretty_print({
54 
+          "file_pointer" => {
55 
+            "file" => "filename",
56 
+            "agent_id" => id
57 
+          }
58 
+        })
59 
+      end
60 
+    end
61 
+
62 
+    def default_options
63 
+      {
64 
+        'mode' => 'read',
65 
+        'access_key_id' => '',
66 
+        'access_key_secret' => '',
67 
+        'watch' => 'true',
68 
+        'bucket' => "",
69 
+        'data' => '{{ data }}'
70 
+      }
71 
+    end
72 
+
73 
+    form_configurable :mode, type: :array, values: %w(read write)
74 
+    form_configurable :access_key_id, roles: :validatable
75 
+    form_configurable :access_key_secret, roles: :validatable
76 
+    form_configurable :region, type: :array, values: %w(us-east-1 us-west-1 us-west-2 eu-west-1 eu-central-1 ap-southeast-1 ap-southeast-2 ap-northeast-1 ap-northeast-2 sa-east-1)
77 
+    form_configurable :watch, type: :array, values: %w(true false)
78 
+    form_configurable :bucket, roles: :completable
79 
+    form_configurable :filename
80 
+    form_configurable :data
81 
+
82 
+    def validate_options
83 
+      if options['mode'].blank? || !['read', 'write'].include?(options['mode'])
84 
+        errors.add(:base, "The 'mode' option is required and must be set to 'read' or 'write'")
85 
+      end
86 
+      if options['bucket'].blank?
87 
+        errors.add(:base, "The 'bucket' option is required.")
88 
+      end
89 
+      if options['region'].blank?
90 
+        errors.add(:base, "The 'region' option is required.")
91 
+      end
92 
+
93 
+      case interpolated['mode']
94 
+      when 'read'
95 
+        if options['watch'].blank? || ![true, false].include?(boolify(options['watch']))
96 
+          errors.add(:base, "The 'watch' option is required and must be set to 'true' or 'false'")
97 
+        end
98 
+      when 'write'
99 
+        if options['filename'].blank?
100 
+          errors.add(:base, "filename must be specified in 'write' mode")
101 
+        end
102 
+        if options['data'].blank?
103 
+          errors.add(:base, "data must be specified in 'write' mode")
104 
+        end
105 
+      end
106 
+    end
107 
+
108 
+    def validate_access_key_id
109 
+      !!buckets
110 
+    end
111 
+
112 
+    def validate_access_key_secret
113 
+      !!buckets
114 
+    end
115 
+
116 
+    def complete_bucket
117 
+      (buckets || []).collect { |room| {text: room.name, id: room.name} }
118 
+    end
119 
+
120 
+    def working?
121 
+      checked_without_error?
122 
+    end
123 
+
124 
+    def check
125 
+      return if interpolated['mode'] != 'read'
126 
+      contents = safely do
127 
+                   get_bucket_contents
128 
+                 end
129 
+      if boolify(interpolated['watch'])
130 
+        watch(contents)
131 
+      else
132 
+        contents.each do |key, _|
133 
+          create_event payload: get_file_pointer(key)
134 
+        end
135 
+      end
136 
+    end
137 
+
138 
+    def get_io(file)
139 
+      client.get_object(bucket: interpolated['bucket'], key: file).body
140 
+    end
141 
+
142 
+    def receive(incoming_events)
143 
+      return if interpolated['mode'] != 'write'
144 
+      incoming_events.each do |event|
145 
+        safely do
146 
+          mo = interpolated(event)
147 
+          client.put_object(bucket: mo['bucket'], key: mo['filename'], body: mo['data'])
148 
+        end
149 
+      end
150 
+    end
151 
+
152 
+    private
153 
+
154 
+    def safely
155 
+      yield
156 
+    rescue Aws::S3::Errors::AccessDenied => e
157 
+      error("Could not access '#{interpolated['bucket']}' #{e.class} #{e.message}")
158 
+    rescue Aws::S3::Errors::ServiceError =>e
159 
+      error("#{e.class}: #{e.message}")
160 
+    end
161 
+
162 
+    def watch(contents)
163 
+      if last_check_at.nil?
164 
+        self.memory['seen_contents'] = contents
165 
+        return
166 
+      end
167 
+
168 
+      new_memory = contents.dup
169 
+
170 
+      memory['seen_contents'].each do |key, etag|
171 
+        if contents[key].blank?
172 
+          create_event payload: get_file_pointer(key).merge(event_type: :removed)
173 
+        elsif contents[key] != etag
174 
+          create_event payload: get_file_pointer(key).merge(event_type: :modified)
175 
+        end
176 
+        contents.delete(key)
177 
+      end
178 
+      contents.each do |key, etag|
179 
+        create_event payload: get_file_pointer(key).merge(event_type: :added)
180 
+      end
181 
+
182 
+      self.memory['seen_contents'] = new_memory
183 
+    end
184 
+
185 
+    def get_bucket_contents
186 
+      contents = {}
187 
+      client.list_objects(bucket: interpolated['bucket']).each do |response|
188 
+        response.contents.each do |file|
189 
+          contents[file.key] = file.etag
190 
+        end
191 
+      end
192 
+      contents
193 
+    end
194 
+
195 
+    def client
196 
+      @client ||= Aws::S3::Client.new(credentials: Aws::Credentials.new(interpolated['access_key_id'], interpolated['access_key_secret']),
197 
+                                      region: interpolated['region'])
198 
+    end
199 
+
200 
+    def buckets(log = false)
201 
+      @buckets ||= client.list_buckets.buckets
202 
+    rescue Aws::S3::Errors::ServiceError => e
203 
+      false
204 
+    end
205 
+  end
206 
+end

+ 15 - 0
db/migrate/20160224120316_add_mode_option_to_ftpsite_agents.rb
Datei anzeigen 

@@ -0,0 +1,15 @@
1 
+class AddModeOptionToFtpsiteAgents < ActiveRecord::Migration
2 
+  def up
3 
+    Agents::FtpsiteAgent.find_each do |agent|
4 
+      agent.options['mode'] = 'read'
5 
+      agent.save!(validate: false)
6 
+    end
7 
+  end
8 
+
9 
+  def down
10 
+    Agents::FtpsiteAgent.find_each do |agent|
11 
+      agent.options.delete 'mode'
12 
+      agent.save!(validate: false)
13 
+    end
14 
+  end
15 
+end

+ 1 - 0
lib/agent_runner.rb
Datei anzeigen 

@@ -118,5 +118,6 @@ end
118 118
119 119 
 require 'agents/twitter_stream_agent'
120 120 
 require 'agents/jabber_agent'
121 
+require 'agents/local_file_agent'
121 122 
 require 'huginn_scheduler'
122 123 
 require 'delayed_job_worker'

+ 1 - 1
lib/utils.rb
Datei anzeigen 

@@ -15,7 +15,7 @@ module Utils
15 15 
   def self.pretty_print(struct, indent = true)
16 16 
     output = JSON.pretty_generate(struct)
17 17 
     if indent
18 
-      output.gsub(/\n/i, "\n    ").tap { |a| p a }
18 
+      output.gsub(/\n/i, "\n    ")
19 19 
     else
20 20 
       output
21 21 
     end

+ 1 - 0
spec/env.test
Datei anzeigen 

@@ -12,3 +12,4 @@ EVERNOTE_OAUTH_KEY=evernoteoauthkey
12 12 
 EVERNOTE_OAUTH_SECRET=evernoteoauthsecret
13 13 
 FAILED_JOBS_TO_KEEP=2
14 14 
 REQUIRE_CONFIRMED_EMAIL=false
15 
+ENABLE_INSECURE_AGENTS=true

+ 244 - 0
spec/models/agents/csv_agent_spec.rb
Datei anzeigen 

@@ -0,0 +1,244 @@
1 
+require 'rails_helper'
2 
+
3 
+describe Agents::CsvAgent do
4 
+  before(:each) do
5 
+    @valid_params = {
6 
+                      'mode' => 'parse',
7 
+                      'separator' => ',',
8 
+                      'use_fields' => '',
9 
+                      'output' => 'event_per_row',
10 
+                      'with_header' => 'true',
11 
+                      'data_path' => '$.data',
12 
+                      'data_key' => 'data'
13 
+                    }
14 
+
15 
+    @checker = Agents::CsvAgent.new(:name => 'somename', :options => @valid_params)
16 
+    @checker.user = users(:jane)
17 
+    @checker.save!
18 
+    @lfa = Agents::LocalFileAgent.new(name: 'local', options: {path: '{{}}', watch: 'false', append: 'false', mode: 'read'})
19 
+    @lfa.user = users(:jane)
20 
+    @lfa.save!
21 
+  end
22 
+
23 
+  it_behaves_like 'FileHandlingConsumer'
24 
+
25 
+  context '#validate_options' do
26 
+    it 'is valid with the given options' do
27 
+      expect(@checker).to be_valid
28 
+    end
29 
+
30 
+    it "requires with_header to be either 'true' or 'false'" do
31 
+      @checker.options['with_header'] = 'true'
32 
+      expect(@checker).to be_valid
33 
+      @checker.options['with_header'] = 'false'
34 
+      expect(@checker).to be_valid
35 
+      @checker.options['with_header'] = 'test'
36 
+      expect(@checker).not_to be_valid
37 
+    end
38 
+
39 
+    it "data_path has to be set in serialize mode" do
40 
+      @checker.options['mode'] = 'serialize'
41 
+      @checker.options['data_path'] = ''
42 
+      expect(@checker).not_to be_valid
43 
+    end
44 
+  end
45 
+
46 
+  context '#working' do
47 
+    it 'is not working without having received an event' do
48 
+      expect(@checker).not_to be_working
49 
+    end
50 
+
51 
+    it 'is working after receiving an event without error' do
52 
+      @checker.last_receive_at = Time.now
53 
+      expect(@checker).to be_working
54 
+    end
55 
+  end
56 
+
57 
+  context '#receive' do
58 
+    after(:all) do
59 
+      FileUtils.rm(File.join(Rails.root, 'tmp', 'csv'))
60 
+    end
61 
+
62 
+    def event_with_contents(contents)
63 
+      path = File.join(Rails.root, 'tmp', 'csv')
64 
+      File.open(path, 'w') do |f|
65 
+        f.write(contents)
66 
+      end
67 
+      Event.new(payload: { 'file_pointer' => {'agent_id' => @lfa.id, 'file' => path } }, user_id: @checker.user_id)
68 
+    end
69 
+
70 
+    context "agent options" do
71 
+      let(:with_headers) { event_with_contents("one,two\n1,2\n2,3") }
72 
+      let(:without_headers) { event_with_contents("1,2\n2,3") }
73 
+
74 
+      context "output" do
75 
+        it "creates one event per row" do
76 
+          @checker.options['output'] = 'event_per_row'
77 
+          expect { @checker.receive([with_headers]) }.to change(Event, :count).by(2)
78 
+          expect(Event.last.payload).to eq(@checker.options['data_key'] => {'one' => '2', 'two' => '3'})
79 
+        end
80 
+
81 
+        it "creates one event per file" do
82 
+          @checker.options['output'] = 'event_per_file'
83 
+          expect { @checker.receive([with_headers]) }.to change(Event, :count).by(1)
84 
+          expect(Event.last.payload).to eq(@checker.options['data_key'] => [{"one"=>"1", "two"=>"2"}, {"one"=>"2", "two"=>"3"}])
85 
+        end
86 
+      end
87 
+
88 
+      context "with_header" do
89 
+        it "works without headers" do
90 
+          @checker.options['with_header'] = 'false'
91 
+          expect { @checker.receive([without_headers]) }.to change(Event, :count).by(2)
92 
+          expect(Event.last.payload).to eq({@checker.options['data_key']=>["2", "3"]})
93 
+        end
94 
+
95 
+        it "works without headers and event_per_file" do
96 
+          @checker.options['with_header'] = 'false'
97 
+          @checker.options['output'] = 'event_per_file'
98 
+          expect { @checker.receive([without_headers]) }.to change(Event, :count).by(1)
99 
+          expect(Event.last.payload).to eq({@checker.options['data_key']=>[['1', '2'], ["2", "3"]]})
100 
+        end
101 
+      end
102 
+
103 
+      context "use_fields" do
104 
+        it "extracts the specified columns" do
105 
+          @checker.options['use_fields'] = 'one'
106 
+          expect { @checker.receive([with_headers]) }.to change(Event, :count).by(2)
107 
+          expect(Event.last.payload).to eq(@checker.options['data_key'] => {'one' => '2'})
108 
+        end
109 
+      end
110 
+
111 
+      context "data_path" do
112 
+        it "can receive the CSV via a regular event" do
113 
+          @checker.options['data_path'] = '$.data'
114 
+          event = Event.new(payload: {'data' => "one,two\r\n1,2\r\n2,3"})
115 
+          expect { @checker.receive([event]) }.to change(Event, :count).by(2)
116 
+          expect(Event.last.payload).to eq(@checker.options['data_key'] => {'one' => '2', 'two' => '3'})
117 
+        end
118 
+      end
119 
+    end
120 
+
121 
+    context "handling different CSV formats" do
122 
+      it "works with windows line endings" do
123 
+        event = event_with_contents("one,two\r\n1,2\r\n2,3")
124 
+        expect { @checker.receive([event]) }.to change(Event, :count).by(2)
125 
+        expect(Event.last.payload).to eq(@checker.options['data_key'] => {'one' => '2', 'two' => '3'})
126 
+      end
127 
+
128 
+      it "works with OSX line endings" do
129 
+        event = event_with_contents("one,two\r1,2\r2,3")
130 
+        expect { @checker.receive([event]) }.to change(Event, :count).by(2)
131 
+        expect(Event.last.payload).to eq(@checker.options['data_key'] => {'one' => '2', 'two' => '3'})
132 
+      end
133 
+
134 
+      it "handles quotes correctly" do
135 
+        event = event_with_contents("\"one\",\"two\"\n1,2\n\"\"2, two\",3")
136 
+        expect { @checker.receive([event]) }.to change(Event, :count).by(2)
137 
+        expect(Event.last.payload).to eq(@checker.options['data_key'] => {'one' => '"2, two', 'two' => '3'})
138 
+      end
139 
+
140 
+      it "works with tab seperated csv" do
141 
+        event = event_with_contents("one\ttwo\r\n1\t2\r\n2\t3")
142 
+        @checker.options['separator'] = '\\t'
143 
+        expect { @checker.receive([event]) }.to change(Event, :count).by(2)
144 
+        expect(Event.last.payload).to eq(@checker.options['data_key'] => {'one' => '2', 'two' => '3'})
145 
+      end
146 
+    end
147 
+
148 
+    context "serializing" do
149 
+      before(:each) do
150 
+        @checker.options['mode'] = 'serialize'
151 
+        @checker.options['data_path'] = '$.data'
152 
+        @checker.options['data_key'] = 'data'
153 
+      end
154 
+
155 
+      it "writes headers when with_header is true" do
156 
+        event = Event.new(payload: { 'data' => {'key' => 'value', 'key2' => 'value2', 'key3' => 'value3'} })
157 
+        expect { @checker.receive([event])}.to change(Event, :count).by(1)
158 
+        expect(Event.last.payload).to eq('data' => "\"key\",\"key2\",\"key3\"\n\"value\",\"value2\",\"value3\"\n")
159 
+      end
160 
+
161 
+      it "writes one row per received event" do
162 
+        event = Event.new(payload: { 'data' => {'key' => 'value', 'key2' => 'value2', 'key3' => 'value3'} })
163 
+        event2 = Event.new(payload: { 'data' => {'key' => '2value', 'key2' => '2value2', 'key3' => '2value3'} })
164 
+        expect { @checker.receive([event, event2])}.to change(Event, :count).by(1)
165 
+        expect(Event.last.payload).to eq('data' => "\"key\",\"key2\",\"key3\"\n\"value\",\"value2\",\"value3\"\n\"2value\",\"2value2\",\"2value3\"\n")
166 
+      end
167 
+
168 
+      it "accepts multiple rows per event" do
169 
+        event = Event.new(payload: { 'data' => [{'key' => 'value', 'key2' => 'value2', 'key3' => 'value3'}, {'key' => '2value', 'key2' => '2value2', 'key3' => '2value3'}] })
170 
+        expect { @checker.receive([event])}.to change(Event, :count).by(1)
171 
+        expect(Event.last.payload).to eq('data' => "\"key\",\"key2\",\"key3\"\n\"value\",\"value2\",\"value3\"\n\"2value\",\"2value2\",\"2value3\"\n")
172 
+      end
173 
+
174 
+      it "does not write the headers when with_header is false" do
175 
+        @checker.options['with_header'] = 'false'
176 
+        event = Event.new(payload: { 'data' => {'key' => 'value', 'key2' => 'value2', 'key3' => 'value3'} })
177 
+        expect { @checker.receive([event])}.to change(Event, :count).by(1)
178 
+        expect(Event.last.payload).to eq('data' => "\"value\",\"value2\",\"value3\"\n")
179 
+      end
180 
+
181 
+      it "only serialize the keys specified in use_fields" do
182 
+        @checker.options['use_fields'] = 'key2, key3'
183 
+        event = Event.new(payload: { 'data' => {'key' => 'value', 'key2' => 'value2', 'key3' => 'value3'} })
184 
+        expect { @checker.receive([event])}.to change(Event, :count).by(1)
185 
+        expect(Event.last.payload).to eq('data' => "\"key2\",\"key3\"\n\"value2\",\"value3\"\n")
186 
+      end
187 
+
188 
+      it "respects the order of use_fields" do
189 
+        @checker.options['use_fields'] = 'key3, key'
190 
+        event = Event.new(payload: { 'data' => {'key' => 'value', 'key2' => 'value2', 'key3' => 'value3'} })
191 
+        expect { @checker.receive([event])}.to change(Event, :count).by(1)
192 
+        expect(Event.last.payload).to eq('data' => "\"key3\",\"key\"\n\"value3\",\"value\"\n")
193 
+      end
194 
+
195 
+      it "respects use_fields and writes no header" do
196 
+        @checker.options['with_header'] = 'false'
197 
+        @checker.options['use_fields'] = 'key2, key3'
198 
+        event = Event.new(payload: { 'data' => {'key' => 'value', 'key2' => 'value2', 'key3' => 'value3'} })
199 
+        expect { @checker.receive([event])}.to change(Event, :count).by(1)
200 
+        expect(Event.last.payload).to eq('data' => "\"value2\",\"value3\"\n")
201 
+      end
202 
+
203 
+      context "arrays" do
204 
+        it "does not write a header" do
205 
+          @checker.options['with_header'] = 'false'
206 
+          event = Event.new(payload: { 'data' => ['value1', 'value2'] })
207 
+          event2 = Event.new(payload: { 'data' => ['value3', 'value4'] })
208 
+          expect { @checker.receive([event, event2])}.to change(Event, :count).by(1)
209 
+          expect(Event.last.payload).to eq('data' => "\"value1\",\"value2\"\n\"value3\",\"value4\"\n")
210 
+        end
211 
+
212 
+        it "handles nested arrays" do
213 
+          event = Event.new(payload: { 'data' => [['value1', 'value2'], ['value3', 'value4']] })
214 
+          expect { @checker.receive([event])}.to change(Event, :count).by(1)
215 
+          expect(Event.last.payload).to eq('data' => "\"value1\",\"value2\"\n\"value3\",\"value4\"\n")
216 
+        end
217 
+      end
218 
+    end
219 
+  end
220 
+
221 
+  context '#event_description' do
222 
+    it "works with event_per_row and headers" do
223 
+      @checker.options['output'] = 'event_per_row'
224 
+      @checker.options['with_header'] = 'true'
225 
+      description = @checker.event_description
226 
+      expect(description).not_to match(/\n\s+\[\n/)
227 
+      expect(description).to include(": {\n")
228 
+    end
229 
+
230 
+    it "works with event_per_file and without headers" do
231 
+      @checker.options['output'] = 'event_per_file'
232 
+      @checker.options['with_header'] = 'false'
233 
+      description = @checker.event_description
234 
+      expect(description).to match(/\n\s+\[\n/)
235 
+      expect(description).not_to include(": {\n")
236 
+    end
237 
+
238 
+    it "shows dummy CSV when in serialize mode" do
239 
+      @checker.options['mode'] = 'serialize'
240 
+      description = @checker.event_description
241 
+      expect(description).to include('"generated\",\"csv')
242 
+    end
243 
+  end
244 
+end

+ 143 - 0
spec/models/agents/ftpsite_agent_spec.rb
Datei anzeigen 

@@ -8,12 +8,74 @@ describe Agents::FtpsiteAgent do
8 8 
         'expected_update_period_in_days' => 1,
9 9 
         'url' => "ftp://ftp.example.org/pub/releases/",
10 10 
         'patterns' => ["example*.tar.gz"],
11 
+        'mode' => 'read',
12 
+        'filename' => 'test',
13 
+        'data' => '{{ data }}'
11 14 
       }
12 15 
       @checker = Agents::FtpsiteAgent.new(:name => "Example", :options => @site, :keep_events_for => 2.days)
13 16 
       @checker.user = users(:bob)
14 17 
       @checker.save!
15 18 
     end
16 19
20 
+    context "#validate_options" do
21 
+      it "requires url to be a valid URI" do
22 
+        @checker.options['url'] = 'not_valid'
23 
+        expect(@checker).not_to be_valid
24 
+      end
25 
+
26 
+      it "allows an URI without a path" do
27 
+        @checker.options['url'] = 'ftp://ftp.example.org'
28 
+        expect(@checker).to be_valid
29 
+      end
30 
+
31 
+      it "does not check the url when liquid output markup is used" do
32 
+        @checker.options['url'] = 'ftp://{{ ftp_host }}'
33 
+        expect(@checker).to be_valid
34 
+      end
35 
+
36 
+      it "requires patterns to be present and not empty array" do
37 
+        @checker.options['patterns'] = ''
38 
+        expect(@checker).not_to be_valid
39 
+        @checker.options['patterns'] = 'not an array'
40 
+        expect(@checker).not_to be_valid
41 
+        @checker.options['patterns'] = []
42 
+        expect(@checker).not_to be_valid
43 
+      end
44 
+
45 
+      it "when present timestamp must be parsable into a Time object instance" do
46 
+        @checker.options['timestamp'] = '2015-01-01 00:00:01'
47 
+        expect(@checker).to be_valid
48 
+        @checker.options['timestamp'] = 'error'
49 
+        expect(@checker).not_to be_valid
50 
+      end
51 
+
52 
+      it "requires mode to be set to 'read' or 'write'" do
53 
+        @checker.options['mode'] = 'write'
54 
+        expect(@checker).to be_valid
55 
+        @checker.options['mode'] = ''
56 
+        expect(@checker).not_to be_valid
57 
+      end
58 
+
59 
+      it 'automatically sets mode to read when the agent is a new record' do
60 
+        checker = Agents::FtpsiteAgent.new(name: 'test', options: @site.except('mode'))
61 
+        checker.user = users(:bob)
62 
+        expect(checker).to be_valid
63 
+        expect(checker.options['mode']).to eq('read')
64 
+      end
65 
+
66 
+      it "requires 'filename' in 'write' mode" do
67 
+        @checker.options['mode'] = 'write'
68 
+        @checker.options['filename'] = ''
69 
+        expect(@checker).not_to be_valid
70 
+      end
71 
+
72 
+      it "requires 'data' in 'write' mode" do
73 
+        @checker.options['mode'] = 'write'
74 
+        @checker.options['data'] = ''
75 
+        expect(@checker).not_to be_valid
76 
+      end
77 
+    end
78 
+
17 79 
     describe "#check" do
18 80
19 81 
       before do
 
@@ -42,6 +104,7 @@ describe Agents::FtpsiteAgent do
42 104 
         }
43 105
44 106 
         expect(Event.last(2).first.payload).to eq({
107 
+          'file_pointer' => { 'file' => 'example-1.1.tar.gz', 'agent_id' => @checker.id },
45 108 
           'url' => 'ftp://ftp.example.org/pub/releases/example-1.1.tar.gz',
46 109 
           'filename' => 'example-1.1.tar.gz',
47 110 
           'timestamp' => '2014-04-01T10:00:00Z',
 
@@ -71,12 +134,14 @@ describe Agents::FtpsiteAgent do
71 134 
         }
72 135
73 136 
         expect(Event.last(2).first.payload).to eq({
137 
+          'file_pointer' => { 'file' => 'example-1.2.tar.gz', 'agent_id' => @checker.id },
74 138 
           'url' => 'ftp://ftp.example.org/pub/releases/example-1.2.tar.gz',
75 139 
           'filename' => 'example-1.2.tar.gz',
76 140 
           'timestamp' => '2014-04-02T10:00:00Z',
77 141 
         })
78 142
79 143 
         expect(Event.last.payload).to eq({
144 
+          'file_pointer' => { 'file' => 'example latest.tar.gz', 'agent_id' => @checker.id },
80 145 
           'url' => 'ftp://ftp.example.org/pub/releases/example%20latest.tar.gz',
81 146 
           'filename' => 'example latest.tar.gz',
82 147 
           'timestamp' => '2014-04-02T10:00:01Z',
 
@@ -113,5 +178,83 @@ describe Agents::FtpsiteAgent do
113 178 
       end
114 179 
     end
115 180
181 
+    context "#open_ftp" do
182 
+      before(:each) do
183 
+        @ftp_mock = mock()
184 
+        mock(@ftp_mock).close
185 
+        mock(@ftp_mock).connect('ftp.example.org', 21)
186 
+        mock(@ftp_mock).passive=(true)
187 
+        mock(Net::FTP).new { @ftp_mock }
188 
+      end
189 
+      context 'with_path' do
190 
+        before(:each) { mock(@ftp_mock).chdir('pub/releases') }
191 
+
192 
+        it "logs in as anonymous when no user and password are given" do
193 
+          mock(@ftp_mock).login('anonymous', 'anonymous@')
194 
+          expect { |b| @checker.open_ftp(@checker.base_uri, &b) }.to yield_with_args(@ftp_mock)
195 
+        end
196 
+
197 
+        it "passes the provided user and password" do
198 
+          @checker.options['url'] = "ftp://user:password@ftp.example.org/pub/releases/"
199 
+          mock(@ftp_mock).login('user', 'password')
200 
+          expect { |b| @checker.open_ftp(@checker.base_uri, &b) }.to yield_with_args(@ftp_mock)
201 
+        end
202 
+      end
203 
+
204 
+      it "does not call chdir when no path is given" do
205 
+        @checker.options['url'] = "ftp://ftp.example.org/"
206 
+        mock(@ftp_mock).login('anonymous', 'anonymous@')
207 
+        expect { |b| @checker.open_ftp(@checker.base_uri, &b) }.to yield_with_args(@ftp_mock)
208 
+      end
209 
+    end
210 
+
211 
+    context "#get_io" do
212 
+      it "returns the contents of the file" do
213 
+        ftp_mock= mock()
214 
+        mock(ftp_mock).getbinaryfile('file', nil).yields('data')
215 
+        mock(@checker).open_ftp(@checker.base_uri).yields(ftp_mock)
216 
+        expect(@checker.get_io('file').read).to eq('data')
217 
+      end
218 
+
219 
+      it "uses the encoding specified in force_encoding to convert the data to UTF-8" do
220 
+        ftp_mock= mock()
221 
+        mock(ftp_mock).getbinaryfile('file', nil).yields('ümlaut'.force_encoding('ISO-8859-15'))
222 
+        mock(@checker).open_ftp(@checker.base_uri).yields(ftp_mock)
223 
+        expect(@checker.get_io('file').read).to eq('ümlaut')
224 
+      end
225 
+
226 
+      it "returns an empty StringIO instance when no data was read" do
227 
+        ftp_mock= mock()
228 
+        mock(ftp_mock).getbinaryfile('file', nil)
229 
+        mock(@checker).open_ftp(@checker.base_uri).yields(ftp_mock)
230 
+        expect(@checker.get_io('file').length).to eq(0)
231 
+      end
232 
+    end
233 
+
234 
+    context "#receive" do
235 
+      before(:each) do
236 
+        @checker.options['mode'] = 'write'
237 
+        @checker.options['filename'] = 'file.txt'
238 
+        @checker.options['data'] = '{{ data }}'
239 
+        @ftp_mock= mock()
240 
+        @stringio = StringIO.new()
241 
+        mock(@checker).open_ftp(@checker.base_uri).yields(@ftp_mock)
242 
+      end
243 
+
244 
+      it "writes the data at data into a file" do
245 
+        mock(StringIO).new('hello world🔥') { @stringio }
246 
+        mock(@ftp_mock).storbinary('STOR file.txt', @stringio, Net::FTP::DEFAULT_BLOCKSIZE)
247 
+        event = Event.new(payload: {'data' => 'hello world🔥'})
248 
+        @checker.receive([event])
249 
+      end
250 
+
251 
+      it "converts the string encoding when force_encoding is specified" do
252 
+        @checker.options['force_encoding'] = 'ISO-8859-1'
253 
+        mock(StringIO).new('hello world?') { @stringio }
254 
+        mock(@ftp_mock).storbinary('STOR file.txt', @stringio, Net::FTP::DEFAULT_BLOCKSIZE)
255 
+        event = Event.new(payload: {'data' => 'hello world🔥'})
256 
+        @checker.receive([event])
257 
+      end
258 
+    end
116 259 
   end
117 260 
 end

+ 276 - 0
spec/models/agents/local_file_agent_spec.rb
Datei anzeigen 

@@ -0,0 +1,276 @@
1 
+require 'rails_helper'
2 
+
3 
+describe Agents::LocalFileAgent do
4 
+  before(:each) do
5 
+    @valid_params = {
6 
+                      'mode' => 'read',
7 
+                      'watch' => 'false',
8 
+                      'append' => 'false',
9 
+                      'path' => File.join(Rails.root, 'tmp', 'spec')
10 
+                    }
11 
+    FileUtils.mkdir_p File.join(Rails.root, 'tmp', 'spec')
12 
+
13 
+    @checker = Agents::LocalFileAgent.new(:name => "somename", :options => @valid_params)
14 
+    @checker.user = users(:jane)
15 
+    @checker.save!
16 
+  end
17 
+
18 
+  after(:all) do
19 
+    FileUtils.rm_r File.join(Rails.root, 'tmp', 'spec')
20 
+  end
21 
+
22 
+  describe "#validate_options" do
23 
+    it "is valid with the given options" do
24 
+      expect(@checker).to be_valid
25 
+    end
26 
+
27 
+    it "requires mode to be either 'read' or 'write'" do
28 
+      @checker.options['mode'] = 'write'
29 
+      expect(@checker).to be_valid
30 
+      @checker.options['mode'] = 'write'
31 
+      expect(@checker).to be_valid
32 
+      @checker.options['mode'] = 'test'
33 
+      expect(@checker).not_to be_valid
34 
+    end
35 
+
36 
+    it "requires the path to be set" do
37 
+      @checker.options['path'] = ''
38 
+      expect(@checker).not_to be_valid
39 
+    end
40 
+
41 
+    it "requires watch to be present" do
42 
+      @checker.options['watch'] = ''
43 
+      expect(@checker).not_to be_valid
44 
+    end
45 
+
46 
+    it "requires watch to be either 'true' or 'false'" do
47 
+      @checker.options['watch'] = 'true'
48 
+      expect(@checker).to be_valid
49 
+      @checker.options['watch'] = 'false'
50 
+      expect(@checker).to be_valid
51 
+      @checker.options['watch'] = 'test'
52 
+      expect(@checker).not_to be_valid
53 
+    end
54 
+
55 
+    it "requires append to be either 'true' or 'false'" do
56 
+      @checker.options['append'] = 'true'
57 
+      expect(@checker).to be_valid
58 
+      @checker.options['append'] = 'false'
59 
+      expect(@checker).to be_valid
60 
+      @checker.options['append'] = 'test'
61 
+      expect(@checker).not_to be_valid
62 
+    end
63 
+  end
64 
+
65 
+  context "#working" do
66 
+    it "is working with no recent errors in read mode" do
67 
+      @checker.last_check_at = Time.now
68 
+      expect(@checker).to be_working
69 
+    end
70 
+
71 
+    it "is working with no recent errors in write mode" do
72 
+      @checker.options['mode'] = 'write'
73 
+      @checker.last_receive_at = Time.now
74 
+      expect(@checker).to be_working
75 
+    end
76 
+  end
77 
+
78 
+  context "#check_path_existance" do
79 
+    it "is truethy when the path exists" do
80 
+      expect(@checker.check_path_existance).to be_truthy
81 
+    end
82 
+
83 
+    it "is falsy when the path does not exist" do
84 
+      @checker.options['path'] = '/doesnotexist'
85 
+      expect(@checker.check_path_existance).to be_falsy
86 
+    end
87 
+
88 
+    it "create a log entry" do
89 
+      @checker.options['path'] = '/doesnotexist'
90 
+      expect { @checker.check_path_existance(true) }.to change(AgentLog, :count).by(1)
91 
+    end
92 
+
93 
+    it "works with non-expanded paths" do
94 
+      @checker.options['path'] = '~'
95 
+      expect(@checker.check_path_existance).to be_truthy
96 
+    end
97 
+  end
98 
+
99 
+  def with_files(*files)
100 
+    files.each { |f| FileUtils.touch(f) }
101 
+    yield
102 
+    files.each { |f| FileUtils.rm(f) }
103 
+  end
104 
+
105 
+  context "#check" do
106 
+    it "does not create events when the directory is empty" do
107 
+      expect { @checker.check }.to change(Event, :count).by(0)
108 
+    end
109 
+
110 
+    it "creates an event for every file in the directory" do
111 
+      with_files(File.join(Rails.root, 'tmp', 'spec', 'one'), File.join(Rails.root, 'tmp', 'spec', 'two')) do
112 
+        expect { @checker.check }.to change(Event, :count).by(2)
113 
+        expect(Event.last.payload.has_key?('file_pointer')).to be_truthy
114 
+      end
115 
+    end
116 
+
117 
+    it "creates an event if the configured file exists" do
118 
+      @checker.options['path'] = File.join(Rails.root, 'tmp', 'spec', 'one')
119 
+      with_files(File.join(Rails.root, 'tmp', 'spec', 'one'), File.join(Rails.root, 'tmp', 'spec', 'two')) do
120 
+        expect { @checker.check }.to change(Event, :count).by(1)
121 
+        payload = Event.last.payload
122 
+        expect(payload.has_key?('file_pointer')).to be_truthy
123 
+        expect(payload['file_pointer']['file']).to eq(@checker.options['path'])
124 
+      end
125 
+    end
126 
+
127 
+    it "does not run when ENABLE_INSECURE_AGENTS is not set to true" do
128 
+      ENV['ENABLE_INSECURE_AGENTS'] = 'false'
129 
+      expect { @checker.check }.to change(AgentLog, :count).by(1)
130 
+      ENV['ENABLE_INSECURE_AGENTS'] = 'true'
131 
+    end
132 
+  end
133 
+
134 
+  context "#event_description" do
135 
+    it "should include event_type when watch is set to true" do
136 
+      @checker.options['watch'] = 'true'
137 
+      expect(@checker.event_description).to include('event_type')
138 
+    end
139 
+
140 
+    it "should not include event_type when watch is set to false" do
141 
+      @checker.options['watch'] = 'false'
142 
+      expect(@checker.event_description).not_to include('event_type')
143 
+    end
144 
+  end
145 
+
146 
+  it "get_io opens the file" do
147 
+    mock(File).open('test', 'r')
148 
+    @checker.get_io('test')
149 
+  end
150 
+
151 
+  context "#start_worker?" do
152 
+    it "reeturns true when watch is true" do
153 
+      @checker.options['watch'] = 'true'
154 
+      expect(@checker.start_worker?).to be_truthy
155 
+    end
156 
+
157 
+    it "returns false when watch is false" do
158 
+      @checker.options['watch'] = 'false'
159 
+      expect(@checker.start_worker?).to be_falsy
160 
+    end
161 
+  end
162 
+
163 
+  context "#receive" do
164 
+    before(:each) do
165 
+      @checker.options['mode'] = 'write'
166 
+      @checker.options['data'] = '{{ data }}'
167 
+      @file_mock = mock()
168 
+    end
169 
+
170 
+    it "writes the data at data into a file" do
171 
+      mock(@file_mock).write('hello world')
172 
+      event = Event.new(payload: {'data' => 'hello world'})
173 
+      mock(File).open(File.join(Rails.root, 'tmp', 'spec'), 'w').yields @file_mock
174 
+      @checker.receive([event])
175 
+    end
176 
+
177 
+    it "appends the data at data onto a file" do
178 
+      mock(@file_mock).write('hello world')
179 
+      @checker.options['append'] = 'true'
180 
+      event = Event.new(payload: {'data' => 'hello world'})
181 
+      mock(File).open(File.join(Rails.root, 'tmp', 'spec'), 'a').yields @file_mock
182 
+      @checker.receive([event])
183 
+    end
184 
+
185 
+    it "does not receive when ENABLE_INSECURE_AGENTS is not set to true" do
186 
+      ENV['ENABLE_INSECURE_AGENTS'] = 'false'
187 
+      expect { @checker.receive([]) }.to change(AgentLog, :count).by(1)
188 
+      ENV['ENABLE_INSECURE_AGENTS'] = 'true'
189 
+    end
190 
+  end
191 
+
192 
+  describe describe Agents::LocalFileAgent::Worker do
193 
+    require 'listen'
194 
+
195 
+    before(:each) do
196 
+      @checker.options['watch'] = true
197 
+      @checker.save
198 
+      @worker = Agents::LocalFileAgent::Worker.new(agent: @checker)
199 
+      @listen_mock = mock()
200 
+    end
201 
+
202 
+    context "#setup" do
203 
+      it "initializes the listen gem" do
204 
+        mock(Listen).to(@checker.options['path'], ignore!: [])
205 
+        @worker.setup
206 
+      end
207 
+    end
208 
+
209 
+    context "#run" do
210 
+      before(:each) do
211 
+        stub(Listen).to { @listen_mock }
212 
+        @worker.setup
213 
+      end
214 
+
215 
+      it "starts to listen to changes in the directory when the path is present" do
216 
+        mock(@worker).sleep
217 
+        mock(@listen_mock).start
218 
+        @worker.run
219 
+      end
220 
+
221 
+      it "does nothing when the path does not exist" do
222 
+        mock(@worker.agent).check_path_existance(true) { false }
223 
+        dont_allow(@listen_mock).start
224 
+        mock(@worker).sleep { raise "Sleeping" }
225 
+        expect { @worker.run }.to raise_exception(RuntimeError, 'Sleeping')
226 
+      end
227 
+    end
228 
+
229 
+    context "#stop" do
230 
+      it "stops the listen gem" do
231 
+        stub(Listen).to { @listen_mock }
232 
+        @worker.setup
233 
+        mock(@listen_mock).stop
234 
+        @worker.stop
235 
+      end
236 
+    end
237 
+
238 
+    context "#callback" do
239 
+      let(:file) { File.join(Rails.root, 'tmp', 'one') }
240 
+      let(:file2) { File.join(Rails.root, 'tmp', 'one2') }
241 
+
242 
+      it "creates an event for modifies files" do
243 
+        expect { @worker.send(:callback, [file], [], [])}.to change(Event, :count).by(1)
244 
+        payload = Event.last.payload
245 
+        expect(payload['event_type']).to eq('modified')
246 
+      end
247 
+
248 
+      it "creates an event for modifies files" do
249 
+        expect { @worker.send(:callback, [], [file], [])}.to change(Event, :count).by(1)
250 
+        payload = Event.last.payload
251 
+        expect(payload['event_type']).to eq('added')
252 
+      end
253 
+
254 
+      it "creates an event for modifies files" do
255 
+        expect { @worker.send(:callback, [], [], [file])}.to change(Event, :count).by(1)
256 
+        payload = Event.last.payload
257 
+        expect(payload['event_type']).to eq('removed')
258 
+      end
259 
+
260 
+      it "creates an event each changed file" do
261 
+        expect { @worker.send(:callback, [], [file], [file2])}.to change(Event, :count).by(2)
262 
+      end
263 
+    end
264 
+
265 
+    context "#listen_options" do
266 
+      it "returns the path when a directory is given" do
267 
+        expect(@worker.send(:listen_options)).to eq([File.join(Rails.root, 'tmp', 'spec'), ignore!: []])
268 
+      end
269 
+
270 
+      it "restricts to only the specified filename" do
271 
+        @worker.agent.options['path'] = File.join(Rails.root, 'tmp', 'one')
272 
+        expect(@worker.send(:listen_options)).to eq([File.join(Rails.root, 'tmp'), { only: /\Aone\z/, ignore!: [] } ])
273 
+      end
274 
+    end
275 
+  end
276 
+end

+ 47 - 0
spec/models/agents/read_file_agent_spec.rb
Datei anzeigen 

@@ -0,0 +1,47 @@
1 
+require 'rails_helper'
2 
+
3 
+describe Agents::ReadFileAgent do
4 
+  before(:each) do
5 
+    @valid_params = {
6 
+                      'data_key' => 'data',
7 
+                    }
8 
+
9 
+    @checker = Agents::ReadFileAgent.new(:name => 'somename', :options => @valid_params)
10 
+    @checker.user = users(:jane)
11 
+    @checker.save!
12 
+  end
13 
+
14 
+  it_behaves_like 'FileHandlingConsumer'
15 
+
16 
+  context '#validate_options' do
17 
+    it 'is valid with the given options' do
18 
+      expect(@checker).to be_valid
19 
+    end
20 
+
21 
+    it "requires data_key to be present" do
22 
+      @checker.options['data_key'] = ''
23 
+      expect(@checker).not_to be_valid
24 
+    end
25 
+  end
26 
+
27 
+  context '#working' do
28 
+    it 'is not working without having received an event' do
29 
+      expect(@checker).not_to be_working
30 
+    end
31 
+
32 
+    it 'is working after receiving an event without error' do
33 
+      @checker.last_receive_at = Time.now
34 
+      expect(@checker).to be_working
35 
+    end
36 
+  end
37 
+
38 
+  context '#receive' do
39 
+    it "emits an event with the contents of the receives files" do
40 
+      event = Event.new(payload: {file_pointer: {agent_id: 111, file: 'test'}})
41 
+      io_mock = mock()
42 
+      mock(@checker).get_io(event) { StringIO.new("testdata") }
43 
+      expect { @checker.receive([event]) }.to change(Event, :count).by(1)
44 
+      expect(Event.last.payload).to eq('data' => 'testdata')
45 
+    end
46 
+  end
47 
+end

+ 220 - 0
spec/models/agents/s3_agent_spec.rb
Datei anzeigen 

@@ -0,0 +1,220 @@
1 
+require 'rails_helper'
2 
+
3 
+describe Agents::S3Agent do
4 
+  before(:each) do
5 
+    @valid_params = {
6 
+                      'mode' => 'read',
7 
+                      'access_key_id' => '32343242',
8 
+                      'access_key_secret' => '1231312',
9 
+                      'watch' => 'false',
10 
+                      'bucket' => 'testbucket',
11 
+                      'region' => 'us-east-1',
12 
+                      'filename' => 'test.txt',
13 
+                      'data' => '{{ data }}'
14 
+                    }
15 
+
16 
+    @checker = Agents::S3Agent.new(:name => "somename", :options => @valid_params)
17 
+    @checker.user = users(:jane)
18 
+    @checker.save!
19 
+  end
20 
+
21 
+  describe "#validate_options" do
22 
+    it "requires the bucket to be set" do
23 
+      @checker.options['bucket'] = ''
24 
+      expect(@checker).not_to be_valid
25 
+    end
26 
+
27 
+    it "requires watch to be present" do
28 
+      @checker.options['watch'] = ''
29 
+      expect(@checker).not_to be_valid
30 
+    end
31 
+
32 
+    it "requires watch to be either 'true' or 'false'" do
33 
+      @checker.options['watch'] = 'true'
34 
+      expect(@checker).to be_valid
35 
+      @checker.options['watch'] = 'false'
36 
+      expect(@checker).to be_valid
37 
+      @checker.options['watch'] = 'test'
38 
+      expect(@checker).not_to be_valid
39 
+    end
40 
+
41 
+    it "requires region to be present" do
42 
+      @checker.options['region'] = ''
43 
+      expect(@checker).not_to be_valid
44 
+    end
45 
+
46 
+    it "requires mode to be set to 'read' or 'write'" do
47 
+      @checker.options['mode'] = 'write'
48 
+      expect(@checker).to be_valid
49 
+      @checker.options['mode'] = ''
50 
+      expect(@checker).not_to be_valid
51 
+    end
52 
+
53 
+    it "requires 'filename' in 'write' mode" do
54 
+      @checker.options['mode'] = 'write'
55 
+      @checker.options['filename'] = ''
56 
+      expect(@checker).not_to be_valid
57 
+    end
58 
+
59 
+    it "requires 'data' in 'write' mode" do
60 
+      @checker.options['mode'] = 'write'
61 
+      @checker.options['data'] = ''
62 
+      expect(@checker).not_to be_valid
63 
+    end
64 
+  end
65 
+
66 
+  describe "#validating" do
67 
+    it "validates the key" do
68 
+      mock(@checker).client { raise Aws::S3::Errors::SignatureDoesNotMatch.new('', '') }
69 
+      expect(@checker.validate_access_key_id).to be_falsy
70 
+    end
71 
+
72 
+    it "validates the secret" do
73 
+      mock(@checker).buckets { true }
74 
+      expect(@checker.validate_access_key_secret).to be_truthy
75 
+    end
76 
+  end
77 
+
78 
+  it "completes the buckets" do
79 
+    mock(@checker).buckets { [OpenStruct.new(name: 'test'), OpenStruct.new(name: 'test2')]}
80 
+    expect(@checker.complete_bucket).to eq([{text: 'test', id: 'test'}, {text: 'test2', id: 'test2'}])
81 
+  end
82 
+
83 
+  context "#working" do
84 
+    it "is working with no recent errors" do
85 
+      @checker.last_check_at = Time.now
86 
+      expect(@checker).to be_working
87 
+    end
88 
+  end
89 
+
90 
+  context "#check" do
91 
+    context "not watching" do
92 
+      it "emits an event for every file" do
93 
+        mock(@checker).get_bucket_contents { {"test"=>"231232", "test2"=>"4564545"} }
94 
+        expect { @checker.check }.to change(Event, :count).by(2)
95 
+        expect(Event.last.payload).to eq({"file_pointer" => {"file"=>"test2", "agent_id"=> @checker.id}})
96 
+      end
97 
+    end
98 
+
99 
+    context "watching" do
100 
+      before(:each) do
101 
+        @checker.options['watch'] = 'true'
102 
+      end
103 
+
104 
+      it "does not emit any events on the first run" do
105 
+        contents = {"test"=>"231232", "test2"=>"4564545"}
106 
+        mock(@checker).get_bucket_contents { contents }
107 
+        expect { @checker.check }.not_to change(Event, :count)
108 
+        expect(@checker.memory).to eq('seen_contents' => contents)
109 
+      end
110 
+
111 
+      context "detecting changes" do
112 
+        before(:each) do
113 
+          contents = {"test"=>"231232", "test2"=>"4564545"}
114 
+          mock(@checker).get_bucket_contents { contents }
115 
+          expect { @checker.check }.not_to change(Event, :count)
116 
+          @checker.last_check_at = Time.now
117 
+        end
118 
+
119 
+        it "emits events for removed files" do
120 
+          contents = {"test"=>"231232"}
121 
+          mock(@checker).get_bucket_contents { contents }
122 
+          expect { @checker.check }.to change(Event, :count).by(1)
123 
+          expect(Event.last.payload).to eq({"file_pointer" => {"file" => "test2", "agent_id"=> @checker.id}, "event_type" => "removed"})
124 
+        end
125 
+
126 
+        it "emits events for modified files" do
127 
+          contents = {"test"=>"231232", "test2"=>"changed"}
128 
+          mock(@checker).get_bucket_contents { contents }
129 
+          expect { @checker.check }.to change(Event, :count).by(1)
130 
+          expect(Event.last.payload).to eq({"file_pointer" => {"file" => "test2", "agent_id"=> @checker.id}, "event_type" => "modified"})
131 
+        end
132 
+        it "emits events for added files" do
133 
+          contents = {"test"=>"231232", "test2"=>"4564545", "test3" => "31231231"}
134 
+          mock(@checker).get_bucket_contents { contents }
135 
+          expect { @checker.check }.to change(Event, :count).by(1)
136 
+          expect(Event.last.payload).to eq({"file_pointer" => {"file" => "test3", "agent_id"=> @checker.id}, "event_type" => "added"})
137 
+        end
138 
+      end
139 
+
140 
+      context "error handling" do
141 
+        it "handles AccessDenied exceptions" do
142 
+          mock(@checker).get_bucket_contents { raise Aws::S3::Errors::AccessDenied.new('', '') }
143 
+          expect { @checker.check }.to change(AgentLog, :count).by(1)
144 
+          expect(AgentLog.last.message).to eq("Could not access 'testbucket' Aws::S3::Errors::AccessDenied ")
145 
+        end
146 
+
147 
+        it "handles generic S3 exceptions" do
148 
+          mock(@checker).get_bucket_contents { raise Aws::S3::Errors::PermanentRedirect.new('', 'error') }
149 
+          expect { @checker.check }.to change(AgentLog, :count).by(1)
150 
+          expect(AgentLog.last.message).to eq("Aws::S3::Errors::PermanentRedirect: error")
151 
+        end
152 
+      end
153 
+    end
154 
+  end
155 
+
156 
+  it "get_io returns a StringIO object" do
157 
+    stringio =StringIO.new
158 
+    mock_response = mock()
159 
+    mock(mock_response).body { stringio }
160 
+    mock_client = mock()
161 
+    mock(mock_client).get_object(bucket: 'testbucket', key: 'testfile') { mock_response }
162 
+    mock(@checker).client { mock_client }
163 
+    @checker.get_io('testfile')
164 
+  end
165 
+
166 
+  context "#get_bucket_contents" do
167 
+    it "returns a hash with the contents of the bucket" do
168 
+      mock_response = mock()
169 
+      mock(mock_response).contents { [OpenStruct.new(key: 'test', etag: '231232'), OpenStruct.new(key: 'test2', etag: '4564545')] }
170 
+      mock_client = mock()
171 
+      mock(mock_client).list_objects(bucket: 'testbucket') { [mock_response] }
172 
+      mock(@checker).client { mock_client }
173 
+      expect(@checker.send(:get_bucket_contents)).to eq({"test"=>"231232", "test2"=>"4564545"})
174 
+    end
175 
+  end
176 
+
177 
+  context "#client" do
178 
+    it "initializes the S3 client correctly" do
179 
+      mock_credential = mock()
180 
+      mock(Aws::Credentials).new('32343242', '1231312') { mock_credential }
181 
+      mock(Aws::S3::Client).new(credentials: mock_credential,
182 
+                                      region: 'us-east-1')
183 
+      @checker.send(:client)
184 
+    end
185 
+  end
186 
+
187 
+  context "#event_description" do
188 
+    it "should include event_type when watch is set to true" do
189 
+      @checker.options['watch'] = 'true'
190 
+      expect(@checker.event_description).to include('event_type')
191 
+    end
192 
+
193 
+    it "should not include event_type when watch is set to false" do
194 
+      @checker.options['watch'] = 'false'
195 
+      expect(@checker.event_description).not_to include('event_type')
196 
+    end
197 
+  end
198 
+
199 
+  context "#receive" do
200 
+    before(:each) do
201 
+      @checker.options['mode'] = 'write'
202 
+      @checker.options['filename'] = 'file.txt'
203 
+      @checker.options['data'] = '{{ data }}'
204 
+    end
205 
+
206 
+    it "writes the data at data into a file" do
207 
+      client_mock = mock()
208 
+      mock(client_mock).put_object(bucket: @checker.options['bucket'], key: @checker.options['filename'], body: 'hello world!')
209 
+      mock(@checker).client { client_mock }
210 
+      event = Event.new(payload: {'data' => 'hello world!'})
211 
+      @checker.receive([event])
212 
+    end
213 
+
214 
+    it "does nothing when mode is set to 'read'" do
215 
+      @checker.options['mode'] = 'read'
216 
+      event = Event.new(payload: {'data' => 'hello world!'})
217 
+      @checker.receive([event])
218 
+    end
219 
+  end
220 
+end

+ 16 - 0
spec/support/shared_examples/file_handling_consumer.rb
Datei anzeigen 

@@ -0,0 +1,16 @@
1 
+require 'rails_helper'
2 
+
3 
+shared_examples_for 'FileHandlingConsumer' do
4 
+  it 'returns a file pointer' do
5 
+    expect(@checker.get_file_pointer('testfile')).to eq(file_pointer: { file: "testfile", agent_id: @checker.id})
6 
+  end
7 
+
8 
+  it 'get_io raises an exception when trying to access an agent of a different user' do
9 
+    @checker2 = @checker.dup
10 
+    @checker2.user = users(:bob)
11 
+    @checker2.save!
12 
+    expect(@checker2.user.id).not_to eq(@checker.user.id)
13 
+    event = Event.new(user: @checker.user, payload: {'file_pointer' => {'file' => 'test', 'agent_id' => @checker2.id}})
14 
+    expect { @checker.get_io(event) }.to raise_error(ActiveRecord::RecordNotFound)
15 
+  end
16 
+end

+ 24 - 0
spec/support/shared_examples/working_helpers.rb
Datei anzeigen 

@@ -50,4 +50,28 @@ shared_examples_for WorkingHelpers do
50 50 
       expect(@agent.received_event_without_error?).to eq(true)
51 51 
     end
52 52 
   end
53 
+
54 
+  describe "checked_without_error?" do
55 
+    before do
56 
+      @agent = described_class.new
57 
+    end
58 
+
59 
+    it "should return false until the first time check ran" do
60 
+      expect(@agent.checked_without_error?).to eq(false)
61 
+      @agent.last_check_at = Time.now
62 
+      expect(@agent.checked_without_error?).to eq(true)
63 
+    end
64 
+
65 
+    it "should return false when the last error occured after the check" do
66 
+      @agent.last_check_at = Time.now - 1.minute
67 
+      @agent.last_error_log_at = Time.now
68 
+      expect(@agent.checked_without_error?).to eq(false)
69 
+    end
70 
+
71 
+    it "should return true when the last check occured after the last error" do
72 
+      @agent.last_check_at = Time.now
73 
+      @agent.last_error_log_at = Time.now - 1.minute
74 
+      expect(@agent.checked_without_error?).to eq(true)
75 
+    end
76 
+  end
53 77 
 end