Strona główna Odkrywaj Pomoc
Zarejestruj się Zaloguj się
J1X
/
huginn
Obserwuj 2
Polub 0
Fork 0

Przeglądaj źródła
Merge remote-tracking branch 'huginn/master' into chef-solo

Dominik Sander 11 lat temu
rodzic
e7fe019195 a6c1b053c1
commit
53ebf6b6aa
7 zmienionych plików z 171 dodań i 46 usunięć
Pokaż statystyki zmian
  1. 16 9
      app/models/agents/trigger_agent.rb
  2. 20 6
      app/models/agents/website_agent.rb
  3. 24 27
      deployment/Vagrantfile
  4. 1 0
      deployment/roles/huginn_development.json
  5. 10 3
      deployment/site-cookbooks/huginn_development/recipes/default.rb
  6. 86 0
      spec/models/agents/trigger_agent_spec.rb
  7. 14 1
      spec/models/agents/website_agent_spec.rb

+ 16 - 9
app/models/agents/trigger_agent.rb
Wyświetl plik 

@@ -11,6 +11,8 @@ module Agents
11 11
12 12 
       The `type` can be one of #{VALID_COMPARISON_TYPES.map { |t| "`#{t}`" }.to_sentence} and compares with the `value`.
13 13
14 
+      The `value` can be a single value or an array of values. In the case of an array, if one or more values match then the rule matches.
15 
+
14 16 
       All rules must match for the Agent to match.  The resulting Event will have a payload message of `message`.  You can include extractions in the message, for example: `I saw a bar of: <foo.bar>`
15 17
16 18 
       Set `expected_receive_period_in_days` to the maximum amount of time that you'd expect to pass between Events being received by this Agent.
 
@@ -49,25 +51,30 @@ module Agents
49 51 
       incoming_events.each do |event|
50 52 
         match = options['rules'].all? do |rule|
51 53 
           value_at_path = Utils.value_at(event['payload'], rule['path'])
52 
-          case rule['type']
54 
+          rule_values = rule['value']
55 
+          rule_values = [rule_values] unless rule_values.is_a?(Array)
56 
+
57 
+          match_found = rule_values.any? do |rule_value|
58 
+            case rule['type']
53 59 
             when "regex"
54 
-              value_at_path.to_s =~ Regexp.new(rule['value'], Regexp::IGNORECASE)
60 
+              value_at_path.to_s =~ Regexp.new(rule_value, Regexp::IGNORECASE)
55 61 
             when "!regex"
56 
-              value_at_path.to_s !~ Regexp.new(rule['value'], Regexp::IGNORECASE)
62 
+              value_at_path.to_s !~ Regexp.new(rule_value, Regexp::IGNORECASE)
57 63 
             when "field>value"
58 
-              value_at_path.to_f > rule['value'].to_f
64 
+              value_at_path.to_f > rule_value.to_f
59 65 
             when "field>=value"
60 
-              value_at_path.to_f >= rule['value'].to_f
66 
+              value_at_path.to_f >= rule_value.to_f
61 67 
             when "field<value"
62 
-              value_at_path.to_f < rule['value'].to_f
68 
+              value_at_path.to_f < rule_value.to_f
63 69 
             when "field<=value"
64 
-              value_at_path.to_f <= rule['value'].to_f
70 
+              value_at_path.to_f <= rule_value.to_f
65 71 
             when "field==value"
66 
-              value_at_path.to_s == rule['value'].to_s
72 
+              value_at_path.to_s == rule_value.to_s
67 73 
             when "field!=value"
68 
-              value_at_path.to_s != rule['value'].to_s
74 
+              value_at_path.to_s != rule_value.to_s
69 75 
             else
70 76 
               raise "Invalid type of #{rule['type']} in TriggerAgent##{id}"
77 
+            end
71 78 
           end
72 79 
         end
73 80

+ 20 - 6
app/models/agents/website_agent.rb
Wyświetl plik 

@@ -4,7 +4,6 @@ require 'date'
4 4
5 5 
 module Agents
6 6 
   class WebsiteAgent < Agent
7 
-    cannot_receive_events!
8 7
9 8 
     default_schedule "every_12h"
10 9 
 
@@ -46,6 +45,8 @@ module Agents
46 45 
       Set `uniqueness_look_back` to limit the number of events checked for uniqueness (typically for performance).  This defaults to the larger of #{UNIQUENESS_LOOK_BACK} or #{UNIQUENESS_FACTOR}x the number of detected received results.
47 46
48 47 
       Set `force_encoding` to an encoding name if the website does not return a Content-Type header with a proper charset.
48 
+
49 
+      The WebsiteAgent can also scrape based on incoming events. It will scrape the url contained in the `url` key of the incoming event payload.
49 50 
     MD
50 51
51 52 
     event_description do
 
@@ -105,19 +106,23 @@ module Agents
105 106 
     end
106 107
107 108 
     def check
108 
-      hydra = Typhoeus::Hydra.new
109 109 
       log "Fetching #{options['url']}"
110 
+      check_url options['url']
111 
+    end
112 
+
113 
+    def check_url(in_url)
114 
+      hydra = Typhoeus::Hydra.new
110 115 
       request_opts = { :followlocation => true }
111 116 
       request_opts[:userpwd] = options['basic_auth'] if options['basic_auth'].present?
112 117
113 118 
       requests = []
114 119
115 
-      if options['url'].kind_of?(Array)
116 
-        options['url'].each do |url|
120 
+      if in_url.kind_of?(Array)
121 
+        in_url.each do |url|
117 122 
            requests.push(Typhoeus::Request.new(url, request_opts))
118 123 
         end
119 124 
       else
120 
-        requests.push(Typhoeus::Request.new(options['url'], request_opts))
125 
+        requests.push(Typhoeus::Request.new(in_url, request_opts))
121 126 
       end
122 127
123 128 
       requests.each do |request|
 
@@ -185,7 +190,7 @@ module Agents
185 190 
               options['extract'].keys.each do |name|
186 191 
                 result[name] = output[name][index]
187 192 
                 if name.to_s == 'url'
188 
-                  result[name] = URI.join(options['url'], result[name]).to_s if (result[name] =~ URI::DEFAULT_PARSER.regexp[:ABS_URI]).nil?
193 
+                  result[name] = URI.join(request.base_url, result[name]).to_s if (result[name] =~ URI::DEFAULT_PARSER.regexp[:ABS_URI]).nil?
189 194 
                 end
190 195 
               end
191 196 
 
@@ -202,6 +207,13 @@ module Agents
202 207 
       end
203 208 
     end
204 209
210 
+    def receive(incoming_events)
211 
+      incoming_events.each do |event|
212 
+        url_to_scrape = event.payload['url']
213 
+        check_url(url_to_scrape) if url_to_scrape =~ /^https?:\/\//i
214 
+      end
215 
+    end
216 
+
205 217 
     private
206 218
207 219 
     # This method returns true if the result should be stored as a new event.
 
@@ -275,5 +287,7 @@ module Agents
275 287 
         false
276 288 
       end
277 289 
     end
290 
+
278 291 
   end
292 
+
279 293 
 end

+ 24 - 27
deployment/Vagrantfile
Wyświetl plik 

@@ -3,37 +3,34 @@
3 3
4 4 
 Vagrant.configure("2") do |config|
5 5 
   config.omnibus.chef_version = :latest
6 
-  config.vm.define :vb do |vb|
7 
-    vb.vm.box = "precise32"
8 
-    vb.vm.box_url = "http://files.vagrantup.com/precise32.box"
9 
-    vb.vm.network :forwarded_port, host: 3000, guest: 3000
10 6
11 
-    vb.vm.provision :chef_solo do |chef|
12 
-      chef.roles_path = "roles"
13 
-      chef.cookbooks_path = ["cookbooks", "site-cookbooks"]
14 
-      chef.add_role("huginn_development")
15 
-    end
7 
+  config.vm.provision :chef_solo do |chef|
8 
+    chef.roles_path = "roles"
9 
+    chef.cookbooks_path = ["cookbooks", "site-cookbooks"]
10 
+    chef.add_role("huginn_development")
11 
+    # chef.add_role("huginn_production")
16 12 
   end
17 13
18 
-  config.vm.define :ec2 do |ec2|
19 
-    ec2.vm.box = "dummy"
20 
-    ec2.vm.box_url = "https://github.com/mitchellh/vagrant-aws/raw/master/dummy.box"
14 
+  config.vm.provider :virtualbox do |vb, override|
15 
+    override.vm.box = "hashicorp/precise64"
16 
+    override.vm.network :forwarded_port, host: 3000, guest: 3000
17 
+  end
18 
+
19 
+  config.vm.provider :parallels do |prl, override|
20 
+    override.vm.box = "parallels/ubuntu-12.04"
21 
+  end
22 
+
23 
+  config.vm.provider :aws do |aws, override|
24 
+    override.vm.box = "dummy"
25 
+    override.vm.box_url = "https://github.com/mitchellh/vagrant-aws/raw/master/dummy.box"
21 26
22 
-    ec2.vm.provider :aws do |aws, override|
23 
-      aws.access_key_id = ""
24 
-      aws.secret_access_key = ""
25 
-      aws.keypair_name = ""
26 
-      aws.region = "us-east-1"
27 
-      aws.ami = "ami-d0f89fb9"
27 
+    aws.access_key_id = ""
28 
+    aws.secret_access_key = ""
29 
+    aws.keypair_name = ""
30 
+    aws.region = "us-east-1"
31 
+    aws.ami = "ami-d0f89fb9"
28 32
29 
-      override.ssh.username = "ubuntu"
30 
-      override.ssh.private_key_path = ""
31 
-    end
32 
-    ec2.vm.provision :chef_solo do |chef|
33 
-      chef.roles_path = "roles"
34 
-      chef.cookbooks_path = ["cookbooks", "site-cookbooks"]
35 
-      chef.add_role("huginn_production")
36 
-
37 
-    end
33 
+    override.ssh.username = "ubuntu"
34 
+    override.ssh.private_key_path = ""
38 35 
   end
39 36 
 end

+ 1 - 0
deployment/roles/huginn_development.json
Wyświetl plik 

@@ -23,6 +23,7 @@
23 23 
              "recipe[git]",
24 24 
              "recipe[apt]",
25 25 
              "recipe[mysql::server]",
26 
+             "recipe[mysql::client]",
26 27 
              "recipe[nodejs::install_from_binary]",
27 28 
              "recipe[huginn_development]"
28 29 
            ]

+ 10 - 3
deployment/site-cookbooks/huginn_development/recipes/default.rb
Wyświetl plik 

@@ -16,12 +16,19 @@ group "huginn" do
16 16 
   action :create
17 17 
 end
18 18
19 
-%w("ruby1.9.1" "ruby1.9.1-dev" "libxslt-dev" "libxml2-dev" "curl" "libmysqlclient-dev").each do |pkg|
19 
+%w("ruby1.9.1" "ruby1.9.1-dev" "libxslt-dev" "libxml2-dev" "curl" "libmysqlclient-dev" "rubygems").each do |pkg|
20 20 
   package pkg do
21 21 
     action :install
22 22 
   end
23 23 
 end
24 24
25 
+bash "Setting default ruby version to 1.9" do
26 
+  code <<-EOH
27 
+    update-alternatives --set ruby /usr/bin/ruby1.9.1
28 
+    update-alternatives --set gem /usr/bin/gem1.9.1
29 
+  EOH
30 
+end
31 
+
25 32 
 git "/home/huginn/huginn" do
26 33 
   repository 'git://github.com/cantino/huginn.git'
27 34 
   reference 'master'
 
@@ -48,7 +55,7 @@ bash "huginn dependencies" do
48 55 
     export LANG="en_US.UTF-8"
49 56 
     export LC_ALL="en_US.UTF-8"
50 57 
     sudo bundle install
51 
-    sed s/REPLACE_ME_NOW\!/$(sudo rake secret)/ .env.example > .env
58 
+    sed s/REPLACE_ME_NOW\!/$(sudo bundle exec rake secret)/ .env.example > .env
52 59 
     sudo bundle exec rake db:create
53 60 
     sudo bundle exec rake db:migrate
54 61 
     sudo bundle exec rake db:seed
 
@@ -59,6 +66,6 @@ bash "huginn has been installed and will start in a minute" do
59 66 
   user "huginn"
60 67 
   cwd "/home/huginn/huginn"
61 68 
   code <<-EOH
62 
-    sudo foreman start
69 
+    sudo nohup foreman start &
63 70 
     EOH
64 71 
 end

+ 86 - 0
spec/models/agents/trigger_agent_spec.rb
Wyświetl plik 

@@ -71,6 +71,28 @@ describe Agents::TriggerAgent do
71 71 
       }.should change { Event.count }.by(1)
72 72 
     end
73 73
74 
+    it "handles array of regex" do
75 
+      @event.payload['foo']['bar']['baz'] = "a222b"
76 
+      @checker.options['rules'][0] = {
77 
+        'type' => "regex",
78 
+        'value' => ["a\\db", "a\\Wb"],
79 
+        'path' => "foo.bar.baz",
80 
+      }
81 
+      lambda {
82 
+        @checker.receive([@event])
83 
+      }.should_not change { Event.count }
84 
+
85 
+      @event.payload['foo']['bar']['baz'] = "a2b"
86 
+      lambda {
87 
+        @checker.receive([@event])
88 
+      }.should change { Event.count }.by(1)
89 
+
90 
+      @event.payload['foo']['bar']['baz'] = "a b"
91 
+      lambda {
92 
+        @checker.receive([@event])
93 
+      }.should change { Event.count }.by(1)
94 
+    end
95 
+
74 96 
     it "handles negated regex" do
75 97 
       @event.payload['foo']['bar']['baz'] = "a2b"
76 98 
       @checker.options['rules'][0] = {
 
@@ -89,6 +111,24 @@ describe Agents::TriggerAgent do
89 111 
       }.should change { Event.count }.by(1)
90 112 
     end
91 113
114 
+    it "handles array of negated regex" do
115 
+      @event.payload['foo']['bar']['baz'] = "a2b"
116 
+      @checker.options['rules'][0] = {
117 
+        'type' => "!regex",
118 
+        'value' => ["a\\db", "a2b"],
119 
+        'path' => "foo.bar.baz",
120 
+      }
121 
+
122 
+      lambda {
123 
+        @checker.receive([@event])
124 
+      }.should_not change { Event.count }
125 
+
126 
+      @event.payload['foo']['bar']['baz'] = "a3b"
127 
+      lambda {
128 
+        @checker.receive([@event])
129 
+      }.should change { Event.count }.by(1)
130 
+    end
131 
+
92 132 
     it "puts can extract values into the message based on paths" do
93 133 
       @checker.receive([@event])
94 134 
       Event.last.payload['message'].should == "I saw 'a2b' from Joe"
 
@@ -109,6 +149,21 @@ describe Agents::TriggerAgent do
109 149 
       }.should_not change { Event.count }
110 150 
     end
111 151
152 
+    it "handles array of numerical comparisons" do
153 
+      @event.payload['foo']['bar']['baz'] = "5"
154 
+      @checker.options['rules'].first['value'] = [6, 3]
155 
+      @checker.options['rules'].first['type'] = "field<value"
156 
+
157 
+      lambda {
158 
+        @checker.receive([@event])
159 
+      }.should change { Event.count }.by(1)
160 
+
161 
+      @checker.options['rules'].first['value'] = [4, 3]
162 
+      lambda {
163 
+        @checker.receive([@event])
164 
+      }.should_not change { Event.count }
165 
+    end
166 
+
112 167 
     it "handles exact comparisons" do
113 168 
       @event.payload['foo']['bar']['baz'] = "hello world"
114 169 
       @checker.options['rules'].first['type'] = "field==value"
 
@@ -124,6 +179,21 @@ describe Agents::TriggerAgent do
124 179 
       }.should change { Event.count }.by(1)
125 180 
     end
126 181
182 
+    it "handles array of exact comparisons" do
183 
+      @event.payload['foo']['bar']['baz'] = "hello world"
184 
+      @checker.options['rules'].first['type'] = "field==value"
185 
+
186 
+      @checker.options['rules'].first['value'] = ["hello there", "hello universe"]
187 
+      lambda {
188 
+        @checker.receive([@event])
189 
+      }.should_not change { Event.count }
190 
+
191 
+      @checker.options['rules'].first['value'] = ["hello world", "hello universe"]
192 
+      lambda {
193 
+        @checker.receive([@event])
194 
+      }.should change { Event.count }.by(1)
195 
+    end
196 
+
127 197 
     it "handles negated comparisons" do
128 198 
       @event.payload['foo']['bar']['baz'] = "hello world"
129 199 
       @checker.options['rules'].first['type'] = "field!=value"
 
@@ -140,6 +210,22 @@ describe Agents::TriggerAgent do
140 210 
       }.should change { Event.count }.by(1)
141 211 
     end
142 212
213 
+    it "handles array of negated comparisons" do
214 
+      @event.payload['foo']['bar']['baz'] = "hello world"
215 
+      @checker.options['rules'].first['type'] = "field!=value"
216 
+      @checker.options['rules'].first['value'] = ["hello world", "hello world"]
217 
+
218 
+      lambda {
219 
+        @checker.receive([@event])
220 
+      }.should_not change { Event.count }
221 
+
222 
+      @checker.options['rules'].first['value'] = ["hello there", "hello world"]
223 
+
224 
+      lambda {
225 
+        @checker.receive([@event])
226 
+      }.should change { Event.count }.by(1)
227 
+    end
228 
+
143 229 
     it "does fine without dots in the path" do
144 230 
       @event.payload = { 'hello' => "world" }
145 231 
       @checker.options['rules'].first['type'] = "field==value"

+ 14 - 1
spec/models/agents/website_agent_spec.rb
Wyświetl plik 

@@ -331,6 +331,19 @@ describe Agents::WebsiteAgent do
331 331 
         end
332 332 
       end
333 333 
     end
334 
+
335 
+    describe "#receive" do
336 
+      it "should scrape from the url element in incoming event payload" do
337 
+        @event = Event.new
338 
+        @event.agent = agents(:bob_rain_notifier_agent)
339 
+        @event.payload = { 'url' => "http://xkcd.com" }
340 
+
341 
+        lambda {
342 
+          @checker.options = @site
343 
+          @checker.receive([@event])
344 
+        }.should change { Event.count }.by(1)
345 
+      end
346 
+    end
334 347 
   end
335 348
336 349 
   describe "checking with http basic auth" do
 
@@ -361,4 +374,4 @@ describe Agents::WebsiteAgent do
361 374 
       end
362 375 
     end
363 376 
   end
364 
-end
377 
+end