app/models/agents/website_agent.rb

@@ -78,6 +78,8 @@ module Agents

                 
              

                       Set `disable_ssl_verification` to `true` to disable ssl verification.
              

                 
              

                +      Set `unzip` to `gzip` to inflate the resource using gzip.
              

                +
              

                       The WebsiteAgent can also scrape based on incoming events. It will scrape the url contained in the `url` key of the incoming event payload. If you specify `merge` as the mode, it will retain the old payload and update it with the new values.
              

                 
              

                       In Liquid templating, the following variable is available:
              
@@ -174,6 +176,9 @@ module Agents

                         if (encoding = interpolated['force_encoding']).present?
              

                           body = body.encode(Encoding::UTF_8, encoding)
              

                         end
              

                +        if interpolated['unzip'] == "gzip"
              

                +          body = ActiveSupport::Gzip.decompress(body)
              

                +        end
              

                         doc = parse(body)
              

                 
              

                         if extract_full_json?
              

spec/models/agents/website_agent_spec.rb

@@ -152,6 +152,38 @@ describe Agents::WebsiteAgent do

                       end
              

                     end
              

                 
              

                +    describe 'unzipping' do
              

                +      it 'should unzip with unzip option' do
              

                +        json = {
              

                +          'response' => {
              

                +            'version' => 2,
              

                +            'title' => "hello!"
              

                +          }
              

                +        }
              

                +        zipped = ActiveSupport::Gzip.compress(json.to_json)
              

                +        stub_request(:any, /gzip/).to_return(:body => zipped, :status => 200)
              

                +        site = {
              

                +          'name' => "Some JSON Response",
              

                +          'expected_update_period_in_days' => "2",
              

                +          'type' => "json",
              

                +          'url' => "http://gzip.com",
              

                +          'mode' => 'on_change',
              

                +          'extract' => {
              

                +            'version' => { 'path' => 'response.version' },
              

                +          },
              

                +          'unzip' => 'gzip',
              

                +        }
              

                +        checker = Agents::WebsiteAgent.new(:name => "Weather Site", :options => site)
              

                +        checker.user = users(:bob)
              

                +        checker.save!
              

                +
              

                +        checker.check
              

                +        event = Event.last
              

                +        puts event.payload
              

                +        expect(event.payload['version']).to eq(2)
              

                +      end
              

                +    end
              

                +
              

                     describe 'encoding' do
              

                       it 'should be forced with force_encoding option' do
              

                         huginn = "\u{601d}\u{8003}"