s3_agent.rb 6.3KB

    module Agents class S3Agent < Agent include FormConfigurable include FileHandling emits_file_pointer! no_bulk_receive! default_schedule 'every_1h' gem_dependency_check { defined?(Aws::S3) } description do <<-MD The S3Agent can watch a bucket for changes or emit an event for every file in that bucket. When receiving events, it writes the data into a file on S3. #{'## Include `aws-sdk-core` in your Gemfile to use this Agent!' if dependencies_missing?} `mode` must be present and either `read` or `write`, in `read` mode the agent checks the S3 bucket for changed files, with `write` it writes received events to a file in the bucket. ### Universal options To use credentials for the `access_key` and `access_key_secret` use the liquid `credential` tag like so `{% credential name-of-credential %}` Select the `region` in which the bucket was created. ### Reading When `watch` is set to `true` the S3Agent will watch the specified `bucket` for changes. An event will be emitted for every detected change. When `watch` is set to `false` the agent will emit an event for every file in the bucket on each sheduled run. #{emitting_file_handling_agent_description} ### Writing Specify the filename to use in `filename`, Liquid interpolation is possible to change the name per event. Use [Liquid](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) templating in `data` to specify which part of the received event should be written. MD end event_description do "Events will looks like this:\n\n %s" % if boolify(interpolated['watch']) Utils.pretty_print({ "file_pointer" => { "file" => "filename", "agent_id" => id }, "event_type" => "modified/added/removed" }) else Utils.pretty_print({ "file_pointer" => { "file" => "filename", "agent_id" => id } }) end end def default_options { 'mode' => 'read', 'access_key_id' => '', 'access_key_secret' => '', 'watch' => 'true', 'bucket' => "", 'data' => '{{ data }}' } end form_configurable :mode, type: :array, values: %w(read write) form_configurable :access_key_id, roles: :validatable form_configurable :access_key_secret, roles: :validatable form_configurable :region, type: :array, values: %w(us-east-1 us-west-1 us-west-2 eu-west-1 eu-central-1 ap-southeast-1 ap-southeast-2 ap-northeast-1 ap-northeast-2 sa-east-1) form_configurable :watch, type: :array, values: %w(true false) form_configurable :bucket, roles: :completable form_configurable :filename form_configurable :data def validate_options if options['mode'].blank? || !['read', 'write'].include?(options['mode']) errors.add(:base, "The 'mode' option is required and must be set to 'read' or 'write'") end if options['bucket'].blank? errors.add(:base, "The 'bucket' option is required.") end if options['region'].blank? errors.add(:base, "The 'region' option is required.") end case interpolated['mode'] when 'read' if options['watch'].blank? || ![true, false].include?(boolify(options['watch'])) errors.add(:base, "The 'watch' option is required and must be set to 'true' or 'false'") end when 'write' if options['filename'].blank? errors.add(:base, "filename must be specified in 'write' mode") end if options['data'].blank? errors.add(:base, "data must be specified in 'write' mode") end end end def validate_access_key_id !!buckets end def validate_access_key_secret !!buckets end def complete_bucket (buckets || []).collect { |room| {text: room.name, id: room.name} } end def working? checked_without_error? end def check return if interpolated['mode'] != 'read' contents = safely do get_bucket_contents end if boolify(interpolated['watch']) watch(contents) else contents.each do |key, _| create_event payload: get_file_pointer(key) end end end def get_io(file) client.get_object(bucket: interpolated['bucket'], key: file).body end def receive(incoming_events) return if interpolated['mode'] != 'write' incoming_events.each do |event| safely do mo = interpolated(event) client.put_object(bucket: mo['bucket'], key: mo['filename'], body: mo['data']) end end end private def safely yield rescue Aws::S3::Errors::AccessDenied => e error("Could not access '#{interpolated['bucket']}' #{e.class} #{e.message}") rescue Aws::S3::Errors::ServiceError =>e error("#{e.class}: #{e.message}") end def watch(contents) if last_check_at.nil? self.memory['seen_contents'] = contents return end new_memory = contents.dup memory['seen_contents'].each do |key, etag| if contents[key].blank? create_event payload: get_file_pointer(key).merge(event_type: :removed) elsif contents[key] != etag create_event payload: get_file_pointer(key).merge(event_type: :modified) end contents.delete(key) end contents.each do |key, etag| create_event payload: get_file_pointer(key).merge(event_type: :added) end self.memory['seen_contents'] = new_memory end def get_bucket_contents contents = {} client.list_objects(bucket: interpolated['bucket']).each do |response| response.contents.each do |file| contents[file.key] = file.etag end end contents end def client @client ||= Aws::S3::Client.new(credentials: Aws::Credentials.new(interpolated['access_key_id'], interpolated['access_key_secret']), region: interpolated['region']) end def buckets(log = false) @buckets ||= client.list_buckets.buckets rescue Aws::S3::Errors::ServiceError => e false end end end