Няма описание http://j1x-huginn.herokuapp.com

ftpsite_agent.rb 5.3KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215
  1. require 'net/ftp'
  2. require 'uri'
  3. require 'time'
  4. module Agents
  5. class FtpsiteAgent < Agent
  6. cannot_receive_events!
  7. default_schedule "every_12h"
  8. description <<-MD
  9. The FtpsiteAgent checks a FTP site and creates Events based on newly uploaded files in a directory.
  10. Specify a `url` that represents a directory of an FTP site to watch, and a list of `patterns` to match against file names.
  11. Login credentials can be included in `url` if authentication is required.
  12. Only files with a last modification time later than the `after` value, if specifed, are notified.
  13. MD
  14. event_description <<-MD
  15. Events look like this:
  16. {
  17. "url": "ftp://example.org/pub/releases/foo-1.2.tar.gz",
  18. "filename": "foo-1.2.tar.gz",
  19. "timestamp": "2014-04-10T22:50:00Z"
  20. }
  21. MD
  22. def working?
  23. event_created_within?(options['expected_update_period_in_days']) && !recent_error_logs?
  24. end
  25. def default_options
  26. {
  27. 'expected_update_period_in_days' => "1",
  28. 'url' => "ftp://example.org/pub/releases/",
  29. 'patterns' => [
  30. 'foo-*.tar.gz',
  31. ],
  32. 'after' => Time.now.iso8601,
  33. }
  34. end
  35. def validate_options
  36. # Check for required fields
  37. begin
  38. url = options['url']
  39. String === url or raise
  40. uri = URI(url)
  41. URI::FTP === uri or raise
  42. errors.add(:base, "url must end with a slash") unless uri.path.end_with?('/')
  43. rescue
  44. errors.add(:base, "url must be a valid FTP URL")
  45. end
  46. patterns = options['patterns']
  47. case patterns
  48. when Array
  49. if patterns.empty?
  50. errors.add(:base, "patterns must not be empty")
  51. end
  52. when nil, ''
  53. errors.add(:base, "patterns must be specified")
  54. else
  55. errors.add(:base, "patterns must be an array")
  56. end
  57. # Check for optional fields
  58. if (timestamp = options['timestamp']).present?
  59. begin
  60. Time.parse(timestamp)
  61. rescue
  62. errors.add(:base, "timestamp cannot be parsed as time")
  63. end
  64. end
  65. if options['expected_update_period_in_days'].present?
  66. errors.add(:base, "Invalid expected_update_period_in_days format") unless is_positive_integer?(options['expected_update_period_in_days'])
  67. end
  68. end
  69. def check
  70. saving_entries do |found|
  71. each_entry { |filename, mtime|
  72. found[filename, mtime]
  73. }
  74. end
  75. end
  76. def each_entry
  77. patterns = options['patterns']
  78. after =
  79. if str = options['after']
  80. Time.parse(str)
  81. else
  82. Time.at(0)
  83. end
  84. open_ftp(base_uri) do |ftp|
  85. log "Listing the directory"
  86. # Do not use a block style call because we need to call other
  87. # commands during iteration.
  88. list = ftp.list('-a')
  89. month2year = {}
  90. list.each do |line|
  91. mon, day, smtn, rest = line.split(' ', 9)[5..-1]
  92. # Remove symlink target part if any
  93. filename = rest[/\A(.+?)(?:\s+->\s|\z)/, 1]
  94. patterns.any? { |pattern|
  95. File.fnmatch?(pattern, filename)
  96. } or next
  97. case smtn
  98. when /:/
  99. if year = month2year[mon]
  100. mtime = Time.parse("#{mon} #{day} #{year} #{smtn} GMT")
  101. else
  102. log "Getting mtime of #{filename}"
  103. mtime = ftp.mtime(filename)
  104. month2year[mon] = mtime.year
  105. end
  106. else
  107. # Do not bother calling MDTM for old files. Losing the
  108. # time part only makes a timestamp go backwards, meaning
  109. # that it will trigger no new event.
  110. mtime = Time.parse("#{mon} #{day} #{smtn} GMT")
  111. end
  112. after < mtime or next
  113. yield filename, mtime
  114. end
  115. end
  116. end
  117. def open_ftp(uri)
  118. ftp = Net::FTP.new
  119. log "Connecting to #{uri.host}#{':%d' % uri.port if uri.port != uri.default_port}"
  120. ftp.connect(uri.host, uri.port)
  121. user =
  122. if str = uri.user
  123. URI.decode(str)
  124. else
  125. 'anonymous'
  126. end
  127. password =
  128. if str = uri.password
  129. URI.decode(str)
  130. else
  131. 'anonymous@'
  132. end
  133. log "Logging in as #{user}"
  134. ftp.login(user, password)
  135. ftp.passive = true
  136. path = uri.path.chomp('/')
  137. log "Changing directory to #{path}"
  138. ftp.chdir(path)
  139. yield ftp
  140. ensure
  141. log "Closing the connection"
  142. ftp.close
  143. end
  144. def base_uri
  145. @base_uri ||= URI(options['url'])
  146. end
  147. def saving_entries
  148. known_entries = memory['known_entries'] || {}
  149. found_entries = {}
  150. new_files = []
  151. yield proc { |filename, mtime|
  152. found_entries[filename] = misotime = mtime.utc.iso8601
  153. unless (prev = known_entries[filename]) && misotime <= prev
  154. new_files << filename
  155. end
  156. }
  157. new_files.sort_by { |filename|
  158. found_entries[filename]
  159. }.each { |filename|
  160. create_event :payload => {
  161. 'url' => (base_uri + filename).to_s,
  162. 'filename' => filename,
  163. 'timestamp' => found_entries[filename],
  164. }
  165. }
  166. memory['known_entries'] = found_entries
  167. save!
  168. end
  169. private
  170. def is_positive_integer?(value)
  171. Integer(value) >= 0
  172. rescue
  173. false
  174. end
  175. end
  176. end