Restart TwitterStreamAgent::Worker every hour to prevent stalling
Dominik Sander
5 changed files with 51 additions and 9 deletions
+ 11
- 1
app/concerns/long_runnable.rb
@@ -51,7 +51,7 @@ module LongRunnable |
||
| 51 | 51 |
end |
| 52 | 52 |
|
| 53 | 53 |
class Worker |
| 54 |
- attr_reader :thread, :id, :agent, :config, :mutex |
|
| 54 |
+ attr_reader :thread, :id, :agent, :config, :mutex, :scheduler |
|
| 55 | 55 |
|
| 56 | 56 |
def initialize(options = {})
|
| 57 | 57 |
@id = options[:id] |
@@ -92,6 +92,12 @@ module LongRunnable |
||
| 92 | 92 |
end |
| 93 | 93 |
end |
| 94 | 94 |
|
| 95 |
+ def restart! |
|
| 96 |
+ stop! |
|
| 97 |
+ setup!(scheduler, mutex) |
|
| 98 |
+ run! |
|
| 99 |
+ end |
|
| 100 |
+ |
|
| 95 | 101 |
def every(*args, &blk) |
| 96 | 102 |
schedule(:every, args, &blk) |
| 97 | 103 |
end |
@@ -100,6 +106,10 @@ module LongRunnable |
||
| 100 | 106 |
schedule(:cron, args, &blk) |
| 101 | 107 |
end |
| 102 | 108 |
|
| 109 |
+ def schedule_in(*args, &blk) |
|
| 110 |
+ schedule(:schedule_in, args, &blk) |
|
| 111 |
+ end |
|
| 112 |
+ |
|
| 103 | 113 |
def boolify(value) |
| 104 | 114 |
agent.send(:boolify, value) |
| 105 | 115 |
end |
+ 8
- 3
app/models/agents/twitter_stream_agent.rb
@@ -150,13 +150,18 @@ module Agents |
||
| 150 | 150 |
end |
| 151 | 151 |
|
| 152 | 152 |
class Worker < LongRunnable::Worker |
| 153 |
- RELOAD_TIMEOUT = 10.minutes |
|
| 153 |
+ RELOAD_TIMEOUT = 60.minutes |
|
| 154 | 154 |
DUPLICATE_DETECTION_LENGTH = 1000 |
| 155 | 155 |
SEPARATOR = /[^\w_\-]+/ |
| 156 | 156 |
|
| 157 | 157 |
def setup |
| 158 | 158 |
require 'twitter/json_stream' |
| 159 | 159 |
@filter_to_agent_map = @config[:filter_to_agent_map] |
| 160 |
+ |
|
| 161 |
+ schedule_in RELOAD_TIMEOUT do |
|
| 162 |
+ puts "--> Restarting TwitterStream #{id}"
|
|
| 163 |
+ restart! |
|
| 164 |
+ end |
|
| 160 | 165 |
end |
| 161 | 166 |
|
| 162 | 167 |
def run |
@@ -199,13 +204,13 @@ module Agents |
||
| 199 | 204 |
|
| 200 | 205 |
stream.on_no_data do |message| |
| 201 | 206 |
STDERR.puts " --> Got no data for awhile; trying to reconnect." |
| 202 |
- stop |
|
| 207 |
+ restart! |
|
| 203 | 208 |
end |
| 204 | 209 |
|
| 205 | 210 |
stream.on_max_reconnects do |timeout, retries| |
| 206 | 211 |
STDERR.puts " --> Oops, tried too many times! <--" |
| 207 | 212 |
sleep 60 |
| 208 |
- stop |
|
| 213 |
+ restart! |
|
| 209 | 214 |
end |
| 210 | 215 |
end |
| 211 | 216 |
|
+ 1
- 1
lib/agent_runner.rb
@@ -16,7 +16,7 @@ class AgentRunner |
||
| 16 | 16 |
@mutex = Mutex.new |
| 17 | 17 |
@scheduler = Rufus::Scheduler.new(frequency: ENV['SCHEDULER_FREQUENCY'].presence || 0.3) |
| 18 | 18 |
|
| 19 |
- @scheduler.every 1 do |
|
| 19 |
+ @scheduler.every 5 do |
|
| 20 | 20 |
restart_dead_workers if @running |
| 21 | 21 |
end |
| 22 | 22 |
|
+ 27
- 1
spec/concerns/long_runnable_spec.rb
@@ -40,7 +40,7 @@ describe LongRunnable do |
||
| 40 | 40 |
describe LongRunnable::Worker do |
| 41 | 41 |
before(:each) do |
| 42 | 42 |
@agent = Object.new |
| 43 |
- @worker = LongRunnable::Worker.new(agent: @agent) |
|
| 43 |
+ @worker = LongRunnable::Worker.new(agent: @agent, id: 'test1234') |
|
| 44 | 44 |
@worker.setup!(Rufus::Scheduler.new, Mutex.new) |
| 45 | 45 |
end |
| 46 | 46 |
|
@@ -84,5 +84,31 @@ describe LongRunnable do |
||
| 84 | 84 |
@worker.stop! |
| 85 | 85 |
end |
| 86 | 86 |
end |
| 87 |
+ |
|
| 88 |
+ context "#restart!" do |
|
| 89 |
+ it "stops, setups and starts the worker" do |
|
| 90 |
+ mock(@worker).stop! |
|
| 91 |
+ mock(@worker).setup!(@worker.scheduler, @worker.mutex) |
|
| 92 |
+ mock(@worker).run! |
|
| 93 |
+ @worker.restart! |
|
| 94 |
+ end |
|
| 95 |
+ end |
|
| 96 |
+ |
|
| 97 |
+ context "scheduling" do |
|
| 98 |
+ it "schedules tasks once" do |
|
| 99 |
+ mock(@worker.scheduler).send(:schedule_in, 1.hour, tag: 'test1234') |
|
| 100 |
+ @worker.schedule_in 1.hour do noop; end |
|
| 101 |
+ end |
|
| 102 |
+ |
|
| 103 |
+ it "schedules repeating tasks" do |
|
| 104 |
+ mock(@worker.scheduler).send(:every, 1.hour, tag: 'test1234') |
|
| 105 |
+ @worker.every 1.hour do noop; end |
|
| 106 |
+ end |
|
| 107 |
+ |
|
| 108 |
+ it "allows the cron syntax" do |
|
| 109 |
+ mock(@worker.scheduler).send(:cron, '0 * * * *', tag: 'test1234') |
|
| 110 |
+ @worker.cron '0 * * * *' do noop; end |
|
| 111 |
+ end |
|
| 112 |
+ end |
|
| 87 | 113 |
end |
| 88 | 114 |
end |
+ 4
- 3
spec/models/agents/twitter_stream_agent_spec.rb
@@ -169,7 +169,8 @@ describe Agents::TwitterStreamAgent do |
||
| 169 | 169 |
@config = {agent: @agent, config: {filter_to_agent_map: {'agent' => [@mock_agent]}}}
|
| 170 | 170 |
@worker = Agents::TwitterStreamAgent::Worker.new(@config) |
| 171 | 171 |
@worker.instance_variable_set(:@recent_tweets, []) |
| 172 |
- @worker.setup |
|
| 172 |
+ mock(@worker).schedule_in(Agents::TwitterStreamAgent::Worker::RELOAD_TIMEOUT) |
|
| 173 |
+ @worker.setup!(nil, Mutex.new) |
|
| 173 | 174 |
end |
| 174 | 175 |
|
| 175 | 176 |
context "#run" do |
@@ -226,7 +227,7 @@ describe Agents::TwitterStreamAgent do |
||
| 226 | 227 |
|
| 227 | 228 |
it "stop when no data was received"do |
| 228 | 229 |
stub_without(:on_no_data).on_no_data.yields |
| 229 |
- mock(@worker).stop |
|
| 230 |
+ mock(@worker).restart! |
|
| 230 | 231 |
mock(STDERR).puts(" --> Got no data for awhile; trying to reconnect.")
|
| 231 | 232 |
@worker.send(:stream!, ['agent'], @agent) |
| 232 | 233 |
end |
@@ -235,7 +236,7 @@ describe Agents::TwitterStreamAgent do |
||
| 235 | 236 |
stub_without(:on_max_reconnects).on_max_reconnects.yields |
| 236 | 237 |
mock(STDERR).puts(" --> Oops, tried too many times! <--")
|
| 237 | 238 |
mock(@worker).sleep(60) |
| 238 |
- mock(@worker).stop |
|
| 239 |
+ mock(@worker).restart! |
|
| 239 | 240 |
@worker.send(:stream!, ['agent'], @agent) |
| 240 | 241 |
end |
| 241 | 242 |
|