huginn/spec/models/agents/human_task_agent_spec.rb

require 'spec_helper'

describe Agents::HumanTaskAgent do
  before do
    @checker = Agents::HumanTaskAgent.new(:name => "my human task agent")
    @checker.options = @checker.default_options
    @checker.user = users(:bob)
    @checker.save!

    @event = Event.new
    @event.agent = agents(:bob_rain_notifier_agent)
    @event.payload = { :foo => { "bar" => { :baz => "a2b" } },
                       :name => "Joe" }
    @event.id = 345

    @checker.should be_valid
  end

  describe "validations" do
    it "validates that trigger_on is 'schedule' or 'event'" do
      @checker.options[:trigger_on] = "foo"
      @checker.should_not be_valid
    end

    it "requires expected_receive_period_in_days when trigger_on is set to 'event'" do
      @checker.options[:trigger_on] = "event"
      @checker.options[:expected_receive_period_in_days] = nil
      @checker.should_not be_valid
      @checker.options[:expected_receive_period_in_days] = 2
      @checker.should be_valid
    end

    it "requires a positive submission_period when trigger_on is set to 'schedule'" do
      @checker.options[:trigger_on] = "schedule"
      @checker.options[:submission_period] = nil
      @checker.should_not be_valid
      @checker.options[:submission_period] = 2
      @checker.should be_valid
    end

    it "requires a hit.title" do
      @checker.options[:hit][:title] = ""
      @checker.should_not be_valid
    end

    it "requires a hit.description" do
      @checker.options[:hit][:description] = ""
      @checker.should_not be_valid
    end

    it "requires hit.assignments" do
      @checker.options[:hit][:assignments] = ""
      @checker.should_not be_valid
      @checker.options[:hit][:assignments] = 0
      @checker.should_not be_valid
      @checker.options[:hit][:assignments] = "moose"
      @checker.should_not be_valid
      @checker.options[:hit][:assignments] = "2"
      @checker.should be_valid
    end

    it "requires hit.questions" do
      old_questions = @checker.options[:hit][:questions]
      @checker.options[:hit][:questions] = nil
      @checker.should_not be_valid
      @checker.options[:hit][:questions] = []
      @checker.should_not be_valid
      @checker.options[:hit][:questions] = [old_questions[0]]
      @checker.should be_valid
    end

    it "requires that all questions have key, name, required, type, and question" do
      old_questions = @checker.options[:hit][:questions]
      @checker.options[:hit][:questions].first[:key] = ""
      @checker.should_not be_valid

      @checker.options[:hit][:questions] = old_questions
      @checker.options[:hit][:questions].first[:name] = ""
      @checker.should_not be_valid

      @checker.options[:hit][:questions] = old_questions
      @checker.options[:hit][:questions].first[:required] = nil
      @checker.should_not be_valid

      @checker.options[:hit][:questions] = old_questions
      @checker.options[:hit][:questions].first[:type] = ""
      @checker.should_not be_valid

      @checker.options[:hit][:questions] = old_questions
      @checker.options[:hit][:questions].first[:question] = ""
      @checker.should_not be_valid
    end

    it "requires that all questions of type 'selection' have a selections array with keys and text" do
      @checker.options[:hit][:questions][0][:selections] = []
      @checker.should_not be_valid
      @checker.options[:hit][:questions][0][:selections] = [{}]
      @checker.should_not be_valid
      @checker.options[:hit][:questions][0][:selections] = [{ :key => "", :text => "" }]
      @checker.should_not be_valid
      @checker.options[:hit][:questions][0][:selections] = [{ :key => "", :text => "hi" }]
      @checker.should_not be_valid
      @checker.options[:hit][:questions][0][:selections] = [{ :key => "hi", :text => "" }]
      @checker.should_not be_valid
      @checker.options[:hit][:questions][0][:selections] = [{ :key => "hi", :text => "hi" }]
      @checker.should be_valid
      @checker.options[:hit][:questions][0][:selections] = [{ :key => "hi", :text => "hi" }, {}]
      @checker.should_not be_valid
    end

    it "requires that 'poll_options' be present and populated when 'combination_mode' is set to 'poll'" do
      @checker.options[:combination_mode] = "poll"
      @checker.should_not be_valid
      @checker.options[:poll_options] = {}
      @checker.should_not be_valid
      @checker.options[:poll_options] = { :title => "Take a poll about jokes",
                                          :instructions => "Rank these by how funny they are",
                                          :assignments => 3,
                                          :row_template => "<$.joke>" }
      @checker.should be_valid
      @checker.options[:poll_options] = { :instructions => "Rank these by how funny they are",
                                          :assignments => 3,
                                          :row_template => "<$.joke>" }
      @checker.should_not be_valid
      @checker.options[:poll_options] = { :title => "Take a poll about jokes",
                                          :assignments => 3,
                                          :row_template => "<$.joke>" }
      @checker.should_not be_valid
      @checker.options[:poll_options] = { :title => "Take a poll about jokes",
                                          :instructions => "Rank these by how funny they are",
                                          :row_template => "<$.joke>" }
      @checker.should_not be_valid
      @checker.options[:poll_options] = { :title => "Take a poll about jokes",
                                          :instructions => "Rank these by how funny they are",
                                          :assignments => 3}
      @checker.should_not be_valid
    end

    it "requires that all questions be of type 'selection' when 'combination_mode' is 'take_majority'" do
      @checker.options[:combination_mode] = "take_majority"
      @checker.should_not be_valid
      @checker.options[:hit][:questions][1][:type] = "selection"
      @checker.options[:hit][:questions][1][:selections] = @checker.options[:hit][:questions][0][:selections]
      @checker.should be_valid
    end

    it "accepts 'take_majority': 'true' for legacy support" do
      @checker.options[:take_majority] = "true"
      @checker.should_not be_valid
      @checker.options[:hit][:questions][1][:type] = "selection"
      @checker.options[:hit][:questions][1][:selections] = @checker.options[:hit][:questions][0][:selections]
      @checker.should be_valid
    end
  end

  describe "when 'trigger_on' is set to 'schedule'" do
    before do
      @checker.options[:trigger_on] = "schedule"
      @checker.options[:submission_period] = "2"
      @checker.options.delete(:expected_receive_period_in_days)
    end

    it "should check for reviewable HITs frequently" do
      mock(@checker).review_hits.twice
      mock(@checker).create_basic_hit.once
      @checker.check
      @checker.check
    end

    it "should create HITs every 'submission_period' hours" do
      now = Time.now
      stub(Time).now { now }
      mock(@checker).review_hits.times(3)
      mock(@checker).create_basic_hit.twice
      @checker.check
      now += 1 * 60 * 60
      @checker.check
      now += 1 * 60 * 60
      @checker.check
    end

    it "should ignore events" do
      mock(@checker).create_basic_hit(anything).times(0)
      @checker.receive([events(:bob_website_agent_event)])
    end
  end

  describe "when 'trigger_on' is set to 'event'" do
    it "should not create HITs during check but should check for reviewable HITs" do
      @checker.options[:submission_period] = "2"
      now = Time.now
      stub(Time).now { now }
      mock(@checker).review_hits.times(3)
      mock(@checker).create_basic_hit.times(0)
      @checker.check
      now += 1 * 60 * 60
      @checker.check
      now += 1 * 60 * 60
      @checker.check
    end

    it "should create HITs based on events" do
      mock(@checker).create_basic_hit(events(:bob_website_agent_event)).times(1)
      @checker.receive([events(:bob_website_agent_event)])
    end
  end

  describe "creating hits" do
    it "can create HITs based on events, interpolating their values" do
      @checker.options[:hit][:title] = "Hi <.name>"
      @checker.options[:hit][:description] = "Make something for <.name>"
      @checker.options[:hit][:questions][0][:name] = "<.name> Question 1"

      question_form = nil
      hitInterface = OpenStruct.new
      hitInterface.id = 123
      mock(hitInterface).question_form(instance_of Agents::HumanTaskAgent::AgentQuestionForm) { |agent_question_form_instance| question_form = agent_question_form_instance }
      mock(RTurk::Hit).create(:title => "Hi Joe").yields(hitInterface) { hitInterface }

      @checker.send :create_basic_hit, @event

      hitInterface.max_assignments.should == @checker.options[:hit][:assignments]
      hitInterface.reward.should == @checker.options[:hit][:reward]
      hitInterface.description.should == "Make something for Joe"

      xml = question_form.to_xml
      xml.should include("<Title>Hi Joe</Title>")
      xml.should include("<Text>Make something for Joe</Text>")
      xml.should include("<DisplayName>Joe Question 1</DisplayName>")

      @checker.memory[:hits][123][:event_id].should == @event.id
    end

    it "works without an event too" do
      @checker.options[:hit][:title] = "Hi <.name>"
      hitInterface = OpenStruct.new
      hitInterface.id = 123
      mock(hitInterface).question_form(instance_of Agents::HumanTaskAgent::AgentQuestionForm)
      mock(RTurk::Hit).create(:title => "Hi").yields(hitInterface) { hitInterface }
      @checker.send :create_basic_hit
      hitInterface.max_assignments.should == @checker.options[:hit][:assignments]
      hitInterface.reward.should == @checker.options[:hit][:reward]
    end
  end

  describe "reviewing HITs" do
    class FakeHit
      def initialize(options = {})
        @options = options
      end

      def assignments
        @options[:assignments] || []
      end

      def max_assignments
        @options[:max_assignments] || 1
      end

      def dispose!
        @disposed = true
      end

      def disposed?
        @disposed
      end
    end

    class FakeAssignment
      attr_accessor :approved

      def initialize(options = {})
        @options = options
      end

      def answers
        @options[:answers] || {}
      end

      def status
        @options[:status] || ""
      end

      def approve!
        @approved = true
      end
    end

    it "should work on multiple HITs" do
      event2 = Event.new
      event2.agent = agents(:bob_rain_notifier_agent)
      event2.payload = { :foo2 => { "bar2" => { :baz2 => "a2b2" } },
                          :name2 => "Joe2" }
      event2.id = 3452

      # It knows about two HITs from two different events.
      @checker.memory[:hits] = {}
      @checker.memory[:hits][:"JH3132836336DHG"] = { :event_id => @event.id }
      @checker.memory[:hits][:"JH39AA63836DHG"] = { :event_id => event2.id }

      hit_ids = %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345]
      mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { hit_ids } } # It sees 3 HITs.

      # It looksup the two HITs that it owns.  Neither are ready yet.
      mock(RTurk::Hit).new("JH3132836336DHG") { FakeHit.new }
      mock(RTurk::Hit).new("JH39AA63836DHG") { FakeHit.new }

      @checker.send :review_hits
    end

    it "shouldn't do anything if an assignment isn't ready" do
      @checker.memory[:hits] = { :"JH3132836336DHG" => { :event_id => @event.id } }
      mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
      assignments = [
        FakeAssignment.new(:status => "Accepted", :answers => {}),
        FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy", "feedback"=>"Take 2"})
      ]
      hit = FakeHit.new(:max_assignments => 2, :assignments => assignments)
      mock(RTurk::Hit).new("JH3132836336DHG") { hit }

      # One of the assignments isn't set to "Submitted", so this should get skipped for now.
      mock.any_instance_of(FakeAssignment).answers.times(0)

      @checker.send :review_hits

      assignments.all? {|a| a.approved == true }.should be_false
      @checker.memory[:hits].should == { :"JH3132836336DHG" => { :event_id => @event.id } }
    end

    it "shouldn't do anything if an assignment is missing" do
      @checker.memory[:hits] = { :"JH3132836336DHG" => { :event_id => @event.id } }
      mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
      assignments = [
        FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy", "feedback"=>"Take 2"})
      ]
      hit = FakeHit.new(:max_assignments => 2, :assignments => assignments)
      mock(RTurk::Hit).new("JH3132836336DHG") { hit }

      # One of the assignments hasn't shown up yet, so this should get skipped for now.
      mock.any_instance_of(FakeAssignment).answers.times(0)

      @checker.send :review_hits

      assignments.all? {|a| a.approved == true }.should be_false
      @checker.memory[:hits].should == { :"JH3132836336DHG" => { :event_id => @event.id } }
    end

    it "should create events when all assignments are ready" do
      @checker.memory[:hits] = { :"JH3132836336DHG" => { :event_id => @event.id } }
      mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
      assignments = [
        FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"neutral", "feedback"=>""}),
        FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy", "feedback"=>"Take 2"})
      ]
      hit = FakeHit.new(:max_assignments => 2, :assignments => assignments)
      hit.should_not be_disposed
      mock(RTurk::Hit).new("JH3132836336DHG") { hit }

      lambda {
        @checker.send :review_hits
      }.should change { Event.count }.by(1)

      assignments.all? {|a| a.approved == true }.should be_true
      hit.should be_disposed

      @checker.events.last.payload[:answers].should == [
        {:sentiment => "neutral", :feedback => ""},
        {:sentiment => "happy", :feedback => "Take 2"}
      ]

      @checker.memory[:hits].should == {}
    end

    describe "taking majority votes" do
      before do
        @checker.options[:combination_mode] = "take_majority"
        @checker.memory[:hits] = { :"JH3132836336DHG" => { :event_id => @event.id } }
        mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
      end

      it "should take the majority votes of all questions" do
        @checker.options[:hit][:questions][1] = {
          :type => "selection",
          :key => "age_range",
          :name => "Age Range",
          :required => "true",
          :question => "Please select your age range:",
          :selections =>
            [
              { :key => "<50", :text => "50 years old or younger" },
              { :key => ">50", :text => "Over 50 years old" }
            ]
        }

        assignments = [
          FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"sad", "age_range"=>"<50"}),
          FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"neutral", "age_range"=>">50"}),
          FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy", "age_range"=>">50"}),
          FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy", "age_range"=>">50"})
        ]
        hit = FakeHit.new(:max_assignments => 4, :assignments => assignments)
        mock(RTurk::Hit).new("JH3132836336DHG") { hit }

        lambda {
          @checker.send :review_hits
        }.should change { Event.count }.by(1)

        assignments.all? {|a| a.approved == true }.should be_true

        @checker.events.last.payload[:answers].should == [
          { :sentiment => "sad", :age_range => "<50" },
          { :sentiment => "neutral", :age_range => ">50" },
          { :sentiment => "happy", :age_range => ">50" },
          { :sentiment => "happy", :age_range => ">50" }
        ]

        @checker.events.last.payload[:counts].should == { :sentiment => { :happy => 2, :sad => 1, :neutral => 1 }, :age_range => { :">50" => 3, :"<50" => 1 } }
        @checker.events.last.payload[:majority_answer].should == { :sentiment => "happy", :age_range => ">50" }
        @checker.events.last.payload.should_not have_key(:average_answer)

        @checker.memory[:hits].should == {}
      end

      it "should also provide an average answer when all questions are numeric" do
        # it should accept 'take_majority': 'true' as well for legacy support.  Demonstrating that here.
        @checker.options.delete :combination_mode
        @checker.options[:take_majority] = "true"

        @checker.options[:hit][:questions] = [
          {
            :type => "selection",
            :key => "rating",
            :name => "Rating",
            :required => "true",
            :question => "Please select a rating:",
            :selections =>
              [
                { :key => "1", :text => "One" },
                { :key => "2", :text => "Two" },
                { :key => "3", :text => "Three" },
                { :key => "4", :text => "Four" },
                { :key => "5.1", :text => "Five Point One" }
              ]
          }
        ]

        assignments = [
          FakeAssignment.new(:status => "Submitted", :answers => { "rating"=>"1" }),
          FakeAssignment.new(:status => "Submitted", :answers => { "rating"=>"3" }),
          FakeAssignment.new(:status => "Submitted", :answers => { "rating"=>"5.1" }),
          FakeAssignment.new(:status => "Submitted", :answers => { "rating"=>"2" }),
          FakeAssignment.new(:status => "Submitted", :answers => { "rating"=>"2" })
        ]
        hit = FakeHit.new(:max_assignments => 5, :assignments => assignments)
        mock(RTurk::Hit).new("JH3132836336DHG") { hit }

        lambda {
          @checker.send :review_hits
        }.should change { Event.count }.by(1)

        assignments.all? {|a| a.approved == true }.should be_true

        @checker.events.last.payload[:answers].should == [
          { :rating => "1" },
          { :rating => "3" },
          { :rating => "5.1" },
          { :rating => "2" },
          { :rating => "2" }
        ]

        @checker.events.last.payload[:counts].should == { :rating => { :"1" => 1, :"2" => 2, :"3" => 1, :"4" => 0, :"5.1" => 1 } }
        @checker.events.last.payload[:majority_answer].should == { :rating => "2" }
        @checker.events.last.payload[:average_answer].should == { :rating => (1 + 2 + 2 + 3 + 5.1) / 5.0 }

        @checker.memory[:hits].should == {}
      end
    end

    describe "creating and reviewing polls" do
      before do
        @checker.options[:combination_mode] = "poll"
        @checker.options[:poll_options] = {
          :title => "Hi!",
          :instructions => "hello!",
          :assignments => 2,
          :row_template => "This is <.sentiment>"
        }
        @event.save!
        mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
      end

      it "creates a poll using the row_template, message, and correct number of assignments" do
        @checker.memory[:hits] = { :"JH3132836336DHG" => { :event_id => @event.id } }

        # Mock out the HIT's submitted assignments.
        assignments = [
          FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"sad",     "feedback"=>"This is my feedback 1"}),
          FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"neutral", "feedback"=>"This is my feedback 2"}),
          FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy",   "feedback"=>"This is my feedback 3"}),
          FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy",   "feedback"=>"This is my feedback 4"})
        ]
        hit = FakeHit.new(:max_assignments => 4, :assignments => assignments)
        mock(RTurk::Hit).new("JH3132836336DHG") { hit }

        @checker.memory[:hits][:"JH3132836336DHG"].should be_present

        # Setup mocks for HIT creation

        question_form = nil
        hitInterface = OpenStruct.new
        hitInterface.id = "JH39AA63836DH12345"
        mock(hitInterface).question_form(instance_of Agents::HumanTaskAgent::AgentQuestionForm) { |agent_question_form_instance| question_form = agent_question_form_instance }
        mock(RTurk::Hit).create(:title => "Hi!").yields(hitInterface) { hitInterface }

        # And finally, the test.

        lambda {
          @checker.send :review_hits
        }.should change { Event.count }.by(0) # it does not emit an event until all poll results are in

        # it approves the existing assignments

        assignments.all? {|a| a.approved == true }.should be_true
        hit.should be_disposed

        # it creates a new HIT for the poll

        hitInterface.max_assignments.should == @checker.options[:poll_options][:assignments]
        hitInterface.description.should == @checker.options[:poll_options][:instructions]

        xml = question_form.to_xml
        xml.should include("<Text>This is happy</Text>")
        xml.should include("<Text>This is neutral</Text>")
        xml.should include("<Text>This is sad</Text>")

        @checker.save
        @checker.reload
        @checker.memory[:hits][:"JH3132836336DHG"].should_not be_present
        @checker.memory[:hits][:"JH39AA63836DH12345"].should be_present
        @checker.memory[:hits][:"JH39AA63836DH12345"][:event_id].should == @event.id
        @checker.memory[:hits][:"JH39AA63836DH12345"][:type].should == :poll
        @checker.memory[:hits][:"JH39AA63836DH12345"][:original_hit].should == "JH3132836336DHG"
        @checker.memory[:hits][:"JH39AA63836DH12345"][:answers].length.should == 4
      end

      it "emits an event when all poll results are in, containing the data from the best answer, plus all others" do
        original_answers = [
          {:sentiment => "sad",     :feedback => "This is my feedback 1"},
          {:sentiment => "neutral", :feedback => "This is my feedback 2"},
          {:sentiment => "happy",   :feedback => "This is my feedback 3"},
          {:sentiment => "happy",   :feedback => "This is my feedback 4"}
        ]

        @checker.memory[:hits] = {
          :JH39AA63836DH12345 => {
            :type => :poll,
            :original_hit => "JH3132836336DHG",
            :answers => original_answers,
            :event_id => 345
          }
        }

        # Mock out the HIT's submitted assignments.
        assignments = [
          FakeAssignment.new(:status => "Submitted", :answers => {"1" => "2", "2" => "5", "3" => "3", "4" => "2"}),
          FakeAssignment.new(:status => "Submitted", :answers => {"1" => "3", "2" => "4", "3" => "1", "4" => "4"})
        ]
        hit = FakeHit.new(:max_assignments => 2, :assignments => assignments)
        mock(RTurk::Hit).new("JH39AA63836DH12345") { hit }

        @checker.memory[:hits][:"JH39AA63836DH12345"].should be_present

        lambda {
          @checker.send :review_hits
        }.should change { Event.count }.by(1)

        # It emits an event

        @checker.events.last.payload[:answers].should == original_answers
        @checker.events.last.payload[:poll].should == [{:"1" => "2", :"2" => "5", :"3" => "3", :"4" => "2"}, {:"1" => "3", :"2" => "4", :"3" => "1", :"4" => "4"}]
        @checker.events.last.payload[:best_answer].should == {:sentiment => "neutral", :feedback => "This is my feedback 2"}

        # it approves the existing assignments

        assignments.all? {|a| a.approved == true }.should be_true
        hit.should be_disposed

        @checker.memory[:hits].should be_empty
      end
    end
  end
end