#!/usr/bin/env perl use strict; use warnings; use FindBin; use lib "$FindBin::Bin/lib"; use Scalar::Util qw{reftype}; use TestRail::API; use Test::LWP::UserAgent::TestRailMock; use Test::Rail::Parser; use Test::More 'tests' => 126; use Test::Fatal qw{exception}; use Test::Deep qw{cmp_deeply}; use Capture::Tiny qw{capture capture_stderr}; #Same song and dance as in TestRail-API.t my $apiurl = $ENV{'TESTRAIL_API_URL'}; my $login = $ENV{'TESTRAIL_USER'}; my $pw = $ENV{'TESTRAIL_PASSWORD'}; my $step_results = $ENV{'TESTRAIL_STEP_RESULTS'}; my $is_mock = (!$apiurl && !$login && !$pw); ($apiurl,$login,$pw) = ('http://testrail.local','teodesian@cpan.org','fake') if $is_mock; my ($debug,$browser); $debug = 1; if ($is_mock) { $browser = $Test::LWP::UserAgent::TestRailMock::mockObject; } #test exceptions... #TODO my $fcontents = " fake.test .. 1..2 ok 1 - STORAGE TANKS SEARED #goo not ok 2 - NOT SO SEARED AFTER ARR "; my $tap; my $opts = { 'tap' => $fcontents, 'apiurl' => $apiurl, 'user' => $login, 'pass' => $pw, 'debug' => $debug, 'browser' => $browser, 'run' => 'TestingSuite', 'project' => 'TestProject', 'merge' => 1, 'step_results' => $step_results, }; my $res = exception { $tap = Test::Rail::Parser->new($opts) }; is($res,undef,"TR Parser doesn't explode on instantiation"); isa_ok($tap,"Test::Rail::Parser"); if (!$res) { $tap->run(); is($tap->{'errors'},0,"No errors encountered uploading case results"); } undef $tap; delete $opts->{'tap'}; $opts->{'source'} = 't/fake.test'; $res = exception { $tap = Test::Rail::Parser->new($opts) }; is($res,undef,"TR Parser doesn't explode on instantiation"); isa_ok($tap,"Test::Rail::Parser"); if (!$res) { $tap->run(); is($tap->{'errors'},0,"No errors encountered uploading case results"); } $fcontents = "fake.test... ok 1 - STORAGE TANKS SEARED # whee not ok 2 - NOT SO SEARED AFTER ARR # Failed test 'NOT SO SEARED AFTER ARR' # at t/fake.test line 10. # Looks like you failed 1 test of 2. "; like($tap->{'raw_output'},qr/SEARED\n# whee.*\n.*AFTER ARR\n\n.*Failed/msxi,"Full raw content uploaded in non step results mode"); #Check that time run is being uploaded my $timeResults = $tap->{'tr_opts'}->{'testrail'}->getTestResults(1); if ( ( reftype($timeResults) || 'undef') eq 'ARRAY') { is( $timeResults->[0]->{'elapsed'}, '2s', "Plugin correctly sets elapsed time"); } else { fail("Could not get test results to check elapsed time!"); } #Check the time formatting routine. is(Test::Rail::Parser::_compute_elapsed(0,0),undef,"Elapsed computation correct at second boundary"); is(Test::Rail::Parser::_compute_elapsed(0,61),'1m 1s',"Elapsed computation correct at minute boundary"); is(Test::Rail::Parser::_compute_elapsed(0,3661),'1h 1m 1s',"Elapsed computation correct at hour boundary"); is(Test::Rail::Parser::_compute_elapsed(0,86461),'24h 1m 1s',"Elapsed computation correct at day boundary"); undef $tap; $opts->{'source'} = 't/faker.test'; $opts->{'run'} = 'OtherOtherSuite'; $opts->{'step_results'} = 'step_results'; $res = exception { $tap = Test::Rail::Parser->new($opts) }; is($res,undef,"TR Parser doesn't explode on instantiation"); isa_ok($tap,"Test::Rail::Parser"); if (!$res) { $tap->run(); is($tap->{'errors'},0,"No errors encountered uploading case results"); is($tap->{'global_status'},5, "Test global result is FAIL when one subtest fails even if there are TODO passes"); subtest 'Timestamp/elapsed printed in step results' => sub { foreach my $result (@{$tap->{'tr_opts'}->{'result_custom_options'}->{'step_results'}}) { like($result->{'content'}, qr/^\[.*\(.*\)\]/i, "Timestamp printed in step results"); } }; } #Default mode undef $tap; delete $opts->{'step_results'}; $res = exception { $tap = Test::Rail::Parser->new($opts) }; is($res,undef,"TR Parser doesn't explode on instantiation"); isa_ok($tap,"Test::Rail::Parser"); if (!$res) { $tap->run(); is($tap->{'errors'},0,"No errors encountered uploading case results"); my @matches = $tap->{'raw_output'} =~ m/^(\[.*\(.*\)\])/msgi; ok(scalar(@matches),"Timestamps present in raw TAP"); } #Default mode undef $tap; $fcontents = " fake.test .. 1..2 ok 1 - STORAGE TANKS SEARED #Subtest NOT SO SEARED AFTER ARR ok 1 - STROGGIFY POPULATION CENTERS not ok 2 - STROGGIFY POPULATION CENTERS #goo not ok 2 - NOT SO SEARED AFTER ARR "; $opts->{'tap'} = $fcontents; delete $opts->{'source'}; delete $opts->{'step_results'}; $opts->{'run'} = 'TestingSuite'; $res = exception { $tap = Test::Rail::Parser->new($opts) }; is($res,undef,"TR Parser doesn't explode on instantiation"); isa_ok($tap,"Test::Rail::Parser"); if (!$res) { $tap->run(); is($tap->{'errors'},0,"No errors encountered uploading case results"); } undef $tap; delete $opts->{'tap'}; $opts->{'source'} = 't/skip.test'; $res = exception { $tap = Test::Rail::Parser->new($opts) }; is($res,undef,"TR Parser doesn't explode on instantiation"); isa_ok($tap,"Test::Rail::Parser"); if (!$res) { $tap->run(); is($tap->{'errors'},0,"No errors encountered uploading case results"); } #Default mode skip (skip_all) undef $tap; $opts->{'source'} = 't/skipall.test'; $res = exception { $tap = Test::Rail::Parser->new($opts) }; is($res,undef,"TR Parser doesn't explode on instantiation"); isa_ok($tap,"Test::Rail::Parser"); if (!$res) { $tap->run(); is($tap->{'errors'},0,"No errors encountered uploading case results"); like( $tap->{raw_output}, qr/cause I can/i, "SKIP_ALL reason recorded"); is($tap->{'global_status'},6, "Test global result is SKIP on skip all"); } #Ok, let's test the plan, config, and spawn bits. undef $tap; $opts->{'run'} = 'hoo hoo I do not exist'; $opts->{'plan'} = 'mah dubz plan'; $opts->{'configs'} = ['testPlatform1']; $res = exception { $tap = Test::Rail::Parser->new($opts) }; isnt($res,undef,"TR Parser explodes on instantiation when asking for run not in plan"); undef $tap; $opts->{'run'} = 'TestingSuite'; $opts->{'configs'} = ['testConfig']; $res = exception { $tap = Test::Rail::Parser->new($opts) }; is($res,undef,"TR Parser doesn't explode on instantiation looking for existing run in plan"); isa_ok($tap,"Test::Rail::Parser"); if (!$res) { $tap->run(); is($tap->{'errors'},0,"No errors encountered uploading case results"); } #Now, test spawning. undef $tap; $opts->{'run'} = 'TestingSuite2'; $opts->{'configs'} = ['testPlatform1']; $opts->{'testsuite_id'} = 9; $res = exception { $tap = Test::Rail::Parser->new($opts) }; is($res,undef,"TR Parser doesn't explode on instantiation when spawning run in plan"); isa_ok($tap,"Test::Rail::Parser"); if (!$res) { $tap->run(); is($tap->{'errors'},0,"No errors encountered uploading case results"); } #Test spawning of builds not in plans. #Now, test spawning. undef $tap; delete $opts->{'testsuite_id'}; delete $opts->{'plan'}; delete $opts->{'configs'}; $opts->{'testsuite'} = 'HAMBURGER-IZE HUMANITY'; $res = exception { $tap = Test::Rail::Parser->new($opts) }; is($res,undef,"TR Parser doesn't explode on instantiation when spawning run in plan"); isa_ok($tap,"Test::Rail::Parser"); if (!$res) { $tap->run(); is($tap->{'errors'},0,"No errors encountered uploading case results"); } #Test spawning of plans and runs. undef $tap; $opts->{'run'} = 'BogoRun'; $opts->{'plan'} = 'BogoPlan'; $opts->{'testsuite_id'} = 9; delete $opts->{'testsuite'}; $res = exception { $tap = Test::Rail::Parser->new($opts) }; is($res,undef,"TR Parser doesn't explode on instantiation when spawning run in plan"); isa_ok($tap,"Test::Rail::Parser"); if (!$res) { $tap->run(); is($tap->{'errors'},0,"No errors encountered uploading case results"); } #Check that per-section spawn works undef $tap; $opts->{'source'} = 't/fake.test'; delete $opts->{'plan'}; $opts->{'sections'} = ['fake.test']; delete $opts->{'step_results'}; $res = exception { $tap = Test::Rail::Parser->new($opts) }; is($res,undef,"TR Parser doesn't explode on instantiation"); isa_ok($tap,"Test::Rail::Parser"); if (!$res) { $tap->run(); is($tap->{'errors'},0,"No errors encountered uploading case results"); } #Check that per-section spawn works undef $tap; $opts->{'plan'} = 'BogoPlan'; $res = exception { $tap = Test::Rail::Parser->new($opts) }; is($res,undef,"TR Parser doesn't explode on instantiation"); isa_ok($tap,"Test::Rail::Parser"); if (!$res) { $tap->run(); is($tap->{'errors'},0,"No errors encountered uploading case results"); } undef $tap; $opts->{'sections'} = ['potzrebie']; delete $opts->{'plan'}; $res = exception { $tap = Test::Rail::Parser->new($opts) }; isnt($res,undef,"TR Parser explodes on instantiation with invalid section"); undef $tap; $opts->{'source'} = 't/notests.test'; delete $opts->{'sections'}; $res = exception { $tap = Test::Rail::Parser->new($opts) }; is($res,undef,"TR Parser doesn't explode on instantiation"); isa_ok($tap,"Test::Rail::Parser"); if (!$res) { $tap->run(); is($tap->{'errors'},0,"No errors encountered uploading case results"); is($tap->{'global_status'},5, "Test global result is FAIL by default on env fail"); } undef $tap; $opts->{'source'} = 't/pass.test'; $res = exception { $tap = Test::Rail::Parser->new($opts) }; is($res,undef,"TR Parser doesn't explode on instantiation"); isa_ok($tap,"Test::Rail::Parser"); if (!$res) { $tap->run(); is($tap->{'errors'},0,"No errors encountered uploading case results"); is($tap->{'global_status'},1, "Test global result is PASS on ok test"); } undef $tap; $opts->{'source'} = 't/todo_pass.test'; $res = exception { $tap = Test::Rail::Parser->new($opts) }; is($res,undef,"TR Parser doesn't explode on instantiation"); isa_ok($tap,"Test::Rail::Parser"); if (!$res) { $tap->run(); is($tap->{'errors'},0,"No errors encountered uploading case results"); is($tap->{'global_status'},8, "Test global result is TODO PASS on todo pass test"); } undef $tap; $opts->{'step_results'} = 'bogus_garbage'; $res = exception { $tap = Test::Rail::Parser->new($opts) }; like($res,qr/invalid step results/i,"Bogus step results name throws"); undef $tap; $opts->{'source'} = 't/todo_pass_and_fail.test'; $opts->{'step_results'} = 'step_results'; $res = exception { $tap = Test::Rail::Parser->new($opts) }; is($res,undef,"TR Parser doesn't explode on instantiation"); isa_ok($tap,"Test::Rail::Parser"); if (!$res) { capture { $tap->run() }; is($tap->{'errors'},1,"Errors encountered uploading case results for case that does not exist in TestRail"); is($tap->{'global_status'},7, "Test global result is TODO FAIL on todo pass & fail test"); my @desired_statuses = qw{1 8 7}; my @got_statuses = map {$_->{'status_id'}} @{$tap->{'tr_opts'}->{'result_custom_options'}->{'step_results'}}; my @desired_expected = ('OK', 'OK', 'OK'); my @got_expected = map {$_->{'expected'}} @{$tap->{'tr_opts'}->{'result_custom_options'}->{'step_results'}}; my @desired_actual = ('OK', 'TODO PASS', 'TODO FAIL'); my @got_actual = map {$_->{'actual'}} @{$tap->{'tr_opts'}->{'result_custom_options'}->{'step_results'}}; cmp_deeply(\@got_expected,\@desired_expected,"Expected status names look OK"); cmp_deeply(\@got_actual,\@desired_actual,"Actual status names look OK"); cmp_deeply(\@got_statuses,\@desired_statuses,"Step result status codes set correctly"); like($tap->{'tr_opts'}->{'test_notes'},qr/ez duz it/i,"TODO reason captured in test notes"); } undef $opts->{'step_results'}; undef $tap; #Check bad plan w/ todo pass logic $fcontents = " todo_pass.test .. 1..2 ok 1 - STORAGE TANKS SEARED #TODO todo pass # goo "; undef $opts->{'source'}; $opts->{'tap'} = $fcontents; $opts->{'step_results'} = 'step_results'; $res = exception { $tap = Test::Rail::Parser->new($opts) }; is($res,undef,"TR Parser doesn't explode on instantiation"); isa_ok($tap,"Test::Rail::Parser"); if (!$res) { $tap->run(); is($tap->{'errors'},0,"No errors encountered uploading case results"); is($tap->{'global_status'},5, "Test global result is FAIL on todo pass test w/ bad plan"); my $srs = $tap->{'tr_opts'}->{'result_custom_options'}->{'step_results'}; is($srs->[-1]->{'content'},"Bad Plan.","Bad plan noted in step results"); } undef $opts->{'step_results'}; #Check instant pizza $fcontents = " todo_pass.test .. 1..2 "; undef $opts->{'source'}; $opts->{'tap'} = $fcontents; $opts->{'step_results'} = 'step_results'; $res = exception { $tap = Test::Rail::Parser->new($opts) }; is($res,undef,"TR Parser doesn't explode on instantiation"); isa_ok($tap,"Test::Rail::Parser"); if (!$res) { $tap->run(); is($tap->{'errors'},0,"No errors encountered uploading case results"); is($tap->{'global_status'},5, "Test global result is FAILURE when insta-bombout occurs"); my $srs = $tap->{'tr_opts'}->{'result_custom_options'}->{'step_results'}; is($srs->[-1]->{'content'},"Bad Plan.","Bad plan noted in step results"); } $opts->{test_bad_status} = 'bogus_status'; $res = exception { $tap = Test::Rail::Parser->new($opts) }; like($res,qr/bogus_status/,"TR Parser explodes on instantiation w bogus status"); $opts->{test_bad_status} = 'blocked'; $res = exception { $tap = Test::Rail::Parser->new($opts) }; is($res,undef,"TR Parser doesn't explode on instantiation"); isa_ok($tap,"Test::Rail::Parser"); if (!$res) { $tap->run(); is($tap->{'errors'},0,"No errors encountered uploading case results"); is($tap->{'global_status'},2, "Test global result is BLOCKED when insta-bombout occurs & custom status set"); } undef $opts->{'step_results'}; #Check unplanned tests $fcontents = " todo_pass.test .. 1..1 ok 1 - STORAGE TANKS SEARED ok 2 - ZIPPPEEE "; $opts->{'tap'} = $fcontents; $res = exception { $tap = Test::Rail::Parser->new($opts) }; is($res,undef,"TR Parser doesn't explode on instantiation"); isa_ok($tap,"Test::Rail::Parser"); if (!$res) { $tap->run(); is($tap->{'errors'},0,"No errors encountered uploading case results w/ unplanned tests"); is($tap->{'global_status'},5, "Test global result is FAIL when unplanned test seen without case-per-ok"); } undef $tap; #Check bad plan w/ todo pass logic $fcontents = " todo_pass.test .. 1..2 ok 1 - STORAGE TANKS SEARED #TODO todo pass # goo % mark_status=todo_fail #Appears tanks weren't so sealed after all "; undef $opts->{'source'}; $opts->{'tap'} = $fcontents; $res = exception { $tap = Test::Rail::Parser->new($opts) }; is($res,undef,"TR Parser doesn't explode on instantiation"); isa_ok($tap,"Test::Rail::Parser"); if (!$res) { $tap->run(); is($tap->{'errors'},0,"No errors encountered uploading case results"); is($tap->{'global_status'},7, "Test global result is respected when using global status override"); } undef $opts->{'tap'}; #Check autoclose functionality against Run with all tests in run status. undef $tap; $opts->{'source'} = 't/skip.test'; $opts->{'run'} = 'FinalRun'; $opts->{'autoclose'} = 1; $res = exception { $tap = Test::Rail::Parser->new($opts) }; is($res,undef,"TR Parser doesn't explode on instantiation"); isa_ok($tap,"Test::Rail::Parser"); if (!$res) { $tap->run(); is($tap->{'errors'},0,"No errors encountered uploading case results"); is($tap->{'run_closed'},1, "Run closed by parser when all tests done"); } #Check autoclose functionality against Run with not all tests in run status. undef $tap; $opts->{'source'} = 't/todo_pass.test'; $opts->{'run'} = 'BogoRun'; $res = exception { $tap = Test::Rail::Parser->new($opts) }; is($res,undef,"TR Parser doesn't explode on instantiation"); isa_ok($tap,"Test::Rail::Parser"); if (!$res) { $tap->run(); is($tap->{'errors'},0,"No errors encountered uploading case results"); is($tap->{'run_closed'},undef, "Run not closed by parser when results are outstanding"); } #Check that autoclose works against plan with all tests in run status undef $tap; $opts->{'source'} = 't/fake.test'; $opts->{'run'} = 'FinalRun'; $opts->{'plan'} = 'FinalPlan'; $opts->{'configs'} = ['testConfig']; $res = exception { $tap = Test::Rail::Parser->new($opts) }; is($res,undef,"TR Parser doesn't explode on instantiation"); isa_ok($tap,"Test::Rail::Parser"); if (!$res) { $tap->run(); is($tap->{'errors'},0,"No errors encountered uploading case results"); is($tap->{'plan_closed'},1, "Plan closed by parser when all tests done"); } #Check that autoclose works against plan with all tests not in run status undef $tap; $opts->{'run'} = 'BogoRun'; $opts->{'plan'} = 'BogoPlan'; $res = exception { $tap = Test::Rail::Parser->new($opts) }; is($res,undef,"TR Parser doesn't explode on instantiation"); isa_ok($tap,"Test::Rail::Parser"); if (!$res) { $tap->run(); is($tap->{'errors'},0,"No errors encountered uploading case results"); is($tap->{'plan_closed'},undef, "Plan not closed by parser when results are outstanding"); } #Plan but no run 'splodes undef $tap; $opts->{'plan'} = 'CompletePlan'; delete $opts->{'run'}; delete $opts->{'configs'}; $res = exception { $tap = Test::Rail::Parser->new($opts) }; like($res,qr/but no run passed/i,"TR Parser explodes on instantiation due to passing plan with no run"); #Check that trying without spawn opts, using completed plan fails undef $tap; $opts->{'plan'} = 'ClosedPlan'; $opts->{'run'} = 'BogoRun'; delete $opts->{'testsuite_id'}; $res = exception { $tap = Test::Rail::Parser->new($opts) }; like($res,qr/plan provided is completed/i,"TR Parser explodes on instantiation due to passing closed plan"); #Check that the above two will just spawn a new plan in these cases $opts->{'testsuite_id'} = 9; $res = exception { $tap = Test::Rail::Parser->new($opts) }; is($res,undef,"TR Parser runs all the way through on completed run when spawning"); #Check that trying without spawn opts, using completed run fails undef $tap; delete $opts->{'testsuite_id'}; delete $opts->{'plan'}; $opts->{'run'} = 'ClosedRun'; $res = exception { $tap = Test::Rail::Parser->new($opts) }; like($res,qr/run provided is completed/i,"TR Parser explodes on instantiation due to passing closed run"); #Check that the above two will just spawn a new run in these cases $opts->{'testsuite_id'} = 9; $res = exception { $tap = Test::Rail::Parser->new($opts) }; is($res,undef,"TR Parser runs all the way through on completed run when spawning"); $fcontents = " todo_pass.test .. 1..2 ok 1 - STORAGE TANKS SEARED #TODO todo pass # goo Bail out! #YOLO "; undef $opts->{'source'}; $opts->{'tap'} = $fcontents; $opts->{'step_results'} = 'step_results'; $res = exception { $tap = Test::Rail::Parser->new($opts) }; is($res,undef,"TR Parser runs all the way through on bailout"); if (!$res) { $tap->run(); is($tap->{'errors'},0,"No errors encountered uploading case results"); is($tap->{'global_status'},5, "Test global result is FAIL on todo pass test w/ bailout"); my $srs = $tap->{'tr_opts'}->{'result_custom_options'}->{'step_results'}; is($srs->[-1]->{'content'},"Bail Out!.","Bailout noted in step results"); } #Check section spawn recursion is done correctly undef $opts->{'tap'}; $opts->{'source'} = 't/pass.test'; $opts->{'testsuite_id'} = 5; $opts->{'project_id'} = 3; $opts->{'run'} = 'zippyRun'; $opts->{'sections'} = ['Recursing section','grandchild']; $res = exception { $tap = Test::Rail::Parser->new($opts) }; is($res,undef,"TR Parser runs all the way through when recursing sections"); if (!$res) { $tap->run(); is($tap->{'errors'},0,"No errors encountered uploading case results"); } #Check configuration group spawn is done correctly undef $opts->{'tap'}; $opts->{'source'} = 't/pass.test'; $opts->{'project_id'} = 9; $opts->{'run'} = 'TestingSuite'; $opts->{'plan'} = 'mah dubz plan'; $opts->{'config_group'} = 'noSuchGroup'; $opts->{'configs'} = ['noSuchConfig']; $opts->{'sections'} = []; $res = exception { $tap = Test::Rail::Parser->new($opts) }; is($res,undef,"TR Parser runs all the way through when spawning configurations"); $fcontents = " todo_pass.test .. ok 1 - STORAGE TANKS SEARED #TODO todo pass # goo Bail out! #YOLO "; undef $opts->{'source'}; $opts->{'tap'} = $fcontents; #Issue 143 my $warns = capture_stderr { $tap = Test::Rail::Parser->new($opts) }; is($warns, '', "No warnings parsing TAP with undef plans"); #Issue 140 $fcontents = " todo_pass.test .. "; undef $opts->{'source'}; $opts->{'tap'} = $fcontents; $warns = capture_stderr { $tap = Test::Rail::Parser->new($opts) }; is($warns, '', "No warnings parsing TAP with NOTESTS");