Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,15 @@ xcode_summary.inline_mode = true
xcode_summary.report 'MyApp.xcresult'
```

You can use `ignore_retried_tests` to intelligently handle retried unit tests.
When enabled, if a test fails but succeeds on retry, the failure will be ignored and not reported as an error.

```ruby
# Ignore test failures that succeeded on retry
xcode_summary.ignore_retried_tests = true
xcode_summary.report 'MyApp.xcresult'
```

You can use `strict` to reporting errors as warnings thereby don't block merge PR.

```ruby
Expand Down
68 changes: 67 additions & 1 deletion lib/xcode_summary/plugin.rb
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,13 @@ class DangerXcodeSummary < Plugin
# @return [Boolean]
attr_accessor :collapse_parallelized_tests

# Defines if retried unit tests should be handled intelligently.
# When enabled, if a test fails but succeeds on retry, the failure will be ignored.
# Defaults to `false`
# @param [Boolean] value
# @return [Boolean]
attr_accessor :ignore_retried_tests

# rubocop:disable Lint/DuplicateMethods
def project_root
root = @project_root || Dir.pwd
Expand Down Expand Up @@ -131,6 +138,10 @@ def collapse_parallelized_tests
@collapse_parallelized_tests || false
end

def ignore_retried_tests
@ignore_retried_tests || false
end

# Pick a Dangerfile plugin for a chosen request_source and cache it
# based on https://github.com/danger/danger/blob/master/lib/danger/plugin_support/plugin.rb#L31
#
Expand All @@ -148,6 +159,9 @@ def plugin
def report(file_path)
if File.exist?(file_path)
xcode_summary = XCResult::Parser.new(path: file_path)
if ignore_retried_tests
@successfully_retried_test_identifiers = extract_all_successfully_retried_test_identifiers(xcode_summary)
end
format_summary(xcode_summary)
else
fail 'summary file not found'
Expand All @@ -161,6 +175,9 @@ def report(file_path)
def warning_error_count(file_path)
if File.exist?(file_path)
xcode_summary = XCResult::Parser.new(path: file_path)
if ignore_retried_tests
@successfully_retried_test_identifiers = extract_all_successfully_retried_test_identifiers(xcode_summary)
end
warning_count = 0
error_count = 0
xcode_summary.actions_invocation_record.actions.each do |action|
Expand Down Expand Up @@ -229,6 +246,17 @@ def messages(xcode_summary)
subtests = action_test_object.all_subtests
subtests_duration = subtests.map(&:duration).sum

if ignore_retried_tests
subtests_without_retry_attempt = subtests.group_by(&:identifier).values.map do |group|
if group.length > 1 && group.any? { |subtest| subtest.test_status == 'Success' }
group.reject { |subtest| subtest.test_status == 'Failure' }
else
group
end
end
subtests = subtests_without_retry_attempt.flatten
end

failed_tests_count = subtests.reject { |test| test.test_status == 'Success' }.count
expected_failed_tests_count = subtests.select { |test| test.test_status == 'Expected Failure' }.count

Expand Down Expand Up @@ -306,19 +334,57 @@ def errors(action)
Result.new(format_warning(result), result.location)
end

successfully_retried_test_identifiers = @successfully_retried_test_identifiers || []
test_failures = [
action.action_result.issues.test_failure_summaries,
action.build_result.issues.test_failure_summaries
].flatten.compact.map do |summary|
if ignore_retried_tests && successfully_retried_test_identifiers.include?(sanitized_test_case_name(summary.test_case_name))
next
end

result = Result.new(summary.message, parse_location(summary.document_location_in_creating_workspace))
Result.new(format_test_failure(result, summary.producing_target, summary.test_case_name),
result.location)
end

results = (errors + test_failures).uniq.reject { |result| result.message.nil? }
results = (errors + test_failures).compact.uniq.reject { |result| result.message.nil? }
results.delete_if(&ignored_results)
end

def extract_all_successfully_retried_test_identifiers(xcode_summary)
successfully_retried_test_identifiers = []
xcode_summary.action_test_plan_summaries.each do |test_plan_summaries|
test_plan_summaries.summaries.each do |summary|
summary.testable_summaries.each do |testable_summary|
testable_summary.tests.each do |test|
next unless test.instance_of? XCResult::ActionTestSummaryGroup

test.all_subtests.group_by(&:identifier).each do |identifier, subtests|
contain_success = subtests.any? { |subtest| subtest.test_status == 'Success' }
if subtests.length > 1 && contain_success
successfully_retried_test_identifiers << identifier
end
end
end
end
end
end
successfully_retried_test_identifiers
end

def sanitized_test_case_name(test_case_name)
# Clean test_case_name to match identifier format
# Sanitize for Swift by replacing "." for "/"
# Sanitize for Objective-C by removing "-", "[", "]", and replacing " " for ?/
test_case_name
.tr('.', '/')
.tr('-', '')
.tr('[', '')
.tr(']', '')
.tr(' ', '/')
end

def parse_location(document_location)
return nil if document_location&.url.nil?

Expand Down
Binary file not shown.
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[{"type":1,"name":"testmanagerd.log"}]
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[{"type":1,"name":"testmanagerd.log"}]
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
29 changes: 29 additions & 0 deletions spec/fixtures/retried_tests.xcresult/Info.plist
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>dateCreated</key>
<date>2025-08-22T14:06:34Z</date>
<key>externalLocations</key>
<array/>
<key>rootId</key>
<dict>
<key>hash</key>
<string>0~r3y1K6zs0Qq8JJIiol3B_pECoVfq_KHDSMUZDI4q-GfZ2wUj5Px08tL__pTotA0eeh332MvqT10WpiAXLW1Rog==</string>
</dict>
<key>storage</key>
<dict>
<key>backend</key>
<string>fileBacked2</string>
<key>compression</key>
<string>standard</string>
</dict>
<key>version</key>
<dict>
<key>major</key>
<integer>3</integer>
<key>minor</key>
<integer>53</integer>
</dict>
</dict>
</plist>
Binary file not shown.
59 changes: 59 additions & 0 deletions spec/xcode_summary_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -314,6 +314,65 @@ module Danger
end
end

# Test retried tests functionality
describe 'with retried tests' do
before do
@dangerfile = testing_dangerfile
@xcode_summary = @dangerfile.xcode_summary
@xcode_summary.env.request_source.pr_json = JSON.parse(File.read('spec/fixtures/pr_json.json'))
@xcode_summary.project_root = '/Users/marcelofabri/SwiftLint/'
end

context 'with ignore_retried_tests enabled' do
before do
@xcode_summary.ignore_retried_tests = true
end

it 'ignores test failures that succeeded on retry' do
@xcode_summary.report('spec/fixtures/retried_tests.xcresult')
# The retried test should not show up as an error since it ultimately succeeded
expect(@dangerfile.status_report[:errors]).to eq []
end

it 'shows test summary with retry information' do
@xcode_summary.test_summary = true
@xcode_summary.report('spec/fixtures/retried_tests.xcresult')
# The test summary should still show execution information
expect(@dangerfile.status_report[:messages].length).to eq 1
expect(@dangerfile.status_report[:messages].first).to match('retried-testTests: Executed 2 tests, with 0 failures (0 expected) in 0.158 (0.158) seconds')
end

it 'shows no error count' do
result = @xcode_summary.warning_error_count('spec/fixtures/retried_tests.xcresult')
expect(result).to eq '{"warnings":0,"errors":0}'
end

it 'defaults to false' do
# Create a fresh instance to test default value
fresh_dangerfile = testing_dangerfile
fresh_xcode_summary = fresh_dangerfile.xcode_summary
expect(fresh_xcode_summary.ignore_retried_tests).to eq false
end
end

context 'with ignore_retried_tests disabled' do
before do
@xcode_summary.ignore_retried_tests = false
end

it 'still reports test failures even if they succeeded on retry' do
@xcode_summary.report('spec/fixtures/retried_tests.xcresult')
# With retry filtering disabled, failed tests should still be reported
expect(@dangerfile.status_report[:errors].length).to eq 1
end

it 'count retry as error' do
result = @xcode_summary.warning_error_count('spec/fixtures/retried_tests.xcresult')
expect(result).to eq '{"warnings":0,"errors":1}'
end
end
end

# Second environment with different request_source
describe 'with bitbucket request_source' do
before do
Expand Down
Loading