Skip to content

Commit 5289cf1

Browse files
AMaini503Aayush Maini
AMaini503
and
Aayush Maini
authored
User/aamaini/go sort requests (#1432)
* Process go process requests in order of depth * Add UTs to verify traversal order in go detector * Fix paths for cross-plat UTs * Fix another path in UT * CR: Change level of a noisy trace * Bump detector version --------- Co-authored-by: Aayush Maini <[email protected]>
1 parent 1b1925e commit 5289cf1

File tree

2 files changed

+249
-6
lines changed

2 files changed

+249
-6
lines changed

‎src/Microsoft.ComponentDetection.Detectors/go/GoComponentDetector.cs‎

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -48,14 +48,14 @@ public GoComponentDetector(
4848

4949
public override IEnumerable<ComponentType> SupportedComponentTypes { get; } = [ComponentType.Go];
5050

51-
public override int Version => 9;
51+
public override int Version => 10;
5252

53-
protected override Task<IObservable<ProcessRequest>> OnPrepareDetectionAsync(
53+
protected async override Task<IObservable<ProcessRequest>> OnPrepareDetectionAsync(
5454
IObservable<ProcessRequest> processRequests,
5555
IDictionary<string, string> detectorArgs,
5656
CancellationToken cancellationToken = default)
5757
{
58-
var goModProcessRequests = processRequests.Where(processRequest =>
58+
var filteredGoProcessRequests = await processRequests.Where(processRequest =>
5959
{
6060
if (Path.GetFileName(processRequest.ComponentStream.Location) != "go.sum")
6161
{
@@ -81,9 +81,15 @@ protected override Task<IObservable<ProcessRequest>> OnPrepareDetectionAsync(
8181
{
8282
goModFile?.Stream.Dispose();
8383
}
84-
});
84+
}).ToList(); // Materialize the filtered items for sorting
8585

86-
return Task.FromResult(goModProcessRequests);
86+
// Sort by depth: shallow files (fewer directory segments) come first
87+
var sortedGoProcessRequests = filteredGoProcessRequests
88+
.OrderBy(pr => pr.ComponentStream.Location.Count(c => c == Path.DirectorySeparatorChar))
89+
.ThenBy(pr => pr.ComponentStream.Location)
90+
.ToList();
91+
92+
return sortedGoProcessRequests.ToObservable();
8793
}
8894

8995
protected override async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary<string, string> detectorArgs, CancellationToken cancellationToken = default)
@@ -108,7 +114,20 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID
108114
case ".MOD":
109115
{
110116
this.Logger.LogDebug("Found Go.mod: {Location}", file.Location);
111-
await this.goParserFactory.CreateParser(GoParserType.GoMod, this.Logger).ParseAsync(singleFileComponentRecorder, file, record);
117+
var wasModParsedSuccessfully = await this.goParserFactory.CreateParser(GoParserType.GoMod, this.Logger).ParseAsync(singleFileComponentRecorder, file, record);
118+
119+
// Check if go.mod was parsed successfully and Go version is >= 1.17 in go.mod
120+
if (wasModParsedSuccessfully &&
121+
!string.IsNullOrEmpty(record.GoModVersion) &&
122+
System.Version.TryParse(record.GoModVersion, out var goVersion) &&
123+
goVersion >= new Version(1, 17))
124+
{
125+
this.projectRoots.Add(projectRootDirectory.FullName);
126+
}
127+
else
128+
{
129+
this.Logger.LogDebug("Not adding {Root} to processed roots: {ParseSuccess} {GoModVersion}", projectRootDirectory.FullName, wasModParsedSuccessfully, record.GoModVersion);
130+
}
112131

113132
if (await this.ShouldRunGoGraphAsync())
114133
{

‎test/Microsoft.ComponentDetection.Detectors.Tests/GoComponentDetectorTests.cs‎

Lines changed: 224 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -983,4 +983,228 @@ public async Task GoModDetector_VerifyLocalReferencesIgnored()
983983
.Should()
984984
.BeEquivalentTo(expectedComponentIds);
985985
}
986+
987+
/// <summary>
988+
/// Verify that nested directories are skipped once root is processed.
989+
/// Assume root GoModVersion is >= 1.17.
990+
/// </summary>
991+
/// <returns>Task.</returns>
992+
[TestMethod]
993+
public async Task GoDetector_GoMod_VerifyNestedRootsUnderGTE117_AreSkipped()
994+
{
995+
var processedFiles = new List<string>();
996+
this.SetupMockGoModParser();
997+
this.mockGoModParser
998+
.Setup(p => p.ParseAsync(It.IsAny<ISingleFileComponentRecorder>(), It.IsAny<IComponentStream>(), It.IsAny<GoGraphTelemetryRecord>()))
999+
.ReturnsAsync(true)
1000+
.Callback<ISingleFileComponentRecorder, IComponentStream, GoGraphTelemetryRecord>((_, file, record) =>
1001+
{
1002+
processedFiles.Add(file.Location);
1003+
record.GoModVersion = "1.18";
1004+
});
1005+
1006+
var root = Path.Combine("C:", "root");
1007+
var (scanResult, componentRecorder) = await this.DetectorTestUtility
1008+
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "a", "go.mod"))
1009+
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "a", "a", "go.mod"))
1010+
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "a", "b", "go.mod"))
1011+
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "b", "go.mod"))
1012+
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "go.mod"))
1013+
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "b", "d", "go.mod"))
1014+
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "b", "a", "go.mod"))
1015+
.ExecuteDetectorAsync();
1016+
1017+
scanResult.ResultCode.Should().Be(ProcessingResultCode.Success);
1018+
processedFiles.Should().ContainSingle();
1019+
processedFiles.Should().OnlyContain(p => p == Path.Combine(root, "go.mod"));
1020+
}
1021+
1022+
/// <summary>
1023+
/// Verify that nested roots under go mod less than 1.17 are not skipped.
1024+
/// </summary>
1025+
/// <returns>Task.</returns>
1026+
[TestMethod]
1027+
public async Task GoDetector_GoMod_VerifyNestedRootsUnderLT117AreNotSkipped()
1028+
{
1029+
var root = Path.Combine("C:", "root");
1030+
var processedFiles = new List<string>();
1031+
this.SetupMockGoModParser();
1032+
this.mockGoModParser
1033+
.Setup(p => p.ParseAsync(It.IsAny<ISingleFileComponentRecorder>(), It.IsAny<IComponentStream>(), It.IsAny<GoGraphTelemetryRecord>()))
1034+
.ReturnsAsync(true)
1035+
.Callback<ISingleFileComponentRecorder, IComponentStream, GoGraphTelemetryRecord>((_, file, record) =>
1036+
{
1037+
processedFiles.Add(file.Location);
1038+
var rootMod = Path.Combine(root, "go.mod");
1039+
var aMod = Path.Combine(root, "a", "go.mod");
1040+
var bMod = Path.Combine(root, "b", "go.mod");
1041+
record.GoModVersion = file.Location switch
1042+
{
1043+
var loc when loc == rootMod => "1.16",
1044+
var loc when loc == aMod => "1.16",
1045+
var loc when loc == bMod => "1.17",
1046+
_ => null,
1047+
};
1048+
});
1049+
1050+
var (scanResult, componentRecorder) = await this.DetectorTestUtility
1051+
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "a", "go.mod"))
1052+
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "a", "a", "go.mod"))
1053+
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "a", "b", "go.mod"))
1054+
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "b", "go.mod"))
1055+
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "go.mod"))
1056+
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "b", "d", "go.mod"))
1057+
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "b", "a", "go.mod"))
1058+
.ExecuteDetectorAsync();
1059+
1060+
scanResult.ResultCode.Should().Be(ProcessingResultCode.Success);
1061+
processedFiles.Should().HaveCount(5);
1062+
processedFiles.Should().ContainInOrder(
1063+
Path.Combine(root, "go.mod"),
1064+
Path.Combine(root, "a", "go.mod"),
1065+
Path.Combine(root, "b", "go.mod"),
1066+
Path.Combine(root, "a", "a", "go.mod"),
1067+
Path.Combine(root, "a", "b", "go.mod"));
1068+
}
1069+
1070+
/// <summary>
1071+
/// Verify that nested roots are not skipped if parent go.mod parsing fails.
1072+
/// </summary>
1073+
/// <returns>Task.</returns>
1074+
[TestMethod]
1075+
public async Task GoDetector_GoMod_VerifyNestedRootsAreNotSkippedIfParentParseFails()
1076+
{
1077+
var processedFiles = new List<string>();
1078+
var root = Path.Combine("C:", "root");
1079+
this.SetupMockGoModParser();
1080+
1081+
this.mockGoModParser
1082+
.Setup(p => p.ParseAsync(It.IsAny<ISingleFileComponentRecorder>(), It.IsAny<IComponentStream>(), It.IsAny<GoGraphTelemetryRecord>()))
1083+
.ReturnsAsync((ISingleFileComponentRecorder recorder, IComponentStream file, GoGraphTelemetryRecord record) =>
1084+
{
1085+
processedFiles.Add(file.Location);
1086+
var aMod = Path.Combine(root, "a", "go.mod");
1087+
var bMod = Path.Combine(root, "b", "go.mod");
1088+
record.GoModVersion = file.Location switch
1089+
{
1090+
var loc when loc == bMod => "1.18",
1091+
_ => "1.16",
1092+
};
1093+
1094+
// Simulate parse failure only for C:\root\a\go.mod
1095+
if (file.Location == aMod)
1096+
{
1097+
return false;
1098+
}
1099+
1100+
return true;
1101+
});
1102+
1103+
var (scanResult, componentRecorder) = await this.DetectorTestUtility
1104+
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "a", "go.mod"))
1105+
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "a", "a", "go.mod"))
1106+
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "a", "b", "go.mod"))
1107+
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "b", "go.mod"))
1108+
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "go.mod"))
1109+
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "b", "d", "go.mod"))
1110+
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "b", "a", "go.mod"))
1111+
.ExecuteDetectorAsync();
1112+
1113+
scanResult.ResultCode.Should().Be(ProcessingResultCode.Success);
1114+
processedFiles.Should().HaveCount(5);
1115+
processedFiles.Should().ContainInOrder(
1116+
Path.Combine(root, "go.mod"),
1117+
Path.Combine(root, "a", "go.mod"),
1118+
Path.Combine(root, "b", "go.mod"),
1119+
Path.Combine(root, "a", "a", "go.mod"),
1120+
Path.Combine(root, "a", "b", "go.mod"));
1121+
}
1122+
1123+
/// <summary>
1124+
/// Verify that nested directories are skipped once root is processed.
1125+
/// Assume root GoModVersion is >= 1.17.
1126+
/// </summary>
1127+
/// <returns>Task.</returns>
1128+
[TestMethod]
1129+
public async Task GoDetector_GoSum_VerifyNestedRootsUnderGoSum_AreSkipped()
1130+
{
1131+
var processedFiles = new List<string>();
1132+
var root = Path.Combine("C:", "root");
1133+
this.envVarService.Setup(x => x.IsEnvironmentVariableValueTrue("DisableGoCliScan")).Returns(false);
1134+
this.SetupMockGoCLIParser();
1135+
this.mockGoCliParser
1136+
.Setup(p => p.ParseAsync(It.IsAny<ISingleFileComponentRecorder>(), It.IsAny<IComponentStream>(), It.IsAny<GoGraphTelemetryRecord>()))
1137+
.ReturnsAsync(true)
1138+
.Callback<ISingleFileComponentRecorder, IComponentStream, GoGraphTelemetryRecord>((_, file, record) =>
1139+
{
1140+
processedFiles.Add(file.Location);
1141+
});
1142+
1143+
var (scanResult, componentRecorder) = await this.DetectorTestUtility
1144+
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "a", "go.mod"))
1145+
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "a", "a", "go.mod"))
1146+
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "a", "b", "go.mod"))
1147+
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "b", "go.mod"))
1148+
.WithFile("go.sum", string.Empty, fileLocation: Path.Combine(root, "go.sum"))
1149+
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "b", "d", "go.mod"))
1150+
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "b", "a", "go.mod"))
1151+
.ExecuteDetectorAsync();
1152+
1153+
scanResult.ResultCode.Should().Be(ProcessingResultCode.Success);
1154+
processedFiles.Should().ContainSingle();
1155+
processedFiles.Should().OnlyContain(p => p == Path.Combine(root, "go.sum"));
1156+
}
1157+
1158+
[TestMethod]
1159+
public async Task GoDetector_GoSum_VerifyNestedRootsAreNotSkippedIfParentParseFails()
1160+
{
1161+
var processedFiles = new List<string>();
1162+
var root = Path.Combine("C:", "root");
1163+
this.envVarService.Setup(x => x.IsEnvironmentVariableValueTrue("DisableGoCliScan")).Returns(false);
1164+
this.SetupMockGoModParser();
1165+
this.SetupMockGoCLIParser();
1166+
this.SetupMockGoSumParser();
1167+
1168+
this.mockGoModParser
1169+
.Setup(p => p.ParseAsync(It.IsAny<ISingleFileComponentRecorder>(), It.IsAny<IComponentStream>(), It.IsAny<GoGraphTelemetryRecord>()))
1170+
.ReturnsAsync((ISingleFileComponentRecorder recorder, IComponentStream file, GoGraphTelemetryRecord record) =>
1171+
{
1172+
processedFiles.Add(file.Location);
1173+
var bMod = Path.Combine(root, "b", "go.mod");
1174+
record.GoModVersion = file.Location switch
1175+
{
1176+
var loc when loc == bMod => "1.18",
1177+
_ => "1.16",
1178+
};
1179+
1180+
return true;
1181+
});
1182+
1183+
this.mockGoCliParser
1184+
.Setup(p => p.ParseAsync(It.IsAny<ISingleFileComponentRecorder>(), It.IsAny<IComponentStream>(), It.IsAny<GoGraphTelemetryRecord>()))
1185+
.ReturnsAsync((ISingleFileComponentRecorder recorder, IComponentStream file, GoGraphTelemetryRecord record) =>
1186+
{
1187+
processedFiles.Add(file.Location);
1188+
return file.Location != Path.Combine(root, "a", "go.sum");
1189+
});
1190+
1191+
var (scanResult, componentRecorder) = await this.DetectorTestUtility
1192+
.WithFile("go.sum", string.Empty, fileLocation: Path.Combine(root, "a", "go.sum"))
1193+
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "a", "a", "go.mod"))
1194+
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "a", "b", "go.mod"))
1195+
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "b", "go.mod"))
1196+
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "go.mod"))
1197+
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "b", "d", "go.mod"))
1198+
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "b", "a", "go.mod"))
1199+
.ExecuteDetectorAsync();
1200+
1201+
scanResult.ResultCode.Should().Be(ProcessingResultCode.Success);
1202+
processedFiles.Should().HaveCount(5);
1203+
processedFiles.Should().ContainInOrder(
1204+
Path.Combine(root, "go.mod"),
1205+
Path.Combine(root, "a", "go.sum"),
1206+
Path.Combine(root, "b", "go.mod"),
1207+
Path.Combine(root, "a", "a", "go.mod"),
1208+
Path.Combine(root, "a", "b", "go.mod"));
1209+
}
9861210
}

0 commit comments

Comments
 (0)