Skip to content

Commit 8a445ca

Browse files
bcarrierbcarrier
authored
Merge pull request #3138 from eugene7646/file_visitor_fix
File visitor exception handling fix
2 parents 5eefb7a + ca14274 commit 8a445ca

File tree

3 files changed

+167
-118
lines changed

3 files changed

+167
-118
lines changed

‎Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJob.java‎

Lines changed: 84 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -89,30 +89,34 @@ public final class AutoIngestJob implements Comparable<AutoIngestJob>, Serializa
8989
*
9090
* @param manifest The manifest for an automated ingest job.
9191
*/
92-
AutoIngestJob(Manifest manifest) {
93-
/*
94-
* Version 0 fields.
95-
*/
96-
this.manifest = manifest;
97-
this.nodeName = "";
98-
this.caseDirectoryPath = "";
99-
this.priority = DEFAULT_PRIORITY;
100-
this.stage = Stage.PENDING;
101-
this.stageStartDate = manifest.getDateFileCreated();
102-
this.dataSourceProcessor = null;
103-
this.ingestJob = null;
104-
this.cancelled = false;
105-
this.completed = false;
106-
this.completedDate = new Date(0);
107-
this.errorsOccurred = false;
108-
109-
/*
110-
* Version 1 fields.
111-
*/
112-
this.version = CURRENT_VERSION;
113-
this.processingStatus = ProcessingStatus.PENDING;
114-
this.numberOfCrashes = 0;
115-
this.stageDetails = this.getProcessingStageDetails();
92+
AutoIngestJob(Manifest manifest) throws AutoIngestJobException {
93+
try {
94+
/*
95+
* Version 0 fields.
96+
*/
97+
this.manifest = manifest;
98+
this.nodeName = "";
99+
this.caseDirectoryPath = "";
100+
this.priority = DEFAULT_PRIORITY;
101+
this.stage = Stage.PENDING;
102+
this.stageStartDate = manifest.getDateFileCreated();
103+
this.dataSourceProcessor = null;
104+
this.ingestJob = null;
105+
this.cancelled = false;
106+
this.completed = false;
107+
this.completedDate = new Date(0);
108+
this.errorsOccurred = false;
109+
110+
/*
111+
* Version 1 fields.
112+
*/
113+
this.version = CURRENT_VERSION;
114+
this.processingStatus = ProcessingStatus.PENDING;
115+
this.numberOfCrashes = 0;
116+
this.stageDetails = this.getProcessingStageDetails();
117+
} catch (Exception ex) {
118+
throw new AutoIngestJobException(String.format("Error creating automated ingest job"), ex);
119+
}
116120
}
117121

118122
/**
@@ -122,30 +126,34 @@ public final class AutoIngestJob implements Comparable<AutoIngestJob>, Serializa
122126
* @param nodeData The coordination service node data for an automated
123127
* ingest job.
124128
*/
125-
AutoIngestJob(AutoIngestJobNodeData nodeData) {
126-
/*
127-
* Version 0 fields.
128-
*/
129-
this.manifest = new Manifest(nodeData.getManifestFilePath(), nodeData.getManifestFileDate(), nodeData.getCaseName(), nodeData.getDeviceId(), nodeData.getDataSourcePath(), Collections.emptyMap());
130-
this.nodeName = nodeData.getProcessingHostName();
131-
this.caseDirectoryPath = nodeData.getCaseDirectoryPath().toString();
132-
this.priority = nodeData.getPriority();
133-
this.stage = nodeData.getProcessingStage();
134-
this.stageStartDate = nodeData.getProcessingStageStartDate();
135-
this.dataSourceProcessor = null; // Transient data not in node data.
136-
this.ingestJob = null; // Transient data not in node data.
137-
this.cancelled = false; // Transient data not in node data.
138-
this.completed = false; // Transient data not in node data.
139-
this.completedDate = nodeData.getCompletedDate();
140-
this.errorsOccurred = nodeData.getErrorsOccurred();
141-
142-
/*
143-
* Version 1 fields.
144-
*/
145-
this.version = CURRENT_VERSION;
146-
this.processingStatus = nodeData.getProcessingStatus();
147-
this.numberOfCrashes = nodeData.getNumberOfCrashes();
148-
this.stageDetails = this.getProcessingStageDetails();
129+
AutoIngestJob(AutoIngestJobNodeData nodeData) throws AutoIngestJobException {
130+
try {
131+
/*
132+
* Version 0 fields.
133+
*/
134+
this.manifest = new Manifest(nodeData.getManifestFilePath(), nodeData.getManifestFileDate(), nodeData.getCaseName(), nodeData.getDeviceId(), nodeData.getDataSourcePath(), Collections.emptyMap());
135+
this.nodeName = nodeData.getProcessingHostName();
136+
this.caseDirectoryPath = nodeData.getCaseDirectoryPath().toString();
137+
this.priority = nodeData.getPriority();
138+
this.stage = nodeData.getProcessingStage();
139+
this.stageStartDate = nodeData.getProcessingStageStartDate();
140+
this.dataSourceProcessor = null; // Transient data not in node data.
141+
this.ingestJob = null; // Transient data not in node data.
142+
this.cancelled = false; // Transient data not in node data.
143+
this.completed = false; // Transient data not in node data.
144+
this.completedDate = nodeData.getCompletedDate();
145+
this.errorsOccurred = nodeData.getErrorsOccurred();
146+
147+
/*
148+
* Version 1 fields.
149+
*/
150+
this.version = CURRENT_VERSION;
151+
this.processingStatus = nodeData.getProcessingStatus();
152+
this.numberOfCrashes = nodeData.getNumberOfCrashes();
153+
this.stageDetails = this.getProcessingStageDetails();
154+
} catch (Exception ex) {
155+
throw new AutoIngestJobException(String.format("Error creating automated ingest job"), ex);
156+
}
149157
}
150158

151159
/**
@@ -622,5 +630,33 @@ Date getStartDate() {
622630
}
623631

624632
}
633+
634+
/**
635+
* Exception thrown when there is a problem creating auto ingest job.
636+
*/
637+
final static class AutoIngestJobException extends Exception {
625638

639+
private static final long serialVersionUID = 1L;
640+
641+
/**
642+
* Constructs an exception to throw when there is a problem creating
643+
* auto ingest job.
644+
*
645+
* @param message The exception message.
646+
*/
647+
private AutoIngestJobException(String message) {
648+
super(message);
649+
}
650+
651+
/**
652+
* Constructs an exception to throw when there is a problem creating
653+
* auto ingest job.
654+
*
655+
* @param message The exception message.
656+
* @param cause The cause of the exception, if it was an exception.
657+
*/
658+
private AutoIngestJobException(String message, Throwable cause) {
659+
super(message, cause);
660+
}
661+
}
626662
}

‎Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java‎

Lines changed: 82 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@
6161
import javax.annotation.concurrent.GuardedBy;
6262
import javax.annotation.concurrent.Immutable;
6363
import javax.annotation.concurrent.ThreadSafe;
64+
import org.openide.util.Exceptions;
6465
import org.openide.util.Lookup;
6566
import org.sleuthkit.autopsy.casemodule.Case;
6667
import org.sleuthkit.autopsy.casemodule.Case.CaseType;
@@ -93,6 +94,7 @@
9394
import org.sleuthkit.autopsy.experimental.configuration.SharedConfiguration.SharedConfigurationException;
9495
import org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor;
9596
import org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException;
97+
import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestJob.AutoIngestJobException;
9698
import org.sleuthkit.autopsy.ingest.IngestJob;
9799
import org.sleuthkit.autopsy.ingest.IngestJob.CancellationReason;
98100
import org.sleuthkit.autopsy.ingest.IngestJobSettings;
@@ -759,7 +761,7 @@ CaseDeletionResult deleteCase(String caseName, Path caseDirectoryPath) {
759761
AutoIngestJob deletedJob = new AutoIngestJob(nodeData);
760762
deletedJob.setProcessingStatus(AutoIngestJob.ProcessingStatus.DELETED);
761763
this.updateCoordinationServiceNode(deletedJob);
762-
} catch (AutoIngestJobNodeData.InvalidDataException ex) {
764+
} catch (AutoIngestJobNodeData.InvalidDataException | AutoIngestJobException ex) {
763765
SYS_LOGGER.log(Level.WARNING, String.format("Invalid auto ingest job node data for %s", manifestPath), ex);
764766
return CaseDeletionResult.PARTIALLY_DELETED;
765767
} catch (InterruptedException | CoordinationServiceException ex) {
@@ -1015,92 +1017,103 @@ public FileVisitResult preVisitDirectory(Path dirPath, BasicFileAttributes dirAt
10151017
* @return TERMINATE if auto ingest is shutting down, CONTINUE if it has
10161018
* not.
10171019
*
1018-
* @throws IOException if an I/O error occurs, but this implementation
1019-
* does not throw.
10201020
*/
10211021
@Override
1022-
public FileVisitResult visitFile(Path filePath, BasicFileAttributes attrs) throws IOException {
1022+
public FileVisitResult visitFile(Path filePath, BasicFileAttributes attrs) {
10231023
if (Thread.currentThread().isInterrupted()) {
10241024
return TERMINATE;
10251025
}
10261026

1027-
Manifest manifest = null;
1028-
for (ManifestFileParser parser : Lookup.getDefault().lookupAll(ManifestFileParser.class)) {
1029-
if (parser.fileIsManifest(filePath)) {
1030-
try {
1031-
manifest = parser.parse(filePath);
1032-
break;
1033-
} catch (ManifestFileParserException ex) {
1034-
SYS_LOGGER.log(Level.SEVERE, String.format("Error attempting to parse %s with parser %s", filePath, parser.getClass().getCanonicalName()), ex);
1027+
try {
1028+
Manifest manifest = null;
1029+
for (ManifestFileParser parser : Lookup.getDefault().lookupAll(ManifestFileParser.class)) {
1030+
if (parser.fileIsManifest(filePath)) {
1031+
try {
1032+
manifest = parser.parse(filePath);
1033+
break;
1034+
} catch (ManifestFileParserException ex) {
1035+
SYS_LOGGER.log(Level.SEVERE, String.format("Error attempting to parse %s with parser %s", filePath, parser.getClass().getCanonicalName()), ex);
1036+
}
1037+
}
1038+
if (Thread.currentThread().isInterrupted()) {
1039+
return TERMINATE;
10351040
}
10361041
}
1042+
10371043
if (Thread.currentThread().isInterrupted()) {
10381044
return TERMINATE;
10391045
}
1040-
}
1041-
1042-
if (Thread.currentThread().isInterrupted()) {
1043-
return TERMINATE;
1044-
}
10451046

1046-
if (null != manifest) {
1047-
/*
1047+
if (null != manifest) {
1048+
/*
10481049
* Update the mapping of case names to manifest paths that is
10491050
* used for case deletion.
1050-
*/
1051-
String caseName = manifest.getCaseName();
1052-
Path manifestPath = manifest.getFilePath();
1053-
if (casesToManifests.containsKey(caseName)) {
1054-
Set<Path> manifestPaths = casesToManifests.get(caseName);
1055-
manifestPaths.add(manifestPath);
1056-
} else {
1057-
Set<Path> manifestPaths = new HashSet<>();
1058-
manifestPaths.add(manifestPath);
1059-
casesToManifests.put(caseName, manifestPaths);
1060-
}
1051+
*/
1052+
String caseName = manifest.getCaseName();
1053+
Path manifestPath = manifest.getFilePath();
1054+
if (casesToManifests.containsKey(caseName)) {
1055+
Set<Path> manifestPaths = casesToManifests.get(caseName);
1056+
manifestPaths.add(manifestPath);
1057+
} else {
1058+
Set<Path> manifestPaths = new HashSet<>();
1059+
manifestPaths.add(manifestPath);
1060+
casesToManifests.put(caseName, manifestPaths);
1061+
}
10611062

1062-
/*
1063+
/*
10631064
* Add a job to the pending jobs queue, the completed jobs list,
10641065
* or do crashed job recovery, as required.
1065-
*/
1066-
try {
1067-
byte[] rawData = coordinationService.getNodeData(CoordinationService.CategoryNode.MANIFESTS, manifestPath.toString());
1068-
if (null != rawData && rawData.length > 0) {
1069-
try {
1070-
AutoIngestJobNodeData nodeData = new AutoIngestJobNodeData(rawData);
1071-
AutoIngestJob.ProcessingStatus processingStatus = nodeData.getProcessingStatus();
1072-
switch (processingStatus) {
1073-
case PENDING:
1074-
addPendingJob(manifest, nodeData);
1075-
break;
1076-
case PROCESSING:
1077-
doRecoveryIfCrashed(manifest, nodeData);
1078-
break;
1079-
case COMPLETED:
1080-
addCompletedJob(manifest, nodeData);
1081-
break;
1082-
case DELETED:
1083-
/*
1066+
*/
1067+
try {
1068+
byte[] rawData = coordinationService.getNodeData(CoordinationService.CategoryNode.MANIFESTS, manifestPath.toString());
1069+
if (null != rawData && rawData.length > 0) {
1070+
try {
1071+
AutoIngestJobNodeData nodeData = new AutoIngestJobNodeData(rawData);
1072+
AutoIngestJob.ProcessingStatus processingStatus = nodeData.getProcessingStatus();
1073+
switch (processingStatus) {
1074+
case PENDING:
1075+
addPendingJob(manifest, nodeData);
1076+
break;
1077+
case PROCESSING:
1078+
doRecoveryIfCrashed(manifest, nodeData);
1079+
break;
1080+
case COMPLETED:
1081+
addCompletedJob(manifest, nodeData);
1082+
break;
1083+
case DELETED:
1084+
/*
10841085
* Ignore jobs marked as "deleted."
1085-
*/
1086-
break;
1087-
default:
1088-
SYS_LOGGER.log(Level.SEVERE, "Unknown ManifestNodeData.ProcessingStatus");
1089-
break;
1086+
*/
1087+
break;
1088+
default:
1089+
SYS_LOGGER.log(Level.SEVERE, "Unknown ManifestNodeData.ProcessingStatus");
1090+
break;
1091+
}
1092+
} catch (AutoIngestJobNodeData.InvalidDataException | AutoIngestJobException ex) {
1093+
SYS_LOGGER.log(Level.SEVERE, String.format("Invalid auto ingest job node data for %s", manifestPath), ex);
1094+
}
1095+
} else {
1096+
try {
1097+
addNewPendingJob(manifest);
1098+
} catch (AutoIngestJobException ex) {
1099+
SYS_LOGGER.log(Level.SEVERE, String.format("Invalid manifest data for %s", manifestPath), ex);
10901100
}
1091-
} catch (AutoIngestJobNodeData.InvalidDataException ex) {
1092-
SYS_LOGGER.log(Level.WARNING, String.format("Invalid auto ingest job node data for %s", manifestPath), ex);
10931101
}
1094-
} else {
1095-
addNewPendingJob(manifest);
1102+
} catch (CoordinationServiceException ex) {
1103+
SYS_LOGGER.log(Level.SEVERE, String.format("Error transmitting node data for %s", manifestPath), ex);
1104+
return CONTINUE;
1105+
} catch (InterruptedException ex) {
1106+
Thread.currentThread().interrupt();
1107+
return TERMINATE;
10961108
}
1097-
} catch (CoordinationServiceException ex) {
1098-
SYS_LOGGER.log(Level.SEVERE, String.format("Error transmitting node data for %s", manifestPath), ex);
1099-
return CONTINUE;
1100-
} catch (InterruptedException ex) {
1101-
Thread.currentThread().interrupt();
1102-
return TERMINATE;
11031109
}
1110+
1111+
} catch (Exception ex) {
1112+
// Catch all unhandled and unexpected exceptions. Otherwise one bad file
1113+
// can stop the entire input folder scanning. Given that the exception is unexpected,
1114+
// I'm hesitant to add logging which requires accessing or de-referencing data.
1115+
SYS_LOGGER.log(Level.SEVERE, "Unexpected exception in file visitor", ex);
1116+
return CONTINUE;
11041117
}
11051118

11061119
if (!Thread.currentThread().isInterrupted()) {
@@ -1122,7 +1135,7 @@ public FileVisitResult visitFile(Path filePath, BasicFileAttributes attrs) throw
11221135
* blocked, i.e., if auto ingest is
11231136
* shutting down.
11241137
*/
1125-
private void addPendingJob(Manifest manifest, AutoIngestJobNodeData nodeData) throws InterruptedException {
1138+
private void addPendingJob(Manifest manifest, AutoIngestJobNodeData nodeData) throws InterruptedException, AutoIngestJobException {
11261139
AutoIngestJob job;
11271140
if (nodeData.getVersion() == AutoIngestJobNodeData.getCurrentVersion()) {
11281141
job = new AutoIngestJob(nodeData);
@@ -1176,7 +1189,7 @@ private void addPendingJob(Manifest manifest, AutoIngestJobNodeData nodeData) th
11761189
* blocked, i.e., if auto ingest is
11771190
* shutting down.
11781191
*/
1179-
private void addNewPendingJob(Manifest manifest) throws InterruptedException {
1192+
private void addNewPendingJob(Manifest manifest) throws InterruptedException, AutoIngestJobException {
11801193
/*
11811194
* Create the coordination service node data for the job. Note that
11821195
* getting the lock will create the node for the job (with no data)
@@ -1218,7 +1231,7 @@ private void addNewPendingJob(Manifest manifest) throws InterruptedException {
12181231
* blocked, i.e., if auto ingest is
12191232
* shutting down.
12201233
*/
1221-
private void doRecoveryIfCrashed(Manifest manifest, AutoIngestJobNodeData nodeData) throws InterruptedException {
1234+
private void doRecoveryIfCrashed(Manifest manifest, AutoIngestJobNodeData nodeData) throws InterruptedException, AutoIngestJobException {
12221235
/*
12231236
* Try to get an exclusive lock on the coordination service node for
12241237
* the job. If the lock cannot be obtained, another host in the auto
@@ -1314,7 +1327,7 @@ private void doRecoveryIfCrashed(Manifest manifest, AutoIngestJobNodeData nodeDa
13141327
* @throws CoordinationServiceException
13151328
* @throws InterruptedException
13161329
*/
1317-
private void addCompletedJob(Manifest manifest, AutoIngestJobNodeData nodeData) throws CoordinationServiceException, InterruptedException {
1330+
private void addCompletedJob(Manifest manifest, AutoIngestJobNodeData nodeData) throws CoordinationServiceException, InterruptedException, AutoIngestJobException {
13181331
Path caseDirectoryPath = PathUtils.findCaseDirectory(rootOutputDirectory, manifest.getCaseName());
13191332
if (null != caseDirectoryPath) {
13201333
AutoIngestJob job;

0 commit comments

Comments
 (0)