Skip to content

Commit 324973a

Browse files
ctruedenctrueden
committed
Merge branch 'download'
This adds two new services: * org.scijava.download.DownloadService * org.scijava.task.TaskService Initial versions are minimal but functional. No UI yet for the TaskService; that comes later.
2 parents 72a7e37 + 68aedd9 commit 324973a

26 files changed

+1648
-128
lines changed

‎pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
</parent>
1111

1212
<artifactId>scijava-common</artifactId>
13-
<version>2.64.1-SNAPSHOT</version>
13+
<version>2.65.0-SNAPSHOT</version>
1414

1515
<name>SciJava Common</name>
1616
<description>SciJava Common is a shared library for SciJava software. It provides a plugin framework, with an extensible mechanism for service discovery, backed by its own annotation processor, so that plugins can be loaded dynamically. It is used by downstream projects in the SciJava ecosystem, such as ImageJ and SCIFIO.</description>

‎src/main/java/org/scijava/download/DefaultDownloadService.java

Lines changed: 220 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,220 @@
1+
/*-
2+
* #%L
3+
* SciJava Common shared library for SciJava software.
4+
* %%
5+
* Copyright (C) 2009 - 2017 Board of Regents of the University of
6+
* Wisconsin-Madison, Broad Institute of MIT and Harvard, Max Planck
7+
* Institute of Molecular Cell Biology and Genetics, University of
8+
* Konstanz, and KNIME GmbH.
9+
* %%
10+
* Redistribution and use in source and binary forms, with or without
11+
* modification, are permitted provided that the following conditions are met:
12+
*
13+
* 1. Redistributions of source code must retain the above copyright notice,
14+
* this list of conditions and the following disclaimer.
15+
* 2. Redistributions in binary form must reproduce the above copyright notice,
16+
* this list of conditions and the following disclaimer in the documentation
17+
* and/or other materials provided with the distribution.
18+
*
19+
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20+
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21+
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22+
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
23+
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24+
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25+
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26+
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27+
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28+
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29+
* POSSIBILITY OF SUCH DAMAGE.
30+
* #L%
31+
*/
32+
33+
package org.scijava.download;
34+
35+
import java.io.IOException;
36+
import java.util.Date;
37+
38+
import org.scijava.io.handle.DataHandle;
39+
import org.scijava.io.handle.DataHandleService;
40+
import org.scijava.io.location.Location;
41+
import org.scijava.plugin.Parameter;
42+
import org.scijava.plugin.Plugin;
43+
import org.scijava.service.AbstractService;
44+
import org.scijava.service.Service;
45+
import org.scijava.task.Task;
46+
import org.scijava.task.TaskService;
47+
48+
/**
49+
* Default implementation of {@link DownloadService}.
50+
*
51+
* @author Curtis Rueden
52+
*/
53+
@Plugin(type = Service.class)
54+
public class DefaultDownloadService extends AbstractService implements
55+
DownloadService
56+
{
57+
58+
@Parameter
59+
private DataHandleService dataHandleService;
60+
61+
@Parameter
62+
private TaskService taskService;
63+
64+
@Override
65+
public Download download(final Location source, final Location destination) {
66+
final Task task = taskService.createTask("Download");
67+
return new DefaultDownload(source, destination, task, () -> {
68+
try (final DataHandle<Location> in = dataHandleService.create(source);
69+
final DataHandle<Location> out = dataHandleService.create(
70+
destination))
71+
{
72+
task.setStatusMessage("Downloading " + source.getURI());
73+
copy(task, in, out);
74+
}
75+
catch (final IOException exc) {
76+
// TODO: Improve error handling:
77+
// 1. Consider a better exception handling design here.
78+
// 2. Retry at least a few times if something goes wrong.
79+
throw new RuntimeException(exc);
80+
}
81+
});
82+
}
83+
84+
@Override
85+
public Download download(final Location source, final Location destination,
86+
final LocationCache cache)
87+
{
88+
if (cache == null || !cache.canCache(source)) {
89+
// Caching this location is not supported.
90+
return download(source, destination);
91+
}
92+
93+
final Task task = taskService.createTask("Download");
94+
return new DefaultDownload(source, destination, task, () -> {
95+
final Location cached = cache.cachedLocation(source);
96+
try (
97+
final DataHandle<Location> sourceHandle = dataHandleService.create(source);
98+
final DataHandle<Location> cachedHandle = dataHandleService.create(cached);
99+
final DataHandle<Location> destHandle = dataHandleService.create(destination)
100+
)
101+
{
102+
if (isCachedHandleValid(source, cache, sourceHandle, cachedHandle)) {
103+
// The data is cached; download from the cached source instead.
104+
task.setStatusMessage("Retrieving " + source.getURI());
105+
copy(task, cachedHandle, destHandle);
106+
}
107+
else {
108+
// Data is not yet cached; write to the destination _and_ the cache.
109+
task.setStatusMessage("Downloading + caching " + source.getURI());
110+
copy(task, sourceHandle, //
111+
new MultiWriteHandle(cachedHandle, destHandle));
112+
}
113+
}
114+
catch (final IOException exc) {
115+
// TODO: Improve error handling:
116+
// 1. Consider a better exception handling design here.
117+
// 2. Retry at least a few times if something goes wrong.
118+
throw new RuntimeException(exc);
119+
}
120+
});
121+
}
122+
123+
// -- Helper methods --
124+
125+
private void copy(final Task task, final DataHandle<Location> in,
126+
final DataHandle<Location> out) throws IOException
127+
{
128+
long length;
129+
try {
130+
length = in.length();
131+
}
132+
catch (final IOException exc) {
133+
// Assume unknown length.
134+
length = 0;
135+
}
136+
if (length > 0) task.setProgressMaximum(length);
137+
138+
final int chunkSize = 64 * 1024; // TODO: Make size configurable.
139+
final byte[] buf = new byte[chunkSize];
140+
while (true) {
141+
if (task.isCanceled()) return;
142+
final int r = in.read(buf);
143+
if (r <= 0) break; // EOF
144+
if (task.isCanceled()) return;
145+
out.write(buf, 0, r);
146+
if (length > 0) task.setProgressValue(task.getProgressValue() + r);
147+
}
148+
}
149+
150+
private boolean isCachedHandleValid(final Location source,
151+
final LocationCache cache, final DataHandle<Location> sourceHandle,
152+
final DataHandle<Location> cachedHandle) throws IOException
153+
{
154+
if (!cachedHandle.exists()) return false; // No cached data is present.
155+
156+
// Compare data lengths.
157+
final long sourceLen = sourceHandle.length();
158+
final long cachedLen = cachedHandle.length();
159+
if (sourceLen >= 0 && cachedLen >= 0 && sourceLen != cachedLen) {
160+
// Original and cached sources report different lengths; cache is invalid.
161+
return false;
162+
}
163+
164+
// Compare last modified timestamps.
165+
final Date sourceDate = sourceHandle.lastModified();
166+
final Date cachedDate = cachedHandle.lastModified();
167+
if (sourceDate != null && cachedDate != null && //
168+
sourceDate.after(cachedDate))
169+
{
170+
// Source was changed after cache was written; cache is invalid.
171+
return false;
172+
}
173+
174+
// Compare checksums.
175+
final String sourceChecksum = sourceHandle.checksum();
176+
final String cachedChecksum = cache.loadChecksum(source);
177+
if (sourceChecksum != null && cachedChecksum != null && //
178+
!sourceChecksum.equals(cachedChecksum))
179+
{
180+
// Checksums do not match; cache is invalid.
181+
return false;
182+
}
183+
184+
// Everything matched; we're all good.
185+
return true;
186+
}
187+
188+
// -- Helper classes --
189+
190+
private class DefaultDownload implements Download {
191+
192+
private Location source;
193+
private Location destination;
194+
private Task task;
195+
196+
private DefaultDownload(final Location source, final Location destination,
197+
final Task task, final Runnable r)
198+
{
199+
this.source = source;
200+
this.destination = destination;
201+
this.task = task;
202+
task.run(r);
203+
}
204+
205+
@Override
206+
public Location source() {
207+
return source;
208+
}
209+
210+
@Override
211+
public Location destination() {
212+
return destination;
213+
}
214+
215+
@Override
216+
public Task task() {
217+
return task;
218+
}
219+
}
220+
}

‎src/main/java/org/scijava/download/DiskLocationCache.java

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
/*-
2+
* #%L
3+
* SciJava Common shared library for SciJava software.
4+
* %%
5+
* Copyright (C) 2009 - 2017 Board of Regents of the University of
6+
* Wisconsin-Madison, Broad Institute of MIT and Harvard, Max Planck
7+
* Institute of Molecular Cell Biology and Genetics, University of
8+
* Konstanz, and KNIME GmbH.
9+
* %%
10+
* Redistribution and use in source and binary forms, with or without
11+
* modification, are permitted provided that the following conditions are met:
12+
*
13+
* 1. Redistributions of source code must retain the above copyright notice,
14+
* this list of conditions and the following disclaimer.
15+
* 2. Redistributions in binary form must reproduce the above copyright notice,
16+
* this list of conditions and the following disclaimer in the documentation
17+
* and/or other materials provided with the distribution.
18+
*
19+
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20+
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21+
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22+
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
23+
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24+
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25+
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26+
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27+
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28+
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29+
* POSSIBILITY OF SUCH DAMAGE.
30+
* #L%
31+
*/
32+
33+
package org.scijava.download;
34+
35+
import java.io.File;
36+
import java.io.IOException;
37+
38+
import org.scijava.io.location.FileLocation;
39+
import org.scijava.io.location.Location;
40+
import org.scijava.util.DigestUtils;
41+
import org.scijava.util.FileUtils;
42+
43+
/**
44+
* A file-based implementation of {@link LocationCache}.
45+
*
46+
* @author Curtis Rueden
47+
*/
48+
public class DiskLocationCache implements LocationCache {
49+
50+
private File baseDir = new File(System.getProperty("user.home") +
51+
File.separator + ".scijava" + File.separator + "cache" + File.separator);
52+
53+
private boolean cacheFileLocations;
54+
55+
// -- DiskLocationCache methods --
56+
57+
public File getBaseDirectory() {
58+
return baseDir;
59+
}
60+
61+
public void setBaseDirectory(final File baseDir) {
62+
if (!baseDir.isDirectory()) {
63+
throw new IllegalArgumentException("Not a directory: " + baseDir);
64+
}
65+
this.baseDir = baseDir;
66+
}
67+
68+
public boolean isFileLocationCachingEnabled() {
69+
return cacheFileLocations;
70+
}
71+
72+
public void setFileLocationCachingEnabled(final boolean enabled) {
73+
// NB: It is possible the input file is stored on a volume which is much
74+
// slower than the local disk cache, so we make this setting configurable.
75+
cacheFileLocations = enabled;
76+
}
77+
78+
// -- LocationCache methods --
79+
80+
@Override
81+
public boolean canCache(final Location source) {
82+
if (source instanceof FileLocation && !isFileLocationCachingEnabled()) {
83+
// The cache is not configured to cache files to other files.
84+
return false;
85+
}
86+
return source.getURI() != null;
87+
}
88+
89+
@Override
90+
public Location cachedLocation(final Location source) {
91+
if (!canCache(source)) {
92+
throw new IllegalArgumentException("Uncacheable source: " + source);
93+
}
94+
return new FileLocation(cachedData(source));
95+
}
96+
97+
@Override
98+
public String loadChecksum(final Location source) throws IOException {
99+
final File cachedChecksum = cachedChecksum(source);
100+
if (!cachedChecksum.exists()) return null;
101+
return DigestUtils.string(FileUtils.readFile(cachedChecksum));
102+
}
103+
104+
@Override
105+
public void saveChecksum(final Location source, final String checksum)
106+
throws IOException
107+
{
108+
final File cachedChecksum = cachedChecksum(source);
109+
FileUtils.writeFile(cachedChecksum, DigestUtils.bytes(checksum));
110+
}
111+
112+
// -- Helper methods --
113+
114+
private File cachedData(final Location source) {
115+
return cachedFile(source, ".data");
116+
}
117+
118+
private File cachedChecksum(final Location source) {
119+
return cachedFile(source, ".checksum");
120+
}
121+
122+
private File cachedFile(final Location source, final String suffix) {
123+
final String hexCode = Integer.toHexString(source.hashCode());
124+
return new File(getBaseDirectory(), hexCode + suffix);
125+
}
126+
}

0 commit comments

Comments
 (0)