Skip to content

Commit 04053f0

Browse files
kpuriIpackpuriIpac
authored
IRSA-7251: Merge PR #1843 from Caltech-IPAC/IRSA-7251-spherex-bugs
IRSA-7251 SPHEREx bug fixes
2 parents 4964225 + b3e84b3 commit 04053f0

File tree

9 files changed

+36
-65
lines changed

9 files changed

+36
-65
lines changed

‎src/firefly/java/edu/caltech/ipac/firefly/core/background/EmailNotification.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ public static void sendNotification(JobInfo jobInfo) {
109109
String sugName = ifNotNull(jobInfo.getAux().getTitle()).getOrElse("Download");
110110
String curlScript = getDownloadURL(makeScript(jobInfo, Curl), makeScriptFilename(Curl, sugName), jobInfo.getMeta().getAppUrl());
111111
String wgetScript = getDownloadURL(makeScript(jobInfo, Wget), makeScriptFilename(Wget, sugName), jobInfo.getMeta().getAppUrl());
112-
String directUrls = getDownloadURL(makeScript(jobInfo, ScriptAttributes.URLsOnly), makeScriptFilename(URLsOnly, sugName), jobInfo.getMeta().getAppUrl());
112+
String directUrls = getDownloadURL(makeScript(jobInfo, ScriptAttributes.URLList), makeScriptFilename(URLList, sugName), jobInfo.getMeta().getAppUrl());
113113
String msg = zipSuccess.formatted(name, curlScript, wgetScript, directUrls, contact);
114114
EMailUtil.sendMessage(new String[]{email}, null, null, subject, msg);
115115
} catch (Exception e) {
@@ -118,7 +118,7 @@ public static void sendNotification(JobInfo jobInfo) {
118118

119119
} else if (type == Job.Type.SCRIPT) {
120120

121-
String msg = success.formatted(name, getUrl(jobInfo, Curl), getUrl(jobInfo, Wget), getUrl(jobInfo, URLsOnly), contact);
121+
String msg = success.formatted(name, getUrl(jobInfo, Curl), getUrl(jobInfo, Wget), getUrl(jobInfo, URLList), contact);
122122
Try.it(() -> EMailUtil.sendMessage(new String[]{email}, null, null, subject, msg))
123123
.getOrElse(e -> Logger.getLogger().error(e));
124124
} else {
@@ -130,7 +130,7 @@ public static void sendNotification(JobInfo jobInfo) {
130130
}
131131

132132
public static String getUrl(JobInfo jobInfo, ScriptAttributes type) {
133-
String matcher = type == URLsOnly ? "list" : type.name().toLowerCase();
133+
String matcher = type == URLList ? "list" : type.name().toLowerCase();
134134
return jobInfo.getResults().stream()
135135
.filter(r -> r.id().contains(matcher))
136136
.map(r -> r.href()).findFirst().orElse(null);

‎src/firefly/java/edu/caltech/ipac/firefly/core/background/ScriptAttributes.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,5 +12,5 @@
1212
/**
1313
* @author Trey Roby
1414
*/
15-
public enum ScriptAttributes {URLsOnly, Unzip, Curl, Wget}
15+
public enum ScriptAttributes {URLList, Unzip, Curl, Wget}
1616

‎src/firefly/java/edu/caltech/ipac/firefly/server/packagedata/DownloadScriptWorker.java‎

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,8 @@
44

55
package edu.caltech.ipac.firefly.server.packagedata;
66

7-
import java.io.File;
8-
import java.util.*;
9-
import java.util.function.Function;
10-
7+
import edu.caltech.ipac.firefly.core.background.Job;
8+
import edu.caltech.ipac.firefly.core.background.JobInfo;
119
import edu.caltech.ipac.firefly.core.background.JobUtil;
1210
import edu.caltech.ipac.firefly.core.background.ScriptAttributes;
1311
import edu.caltech.ipac.firefly.data.DownloadRequest;
@@ -20,17 +18,17 @@
2018
import edu.caltech.ipac.firefly.server.util.Logger;
2119
import edu.caltech.ipac.firefly.server.ws.WsServerParams;
2220
import edu.caltech.ipac.firefly.server.ws.WsServerUtils;
23-
import edu.caltech.ipac.firefly.core.background.Job;
24-
import edu.caltech.ipac.firefly.core.background.JobInfo;
21+
22+
import java.io.File;
23+
import java.util.List;
2524

2625
import static edu.caltech.ipac.firefly.core.Util.Opt.ifNotNull;
27-
import static edu.caltech.ipac.firefly.core.background.JobManager.updateJobInfo;
26+
import static edu.caltech.ipac.firefly.core.Util.Try;
2827
import static edu.caltech.ipac.firefly.core.background.ScriptAttributes.*;
2928
import static edu.caltech.ipac.firefly.server.servlets.AnyFileDownload.getDownloadURL;
3029
import static edu.caltech.ipac.firefly.server.util.DownloadScript.makeScriptFilename;
3130
import static edu.caltech.ipac.firefly.server.ws.WsServerParams.WS_SERVER_PARAMS.CURRENTRELPATH;
3231
import static edu.caltech.ipac.util.StringUtils.isEmpty;
33-
import static edu.caltech.ipac.firefly.core.Util.Try;
3432

3533
/**
3634
* Downloads a script with products from the List<FileInfo>
@@ -83,18 +81,18 @@ public String doCommand(SrvParam params) throws Exception {
8381

8482
File curlScript = makeScript(result, Curl, dataDesc);
8583
File wgetScript = makeScript(result, Wget, dataDesc);
86-
File urlsFile = makeScript(result, URLsOnly, dataDesc);
84+
File urlsFile = makeScript(result, URLList, dataDesc);
8785

8886
// handle 'save to Workspace' option: pushes downloaded script (.sh or .txt) to workspace (LLY: should probably ignore. what is the use case?)
8987
if (!isEmpty(wsDestPath)) {
9088
Try.it(() -> new WsServerUtils().putFile(new WsServerParams().set(CURRENTRELPATH, wsDestPath + makeScriptFilename(Curl, suggestedName)), curlScript));
9189
Try.it(() -> new WsServerUtils().putFile(new WsServerParams().set(CURRENTRELPATH, wsDestPath + makeScriptFilename(Wget, suggestedName)), wgetScript));
92-
Try.it(() -> new WsServerUtils().putFile(new WsServerParams().set(CURRENTRELPATH, wsDestPath + makeScriptFilename(URLsOnly, suggestedName)), urlsFile));
90+
Try.it(() -> new WsServerUtils().putFile(new WsServerParams().set(CURRENTRELPATH, wsDestPath + makeScriptFilename(URLList, suggestedName)), urlsFile));
9391
}
9492

9593
String curlScriptUrl = getDownloadURL(curlScript, makeScriptFilename(Curl, suggestedName));
9694
String wgetScriptUrl = getDownloadURL(wgetScript, makeScriptFilename(Wget, suggestedName));
97-
String urlsFileUrl = getDownloadURL(urlsFile, makeScriptFilename(URLsOnly, suggestedName));
95+
String urlsFileUrl = getDownloadURL(urlsFile, makeScriptFilename(URLList, suggestedName));
9896

9997
sendJobUpdate(ji -> {
10098
String summary = String.format("%,d files were processed.", totalFiles);

‎src/firefly/java/edu/caltech/ipac/firefly/server/query/ObsCorePackager.java‎

Lines changed: 2 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -173,28 +173,6 @@ public static List<FileInfo> parseDatalink(URL url, DownloadRequest request, Str
173173
Map<String, ServDescUrl> serDefUrls = new HashMap<>();
174174

175175
for (DataGroup dg : groups) {
176-
boolean makeDataLinkFolder = false;
177-
int countValidDLFiles = 0;
178-
//do one initial pass through semantics column to determine if we need to create a folder for datalink files
179-
for (int i=0; i < dg.size(); i++) {
180-
String sem = Objects.toString(dg.getData(SEMANTICS, i), null);
181-
if (sem == null) {
182-
continue;
183-
}
184-
185-
if (products != null) { // Check if products exist and contains sem
186-
if (Arrays.asList(products).contains(sem) || Arrays.asList(products).contains("*")) { //* refers to all data products
187-
countValidDLFiles++;
188-
}
189-
}
190-
else countValidDLFiles++;
191-
192-
if (countValidDLFiles > 1) {
193-
makeDataLinkFolder = true;
194-
break;
195-
}
196-
}
197-
198176
for (int i=0; i < dg.size(); i++) {
199177
//if (indices != null && !indices.contains(i)) continue;
200178
String accessUrl = Objects.toString(dg.getData(ACCESS_URL, i), null);
@@ -228,10 +206,10 @@ public static List<FileInfo> parseDatalink(URL url, DownloadRequest request, Str
228206
String extension = "unknown";
229207
extension = content_type != null ? getExtFromURL(content_type) : getExtFromURL(productUrl);
230208
ext_file_name += "." + extension;
231-
ext_file_name = (isFlattenedStructure || (!makeDataLinkFolder)) ?
209+
ext_file_name = (isFlattenedStructure) ?
232210
ext_file_name : prepend_file_name + "/" + ext_file_name;
233211
} else {
234-
ext_file_name = (isFlattenedStructure || (!makeDataLinkFolder)) ?
212+
ext_file_name = (isFlattenedStructure) ?
235213
ext_file_name : (prepend_file_name + "/" + (fileName != null ? fileName : ""));
236214
}
237215

‎src/firefly/java/edu/caltech/ipac/firefly/server/util/DownloadScript.java‎

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -9,20 +9,12 @@
99
import edu.caltech.ipac.firefly.server.packagedata.FileGroup;
1010
import edu.caltech.ipac.table.IpacTableUtil;
1111
import edu.caltech.ipac.util.AppProperties;
12-
import edu.caltech.ipac.util.StringUtils;
13-
import edu.caltech.ipac.util.download.URLDownload;
14-
15-
import java.io.BufferedWriter;
16-
import java.io.File;
17-
import java.io.FileWriter;
18-
import java.io.IOException;
19-
import java.io.PrintWriter;
20-
import java.io.Serializable;
12+
13+
import java.io.*;
2114
import java.util.ArrayList;
2215
import java.util.Arrays;
2316
import java.util.Date;
2417
import java.util.List;
25-
import java.util.function.Function;
2618

2719
import static edu.caltech.ipac.firefly.core.Util.Opt.ifNotNull;
2820
import static edu.caltech.ipac.firefly.core.background.ScriptAttributes.*;
@@ -41,7 +33,9 @@ public class DownloadScript {
4133
*
4234
* Date: %s
4335
*
44-
* Download %s from %s
36+
* Script to download %s data from %s
37+
*
38+
* The script below defines a subroutine to download the files
4539
*
4640
************************************************
4741
@@ -222,8 +216,8 @@ public static void createScript(File outFile,
222216

223217
try (PrintWriter writer = new PrintWriter(new BufferedWriter(new FileWriter(outFile), IpacTableUtil.FILE_IO_BUFFER_SIZE))) {
224218

225-
if (is(URLsOnly, attribs)) { // most tools do not natively support comments; maybe we should exclude them.
226-
writer.printf("# Date: %s --- Download %s from %s \n", new Date(), dataDesc, FROM_ORG);
219+
if (is(URLList, attribs)) { // most tools do not natively support comments; maybe we should exclude them.
220+
writer.printf("# Date: %s --- Download %s data from %s \n", new Date(), dataDesc, FROM_ORG);
227221
urlInfos.forEach(pi -> writer.println(pi.getHref()));
228222
return;
229223
}
@@ -248,7 +242,7 @@ public static void createScript(File outFile,
248242
}
249243

250244
public static String makeScriptFilename(ScriptAttributes type, String suggName) {
251-
String ext = type.equals(URLsOnly) ? "txt" : "sh";
245+
String ext = type.equals(URLList) ? "txt" : "sh";
252246
return "%s-%s.%s".formatted(suggName, type.name().toLowerCase(), ext);
253247
};
254248

‎src/firefly/js/core/background/BackgroundUtil.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -171,7 +171,7 @@ export function getErrMsg(jobInfo) {
171171
return jobInfo?.errorSummary?.message;
172172
}
173173

174-
export const SCRIPT_ATTRIB = new Enum(['URLsOnly', 'Unzip', 'Ditto', 'Curl', 'Wget', 'RemoveZip']);
174+
export const SCRIPT_ATTRIB = new Enum(['URLList', 'Unzip', 'Ditto', 'Curl', 'Wget', 'RemoveZip']);
175175

176176
export function doPackageRequest({dlRequest, searchRequest, selectInfo, bgKey, downloadType, onComplete}) {
177177

‎src/firefly/js/templates/common/ttFeatureWatchers.js‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,7 @@ export const PrepareDownload = React.memo(({table_id, tbl_title, viewerId, showF
131131
'#auxiliary': 'Auxiliary',
132132
'#progenitor': 'Progenitor',
133133
'#thumbnail': 'Thumbnail',
134+
'#calibration': 'Calibration',
134135
'#preview': 'Preview',
135136
'#package': 'Package',
136137
'#weight': 'Weight'
@@ -150,7 +151,7 @@ export const PrepareDownload = React.memo(({table_id, tbl_title, viewerId, showF
150151
options = []; //fallback only to "All data" only below
151152
}
152153
//ensure '*' ("All data") is always included, even as a fallback when semList is empty
153-
options.push({ label: 'All data', value: '*' });
154+
options.push({ label: 'All Data', value: '*' });
154155
return options;
155156
}, [semList, labelMap]);
156157

‎src/firefly/js/ui/ScriptDownloadDialog.jsx‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ class ScripDownloadPanel extends PureComponent {
3939

4040
constructor(props) {
4141
super(props);
42-
this.state = {urlsOnly: false};
42+
this.state = {urlList: false};
4343
this.onSubmit = this.onSubmit.bind(this);
4444
}
4545

@@ -56,8 +56,8 @@ class ScripDownloadPanel extends PureComponent {
5656
storeUpdate() {
5757
if (!this.isUnmounted) {
5858
const fields = FieldGroupUtils.getGroupFields('ScriptDownloadDialog');
59-
const urlsOnly = FieldGroupUtils.getFldValue(fields, 'URLsOnly');
60-
this.setState({urlsOnly});
59+
const urlList = FieldGroupUtils.getFldValue(fields, 'URLList');
60+
this.setState({urlList});
6161
}
6262
}
6363

@@ -73,7 +73,7 @@ class ScripDownloadPanel extends PureComponent {
7373

7474
render() {
7575
const {help_id, jobId} = this.props;
76-
const {urlsOnly} = this.state;
76+
const {urlList} = this.state;
7777
return (
7878
<div style = {{margin: '4px'}}>
7979
<FormPanel
@@ -84,7 +84,7 @@ class ScripDownloadPanel extends PureComponent {
8484
help_id={help_id}>
8585

8686
<FieldGroup groupKey={'ScriptDownloadDialog'}>
87-
<div style={{visibility: (urlsOnly ? 'hidden' : 'visible')}}>
87+
<div style={{visibility: (urlList ? 'hidden' : 'visible')}}>
8888
<ListBoxInputField
8989
fieldKey ='downloader'
9090
initialState = {{
@@ -112,10 +112,10 @@ class ScripDownloadPanel extends PureComponent {
112112
</div>
113113

114114
<CheckboxGroupInputField
115-
fieldKey='URLsOnly'
115+
fieldKey='URLList'
116116
initialState= {{label : ' '}}
117117
options={[
118-
{label: 'Download just a list of URLs', value: 'URLsOnly'}
118+
{label: 'Download just a list of URLs', value: 'URLList'}
119119
]}
120120
/>
121121
</FieldGroup>

‎src/firefly/test/edu/caltech/ipac/firefly/util/DownloadScriptTest.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,19 +29,19 @@ public void tearDown() {
2929
}
3030

3131
@Test
32-
public void urlsOnly() throws Exception {
32+
public void urlList() throws Exception {
3333
List<FileGroup> fileInfoList = List.of(new FileGroup(List.of(
3434
new FileInfo("https://example.com/file1.txt", "mydata/file1.txt", 0),
3535
new FileInfo("https://example.com/file2.txt", "mydata/file1.txt", 0)
3636
)));
3737

38-
DownloadScript.createScript(tempScript, "Test Data", fileInfoList, URLsOnly);
38+
DownloadScript.createScript(tempScript, "Test Data", fileInfoList, URLList);
3939

4040
List<String> lines = Files.readAllLines(tempScript.toPath());
4141

4242
assertTrue("File1 URL should be present", lines.contains("https://example.com/file1.txt"));
4343
assertTrue("File2 URL should be present", lines.contains("https://example.com/file2.txt"));
44-
assertFalse("Script header should not exist in URLsOnly mode", lines.stream().anyMatch(line -> line.startsWith("#!")));
44+
assertFalse("Script header should not exist in URLList mode", lines.stream().anyMatch(line -> line.startsWith("#!")));
4545
}
4646

4747
@Test

0 commit comments

Comments
 (0)