Firefly-1815: upload a file via an action, usally firefly_client

 - includes clean up and response to feedback

Author: robyww

src/firefly/java/edu/caltech/ipac/firefly/server/servlets/AnyFileUpload.java

@@ -63,12 +63,14 @@ public class AnyFileUpload extends BaseHttpServlet {
     private static final String WORKSPACE_PUT = "workspacePut";
     private static final String WS_CMD = "wsCmd";
     public  static final String ANALYZER_ID = "analyzerId";
+    public  static final String NAME = "name";
     /** use the hips cache hierarchy */
     public  static final String HIPS_CACHE = "hipsCache";
     /** load from a URL */
     private static final String URL = "URL";
     /** load from a WebPlotRequest */
     private static final String WEB_PLOT_REQUEST = "webPlotRequest";
+    private static final String FILE_ON_SERVER = "fileOnServer";
     /** run file analysis and return an analysis json object */
     private static final String FILE_ANALYSIS= "fileAnalysis";
 
@@ -116,7 +118,7 @@ public static void doFileUpload(HttpServletRequest req, HttpServletResponse res)
             int responseCode= statusFileInfo.getResponseCode();
 
             if (responseCode>=400) {
-                res.sendError(statusFileInfo.getResponseCode(), statusFileInfo.getResponseCodeMsg());
+                res.sendError(responseCode, statusFileInfo.getResponseCodeMsg());
                 return;
             }
 
@@ -144,7 +146,6 @@ public static void doFileUpload(HttpServletRequest req, HttpServletResponse res)
 
             // returns the fileCacheKey or full analysis json
             String returnVal= analyzeFile ? callAnalysis(sp,statusFileInfo,uploadFileInfo,fileCacheKey) : fileCacheKey;
-            if (responseCode>=400) throw new Exception(codeFailMsg(responseCode));
 
             sendReturnMsg(res, 200, null, returnVal);
             Counters.getInstance().increment(Counters.Category.Upload, uploadFileInfo.getContentType());
@@ -175,7 +176,7 @@ private static boolean isUrlFail(FileInfo statusFileInfo) {
     private static UploadFileInfo makeUploadFileInfo(FileInfo statusFileInfo, String fname) {
         File file= statusFileInfo.getFile();
         return new UploadFileInfo(ServerContext.replaceWithPrefix(file), file, fname!=null ? fname : file.getName(),
-                statusFileInfo.getContentType());
+                statusFileInfo.getContentType(), statusFileInfo.getResponseCode());
     }
 
     private static void updateFeedback(String statusKey, long totalRead) {
@@ -225,6 +226,7 @@ private static Result retrieveFile(SrvParam sp, FileItemStream uploadedItem) thr
         String wsCmd = sp.getOptional(WS_CMD);
         String fromUrl = sp.getOptional(URL);
         WebPlotRequest fromWPR= sp.getOptionalWebPlotRequest(WEB_PLOT_REQUEST);
+        String fileOnServer= sp.getOptional(FILE_ON_SERVER);
         boolean hipsCache = sp.getOptionalBoolean(HIPS_CACHE,false);
 
         UploadFileInfo uploadFileInfo;
@@ -254,6 +256,14 @@ private static Result retrieveFile(SrvParam sp, FileItemStream uploadedItem) thr
             if (retrieve==null) throw new Exception("Could not determine how to retrieve file");
             statusFileInfo = retrieve.getFile(fromWPR,false);
             uploadFileInfo= makeUploadFileInfo(statusFileInfo,null);
+        } else if (fileOnServer != null) {
+            File f= ServerContext.convertToFile(fileOnServer);
+            if (!f.canRead()) {
+                return new Result(new FileInfo(404),null);
+            }
+            String name= sp.getOptional(NAME,f.getName());
+            uploadFileInfo= new UploadFileInfo(fileOnServer, f,name,null);
+            statusFileInfo= new FileInfo(uploadFileInfo.getFile());
         } else if (uploadedItem != null) {
             // it's a stream from multipart.. write it to disk
             String name = uploadedItem.getName();

src/firefly/java/edu/caltech/ipac/firefly/server/util/multipart/UploadFileInfo.java

@@ -3,6 +3,8 @@
  */
 package edu.caltech.ipac.firefly.server.util.multipart;
 
+import edu.caltech.ipac.firefly.data.FileInfo;
+
 import java.io.File;
 import java.io.Serializable;
 
@@ -17,13 +19,27 @@ public class UploadFileInfo implements Serializable {
     private File file;
     private String fileName;
     private String contentType;
+    private int responseCode;
     private long size;
+    private transient FileInfo fileInfo;
+
+    public UploadFileInfo(String pname, File file, String fileName, String contentType, int responseCode) {
+        this.pname = pname;
+        this.file = file;
+        this.fileName = fileName;
+        this.contentType = contentType;
+        this.responseCode = responseCode;
+        if (file != null && file.exists()) {
+            size = file.length();
+        }
+    }
 
     public UploadFileInfo(String pname, File file, String fileName, String contentType) {
         this.pname = pname;
         this.file = file;
         this.fileName = fileName;
         this.contentType = contentType;
+        this.responseCode = 200;
         if (file != null && file.exists()) {
             size = file.length();
         }
@@ -53,6 +69,8 @@ public void setFileName(String fileName) {
         this.fileName = fileName;
     }
 
+    public int getResponseCode() {return responseCode;}
+
     @Override
     public String toString() {
         return "FileEntry{" + fileName + "[" + size + "]}";

src/firefly/js/api/webApiCommands/ImageCommands.js

@@ -21,7 +21,7 @@ const imageOverview= {
     ],
     allowAdditionalParameters: true,
     parameters: {
-        url : 'URL of the FITS image',
+        url : 'URL to the FITS image',
         title : 'title to show',
         service : 'one of IRIS, ISSA, DSS, SDSS, TWOMASS, MSX-ATLAS, WISE, ATLAS,ZTF, PTF',
         SurveyKey : [

src/firefly/js/api/webApiCommands/ViewerWebApiCommands.js

@@ -1,4 +1,5 @@
 import {isEmpty} from 'lodash';
+import {getAnyFileLoadCommands} from './AnyFileLoadCommands';
 import {getImageCommands} from './ImageCommands';
 import {getLsdbCommands} from './LsdbCommands';
 import {getTabCommands} from './TabCommands';
@@ -16,7 +17,8 @@ export function getFireflyViewerWebApiCommands(cmdNameList, tapPanelList=[]) {
         ...getImageCommands(),
         ...getTableCommands(),
         ...getTapCommands(tapPanelList),
-        ...getLsdbCommands()
+        ...getLsdbCommands(),
+        ...getAnyFileLoadCommands()
     ];
 
     if (isEmpty(cmdNameList)) return allCommands;

src/firefly/js/core/AppDataCntlr.js

@@ -37,6 +37,7 @@ export const LOAD_SEARCHES = `${APP_DATA_PATH}.loadSearches`;
 export const NOTIFY_REMOTE_APP_READY = `${APP_DATA_PATH}.notifyRemoteAppReady`;
 export const FORM_SUBMIT = `${APP_DATA_PATH}.formSubmit`;
 export const FORM_CANCEL = `${APP_DATA_PATH}.formCancel`;
+export const EXTERNAL_UPLOAD = `${APP_DATA_PATH}.externalUpload`;
 
 /** fired when there's a connection is added/removed from this channel.  useful for tracking connections in channel, etc   */
 export const WS_CONN_UPDATED = `${APP_DATA_PATH}.wsConnUpdated`;
@@ -46,7 +47,7 @@ export const GRAB_WINDOW_FOCUS = `${APP_DATA_PATH}.grabFocus`;
 
 /** the extension to add to a channel string to make a viewer channel */
 const CHANNEL_VIEWER_EXTENSION = '__viewer';
-const channel_matcher = new RegExp(`(.+)${CHANNEL_VIEWER_EXTENSION}(?:-(.+))?`)
+const channel_matcher = new RegExp(`(.+)${CHANNEL_VIEWER_EXTENSION}(?:-(.+))?`);
 
 /** @type {SearchInfo} */
 const searchInfo = {};
@@ -209,6 +210,7 @@ export function dispatchFormCancel(payload) {
 /**
  * Given a channel string return a version of the will be the channel for a viewer
  * @param {String} channel
+ * @param {String} file
  * @return {string}
  */
 export function makeViewerChannel(channel, file) {

src/firefly/js/core/BootstrapRegistry.js

@@ -55,6 +55,7 @@ import ImageLineBasedFootprint from '../drawingLayers/ImageLineBasedFootprint.js
 import {dispatchAddSaga, masterSaga} from './MasterSaga.js';
 import {watchReadout} from '../visualize/saga/MouseReadoutWatch.js';
 import {addExtensionWatcher} from './messaging/ExternalAccessWatcher.js';
+import {initHandleExternalUpload} from '../ui/FileUploadProcessor';
 
 
 const USE_LOGGING_MIDDLEWARE= false; // logging middleware is useful for debugging but we don't use if much
@@ -132,6 +133,7 @@ export const getBootstrapRegistry= once(() => {
         sagaMiddleware.run(masterSaga);
         dispatchAddSaga( watchReadout);
         addExtensionWatcher();
+        initHandleExternalUpload();
     };
 
     const registerCntlr= (cntlr = {}) => {

src/firefly/js/rpc/CoreServices.js

@@ -3,7 +3,7 @@
  */
 
 
-import {isString} from 'lodash';
+import {isEmpty, isString} from 'lodash';
 import {ServerParams} from '../data/ServerParams.js';
 import {doJsonRequest} from '../core/JsonUtils.js';
 import {WebPlotRequest} from '../visualize/WebPlotRequest';
@@ -52,7 +52,7 @@ export function getTextFile(url,maxSize) {
  * @return {Promise}
  */
 export async function upload(item, fileAnalysis= false, params={}) {
-    const fetchParam= item && buildUploadParam(item);
+    const fetchParam= (item || !isEmpty(params)) && buildUploadParam(item, params);
     if (!fetchParam) {
         const msg= item ? 'Did not recognize item to upload: must be URL (String), File, Blob, or WebPlotRequest' :
                           'item parameter not given';
@@ -66,11 +66,17 @@ export async function upload(item, fileAnalysis= false, params={}) {
 /**
  * return the correct upload parameter based on the type that is passed
  * @param {WebPlotRequest|Blob|File|String} item - the parameter to evaluate
+ * @param {Object} [params] - this will override the analysis of item
  * @return {Object|undefined} an object with the correct parameter to pass to the upload,
  * or undefined if it is not the correct parameter type
  */
-function buildUploadParam(item) {
-    if (isString(item)) return {URL: item};
+function buildUploadParam(item, params={}) {
+    const overrideKeys= ['URL', 'fileOnServer', 'file', 'webPlotRequest'];
+    const overrideUploadParamKey= Object.keys(params).find( (k) => overrideKeys.includes(k));
+    if (overrideUploadParamKey) return {[overrideUploadParamKey]:params[overrideUploadParamKey]};
+    if (isString(item)) {
+       return (item.startsWith('${') || item.startsWith('/')) ? {fileOnServer:item} : {URL: item};
+    }
     else if (WebPlotRequest.isWPR(item)) return {webPlotRequest: item.toString()};
     else if (item instanceof Blob)  return {file:item}; // handles blob or file
 }

src/firefly/js/ui/FileUpload.jsx

@@ -158,10 +158,13 @@ function doUrlAnalysis(value, fireValueChange, type, fileAnalysis) {
 
 function handleChange(ev, fireValueChange, type, fileAnalysis) {
     let file = ev?.target?.files?.[0];
-    let displayValue = ev?.target?.value;
-    if (ev.type === 'drop') { //drag drop files - instead of picking file from 'Choose File'
-        file = Array.from(ev.dataTransfer.files)[0];
-        displayValue = file?.name;
+    let displayValue;
+    if (ev) {
+        displayValue = ev.target?.value;
+        if (ev.type === 'drop') { //drag drop files - instead of picking file from 'Choose File'
+            file = Array.from(ev.dataTransfer.files)[0];
+            displayValue = ev.transferIsUrl ?  ev?.dataTransfer?.files?.[0] : file?.name;
+        }
     }
     fireValueChange({
         displayValue,

src/firefly/js/ui/FileUploadDropdown.jsx

@@ -2,38 +2,29 @@
  * License information at https://github.com/Caltech-IPAC/firefly/blob/master/License.txt
  */
 
-import {Box, Skeleton} from '@mui/joy';
 import React, {useState} from 'react';
-import {FormPanel} from './FormPanel.jsx';
-import {FileUploadViewPanel, resultFail} from '../visualize/ui/FileUploadViewPanel.jsx';
+import {Box, Skeleton} from '@mui/joy';
 import {getAppOptions} from 'firefly/api/ApiUtil.js';
-import DialogRootContainer from 'firefly/ui/DialogRootContainer.jsx';
 import {dispatchHideDialog, dispatchShowDialog} from 'firefly/core/ComponentCntlr.js';
-import {PopupPanel} from 'firefly/ui/PopupPanel.jsx';
-import {resultSuccess} from 'firefly/ui/FileUploadProcessor';
+import DialogRootContainer from 'firefly/ui/DialogRootContainer.jsx';
 import {FieldGroup} from 'firefly/ui/FieldGroup';
-import {DATA_LINK_TABLES, IMAGES, MOC_TABLES, REGIONS, SPECTRUM_TABLES, TABLES, UWS} from 'firefly/ui/FileUploadUtil';
+import {defaultAcceptList, resultSuccess} from 'firefly/ui/FileUploadProcessor';
+import {TABLES} from 'firefly/ui/FileUploadUtil';
+import {PopupPanel} from 'firefly/ui/PopupPanel.jsx';
+import {FileUploadViewPanel, resultFail} from '../visualize/ui/FileUploadViewPanel.jsx';
+import {FormPanel} from './FormPanel.jsx';
 
 const panelKey = 'FileUploadAnalysis';
-
-const defaultAcceptList = [
-    TABLES,
-    REGIONS,
-    DATA_LINK_TABLES,
-    SPECTRUM_TABLES,
-    MOC_TABLES,
-    IMAGES,
-    UWS
-];
-
-const tableOnlyDefaultAcceptList = [
-    TABLES
-];
+const tableOnlyDefaultAcceptList = [ TABLES ];
 
 export const FileUploadDropdown= ({sx, onCancel, onSubmit=resultSuccess, keepState=true,
                                       initArgs,
-                                      groupKey=panelKey, acceptList= getAppOptions()?.uploadPanelLimit==='tablesOnly'?
-        tableOnlyDefaultAcceptList: defaultAcceptList, acceptOneItem=false}) =>{
+                                      groupKey=panelKey,
+                                      acceptOneItem=false,
+                                      acceptList= getAppOptions()?.uploadPanelLimit==='tablesOnly'
+                                          ? tableOnlyDefaultAcceptList
+                                          : defaultAcceptList,
+                                      }) =>{
     const [submitText,setSubmitText]= useState('Load');
     const [doMask, changeMasking]= useState(() => false);
     const helpId = getAppOptions()?.uploadPanelHelpId ?? 'basics.searching';
@@ -56,7 +47,7 @@ export const FileUploadDropdown= ({sx, onCancel, onSubmit=resultSuccess, keepSta
                     }}>
 
                     <FileUploadViewPanel {...{setSubmitText, acceptList, acceptOneItem,
-                        externalDropEvent:initArgs?.searchParams?.dropEvent}}/>
+                        externalDropEvent:initArgs?.searchParams?.dropEvent, initArgs}}/>
                 </FormPanel>
             </FieldGroup>
             { doMask && <Skeleton sx={{inset:0, zIndex:10}}/> }