From 6a9a6c823c9e5d63ba0628ac1f3be298b0fac29e Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 10 Aug 2021 18:07:08 -0600 Subject: [PATCH 1/2] Changes for remote systems with shared file system --- pysqa/utils/remote.py | 76 ++++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 37 deletions(-) diff --git a/pysqa/utils/remote.py b/pysqa/utils/remote.py index 1b3066cc..ec795b80 100644 --- a/pysqa/utils/remote.py +++ b/pysqa/utils/remote.py @@ -50,7 +50,7 @@ def submit_job( run_time_max=None, command=None, ): - self._transfer_data_to_remote(working_directory=working_directory) + # self._transfer_data_to_remote(working_directory=working_directory) output = self._execute_remote_command(command=command) return int(output.split()[-1]) @@ -103,44 +103,46 @@ def get_job_from_remote(self, working_directory, delete_remote=False): """ Get the results of the calculation - this is necessary when the calculation was executed on a remote host. """ - working_directory = os.path.abspath(os.path.expanduser(working_directory)) - remote_working_directory = self._get_remote_working_dir( - working_directory=working_directory - ) - remote_dict = json.loads( - self._execute_remote_command( - command="python -m pysqa.cmd --list --working_directory " - + remote_working_directory - ) - ) - for d in remote_dict["dirs"]: - local_dir = self._get_file_transfer( - file=d, local_dir=remote_working_directory, remote_dir=working_directory - ) - os.makedirs(local_dir, exist_ok=True) - file_dict = {} - for f in remote_dict["files"]: - local_file = self._get_file_transfer( - file=f, local_dir=remote_working_directory, remote_dir=working_directory - ) - file_dict[local_file] = f - self._transfer_files(file_dict=file_dict, sftp=None, transfer_back=True) - if delete_remote: - self._execute_remote_command(command="rm -r " + remote_working_directory) + pass + # working_directory = os.path.abspath(os.path.expanduser(working_directory)) + # remote_working_directory = self._get_remote_working_dir( + # working_directory=working_directory + # ) + # remote_dict = json.loads( + # self._execute_remote_command( + # command="python -m pysqa.cmd --list --working_directory " + # + remote_working_directory + # ) + # ) + # for d in remote_dict["dirs"]: + # local_dir = self._get_file_transfer( + # file=d, local_dir=remote_working_directory, remote_dir=working_directory + # ) + # os.makedirs(local_dir, exist_ok=True) + # file_dict = {} + # for f in remote_dict["files"]: + # local_file = self._get_file_transfer( + # file=f, local_dir=remote_working_directory, remote_dir=working_directory + # ) + # file_dict[local_file] = f + # self._transfer_files(file_dict=file_dict, sftp=None, transfer_back=True) + # if delete_remote: + # self._execute_remote_command(command="rm -r " + remote_working_directory) def transfer_file(self, file, transfer_back=False, delete_remote=False): - working_directory = os.path.abspath(os.path.expanduser(file)) - remote_working_directory = self._get_remote_working_dir( - working_directory=working_directory - ) - self._create_remote_dir(directory=os.path.dirname(remote_working_directory)) - self._transfer_files( - file_dict={working_directory: remote_working_directory}, - sftp=None, - transfer_back=transfer_back, - ) - if delete_remote and transfer_back: - self._execute_remote_command(command="rm " + remote_working_directory) + pass + # working_directory = os.path.abspath(os.path.expanduser(file)) + # remote_working_directory = self._get_remote_working_dir( + # working_directory=working_directory + # ) + # self._create_remote_dir(directory=os.path.dirname(remote_working_directory)) + # self._transfer_files( + # file_dict={working_directory: remote_working_directory}, + # sftp=None, + # transfer_back=transfer_back, + # ) + # if delete_remote and transfer_back: + # self._execute_remote_command(command="rm " + remote_working_directory) def __del__(self): if self._ssh_connection is not None: From ca0cb43b82d093aa03052cf4d83cf12e92c38152 Mon Sep 17 00:00:00 2001 From: pyiron-runner Date: Thu, 13 Oct 2022 20:01:08 +0000 Subject: [PATCH 2/2] Format black --- pysqa/utils/remote.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysqa/utils/remote.py b/pysqa/utils/remote.py index ec795b80..e143487e 100644 --- a/pysqa/utils/remote.py +++ b/pysqa/utils/remote.py @@ -130,7 +130,7 @@ def get_job_from_remote(self, working_directory, delete_remote=False): # self._execute_remote_command(command="rm -r " + remote_working_directory) def transfer_file(self, file, transfer_back=False, delete_remote=False): - pass + pass # working_directory = os.path.abspath(os.path.expanduser(file)) # remote_working_directory = self._get_remote_working_dir( # working_directory=working_directory