[Dms-commit] Changes to dms/src/RemoteJob.py [refactoring]

[Julien Barbot]

Index: dms/src/RemoteJob.py
diff -u dms/src/RemoteJob.py:1.10.2.3 dms/src/RemoteJob.py:1.10.2.4
--- dms/src/RemoteJob.py:1.10.2.3       Sun Mar 27 21:01:07 2005
+++ dms/src/RemoteJob.py        Sat Apr  2 17:15:01 2005
@@ -28,7 +28,6 @@
     """
     # a lock that is shared by all RemoteJob instances
     # in order to prevent the creation of same temporary file names
-    tmp_file_creation_lock = threading.Lock()
     nb_job = 0
 
     def __init__(self, socket, scheduler_address, scheduer_port):
@@ -104,39 +103,43 @@
         # send stderr messages
         ProtocolUtils.send_data(self.__client_socket, msg_stderr)
   
-    def send_result_files_back_to_client(\
-        self,\
-        input_files,\
-        output_files):
+    def send_result_files_back_to_client(self,
+                                         output_filenames,
+                                         output_files):
         """Send the result (output files and output messages) to the client,
         and replace the temporary input file names in the various messages by
         the original ones."""
 
-        #clean temporary input files created
-        for tmp_input_file in input_files.values():
-            syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, "Removing tem 
file : "\
-                          + tmp_input_file)
-            os.unlink(tmp_input_file)
         # FIXME: don't forget to send stdout and stderr
         
         # then send output files back to the client
-        for original_output_file_name in output_files.keys():
+        for (original_output_file_name, tmp_output_file_name) in 
output_filenames.items():
             self.__client_socket.send(Protocol.FILE_COMMAND)
-
+            
             ProtocolUtils.send_data(self.__client_socket, 
original_output_file_name)
-
             # send the file
-            tmp_output_file = open(\
-                output_files[original_output_file_name], 'rb')
+            # If this not a NamedTemporaryFile
+            if original_output_file_name not in output_files.keys():
+                tmp_output_file = open(tmp_output_file_name, 'rb')
+
+            else:
+                tmp_output_file = output_files[original_output_file_name]
+
             if __debug__:
-                syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, "Sending 
file : "\
-                              + output_files[original_output_file_name])
-            ProtocolUtils.send_data(self.__client_socket, 
tmp_output_file.read())                 
-            tmp_output_file.close()
-            os.unlink(output_files[original_output_file_name])
+                syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, "Sending 
file : %s" %
+                              tmp_output_file_name)
+
+            tmp_output_file.seek(0)
+            ProtocolUtils.send_data(self.__client_socket, 
tmp_output_file.read())
+
+            # If this not a NamedTemporaryFile
+            if original_output_file_name not in output_files.keys():
+                tmp_output_file.close()
+                os.unlink(tmp_output_file_name)
+
             if __debug__:
-                syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, "File sent 
: "\
-                              + output_files[original_output_file_name])
+                syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, "File sent 
: %s"
+                              % tmp_output_file_name)
         
     def compiler_job(self):
         """This method is called when a remote client
@@ -156,50 +159,46 @@
         compiler_command_line = ProtocolUtils.recv_data(self.__client_socket)
         if __debug__:
             syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,
-                          "Compilation command line received : "\
-                          + compiler_command_line)
+                          "Compilation command line received : %s"
+                          % compiler_command_line)
         compiler_command = CompilerCommandFactory.build_compiler_instance\
                            (compiler_command_line.split())
 
 
         # get the content of the input files used in the command line we
         # have just received
+        input_temp_filenames = {}
+        # keep a reference on temporary files to prevent it to be garbage 
collected
         input_temp_files = {}
         output_temp_files = {}
+        output_temp_filenames = {}
         command = self.__client_socket.recv(Protocol.COMMAND_TYPE_SIZE)
                                             
         while command == Protocol.FILE_COMMAND:
 
             file_name = ProtocolUtils.recv_data(self.__client_socket)
 
-            # FIXME: do we need to create the file inside the
-            # critical section ?
-            RemoteJob.tmp_file_creation_lock.acquire()
-            if __debug__:
-                syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,\
-                              "Creating temporary file.")
-            tmp_file_name = tempfile.mktemp(\
-                FileUtils.get_file_extension(file_name))
-            tmp_file = open(tmp_file_name, 'w')
-            RemoteJob.tmp_file_creation_lock.release()
+            tmp_file = tempfile.NamedTemporaryFile("w+b", -1, 
FileUtils.get_file_extension(file_name))
+
             if __debug__:
                 syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,\
-                              "Temporary file created.")
+                              "Temporary file created: %s." % tmp_file.name)
 
-            input_temp_files[file_name] = tmp_file_name
+            input_temp_filenames[file_name] = tmp_file.name
+            input_temp_files[file_name] = tmp_file
+            
             tmp_file.write(ProtocolUtils.recv_data(self.__client_socket))
             tmp_file.flush()
-            tmp_file.close()
+
             command = self.__client_socket.recv(Protocol.COMMAND_TYPE_SIZE)
-                                                
 
         # replace original input files in the command line by the
         # temporary ones
-        compiler_command.replace_input_files(input_temp_files)
+        compiler_command.replace_input_files(input_temp_filenames)
         if __debug__:
             syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \
-                          "New compilation command line :"\
-                          + " ".join(compiler_command.get_command_args()))
+                          "New compilation command line : %s"\
+                           % " ".join(compiler_command.get_command_args()))
 
 
         # FIXME We should not use "-o" here, this is compiler dependant
@@ -211,20 +210,18 @@
                 index_output = compiler_command.get_command_args().index("-o") 
\
                                + 1
                 
-                # FIXME: do we need to create the file inside the
-                # critical section ?
-                RemoteJob.tmp_file_creation_lock.acquire()
-                tmp_output_file = tempfile.mktemp()
-                tmp_output_file_hdl = open(tmp_output_file, 'w')
-                tmp_output_file_hdl.close()
-                RemoteJob.tmp_file_creation_lock.release()
+                tmp_output_file = tempfile.NamedTemporaryFile()
+
                 # associate the output tmp file with the original one
                 output_file_name = compiler_command.get_command_args()\
                                   [index_output]
+
+                output_temp_filenames[output_file_name] = tmp_output_file.name
                 output_temp_files[output_file_name] = tmp_output_file
+                
                 # replace the output file name in the command line
                 compiler_command.get_command_args(\
-                    )[index_output] = tmp_output_file
+                    )[index_output] = tmp_output_file.name
             except IndexError:
                 # if there is no file name after the -o option,
                 # it means that the command line is wrong, but we
@@ -236,22 +233,25 @@
             if __debug__:
                 syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \
                               "No -o option in command line.")
-            for original_input_file_name in input_temp_files.keys():
+            for original_input_file_name in input_temp_filenames.keys():
                 stop_step = compiler_command.get_stop_step()
+
                 orig_output_file_name = compiler_command.\
                                         get_output_file_name_for_step(\
                     original_input_file_name,\
                     stop_step)
-                output_temp_files[\
+
+                # FIXME: Here there is a problem ... we do not give a temp 
filename
+                output_temp_filenames[\
                     orig_output_file_name] = compiler_command.\
-                    get_output_file_name_for_step(\
-                    input_temp_files[original_input_file_name],\
+                    get_output_file_name_for_step(
+                    input_temp_filenames[original_input_file_name],
                     stop_step)
                 
                 if __debug__:
                     syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \
-                                  "File to return to the client : "\
-                                  + output_temp_files[orig_output_file_name])
+                                  "File to return to the client : %s" %
+                                  output_temp_filenames[orig_output_file_name])
                 
         # execute the command in a subshell and get the stdout and stderr 
output
         if __debug__:
@@ -271,8 +271,9 @@
         proc.stdout.close()
         exit_code = proc.wait()
 
-        self.send_output_messages(msg_stdout, msg_stderr, input_temp_files,\
-                                  output_temp_files)
+        self.send_output_messages(msg_stdout, msg_stderr,
+                                  input_temp_filenames,
+                                  output_temp_filenames)
 
         if os.WIFEXITED(exit_code):
             exit_code = os.WEXITSTATUS(exit_code)
@@ -283,7 +284,7 @@
                           "Exit code : " + str(exit_code))
         if exit_code == 0:
             # send the result (output files and output messages) to the client
-            self.send_result_files_back_to_client(input_temp_files,\
+            self.send_result_files_back_to_client(output_temp_filenames,
                                                   output_temp_files)
             if __debug__:
                 syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,\