[2] | 1 | import os, tempfile, StringIO |
---|
| 2 | from cgi import FieldStorage |
---|
| 3 | from galaxy import datatypes, util |
---|
| 4 | from galaxy.util.odict import odict |
---|
| 5 | from galaxy.datatypes import sniff |
---|
| 6 | from galaxy.util.json import to_json_string |
---|
| 7 | from galaxy.model.orm import eagerload_all |
---|
| 8 | |
---|
| 9 | import logging |
---|
| 10 | log = logging.getLogger( __name__ ) |
---|
| 11 | |
---|
| 12 | def persist_uploads( params ): |
---|
| 13 | """ |
---|
| 14 | Turn any uploads in the submitted form to persisted files. |
---|
| 15 | """ |
---|
| 16 | if 'files' in params: |
---|
| 17 | new_files = [] |
---|
| 18 | temp_files = [] |
---|
| 19 | for upload_dataset in params['files']: |
---|
| 20 | f = upload_dataset['file_data'] |
---|
| 21 | if isinstance( f, FieldStorage ): |
---|
| 22 | assert not isinstance( f.file, StringIO.StringIO ) |
---|
| 23 | assert f.file.name != '<fdopen>' |
---|
| 24 | local_filename = util.mkstemp_ln( f.file.name, 'upload_file_data_' ) |
---|
| 25 | f.file.close() |
---|
| 26 | upload_dataset['file_data'] = dict( filename = f.filename, |
---|
| 27 | local_filename = local_filename ) |
---|
| 28 | elif type( f ) == dict and 'filename' and 'local_filename' not in f: |
---|
| 29 | raise Exception( 'Uploaded file was encoded in a way not understood by Galaxy.' ) |
---|
| 30 | if upload_dataset['url_paste'].strip() != '': |
---|
| 31 | upload_dataset['url_paste'], is_multi_byte = datatypes.sniff.stream_to_file( StringIO.StringIO( upload_dataset['url_paste'] ), prefix="strio_url_paste_" ) |
---|
| 32 | else: |
---|
| 33 | upload_dataset['url_paste'] = None |
---|
| 34 | new_files.append( upload_dataset ) |
---|
| 35 | params['files'] = new_files |
---|
| 36 | return params |
---|
| 37 | def handle_library_params( trans, params, folder_id, replace_dataset=None ): |
---|
| 38 | # FIXME: the received params has already been parsed by util.Params() by the time it reaches here, |
---|
| 39 | # so no complex objects remain. This is not good because it does not allow for those objects to be |
---|
| 40 | # manipulated here. The receivd params should be the original kwd from the initial request. |
---|
| 41 | library_bunch = util.bunch.Bunch() |
---|
| 42 | library_bunch.replace_dataset = replace_dataset |
---|
| 43 | library_bunch.message = params.get( 'ldda_message', '' ) |
---|
| 44 | # See if we have any template field contents |
---|
| 45 | library_bunch.template_field_contents = [] |
---|
| 46 | template_id = params.get( 'template_id', None ) |
---|
| 47 | library_bunch.folder = trans.sa_session.query( trans.app.model.LibraryFolder ).get( trans.security.decode_id( folder_id ) ) |
---|
| 48 | # We are inheriting the folder's info_association, so we may have received inherited contents or we may have redirected |
---|
| 49 | # here after the user entered template contents ( due to errors ). |
---|
| 50 | if template_id not in [ None, 'None' ]: |
---|
| 51 | library_bunch.template = trans.sa_session.query( trans.app.model.FormDefinition ).get( template_id ) |
---|
| 52 | for field_index in range( len( library_bunch.template.fields ) ): |
---|
| 53 | field_name = 'field_%i' % field_index |
---|
| 54 | if params.get( field_name, False ): |
---|
| 55 | field_value = util.restore_text( params.get( field_name, '' ) ) |
---|
| 56 | library_bunch.template_field_contents.append( field_value ) |
---|
| 57 | else: |
---|
| 58 | library_bunch.template = None |
---|
| 59 | library_bunch.roles = [] |
---|
| 60 | for role_id in util.listify( params.get( 'roles', [] ) ): |
---|
| 61 | role = trans.sa_session.query( trans.app.model.Role ).get( role_id ) |
---|
| 62 | library_bunch.roles.append( role ) |
---|
| 63 | return library_bunch |
---|
| 64 | def get_precreated_datasets( trans, params, data_obj, controller='root' ): |
---|
| 65 | """ |
---|
| 66 | Get any precreated datasets (when using asynchronous uploads). |
---|
| 67 | """ |
---|
| 68 | rval = [] |
---|
| 69 | async_datasets = [] |
---|
| 70 | if params.get( 'async_datasets', None ) not in ["None", "", None]: |
---|
| 71 | async_datasets = params['async_datasets'].split(',') |
---|
| 72 | current_user_roles = trans.get_current_user_roles() |
---|
| 73 | for id in async_datasets: |
---|
| 74 | try: |
---|
| 75 | data = trans.sa_session.query( data_obj ).get( int( id ) ) |
---|
| 76 | except: |
---|
| 77 | log.exception( 'Unable to load precreated dataset (%s) sent in upload form' % id ) |
---|
| 78 | continue |
---|
| 79 | if data_obj is trans.app.model.HistoryDatasetAssociation: |
---|
| 80 | if trans.user is None and trans.galaxy_session.current_history != data.history: |
---|
| 81 | log.error( 'Got a precreated dataset (%s) but it does not belong to anonymous user\'s current session (%s)' % ( data.id, trans.galaxy_session.id ) ) |
---|
| 82 | elif data.history.user != trans.user: |
---|
| 83 | log.error( 'Got a precreated dataset (%s) but it does not belong to current user (%s)' % ( data.id, trans.user.id ) ) |
---|
| 84 | else: |
---|
| 85 | rval.append( data ) |
---|
| 86 | elif data_obj is trans.app.model.LibraryDatasetDatasetAssociation: |
---|
| 87 | if controller == 'library' and not trans.app.security_agent.can_add_library_item( current_user_roles, data.library_dataset.folder ): |
---|
| 88 | log.error( 'Got a precreated dataset (%s) but this user (%s) is not allowed to write to it' % ( data.id, trans.user.id ) ) |
---|
| 89 | else: |
---|
| 90 | rval.append( data ) |
---|
| 91 | return rval |
---|
| 92 | def get_precreated_dataset( precreated_datasets, name ): |
---|
| 93 | """ |
---|
| 94 | Return a dataset matching a name from the list of precreated (via async |
---|
| 95 | upload) datasets. If there's more than one upload with the exact same |
---|
| 96 | name, we need to pop one (the first) so it isn't chosen next time. |
---|
| 97 | """ |
---|
| 98 | names = [ d.name for d in precreated_datasets ] |
---|
| 99 | if names.count( name ) > 0: |
---|
| 100 | return precreated_datasets.pop( names.index( name ) ) |
---|
| 101 | else: |
---|
| 102 | return None |
---|
| 103 | def cleanup_unused_precreated_datasets( precreated_datasets ): |
---|
| 104 | for data in precreated_datasets: |
---|
| 105 | log.info( 'Cleaned up unclaimed precreated dataset (%s).' % ( data.id ) ) |
---|
| 106 | data.state = data.states.ERROR |
---|
| 107 | data.info = 'No file contents were available.' |
---|
| 108 | |
---|
| 109 | def new_history_upload( trans, uploaded_dataset, state=None ): |
---|
| 110 | hda = trans.app.model.HistoryDatasetAssociation( name = uploaded_dataset.name, |
---|
| 111 | extension = uploaded_dataset.file_type, |
---|
| 112 | dbkey = uploaded_dataset.dbkey, |
---|
| 113 | history = trans.history, |
---|
| 114 | create_dataset = True, |
---|
| 115 | sa_session = trans.sa_session ) |
---|
| 116 | if state: |
---|
| 117 | hda.state = state |
---|
| 118 | else: |
---|
| 119 | hda.state = hda.states.QUEUED |
---|
| 120 | trans.sa_session.add( hda ) |
---|
| 121 | trans.sa_session.flush() |
---|
| 122 | trans.history.add_dataset( hda, genome_build = uploaded_dataset.dbkey ) |
---|
| 123 | permissions = trans.app.security_agent.history_get_default_permissions( trans.history ) |
---|
| 124 | trans.app.security_agent.set_all_dataset_permissions( hda.dataset, permissions ) |
---|
| 125 | trans.sa_session.flush() |
---|
| 126 | return hda |
---|
| 127 | def new_library_upload( trans, cntrller, uploaded_dataset, library_bunch, state=None ): |
---|
| 128 | current_user_roles = trans.get_current_user_roles() |
---|
| 129 | if not ( ( trans.user_is_admin() and cntrller in [ 'library_admin', 'api' ] ) or trans.app.security_agent.can_add_library_item( current_user_roles, library_bunch.folder ) ): |
---|
| 130 | # This doesn't have to be pretty - the only time this should happen is if someone's being malicious. |
---|
| 131 | raise Exception( "User is not authorized to add datasets to this library." ) |
---|
| 132 | folder = library_bunch.folder |
---|
| 133 | if uploaded_dataset.get( 'in_folder', False ): |
---|
| 134 | # Create subfolders if desired |
---|
| 135 | for name in uploaded_dataset.in_folder.split( os.path.sep ): |
---|
| 136 | trans.sa_session.refresh( folder ) |
---|
| 137 | matches = filter( lambda x: x.name == name, active_folders( trans, folder ) ) |
---|
| 138 | if matches: |
---|
| 139 | folder = matches[0] |
---|
| 140 | else: |
---|
| 141 | new_folder = trans.app.model.LibraryFolder( name=name, description='Automatically created by upload tool' ) |
---|
| 142 | new_folder.genome_build = util.dbnames.default_value |
---|
| 143 | folder.add_folder( new_folder ) |
---|
| 144 | trans.sa_session.add( new_folder ) |
---|
| 145 | trans.sa_session.flush() |
---|
| 146 | trans.app.security_agent.copy_library_permissions( folder, new_folder ) |
---|
| 147 | folder = new_folder |
---|
| 148 | if library_bunch.replace_dataset: |
---|
| 149 | ld = library_bunch.replace_dataset |
---|
| 150 | else: |
---|
| 151 | ld = trans.app.model.LibraryDataset( folder=folder, name=uploaded_dataset.name ) |
---|
| 152 | trans.sa_session.add( ld ) |
---|
| 153 | trans.sa_session.flush() |
---|
| 154 | trans.app.security_agent.copy_library_permissions( folder, ld ) |
---|
| 155 | ldda = trans.app.model.LibraryDatasetDatasetAssociation( name = uploaded_dataset.name, |
---|
| 156 | extension = uploaded_dataset.file_type, |
---|
| 157 | dbkey = uploaded_dataset.dbkey, |
---|
| 158 | library_dataset = ld, |
---|
| 159 | user = trans.user, |
---|
| 160 | create_dataset = True, |
---|
| 161 | sa_session = trans.sa_session ) |
---|
| 162 | trans.sa_session.add( ldda ) |
---|
| 163 | if state: |
---|
| 164 | ldda.state = state |
---|
| 165 | else: |
---|
| 166 | ldda.state = ldda.states.QUEUED |
---|
| 167 | ldda.message = library_bunch.message |
---|
| 168 | trans.sa_session.flush() |
---|
| 169 | # Permissions must be the same on the LibraryDatasetDatasetAssociation and the associated LibraryDataset |
---|
| 170 | trans.app.security_agent.copy_library_permissions( ld, ldda ) |
---|
| 171 | if library_bunch.replace_dataset: |
---|
| 172 | # Copy the Dataset level permissions from replace_dataset to the new LibraryDatasetDatasetAssociation.dataset |
---|
| 173 | trans.app.security_agent.copy_dataset_permissions( library_bunch.replace_dataset.library_dataset_dataset_association.dataset, ldda.dataset ) |
---|
| 174 | else: |
---|
| 175 | # Copy the current user's DefaultUserPermissions to the new LibraryDatasetDatasetAssociation.dataset |
---|
| 176 | trans.app.security_agent.set_all_dataset_permissions( ldda.dataset, trans.app.security_agent.user_get_default_permissions( trans.user ) ) |
---|
| 177 | folder.add_library_dataset( ld, genome_build=uploaded_dataset.dbkey ) |
---|
| 178 | trans.sa_session.add( folder ) |
---|
| 179 | trans.sa_session.flush() |
---|
| 180 | ld.library_dataset_dataset_association_id = ldda.id |
---|
| 181 | trans.sa_session.add( ld ) |
---|
| 182 | trans.sa_session.flush() |
---|
| 183 | # Handle template included in the upload form, if any. If the upload is not asynchronous ( e.g., URL paste ), |
---|
| 184 | # then the template and contents will be included in the library_bunch at this point. If the upload is |
---|
| 185 | # asynchronous ( e.g., uploading a file ), then the template and contents will be included in the library_bunch |
---|
| 186 | # in the get_uploaded_datasets() method below. |
---|
| 187 | if library_bunch.template and library_bunch.template_field_contents: |
---|
| 188 | # Since information templates are inherited, the template fields can be displayed on the upload form. |
---|
| 189 | # If the user has added field contents, we'll need to create a new form_values and info_association |
---|
| 190 | # for the new library_dataset_dataset_association object. |
---|
| 191 | # Create a new FormValues object, using the template we previously retrieved |
---|
| 192 | form_values = trans.app.model.FormValues( library_bunch.template, library_bunch.template_field_contents ) |
---|
| 193 | trans.sa_session.add( form_values ) |
---|
| 194 | trans.sa_session.flush() |
---|
| 195 | # Create a new info_association between the current ldda and form_values |
---|
| 196 | # TODO: Currently info_associations at the ldda level are not inheritable to the associated LibraryDataset, |
---|
| 197 | # we need to figure out if this is optimal |
---|
| 198 | info_association = trans.app.model.LibraryDatasetDatasetInfoAssociation( ldda, library_bunch.template, form_values ) |
---|
| 199 | trans.sa_session.add( info_association ) |
---|
| 200 | trans.sa_session.flush() |
---|
| 201 | # If roles were selected upon upload, restrict access to the Dataset to those roles |
---|
| 202 | if library_bunch.roles: |
---|
| 203 | for role in library_bunch.roles: |
---|
| 204 | dp = trans.app.model.DatasetPermissions( trans.app.security_agent.permitted_actions.DATASET_ACCESS.action, ldda.dataset, role ) |
---|
| 205 | trans.sa_session.add( dp ) |
---|
| 206 | trans.sa_session.flush() |
---|
| 207 | return ldda |
---|
| 208 | def new_upload( trans, cntrller, uploaded_dataset, library_bunch=None, state=None ): |
---|
| 209 | if library_bunch: |
---|
| 210 | return new_library_upload( trans, cntrller, uploaded_dataset, library_bunch, state ) |
---|
| 211 | else: |
---|
| 212 | return new_history_upload( trans, uploaded_dataset, state ) |
---|
| 213 | def get_uploaded_datasets( trans, cntrller, params, precreated_datasets, dataset_upload_inputs, library_bunch=None ): |
---|
| 214 | uploaded_datasets = [] |
---|
| 215 | for dataset_upload_input in dataset_upload_inputs: |
---|
| 216 | uploaded_datasets.extend( dataset_upload_input.get_uploaded_datasets( trans, params ) ) |
---|
| 217 | for uploaded_dataset in uploaded_datasets: |
---|
| 218 | data = get_precreated_dataset( precreated_datasets, uploaded_dataset.name ) |
---|
| 219 | if not data: |
---|
| 220 | data = new_upload( trans, cntrller, uploaded_dataset, library_bunch ) |
---|
| 221 | else: |
---|
| 222 | data.extension = uploaded_dataset.file_type |
---|
| 223 | data.dbkey = uploaded_dataset.dbkey |
---|
| 224 | trans.sa_session.add( data ) |
---|
| 225 | trans.sa_session.flush() |
---|
| 226 | if library_bunch: |
---|
| 227 | library_bunch.folder.genome_build = uploaded_dataset.dbkey |
---|
| 228 | trans.sa_session.add( library_bunch.folder ) |
---|
| 229 | # Handle template included in the upload form, if any. If the upload is asynchronous ( e.g., file upload ), |
---|
| 230 | # then the template and contents will be included in the library_bunch at this point. If the upload is |
---|
| 231 | # not asynchronous ( e.g., URL paste ), then the template and contents will be included in the library_bunch |
---|
| 232 | # in the new_library_upload() method above. |
---|
| 233 | if library_bunch.template and library_bunch.template_field_contents: |
---|
| 234 | # Since information templates are inherited, the template fields can be displayed on the upload form. |
---|
| 235 | # If the user has added field contents, we'll need to create a new form_values and info_association |
---|
| 236 | # for the new library_dataset_dataset_association object. |
---|
| 237 | # Create a new FormValues object, using the template we previously retrieved |
---|
| 238 | form_values = trans.app.model.FormValues( library_bunch.template, library_bunch.template_field_contents ) |
---|
| 239 | trans.sa_session.add( form_values ) |
---|
| 240 | trans.sa_session.flush() |
---|
| 241 | # Create a new info_association between the current ldda and form_values |
---|
| 242 | # TODO: Currently info_associations at the ldda level are not inheritable to the associated LibraryDataset, |
---|
| 243 | # we need to figure out if this is optimal |
---|
| 244 | info_association = trans.app.model.LibraryDatasetDatasetInfoAssociation( data, library_bunch.template, form_values ) |
---|
| 245 | trans.sa_session.add( info_association ) |
---|
| 246 | trans.sa_session.flush() |
---|
| 247 | else: |
---|
| 248 | trans.history.genome_build = uploaded_dataset.dbkey |
---|
| 249 | uploaded_dataset.data = data |
---|
| 250 | return uploaded_datasets |
---|
| 251 | def create_paramfile( trans, uploaded_datasets ): |
---|
| 252 | """ |
---|
| 253 | Create the upload tool's JSON "param" file. |
---|
| 254 | """ |
---|
| 255 | json_file = tempfile.mkstemp() |
---|
| 256 | json_file_path = json_file[1] |
---|
| 257 | json_file = os.fdopen( json_file[0], 'w' ) |
---|
| 258 | for uploaded_dataset in uploaded_datasets: |
---|
| 259 | data = uploaded_dataset.data |
---|
| 260 | if uploaded_dataset.type == 'composite': |
---|
| 261 | # we need to init metadata before the job is dispatched |
---|
| 262 | data.init_meta() |
---|
| 263 | for meta_name, meta_value in uploaded_dataset.metadata.iteritems(): |
---|
| 264 | setattr( data.metadata, meta_name, meta_value ) |
---|
| 265 | trans.sa_session.add( data ) |
---|
| 266 | trans.sa_session.flush() |
---|
| 267 | json = dict( file_type = uploaded_dataset.file_type, |
---|
| 268 | dataset_id = data.dataset.id, |
---|
| 269 | dbkey = uploaded_dataset.dbkey, |
---|
| 270 | type = uploaded_dataset.type, |
---|
| 271 | metadata = uploaded_dataset.metadata, |
---|
| 272 | primary_file = uploaded_dataset.primary_file, |
---|
| 273 | composite_file_paths = uploaded_dataset.composite_files, |
---|
| 274 | composite_files = dict( [ ( k, v.__dict__ ) for k, v in data.datatype.get_composite_files( data ).items() ] ) ) |
---|
| 275 | else: |
---|
| 276 | try: |
---|
| 277 | is_binary = uploaded_dataset.datatype.is_binary |
---|
| 278 | except: |
---|
| 279 | is_binary = None |
---|
| 280 | try: |
---|
| 281 | link_data_only = uploaded_dataset.link_data_only |
---|
| 282 | except: |
---|
| 283 | link_data_only = False |
---|
| 284 | json = dict( file_type = uploaded_dataset.file_type, |
---|
| 285 | ext = uploaded_dataset.ext, |
---|
| 286 | name = uploaded_dataset.name, |
---|
| 287 | dataset_id = data.dataset.id, |
---|
| 288 | dbkey = uploaded_dataset.dbkey, |
---|
| 289 | type = uploaded_dataset.type, |
---|
| 290 | is_binary = is_binary, |
---|
| 291 | link_data_only = link_data_only, |
---|
| 292 | space_to_tab = uploaded_dataset.space_to_tab, |
---|
| 293 | path = uploaded_dataset.path ) |
---|
| 294 | json_file.write( to_json_string( json ) + '\n' ) |
---|
| 295 | json_file.close() |
---|
| 296 | return json_file_path |
---|
| 297 | def create_job( trans, params, tool, json_file_path, data_list, folder=None, return_job=False ): |
---|
| 298 | """ |
---|
| 299 | Create the upload job. |
---|
| 300 | """ |
---|
| 301 | job = trans.app.model.Job() |
---|
| 302 | galaxy_session = trans.get_galaxy_session() |
---|
| 303 | if type( galaxy_session ) == trans.model.GalaxySession: |
---|
| 304 | job.session_id = galaxy_session.id |
---|
| 305 | if trans.user is not None: |
---|
| 306 | job.user_id = trans.user.id |
---|
| 307 | if folder: |
---|
| 308 | job.library_folder_id = folder.id |
---|
| 309 | else: |
---|
| 310 | job.history_id = trans.history.id |
---|
| 311 | job.tool_id = tool.id |
---|
| 312 | job.tool_version = tool.version |
---|
| 313 | job.state = job.states.UPLOAD |
---|
| 314 | trans.sa_session.add( job ) |
---|
| 315 | trans.sa_session.flush() |
---|
| 316 | log.info( 'tool %s created job id %d' % ( tool.id, job.id ) ) |
---|
| 317 | trans.log_event( 'created job id %d' % job.id, tool_id=tool.id ) |
---|
| 318 | |
---|
| 319 | for name, value in tool.params_to_strings( params, trans.app ).iteritems(): |
---|
| 320 | job.add_parameter( name, value ) |
---|
| 321 | job.add_parameter( 'paramfile', to_json_string( json_file_path ) ) |
---|
| 322 | if folder: |
---|
| 323 | for i, dataset in enumerate( data_list ): |
---|
| 324 | job.add_output_library_dataset( 'output%i' % i, dataset ) |
---|
| 325 | else: |
---|
| 326 | for i, dataset in enumerate( data_list ): |
---|
| 327 | job.add_output_dataset( 'output%i' % i, dataset ) |
---|
| 328 | job.state = job.states.NEW |
---|
| 329 | trans.sa_session.add( job ) |
---|
| 330 | trans.sa_session.flush() |
---|
| 331 | |
---|
| 332 | # Queue the job for execution |
---|
| 333 | trans.app.job_queue.put( job.id, tool ) |
---|
| 334 | trans.log_event( "Added job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id ) |
---|
| 335 | output = odict() |
---|
| 336 | for i, v in enumerate( data_list ): |
---|
| 337 | output[ 'output%i' % i ] = v |
---|
| 338 | if return_job: |
---|
| 339 | return job, output |
---|
| 340 | else: |
---|
| 341 | return output |
---|
| 342 | def active_folders( trans, folder ): |
---|
| 343 | # Stolen from galaxy.web.controllers.library_common (importing from which causes a circular issues). |
---|
| 344 | # Much faster way of retrieving all active sub-folders within a given folder than the |
---|
| 345 | # performance of the mapper. This query also eagerloads the permissions on each folder. |
---|
| 346 | return trans.sa_session.query( trans.app.model.LibraryFolder ) \ |
---|
| 347 | .filter_by( parent=folder, deleted=False ) \ |
---|
| 348 | .options( eagerload_all( "actions" ) ) \ |
---|
| 349 | .order_by( trans.app.model.LibraryFolder.table.c.name ) \ |
---|
| 350 | .all() |
---|