root/galaxy-central/lib/galaxy/tools/actions/upload_common.py

リビジョン 2, 19.3 KB (コミッタ: hatakeyama, 14 年 前)

import galaxy-central

行番号 
1import os, tempfile, StringIO
2from cgi import FieldStorage
3from galaxy import datatypes, util
4from galaxy.util.odict import odict
5from galaxy.datatypes import sniff
6from galaxy.util.json import to_json_string
7from galaxy.model.orm import eagerload_all
8
9import logging
10log = logging.getLogger( __name__ )
11
12def persist_uploads( params ):
13    """
14    Turn any uploads in the submitted form to persisted files.
15    """
16    if 'files' in params:
17        new_files = []
18        temp_files = []
19        for upload_dataset in params['files']:
20            f = upload_dataset['file_data']
21            if isinstance( f, FieldStorage ):
22                assert not isinstance( f.file, StringIO.StringIO )
23                assert f.file.name != '<fdopen>'
24                local_filename = util.mkstemp_ln( f.file.name, 'upload_file_data_' )
25                f.file.close()
26                upload_dataset['file_data'] = dict( filename = f.filename,
27                                                    local_filename = local_filename )
28            elif type( f ) == dict and 'filename' and 'local_filename' not in f:
29                raise Exception( 'Uploaded file was encoded in a way not understood by Galaxy.' )
30            if upload_dataset['url_paste'].strip() != '':
31                upload_dataset['url_paste'], is_multi_byte = datatypes.sniff.stream_to_file( StringIO.StringIO( upload_dataset['url_paste'] ), prefix="strio_url_paste_" )
32            else:
33                upload_dataset['url_paste'] = None
34            new_files.append( upload_dataset )
35        params['files'] = new_files
36    return params
37def handle_library_params( trans, params, folder_id, replace_dataset=None ):
38    # FIXME: the received params has already been parsed by util.Params() by the time it reaches here,
39    # so no complex objects remain.  This is not good because it does not allow for those objects to be
40    # manipulated here.  The receivd params should be the original kwd from the initial request.
41    library_bunch = util.bunch.Bunch()
42    library_bunch.replace_dataset = replace_dataset
43    library_bunch.message = params.get( 'ldda_message', '' )
44    # See if we have any template field contents
45    library_bunch.template_field_contents = []
46    template_id = params.get( 'template_id', None )
47    library_bunch.folder = trans.sa_session.query( trans.app.model.LibraryFolder ).get( trans.security.decode_id( folder_id ) )
48    # We are inheriting the folder's info_association, so we may have received inherited contents or we may have redirected
49    # here after the user entered template contents ( due to errors ).
50    if template_id not in [ None, 'None' ]:
51        library_bunch.template = trans.sa_session.query( trans.app.model.FormDefinition ).get( template_id )
52        for field_index in range( len( library_bunch.template.fields ) ):
53            field_name = 'field_%i' % field_index
54            if params.get( field_name, False ):
55                field_value = util.restore_text( params.get( field_name, ''  ) )
56                library_bunch.template_field_contents.append( field_value )
57    else:
58        library_bunch.template = None
59    library_bunch.roles = []
60    for role_id in util.listify( params.get( 'roles', [] ) ):
61        role = trans.sa_session.query( trans.app.model.Role ).get( role_id )
62        library_bunch.roles.append( role )
63    return library_bunch
64def get_precreated_datasets( trans, params, data_obj, controller='root' ):
65    """
66    Get any precreated datasets (when using asynchronous uploads).
67    """
68    rval = []
69    async_datasets = []
70    if params.get( 'async_datasets', None ) not in ["None", "", None]:
71        async_datasets = params['async_datasets'].split(',')
72    current_user_roles = trans.get_current_user_roles()
73    for id in async_datasets:
74        try:
75            data = trans.sa_session.query( data_obj ).get( int( id ) )
76        except:
77            log.exception( 'Unable to load precreated dataset (%s) sent in upload form' % id )
78            continue
79        if data_obj is trans.app.model.HistoryDatasetAssociation:
80            if trans.user is None and trans.galaxy_session.current_history != data.history:
81                log.error( 'Got a precreated dataset (%s) but it does not belong to anonymous user\'s current session (%s)' % ( data.id, trans.galaxy_session.id ) )
82            elif data.history.user != trans.user:
83                log.error( 'Got a precreated dataset (%s) but it does not belong to current user (%s)' % ( data.id, trans.user.id ) )
84            else:
85                rval.append( data )
86        elif data_obj is trans.app.model.LibraryDatasetDatasetAssociation:
87            if controller == 'library' and not trans.app.security_agent.can_add_library_item( current_user_roles, data.library_dataset.folder ):
88                log.error( 'Got a precreated dataset (%s) but this user (%s) is not allowed to write to it' % ( data.id, trans.user.id ) )
89            else:
90                rval.append( data )
91    return rval
92def get_precreated_dataset( precreated_datasets, name ):
93    """
94    Return a dataset matching a name from the list of precreated (via async
95    upload) datasets. If there's more than one upload with the exact same
96    name, we need to pop one (the first) so it isn't chosen next time.
97    """
98    names = [ d.name for d in precreated_datasets ]
99    if names.count( name ) > 0:
100        return precreated_datasets.pop( names.index( name ) )
101    else:
102        return None
103def cleanup_unused_precreated_datasets( precreated_datasets ):
104    for data in precreated_datasets:
105        log.info( 'Cleaned up unclaimed precreated dataset (%s).' % ( data.id ) )
106        data.state = data.states.ERROR
107        data.info = 'No file contents were available.'
108
109def new_history_upload( trans, uploaded_dataset, state=None ):
110    hda = trans.app.model.HistoryDatasetAssociation( name = uploaded_dataset.name,
111                                                     extension = uploaded_dataset.file_type,
112                                                     dbkey = uploaded_dataset.dbkey,
113                                                     history = trans.history,
114                                                     create_dataset = True,
115                                                     sa_session = trans.sa_session )
116    if state:
117        hda.state = state
118    else:
119        hda.state = hda.states.QUEUED
120    trans.sa_session.add( hda )
121    trans.sa_session.flush()
122    trans.history.add_dataset( hda, genome_build = uploaded_dataset.dbkey )
123    permissions = trans.app.security_agent.history_get_default_permissions( trans.history )
124    trans.app.security_agent.set_all_dataset_permissions( hda.dataset, permissions )
125    trans.sa_session.flush()
126    return hda
127def new_library_upload( trans, cntrller, uploaded_dataset, library_bunch, state=None ):
128    current_user_roles = trans.get_current_user_roles()
129    if not ( ( trans.user_is_admin() and cntrller in [ 'library_admin', 'api' ] ) or trans.app.security_agent.can_add_library_item( current_user_roles, library_bunch.folder ) ):
130        # This doesn't have to be pretty - the only time this should happen is if someone's being malicious.
131        raise Exception( "User is not authorized to add datasets to this library." )
132    folder = library_bunch.folder
133    if uploaded_dataset.get( 'in_folder', False ):
134        # Create subfolders if desired
135        for name in uploaded_dataset.in_folder.split( os.path.sep ):
136            trans.sa_session.refresh( folder )
137            matches = filter( lambda x: x.name == name, active_folders( trans, folder ) )
138            if matches:
139                folder = matches[0]
140            else:
141                new_folder = trans.app.model.LibraryFolder( name=name, description='Automatically created by upload tool' )
142                new_folder.genome_build = util.dbnames.default_value
143                folder.add_folder( new_folder )
144                trans.sa_session.add( new_folder )
145                trans.sa_session.flush()
146                trans.app.security_agent.copy_library_permissions( folder, new_folder )
147                folder = new_folder
148    if library_bunch.replace_dataset:
149        ld = library_bunch.replace_dataset
150    else:
151        ld = trans.app.model.LibraryDataset( folder=folder, name=uploaded_dataset.name )
152        trans.sa_session.add( ld )
153        trans.sa_session.flush()
154        trans.app.security_agent.copy_library_permissions( folder, ld )
155    ldda = trans.app.model.LibraryDatasetDatasetAssociation( name = uploaded_dataset.name,
156                                                             extension = uploaded_dataset.file_type,
157                                                             dbkey = uploaded_dataset.dbkey,
158                                                             library_dataset = ld,
159                                                             user = trans.user,
160                                                             create_dataset = True,
161                                                             sa_session = trans.sa_session )
162    trans.sa_session.add( ldda )
163    if state:
164        ldda.state = state
165    else:
166        ldda.state = ldda.states.QUEUED
167    ldda.message = library_bunch.message
168    trans.sa_session.flush()
169    # Permissions must be the same on the LibraryDatasetDatasetAssociation and the associated LibraryDataset
170    trans.app.security_agent.copy_library_permissions( ld, ldda )
171    if library_bunch.replace_dataset:
172        # Copy the Dataset level permissions from replace_dataset to the new LibraryDatasetDatasetAssociation.dataset
173        trans.app.security_agent.copy_dataset_permissions( library_bunch.replace_dataset.library_dataset_dataset_association.dataset, ldda.dataset )
174    else:
175        # Copy the current user's DefaultUserPermissions to the new LibraryDatasetDatasetAssociation.dataset
176        trans.app.security_agent.set_all_dataset_permissions( ldda.dataset, trans.app.security_agent.user_get_default_permissions( trans.user ) )
177        folder.add_library_dataset( ld, genome_build=uploaded_dataset.dbkey )
178        trans.sa_session.add( folder )
179        trans.sa_session.flush()
180    ld.library_dataset_dataset_association_id = ldda.id
181    trans.sa_session.add( ld )
182    trans.sa_session.flush()
183    # Handle template included in the upload form, if any.  If the upload is not asynchronous ( e.g., URL paste ),
184    # then the template and contents will be included in the library_bunch at this point.  If the upload is
185    # asynchronous ( e.g., uploading a file ), then the template and contents will be included in the library_bunch
186    # in the get_uploaded_datasets() method below.
187    if library_bunch.template and library_bunch.template_field_contents:
188        # Since information templates are inherited, the template fields can be displayed on the upload form.
189        # If the user has added field contents, we'll need to create a new form_values and info_association
190        # for the new library_dataset_dataset_association object.
191        # Create a new FormValues object, using the template we previously retrieved
192        form_values = trans.app.model.FormValues( library_bunch.template, library_bunch.template_field_contents )
193        trans.sa_session.add( form_values )
194        trans.sa_session.flush()
195        # Create a new info_association between the current ldda and form_values
196        # TODO: Currently info_associations at the ldda level are not inheritable to the associated LibraryDataset,
197        # we need to figure out if this is optimal
198        info_association = trans.app.model.LibraryDatasetDatasetInfoAssociation( ldda, library_bunch.template, form_values )
199        trans.sa_session.add( info_association )
200        trans.sa_session.flush()
201    # If roles were selected upon upload, restrict access to the Dataset to those roles
202    if library_bunch.roles:
203        for role in library_bunch.roles:
204            dp = trans.app.model.DatasetPermissions( trans.app.security_agent.permitted_actions.DATASET_ACCESS.action, ldda.dataset, role )
205            trans.sa_session.add( dp )
206            trans.sa_session.flush()
207    return ldda
208def new_upload( trans, cntrller, uploaded_dataset, library_bunch=None, state=None ):
209    if library_bunch:
210        return new_library_upload( trans, cntrller, uploaded_dataset, library_bunch, state )
211    else:
212        return new_history_upload( trans, uploaded_dataset, state )
213def get_uploaded_datasets( trans, cntrller, params, precreated_datasets, dataset_upload_inputs, library_bunch=None ):
214    uploaded_datasets = []
215    for dataset_upload_input in dataset_upload_inputs:
216        uploaded_datasets.extend( dataset_upload_input.get_uploaded_datasets( trans, params ) )
217    for uploaded_dataset in uploaded_datasets:
218        data = get_precreated_dataset( precreated_datasets, uploaded_dataset.name )
219        if not data:
220            data = new_upload( trans, cntrller, uploaded_dataset, library_bunch )
221        else:
222            data.extension = uploaded_dataset.file_type
223            data.dbkey = uploaded_dataset.dbkey
224            trans.sa_session.add( data )
225            trans.sa_session.flush()
226            if library_bunch:
227                library_bunch.folder.genome_build = uploaded_dataset.dbkey
228                trans.sa_session.add( library_bunch.folder )
229                # Handle template included in the upload form, if any.  If the upload is asynchronous ( e.g., file upload ),
230                # then the template and contents will be included in the library_bunch at this point.  If the upload is
231                # not asynchronous ( e.g., URL paste ), then the template and contents will be included in the library_bunch
232                # in the new_library_upload() method above.
233                if library_bunch.template and library_bunch.template_field_contents:
234                    # Since information templates are inherited, the template fields can be displayed on the upload form.
235                    # If the user has added field contents, we'll need to create a new form_values and info_association
236                    # for the new library_dataset_dataset_association object.
237                    # Create a new FormValues object, using the template we previously retrieved
238                    form_values = trans.app.model.FormValues( library_bunch.template, library_bunch.template_field_contents )
239                    trans.sa_session.add( form_values )
240                    trans.sa_session.flush()
241                    # Create a new info_association between the current ldda and form_values
242                    # TODO: Currently info_associations at the ldda level are not inheritable to the associated LibraryDataset,
243                    # we need to figure out if this is optimal
244                    info_association = trans.app.model.LibraryDatasetDatasetInfoAssociation( data, library_bunch.template, form_values )
245                    trans.sa_session.add( info_association )
246                trans.sa_session.flush()
247            else:
248                trans.history.genome_build = uploaded_dataset.dbkey
249        uploaded_dataset.data = data
250    return uploaded_datasets
251def create_paramfile( trans, uploaded_datasets ):
252    """
253    Create the upload tool's JSON "param" file.
254    """
255    json_file = tempfile.mkstemp()
256    json_file_path = json_file[1]
257    json_file = os.fdopen( json_file[0], 'w' )
258    for uploaded_dataset in uploaded_datasets:
259        data = uploaded_dataset.data
260        if uploaded_dataset.type == 'composite':
261            # we need to init metadata before the job is dispatched
262            data.init_meta()
263            for meta_name, meta_value in uploaded_dataset.metadata.iteritems():
264                setattr( data.metadata, meta_name, meta_value )
265            trans.sa_session.add( data )
266            trans.sa_session.flush()
267            json = dict( file_type = uploaded_dataset.file_type,
268                         dataset_id = data.dataset.id,
269                         dbkey = uploaded_dataset.dbkey,
270                         type = uploaded_dataset.type,
271                         metadata = uploaded_dataset.metadata,
272                         primary_file = uploaded_dataset.primary_file,
273                         composite_file_paths = uploaded_dataset.composite_files,
274                         composite_files = dict( [ ( k, v.__dict__ ) for k, v in data.datatype.get_composite_files( data ).items() ] ) )
275        else:
276            try:
277                is_binary = uploaded_dataset.datatype.is_binary
278            except:
279                is_binary = None
280            try:
281                link_data_only = uploaded_dataset.link_data_only
282            except:
283                link_data_only = False
284            json = dict( file_type = uploaded_dataset.file_type,
285                         ext = uploaded_dataset.ext,
286                         name = uploaded_dataset.name,
287                         dataset_id = data.dataset.id,
288                         dbkey = uploaded_dataset.dbkey,
289                         type = uploaded_dataset.type,
290                         is_binary = is_binary,
291                         link_data_only = link_data_only,
292                         space_to_tab = uploaded_dataset.space_to_tab,
293                         path = uploaded_dataset.path )
294        json_file.write( to_json_string( json ) + '\n' )
295    json_file.close()
296    return json_file_path
297def create_job( trans, params, tool, json_file_path, data_list, folder=None, return_job=False ):
298    """
299    Create the upload job.
300    """
301    job = trans.app.model.Job()
302    galaxy_session = trans.get_galaxy_session()
303    if type( galaxy_session ) == trans.model.GalaxySession:
304        job.session_id = galaxy_session.id
305    if trans.user is not None:
306        job.user_id = trans.user.id
307    if folder:
308        job.library_folder_id = folder.id
309    else:
310        job.history_id = trans.history.id
311    job.tool_id = tool.id
312    job.tool_version = tool.version
313    job.state = job.states.UPLOAD
314    trans.sa_session.add( job )
315    trans.sa_session.flush()
316    log.info( 'tool %s created job id %d' % ( tool.id, job.id ) )
317    trans.log_event( 'created job id %d' % job.id, tool_id=tool.id )
318
319    for name, value in tool.params_to_strings( params, trans.app ).iteritems():
320        job.add_parameter( name, value )
321    job.add_parameter( 'paramfile', to_json_string( json_file_path ) )
322    if folder:
323        for i, dataset in enumerate( data_list ):
324            job.add_output_library_dataset( 'output%i' % i, dataset )
325    else:
326        for i, dataset in enumerate( data_list ):
327            job.add_output_dataset( 'output%i' % i, dataset )
328    job.state = job.states.NEW
329    trans.sa_session.add( job )
330    trans.sa_session.flush()
331
332    # Queue the job for execution
333    trans.app.job_queue.put( job.id, tool )
334    trans.log_event( "Added job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id )
335    output = odict()
336    for i, v in enumerate( data_list ):
337        output[ 'output%i' % i ] = v
338    if return_job:
339        return job, output
340    else:
341        return output
342def active_folders( trans, folder ):
343    # Stolen from galaxy.web.controllers.library_common (importing from which causes a circular issues).
344    # Much faster way of retrieving all active sub-folders within a given folder than the
345    # performance of the mapper.  This query also eagerloads the permissions on each folder.
346    return trans.sa_session.query( trans.app.model.LibraryFolder ) \
347                           .filter_by( parent=folder, deleted=False ) \
348                           .options( eagerload_all( "actions" ) ) \
349                           .order_by( trans.app.model.LibraryFolder.table.c.name ) \
350                           .all()
Note: リポジトリブラウザについてのヘルプは TracBrowser を参照してください。