root/galaxy-central/lib/galaxy/datatypes/metadata.py

リビジョン 2, 30.5 KB (コミッタ: hatakeyama, 14 年 前)

import galaxy-central

行番号 
1import sys, logging, copy, shutil, weakref, cPickle, tempfile, os
2
3from galaxy.util import string_as_bool, relpath, stringify_dictionary_keys, listify
4from galaxy.util.odict import odict
5from galaxy.web import form_builder
6import galaxy.model
7from sqlalchemy.orm import object_session
8
9import pkg_resources
10pkg_resources.require("simplejson")
11import simplejson
12
13log = logging.getLogger( __name__ )
14
15STATEMENTS = "__galaxy_statements__" #this is the name of the property in a Datatype class where new metadata spec element Statements are stored
16
17DATABASE_CONNECTION_AVAILABLE = True #When False, certain metadata parameter types (see FileParameter) will behave differently
18
19class Statement( object ):
20    """
21    This class inserts its target into a list in the surrounding
22    class.  the data.Data class has a metaclass which executes these
23    statements.  This is how we shove the metadata element spec into
24    the class.
25    """
26    def __init__( self, target ):
27        self.target = target
28    def __call__( self, *args, **kwargs ):
29        class_locals = sys._getframe( 1 ).f_locals #get the locals dictionary of the frame object one down in the call stack (i.e. the Datatype class calling MetadataElement)
30        statements = class_locals.setdefault( STATEMENTS, [] ) #get and set '__galaxy_statments__' to an empty list if not in locals dict
31        statements.append( ( self, args, kwargs ) ) #add Statement containing info to populate a MetadataElementSpec
32    @classmethod
33    def process( cls, element ):
34        for statement, args, kwargs in getattr( element, STATEMENTS, [] ):
35            statement.target( element, *args, **kwargs ) #statement.target is MetadataElementSpec, element is a Datatype class
36
37
38class MetadataCollection( object ):
39    """
40    MetadataCollection is not a collection at all, but rather a proxy
41    to the real metadata which is stored as a Dictionary. This class
42    handles processing the metadata elements when they are set and
43    retrieved, returning default values in cases when metadata is not set.
44    """
45    def __init__(self, parent ):
46        self.parent = parent
47        #initialize dict if needed
48        if self.parent._metadata is None:
49            self.parent._metadata = {}
50    def get_parent( self ):
51        if "_parent" in self.__dict__:
52            return self.__dict__["_parent"]()
53        return None
54    def set_parent( self, parent ):
55        self.__dict__["_parent"] = weakref.ref( parent ) # use weakref to prevent a circular reference interfering with garbage collection: hda/lda (parent) <--> MetadataCollection (self) ; needs to be hashable, so cannot use proxy.
56    parent = property( get_parent, set_parent )
57    @property
58    def spec( self ):
59        return self.parent.datatype.metadata_spec
60    def __iter__( self ):
61        return self.parent._metadata.__iter__()
62    def get( self, key, default=None ):
63        try:
64            return self.__getattr__( key ) or default
65        except:
66            return default
67    def items(self):
68        return iter( [ ( k, self.get( k ) ) for k in self.spec.iterkeys() ] )
69    def __str__(self):
70        return dict( self.items() ).__str__()
71    def __nonzero__( self ):
72        return bool( self.parent._metadata )
73    def __getattr__( self, name ):
74        if name in self.spec:
75            if name in self.parent._metadata:
76                return self.spec[name].wrap( self.parent._metadata[name] )
77            return self.spec[name].wrap( self.spec[name].default )
78        if name in self.parent._metadata:
79            return self.parent._metadata[name]
80    def __setattr__( self, name, value ):
81        if name == "parent":
82            return self.set_parent( value )
83        else:
84            if name in self.spec:
85                self.parent._metadata[name] = self.spec[name].unwrap( value )
86            else:
87                self.parent._metadata[name] = value
88    def element_is_set( self, name ):
89        return bool( self.parent._metadata.get( name, False ) )
90    def get_html_by_name( self, name, **kwd ):
91        if name in self.spec:
92            rval = self.spec[name].param.get_html( value=getattr( self, name ), context=self, **kwd )
93            if rval is None:
94                return self.spec[name].no_value
95            return rval
96    def make_dict_copy( self, to_copy ):
97        """Makes a deep copy of input iterable to_copy according to self.spec"""
98        rval = {}
99        for key, value in to_copy.items():
100            if key in self.spec:
101                rval[key] = self.spec[key].param.make_copy( value, target_context=self, source_context=to_copy )
102        return rval
103    def from_JSON_dict( self, filename ):
104        dataset = self.parent
105        log.debug( 'loading metadata from file for: %s %s' % ( dataset.__class__.__name__, dataset.id ) )
106        JSONified_dict = simplejson.load( open( filename ) )
107        for name, spec in self.spec.items():
108            if name in JSONified_dict:
109                dataset._metadata[ name ] = spec.param.from_external_value( JSONified_dict[ name ], dataset )
110            elif name in dataset._metadata:
111                #if the metadata value is not found in our externally set metadata but it has a value in the 'old'
112                #metadata associated with our dataset, we'll delete it from our dataset's metadata dict
113                del dataset._metadata[ name ]
114    def to_JSON_dict( self, filename ):
115        #galaxy.model.customtypes.json_encoder.encode()
116        meta_dict = {}
117        dataset_meta_dict = self.parent._metadata
118        for name, spec in self.spec.items():
119            if name in dataset_meta_dict:
120                meta_dict[ name ] = spec.param.to_external_value( dataset_meta_dict[ name ] )
121        simplejson.dump( meta_dict, open( filename, 'wb+' ) )
122    def __getstate__( self ):
123        return None #cannot pickle a weakref item (self._parent), when data._metadata_collection is None, it will be recreated on demand
124
125class MetadataSpecCollection( odict ):
126    """
127    A simple extension of dict which allows cleaner access to items
128    and allows the values to be iterated over directly as if it were a
129    list.  append() is also implemented for simplicity and does not
130    "append".
131    """
132    def __init__( self, dict = None ):
133        odict.__init__( self, dict = None )
134    def append( self, item ):
135        self[item.name] = item
136    def iter( self ):
137        return self.itervalues()
138    def __getattr__( self, name ):
139        return self.get( name )
140
141class MetadataParameter( object ):
142    def __init__( self, spec ):
143        self.spec = spec
144       
145    def get_html_field( self, value=None, context={}, other_values={}, **kwd ):       
146        return form_builder.TextField( self.spec.name, value=value )
147   
148    def get_html( self, value, context={}, other_values={}, **kwd ):
149        """
150        The "context" is simply the metadata collection/bunch holding
151        this piece of metadata. This is passed in to allow for
152        metadata to validate against each other (note: this could turn
153        into a huge, recursive mess if not done with care). For
154        example, a column assignment should validate against the
155        number of columns in the dataset.
156        """
157        if self.spec.get("readonly"):
158            return value
159        if self.spec.get("optional"):
160            checked = False
161            if value: checked = "true"
162            checkbox = form_builder.CheckboxField( "is_" + self.spec.name, checked=checked )
163            return checkbox.get_html() + self.get_html_field( value=value, context=context, other_values=other_values, **kwd ).get_html()
164        else:
165            return self.get_html_field( value=value, context=context, other_values=other_values, **kwd ).get_html()
166   
167    def to_string( self, value ):
168        return str( value )
169   
170    def make_copy( self, value, target_context = None, source_context = None ):
171        return copy.deepcopy( value )
172   
173    @classmethod
174    def marshal ( cls, value ):
175        """
176        This method should/can be overridden to convert the incoming
177        value to whatever type it is supposed to be.
178        """
179        return value
180
181    def validate( self, value ):
182        """
183        Throw an exception if the value is invalid.
184        """
185        pass
186
187
188    def unwrap( self, form_value ):
189        """
190        Turns a value into its storable form.
191        """
192        value = self.marshal( form_value )
193        self.validate( value )
194        return value
195   
196    def wrap( self, value ):
197        """
198        Turns a value into its usable form.
199        """
200        return value
201
202    def from_external_value( self, value, parent ):
203        """
204        Turns a value read from an external dict into its value to be pushed directly into the metadata dict.
205        """
206        return value
207    def to_external_value( self, value ):
208        """
209        Turns a value read from a metadata into its value to be pushed directly into the external dict.
210        """
211        return value
212
213class MetadataElementSpec( object ):
214    """
215    Defines a metadata element and adds it to the metadata_spec (which
216    is a MetadataSpecCollection) of datatype.
217    """
218
219    def __init__( self, datatype, name=None, desc=None, param=MetadataParameter, default=None, no_value = None, visible=True, set_in_upload = False, **kwargs ):
220        self.name = name
221        self.desc = desc or name
222        self.default = default
223        self.no_value = no_value
224        self.visible = visible
225        self.set_in_upload = set_in_upload
226        # Catch-all, allows for extra attributes to be set
227        self.__dict__.update(kwargs)
228        #set up param last, as it uses values set above
229        self.param = param( self )
230        datatype.metadata_spec.append( self ) #add spec element to the spec
231    def get( self, name ):
232        return self.__dict__.get(name, None)
233    def wrap( self, value ):
234        """
235        Turns a stored value into its usable form.
236        """
237        return self.param.wrap( value )
238    def unwrap( self, value ):
239        """
240        Turns an incoming value into its storable form.
241        """
242        return self.param.unwrap( value )
243
244MetadataElement = Statement( MetadataElementSpec )
245
246"""
247MetadataParameter sub-classes.
248"""
249
250class SelectParameter( MetadataParameter ):
251    def __init__( self, spec ):
252        MetadataParameter.__init__( self, spec )
253        self.values = self.spec.get( "values" )
254        self.multiple = string_as_bool( self.spec.get( "multiple" ) )
255   
256    def to_string( self, value ):
257        if value in [ None, [] ]:
258            return str( self.spec.no_value )
259        if not isinstance( value, list ):
260            value = [value]
261        return ",".join( map( str, value ) )
262   
263    def get_html_field( self, value=None, context={}, other_values={}, values=None, **kwd ):       
264        field = form_builder.SelectField( self.spec.name, multiple=self.multiple, display=self.spec.get("display") )
265        if self.values:
266            value_list = self.values
267        elif values:
268            value_list = values
269        elif value:
270            value_list = [ ( v, v ) for v in listify( value )]
271        else:
272            value_list = []
273        for val, label in value_list:
274            try:
275                if ( self.multiple and val in value ) or ( not self.multiple and val == value ):
276                    field.add_option( label, val, selected=True )
277                else:
278                    field.add_option( label, val, selected=False )
279            except TypeError:
280                field.add_option( val, label, selected=False )
281        return field
282
283    def get_html( self, value, context={}, other_values={}, values=None, **kwd ):
284        if self.spec.get("readonly"):
285            if value in [ None, [] ]:
286                return str( self.spec.no_value )
287            return ", ".join( map( str, value ) )
288        return MetadataParameter.get_html( self, value, context=context, other_values=other_values, values=values, **kwd )
289
290    def wrap( self, value ):
291        value = self.marshal( value ) #do we really need this (wasteful)? - yes because we are not sure that all existing selects have been stored previously as lists. Also this will handle the case where defaults/no_values are specified and are single non-list values.
292        if self.multiple:
293            return value
294        elif value:
295            return value[0] #single select, only return the first value
296        return None
297
298    @classmethod
299    def marshal( cls, value ):
300        # Store select as list, even if single item
301        if value is None: return []
302        if not isinstance( value, list ): return [value]
303        return value
304
305class DBKeyParameter( SelectParameter ):
306    def get_html_field( self, value=None, context={}, other_values={}, values=None, **kwd):
307        try:
308            values = kwd['trans'].db_builds
309        except KeyError:
310            pass
311        return super(DBKeyParameter, self).get_html_field( value, context, other_values, values, **kwd)
312    def get_html( self, value=None, context={}, other_values={}, values=None, **kwd):       
313        try:
314            values = kwd['trans'].db_builds
315        except KeyError:
316            pass
317        return super(DBKeyParameter, self).get_html( value, context, other_values, values, **kwd)
318
319class RangeParameter( SelectParameter ):
320    def __init__( self, spec ):
321        SelectParameter.__init__( self, spec )
322        # The spec must be set with min and max values
323        self.min = spec.get( "min" ) or 1
324        self.max = spec.get( "max" ) or 1
325        self.step = self.spec.get( "step" ) or 1
326
327    def get_html_field( self, value=None, context={}, other_values={}, values=None, **kwd ):
328        if values is None:
329            values = zip( range( self.min, self.max, self.step ), range( self.min, self.max, self.step ))
330        return SelectParameter.get_html_field( self, value=value, context=context, other_values=other_values, values=values, **kwd )
331   
332    def get_html( self, value, context={}, other_values={}, values=None, **kwd ):
333        if values is None:
334            values = zip( range( self.min, self.max, self.step ), range( self.min, self.max, self.step ))
335        return SelectParameter.get_html( self, value, context=context, other_values=other_values, values=values, **kwd )
336   
337    @classmethod
338    def marshal( cls, value ):
339        value = SelectParameter.marshal( value )
340        values = [ int(x) for x in value ]
341        return values
342   
343class ColumnParameter( RangeParameter ):
344   
345    def get_html_field( self, value=None, context={}, other_values={}, values=None, **kwd ):
346        if values is None and context:
347            column_range = range( 1, context.columns+1, 1 )
348            values = zip( column_range, column_range )
349        return RangeParameter.get_html_field( self, value=value, context=context, other_values=other_values, values=values, **kwd )
350   
351    def get_html( self, value, context={}, other_values={}, values=None, **kwd ):
352        if values is None and context:
353            column_range = range( 1, context.columns+1, 1 )
354            values = zip( column_range, column_range )
355        return RangeParameter.get_html( self, value, context=context, other_values=other_values, values=values, **kwd )
356   
357class ColumnTypesParameter( MetadataParameter ):
358   
359    def to_string( self, value ):
360        return ",".join( map( str, value ) )
361
362class PythonObjectParameter( MetadataParameter ):
363   
364    def to_string( self, value ):
365        if not value:
366            return self.spec._to_string( self.spec.no_value )
367        return self.spec._to_string( value )
368   
369    def get_html_field( self, value=None, context={}, other_values={}, **kwd ):
370        return form_builder.TextField( self.spec.name, value=self._to_string( value ) )
371
372    def get_html( self, value=None, context={}, other_values={}, **kwd ):
373        return str( self )
374
375    @classmethod
376    def marshal( cls, value ):
377        return value
378
379class FileParameter( MetadataParameter ):
380   
381    def to_string( self, value ):
382        if not value:
383            return str( self.spec.no_value )
384        return value.file_name
385   
386    def get_html_field( self, value=None, context={}, other_values={}, **kwd ):
387        return form_builder.TextField( self.spec.name, value=str( value.id ) )
388
389    def get_html( self, value=None, context={}, other_values={}, **kwd ):
390        return "<div>No display available for Metadata Files</div>"
391
392    def wrap( self, value ):
393        if value is None:
394            return None
395        if isinstance( value, galaxy.model.MetadataFile ) or isinstance( value, MetadataTempFile ):
396            return value
397        if DATABASE_CONNECTION_AVAILABLE:
398            try:
399                # FIXME: this query requires a monkey patch in assignmapper.py since
400                # MetadataParameters do not have a handle to the sqlalchemy session
401                return galaxy.model.MetadataFile.get( value )
402            except:
403                #value was not a valid id
404                return None
405        else:
406            mf = galaxy.model.MetadataFile()
407            mf.id = value #we assume this is a valid id, since we cannot check it
408            return mf
409    def make_copy( self, value, target_context, source_context ):
410        value = self.wrap( value )
411        if value:
412            new_value = galaxy.model.MetadataFile( dataset = target_context.parent, name = self.spec.name )
413            object_session( target_context.parent ).add( new_value )
414            object_session( target_context.parent ).flush()
415            shutil.copy( value.file_name, new_value.file_name )
416            return self.unwrap( new_value )
417        return None
418   
419    @classmethod
420    def marshal( cls, value ):
421        if isinstance( value, galaxy.model.MetadataFile ):
422            value = value.id
423        return value
424   
425    def from_external_value( self, value, parent ):
426        """
427        Turns a value read from a external dict into its value to be pushed directly into the metadata dict.
428        """
429        if MetadataTempFile.is_JSONified_value( value ):
430            value = MetadataTempFile.from_JSON( value )
431        if isinstance( value, MetadataTempFile ):
432            mf = parent.metadata.get( self.spec.name, None)
433            if mf is None:
434                mf = self.new_file( dataset = parent, **value.kwds )
435            shutil.move( value.file_name, mf.file_name )
436            value = mf.id
437        return value
438    def to_external_value( self, value ):
439        """
440        Turns a value read from a metadata into its value to be pushed directly into the external dict.
441        """
442        if isinstance( value, galaxy.model.MetadataFile ):
443            value = value.id
444        elif isinstance( value, MetadataTempFile ):
445            value = MetadataTempFile.to_JSON( value )
446        return value
447   
448    def new_file( self, dataset = None, **kwds ):
449        if DATABASE_CONNECTION_AVAILABLE:
450            mf = galaxy.model.MetadataFile( name = self.spec.name, dataset = dataset, **kwds )
451            object_session( dataset ).add( mf )
452            object_session( dataset ).flush() #flush to assign id
453            return mf
454        else:
455            #we need to make a tmp file that is accessable to the head node,
456            #we will be copying its contents into the MetadataFile objects filename after restoring from JSON
457            #we do not include 'dataset' in the kwds passed, as from_JSON_value() will handle this for us
458            return MetadataTempFile( **kwds )
459
460#This class is used when a database file connection is not available
461class MetadataTempFile( object ):
462    tmp_dir = 'database/tmp' #this should be overwritten as necessary in calling scripts
463    def __init__( self, **kwds ):
464        self.kwds = kwds
465        self._filename = None
466    @property
467    def file_name( self ):
468        if self._filename is None:
469            #we need to create a tmp file, accessable across all nodes/heads, save the name, and return it
470            self._filename = relpath( tempfile.NamedTemporaryFile( dir = self.tmp_dir, prefix = "metadata_temp_file_" ).name )
471            open( self._filename, 'wb+' ) #create an empty file, so it can't be reused using tempfile
472        return self._filename
473    def to_JSON( self ):
474        return { '__class__':self.__class__.__name__, 'filename':self.file_name, 'kwds':self.kwds }
475    @classmethod
476    def from_JSON( cls, json_dict ):
477        #need to ensure our keywords are not unicode
478        rval = cls( **stringify_dictionary_keys( json_dict['kwds'] ) )
479        rval._filename = json_dict['filename']
480        return rval
481    @classmethod
482    def is_JSONified_value( cls, value ):
483        return ( isinstance( value, dict ) and value.get( '__class__', None ) == cls.__name__ )
484    @classmethod
485    def cleanup_from_JSON_dict_filename( cls, filename ):
486        try:
487            for key, value in simplejson.load( open( filename ) ).items():
488                if cls.is_JSONified_value( value ):
489                    value = cls.from_JSON( value )
490                if isinstance( value, cls ) and os.path.exists( value.file_name ):
491                    log.debug( 'Cleaning up abandoned MetadataTempFile file: %s' % value.file_name )
492                    os.unlink( value.file_name )
493        except Exception, e:
494            log.debug( 'Failed to cleanup MetadataTempFile temp files from %s: %s' % ( filename, e ) )
495
496#Class with methods allowing set_meta() to be called externally to the Galaxy head
497class JobExternalOutputMetadataWrapper( object ):
498    #this class allows access to external metadata filenames for all outputs associated with a job
499    #We will use JSON as the medium of exchange of information, except for the DatasetInstance object which will use pickle (in the future this could be JSONified as well)
500    def __init__( self, job ):
501        self.job_id = job.id
502    def get_output_filenames_by_dataset( self, dataset, sa_session ):
503        if isinstance( dataset, galaxy.model.HistoryDatasetAssociation ):
504            return sa_session.query( galaxy.model.JobExternalOutputMetadata ) \
505                             .filter_by( job_id = self.job_id, history_dataset_association_id = dataset.id ) \
506                             .first() #there should only be one or None
507        elif isinstance( dataset, galaxy.model.LibraryDatasetDatasetAssociation ):
508            return sa_session.query( galaxy.model.JobExternalOutputMetadata ) \
509                             .filter_by( job_id = self.job_id, library_dataset_dataset_association_id = dataset.id ) \
510                             .first() #there should only be one or None
511        return None
512    def get_dataset_metadata_key( self, dataset ):
513        # Set meta can be called on library items and history items,
514        # need to make different keys for them, since ids can overlap
515        return "%s_%d" % ( dataset.__class__.__name__, dataset.id )
516    def setup_external_metadata( self, datasets, sa_session, exec_dir=None, tmp_dir=None, dataset_files_path=None,
517                                 output_fnames=None, config_root=None, datatypes_config=None, job_metadata=None, kwds={} ):
518        #fill in metadata_files_dict and return the command with args required to set metadata
519        def __metadata_files_list_to_cmd_line( metadata_files ):
520            def __get_filename_override():
521                if output_fnames:
522                    for dataset_path in output_fnames:
523                        if dataset_path.false_path and dataset_path.real_path == metadata_files.dataset.file_name:
524                            return dataset_path.false_path
525                return ""
526            return "%s,%s,%s,%s,%s,%s" % ( metadata_files.filename_in, metadata_files.filename_kwds, metadata_files.filename_out, metadata_files.filename_results_code, __get_filename_override(), metadata_files.filename_override_metadata )
527        if not isinstance( datasets, list ):
528            datasets = [ datasets ]
529        if exec_dir is None:
530            exec_dir = os.path.abspath( os.getcwd() )
531        if tmp_dir is None:
532            tmp_dir = MetadataTempFile.tmp_dir
533        if dataset_files_path is None:
534            dataset_files_path = galaxy.model.Dataset.file_path
535        if config_root is None:
536            config_root = os.path.abspath( os.getcwd() )
537        if datatypes_config is None:
538            datatypes_config = 'datatypes_conf.xml'
539        metadata_files_list = []
540        for dataset in datasets:
541            key = self.get_dataset_metadata_key( dataset )
542            #future note:
543            #wonkiness in job execution causes build command line to be called more than once
544            #when setting metadata externally, via 'auto-detect' button in edit attributes, etc.,
545            #we don't want to overwrite (losing the ability to cleanup) our existing dataset keys and files,
546            #so we will only populate the dictionary once
547            metadata_files = self.get_output_filenames_by_dataset( dataset, sa_session )
548            if not metadata_files:
549                metadata_files = galaxy.model.JobExternalOutputMetadata( dataset = dataset)
550                metadata_files.job_id = self.job_id
551                #we are using tempfile to create unique filenames, tempfile always returns an absolute path
552                #we will use pathnames relative to the galaxy root, to accommodate instances where the galaxy root
553                #is located differently, i.e. on a cluster node with a different filesystem structure
554               
555                #file to store existing dataset
556                metadata_files.filename_in = relpath( tempfile.NamedTemporaryFile( dir = tmp_dir, prefix = "metadata_in_%s_" % key ).name )
557               
558                #FIXME: HACK
559                #sqlalchemy introduced 'expire_on_commit' flag for sessionmaker at version 0.5x
560                #This may be causing the dataset attribute of the dataset_association object to no-longer be loaded into memory when needed for pickling.
561                #For now, we'll simply 'touch' dataset_association.dataset to force it back into memory.
562                dataset.dataset #force dataset_association.dataset to be loaded before pickling
563                #A better fix could be setting 'expire_on_commit=False' on the session, or modifying where commits occur, or ?
564               
565                cPickle.dump( dataset, open( metadata_files.filename_in, 'wb+' ) )
566                #file to store metadata results of set_meta()
567                metadata_files.filename_out = relpath( tempfile.NamedTemporaryFile( dir = tmp_dir, prefix = "metadata_out_%s_" % key ).name )
568                open( metadata_files.filename_out, 'wb+' ) # create the file on disk, so it cannot be reused by tempfile (unlikely, but possible)
569                #file to store a 'return code' indicating the results of the set_meta() call
570                #results code is like (True/False - if setting metadata was successful/failed , exception or string of reason of success/failure )
571                metadata_files.filename_results_code = relpath( tempfile.NamedTemporaryFile( dir = tmp_dir, prefix = "metadata_results_%s_" % key ).name )
572                simplejson.dump( ( False, 'External set_meta() not called' ), open( metadata_files.filename_results_code, 'wb+' ) ) # create the file on disk, so it cannot be reused by tempfile (unlikely, but possible)
573                #file to store kwds passed to set_meta()
574                metadata_files.filename_kwds = relpath( tempfile.NamedTemporaryFile( dir = tmp_dir, prefix = "metadata_kwds_%s_" % key ).name )
575                simplejson.dump( kwds, open( metadata_files.filename_kwds, 'wb+' ), ensure_ascii=True )
576                #existing metadata file parameters need to be overridden with cluster-writable file locations
577                metadata_files.filename_override_metadata = relpath( tempfile.NamedTemporaryFile( dir = tmp_dir, prefix = "metadata_override_%s_" % key ).name )
578                open( metadata_files.filename_override_metadata, 'wb+' ) # create the file on disk, so it cannot be reused by tempfile (unlikely, but possible)
579                override_metadata = []
580                for meta_key, spec_value in dataset.metadata.spec.iteritems():
581                    if isinstance( spec_value.param, FileParameter ) and dataset.metadata.get( meta_key, None ) is not None:
582                        metadata_temp = MetadataTempFile()
583                        shutil.copy( dataset.metadata.get( meta_key, None ).file_name, metadata_temp.file_name )
584                        override_metadata.append( ( meta_key, metadata_temp.to_JSON() ) )
585                simplejson.dump( override_metadata, open( metadata_files.filename_override_metadata, 'wb+' ) )
586                #add to session and flush
587                sa_session.add( metadata_files )
588                sa_session.flush()
589            metadata_files_list.append( metadata_files )
590        #return command required to build
591        return "%s %s %s %s %s %s %s" % ( os.path.join( exec_dir, 'set_metadata.sh' ), dataset_files_path, tmp_dir, config_root, datatypes_config, job_metadata, " ".join( map( __metadata_files_list_to_cmd_line, metadata_files_list ) ) )
592   
593    def external_metadata_set_successfully( self, dataset, sa_session ):
594        metadata_files = self.get_output_filenames_by_dataset( dataset, sa_session )
595        if not metadata_files:
596            return False # this file doesn't exist
597        rval, rstring = simplejson.load( open( metadata_files.filename_results_code ) )
598        if not rval:
599            log.debug( 'setting metadata externally failed for %s %s: %s' % ( dataset.__class__.__name__, dataset.id, rstring ) )
600        return rval
601   
602    def cleanup_external_metadata( self, sa_session ):
603        log.debug( 'Cleaning up external metadata files' )
604        for metadata_files in sa_session.query( galaxy.model.Job ).get( self.job_id ).external_output_metadata:
605            #we need to confirm that any MetadataTempFile files were removed, if not we need to remove them
606            #can occur if the job was stopped before completion, but a MetadataTempFile is used in the set_meta
607            MetadataTempFile.cleanup_from_JSON_dict_filename( metadata_files.filename_out )
608            dataset_key = self.get_dataset_metadata_key( metadata_files.dataset )
609            for key, fname in [ ( 'filename_in', metadata_files.filename_in ), ( 'filename_out', metadata_files.filename_out ), ( 'filename_results_code', metadata_files.filename_results_code ), ( 'filename_kwds', metadata_files.filename_kwds ), ( 'filename_override_metadata', metadata_files.filename_override_metadata ) ]:
610                try:
611                    os.remove( fname )
612                except Exception, e:
613                    log.debug( 'Failed to cleanup external metadata file (%s) for %s: %s' % ( key, dataset_key, e ) )
614    def set_job_runner_external_pid( self, pid, sa_session ):
615        for metadata_files in sa_session.query( galaxy.model.Job ).get( self.job_id ).external_output_metadata:
616            metadata_files.job_runner_external_pid = pid
617            sa_session.add( metadata_files )
618            sa_session.flush()
Note: リポジトリブラウザについてのヘルプは TracBrowser を参照してください。