[2] | 1 | """ |
---|
| 2 | Execute an external process to set_meta() on a provided list of pickled datasets. |
---|
| 3 | |
---|
| 4 | This should not be called directly! Use the set_metadata.sh script in Galaxy's |
---|
| 5 | top level directly. |
---|
| 6 | |
---|
| 7 | """ |
---|
| 8 | |
---|
| 9 | import logging |
---|
| 10 | logging.basicConfig() |
---|
| 11 | log = logging.getLogger( __name__ ) |
---|
| 12 | |
---|
| 13 | import os, sys, cPickle |
---|
| 14 | assert sys.version_info[:2] >= ( 2, 4 ) |
---|
| 15 | |
---|
| 16 | new_path = [ os.path.join( os.getcwd(), "lib" ) ] |
---|
| 17 | new_path.extend( sys.path[1:] ) # remove scripts/ from the path |
---|
| 18 | sys.path = new_path |
---|
| 19 | |
---|
| 20 | from galaxy import eggs |
---|
| 21 | import pkg_resources |
---|
| 22 | pkg_resources.require("simplejson") |
---|
| 23 | import simplejson |
---|
| 24 | import galaxy.model.mapping #need to load this before we unpickle, in order to setup properties assigned by the mappers |
---|
| 25 | galaxy.model.Job() #this looks REAL stupid, but it is REQUIRED in order for SA to insert parameters into the classes defined by the mappers --> it appears that instantiating ANY mapper'ed class would suffice here |
---|
| 26 | galaxy.datatypes.metadata.DATABASE_CONNECTION_AVAILABLE = False #Let metadata know that there is no database connection, and to just assume object ids are valid |
---|
| 27 | from galaxy.util import stringify_dictionary_keys |
---|
| 28 | from galaxy.util.json import from_json_string |
---|
| 29 | from sqlalchemy.orm import clear_mappers |
---|
| 30 | |
---|
| 31 | def __main__(): |
---|
| 32 | file_path = sys.argv.pop( 1 ) |
---|
| 33 | tmp_dir = sys.argv.pop( 1 ) |
---|
| 34 | galaxy.model.Dataset.file_path = file_path |
---|
| 35 | galaxy.datatypes.metadata.MetadataTempFile.tmp_dir = tmp_dir |
---|
| 36 | |
---|
| 37 | # Set up datatypes registry |
---|
| 38 | config_root = sys.argv.pop( 1 ) |
---|
| 39 | datatypes_config = sys.argv.pop( 1 ) |
---|
| 40 | galaxy.model.set_datatypes_registry( galaxy.datatypes.registry.Registry( config_root, datatypes_config ) ) |
---|
| 41 | |
---|
| 42 | job_metadata = sys.argv.pop( 1 ) |
---|
| 43 | ext_override = dict() |
---|
| 44 | if job_metadata != "None" and os.path.exists( job_metadata ): |
---|
| 45 | for line in open( job_metadata, 'r' ): |
---|
| 46 | try: |
---|
| 47 | line = stringify_dictionary_keys( from_json_string( line ) ) |
---|
| 48 | assert line['type'] == 'dataset' |
---|
| 49 | ext_override[line['dataset_id']] = line['ext'] |
---|
| 50 | except: |
---|
| 51 | continue |
---|
| 52 | for filenames in sys.argv[1:]: |
---|
| 53 | fields = filenames.split( ',' ) |
---|
| 54 | filename_in = fields.pop( 0 ) |
---|
| 55 | filename_kwds = fields.pop( 0 ) |
---|
| 56 | filename_out = fields.pop( 0 ) |
---|
| 57 | filename_results_code = fields.pop( 0 ) |
---|
| 58 | dataset_filename_override = fields.pop( 0 ) |
---|
| 59 | #Need to be careful with the way that these parameters are populated from the filename splitting, |
---|
| 60 | #because if a job is running when the server is updated, any existing external metadata command-lines |
---|
| 61 | #will not have info about the newly added override_metadata file |
---|
| 62 | if fields: |
---|
| 63 | override_metadata = fields.pop( 0 ) |
---|
| 64 | else: |
---|
| 65 | override_metadata = None |
---|
| 66 | try: |
---|
| 67 | dataset = cPickle.load( open( filename_in ) ) #load DatasetInstance |
---|
| 68 | if dataset_filename_override: |
---|
| 69 | dataset.dataset.external_filename = dataset_filename_override |
---|
| 70 | if ext_override.get( dataset.dataset.id, None ): |
---|
| 71 | dataset.extension = ext_override[ dataset.dataset.id ] |
---|
| 72 | #Metadata FileParameter types may not be writable on a cluster node, and are therefore temporarily substituted with MetadataTempFiles |
---|
| 73 | if override_metadata: |
---|
| 74 | override_metadata = simplejson.load( open( override_metadata ) ) |
---|
| 75 | for metadata_name, metadata_file_override in override_metadata: |
---|
| 76 | if galaxy.datatypes.metadata.MetadataTempFile.is_JSONified_value( metadata_file_override ): |
---|
| 77 | metadata_file_override = galaxy.datatypes.metadata.MetadataTempFile.from_JSON( metadata_file_override ) |
---|
| 78 | setattr( dataset.metadata, metadata_name, metadata_file_override ) |
---|
| 79 | kwds = stringify_dictionary_keys( simplejson.load( open( filename_kwds ) ) )#load kwds; need to ensure our keywords are not unicode |
---|
| 80 | dataset.datatype.set_meta( dataset, **kwds ) |
---|
| 81 | dataset.metadata.to_JSON_dict( filename_out ) # write out results of set_meta |
---|
| 82 | simplejson.dump( ( True, 'Metadata has been set successfully' ), open( filename_results_code, 'wb+' ) ) #setting metadata has succeeded |
---|
| 83 | except Exception, e: |
---|
| 84 | simplejson.dump( ( False, str( e ) ), open( filename_results_code, 'wb+' ) ) #setting metadata has failed somehow |
---|
| 85 | clear_mappers() |
---|
| 86 | |
---|
| 87 | __main__() |
---|