[2] | 1 | """ |
---|
| 2 | Support for constructing and viewing custom "track" browsers within Galaxy. |
---|
| 3 | |
---|
| 4 | Track browsers are currently transient -- nothing is stored to the database |
---|
| 5 | when a browser is created. Building a browser consists of selecting a set |
---|
| 6 | of datasets associated with the same dbkey to display. Once selected, jobs |
---|
| 7 | are started to create any necessary indexes in the background, and the user |
---|
| 8 | is redirected to the browser interface, which loads the appropriate datasets. |
---|
| 9 | |
---|
| 10 | """ |
---|
| 11 | |
---|
| 12 | import re, pkg_resources |
---|
| 13 | pkg_resources.require( "bx-python" ) |
---|
| 14 | |
---|
| 15 | from bx.seq.twobit import TwoBitFile |
---|
| 16 | from galaxy import model |
---|
| 17 | from galaxy.util.json import to_json_string, from_json_string |
---|
| 18 | from galaxy.web.base.controller import * |
---|
| 19 | from galaxy.web.framework import simplejson |
---|
| 20 | from galaxy.web.framework.helpers import grids |
---|
| 21 | from galaxy.util.bunch import Bunch |
---|
| 22 | |
---|
| 23 | from galaxy.visualization.tracks.data_providers import * |
---|
| 24 | |
---|
| 25 | # Message strings returned to browser |
---|
| 26 | messages = Bunch( |
---|
| 27 | PENDING = "pending", |
---|
| 28 | NO_DATA = "no data", |
---|
| 29 | NO_CHROMOSOME = "no chromosome", |
---|
| 30 | NO_CONVERTER = "no converter", |
---|
| 31 | DATA = "data", |
---|
| 32 | ERROR = "error" |
---|
| 33 | ) |
---|
| 34 | |
---|
| 35 | class DatasetSelectionGrid( grids.Grid ): |
---|
| 36 | class DbKeyColumn( grids.GridColumn ): |
---|
| 37 | def filter( self, trans, user, query, dbkey ): |
---|
| 38 | """ Filter by dbkey. """ |
---|
| 39 | # use raw SQL b/c metadata is a BLOB |
---|
| 40 | dbkey = dbkey.replace("'", "\\'") |
---|
| 41 | return query.filter( or_( "metadata like '%%\"dbkey\": [\"%s\"]%%'" % dbkey, "metadata like '%%\"dbkey\": \"%s\"%%'" % dbkey ) ) |
---|
| 42 | |
---|
| 43 | # Grid definition. |
---|
| 44 | available_tracks = None |
---|
| 45 | title = "Add Tracks" |
---|
| 46 | template = "/tracks/add_tracks.mako" |
---|
| 47 | async_template = "/page/select_items_grid_async.mako" |
---|
| 48 | model_class = model.HistoryDatasetAssociation |
---|
| 49 | default_filter = { "deleted" : "False" , "shared" : "All" } |
---|
| 50 | default_sort_key = "name" |
---|
| 51 | use_async = True |
---|
| 52 | use_paging = False |
---|
| 53 | columns = [ |
---|
| 54 | grids.TextColumn( "Name", key="name", model_class=model.HistoryDatasetAssociation ), |
---|
| 55 | grids.TextColumn( "Filetype", key="extension", model_class=model.HistoryDatasetAssociation ), |
---|
| 56 | DbKeyColumn( "Dbkey", key="dbkey", model_class=model.HistoryDatasetAssociation, visible=False ) |
---|
| 57 | ] |
---|
| 58 | columns.append( |
---|
| 59 | grids.MulticolFilterColumn( "Search", cols_to_filter=[ columns[0], columns[1] ], |
---|
| 60 | key="free-text-search", visible=False, filterable="standard" ) |
---|
| 61 | ) |
---|
| 62 | |
---|
| 63 | def build_initial_query( self, trans, **kwargs ): |
---|
| 64 | return trans.sa_session.query( self.model_class ).join( model.History.table).join( model.Dataset.table ) |
---|
| 65 | def apply_query_filter( self, trans, query, **kwargs ): |
---|
| 66 | if self.available_tracks is None: |
---|
| 67 | self.available_tracks = trans.app.datatypes_registry.get_available_tracks() |
---|
| 68 | return query.filter( model.History.user == trans.user ) \ |
---|
| 69 | .filter( model.HistoryDatasetAssociation.extension.in_(self.available_tracks) ) \ |
---|
| 70 | .filter( model.Dataset.state == model.Dataset.states.OK ) \ |
---|
| 71 | .filter( model.History.deleted == False ) \ |
---|
| 72 | .filter( model.HistoryDatasetAssociation.deleted == False ) |
---|
| 73 | |
---|
| 74 | class TracksterSelectionGrid( grids.Grid ): |
---|
| 75 | |
---|
| 76 | # Grid definition. |
---|
| 77 | title = "Insert into visualization" |
---|
| 78 | template = "/tracks/add_to_viz.mako" |
---|
| 79 | async_template = "/page/select_items_grid_async.mako" |
---|
| 80 | model_class = model.Visualization |
---|
| 81 | default_filter = { "deleted" : "False" , "shared" : "All" } |
---|
| 82 | default_sort_key = "title" |
---|
| 83 | use_async = True |
---|
| 84 | use_paging = False |
---|
| 85 | columns = [ |
---|
| 86 | grids.TextColumn( "Title", key="title", model_class=model.Visualization ), |
---|
| 87 | grids.TextColumn( "Dbkey", key="dbkey", model_class=model.Visualization ) |
---|
| 88 | ] |
---|
| 89 | columns.append( |
---|
| 90 | grids.MulticolFilterColumn( "Search", cols_to_filter=[ columns[0] ], |
---|
| 91 | key="free-text-search", visible=False, filterable="standard" ) |
---|
| 92 | ) |
---|
| 93 | |
---|
| 94 | def build_initial_query( self, trans, **kwargs ): |
---|
| 95 | return trans.sa_session.query( self.model_class ) |
---|
| 96 | def apply_query_filter( self, trans, query, **kwargs ): |
---|
| 97 | return query.filter( self.model_class.user_id == trans.user.id ) |
---|
| 98 | |
---|
| 99 | class TracksController( BaseController, UsesVisualization ): |
---|
| 100 | """ |
---|
| 101 | Controller for track browser interface. Handles building a new browser from |
---|
| 102 | datasets in the current history, and display of the resulting browser. |
---|
| 103 | """ |
---|
| 104 | |
---|
| 105 | available_tracks = None |
---|
| 106 | available_genomes = None |
---|
| 107 | |
---|
| 108 | def _init_references(self, trans): |
---|
| 109 | avail_genomes = {} |
---|
| 110 | for line in open( os.path.join( trans.app.config.tool_data_path, "twobit.loc" ) ): |
---|
| 111 | if line.startswith("#"): continue |
---|
| 112 | val = line.split() |
---|
| 113 | if len(val) == 2: |
---|
| 114 | key, path = val |
---|
| 115 | avail_genomes[key] = path |
---|
| 116 | self.available_genomes = avail_genomes |
---|
| 117 | |
---|
| 118 | @web.expose |
---|
| 119 | @web.require_login() |
---|
| 120 | def index( self, trans, **kwargs ): |
---|
| 121 | config = {} |
---|
| 122 | |
---|
| 123 | return trans.fill_template( "tracks/browser.mako", config=config, add_dataset=kwargs.get("dataset_id", None), \ |
---|
| 124 | default_dbkey=kwargs.get("default_dbkey", None) ) |
---|
| 125 | |
---|
| 126 | @web.expose |
---|
| 127 | @web.require_login() |
---|
| 128 | def new_browser( self, trans, **kwargs ): |
---|
| 129 | return trans.fill_template( "tracks/new_browser.mako", dbkeys=self._get_dbkeys( trans ), default_dbkey=kwargs.get("default_dbkey", None) ) |
---|
| 130 | |
---|
| 131 | @web.json |
---|
| 132 | @web.require_login() |
---|
| 133 | def add_track_async(self, trans, id): |
---|
| 134 | dataset_id = trans.security.decode_id( id ) |
---|
| 135 | |
---|
| 136 | hda_query = trans.sa_session.query( model.HistoryDatasetAssociation ) |
---|
| 137 | dataset = hda_query.get( dataset_id ) |
---|
| 138 | track_type, _ = dataset.datatype.get_track_type() |
---|
| 139 | track_data_provider_class = get_data_provider( original_dataset=dataset ) |
---|
| 140 | track_data_provider = track_data_provider_class( original_dataset=dataset ) |
---|
| 141 | |
---|
| 142 | track = { |
---|
| 143 | "track_type": track_type, |
---|
| 144 | "name": dataset.name, |
---|
| 145 | "dataset_id": dataset.id, |
---|
| 146 | "prefs": {}, |
---|
| 147 | "filters": track_data_provider.get_filters() |
---|
| 148 | } |
---|
| 149 | return track |
---|
| 150 | |
---|
| 151 | @web.expose |
---|
| 152 | @web.require_login() |
---|
| 153 | def browser(self, trans, id, chrom="", **kwargs): |
---|
| 154 | """ |
---|
| 155 | Display browser for the datasets listed in `dataset_ids`. |
---|
| 156 | """ |
---|
| 157 | decoded_id = trans.security.decode_id( id ) |
---|
| 158 | session = trans.sa_session |
---|
| 159 | vis = session.query( model.Visualization ).get( decoded_id ) |
---|
| 160 | viz_config = self.get_visualization_config( trans, vis ) |
---|
| 161 | |
---|
| 162 | new_dataset = kwargs.get("dataset_id", None) |
---|
| 163 | if new_dataset is not None: |
---|
| 164 | if trans.security.decode_id(new_dataset) in [ d["dataset_id"] for d in viz_config.get("tracks") ]: |
---|
| 165 | new_dataset = None # Already in browser, so don't add |
---|
| 166 | return trans.fill_template( 'tracks/browser.mako', config=viz_config, add_dataset=new_dataset ) |
---|
| 167 | |
---|
| 168 | @web.json |
---|
| 169 | def chroms(self, trans, vis_id=None, dbkey=None ): |
---|
| 170 | """ |
---|
| 171 | Returns a naturally sorted list of chroms/contigs for either a given visualization or a given dbkey. |
---|
| 172 | """ |
---|
| 173 | def check_int(s): |
---|
| 174 | if s.isdigit(): |
---|
| 175 | return int(s) |
---|
| 176 | else: |
---|
| 177 | return s |
---|
| 178 | |
---|
| 179 | def split_by_number(s): |
---|
| 180 | return [ check_int(c) for c in re.split('([0-9]+)', s) ] |
---|
| 181 | |
---|
| 182 | # Must specify either vis_id or dbkey. |
---|
| 183 | if not vis_id and not dbkey: |
---|
| 184 | return trans.show_error_message("No visualization id or dbkey specified.") |
---|
| 185 | |
---|
| 186 | # Need to get user and dbkey in order to get chroms data. |
---|
| 187 | if vis_id: |
---|
| 188 | # Use user, dbkey from viz. |
---|
| 189 | visualization = self.get_visualization( trans, vis_id, check_ownership=False, check_accessible=True ) |
---|
| 190 | visualization.config = self.get_visualization_config( trans, visualization ) |
---|
| 191 | vis_user = visualization.user |
---|
| 192 | vis_dbkey = visualization.dbkey |
---|
| 193 | else: |
---|
| 194 | # No vis_id, so visualization is new. User is current user, dbkey must be given. |
---|
| 195 | vis_user = trans.user |
---|
| 196 | vis_dbkey = dbkey |
---|
| 197 | |
---|
| 198 | # Get chroms data. |
---|
| 199 | chroms = self._chroms( trans, vis_user, vis_dbkey ) |
---|
| 200 | |
---|
| 201 | # Check for reference chrom |
---|
| 202 | if self.available_genomes is None: self._init_references(trans) |
---|
| 203 | |
---|
| 204 | to_sort = [{ 'chrom': chrom, 'len': length } for chrom, length in chroms.iteritems()] |
---|
| 205 | to_sort.sort(lambda a,b: cmp( split_by_number(a['chrom']), split_by_number(b['chrom']) )) |
---|
| 206 | return { 'reference': vis_dbkey in self.available_genomes, 'chrom_info': to_sort } |
---|
| 207 | |
---|
| 208 | def _chroms( self, trans, user, dbkey ): |
---|
| 209 | """ |
---|
| 210 | Helper method that returns chrom lengths for a given user and dbkey. |
---|
| 211 | """ |
---|
| 212 | len_file = None |
---|
| 213 | len_ds = None |
---|
| 214 | # If there is any dataset in the history of extension `len`, this will use it |
---|
| 215 | if 'dbkeys' in user.preferences: |
---|
| 216 | user_keys = from_json_string( user.preferences['dbkeys'] ) |
---|
| 217 | if dbkey in user_keys: |
---|
| 218 | len_file = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( user_keys[dbkey]['len'] ).file_name |
---|
| 219 | |
---|
| 220 | if not len_file: |
---|
| 221 | len_ds = trans.db_dataset_for( dbkey ) |
---|
| 222 | if not len_ds: |
---|
| 223 | len_file = os.path.join( trans.app.config.tool_data_path, 'shared','ucsc','chrom', "%s.len" % dbkey ) |
---|
| 224 | else: |
---|
| 225 | len_file = len_ds.file_name |
---|
| 226 | manifest = {} |
---|
| 227 | if not os.path.exists( len_file ): |
---|
| 228 | return None |
---|
| 229 | for line in open( len_file ): |
---|
| 230 | if line.startswith("#"): continue |
---|
| 231 | line = line.rstrip("\r\n") |
---|
| 232 | fields = line.split("\t") |
---|
| 233 | manifest[fields[0]] = int(fields[1]) |
---|
| 234 | return manifest |
---|
| 235 | |
---|
| 236 | @web.json |
---|
| 237 | def reference( self, trans, dbkey, chrom, low, high, **kwargs ): |
---|
| 238 | if self.available_genomes is None: self._init_references(trans) |
---|
| 239 | |
---|
| 240 | if dbkey not in self.available_genomes: |
---|
| 241 | return None |
---|
| 242 | |
---|
| 243 | try: |
---|
| 244 | twobit = TwoBitFile( open(self.available_genomes[dbkey]) ) |
---|
| 245 | except IOError: |
---|
| 246 | return None |
---|
| 247 | |
---|
| 248 | if chrom in twobit: |
---|
| 249 | return twobit[chrom].get(int(low), int(high)) |
---|
| 250 | |
---|
| 251 | return None |
---|
| 252 | |
---|
| 253 | @web.json |
---|
| 254 | def data( self, trans, dataset_id, chrom, low, high, **kwargs ): |
---|
| 255 | """ |
---|
| 256 | Called by the browser to request a block of data |
---|
| 257 | """ |
---|
| 258 | dataset = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( dataset_id ) |
---|
| 259 | if not dataset or not chrom: |
---|
| 260 | return messages.NO_DATA |
---|
| 261 | if dataset.state == trans.app.model.Job.states.ERROR: |
---|
| 262 | return messages.ERROR |
---|
| 263 | if dataset.state != trans.app.model.Job.states.OK: |
---|
| 264 | return messages.PENDING |
---|
| 265 | |
---|
| 266 | track_type, data_sources = dataset.datatype.get_track_type() |
---|
| 267 | for source_type, data_source in data_sources.iteritems(): |
---|
| 268 | try: |
---|
| 269 | converted_dataset = dataset.get_converted_dataset(trans, data_source) |
---|
| 270 | except ValueError: |
---|
| 271 | return messages.NO_CONVERTER |
---|
| 272 | |
---|
| 273 | # Need to check states again for the converted version |
---|
| 274 | if converted_dataset and converted_dataset.state == model.Dataset.states.ERROR: |
---|
| 275 | job_id = trans.sa_session.query( trans.app.model.JobToOutputDatasetAssociation ).filter_by( dataset_id=converted_dataset.id ).first().job_id |
---|
| 276 | job = trans.sa_session.query( trans.app.model.Job ).get( job_id ) |
---|
| 277 | return { 'kind': messages.ERROR, 'message': job.stderr } |
---|
| 278 | |
---|
| 279 | if not converted_dataset or converted_dataset.state != model.Dataset.states.OK: |
---|
| 280 | return messages.PENDING |
---|
| 281 | |
---|
| 282 | extra_info = None |
---|
| 283 | if 'index' in data_sources: |
---|
| 284 | # Have to choose between indexer and data provider |
---|
| 285 | indexer = get_data_provider( name=data_sources['index'] )( dataset.get_converted_dataset(trans, data_sources['index']), dataset ) |
---|
| 286 | summary = indexer.get_summary( chrom, low, high, **kwargs ) |
---|
| 287 | if summary is not None and kwargs.get("mode", "Auto") == "Auto": |
---|
| 288 | # Only check for summary if it's Auto mode (which is the default) |
---|
| 289 | if summary == "no_detail": |
---|
| 290 | kwargs["no_detail"] = True # meh |
---|
| 291 | extra_info = "no_detail" |
---|
| 292 | else: |
---|
| 293 | frequencies, max_v, avg_v, delta = summary |
---|
| 294 | return { 'dataset_type': data_sources['index'], 'data': frequencies, 'max': max_v, 'avg': avg_v, 'delta': delta } |
---|
| 295 | |
---|
| 296 | # Get data provider. |
---|
| 297 | tracks_dataset_type = data_sources['data'] |
---|
| 298 | data_provider_class = get_data_provider( name=tracks_dataset_type, original_dataset=dataset ) |
---|
| 299 | data_provider = data_provider_class( dataset.get_converted_dataset(trans, tracks_dataset_type), dataset ) |
---|
| 300 | |
---|
| 301 | # Get and return data from data_provider. |
---|
| 302 | data = data_provider.get_data( chrom, low, high, **kwargs ) |
---|
| 303 | message = None |
---|
| 304 | if isinstance(data, dict) and 'message' in data: |
---|
| 305 | message = data['message'] |
---|
| 306 | tracks_dataset_type = data.get( 'data_type', tracks_dataset_type ) |
---|
| 307 | track_data = data['data'] |
---|
| 308 | else: |
---|
| 309 | track_data = data |
---|
| 310 | return { 'dataset_type': tracks_dataset_type, 'extra_info': extra_info, 'data': track_data, 'message': message } |
---|
| 311 | |
---|
| 312 | |
---|
| 313 | @web.json |
---|
| 314 | def save( self, trans, **kwargs ): |
---|
| 315 | session = trans.sa_session |
---|
| 316 | vis_id = "undefined" |
---|
| 317 | if 'vis_id' in kwargs: |
---|
| 318 | vis_id = kwargs['vis_id'].strip('"') |
---|
| 319 | dbkey = kwargs['dbkey'] |
---|
| 320 | |
---|
| 321 | if vis_id == "undefined": # new vis |
---|
| 322 | vis = model.Visualization() |
---|
| 323 | vis.user = trans.user |
---|
| 324 | vis.title = kwargs['vis_title'] |
---|
| 325 | vis.type = "trackster" |
---|
| 326 | vis.dbkey = dbkey |
---|
| 327 | session.add( vis ) |
---|
| 328 | else: |
---|
| 329 | decoded_id = trans.security.decode_id( vis_id ) |
---|
| 330 | vis = session.query( model.Visualization ).get( decoded_id ) |
---|
| 331 | |
---|
| 332 | decoded_payload = simplejson.loads( kwargs['payload'] ) |
---|
| 333 | vis_rev = model.VisualizationRevision() |
---|
| 334 | vis_rev.visualization = vis |
---|
| 335 | vis_rev.title = vis.title |
---|
| 336 | vis_rev.dbkey = dbkey |
---|
| 337 | tracks = [] |
---|
| 338 | for track in decoded_payload: |
---|
| 339 | tracks.append( { "dataset_id": str(track['dataset_id']), |
---|
| 340 | "name": track['name'], |
---|
| 341 | "track_type": track['track_type'], |
---|
| 342 | "prefs": track['prefs'] |
---|
| 343 | } ) |
---|
| 344 | vis_rev.config = { "tracks": tracks } |
---|
| 345 | vis.latest_revision = vis_rev |
---|
| 346 | session.add( vis_rev ) |
---|
| 347 | session.flush() |
---|
| 348 | return trans.security.encode_id(vis.id) |
---|
| 349 | |
---|
| 350 | data_grid = DatasetSelectionGrid() |
---|
| 351 | tracks_grid = TracksterSelectionGrid() |
---|
| 352 | |
---|
| 353 | @web.expose |
---|
| 354 | @web.require_login( "see all available datasets" ) |
---|
| 355 | def list_datasets( self, trans, **kwargs ): |
---|
| 356 | """List all datasets that can be added as tracks""" |
---|
| 357 | |
---|
| 358 | # Render the list view |
---|
| 359 | return self.data_grid( trans, **kwargs ) |
---|
| 360 | |
---|
| 361 | @web.expose |
---|
| 362 | def list_tracks( self, trans, **kwargs ): |
---|
| 363 | return self.tracks_grid( trans, **kwargs ) |
---|
| 364 | |
---|