[2] | 1 | """ |
---|
| 2 | Manage Galaxy eggs |
---|
| 3 | """ |
---|
| 4 | |
---|
| 5 | import os, sys, shutil, glob, urllib, urllib2, ConfigParser, HTMLParser, zipimport, zipfile |
---|
| 6 | |
---|
| 7 | import logging |
---|
| 8 | log = logging.getLogger( __name__ ) |
---|
| 9 | log.addHandler( logging.NullHandler() ) |
---|
| 10 | |
---|
| 11 | import pkg_resources |
---|
| 12 | |
---|
| 13 | galaxy_dir = os.path.abspath( os.path.join( os.path.dirname( __file__ ), '..', '..', '..' ) ) |
---|
| 14 | py = 'py%s' % sys.version[:3] |
---|
| 15 | |
---|
| 16 | class EggNotFetchable( Exception ): |
---|
| 17 | def __init__( self, eggs ): |
---|
| 18 | if type( eggs ) in ( list, tuple ): |
---|
| 19 | self.eggs = eggs |
---|
| 20 | else: |
---|
| 21 | self.eggs = [ eggs ] |
---|
| 22 | def __str__( self ): |
---|
| 23 | return ' '.join( self.eggs ) |
---|
| 24 | |
---|
| 25 | # need the options to remain case sensitive |
---|
| 26 | class CaseSensitiveConfigParser( ConfigParser.SafeConfigParser ): |
---|
| 27 | def optionxform( self, optionstr ): |
---|
| 28 | return optionstr |
---|
| 29 | |
---|
| 30 | # so we can actually detect failures |
---|
| 31 | class URLRetriever( urllib.FancyURLopener ): |
---|
| 32 | def http_error_default( *args ): |
---|
| 33 | urllib.URLopener.http_error_default( *args ) |
---|
| 34 | |
---|
| 35 | class Egg( object ): |
---|
| 36 | """ |
---|
| 37 | Contains information about locating and downloading eggs. |
---|
| 38 | """ |
---|
| 39 | def __init__( self, name=None, version=None, tag=None, url=None, platform=None ): |
---|
| 40 | self.name = name |
---|
| 41 | self.version = version |
---|
| 42 | self.tag = tag |
---|
| 43 | self.url = url |
---|
| 44 | self.platform = platform |
---|
| 45 | self.distribution = None |
---|
| 46 | self.dir = None |
---|
| 47 | if self.name is not None and self.version is not None: |
---|
| 48 | self.set_distribution() |
---|
| 49 | def set_dir( self ): |
---|
| 50 | self.dir = os.path.join( galaxy_dir, 'eggs' ) |
---|
| 51 | if not os.path.exists( self.dir ): |
---|
| 52 | os.makedirs( self.dir ) |
---|
| 53 | def set_distribution( self ): |
---|
| 54 | """ |
---|
| 55 | Stores a pkg_resources Distribution object for reference later |
---|
| 56 | """ |
---|
| 57 | if self.dir is None: |
---|
| 58 | self.set_dir() |
---|
| 59 | tag = self.tag or '' |
---|
| 60 | self.distribution = pkg_resources.Distribution.from_filename( |
---|
| 61 | os.path.join( self.dir, '-'.join( ( self.name, self.version + tag, self.platform ) ) + '.egg' ) ) |
---|
| 62 | @property |
---|
| 63 | def path( self ): |
---|
| 64 | """ |
---|
| 65 | Return the path of the egg, if it exists, or None |
---|
| 66 | """ |
---|
| 67 | if env[self.distribution.project_name]: |
---|
| 68 | return env[self.distribution.project_name][0].location |
---|
| 69 | return None |
---|
| 70 | def fetch( self, requirement ): |
---|
| 71 | """ |
---|
| 72 | fetch() serves as the install method to pkg_resources.working_set.resolve() |
---|
| 73 | """ |
---|
| 74 | def find_alternative(): |
---|
| 75 | """ |
---|
| 76 | Some platforms (e.g. Solaris) support eggs compiled on older platforms |
---|
| 77 | """ |
---|
| 78 | class LinkParser( HTMLParser.HTMLParser ): |
---|
| 79 | """ |
---|
| 80 | Finds links in what should be an Apache-style directory index |
---|
| 81 | """ |
---|
| 82 | def __init__( self ): |
---|
| 83 | HTMLParser.HTMLParser.__init__( self ) |
---|
| 84 | self.links = [] |
---|
| 85 | def handle_starttag( self, tag, attrs ): |
---|
| 86 | if tag == 'a' and 'href' in dict( attrs ): |
---|
| 87 | self.links.append( dict( attrs )['href'] ) |
---|
| 88 | parser = LinkParser() |
---|
| 89 | try: |
---|
| 90 | parser.feed( urllib2.urlopen( self.url + '/' ).read() ) |
---|
| 91 | except urllib2.HTTPError, e: |
---|
| 92 | if e.code == 404: |
---|
| 93 | return None |
---|
| 94 | parser.close() |
---|
| 95 | for link in parser.links: |
---|
| 96 | file = urllib.unquote( link ).rsplit( '/', 1 )[-1] |
---|
| 97 | tmp_dist = pkg_resources.Distribution.from_filename( file ) |
---|
| 98 | if tmp_dist.platform is not None and \ |
---|
| 99 | self.distribution.project_name == tmp_dist.project_name and \ |
---|
| 100 | self.distribution.version == tmp_dist.version and \ |
---|
| 101 | self.distribution.py_version == tmp_dist.py_version and \ |
---|
| 102 | pkg_resources.compatible_platforms( tmp_dist.platform, pkg_resources.get_platform() ): |
---|
| 103 | return file |
---|
| 104 | return None |
---|
| 105 | if self.url is None: |
---|
| 106 | return None |
---|
| 107 | alternative = None |
---|
| 108 | try: |
---|
| 109 | url = self.url + '/' + self.distribution.egg_name() + '.egg' |
---|
| 110 | URLRetriever().retrieve( url, self.distribution.location ) |
---|
| 111 | log.debug( "Fetched %s" % url ) |
---|
| 112 | except IOError, e: |
---|
| 113 | if e[1] == 404 and self.distribution.platform != py: |
---|
| 114 | alternative = find_alternative() |
---|
| 115 | if alternative is None: |
---|
| 116 | return None |
---|
| 117 | else: |
---|
| 118 | return None |
---|
| 119 | if alternative is not None: |
---|
| 120 | try: |
---|
| 121 | url = '/'.join( ( self.url, alternative ) ) |
---|
| 122 | URLRetriever().retrieve( url, os.path.join( self.dir, alternative ) ) |
---|
| 123 | log.debug( "Fetched %s" % url ) |
---|
| 124 | except IOError, e: |
---|
| 125 | return None |
---|
| 126 | self.platform = alternative.split( '-', 2 )[-1].rsplit( '.egg', 1 )[0] |
---|
| 127 | self.set_distribution() |
---|
| 128 | self.unpack_if_needed() |
---|
| 129 | self.remove_doppelgangers() |
---|
| 130 | global env |
---|
| 131 | env = get_env() # reset the global Environment object now that we've obtained a new egg |
---|
| 132 | return self.distribution |
---|
| 133 | def unpack_if_needed( self ): |
---|
| 134 | meta = pkg_resources.EggMetadata( zipimport.zipimporter( self.distribution.location ) ) |
---|
| 135 | if meta.has_metadata( 'not-zip-safe' ): |
---|
| 136 | unpack_zipfile( self.distribution.location, self.distribution.location + "-tmp" ) |
---|
| 137 | os.remove( self.distribution.location ) |
---|
| 138 | os.rename( self.distribution.location + "-tmp", self.distribution.location ) |
---|
| 139 | def remove_doppelgangers( self ): |
---|
| 140 | doppelgangers = glob.glob( os.path.join( self.dir, "%s-*-%s.egg" % ( self.name, self.platform ) ) ) |
---|
| 141 | if self.distribution.location in doppelgangers: |
---|
| 142 | doppelgangers.remove( self.distribution.location ) |
---|
| 143 | for doppelganger in doppelgangers: |
---|
| 144 | remove_file_or_path( doppelganger ) |
---|
| 145 | log.debug( "Removed conflicting egg: %s" % doppelganger ) |
---|
| 146 | def resolve( self ): |
---|
| 147 | try: |
---|
| 148 | return pkg_resources.working_set.resolve( ( self.distribution.as_requirement(), ), env, self.fetch ) |
---|
| 149 | except pkg_resources.DistributionNotFound, e: |
---|
| 150 | # If this statement is true, it means we do have the requested egg, |
---|
| 151 | # just not one (or more) of its deps. |
---|
| 152 | if e.args[0].project_name != self.distribution.project_name: |
---|
| 153 | log.warning( "Warning: %s (a dependant egg of %s) cannot be fetched" % ( e.args[0].project_name, self.distribution.project_name ) ) |
---|
| 154 | return ( self.distribution, ) |
---|
| 155 | else: |
---|
| 156 | raise EggNotFetchable( self ) |
---|
| 157 | except pkg_resources.VersionConflict, e: |
---|
| 158 | # there's a conflicting egg on the path, remove it |
---|
| 159 | dist = e.args[0] |
---|
| 160 | # use the canonical path for comparisons |
---|
| 161 | location = os.path.realpath( dist.location ) |
---|
| 162 | for entry in pkg_resources.working_set.entries: |
---|
| 163 | if os.path.realpath( entry ) == location: |
---|
| 164 | pkg_resources.working_set.entries.remove( entry ) |
---|
| 165 | break |
---|
| 166 | else: |
---|
| 167 | location = entry = None |
---|
| 168 | del pkg_resources.working_set.by_key[dist.key] |
---|
| 169 | if entry is not None: |
---|
| 170 | pkg_resources.working_set.entry_keys[entry] = [] |
---|
| 171 | if entry in sys.path: |
---|
| 172 | sys.path.remove(entry) |
---|
| 173 | r = pkg_resources.working_set.resolve( ( self.distribution.as_requirement(), ), env, self.fetch ) |
---|
| 174 | if location is not None and not location.endswith( '.egg' ): |
---|
| 175 | # re-add the path if it's a non-egg dir, in case more deps live there |
---|
| 176 | pkg_resources.working_set.entries.append( location ) |
---|
| 177 | sys.path.append( location ) |
---|
| 178 | return r |
---|
| 179 | def require( self ): |
---|
| 180 | try: |
---|
| 181 | dists = self.resolve() |
---|
| 182 | for dist in dists: |
---|
| 183 | pkg_resources.working_set.add( dist ) |
---|
| 184 | return dists |
---|
| 185 | except: |
---|
| 186 | raise |
---|
| 187 | |
---|
| 188 | class Crate( object ): |
---|
| 189 | """ |
---|
| 190 | Reads the eggs.ini file for use with checking and fetching. |
---|
| 191 | """ |
---|
| 192 | config_file = os.path.join( galaxy_dir, 'eggs.ini' ) |
---|
| 193 | def __init__( self, platform=None ): |
---|
| 194 | self.eggs = {} |
---|
| 195 | self.config = CaseSensitiveConfigParser() |
---|
| 196 | self.repo = None |
---|
| 197 | self.no_auto = [] |
---|
| 198 | self.platform = platform |
---|
| 199 | self.py_platform = None |
---|
| 200 | if platform is not None: |
---|
| 201 | self.py_platform = platform.split( '-' )[0] |
---|
| 202 | self.galaxy_config = GalaxyConfig() |
---|
| 203 | self.parse() |
---|
| 204 | def parse( self ): |
---|
| 205 | self.config.read( Crate.config_file ) |
---|
| 206 | self.repo = self.config.get( 'general', 'repository' ) |
---|
| 207 | self.no_auto = self.config.get( 'general', 'no_auto' ).split() |
---|
| 208 | self.parse_egg_section( self.config.items( 'eggs:platform' ), self.config.items( 'tags' ), True ) |
---|
| 209 | self.parse_egg_section( self.config.items( 'eggs:noplatform' ), self.config.items( 'tags' ) ) |
---|
| 210 | def parse_egg_section( self, eggs, tags, full_platform=False, egg_class=Egg ): |
---|
| 211 | for name, version in eggs: |
---|
| 212 | tag = dict( tags ).get( name, '' ) |
---|
| 213 | url = '/'.join( ( self.repo, name ) ) |
---|
| 214 | if full_platform: |
---|
| 215 | platform = self.platform or '-'.join( ( py, pkg_resources.get_platform() ) ) |
---|
| 216 | else: |
---|
| 217 | platform = self.py_platform or py |
---|
| 218 | egg = egg_class( name, version, tag, url, platform ) |
---|
| 219 | self.eggs[name] = egg |
---|
| 220 | @property |
---|
| 221 | def config_missing( self ): |
---|
| 222 | """ |
---|
| 223 | Return true if any eggs are missing, conditional on options set in the |
---|
| 224 | Galaxy config file. |
---|
| 225 | """ |
---|
| 226 | for egg in self.config_eggs: |
---|
| 227 | if not egg.path: |
---|
| 228 | return True |
---|
| 229 | return False |
---|
| 230 | @property |
---|
| 231 | def all_missing( self ): |
---|
| 232 | """ |
---|
| 233 | Return true if any eggs in the eggs config file are missing. |
---|
| 234 | """ |
---|
| 235 | for egg in self.all_eggs: |
---|
| 236 | if not os.path.exists( egg.distribution.location ): |
---|
| 237 | return True |
---|
| 238 | return False |
---|
| 239 | @property |
---|
| 240 | def config_names( self ): |
---|
| 241 | """ |
---|
| 242 | Return a list of names of all eggs in the crate that are needed based |
---|
| 243 | on the options set in the Galaxy config file. |
---|
| 244 | """ |
---|
| 245 | return [ egg.name for egg in self.config_eggs ] |
---|
| 246 | @property |
---|
| 247 | def all_names( self ): |
---|
| 248 | """ |
---|
| 249 | Return a list of names of all eggs in the crate. |
---|
| 250 | """ |
---|
| 251 | return [ egg.name for egg in self.all_eggs ] |
---|
| 252 | @property |
---|
| 253 | def config_eggs( self ): |
---|
| 254 | """ |
---|
| 255 | Return a list of all eggs in the crate that are needed based on the |
---|
| 256 | options set in the Galaxy config file. |
---|
| 257 | """ |
---|
| 258 | return [ egg for egg in self.eggs.values() if self.galaxy_config.check_conditional( egg.name ) ] |
---|
| 259 | @property |
---|
| 260 | def all_eggs( self ): |
---|
| 261 | """ |
---|
| 262 | Return a list of all eggs in the crate. |
---|
| 263 | """ |
---|
| 264 | rval = [] |
---|
| 265 | for egg in self.eggs.values(): |
---|
| 266 | if egg.name not in self.galaxy_config.always_conditional: |
---|
| 267 | rval.append( egg ) |
---|
| 268 | return rval |
---|
| 269 | def __getitem__( self, name ): |
---|
| 270 | """ |
---|
| 271 | Return a specific egg. |
---|
| 272 | """ |
---|
| 273 | name = name.replace( '-', '_' ) |
---|
| 274 | return self.eggs[name] |
---|
| 275 | def resolve( self, all=False ): |
---|
| 276 | """ |
---|
| 277 | Try to resolve (e.g. fetch) all eggs in the crate. |
---|
| 278 | """ |
---|
| 279 | if all: |
---|
| 280 | eggs = self.all_eggs |
---|
| 281 | else: |
---|
| 282 | eggs = self.config_eggs |
---|
| 283 | eggs = filter( lambda x: x.name not in self.no_auto, eggs ) |
---|
| 284 | missing = [] |
---|
| 285 | for egg in eggs: |
---|
| 286 | try: |
---|
| 287 | egg.resolve() |
---|
| 288 | except EggNotFetchable: |
---|
| 289 | missing.append( egg ) |
---|
| 290 | if missing: |
---|
| 291 | raise EggNotFetchable( missing ) |
---|
| 292 | |
---|
| 293 | class GalaxyConfig( object ): |
---|
| 294 | config_file = os.path.join( galaxy_dir, "universe_wsgi.ini" ) |
---|
| 295 | always_conditional = ( 'GeneTrack', 'pysam' ) |
---|
| 296 | def __init__( self ): |
---|
| 297 | self.config = ConfigParser.ConfigParser() |
---|
| 298 | if self.config.read( GalaxyConfig.config_file ) == []: |
---|
| 299 | raise Exception( "error: unable to read Galaxy config from %s" % GalaxyConfig.config_file ) |
---|
| 300 | def check_conditional( self, egg_name ): |
---|
| 301 | def check_pysam(): |
---|
| 302 | # can't build pysam on solaris < 10 |
---|
| 303 | plat = pkg_resources.get_platform().split( '-' ) |
---|
| 304 | if plat[0] == 'solaris': |
---|
| 305 | minor = plat[1].split('.')[1] |
---|
| 306 | if int( minor ) < 10: |
---|
| 307 | return False |
---|
| 308 | return True |
---|
| 309 | if egg_name == "pysqlite": |
---|
| 310 | # SQLite is different since it can be specified in two config vars and defaults to True |
---|
| 311 | try: |
---|
| 312 | return self.config.get( "app:main", "database_connection" ).startswith( "sqlite://" ) |
---|
| 313 | except: |
---|
| 314 | return True |
---|
| 315 | else: |
---|
| 316 | try: |
---|
| 317 | return { "psycopg2": lambda: self.config.get( "app:main", "database_connection" ).startswith( "postgres://" ), |
---|
| 318 | "MySQL_python": lambda: self.config.get( "app:main", "database_connection" ).startswith( "mysql://" ), |
---|
| 319 | "DRMAA_python": lambda: "sge" in self.config.get( "app:main", "start_job_runners" ).split(","), |
---|
| 320 | "drmaa": lambda: ( "drmaa" in self.config.get( "app:main", "start_job_runners" ).split(",") ) and sys.version_info[:2] >= ( 2, 5 ), |
---|
| 321 | "pbs_python": lambda: "pbs" in self.config.get( "app:main", "start_job_runners" ).split(","), |
---|
| 322 | "threadframe": lambda: self.config.get( "app:main", "use_heartbeat" ), |
---|
| 323 | "guppy": lambda: self.config.get( "app:main", "use_memdump" ), |
---|
| 324 | "GeneTrack": lambda: sys.version_info[:2] >= ( 2, 5 ), |
---|
| 325 | "pysam": check_pysam() |
---|
| 326 | }.get( egg_name, lambda: True )() |
---|
| 327 | except: |
---|
| 328 | return False |
---|
| 329 | |
---|
| 330 | def get_env(): |
---|
| 331 | env = pkg_resources.Environment( platform=pkg_resources.get_platform() ) |
---|
| 332 | for dist in pkg_resources.find_distributions( os.path.join( galaxy_dir, 'eggs' ), False ): |
---|
| 333 | env.add( dist ) |
---|
| 334 | return env |
---|
| 335 | env = get_env() |
---|
| 336 | |
---|
| 337 | def require( req_str ): |
---|
| 338 | c = Crate() |
---|
| 339 | req = pkg_resources.Requirement.parse( req_str ) |
---|
| 340 | try: |
---|
| 341 | return c[req.project_name].require() |
---|
| 342 | except KeyError: |
---|
| 343 | # not a galaxy-owned dependency |
---|
| 344 | return pkg_resources.working_set.require( req_str ) |
---|
| 345 | except EggNotFetchable, e: |
---|
| 346 | raise EggNotFetchable( str( [ egg.name for egg in e.eggs ] ) ) |
---|
| 347 | pkg_resources.require = require |
---|
| 348 | |
---|
| 349 | def unpack_zipfile( filename, extract_dir, ignores=[] ): |
---|
| 350 | z = zipfile.ZipFile(filename) |
---|
| 351 | try: |
---|
| 352 | for info in z.infolist(): |
---|
| 353 | name = info.filename |
---|
| 354 | perm = (info.external_attr >> 16L) & 0777 |
---|
| 355 | # don't extract absolute paths or ones with .. in them |
---|
| 356 | if name.startswith('/') or '..' in name: |
---|
| 357 | continue |
---|
| 358 | target = os.path.join(extract_dir, *name.split('/')) |
---|
| 359 | if not target: |
---|
| 360 | continue |
---|
| 361 | for ignore in ignores: |
---|
| 362 | if ignore in name: |
---|
| 363 | continue |
---|
| 364 | if name.endswith('/'): |
---|
| 365 | # directory |
---|
| 366 | pkg_resources.ensure_directory(target) |
---|
| 367 | else: |
---|
| 368 | # file |
---|
| 369 | pkg_resources.ensure_directory(target) |
---|
| 370 | data = z.read(info.filename) |
---|
| 371 | f = open(target,'wb') |
---|
| 372 | try: |
---|
| 373 | f.write(data) |
---|
| 374 | finally: |
---|
| 375 | f.close() |
---|
| 376 | del data |
---|
| 377 | try: |
---|
| 378 | if not os.path.islink(): |
---|
| 379 | os.chmod(target, mode) |
---|
| 380 | except: |
---|
| 381 | pass |
---|
| 382 | finally: |
---|
| 383 | z.close() |
---|
| 384 | |
---|
| 385 | def remove_file_or_path( f ): |
---|
| 386 | if os.path.isdir( f ): |
---|
| 387 | shutil.rmtree( f ) |
---|
| 388 | else: |
---|
| 389 | os.remove( f ) |
---|