[2] | 1 | """ |
---|
| 2 | Manage Galaxy eggs |
---|
| 3 | """ |
---|
| 4 | |
---|
| 5 | import os, sys, shutil, tempfile, subprocess, urlparse, urllib |
---|
| 6 | from __init__ import Egg, Crate, URLRetriever, galaxy_dir, py, unpack_zipfile, EggNotFetchable |
---|
| 7 | from distutils.sysconfig import get_config_var |
---|
| 8 | |
---|
| 9 | import tarfile, zipfile, zlib |
---|
| 10 | arctypes = ( 'tar.gz', 'tgz', 'zip' ) |
---|
| 11 | |
---|
| 12 | import logging |
---|
| 13 | log = logging.getLogger( __name__ ) |
---|
| 14 | log.addHandler( logging.NullHandler() ) |
---|
| 15 | |
---|
| 16 | import pkg_resources |
---|
| 17 | |
---|
| 18 | class ScrambleFailure( Exception ): |
---|
| 19 | def __init__( self, eggs, msg=None ): |
---|
| 20 | if type( eggs ) in ( list, tuple ): |
---|
| 21 | self.eggs = eggs |
---|
| 22 | else: |
---|
| 23 | self.eggs = [ eggs ] |
---|
| 24 | self.msg = msg |
---|
| 25 | def __str__( self ): |
---|
| 26 | return self.msg or ' '.join( self.eggs ) |
---|
| 27 | |
---|
| 28 | class ScrambleEgg( Egg ): |
---|
| 29 | """ |
---|
| 30 | Contains information about scrambling eggs. |
---|
| 31 | """ |
---|
| 32 | scramble_dir = os.path.join( galaxy_dir, 'scripts', 'scramble' ) |
---|
| 33 | archive_dir = os.path.join( scramble_dir, 'archives' ) |
---|
| 34 | script_dir = os.path.join( scramble_dir, 'scripts' ) |
---|
| 35 | build_dir = os.path.join( scramble_dir, 'build' ) |
---|
| 36 | ez_setup = os.path.join( scramble_dir, 'lib', 'ez_setup.py' ) |
---|
| 37 | ez_setup_url = 'http://peak.telecommunity.com/dist/ez_setup.py' |
---|
| 38 | def __init__( self, *args, **kwargs ): |
---|
| 39 | Egg.__init__( self, *args, **kwargs ) |
---|
| 40 | self.sources = [] |
---|
| 41 | self.dependencies = [] |
---|
| 42 | self.buildpath = None |
---|
| 43 | self.source_path = None |
---|
| 44 | self.py = py |
---|
| 45 | self.build_host = None |
---|
| 46 | self.python = sys.executable |
---|
| 47 | def scramble( self ): |
---|
| 48 | if self.path: |
---|
| 49 | log.warning( "%s(): Egg already exists, remove to force rebuild:" % sys._getframe().f_code.co_name ) |
---|
| 50 | log.warning( " %s" % self.path ) |
---|
| 51 | return |
---|
| 52 | self.fetch_source() |
---|
| 53 | self.unpack_source() |
---|
| 54 | self.copy_build_script() |
---|
| 55 | if not os.path.exists( ScrambleEgg.ez_setup ): |
---|
| 56 | URLRetriever().retrieve( ScrambleEgg.ez_setup_url, ScrambleEgg.ez_setup ) |
---|
| 57 | shutil.copyfile( ScrambleEgg.ez_setup, os.path.join( self.buildpath, 'ez_setup.py' ) ) |
---|
| 58 | self.run_scramble_script() |
---|
| 59 | new_egg = os.path.join( self.buildpath, 'dist', os.path.basename( self.distribution.location ) ) |
---|
| 60 | if not os.path.exists( new_egg ): |
---|
| 61 | raise ScrambleFailure( self, "%s(): Egg build for %s did not appear to fail, but no egg found to copy from expected path:\n %s" % ( sys._getframe().f_code.co_name, self.name, new_egg ) ) |
---|
| 62 | shutil.copyfile( new_egg, self.distribution.location ) |
---|
| 63 | log.warning( "%s(): Copied egg to:" % sys._getframe().f_code.co_name ) |
---|
| 64 | log.warning( " %s" % self.distribution.location ) |
---|
| 65 | self.unpack_if_needed() |
---|
| 66 | self.remove_doppelgangers() |
---|
| 67 | # scramble helper methods |
---|
| 68 | def get_tld( self, names ): |
---|
| 69 | tld = names[0].split( os.path.sep, 1 )[0] |
---|
| 70 | for name in names: |
---|
| 71 | try: |
---|
| 72 | assert tld == name.split( os.path.sep, 1 )[0] |
---|
| 73 | except: |
---|
| 74 | raise Exception( "%s(): Archive contains multiple top-level directories!" % sys._getframe().f_code.co_name ) |
---|
| 75 | return tld |
---|
| 76 | def fetch_one( self, urls ): |
---|
| 77 | """ |
---|
| 78 | Fetches the first available archive out of a list. |
---|
| 79 | """ |
---|
| 80 | for url in urls: |
---|
| 81 | file = os.path.join( ScrambleEgg.archive_dir, ( urllib.unquote( url ).rsplit( '/', 1 ) )[-1] ) |
---|
| 82 | if os.path.exists( file ): |
---|
| 83 | log.warning( "%s(): Using existing source, remove to download again:" % sys._getframe().f_code.co_name ) |
---|
| 84 | log.warning( " %s" % file ) |
---|
| 85 | return file |
---|
| 86 | # if we don't have one, get one |
---|
| 87 | for url in urls: |
---|
| 88 | file = os.path.join( ScrambleEgg.archive_dir, ( urllib.unquote( url ).rsplit( '/', 1 ) )[-1] ) |
---|
| 89 | try: |
---|
| 90 | log.debug( "%s(): Trying to fetch:" % sys._getframe().f_code.co_name ) |
---|
| 91 | log.debug( " %s" % url ) |
---|
| 92 | URLRetriever().retrieve( url, file + '.download' ) |
---|
| 93 | shutil.move( file + '.download', file ) |
---|
| 94 | log.debug( "%s(): Fetched to:" % sys._getframe().f_code.co_name ) |
---|
| 95 | log.debug( " %s" % file ) |
---|
| 96 | return file |
---|
| 97 | except IOError, e: |
---|
| 98 | if e[1] != 404: |
---|
| 99 | raise |
---|
| 100 | else: |
---|
| 101 | return None |
---|
| 102 | def fetch_source( self ): |
---|
| 103 | """ |
---|
| 104 | Get egg (and dependent) source |
---|
| 105 | """ |
---|
| 106 | if not os.path.exists( ScrambleEgg.archive_dir ): |
---|
| 107 | os.makedirs( ScrambleEgg.archive_dir ) |
---|
| 108 | urls = [] |
---|
| 109 | url_base = self.url + '/' + '-'.join( ( self.name, self.version ) ) |
---|
| 110 | urls.extend( map( lambda x: '.'.join( ( url_base, x ) ), arctypes ) ) |
---|
| 111 | if self.tag: |
---|
| 112 | urls.extend( map( lambda x: '.'.join( ( url_base + self.tag, x ) ), arctypes ) ) |
---|
| 113 | self.source_path = self.fetch_one( urls ) |
---|
| 114 | if self.source_path is None: |
---|
| 115 | raise Exception( "%s(): Couldn't find a suitable source archive for %s %s from %s" % ( sys._getframe().f_code.co_name, self.name, self.version, self.url ) ) |
---|
| 116 | for url in self.sources: |
---|
| 117 | if not urlparse.urlparse( url )[0]: |
---|
| 118 | url = self.url + '/' + url.lstrip( '/' ) |
---|
| 119 | urls = [ url ] |
---|
| 120 | urls.extend( map( lambda x: '.'.join( ( url, x ) ), arctypes ) ) # allows leaving off the extension and we'll try to find one |
---|
| 121 | file = self.fetch_one( urls ) |
---|
| 122 | if file is None: |
---|
| 123 | raise Exception( "%s(): Couldn't fetch extra source for %s, check path in %s. URL(s) attempted output above." % ( sys._getframe().f_code.co_name, self.name, Crate.config_file, ) ) |
---|
| 124 | def unpack_source( self ): |
---|
| 125 | unpack_dir = os.path.join( ScrambleEgg.build_dir, self.platform ) |
---|
| 126 | if not os.path.exists( unpack_dir ): |
---|
| 127 | os.makedirs( unpack_dir ) |
---|
| 128 | self.buildpath = os.path.join( unpack_dir, self.name ) |
---|
| 129 | if os.path.exists( self.buildpath ): |
---|
| 130 | log.warning( "%s(): Removing old build directory at:" % sys._getframe().f_code.co_name ) |
---|
| 131 | log.warning( " %s" % self.buildpath ) |
---|
| 132 | shutil.rmtree( self.buildpath ) |
---|
| 133 | if tarfile.is_tarfile( self.source_path ): |
---|
| 134 | self.unpack_tar() |
---|
| 135 | elif zipfile.is_zipfile( self.source_path ): |
---|
| 136 | self.unpack_zip() |
---|
| 137 | else: |
---|
| 138 | raise Exception( "%s(): Unknown archive file type for %s" % ( sys._getframe().f_code.co_name, source_path ) ) |
---|
| 139 | log.warning( "%s(): Unpacked to:" % sys._getframe().f_code.co_name ) |
---|
| 140 | log.warning( " %s" % self.buildpath ) |
---|
| 141 | def unpack_zip( self ): |
---|
| 142 | unpack_path = os.path.dirname( self.buildpath ) |
---|
| 143 | tld = self.get_tld( zipfile.ZipFile( self.source_path, 'r' ).namelist() ) |
---|
| 144 | unpack_zipfile( self.source_path, unpack_path, ( 'ez_setup', ) ) |
---|
| 145 | os.rename( os.path.join( unpack_path, tld ), self.buildpath ) |
---|
| 146 | def unpack_tar( self ): |
---|
| 147 | unpack_path = os.path.dirname( self.buildpath ) |
---|
| 148 | t = tarfile.open( self.source_path, "r" ) |
---|
| 149 | members = filter( lambda x: "ez_setup" not in x.name and "pax_global_header" != x.name, t.getmembers() ) |
---|
| 150 | tld = self.get_tld( [ x.name for x in members ] ) |
---|
| 151 | cur = os.getcwd() |
---|
| 152 | os.chdir( unpack_path ) |
---|
| 153 | for member in members: |
---|
| 154 | t.extract( member ) |
---|
| 155 | t.close() |
---|
| 156 | os.rename( tld, self.name ) |
---|
| 157 | os.chdir( cur ) |
---|
| 158 | def copy_build_script( self ): |
---|
| 159 | # will try: |
---|
| 160 | # bx_python-py2.4-solaris-2.11-i86pc.py |
---|
| 161 | # bx_python-py2.4-solaris.py |
---|
| 162 | # bx_python-solaris-2.11-i86pc.py |
---|
| 163 | # bx_python-solaris.py |
---|
| 164 | # bx_python-py2.4.py |
---|
| 165 | # bx_python.py |
---|
| 166 | # generic.py |
---|
| 167 | platform = self.platform.replace( '-ucs2', '' ).replace( '-ucs4', '' ) # ucs is unimportant here |
---|
| 168 | build_scripts = ( |
---|
| 169 | "%s-%s.py" % ( self.name, platform ), |
---|
| 170 | "%s-%s.py" % ( self.name, '-'.join( platform.split( '-' )[:2] ) ), |
---|
| 171 | "%s-%s.py" % ( self.name, '-'.join( platform.split( '-' )[1:] ) ), |
---|
| 172 | "%s-%s.py" % ( self.name, platform.split( '-' )[:2][-1] ), |
---|
| 173 | "%s-%s.py" % ( self.name, platform.split( '-' )[0] ), |
---|
| 174 | "%s.py" % self.name, |
---|
| 175 | "generic.py" ) |
---|
| 176 | for build_script in build_scripts: |
---|
| 177 | build_script = os.path.join( ScrambleEgg.script_dir, build_script ) |
---|
| 178 | if os.path.exists( build_script ): |
---|
| 179 | log.warning( "%s(): Using build script %s" % ( sys._getframe().f_code.co_name, build_script ) ) |
---|
| 180 | break |
---|
| 181 | shutil.copyfile( build_script, os.path.join( self.buildpath, "scramble.py" ) ) |
---|
| 182 | verfile = open( os.path.join( self.buildpath, ".galaxy_ver" ), "w" ) |
---|
| 183 | verfile.write( self.version + '\n' ) |
---|
| 184 | verfile.close() |
---|
| 185 | if self.tag is not None: |
---|
| 186 | tagfile = open( os.path.join( self.buildpath, ".galaxy_tag" ), "w" ) |
---|
| 187 | tagfile.write( self.tag + '\n' ) |
---|
| 188 | tagfile.close() |
---|
| 189 | if self.dependencies: |
---|
| 190 | depfile = open( os.path.join( self.buildpath, ".galaxy_deps" ), "w" ) |
---|
| 191 | for dependency in self.dependencies: |
---|
| 192 | depfile.write( dependency + '\n' ) |
---|
| 193 | depfile.close() |
---|
| 194 | def run_scramble_script( self ): |
---|
| 195 | log.warning( "%s(): Beginning build" % sys._getframe().f_code.co_name ) |
---|
| 196 | # subprocessed to sterilize the env |
---|
| 197 | cmd = "%s -ES %s" % ( self.python, "scramble.py" ) |
---|
| 198 | log.debug( '%s(): Executing in %s:' % ( sys._getframe().f_code.co_name, self.buildpath ) ) |
---|
| 199 | log.debug( ' %s' % cmd ) |
---|
| 200 | p = subprocess.Popen( args = cmd, shell = True, cwd = self.buildpath ) |
---|
| 201 | r = p.wait() |
---|
| 202 | if r != 0: |
---|
| 203 | if sys.platform == 'sunos5' and get_config_var('CC').endswith('pycc') and not os.environ.get( 'PYCC_CC', None ): |
---|
| 204 | log.error( "%s(): Your python interpreter was compiled with Sun's pycc" % sys._getframe().f_code.co_name ) |
---|
| 205 | log.error( " pseudo-compiler. You may need to set PYCC_CC and PYCC_CXX in your" ) |
---|
| 206 | log.error( " environment if your compiler is in a non-standard location." ) |
---|
| 207 | raise ScrambleFailure( self, "%s(): Egg build failed for %s %s" % ( sys._getframe().f_code.co_name, self.name, self.version ) ) |
---|
| 208 | |
---|
| 209 | class ScrambleCrate( Crate ): |
---|
| 210 | """ |
---|
| 211 | Reads the eggs.ini file for use with scrambling eggs. |
---|
| 212 | """ |
---|
| 213 | def parse( self ): |
---|
| 214 | Crate.parse( self ) |
---|
| 215 | # get dependency sources |
---|
| 216 | for egg in self.eggs.values(): |
---|
| 217 | try: |
---|
| 218 | egg.sources = self.config.get( "source", egg.name ).split() |
---|
| 219 | except: |
---|
| 220 | egg.sources = [] |
---|
| 221 | try: |
---|
| 222 | egg.dependencies = self.config.get( "dependencies", egg.name ).split() |
---|
| 223 | except: |
---|
| 224 | egg.dependencies = [] |
---|
| 225 | def parse_egg_section( self, *args, **kwargs ): |
---|
| 226 | kwargs['egg_class'] = ScrambleEgg |
---|
| 227 | Crate.parse_egg_section( self, *args, **kwargs ) |
---|
| 228 | def scramble( self, all=False ): |
---|
| 229 | if all: |
---|
| 230 | eggs = self.all_eggs |
---|
| 231 | else: |
---|
| 232 | eggs = self.config_eggs |
---|
| 233 | eggs = filter( lambda x: x.name not in self.no_auto, eggs ) |
---|
| 234 | failed = [] |
---|
| 235 | for egg in eggs: |
---|
| 236 | try: |
---|
| 237 | egg.scramble() |
---|
| 238 | except Exception, e: |
---|
| 239 | failed.append( egg ) |
---|
| 240 | last_exc = e |
---|
| 241 | if failed: |
---|
| 242 | if len( failed ) == 1: |
---|
| 243 | raise last_exc # only 1 failure out of the crate, be more informative |
---|
| 244 | else: |
---|
| 245 | raise ScrambleFailure( failed ) |
---|