root/galaxy-central/lib/galaxy/eggs/scramble.py

リビジョン 2, 11.2 KB (コミッタ: hatakeyama, 14 年 前)

import galaxy-central

行番号 
1"""
2Manage Galaxy eggs
3"""
4
5import os, sys, shutil, tempfile, subprocess, urlparse, urllib
6from __init__ import Egg, Crate, URLRetriever, galaxy_dir, py, unpack_zipfile, EggNotFetchable
7from distutils.sysconfig import get_config_var
8
9import tarfile, zipfile, zlib
10arctypes = ( 'tar.gz', 'tgz', 'zip' )
11
12import logging
13log = logging.getLogger( __name__ )
14log.addHandler( logging.NullHandler() )
15
16import pkg_resources
17
18class ScrambleFailure( Exception ):
19    def __init__( self, eggs, msg=None ):
20        if type( eggs ) in ( list, tuple ):
21            self.eggs = eggs
22        else:
23            self.eggs = [ eggs ]
24        self.msg = msg
25    def __str__( self ):
26        return self.msg or ' '.join( self.eggs )
27
28class ScrambleEgg( Egg ):
29    """
30    Contains information about scrambling eggs.
31    """
32    scramble_dir = os.path.join( galaxy_dir, 'scripts', 'scramble' )
33    archive_dir = os.path.join( scramble_dir, 'archives' )
34    script_dir = os.path.join( scramble_dir, 'scripts' )
35    build_dir = os.path.join( scramble_dir, 'build' )
36    ez_setup = os.path.join( scramble_dir, 'lib', 'ez_setup.py' )
37    ez_setup_url = 'http://peak.telecommunity.com/dist/ez_setup.py'
38    def __init__( self, *args, **kwargs ):
39        Egg.__init__( self, *args, **kwargs )
40        self.sources = []
41        self.dependencies = []
42        self.buildpath = None
43        self.source_path = None
44        self.py = py
45        self.build_host = None
46        self.python = sys.executable
47    def scramble( self ):
48        if self.path:
49            log.warning( "%s(): Egg already exists, remove to force rebuild:" % sys._getframe().f_code.co_name )
50            log.warning( "  %s" % self.path )
51            return
52        self.fetch_source()
53        self.unpack_source()
54        self.copy_build_script()
55        if not os.path.exists( ScrambleEgg.ez_setup ):
56            URLRetriever().retrieve( ScrambleEgg.ez_setup_url, ScrambleEgg.ez_setup )
57        shutil.copyfile( ScrambleEgg.ez_setup, os.path.join( self.buildpath, 'ez_setup.py' ) )
58        self.run_scramble_script()
59        new_egg = os.path.join( self.buildpath, 'dist', os.path.basename( self.distribution.location ) )
60        if not os.path.exists( new_egg ):
61            raise ScrambleFailure( self, "%s(): Egg build for %s did not appear to fail, but no egg found to copy from expected path:\n  %s" % ( sys._getframe().f_code.co_name, self.name, new_egg ) )
62        shutil.copyfile( new_egg, self.distribution.location )
63        log.warning( "%s(): Copied egg to:" % sys._getframe().f_code.co_name )
64        log.warning( "  %s" % self.distribution.location )
65        self.unpack_if_needed()
66        self.remove_doppelgangers()
67    # scramble helper methods
68    def get_tld( self, names ):
69        tld = names[0].split( os.path.sep, 1 )[0]
70        for name in names:
71            try:
72                assert tld == name.split( os.path.sep, 1 )[0]
73            except:
74                raise Exception( "%s(): Archive contains multiple top-level directories!" % sys._getframe().f_code.co_name )
75        return tld
76    def fetch_one( self, urls ):
77        """
78        Fetches the first available archive out of a list.
79        """
80        for url in urls:
81            file = os.path.join( ScrambleEgg.archive_dir, ( urllib.unquote( url ).rsplit( '/', 1 ) )[-1] )
82            if os.path.exists( file ):
83                log.warning( "%s(): Using existing source, remove to download again:" % sys._getframe().f_code.co_name )
84                log.warning( "  %s" % file )
85                return file
86        # if we don't have one, get one
87        for url in urls:
88            file = os.path.join( ScrambleEgg.archive_dir, ( urllib.unquote( url ).rsplit( '/', 1 ) )[-1] )
89            try:
90                log.debug( "%s(): Trying to fetch:" % sys._getframe().f_code.co_name )
91                log.debug( "  %s" % url )
92                URLRetriever().retrieve( url, file + '.download' )
93                shutil.move( file + '.download', file )
94                log.debug( "%s(): Fetched to:" % sys._getframe().f_code.co_name )
95                log.debug( "  %s" % file )
96                return file
97            except IOError, e:
98                if e[1] != 404:
99                    raise
100        else:
101            return None
102    def fetch_source( self ):
103        """
104        Get egg (and dependent) source
105        """
106        if not os.path.exists( ScrambleEgg.archive_dir ):
107            os.makedirs( ScrambleEgg.archive_dir )
108        urls = []
109        url_base = self.url + '/' + '-'.join( ( self.name, self.version ) )
110        urls.extend( map( lambda x: '.'.join( ( url_base, x ) ), arctypes ) )
111        if self.tag:
112            urls.extend( map( lambda x: '.'.join( ( url_base + self.tag, x ) ), arctypes ) )
113        self.source_path = self.fetch_one( urls )
114        if self.source_path is None:
115            raise Exception( "%s(): Couldn't find a suitable source archive for %s %s from %s" % ( sys._getframe().f_code.co_name, self.name, self.version, self.url ) )
116        for url in self.sources:
117            if not urlparse.urlparse( url )[0]:
118                url = self.url + '/' + url.lstrip( '/' )
119            urls = [ url ]
120            urls.extend( map( lambda x: '.'.join( ( url, x ) ), arctypes ) ) # allows leaving off the extension and we'll try to find one
121            file = self.fetch_one( urls )
122            if file is None:
123                raise Exception( "%s(): Couldn't fetch extra source for %s, check path in %s.  URL(s) attempted output above." % ( sys._getframe().f_code.co_name, self.name, Crate.config_file, ) )
124    def unpack_source( self ):
125        unpack_dir = os.path.join( ScrambleEgg.build_dir, self.platform )
126        if not os.path.exists( unpack_dir ):
127            os.makedirs( unpack_dir )
128        self.buildpath = os.path.join( unpack_dir, self.name )
129        if os.path.exists( self.buildpath ):
130            log.warning( "%s(): Removing old build directory at:" % sys._getframe().f_code.co_name )
131            log.warning( "  %s" % self.buildpath )
132            shutil.rmtree( self.buildpath )
133        if tarfile.is_tarfile( self.source_path ):
134            self.unpack_tar()
135        elif zipfile.is_zipfile( self.source_path ):
136            self.unpack_zip()
137        else:
138            raise Exception( "%s(): Unknown archive file type for %s" % ( sys._getframe().f_code.co_name, source_path ) )
139        log.warning( "%s(): Unpacked to:" % sys._getframe().f_code.co_name )
140        log.warning( "  %s" % self.buildpath )
141    def unpack_zip( self ):
142        unpack_path = os.path.dirname( self.buildpath )
143        tld = self.get_tld( zipfile.ZipFile( self.source_path, 'r' ).namelist() )
144        unpack_zipfile( self.source_path, unpack_path, ( 'ez_setup', ) )
145        os.rename( os.path.join( unpack_path, tld ), self.buildpath )
146    def unpack_tar( self ):
147        unpack_path = os.path.dirname( self.buildpath )
148        t = tarfile.open( self.source_path, "r" )
149        members = filter( lambda x: "ez_setup" not in x.name and "pax_global_header" != x.name, t.getmembers() )
150        tld = self.get_tld( [ x.name for x in members ] )
151        cur = os.getcwd()
152        os.chdir( unpack_path )
153        for member in members:
154            t.extract( member )
155        t.close()
156        os.rename( tld, self.name )
157        os.chdir( cur )
158    def copy_build_script( self ):
159        # will try:
160        #   bx_python-py2.4-solaris-2.11-i86pc.py
161        #   bx_python-py2.4-solaris.py
162        #   bx_python-solaris-2.11-i86pc.py
163        #   bx_python-solaris.py
164        #   bx_python-py2.4.py
165        #   bx_python.py
166        #   generic.py
167        platform = self.platform.replace( '-ucs2', '' ).replace( '-ucs4', '' ) # ucs is unimportant here
168        build_scripts = (
169            "%s-%s.py" % ( self.name, platform ),
170            "%s-%s.py" % ( self.name, '-'.join( platform.split( '-' )[:2] ) ),
171            "%s-%s.py" % ( self.name, '-'.join( platform.split( '-' )[1:] ) ),
172            "%s-%s.py" % ( self.name, platform.split( '-' )[:2][-1] ),
173            "%s-%s.py" % ( self.name, platform.split( '-' )[0] ),
174            "%s.py" % self.name,
175            "generic.py" )
176        for build_script in build_scripts:
177            build_script = os.path.join( ScrambleEgg.script_dir, build_script )
178            if os.path.exists( build_script ):
179                log.warning( "%s(): Using build script %s" % ( sys._getframe().f_code.co_name, build_script ) )
180                break
181        shutil.copyfile( build_script, os.path.join( self.buildpath, "scramble.py" ) )
182        verfile = open( os.path.join( self.buildpath, ".galaxy_ver" ), "w" )
183        verfile.write( self.version + '\n' )
184        verfile.close()
185        if self.tag is not None:
186            tagfile = open( os.path.join( self.buildpath, ".galaxy_tag" ), "w" )
187            tagfile.write( self.tag + '\n' )
188            tagfile.close()
189        if self.dependencies:
190            depfile = open( os.path.join( self.buildpath, ".galaxy_deps" ), "w" )
191            for dependency in self.dependencies:
192                depfile.write( dependency + '\n' )
193            depfile.close()
194    def run_scramble_script( self ):
195        log.warning( "%s(): Beginning build" % sys._getframe().f_code.co_name )
196        # subprocessed to sterilize the env
197        cmd = "%s -ES %s" % ( self.python, "scramble.py" )
198        log.debug( '%s(): Executing in %s:' % ( sys._getframe().f_code.co_name, self.buildpath ) )
199        log.debug( '  %s' % cmd )
200        p = subprocess.Popen( args = cmd, shell = True, cwd = self.buildpath )
201        r = p.wait()
202        if r != 0:
203            if sys.platform == 'sunos5' and get_config_var('CC').endswith('pycc') and not os.environ.get( 'PYCC_CC', None ):
204                log.error( "%s(): Your python interpreter was compiled with Sun's pycc" % sys._getframe().f_code.co_name )
205                log.error( "  pseudo-compiler.  You may need to set PYCC_CC and PYCC_CXX in your" )
206                log.error( "  environment if your compiler is in a non-standard location." )
207            raise ScrambleFailure( self, "%s(): Egg build failed for %s %s" % ( sys._getframe().f_code.co_name, self.name, self.version ) )
208
209class ScrambleCrate( Crate ):
210    """
211    Reads the eggs.ini file for use with scrambling eggs.
212    """
213    def parse( self ):
214        Crate.parse( self )
215        # get dependency sources
216        for egg in self.eggs.values():
217            try:
218                egg.sources = self.config.get( "source", egg.name ).split()
219            except:
220                egg.sources = []
221            try:
222                egg.dependencies = self.config.get( "dependencies", egg.name ).split()
223            except:
224                egg.dependencies = []
225    def parse_egg_section( self, *args, **kwargs ):
226        kwargs['egg_class'] = ScrambleEgg
227        Crate.parse_egg_section( self, *args, **kwargs )
228    def scramble( self, all=False ):
229        if all:
230            eggs = self.all_eggs
231        else:
232            eggs = self.config_eggs
233        eggs = filter( lambda x: x.name not in self.no_auto, eggs )
234        failed = []
235        for egg in eggs:
236            try:
237                egg.scramble()
238            except Exception, e:
239                failed.append( egg )
240                last_exc = e
241        if failed:
242            if len( failed ) == 1:
243                raise last_exc # only 1 failure out of the crate, be more informative
244            else:
245                raise ScrambleFailure( failed )
Note: リポジトリブラウザについてのヘルプは TracBrowser を参照してください。