root/galaxy-central/lib/galaxy/eggs/__init__.py @ 2

リビジョン 2, 15.6 KB (コミッタ: hatakeyama, 14 年 前)

import galaxy-central
行番号 
1"""
2Manage Galaxy eggs
3"""
4
5import os, sys, shutil, glob, urllib, urllib2, ConfigParser, HTMLParser, zipimport, zipfile
6
7import logging
8log = logging.getLogger( __name__ )
9log.addHandler( logging.NullHandler() )
10
11import pkg_resources
12
13galaxy_dir = os.path.abspath( os.path.join( os.path.dirname( __file__ ), '..', '..', '..' ) )
14py = 'py%s' % sys.version[:3]
15
16class EggNotFetchable( Exception ):
17    def __init__( self, eggs ):
18        if type( eggs ) in ( list, tuple ):
19            self.eggs = eggs
20        else:
21            self.eggs = [ eggs ]
22    def __str__( self ):
23        return ' '.join( self.eggs )
24
25# need the options to remain case sensitive
26class CaseSensitiveConfigParser( ConfigParser.SafeConfigParser ):
27    def optionxform( self, optionstr ):
28        return optionstr
29
30# so we can actually detect failures
31class URLRetriever( urllib.FancyURLopener ):
32    def http_error_default( *args ):
33        urllib.URLopener.http_error_default( *args )
34
35class Egg( object ):
36    """
37    Contains information about locating and downloading eggs.
38    """
39    def __init__( self, name=None, version=None, tag=None, url=None, platform=None ):
40        self.name = name
41        self.version = version
42        self.tag = tag
43        self.url = url
44        self.platform = platform
45        self.distribution = None
46        self.dir = None
47        if self.name is not None and self.version is not None:
48            self.set_distribution()
49    def set_dir( self ):
50        self.dir = os.path.join( galaxy_dir, 'eggs' )
51        if not os.path.exists( self.dir ):
52            os.makedirs( self.dir )
53    def set_distribution( self ):
54        """
55        Stores a pkg_resources Distribution object for reference later
56        """
57        if self.dir is None:
58            self.set_dir()
59        tag = self.tag or ''
60        self.distribution = pkg_resources.Distribution.from_filename(
61                os.path.join( self.dir, '-'.join( ( self.name, self.version + tag, self.platform ) ) + '.egg' ) )
62    @property
63    def path( self ):
64        """
65        Return the path of the egg, if it exists, or None
66        """
67        if env[self.distribution.project_name]:
68            return env[self.distribution.project_name][0].location
69        return None
70    def fetch( self, requirement ):
71        """
72        fetch() serves as the install method to pkg_resources.working_set.resolve()
73        """
74        def find_alternative():
75            """
76            Some platforms (e.g. Solaris) support eggs compiled on older platforms
77            """
78            class LinkParser( HTMLParser.HTMLParser ):
79                """
80                Finds links in what should be an Apache-style directory index
81                """
82                def __init__( self ):
83                    HTMLParser.HTMLParser.__init__( self )
84                    self.links = []
85                def handle_starttag( self, tag, attrs ):
86                    if tag == 'a' and 'href' in dict( attrs ):
87                        self.links.append( dict( attrs )['href'] )
88            parser = LinkParser()
89            try:
90                parser.feed( urllib2.urlopen( self.url + '/' ).read() )
91            except urllib2.HTTPError, e:
92                if e.code == 404:
93                    return None
94            parser.close()
95            for link in parser.links:
96                file = urllib.unquote( link ).rsplit( '/', 1 )[-1]
97                tmp_dist = pkg_resources.Distribution.from_filename( file )
98                if tmp_dist.platform is not None and \
99                        self.distribution.project_name == tmp_dist.project_name and \
100                        self.distribution.version == tmp_dist.version and \
101                        self.distribution.py_version == tmp_dist.py_version and \
102                        pkg_resources.compatible_platforms( tmp_dist.platform, pkg_resources.get_platform() ):
103                    return file
104            return None
105        if self.url is None:
106            return None
107        alternative = None
108        try:
109            url = self.url + '/' + self.distribution.egg_name() + '.egg'
110            URLRetriever().retrieve( url, self.distribution.location )
111            log.debug( "Fetched %s" % url )
112        except IOError, e:
113            if e[1] == 404 and self.distribution.platform != py:
114                alternative = find_alternative()
115                if alternative is None:
116                    return None
117            else:
118                return None
119        if alternative is not None:
120            try:
121                url = '/'.join( ( self.url, alternative ) )
122                URLRetriever().retrieve( url, os.path.join( self.dir, alternative ) )
123                log.debug( "Fetched %s" % url )
124            except IOError, e:
125                return None
126            self.platform = alternative.split( '-', 2 )[-1].rsplit( '.egg', 1 )[0]
127            self.set_distribution()
128        self.unpack_if_needed()
129        self.remove_doppelgangers()
130        global env
131        env = get_env() # reset the global Environment object now that we've obtained a new egg
132        return self.distribution
133    def unpack_if_needed( self ):
134        meta = pkg_resources.EggMetadata( zipimport.zipimporter( self.distribution.location ) )   
135        if meta.has_metadata( 'not-zip-safe' ):
136            unpack_zipfile( self.distribution.location, self.distribution.location + "-tmp" )
137            os.remove( self.distribution.location )
138            os.rename( self.distribution.location + "-tmp", self.distribution.location )
139    def remove_doppelgangers( self ):
140        doppelgangers = glob.glob( os.path.join( self.dir, "%s-*-%s.egg" % ( self.name, self.platform ) ) )
141        if self.distribution.location in doppelgangers:
142            doppelgangers.remove( self.distribution.location )
143        for doppelganger in doppelgangers:
144            remove_file_or_path( doppelganger )
145            log.debug( "Removed conflicting egg: %s" % doppelganger )
146    def resolve( self ):
147        try:
148            return pkg_resources.working_set.resolve( ( self.distribution.as_requirement(), ), env, self.fetch )
149        except pkg_resources.DistributionNotFound, e:
150            # If this statement is true, it means we do have the requested egg,
151            # just not one (or more) of its deps.
152            if e.args[0].project_name != self.distribution.project_name:
153                log.warning( "Warning: %s (a dependant egg of %s) cannot be fetched" % ( e.args[0].project_name, self.distribution.project_name ) )
154                return ( self.distribution, )
155            else:
156                raise EggNotFetchable( self )
157        except pkg_resources.VersionConflict, e:
158            # there's a conflicting egg on the path, remove it
159            dist = e.args[0]
160            # use the canonical path for comparisons
161            location = os.path.realpath( dist.location )
162            for entry in pkg_resources.working_set.entries:
163                if os.path.realpath( entry ) == location:
164                    pkg_resources.working_set.entries.remove( entry )
165                    break
166            else:
167                location = entry = None
168            del pkg_resources.working_set.by_key[dist.key]
169            if entry is not None:
170                pkg_resources.working_set.entry_keys[entry] = []
171                if entry in sys.path:
172                    sys.path.remove(entry)
173            r = pkg_resources.working_set.resolve( ( self.distribution.as_requirement(), ), env, self.fetch )
174            if location is not None and not location.endswith( '.egg' ):
175                # re-add the path if it's a non-egg dir, in case more deps live there
176                pkg_resources.working_set.entries.append( location )
177                sys.path.append( location )
178            return r
179    def require( self ):
180        try:
181            dists = self.resolve()
182            for dist in dists:
183                pkg_resources.working_set.add( dist )
184            return dists
185        except:
186            raise
187
188class Crate( object ):
189    """
190    Reads the eggs.ini file for use with checking and fetching.
191    """
192    config_file = os.path.join( galaxy_dir, 'eggs.ini' )
193    def __init__( self, platform=None ):
194        self.eggs = {}
195        self.config = CaseSensitiveConfigParser()
196        self.repo = None
197        self.no_auto = []
198        self.platform = platform
199        self.py_platform = None
200        if platform is not None:
201            self.py_platform = platform.split( '-' )[0]
202        self.galaxy_config = GalaxyConfig()
203        self.parse()
204    def parse( self ):
205        self.config.read( Crate.config_file )
206        self.repo = self.config.get( 'general', 'repository' )
207        self.no_auto = self.config.get( 'general', 'no_auto' ).split()
208        self.parse_egg_section( self.config.items( 'eggs:platform' ), self.config.items( 'tags' ), True )
209        self.parse_egg_section( self.config.items( 'eggs:noplatform' ), self.config.items( 'tags' ) )
210    def parse_egg_section( self, eggs, tags, full_platform=False, egg_class=Egg ):
211        for name, version in eggs:
212            tag = dict( tags ).get( name, '' )
213            url = '/'.join( ( self.repo, name ) )
214            if full_platform:
215                platform = self.platform or '-'.join( ( py, pkg_resources.get_platform() ) )
216            else:
217                platform = self.py_platform or py
218            egg = egg_class( name, version, tag, url, platform )
219            self.eggs[name] = egg
220    @property
221    def config_missing( self ):
222        """
223        Return true if any eggs are missing, conditional on options set in the
224        Galaxy config file.
225        """
226        for egg in self.config_eggs:
227            if not egg.path:
228                return True
229        return False
230    @property
231    def all_missing( self ):
232        """
233        Return true if any eggs in the eggs config file are missing.
234        """
235        for egg in self.all_eggs:
236            if not os.path.exists( egg.distribution.location ):
237                return True
238        return False
239    @property
240    def config_names( self ):
241        """
242        Return a list of names of all eggs in the crate that are needed based
243        on the options set in the Galaxy config file.
244        """
245        return [ egg.name for egg in self.config_eggs ]
246    @property
247    def all_names( self ):
248        """
249        Return a list of names of all eggs in the crate.
250        """
251        return [ egg.name for egg in self.all_eggs ]
252    @property
253    def config_eggs( self ):
254        """
255        Return a list of all eggs in the crate that are needed based on the
256        options set in the Galaxy config file.
257        """
258        return [ egg for egg in self.eggs.values() if self.galaxy_config.check_conditional( egg.name ) ]
259    @property
260    def all_eggs( self ):
261        """
262        Return a list of all eggs in the crate.
263        """
264        rval = []
265        for egg in self.eggs.values():
266            if egg.name not in self.galaxy_config.always_conditional:
267                rval.append( egg )
268        return rval
269    def __getitem__( self, name ):
270        """
271        Return a specific egg.
272        """
273        name = name.replace( '-', '_' )
274        return self.eggs[name]
275    def resolve( self, all=False ):
276        """
277        Try to resolve (e.g. fetch) all eggs in the crate.
278        """
279        if all:
280            eggs = self.all_eggs
281        else:
282            eggs = self.config_eggs
283        eggs = filter( lambda x: x.name not in self.no_auto, eggs )
284        missing = []
285        for egg in eggs:
286            try:
287                egg.resolve()
288            except EggNotFetchable:
289                missing.append( egg )
290        if missing:
291            raise EggNotFetchable( missing )
292
293class GalaxyConfig( object ):
294    config_file = os.path.join( galaxy_dir, "universe_wsgi.ini" )
295    always_conditional = ( 'GeneTrack', 'pysam' )
296    def __init__( self ):
297        self.config = ConfigParser.ConfigParser()
298        if self.config.read( GalaxyConfig.config_file ) == []:
299            raise Exception( "error: unable to read Galaxy config from %s" % GalaxyConfig.config_file )
300    def check_conditional( self, egg_name ):
301        def check_pysam():
302            # can't build pysam on solaris < 10
303            plat = pkg_resources.get_platform().split( '-' )
304            if plat[0] == 'solaris':
305                minor = plat[1].split('.')[1]
306                if int( minor ) < 10:
307                    return False
308            return True
309        if egg_name == "pysqlite":
310            # SQLite is different since it can be specified in two config vars and defaults to True
311            try:
312                return self.config.get( "app:main", "database_connection" ).startswith( "sqlite://" )
313            except:
314                return True
315        else:
316            try:
317                return { "psycopg2":        lambda: self.config.get( "app:main", "database_connection" ).startswith( "postgres://" ),
318                         "MySQL_python":    lambda: self.config.get( "app:main", "database_connection" ).startswith( "mysql://" ),
319                         "DRMAA_python":    lambda: "sge" in self.config.get( "app:main", "start_job_runners" ).split(","),
320                         "drmaa":           lambda: ( "drmaa" in self.config.get( "app:main", "start_job_runners" ).split(",") ) and sys.version_info[:2] >= ( 2, 5 ),
321                         "pbs_python":      lambda: "pbs" in self.config.get( "app:main", "start_job_runners" ).split(","),
322                         "threadframe":     lambda: self.config.get( "app:main", "use_heartbeat" ),
323                         "guppy":           lambda: self.config.get( "app:main", "use_memdump" ),
324                         "GeneTrack":       lambda: sys.version_info[:2] >= ( 2, 5 ),
325                         "pysam":           check_pysam()
326                       }.get( egg_name, lambda: True )()
327            except:
328                return False
329
330def get_env():
331    env = pkg_resources.Environment( platform=pkg_resources.get_platform() )
332    for dist in pkg_resources.find_distributions( os.path.join( galaxy_dir, 'eggs' ), False ):
333        env.add( dist )
334    return env
335env = get_env()
336
337def require( req_str ):
338    c = Crate()
339    req = pkg_resources.Requirement.parse( req_str )
340    try:
341        return c[req.project_name].require()
342    except KeyError:
343        # not a galaxy-owned dependency
344        return pkg_resources.working_set.require( req_str )
345    except EggNotFetchable, e:
346        raise EggNotFetchable( str( [ egg.name for egg in e.eggs ] ) )
347pkg_resources.require = require
348
349def unpack_zipfile( filename, extract_dir, ignores=[] ):
350    z = zipfile.ZipFile(filename)
351    try:
352        for info in z.infolist():
353            name = info.filename
354            perm = (info.external_attr >> 16L) & 0777
355            # don't extract absolute paths or ones with .. in them
356            if name.startswith('/') or '..' in name:
357                continue
358            target = os.path.join(extract_dir, *name.split('/'))
359            if not target:
360                continue
361            for ignore in ignores:
362                if ignore in name:
363                    continue
364            if name.endswith('/'):
365                # directory
366                pkg_resources.ensure_directory(target)
367            else:
368                # file
369                pkg_resources.ensure_directory(target)
370                data = z.read(info.filename)
371                f = open(target,'wb')
372                try:
373                    f.write(data)
374                finally:
375                    f.close()
376                    del data
377                try:
378                    if not os.path.islink():
379                        os.chmod(target, mode)
380                except:
381                    pass
382    finally:
383        z.close()
384
385def remove_file_or_path( f ):
386    if os.path.isdir( f ):
387        shutil.rmtree( f )
388    else:
389        os.remove( f )
Note: リポジトリブラウザについてのヘルプは TracBrowser を参照してください。