[2] | 1 | import sys, os.path, logging |
---|
| 2 | |
---|
| 3 | from galaxy import eggs |
---|
| 4 | |
---|
| 5 | import pkg_resources |
---|
| 6 | pkg_resources.require( "sqlalchemy-migrate" ) |
---|
| 7 | |
---|
| 8 | from migrate.versioning import repository, schema |
---|
| 9 | from sqlalchemy import * |
---|
| 10 | from sqlalchemy.exc import NoSuchTableError |
---|
| 11 | |
---|
| 12 | log = logging.getLogger( __name__ ) |
---|
| 13 | |
---|
| 14 | # path relative to galaxy |
---|
| 15 | migrate_repository_directory = os.path.dirname( __file__ ).replace( os.getcwd() + os.path.sep, '', 1 ) |
---|
| 16 | migrate_repository = repository.Repository( migrate_repository_directory ) |
---|
| 17 | dialect_to_egg = { |
---|
| 18 | "sqlite" : "pysqlite>=2", |
---|
| 19 | "postgres" : "psycopg2", |
---|
| 20 | "mysql" : "MySQL_python" |
---|
| 21 | } |
---|
| 22 | |
---|
| 23 | def create_or_verify_database( url, engine_options={} ): |
---|
| 24 | """ |
---|
| 25 | Check that the database is use-able, possibly creating it if empty (this is |
---|
| 26 | the only time we automatically create tables, otherwise we force the |
---|
| 27 | user to do it using the management script so they can create backups). |
---|
| 28 | |
---|
| 29 | 1) Empty database --> initialize with latest version and return |
---|
| 30 | 2) Database older than migration support --> fail and require manual update |
---|
| 31 | 3) Database at state where migrate support introduced --> add version control information but make no changes (might still require manual update) |
---|
| 32 | 4) Database versioned but out of date --> fail with informative message, user must run "sh manage_db.sh upgrade" |
---|
| 33 | |
---|
| 34 | """ |
---|
| 35 | |
---|
| 36 | dialect = ( url.split( ':', 1 ) )[0] |
---|
| 37 | try: |
---|
| 38 | egg = dialect_to_egg[dialect] |
---|
| 39 | try: |
---|
| 40 | pkg_resources.require( egg ) |
---|
| 41 | log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) ) |
---|
| 42 | except: |
---|
| 43 | # If the module is in the path elsewhere (i.e. non-egg), it'll still load. |
---|
| 44 | log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) ) |
---|
| 45 | except KeyError: |
---|
| 46 | # Let this go, it could possibly work with db's we don't support |
---|
| 47 | log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) |
---|
| 48 | |
---|
| 49 | # Create engine and metadata |
---|
| 50 | engine = create_engine( url, **engine_options ) |
---|
| 51 | meta = MetaData( bind=engine ) |
---|
| 52 | # Try to load dataset table |
---|
| 53 | try: |
---|
| 54 | dataset_table = Table( "dataset", meta, autoload=True ) |
---|
| 55 | except NoSuchTableError: |
---|
| 56 | # No 'dataset' table means a completely uninitialized database, which |
---|
| 57 | # is fine, init the database in a versioned state |
---|
| 58 | log.info( "No database, initializing" ) |
---|
| 59 | # Database might or might not be versioned |
---|
| 60 | try: |
---|
| 61 | # Declare the database to be under a repository's version control |
---|
| 62 | db_schema = schema.ControlledSchema.create( engine, migrate_repository ) |
---|
| 63 | except: |
---|
| 64 | # The database is already under version control |
---|
| 65 | db_schema = schema.ControlledSchema( engine, migrate_repository ) |
---|
| 66 | # Apply all scripts to get to current version |
---|
| 67 | migrate_to_current_version( engine, db_schema ) |
---|
| 68 | return |
---|
| 69 | try: |
---|
| 70 | hda_table = Table( "history_dataset_association", meta, autoload=True ) |
---|
| 71 | except NoSuchTableError: |
---|
| 72 | raise Exception( "Your database is older than hg revision 1464:c7acaa1bb88f and will need to be updated manually" ) |
---|
| 73 | # There is a 'history_dataset_association' table, so we (hopefully) have |
---|
| 74 | # version 1 of the database, but without the migrate_version table. This |
---|
| 75 | # happens if the user has a build from right before migration was added. |
---|
| 76 | # Verify that this is true, if it is any older they'll have to update |
---|
| 77 | # manually |
---|
| 78 | if 'copied_from_history_dataset_association_id' not in hda_table.c: |
---|
| 79 | # The 'copied_from_history_dataset_association_id' column was added in |
---|
| 80 | # rev 1464:c7acaa1bb88f. This is the oldest revision we currently do |
---|
| 81 | # automated versioning for, so stop here |
---|
| 82 | raise Exception( "Your database is older than hg revision 1464:c7acaa1bb88f and will need to be updated manually" ) |
---|
| 83 | # At revision 1464:c7acaa1bb88f or greater (database version 1), make sure |
---|
| 84 | # that the db has version information. This is the trickiest case -- we |
---|
| 85 | # have a database but no version control, and are assuming it is a certain |
---|
| 86 | # version. If the user has postion version 1 changes this could cause |
---|
| 87 | # problems |
---|
| 88 | try: |
---|
| 89 | version_table = Table( "migrate_version", meta, autoload=True ) |
---|
| 90 | except NoSuchTableError: |
---|
| 91 | # The database exists but is not yet under migrate version control, so init with version 1 |
---|
| 92 | log.info( "Adding version control to existing database" ) |
---|
| 93 | try: |
---|
| 94 | metadata_file_table = Table( "metadata_file", meta, autoload=True ) |
---|
| 95 | schema.ControlledSchema.create( engine, migrate_repository, version=2 ) |
---|
| 96 | except NoSuchTableError: |
---|
| 97 | schema.ControlledSchema.create( engine, migrate_repository, version=1 ) |
---|
| 98 | # Verify that the code and the DB are in sync |
---|
| 99 | db_schema = schema.ControlledSchema( engine, migrate_repository ) |
---|
| 100 | if migrate_repository.versions.latest != db_schema.version: |
---|
| 101 | raise Exception( "Your database has version '%d' but this code expects version '%d'. Please backup your database and then migrate the schema by running 'sh manage_db.sh upgrade'." |
---|
| 102 | % ( db_schema.version, migrate_repository.versions.latest ) ) |
---|
| 103 | else: |
---|
| 104 | log.info( "At database version %d" % db_schema.version ) |
---|
| 105 | |
---|
| 106 | def migrate_to_current_version( engine, schema ): |
---|
| 107 | # Changes to get to current version |
---|
| 108 | changeset = schema.changeset( None ) |
---|
| 109 | for ver, change in changeset: |
---|
| 110 | nextver = ver + changeset.step |
---|
| 111 | log.info( 'Migrating %s -> %s... ' % ( ver, nextver ) ) |
---|
| 112 | old_stdout = sys.stdout |
---|
| 113 | class FakeStdout( object ): |
---|
| 114 | def __init__( self ): |
---|
| 115 | self.buffer = [] |
---|
| 116 | def write( self, s ): |
---|
| 117 | self.buffer.append( s ) |
---|
| 118 | def flush( self ): |
---|
| 119 | pass |
---|
| 120 | sys.stdout = FakeStdout() |
---|
| 121 | try: |
---|
| 122 | schema.runchange( ver, change, changeset.step ) |
---|
| 123 | finally: |
---|
| 124 | for message in "".join( sys.stdout.buffer ).split( "\n" ): |
---|
| 125 | log.info( message ) |
---|
| 126 | sys.stdout = old_stdout |
---|