def execute_safely(manager, statement): try: manager.execute(statement) except Exception: print 'Statement %r failed (this is not fatal)' % statement def delete_duplicates(manager, table, first_id, second_id): rows = manager.execute( 'SELECT %s, %s, COUNT(1) AS count FROM %s ' 'GROUP BY %s, %s HAVING count > 1' % (first_id, second_id, table, first_id, second_id)) for first_id_value, second_id_value, count_unused in rows: manager.execute('DELETE FROM %s ' 'WHERE %s = %%s AND %s = %%s LIMIT 1' % (table, first_id, second_id), first_id_value, second_id_value) if rows: print 'Deleted %s duplicate rows from %s' % (len(rows), table) def delete_invalid_foriegn_keys(manager, pivot_table, foreign_key_field, destination_table): manager.execute( 'DELETE %(table)s.* FROM %(table)s ' 'LEFT JOIN %(destination_table)s ' 'ON %(table)s.%(field)s = %(destination_table)s.id ' 'WHERE %(destination_table)s.id IS NULL' % dict(table=pivot_table, field=foreign_key_field, destination_table=destination_table)) deleted_count = manager._database.rowcount if deleted_count: print ('Deleted %s invalid foreign key references from %s (%s)' % (deleted_count, pivot_table, foreign_key_field)) def unique_index_name(table): return table + '_both_ids' def basic_index_name(table, field): if field == 'aclgroup_id': field = 'acl_group_id' return table + '_' + field def create_unique_index(manager, pivot_table, first_field, second_field): index_name = unique_index_name(pivot_table) manager.execute('CREATE UNIQUE INDEX %s ON %s (%s, %s)' % (index_name, pivot_table, first_field, second_field)) # these indices are in the migrations but may not exist for historical # reasons old_index_name = basic_index_name(pivot_table, first_field) execute_safely(manager, 'DROP INDEX %s ON %s' % (old_index_name, pivot_table)) def drop_unique_index(manager, pivot_table, first_field): index_name = unique_index_name(pivot_table) manager.execute('DROP INDEX %s ON %s' % (index_name, pivot_table)) old_index_name = basic_index_name(pivot_table, first_field) manager.execute('CREATE INDEX %s ON %s (%s)' % (old_index_name, pivot_table, first_field)) def foreign_key_name(table, field): return '_'.join([table, field, 'fk']) def create_foreign_key_constraint(manager, table, field, destination_table): key_name = foreign_key_name(table, field) manager.execute('ALTER TABLE %s ADD CONSTRAINT %s FOREIGN KEY (%s) ' 'REFERENCES %s (id) ON DELETE NO ACTION' % (table, key_name, field, destination_table)) def drop_foreign_key_constraint(manager, table, field): key_name = foreign_key_name(table, field) manager.execute('ALTER TABLE %s DROP FOREIGN KEY %s' % (table, key_name)) def cleanup_m2m_pivot(manager, pivot_table, first_field, first_table, second_field, second_table, create_unique): delete_duplicates(manager, pivot_table, first_field, second_field) delete_invalid_foriegn_keys(manager, pivot_table, first_field, first_table) delete_invalid_foriegn_keys(manager, pivot_table, second_field, second_table) if create_unique: # first field is the more commonly used one, so we'll replace the # less-commonly-used index with the larger unique index create_unique_index(manager, pivot_table, second_field, first_field) create_foreign_key_constraint(manager, pivot_table, first_field, first_table) create_foreign_key_constraint(manager, pivot_table, second_field, second_table) def reverse_cleanup_m2m_pivot(manager, pivot_table, first_field, second_field, drop_unique): drop_foreign_key_constraint(manager, pivot_table, second_field) drop_foreign_key_constraint(manager, pivot_table, first_field) if drop_unique: drop_unique_index(manager, pivot_table, second_field) TABLES = ( ('hosts_labels', 'host_id', 'hosts', 'label_id', 'labels', True), ('acl_groups_hosts', 'host_id', 'hosts', 'aclgroup_id', 'acl_groups', True), ('acl_groups_users', 'user_id', 'users', 'aclgroup_id', 'acl_groups', True), ('autotests_dependency_labels', 'test_id', 'autotests', 'label_id', 'labels', False), ('jobs_dependency_labels', 'job_id', 'jobs', 'label_id', 'labels', False), ('ineligible_host_queues', 'job_id', 'jobs', 'host_id', 'hosts', True), ) def migrate_up(manager): for (table, first_field, first_table, second_field, second_table, create_unique) in TABLES: cleanup_m2m_pivot(manager, table, first_field, first_table, second_field, second_table, create_unique) def migrate_down(manager): for (table, first_field, first_table, second_field, second_table, drop_unique) in reversed(TABLES): reverse_cleanup_m2m_pivot(manager, table, first_field, second_field, drop_unique)