/**
 * Copyright (C) 2010-2015 The Roslin Institute <contact andy.law@roslin.ed.ac.uk>
 *
 * This file is part of JEnsembl: a Java API to Ensembl data sources developed by the
 * Bioinformatics Group at The Roslin Institute, The Royal (Dick) School of
 * Veterinary Studies, University of Edinburgh.
 *
 * Project hosted at: http://jensembl.sourceforge.net
 *
 * This is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License (version 3) as published by
 * the Free Software Foundation.
 *
 * This software is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * in this software distribution. If not, see: http://opensource.org/licenses/gpl-3.0.html
 */
package uk.ac.roslin.ensembl.dao.database;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.util.*;
import java.util.Map.Entry;
import org.apache.ibatis.session.SqlSession;
import org.apache.ibatis.session.SqlSessionFactory;
import org.apache.ibatis.session.SqlSessionFactoryBuilder;
import org.biojava3.core.sequence.transcription.TranscriptionEngine;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import uk.ac.roslin.ensembl.config.*;
import uk.ac.roslin.ensembl.config.DBConnection.DataSource;
import uk.ac.roslin.ensembl.exception.ConfigurationException;
import uk.ac.roslin.ensembl.exception.DAOException;
import uk.ac.roslin.ensembl.exception.NonUniqueException;
import uk.ac.roslin.ensembl.mapper.DatabaseMapper;
import uk.ac.roslin.ensembl.model.core.CollectionSpecies;
import uk.ac.roslin.ensembl.model.core.Species;
import uk.ac.roslin.ensembl.model.database.*;

public class DBRegistry
        implements Registry {

    final static Logger LOGGER = LoggerFactory.getLogger(DBRegistry.class);
    final static String AmbiguitySeparator = "|OR|";
    protected SqlSessionFactory sqlMapper;
    protected Properties configProperties = new Properties();
    protected SchemaVersion schemaVersion = null;
    protected DBConnection dbConnection = null;
    private String mostRecentEnsemblVersion;
//    private String mostRecentGenomesVersion;
    //incase the datasource does not include the 'current' dbVersion
    // the 'schema' Version
    private int highestEnsemblSchemaVersion = 0;
    //the DB dbVersion
    private int highestReleaseVersion = 0;
//    private int currentDatasourceVersion = 0;
    //the versions of core databases for a species or collection name
    private HashMap<String, TreeSet<Integer>> databaseVersions = new HashMap<String, TreeSet<Integer>>();
    protected TreeSet<Integer> knownSchemaVersions = new TreeSet<Integer>();
    //the mybatis configProperties for database connection
    private String baseMybatis;
    //to read the mybatis regConfig
    protected Reader reader;

    //these two hashes hold all the species....keyed on the meta.key = species.productionname
    //(databasestylename)
    protected TreeMap<String, DBSpecies> speciesHash = new TreeMap<String, DBSpecies>();
    //for collection species without GCAccessions (mostly prior to the use of non-taxonomic collections)
    protected TreeMap<String, DBCollectionSpecies> oldstyle_collectionspeciesHashByDBName = new TreeMap<String, DBCollectionSpecies>();
//    //only put species made from new style collections ( or rather, any species with a GCAccession) in here

    protected TreeMap<String, DBCollectionSpecies> collectionspeciesHashByGCAccession = new TreeMap<String, DBCollectionSpecies>();

    protected List<DBCollection> collections = new ArrayList<DBCollection>();
    //we need a separate one of these for each collection (dbVersion) as they will grow
    protected TreeMap<String, TreeMap<String, DBCollection>> collectionNameDBVersionHash = new TreeMap<String, TreeMap<String, DBCollection>>();
    //the aliases are now all added as lower case versions
    protected TreeMap<String, String> ensemblNameForAliasHash = new TreeMap<String, String>();

    //new style ( post v16) collection species should all be indexed by GCAccession
    //hence we need a hash of alias to Accession
    protected TreeMap<String, String> assemblyAccessionStemForAliasHash = new TreeMap<String, String>();

    //the databases
    protected TreeSet<DBDatabase> allDatabases = new TreeSet<DBDatabase>();
    protected TreeSet<DBSingleSpeciesDatabase> singleSpeciesDatabases = new TreeSet<DBSingleSpeciesDatabase>();
    protected TreeSet<DBComparisonDatabase> comparisonDatabases = new TreeSet<DBComparisonDatabase>();
    //HashMap of comparison databases indexed on 'group' , then on 'dbVersion' (not schema dbVersion)
    protected HashMap<EnsemblComparaDivision, HashMap<String, DBComparisonDatabase>> comparaDBByDivision
            = new HashMap<EnsemblComparaDivision, HashMap<String, DBComparisonDatabase>>();
    protected TreeSet<DBCollectionDatabase> collectionDatabases = new TreeSet<DBCollectionDatabase>();
    protected String newline = (System.getProperty("line.separator") != null) ? System.getProperty("line.separator") : "\r\n";
    protected StringBuilder tooNewDB = new StringBuilder();
    protected StringBuilder unknownDB = new StringBuilder();
    protected RegistryConfiguration regConfig = null;
    DataSource datasourceType;
    protected boolean alreadyInitialized = false;
    protected boolean alreadyParsed = false;
    protected HashMap<String, String> renamedDBs = new HashMap<String, String>();

    private HashMap<Integer, TranscriptionEngine> transcriptionEngines = new HashMap<Integer, TranscriptionEngine>();

    public static DBRegistry createEmptyRegistry() {
        return new DBRegistry();
    }

    public static DBRegistry createRegistryForDataSource(DataSource type) throws ConfigurationException, DAOException {
        if (type == null) {
            throw new ConfigurationException("Invalid Datasource");
        }
        if (DataSource.ENSEMBLBACTERIA.equals(type)) {
            throw new ConfigurationException("Ensembl Bacteria data sources are too large to be loaded en masse. "
                    +"Use '.createRegistryForDataSourceCurrentRelease()' to load the current release; "
                    +"or create an unitialized registry to interrogate available versions and then load a single release.");
        }
        DBRegistry dbRegistry = new DBRegistry();
        dbRegistry.setConfiguration(type);
        dbRegistry.initialize(null);
        return dbRegistry;
    }

    public static DBRegistry createUninitializedRegistryForDataSource(DataSource type) throws ConfigurationException, DAOException {
        if (type == null) {
            throw new ConfigurationException("Invalid Datasource");
        }
        DBRegistry dbRegistry = new DBRegistry();
        dbRegistry.setConfiguration(type);
        dbRegistry.parseWithoutInitializing();
        return dbRegistry;
    }

    public static DBRegistry createRegistryForConfiguration(RegistryConfiguration conf) throws ConfigurationException, DAOException {
        if (conf == null) {
            throw new ConfigurationException("Invalid Configuration");
        }
        DBRegistry dbRegistry = new DBRegistry();
        dbRegistry.setConfiguration(conf);
        dbRegistry.initialize(null);
        return dbRegistry;
    }

    public static DBRegistry createUninitializedRegistryForConfiguration(RegistryConfiguration conf) throws ConfigurationException, DAOException {
        if (conf == null) {
            throw new ConfigurationException("Invalid Configuration");
        }
        DBRegistry dbRegistry = new DBRegistry();
        dbRegistry.setConfiguration(conf);
        dbRegistry.parseWithoutInitializing();
        return dbRegistry;
    }

    public static DBRegistry createRegistryForDataSourceCurrentRelease(DataSource type) throws ConfigurationException, DAOException {
        if (type == null) {
            throw new ConfigurationException("Invalid Datasource");
        }

        // would be simpler just to do
        //SchemaVersion versionProps = new SchemaVersion(); 
        //versionProps.getCurrentGenomesVersion()
        //versionProps.getCurrentEnsemblVersion()
        DBRegistry dbRegistry = new DBRegistry();
        dbRegistry.setConfiguration(type);
        dbRegistry.parseWithoutInitializing();
        dbRegistry.knownSchemaVersions.clear();
        //allows backpedaling if actual sorce is older than config 
        //(e.g. ensemblgenomes updates a month later than ensembl)
        dbRegistry.knownSchemaVersions.add(dbRegistry.highestEnsemblSchemaVersion);
        dbRegistry.initialize(dbRegistry.highestReleaseVersion);
        return dbRegistry;
    }

    public static DBRegistry createRegistryForDataSourceAtReleaseVersion(DataSource type, Integer release) throws ConfigurationException, DAOException {
        if (type == null) {
            throw new ConfigurationException("Invalid Datasource");
        }
        if (release == null) {
            throw new ConfigurationException("Invalid schema/release");
        }
        DBRegistry dbRegistry = new DBRegistry();
        dbRegistry.setConfiguration(type);
        dbRegistry.parseWithoutInitializing();
        if (type.equals(DataSource.ENSEMBLDB)) {
            if (!dbRegistry.knownSchemaVersions.contains(release)) {
                throw new ConfigurationException(release + " not a known schema/release");
            }
        }
        dbRegistry.initialize(release);
        return dbRegistry;
    }

    /**
     * Parameterless public constructor for DBRegistry object
     */
    private DBRegistry() {
    }

    public void setConfiguration(RegistryConfiguration conf) throws ConfigurationException, DAOException {
        this.regConfig = conf;
        this.renamedDBs = this.regConfig.getDb().getRenamedDBs();
        this.datasourceType = regConfig.getType();
        this.dbConnection = regConfig.getDb();
        this.configProperties.putAll(dbConnection.getConfigurationProperties());
        this.schemaVersion = regConfig.getSchema();
        this.configProperties.putAll(schemaVersion.getConfigurationProperties());
        this.mostRecentEnsemblVersion = this.schemaVersion.getCurrentEnsemblVersion();
//        this.mostRecentGenomesVersion = this.schemaVersion.getCurrentGenomesVersion();

//        //for ensemble genomes we have a DBrelease_schema_build
//        //but for ensembl we have DBrelease_build, where DBrelease = schema
//        
//        if (this.datasourceType==DataSource.ENSEMBLGENOMES) {
//            this.currentDatasourceVersion = Integer.parseInt(this.mostRecentGenomesVersion);
//        } else {
//            this.currentDatasourceVersion = Integer.parseInt(this.mostRecentEnsemblVersion);
//        }
        String[] versions = schemaVersion.getRegisteredSchemas();

        for (int i = 0; i < versions.length; i++) {
            this.knownSchemaVersions.add(Integer.parseInt(versions[i]));
        }

        LOGGER.info("This application is configured to use schema version " + mostRecentEnsemblVersion
                + " of Ensembl, and knows about the schema versions: " + Arrays.toString(versions));

        // mybatis configProperties for databaseDAO
        this.baseMybatis = this.schemaVersion.getBaseMybatis();
        try {
            reader = new InputStreamReader(this.getClass().getClassLoader().getResourceAsStream(baseMybatis));
            //this doesn't work once you bundle up
            //reader = Resources.getResourceAsReader(baseMybatis);
        } catch (Exception ex) {
            throw new ConfigurationException("Fail to read Mybatis Configuration for initial Database connection", ex);
        }
        try {
            sqlMapper = new SqlSessionFactoryBuilder().build(reader, "current", configProperties);
            if (sqlMapper == null) {
                throw new Exception("SqlSessionFactory is null");
            }
        } catch (Exception e) {
            throw new ConfigurationException("The DBRegistry is unable to make a SqlSessionFactory", e);
        }
    }

    public void setConfiguration(DataSource type) throws ConfigurationException, DAOException {
        this.setConfiguration(new RegistryConfiguration(type));
    }

    private void parseWithoutInitializing() throws ConfigurationException, DAOException {

        if (this.alreadyInitialized) {
            throw new ConfigurationException("Tried to re-initialize a DBRegistry");
        }

        LOGGER.info("Registry is reading information about databases available from this source ("
                + this.dbConnection.getConfigurationProperties().getProperty("url") + "):");

        for (String s : this.getDatabases()) {

            if (dbConnection.isDBNameValid(s)) {
                try {
                    DBDatabase db = this.makeDatabase(s.trim());

                    if (db.intSchemaVersion > Integer.parseInt(this.mostRecentEnsemblVersion)) {
                        LOGGER.warn("IGNORING Database " + s + " with higher schema number ("
                                + db.getSchemaVersion() + ") than application configuration (" + this.mostRecentEnsemblVersion + ").");
                        tooNewDB.append("\t"+s + newline);
                    } else if (!this.knownSchemaVersions.contains(Integer.parseInt(db.getSchemaVersion()))) {
                        LOGGER.warn("IGNORING Database " + s + " with schema number ("
                                + db.getSchemaVersion() + ") not found in the application configuration.");
                        unknownDB.append("\t"+s + newline);
                    } else {
                        this.pseudoRegisterDatabase(db);
                    }

                } catch (Exception e) {
                    LOGGER.warn("Unspecified error thrown trying to make a Database object for '" + s + "'", e);
                }
            }
        }
        this.alreadyParsed=true;

    }

    /**
     * private initialisation method for the registry which connects to ensembl
     * via a DBDatasourceDAO and retrieves details on all the current tables and
     * creates database objects of the appropriate type for each. Also retrieves
     * dbSpeciesName aliases for all the current release dbSpeciesName. And
     * finds out what dbSpeciesName are present in multispecies databases.
     *
     * @param release
     * @throws DAOException, ConfigurationException
     * @throws uk.ac.roslin.ensembl.exception.ConfigurationException
     */
    public void initialize(Integer release) throws DAOException, ConfigurationException {

        if (this.alreadyInitialized) {
            throw new ConfigurationException("Tried to re-initialize a DBRegistry");
        }

        //populates the names of all the databases available, excluding marts, test, mysql and information_schema
        LOGGER.info("Registry is loading and sorting all databases available from this source ("
                + this.dbConnection.getConfigurationProperties().getProperty("url") + "):");

        if (release != null) {
            LOGGER.info("Loading databases only for release " + release);
        }

        tooNewDB = new StringBuilder();
        unknownDB = new StringBuilder();

        if (release != null) {
            this.highestReleaseVersion = release;
        }

        /*
         * note by default translation by this default engine is configured to convert 
         * non-Methionine initiations to Methionine. This cannot be alterred so you need 
         * to make an appropriate different Engine to use in place of the default
         * TranscriptionEngine.Builder b = new TranscriptionEngine.Builder();
         * b.table(1).initMet(false).trimStop(true);
         * TranscriptionEngine alternativeDefault = b.build();
         * 
         * this could then be handed to  (DATranlsation).getProteinSequence(TranscriptionEngine trancriptionEngine)
         * or (DNASequence).getRNASequence().getProteinSequence(engine)
         */
        transcriptionEngines.put(1, TranscriptionEngine.getDefault());

        for (String s : this.getDatabases()) {

            if (dbConnection.isDBNameValid(s)) {
//                 if (s.startsWith("bacillus_collection_core_")  
//                      || s.startsWith("bacteria_21_collection_core_") ) { 
//                    && (s.startsWith("escherichia_shigella_collection_core_16")
//                        || s.startsWith("bacteria_22"))) {
                try {
                    DBDatabase db = this.makeDatabase(s.trim());

                    if (release != null && db.getIntDBVersion() != release) {
                        continue;
                    }

                    if (db.intSchemaVersion > Integer.parseInt(this.mostRecentEnsemblVersion)) {
                        LOGGER.warn("IGNORING Database " + s + " with higher schema number ("
                                + db.getSchemaVersion() + ") than application configuration (" + this.mostRecentEnsemblVersion + ").");
                        tooNewDB.append("\t"+s + newline);
                    } //                    //might drop this test, we may drop 'mostRecentGenomes'....
                    //                    else if ((this.datasourceType == DataSource.ENSEMBLGENOMES)
                    //                            && db.intDBVersion > Integer.parseInt(this.mostRecentGenomesVersion)) {
                    //                        LOGGER.warn("BEWARE: Adding Database " + s + " with higher release number ("
                    //                                + db.intDBVersion + ") than application configuration (" + this.mostRecentGenomesVersion + "). "
                    //                                + "This database can be accessed by schema or release number but will not be returned by default methods that return the current release version.");
                    //                        this.aboveCurrentReleaseDB.append(s + newline);
                    //                        this.registerDatabase(db);
                    ////                        LOGGER.warn("IGNORING Database " + s + " with higher release number ("
                    ////                                + db.intDBVersion + ") than application configuration (" + this.mostRecentGenomesVersion + ").");
                    ////                        tooNewDB.append("\t"+s + newline);
                    //                    } 
                    else if (!this.knownSchemaVersions.contains(Integer.parseInt(db.getSchemaVersion()))) {
                        LOGGER.warn("IGNORING Database " + s + " with schema number ("
                                + db.getSchemaVersion() + ") not found in the application configuration.");
                        unknownDB.append("\t"+s + newline);
                    } else {
                        this.registerDatabase(db);
                    }

                } catch (Exception e) {
                    LOGGER.warn("Unspecified error thrown trying to make a Database object for '" + s + "'", e);
                }
            }
        }

        if (allDatabases.isEmpty()) {
            throw new ConfigurationException("No Valid Databases Loaded");
        }

        if (release != null) {
            DBDatabase first = allDatabases.first();
            if (first != null) {
                this.mostRecentEnsemblVersion = first.schemaVersion;
                this.highestEnsemblSchemaVersion = first.intSchemaVersion;
                this.highestReleaseVersion = release;
                this.knownSchemaVersions.clear();
                this.knownSchemaVersions.add(release);
            }
        }

        //change this to get the most recent release rather than the 'current release'
        //this should sort the noted BUG below too
        //get all the core databases for the current release
        //so that we can create Species database objects for each of these
        //filling in the aliases and some other properties
        //we then put these in a hash against genus_species
        //BUG: if the datasource doesnt have any current releases
        //e.g. if configured to load ensembldb_archives
        //we wont make any species -
        //need a work around if this is the case - to use the highest release dbVersion found
        TreeSet<DBSingleSpeciesDatabase> badDatabases = new TreeSet<DBSingleSpeciesDatabase>();

        // go through all the SSdatabases and select the current core releases
        // and make a species object from these
        for (DBSingleSpeciesDatabase d : this.singleSpeciesDatabases) {

            //can't use this method anymore cos can throw a nonUniqueException
//            if (d.type == EnsemblDBType.core
//                    && d.getDBVersion().equalsIgnoreCase(this.getMostRecentDBVersion(d.getdBClassifier()))) {     
            String mostRecentVersion = null;

            if (this.databaseVersions.containsKey(d.getdBClassifier())) {
                mostRecentVersion = this.databaseVersions.get(d.getdBClassifier()).last().toString();
            } else {
                //if this isnt a db classifier as stored in the version hash - bail out
                continue;
            }

            //if this is the most recent core database - we make the species
            if (d.type == EnsemblDBType.core
                    && d.getDBVersion().equalsIgnoreCase(mostRecentVersion)) {

                DBSpecies spData = null;

                //this can fail if there is something wrong with the database!
                try {
                    spData = this.createSpecies((DBSingleSpeciesCoreDatabase) d);
                } catch (DAOException dAOE) {
                    LOGGER.info("Failed to create a species for current core database: " + d.getdBName()
                            + ": Removed this database!", dAOE);
                }
                if (spData != null && spData.getDatabaseStyleName() != null && !spData.getDatabaseStyleName().isEmpty()) {
                    d.setSpecies(spData);
                    speciesHash.put(spData.getDatabaseStyleName(), spData);
                } else {
                    //store this as a bad (current) database
                    badDatabases.add(d);
                }
            }

        }

        //remove any databases that i cant make a species for!
        this.singleSpeciesDatabases.removeAll(badDatabases);
        this.allDatabases.removeAll(badDatabases);

        // the collections will add members with time - so each collectionDB needs
        //its own Collection
        //get all the collection_core databases 
        //so that we can create Collection database objects for each of these
        //and populate it with CollectionSpecies objects for this collection
        // however we will reuse existing CollectionSpecies objects
        // and only add new ones to the registry
        TreeSet<DBCollectionDatabase> badCollectionDatabases = new TreeSet<DBCollectionDatabase>();

        for (DBCollectionDatabase d : this.collectionDatabases) {
            if (d.type == EnsemblDBType.collection_core) {

                // make a registrycollection for each collection core database
                //as these may differ over time
                DBCollection thisCollection = null;
                try {
                    //creates (or reuses) all the species in the collection
                    thisCollection = this.getCollectionSpeciesProperties((CollectionCoreDatabase) d);
                } catch (DAOException ex) {
                    LOGGER.info("Failed to create a collectionfor current core database: " + d.getdBName()
                            + ": Removed this database!", ex);
                }

                if (thisCollection != null) {
                    this.collections.add(thisCollection);
                    //d.setCollection(r);
                    if (this.collectionNameDBVersionHash.containsKey(thisCollection.getCollectionName())) {
                        this.collectionNameDBVersionHash.get(thisCollection.getCollectionName()).put(d.getDBVersion(), thisCollection);
                    } else {
                        TreeMap<String, DBCollection> map = new TreeMap<String, DBCollection>();
                        map.put(d.getDBVersion(), thisCollection);
                        this.collectionNameDBVersionHash.put(thisCollection.getCollectionName(), map);
                    }
                } else {
                    badCollectionDatabases.add(d);
                }
            }
        }

        this.collectionDatabases.removeAll(badCollectionDatabases);
        this.allDatabases.removeAll(badCollectionDatabases);

        // we need to set the collections on the other types of collection databases
        for (DBCollectionDatabase d : this.collectionDatabases) {
            if (d.type != EnsemblDBType.collection_core) {
                String dbVersion = d.getDBVersion();
                String cName = d.getCollectionName();
                if (this.collectionNameDBVersionHash.containsKey(cName)
                        && this.collectionNameDBVersionHash.get(cName).containsKey(dbVersion)) {
                    d.setCollection(this.collectionNameDBVersionHash.get(cName).get(dbVersion));
                }
            }
        }

        //add aliases for renamed databases
        //only for single species databases
        for (Map.Entry<String, String> entry : this.renamedDBs.entrySet()) {
            if (speciesHash.containsKey(entry.getValue())) {
                speciesHash.get(entry.getValue()).getAliases().add(entry.getKey());
                speciesHash.get(entry.getValue()).getAliases().add(entry.getKey().replace("_", " "));
            }
        }

        //we now populate the look up hash for aliases
        //and at the same time add all the relevant databases to each DBSpecies
        //and vice versa
        for (DBSpecies sp : speciesHash.values()) {
            for (String s : sp.getAliases()) {
                if (this.ensemblNameForAliasHash.containsKey(s)) {
                    String n = this.ensemblNameForAliasHash.get(s).concat(AmbiguitySeparator + sp.getDatabaseStyleName());
                    this.ensemblNameForAliasHash.put(s, n);
                } else {
                    this.ensemblNameForAliasHash.put(s, sp.getDatabaseStyleName());
                }
            }
            for (DBSingleSpeciesDatabase d : this.singleSpeciesDatabases) {
                if (d.getDbSpeciesName().equalsIgnoreCase(sp.getDatabaseStyleName())) {
                    sp.addDatabase(d);
                    if (d.getSpecies() == null) {
                        d.setSpecies(sp);
                    }
                }
            }
        }

        for (DBCollection coll : this.collections) {
            //this sets all the species to have the most recent dbstyle name, and all the TreeSets 
            //in the collections to be remade
            coll.resortSpecies();
        }

        for (DBCollectionSpecies sp : oldstyle_collectionspeciesHashByDBName.values()) {
            for (String s : sp.getAliases()) {
                if (this.ensemblNameForAliasHash.containsKey(s)) {
                    String n = this.ensemblNameForAliasHash.get(s).concat(AmbiguitySeparator + sp.getDatabaseStyleName());
                    this.ensemblNameForAliasHash.put(s, n);
                } else {
                    this.ensemblNameForAliasHash.put(s, sp.getDatabaseStyleName());
                }
            }
            for (DBCollectionDatabase d : this.collectionDatabases) {
                try {
                    if (EnsemblDBType.getCollectionDatabaseTypes().contains(d.getType())
                            && d.getCollection().getSpecies().contains(sp)) {
                        sp.addDatabase(d);
                    }
                } catch (Exception e) {
                }
            }
        }

        for (DBCollectionSpecies sp : collectionspeciesHashByGCAccession.values()) {
            for (String s : sp.getAliases()) {

                if (this.assemblyAccessionStemForAliasHash.containsKey(s)) {
                    String n = this.assemblyAccessionStemForAliasHash.get(s).concat(AmbiguitySeparator + sp.getAssemblyAccessionStem());
                    this.assemblyAccessionStemForAliasHash.put(s, n);
                } else {
                    this.assemblyAccessionStemForAliasHash.put(s, sp.getAssemblyAccessionStem());
                }
            }
            for (DBCollectionDatabase d : this.collectionDatabases) {
                try {

                    if (EnsemblDBType.getCollectionDatabaseTypes().contains(d.getType())
                            && d.getCollection().getSpecies().contains(sp)) {
                        sp.addDatabase(d);
                    }
                } catch (Exception e) {
                }
            }
        }

        //add all the relevant databases to each DBCollection
        for (Entry<String, TreeMap<String, DBCollection>> e : this.collectionNameDBVersionHash.entrySet()) {
            String name = e.getKey();

            for (Entry<String, DBCollection> ei : e.getValue().entrySet()) {
                String dbversion = ei.getKey();
                DBCollection collection = ei.getValue();
                TreeSet<CollectionDatabase> temp = new TreeSet<CollectionDatabase>();
                for (DBCollectionDatabase d : this.collectionDatabases) {
                    if (d.getCollectionName().equals(name) && d.getDBVersion().equalsIgnoreCase(dbversion)) {
                        temp.add(d);
                    }
                }
                collection.addDatabases(temp);
            }
        }
        this.alreadyInitialized = true;
    }

    /**
     * Reports the available database versions for this Registry.
     * Will successfully report for either initialized or uninitialized Registries.
     * since v74
     * @return StringBuilder
     */
    @Override
    public StringBuilder getVersionReport() {

        StringBuilder out = new StringBuilder();
        
        out.append(newline+"VERSION REPORT"+newline+"--------------"+newline+newline);

        if (datasourceType == null) {
            return out.append("\tWARNING: This Registry has invalid DataSource.");
        }
        if (dbConnection == null) {
            return out.append("\tWARNING: This Registry has invalid Database Connection.");
        }
       
        if (!this.alreadyInitialized) {
            out.append("\tWARNING: This registry is uninitialized."+newline);
        }

        out.append("Registry for: [" + this.datasourceType.toString() + "] " + this.dbConnection.getConfigurationProperties().getProperty("url") + newline);

        out.append("Most recent configured Ensembl Release Schema: " + this.mostRecentEnsemblVersion + newline);
        out.append("Most recent Schema Version available in Datasource: " + this.highestEnsemblSchemaVersion + newline);
        if ( this.datasourceType==DataSource.ENSEMBLGENOMES) {
            out.append("Most recent configured ensemblgenomes release available in Datasource: " + this.highestReleaseVersion + newline);
        }
        if ( this.datasourceType==DataSource.ENSEMBLBACTERIA) {
            out.append("Most recent configured ensemblbacteria release available in Datasource: " + this.highestReleaseVersion + newline);
        }

        out.append("Known Schemas (Ensembl releases)" + newline + "\t");

        for (Integer i : this.getKnownSchemaVersions()) {
            out.append(i + ", ");
        };

        int end = out.lastIndexOf(",");
        out.delete(end, end + 1);
        out.append(newline);
        
        out.append(dbConnection.report() + newline);

        return out;
    } 
    
        /**
     * Reports the connection and available database versions etc. for an initialized Registry.
     * Lists databases for which this API is not configured (including those too old or too recent).
     * @return StringBuilder
     */
    @Override
    public StringBuilder getBriefRegistryReport() {
        StringBuilder out = new StringBuilder();
        
        out.append(newline+"REGISTRY REPORT"+newline+"---------------"+newline+newline);        
        
        if (this.regConfig == null) {
            out.append("\tWARNING: This Registry object has not been configured."+newline+newline);
            return out;
        }

        out = this.getVersionReport();
        
        if (tooNewDB.toString() != null && !tooNewDB.toString().isEmpty()) {
            out.append("WARNING: Some database releases were too recent for the current JEnsembl configuration (i.e. using unknown schema):"
                    + newline+ newline + tooNewDB.toString());
        }

        if (unknownDB.toString() != null && !unknownDB.toString().isEmpty()) {
            out.append("WARNING: Some database releases were not known in the current JEnsembl configuration:"
                    + newline + newline+ unknownDB.toString());
        }

        if ((unknownDB.toString() == null || unknownDB.toString().isEmpty())
                && (tooNewDB.toString() == null || tooNewDB.toString().isEmpty())) {
            out.append("No database releases were not known in the current JEnsembl configuration."
                    + newline);
        }
        
       if (!this.alreadyInitialized) {
            out.append(newline+"\tWARNING: This configured Registry has not yet been initialized" + newline);
            return out;
        }

        return out;
    }

    /**
     * Reports the connection, available database versions, species and actual database names for an initialized Registry.
     * Lists databases for which this API is not configured (including those too old or too recent). 
     * @return File
     * @throws java.io.IOException
     */
    @Override
    public File getRegistryReport() throws IOException {
        
        File file = File.createTempFile("RegistryReport", ".txt");
        FileWriter fileWriter = new FileWriter(file);
        BufferedWriter out = new BufferedWriter(fileWriter, 8192);
        

        if (this.regConfig == null) {
            out.write(newline+"REGISTRY REPORT"+newline+"---------------"+newline+newline);                    
            out.write("\tWARNING: This Registry object has not been configured.");
            
            out.flush();
            out.close();
            return file;
        }

        out.write(this.getBriefRegistryReport().toString());
        
        if (!this.alreadyInitialized) {
                        
            out.flush();
            out.close();
            return file;
        }
        
        out.append(newline+"REGISTRY DETAILS"+newline+newline);        
        
        out.append("Species represented in Datasource (with aliases):" + newline);
        out.append("SPECIES in unique databases" + newline );
        out.append("---------------------------" + newline + newline );

        for (DBSpecies s : this.speciesHash.values()) {
            out.append(s.getSpeciesBinomial() + " [" + s.getDatabaseStyleName() + "] (");
            for (String st : s.getAliases()) {
                out.append(st + ", ");
            }
            try {
                out.append(")" + newline + "\tMost Recent Core Database: " + s.getMostRecentCoreDatabase().getdBName() + newline);

                for (FeatureType key : ((DBSingleSpeciesCoreDatabase) s.getMostRecentCoreDatabase()).getBuildLevels().keySet()) {
                    out.append("\t\t"+key.toString() + " : " + ((DBSingleSpeciesCoreDatabase) s.getMostRecentCoreDatabase()).getBuildLevels().get(key) +newline);
                }

            } catch (Exception e) {
                out.append(")" + newline + "\tNo Current Core Database" + newline);
            }
        }

        out.append(newline + "Old Style SPECIES in collection databases" + newline );
        out.append("-------------------------------" + newline+newline );

        if (this.oldstyle_collectionspeciesHashByDBName.values().isEmpty()) {
            out.append(newline + "NONE" + newline);
        }

        for (DBCollectionSpecies s : this.oldstyle_collectionspeciesHashByDBName.values()) {
            out.append(s.getSpeciesBinomial() + " [" + s.getDatabaseStyleName() + "] (");
            for (String st : s.getAliases()) {
                out.append(st + ", ");
            }
            try {
                out.append(")" + newline + "\tCurrent Core Database: " + s.getMostRecentCoreDatabase().getdBName() + newline);
            } catch (Exception e) {
                out.append(")" + newline + "\tNo Current Core Database" + newline);
            }
        }

        out.append(newline+"MULTI SPECIES DATABASES: ");
        out.append(newline+"-----------------------"+newline);

        for (EnsemblDBType t : EnsemblDBType.getSpeciesComparisonDatabaseTypes()) {
            out.append(t.toString() + ":"+newline);
            for (Database d : this.getDatabasesByType(t)) {
                out.append("\t" + d.getdBName() + newline);
            }
        }

        out.append(newline+"SINGLE SPECIES DATABASE DETAILS:");
        out.append(newline+"--------------------------------"+newline);

        for (DBSpecies sp : this.speciesHash.values()) {
            String id = sp.getSpeciesBinomial();
            out.append(id + newline+"--------------------"+newline);

            out.append(newline+"\tMost Recent Versions:"+newline);

//            for (Database d : sp.getDatabasesByVersion(this.getMostRecentDBVersion())) {
//                out.append("\t\t" + d.getdBName()+newline);
//            }
            for (Database d : sp.getDatabasesByVersion(sp.getMostRecentCoreDatabase().getDBVersion())) {
                out.append("\t\t" + d.getdBName() +newline);
            }

            out.append("\tBy Type:"+newline);
            for (EnsemblDBType t : EnsemblDBType.getSingleSpeciesDatabaseTypes()) {

                out.append("\t" + t.toString()+newline);
                if (sp.getDatabasesByType(t).isEmpty()) {
                    out.append("\t\tNONE"+newline);
                } else {
                    for (Database d : sp.getDatabasesByType(t)) {
                        out.append("\t\t" + d.getdBName() +newline);
                        if (t == EnsemblDBType.core) {
                            out.append("\t\t\tAssembly: " + ((SingleSpeciesCoreDatabase) d).getAssemblyName()
                                    + " [" + ((SingleSpeciesCoreDatabase) d).getAssemblyAccession()
                                    + "]"+newline);
                        }
                    }
                }
            }

        }
        out.append(newline+"COLLECTION SPECIES DATABASE DETAILS:"+newline);
        out.append("------------------------------------"+newline+newline);

        out.append(newline+"NEW STYLE SPECIES ID (using GC Accession, typically post release 17):"+newline+newline);
        
        if (this.collectionspeciesHashByGCAccession.values().isEmpty()) {
            out.append("\tNONE"+newline);
        }        
        
        for (DBCollectionSpecies sp : this.collectionspeciesHashByGCAccession.values()) {
            String id = sp.getSpeciesBinomial();
            String acc = sp.getAssemblyAccessionStem()!=null && !sp.getAssemblyAccessionStem().isEmpty() ? sp.getAssemblyAccessionStem():" ";
            
            out.append(id+" (assembly accession:"+acc+")"  +newline+"---------------------"+newline);

            out.append(newline+"\tCurrent Versions:"+newline);
            for (Database d : sp.getDatabasesByVersion(sp.getMostRecentCoreDatabase().getDBVersion())) {
                out.append("\t\t" + d.getdBName() +newline);
            }
            out.append("\tBy Type:"+newline);
            for (EnsemblDBType t : EnsemblDBType.getCollectionDatabaseTypes()) {

                out.append("\t" + t.toString() +newline);
                if (sp.getDatabasesByType(t).isEmpty()) {
                    out.append("\t\tNONE"+newline);
                } else {
                    for (Database d : sp.getDatabasesByType(t)) {
                        out.append("\t\t" + d.getdBName() +newline);
                    }
                }
            }
        }        
        
        
        out.append(newline+"OLD STYLE SPECIES ID (using name matching, typically pre release 17):"+newline+newline);
        
        if (this.oldstyle_collectionspeciesHashByDBName.values().isEmpty()) {
            out.append("\tNONE"+newline);
        }

        for (DBCollectionSpecies sp : this.oldstyle_collectionspeciesHashByDBName.values()) {
            String id = sp.getSpeciesBinomial();
            String acc = sp.getAssemblyAccessionStem()!=null && !sp.getAssemblyAccessionStem().isEmpty() ? sp.getAssemblyAccessionStem():" ";
            
            out.append(id+" (assembly accession:"+acc+")"  +newline+"---------------------"+newline);

            out.append(newline+"\tCurrent Versions:"+newline);
            for (Database d : sp.getDatabasesByVersion(sp.getMostRecentCoreDatabase().getDBVersion())) {
                out.append("\t\t" + d.getdBName() +newline);
            }
            out.append("\tBy Type:"+newline);
            for (EnsemblDBType t : EnsemblDBType.getCollectionDatabaseTypes()) {

                out.append("\t" + t.toString() +newline);
                if (sp.getDatabasesByType(t).isEmpty()) {
                    out.append("\t\tNONE"+newline);
                } else {
                    for (Database d : sp.getDatabasesByType(t)) {
                        out.append("\t\t" + d.getdBName() +newline);
                    }
                }
            }
        }

        out.append(newline+"Core Collection Databases: Species and build information"+newline);
        out.append("--------------------------------------------------------"+newline);

        if (this.collectionDatabases.isEmpty()) {
            out.append(newline+"NONE"+newline);
        }

        for (DBCollectionDatabase d : this.collectionDatabases) {
            if (d.getType().equals(EnsemblDBType.collection_core)) {

                out.append(newline+"CORECOLLECTION DB: " + d.dBName +newline);

                for (DBCollectionSpecies species : d.getCollection().getSpecies()) {

                    DBCollectionCoreDatabase db = (DBCollectionCoreDatabase) d;
                    out.append("\tSPECIES: " + species.commonName +newline);

                    //add a safety check
                    if (db.getBuildLevels(species) != null) {
                        for (FeatureType key : db.getBuildLevels(species).keySet()) {
                            out.append("\t\t" + key.toString() + " : " + db.getBuildLevels(species).get(key) +newline);
                        }
                    }

                }
            }
        }

             out.flush();
            out.close();
            return file;
    }    
    
    @Override
    public Properties getConfigProperties() {
        return configProperties;
    }    
    
    @Override
    public String getMostRecentEnsemblVersion() {
        return mostRecentEnsemblVersion;
    }
    
    //@Override
    public String getMostRecentDBVersion(String speciesOrCollection) throws NonUniqueException {

        if (speciesOrCollection == null || speciesOrCollection.isEmpty()) {
            return "";
        }

        String dbName = speciesOrCollection;

        //can throw nonuniqueexception
        //it shouldnt do this if the 'speciesOrCollection' input is a db classifier
        if (this.getEnsemblNameForAlias(speciesOrCollection) != null) {
            dbName = this.getEnsemblNameForAlias(speciesOrCollection);
        }

        if (this.databaseVersions.containsKey(dbName)) {
            return this.databaseVersions.get(dbName).last().toString();
        } else {
            return "";
        }

    }    

    public TreeSet<Integer> getKnownSchemaVersions() {
        return knownSchemaVersions;
    }

    @Override
    public int getHighestEnsemblSchemaVersion() throws DAOException, ConfigurationException {
        if (this.alreadyInitialized || this.alreadyParsed) {
            return highestEnsemblSchemaVersion;
        } else {
            if (this.datasourceType != null) {
                this.parseWithoutInitializing();
            } else {
                throw new ConfigurationException("Attempt to retrieve HighestEnsemblSchemaVersion from uninitialized database.");
            }
            return highestEnsemblSchemaVersion;
        }
    }

    @Override
    public int getHighestReleaseVersion() throws DAOException, ConfigurationException {
        if (this.alreadyInitialized || this.alreadyParsed) {
            return highestReleaseVersion;
        } else {
            if (this.datasourceType != null) {
                this.parseWithoutInitializing();
            } else {
                throw new ConfigurationException("Attempt to retrieve HighestReleaseVersion from uninitialized database.");
            }
            return highestReleaseVersion;
        }
    }    
    
    // Database look up methods
    @Override
    public DBDatabase getDatabase(String species_name_or_alias) throws NonUniqueException {
        return getDatabase(species_name_or_alias, null, null);
    }

    @Override
    public DBDatabase getDatabase(String species_name_or_alias, String db_version) throws NonUniqueException {
        return getDatabase(species_name_or_alias, null, db_version);
    }

    @Override
    public DBDatabase getDatabase(String speciesNameOrAlias_collectionName_comparaDivision,
            DatabaseType database_type, String db_version) throws NonUniqueException {

        String name = speciesNameOrAlias_collectionName_comparaDivision;

        DBDatabase out = null;

        //default to CORE type if not provided
        EnsemblDBType type = (database_type != null
                && database_type.toString() != null
                && !database_type.toString().equals(""))
                ? (EnsemblDBType) database_type
                : EnsemblDBType.core;

        if (name == null || name.isEmpty()) {
            if (EnsemblDBType.getSpeciesComparisonDatabaseTypes().contains(type)) {
                name = "multi";
            } else {
                //deff out if no name
                return out;
            }
        }

        //default to current dbVersion if not provided
        String version = (db_version != null
                && !db_version.isEmpty()
                && !db_version.equals("current"))
                ? db_version
                //                : this.getMostRecentDBVersion();
                : this.getMostRecentDBVersion(name);

        String speciesOrGroup = "";

        if (EnsemblDBType.getSpeciesComparisonDatabaseTypes().contains(type)) {
            //no look up possible
            speciesOrGroup = name;
        } else if (EnsemblDBType.getSingleSpeciesDatabaseTypes().contains(type)) {
            //do a look up for aliases
            speciesOrGroup = this.getEnsemblNameForAlias(name);
            if (speciesOrGroup == null) {
                //deff out
                return out;
            }
        } else if (EnsemblDBType.getCollectionDatabaseTypes().contains(type)) {
            //no look up possible
            speciesOrGroup = name;
        }

        if (EnsemblDBType.getSingleSpeciesDatabaseTypes().contains(type)) {
            for (DBSingleSpeciesDatabase d : this.singleSpeciesDatabases) {
                if (d.getDbSpeciesName().equals(speciesOrGroup)
                        && d.getDBVersion().equals(version)
                        && d.getType() == type) {
                    return d;
                }
            }
        } else if (EnsemblDBType.getSpeciesComparisonDatabaseTypes().contains(type)) {

            EnsemblComparaDivision gp = EnsemblComparaDivision.getEnsemblComparaDivision(speciesOrGroup);
            for (DBComparisonDatabase d : this.comparisonDatabases) {
                if (d.getComparisonDivision().equals(gp)
                        && d.getDBVersion().equals(version)
                        && d.getType() == type) {
                    return d;
                }
            }
        } //this is only working on Groups not ind species
        else if (EnsemblDBType.getCollectionDatabaseTypes().contains(type)) {
            for (DBCollectionDatabase d : this.collectionDatabases) {
                if (d.getCollectionName().equals(speciesOrGroup)
                        && d.getDBVersion().equals(version)
                        && d.getType() == type) {
                    return d;
                }
            }
        }

        return out;
    }

    @Override
    public ComparisonDatabase getComparaDatabase(EnsemblComparaDivision comparaDivision, String db_version) {

        if (this.comparaDBByDivision.get(comparaDivision) != null) {
            return this.comparaDBByDivision.get(comparaDivision).get(db_version);
        } else {
            return null;
        }
    }

    @Override
    public DBDatabase getDatabaseForFullName(String fullname) {

        DBDatabase out = null;

        if (fullname == null || fullname.isEmpty()) {
            return out;
        }

        for (DBDatabase d : this.allDatabases) {
            if (d.getdBName().equals(fullname)) {
                out = d;
                break;
            }
        }
        return out;
    }

    @Override
    public TreeSet<DBDatabase> getDatabasesByType(DatabaseType type) {
        TreeSet<DBDatabase> out = new TreeSet<DBDatabase>();

        for (DBDatabase d : this.allDatabases) {
            if (d.getType() == type) {
                out.add(d);
            }
        }
        return out;
    }

    @Override
    public String findMybatisSchemaForSchemaVersion(DatabaseType type, String schema_version) {

        String out = null;

        try {
            out = schemaVersion.getMybatisSchemaPath(type.toString(), schema_version);
        } catch (Exception e) {
        }

        //
        if (out == null) {
            try {
                out = schemaVersion.getMybatisSchemaPath(type.toString(), schemaVersion.getCurrentEnsemblVersion());
            } catch (Exception e) {
            }
        }

        return out;
    }

    /**
     * Looks up the ensembl_genus_species_name for the given alias from the
     * ensemblNameForAliasHash hash of aliases in this Registry.
     *
     * @param alias String
     * @return String the ensembl_genus_species_name
     */
    @Override
    public String getEnsemblNameForAlias(String alias) throws NonUniqueException {
        if (alias == null || alias.isEmpty()) {
            return null;
        }
        //if the entered 'alias' is a good ensembl name
        if (ensemblNameForAliasHash.containsValue(alias)) {
            return alias;
        }
        //else look for the lowercase version of 'alias' as an alias
        String get = ensemblNameForAliasHash.get(alias.toLowerCase());
        if (get == null) {
            return null;
        } else if (get.contains(DBRegistry.AmbiguitySeparator)) {

            Collection<String> allHits = new HashSet<String>();
            String[] split = get.split("\\|OR\\|");
            allHits.addAll(Arrays.asList(split));
            throw new NonUniqueException("Alias: '" + alias + "' matches more than one Ensembl Name."
                    + "You can examine the list of species by printing for (Object o: NUN.getAllHits() { "
                    + "System.out.println(((Species) o).getDatabaseStyleName()); }", allHits);
        } else {
            return get;
        }
    }

/**
     * Looks up the GC_assembly_accession_stem for the given alias of a
     * collectionspecies. Post v16 this is the preferred identifier for a
     * CollectionSpecies.
     *
     * @param alias String
     * @return String the GC_assembly_accession_stem
     */
    public String getGCAssemblyAccessionForAlias(String alias) throws NonUniqueException {
        if (alias == null || alias.isEmpty()) {
            return null;
        }

        String get = assemblyAccessionStemForAliasHash.get(alias.toLowerCase());
        if (get == null) {
            return null;
        } else if (get.contains(DBRegistry.AmbiguitySeparator)) {

            Collection<String> allHits = new HashSet<String>();
            String[] split = get.split("\\|OR\\|");
            allHits.addAll(Arrays.asList(split));
            throw new NonUniqueException("Alias matches more than one Ensembl Name."
                    + "You can examine the list of species by printing for (Object o: NUN.getAllHits() { "
                    + "System.out.println(((Species) o).getDatabaseStyleName()); }", allHits);
        } else {
            return get;
        }

    }    
    
    public Set<String> getEnsemblNamesForAliasBeginning(String alias) {
        if (alias == null || alias.isEmpty()) {
            return null;
        }
        
        Set<String> out = new HashSet<String>();
        
        //look for the lowercase version of 'alias' as an alias
        String search = alias.toLowerCase();
        
        for (Entry<String,String> e:ensemblNameForAliasHash.entrySet() ) {
            if (e.getKey().startsWith(search)) {               
                if (e.getValue().contains(DBRegistry.AmbiguitySeparator)) {
                    String[] split = e.getValue().split("\\|OR\\|");
                    out.addAll(Arrays.asList(split));
                } else {                
                    out.add(e.getValue());
                }
            }
        }
        return out;
    }
    
    public Set<String> getEnsemblNamesForAliasContaining(String alias) {
        if (alias == null || alias.isEmpty()) {
            return null;
        }
        
        Set<String> out = new HashSet<String>();
        
        //look for the lowercase version of 'alias' as an alias
        String search = alias.toLowerCase();
        
        for (Entry<String,String> e:ensemblNameForAliasHash.entrySet() ) {
            if (e.getKey().contains(search)) {               
                if (e.getValue().contains(DBRegistry.AmbiguitySeparator)) {
                    String[] split = e.getValue().split("\\|OR\\|");
                    out.addAll(Arrays.asList(split));
                } else {                
                    out.add(e.getValue());
                }
            }
        }
        return out;
    }
    
    public Set<String> getGCAssemblyAccessionsForAliasBeginning(String alias)  {
        if (alias == null || alias.isEmpty()) {
            return null;
        }

        Set<String> out = new HashSet<String>();
        
        //look for the lowercase version of 'alias' as an alias
        String search = alias.toLowerCase();
        
        for (Entry<String,String> e:assemblyAccessionStemForAliasHash.entrySet() ) {
            if (e.getKey().startsWith(search)) {               
                if (e.getValue().contains(DBRegistry.AmbiguitySeparator)) {
                    String[] split = e.getValue().split("\\|OR\\|");
                    out.addAll(Arrays.asList(split));
                } else {                
                    out.add(e.getValue());
                }
            }
        }
        return out;        
    } 
    
    public Set<String> getGCAssemblyAccessionsForAliasContaining(String alias)  {
        if (alias == null || alias.isEmpty()) {
            return null;
        }

        Set<String> out = new HashSet<String>();
        
        //look for the lowercase version of 'alias' as an alias
        String search = alias.toLowerCase();
        
        for (Entry<String,String> e:assemblyAccessionStemForAliasHash.entrySet() ) {
            if (e.getKey().contains(search)) {               
                if (e.getValue().contains(DBRegistry.AmbiguitySeparator)) {
                    String[] split = e.getValue().split("\\|OR\\|");
                    out.addAll(Arrays.asList(split));
                } else {                
                    out.add(e.getValue());
                }
            }
        }
        return out;        
    } 
    
    public Set<? extends Species> getSpeciesForAliasBeginning(String alias) {
        
        if (this.getDatasourceType().equals(DataSource.ENSEMBLBACTERIA)) {
            return this.getBacterialSpeciesForAliasBeginning(alias);
        }
        
        if (alias == null || alias.isEmpty()) {
            return null;
        }
        
        Set<DBSpecies> out = new HashSet<DBSpecies>();
        
        //look for the lowercase version of 'alias' as an alias
        String search = alias.toLowerCase();
        
        for (Entry<String,String> e:ensemblNameForAliasHash.entrySet() ) {
            if (e.getKey().startsWith(search)) {               
                if (e.getValue().contains(DBRegistry.AmbiguitySeparator)) {
                    String[] split = e.getValue().split("\\|OR\\|");
                    for (String s:split) {
                        out.add(this.speciesHash.get(s));
                    }
                } else {                
                    out.add(this.speciesHash.get(e.getValue()));
                }
            }
        }
        return out;
    }
    
    public Set<? extends Species> getSpeciesForAliasContaining(String alias) {
        
        if (this.getDatasourceType().equals(DataSource.ENSEMBLBACTERIA)) {
            return this.getBacterialSpeciesForAliasContaining(alias);
        }
        
        if (alias == null || alias.isEmpty()) {
            return null;
        }
        
        Set<DBSpecies> out = new HashSet<DBSpecies>();
        
        //look for the lowercase version of 'alias' as an alias
        String search = alias.toLowerCase();
        
        for (Entry<String,String> e:ensemblNameForAliasHash.entrySet() ) {
            if (e.getKey().contains(search)) {               
                if (e.getValue().contains(DBRegistry.AmbiguitySeparator)) {
                    String[] split = e.getValue().split("\\|OR\\|");
                    for (String s:split) {
                        out.add(this.speciesHash.get(s));
                    }
                } else {                
                    out.add(this.speciesHash.get(e.getValue()));
                }
            }
        }
        return out;
    }
    
    public Set<? extends CollectionSpecies> getBacterialSpeciesForAliasBeginning(String alias) {
        
        if (this.highestReleaseVersion < 17) {
            return this.getOldStyleBacterialSpeciesForAliasBeginning(alias);
        }

        if (alias == null || alias.isEmpty()) {
            return null;
        }
        
        Set<DBCollectionSpecies> out = new HashSet<DBCollectionSpecies>();
        
        //look for the lowercase version of 'alias' as an alias
        String search = alias.toLowerCase();
        
        for (Entry<String,String> e:assemblyAccessionStemForAliasHash.entrySet() ) {
            if (e.getKey().startsWith(search)) {               
                if (e.getValue().contains(DBRegistry.AmbiguitySeparator)) {
                    String[] split = e.getValue().split("\\|OR\\|");
                    for (String s:split) {
                        out.add(this.collectionspeciesHashByGCAccession.get(s));
                    }
                } else {                
                    out.add(this.collectionspeciesHashByGCAccession.get(e.getValue()));
                }
            }
        }
        return out;
    }
    
    public Set<? extends CollectionSpecies> getBacterialSpeciesForAliasContaining(String alias) {
        
        if (this.highestReleaseVersion < 17) {
            return this.getOldStyleBacterialSpeciesForAliasContaining(alias);
        }

        if (alias == null || alias.isEmpty()) {
            return null;
        }
        
        Set<DBCollectionSpecies> out = new HashSet<DBCollectionSpecies>();
        
        //look for the lowercase version of 'alias' as an alias
        String search = alias.toLowerCase();
        
        for (Entry<String,String> e:assemblyAccessionStemForAliasHash.entrySet() ) {
            if (e.getKey().contains(search)) {               
                if (e.getValue().contains(DBRegistry.AmbiguitySeparator)) {
                    String[] split = e.getValue().split("\\|OR\\|");
                    for (String s:split) {
                        out.add(this.collectionspeciesHashByGCAccession.get(s));
                    }
                } else {                
                    out.add(this.collectionspeciesHashByGCAccession.get(e.getValue()));
                }
            }
        }
        return out;
    }
    
    public Set<? extends CollectionSpecies> getOldStyleBacterialSpeciesForAliasBeginning(String alias)  {
        if (alias == null || alias.isEmpty()) {
            return null;
        }

        Set<CollectionSpecies> out = new HashSet<CollectionSpecies>();
        
        //look for the lowercase version of 'alias' as an alias
        String search = alias.toLowerCase();
        
        for (Entry<String,String> e:ensemblNameForAliasHash.entrySet() ) {
            if (e.getKey().startsWith(search)) {               
                if (e.getValue().contains(DBRegistry.AmbiguitySeparator)) {
                    String[] split = e.getValue().split("\\|OR\\|");                   
                    for (String s:split) {
                      out.add(this.oldstyle_collectionspeciesHashByDBName.get(s) );
                    }
                } else {                
                    out.add(this.oldstyle_collectionspeciesHashByDBName.get(e.getValue()));
                }
            }
        }
        return out;        
    }
    
    public Set<? extends CollectionSpecies> getOldStyleBacterialSpeciesForAliasContaining(String alias)  {
        if (alias == null || alias.isEmpty()) {
            return null;
        }

        Set<CollectionSpecies> out = new HashSet<CollectionSpecies>();
        
        //look for the lowercase version of 'alias' as an alias
        String search = alias.toLowerCase();
        
        for (Entry<String,String> e:ensemblNameForAliasHash.entrySet() ) {
            if (e.getKey().contains(search)) {               
                if (e.getValue().contains(DBRegistry.AmbiguitySeparator)) {
                    String[] split = e.getValue().split("\\|OR\\|");                   
                    for (String s:split) {
                      out.add(this.oldstyle_collectionspeciesHashByDBName.get(s) );
                    }
                } else {                
                    out.add(this.oldstyle_collectionspeciesHashByDBName.get(e.getValue()));
                }
            }
        }
        return out;        
    }

    /**
     * Method for an adhoc alias to be added by the user in this session. Adds
     * the alias to the aliases TreeSet of the cognate DBSpecies and to the
     * ensemblNameForAliasHash hash of aliases in this Registry
     *
     * @param ensembl_genus_species_name Ensmebl databse dbVersion of the name,
     * e.g. 'homo_sapiens'
     * @param alias any string the user wants to use an alias for this
     * @return boolean true if successful
     * @throws Exception if update fails, with meaning full message
     */
    @Override
    public boolean addSessionAlias(String ensembl_genus_species_name, String alias) throws Exception {

        if (ensembl_genus_species_name == null || alias == null || ensembl_genus_species_name.isEmpty() || alias.isEmpty()) {
            throw new Exception("Failed to add alias: " + alias + " for species: " + ensembl_genus_species_name);
        }
        Species spData = null;

        try {
            spData = this.speciesHash.get(ensembl_genus_species_name);
            if (spData == null) {
                spData = this.getCSpeciesByGCAccessionStem(this.assemblyAccessionStemForAliasHash.get(ensembl_genus_species_name));
                if (spData == null) {
                    spData = this.oldstyle_collectionspeciesHashByDBName.get(ensembl_genus_species_name);
                }
                if (spData == null) {
                    throw new Exception("No species in ensembl registry called: " + ensembl_genus_species_name);
                }
            }

        } catch (Exception e) {
            throw new Exception("Failed to add alias: " + alias + " for species: " + ensembl_genus_species_name, e);
        }
        return this.addSessionAlias(spData, alias);
    }

    @Override
    public boolean addSessionAlias(Species sp, String alias) throws Exception {
        if (sp == null || alias == null || alias.isEmpty()) {
            throw new Exception("Failed to add alias: " + alias + " for species: " + sp);
        }

        if (this.getGCAccessionedCollectionSpecies().contains(sp)) {
            sp.getAliases().add(alias.toLowerCase());
            this.assemblyAccessionStemForAliasHash.put(alias.toLowerCase(), sp.getAssemblyAccessionStem());
        } else {
            sp.getAliases().add(alias.toLowerCase());
            if (this.ensemblNameForAliasHash.containsKey(alias.toLowerCase())) {
                String n = this.ensemblNameForAliasHash.get(alias.toLowerCase()).concat(AmbiguitySeparator + sp.getDatabaseStyleName());
                this.ensemblNameForAliasHash.put(alias.toLowerCase(), n);
            } else {
                this.ensemblNameForAliasHash.put(alias.toLowerCase(), sp.getDatabaseStyleName());
            }
        }
        return true;
    }

    /**
     * Returns a HashSet<DBSpecies> matching the given alias. If name is not a
     * valid String, returns null, if no species are found, returns empty Set.
     * If Name matches any DBSpecies, these are returned without searching for
     * DBCollectionSpecies. Note if we hit a CollectionSpecies - any more recent
     * Species with GCA (ENA) Identifiers will be returned before searching for
     * older Species (lacking GCA IDs) which match the alias. If you want to be
     * sure to search for all alias hits use getSetOfEverySpeciesByAlias(String
     * alias_or_name).
     *
     * @param alias_or_name
     * @return a Collection<? extends Species> of DBSpecies.
     */
    @Override
    public Collection<? extends Species> getSetOfSpeciesByAlias(String alias_or_name) {
        HashSet<DBSpecies> ret = new HashSet<DBSpecies>();

        if (alias_or_name == null || alias_or_name.isEmpty()) {
            return null;
        }

        //first look for DBSpecies
        DBSpecies out = null;
        String name = null;
        try {
            name = this.getEnsemblNameForAlias(alias_or_name);
            out = (name != null) ? this.speciesHash.get(name) : null;
            if (out != null) {
                ret.add(out);
            }
        } catch (NonUniqueException ex) {
            for (String acc : (Collection<String>) ex.getAllHits()) {
                if (this.speciesHash.get(acc) != null) {
                    ret.add(this.speciesHash.get(acc));
                }
            }
        }
        if (!ret.isEmpty()) {
            return ret;
        }

        //if no DBSpecies hit
        //we next look for a recent type of CollectionSpecies 
        String gcAccession = null;
        try {
            gcAccession = this.getGCAssemblyAccessionForAlias(alias_or_name);
            out = (gcAccession != null) ? this.collectionspeciesHashByGCAccession.get(gcAccession)
                    : null;
            if (out != null) {
                ret.add(out);
            } else {
                //if the alias hasn't pulled out a recent CollectionSpecies = fall back to look for an older one
                name = this.getEnsemblNameForAlias(alias_or_name);
                out = (name != null) ? this.oldstyle_collectionspeciesHashByDBName.get(name) : null;
                if (out != null) {
                    ret.add(out);
                }
            }
        } catch (NonUniqueException ex) {
            for (String acc : (Collection<String>) ex.getAllHits()) {
                if (collectionspeciesHashByGCAccession.get(acc) != null) {
                    ret.add(this.collectionspeciesHashByGCAccession.get(acc));
                }
            }
            if (ret.isEmpty()) {
                //if the alias doesn't pulled out recent CollectionSpecies = fall back to look for older ones
                for (String acc : (Collection<String>) ex.getAllHits()) {
                    if (this.oldstyle_collectionspeciesHashByDBName.get(acc) != null) {
                        ret.add(this.oldstyle_collectionspeciesHashByDBName.get(acc));
                    }
                }
            }
        }
        return ret;
    }

    /**
     * Returns a HashSet<DBSpecies> of all DBSpecies and old and newstyle
     * CollectionSpecies matching the given alias. If name is not a valid
     * String, returns null, if no species are found, returns empty Set.
     *
     * @param alias_or_name
     * @return a Collection<? extends Species> of DBSpecies.
     */
    @Override
    public Collection<? extends Species> getSetOfEverySpeciesByAlias(String alias_or_name) {
        HashSet<DBSpecies> ret = new HashSet<DBSpecies>();

        if (alias_or_name == null || alias_or_name.isEmpty()) {
            return null;
        }

        DBSpecies out = null;

        String name = null;
        try {
            name = this.getEnsemblNameForAlias(alias_or_name);
            out = (name != null) ? this.speciesHash.get(name) : null;
            if (out != null) {
                ret.add(out);
            }
            out = (name != null) ? this.oldstyle_collectionspeciesHashByDBName.get(name)
                    : null;
            if (out != null) {
                ret.add(out);
            }
        } catch (NonUniqueException ex) {
            for (String acc : (Collection<String>) ex.getAllHits()) {
                if (this.speciesHash.get(acc) != null) {
                    ret.add(this.speciesHash.get(acc));
                }
                if (this.oldstyle_collectionspeciesHashByDBName.get(acc) != null) {
                    ret.add(this.oldstyle_collectionspeciesHashByDBName.get(acc));
                }
            }
        }

        String gcAccession = null;
        try {
            gcAccession = this.getGCAssemblyAccessionForAlias(alias_or_name);
            out = (gcAccession != null) ? this.collectionspeciesHashByGCAccession.get(gcAccession)
                    : null;
            if (out != null) {
                ret.add(out);
            }
        } catch (NonUniqueException ex) {
            for (String acc : (Collection<String>) ex.getAllHits()) {
                if (collectionspeciesHashByGCAccession.get(acc) != null) {
                    ret.add(this.collectionspeciesHashByGCAccession.get(acc));
                }
            }
        }
        return ret;
    }

    /**
     * Wraps getSetOfSpeciesByAlias(String alias_or_name) to return a single
     * species if the returned Collection has only one member. If no species are
     * found, returns null. If more than one species is found, it throws a
     * NonUniqueException which holds the HashSet<DBSpecies> of results. Note if
     * we hit a CollectionSpecies - more recent Species with GCA (ENA)
     * Identifiers will be returned in preference to older Species lacking IDs.
     *
     * @param alias_or_name
     * @return
     * @throws NonUniqueException
     */
    @Override
    public DBSpecies getSpeciesByAlias(String alias_or_name) throws NonUniqueException {
        HashSet<DBSpecies> ret = (HashSet<DBSpecies>) this.getSetOfSpeciesByAlias(alias_or_name);

        if (ret == null || ret.isEmpty()) {
            return null;
        }

        if (ret.size() == 1) {
            return ret.iterator().next();
        } else {
            throw new NonUniqueException("More than one species with this alias is found. "
                    + "You can examine the list of species by printing for (Object o: NUN.getAllHits() { "
                    + "System.out.println(((Species) o).getDatabaseStyleName()); }", ret);
        }

    }

    /**
     * Wraps getSetOfEverySpeciesByAlias(String alias_or_name) to return a
     * single species if the returned Collection has only one member with the
     * desired alias and version. If no species are found, returns null. If more
     * than one species is found for the datasource version, it throws a
     * NonUniqueException which holds the HashSet<DBSpecies> of results.
     *
     * @param alias_or_name
     * @return
     * @throws NonUniqueException
     */
    @Override
    public DBSpecies getSpeciesByAlias(String alias_or_name, String version) throws NonUniqueException {
        HashSet<DBSpecies> ret = (HashSet<DBSpecies>) this.getSetOfEverySpeciesByAlias(alias_or_name);
        HashSet<DBSpecies> versions = new HashSet<DBSpecies>();

        if (ret == null || ret.isEmpty()) {
            return null;
        } else if (ret.size() == 1) {
            DBSpecies o = ret.iterator().next();
            if (o.getDBVersions().contains(version)) {
                return o;
            } else {
                return null;
            }
        } else {
            for (DBSpecies sp : ret) {
                if (sp.getDBVersions().contains(version)) {
                    versions.add(sp);
                }
            }
            if (versions.isEmpty()) {
                return null;
            } else if (versions.size() == 1) {
                return versions.iterator().next();
            } else if (versions.size() > 1) {
                throw new NonUniqueException("More than one species with alias '"
                        + alias_or_name + "' is found for version '" + version + "' of the datasource. "
                        + "You can examine the list of species by printing for (Object o: NUN.getAllHits() { "
                        + "System.out.println(((Species) o).getDatabaseStyleName()); }", versions);
            }
        }
        return null;
    }

    /**
     * Returns the most recent Species matching the alias. Return null if no
     * match, throws NonUniqueException which holds the HashSet<DBSpecies> of
     * results if more than one, equally recent species is found.
     *
     * @param alias_or_name
     * @return
     * @throws NonUniqueException
     */
    @Override
    public DBSpecies getMostRecentSpeciesByAlias(String alias_or_name) throws NonUniqueException {

        HashSet<DBSpecies> temp = (HashSet<DBSpecies>) this.getSetOfEverySpeciesByAlias(alias_or_name);
        TreeMap<Integer, HashSet<DBSpecies>> versionMap = new TreeMap<Integer, HashSet<DBSpecies>>();

        HashSet<DBSpecies> out = null;

        if (temp == null || temp.isEmpty()) {
            return null;
        }

        if (temp.size() == 1) {
            return temp.iterator().next();
        } else {
            for (DBSpecies sp : temp) {

                if (!versionMap.containsKey(sp.getHighestDBRelease())) {
                    versionMap.put(sp.getHighestDBRelease(), new HashSet<DBSpecies>());
                }
                versionMap.get(sp.getHighestDBRelease()).add(sp);
            }
        }

        out = versionMap.lastEntry().getValue();

        if (out.size() == 1) {
            return out.iterator().next();
        } else {
            throw new NonUniqueException("More than one species with this alias is found. "
                    + "You can examine the list of species by printing for (Object o: NUN.getAllHits() { "
                    + "System.out.println(((Species) o).getDatabaseStyleName()); }", out);
        }

    }

    @Override
    public DBSpecies getSpeciesByEnsemblName(String ensemblName) {
        if (ensemblName == null || ensemblName.isEmpty()) {
            return null;
        }

        DBSpecies out = null;
        out = this.speciesHash.get(ensemblName);
        if (out == null) {
            out = this.oldstyle_collectionspeciesHashByDBName.get(ensemblName);
        }

        return out;
    }

    public DBCollectionSpecies getCSpeciesByGCAccessionStem(String key) {
        if (key == null || key.isEmpty()) {
            return null;
        }
        return this.collectionspeciesHashByGCAccession.get(key);
    }

    @Override
    public DBCollection getMostRecentCollection(String name) {
        return this.getCollection(name, null);
    }

    @Override
    public DBCollection getCollection(String name, String dbVersion) {
        DBCollection out = null;

        if (name == null || name.isEmpty()) {
            return out;
        }

        if (dbVersion == null || dbVersion.isEmpty()) {
            if (this.databaseVersions.containsKey(name)) {
                dbVersion = this.databaseVersions.get(name).last().toString();
            }
        }

        if (dbVersion == null || dbVersion.isEmpty()) {
            return out;
        }

        if (this.collectionNameDBVersionHash.containsKey(name)) {
            if (this.collectionNameDBVersionHash.get(name).containsKey(dbVersion)) {
                out = this.collectionNameDBVersionHash.get(name).get(dbVersion);
            }
        }
        return out;
    }

    private void registerDatabase(DBDatabase db) {
        if (db instanceof DBCollectionDatabase) {
            this.collectionDatabases.add((DBCollectionDatabase) db);
            if (db.intSchemaVersion > this.highestEnsemblSchemaVersion) {
                this.highestEnsemblSchemaVersion = db.intSchemaVersion;
            }
            if (db.intDBVersion > this.highestReleaseVersion) {
                this.highestReleaseVersion = db.intDBVersion;
            }
        } else if (db instanceof DBSingleSpeciesDatabase) {
            this.singleSpeciesDatabases.add((DBSingleSpeciesDatabase) db);
            if (db.intSchemaVersion > this.highestEnsemblSchemaVersion) {
                this.highestEnsemblSchemaVersion = db.intSchemaVersion;
            }
            if (db.intDBVersion > this.highestReleaseVersion) {
                this.highestReleaseVersion = db.intDBVersion;
            }
        } else if (db instanceof DBComparisonDatabase) {
            this.comparisonDatabases.add((DBComparisonDatabase) db);

            if (db.getType().equals(EnsemblDBType.compara)) {

                if (!this.comparaDBByDivision.containsKey(((DBComparisonDatabase) db).getComparisonDivision())) {
                    this.comparaDBByDivision.put(((DBComparisonDatabase) db).getComparisonDivision(),
                            new HashMap<String, DBComparisonDatabase>());
                }
                this.comparaDBByDivision.get(((DBComparisonDatabase) db).getComparisonDivision()).put(
                        db.getDBVersion(), (DBComparisonDatabase) db);

            }

        }

        //registering versions of this database
        if (!this.databaseVersions.containsKey(db.getdBClassifier())) {
            this.databaseVersions.put(db.getdBClassifier(), new TreeSet<Integer>());
        }
        this.databaseVersions.get(db.getdBClassifier()).add(db.getIntDBVersion());

        this.allDatabases.add(db);
    }

    private void pseudoRegisterDatabase(DBDatabase db) {
        if (db instanceof DBCollectionDatabase
                || db instanceof DBSingleSpeciesDatabase) {

            if (db.intSchemaVersion > this.highestEnsemblSchemaVersion) {
                this.highestEnsemblSchemaVersion = db.intSchemaVersion;
            }
            if (db.intDBVersion > this.highestReleaseVersion) {
                this.highestReleaseVersion = db.intDBVersion;
            }
        }
    }

    @Override
    public Collection<DBCollection> getCollectionRegistriesByName(String name) {
        if (name == null || name.isEmpty()) {
            return new ArrayList<DBCollection>();
        }
        if (collectionNameDBVersionHash.containsKey(name)) {
            return new ArrayList<DBCollection>(collectionNameDBVersionHash.get(name).values());
        } else {
            return new ArrayList<DBCollection>();
        }
    }

    @Override
    public Collection<DBCollection> getCollectionsByDBVersion(String dbVersion) {

        Collection<DBCollection> out = new ArrayList<DBCollection>();
        if (dbVersion == null || dbVersion.isEmpty()) {
            return out;
        }

        for (String name : this.collectionNameDBVersionHash.keySet()) {

            if (this.collectionNameDBVersionHash.get(name).containsKey(dbVersion)) {
                out.add(this.collectionNameDBVersionHash.get(name).get(dbVersion));
            }
        }
        return out;
    }

    @Override
    public List<DBCollection> getCollections() {
        return collections;
    }

    /**
     * Fetches the list of all DBSpecies objects for this Registry.
     *
     * @return
     */
    @Override
    public List<DBSpecies> getSpecies() {
        return new ArrayList<DBSpecies>(speciesHash.values());
    }

    /**
     * Returns the combined list of DBCollectionSpecies that are indexed by
     * dbstylename (old_style_ and those indexed by GCAssemblyAccession (new
     * style, post v16)
     *
     * @return
     */
    @Override
    public List<DBCollectionSpecies> getCollectionSpecies() {

        ArrayList<DBCollectionSpecies> out = new ArrayList<DBCollectionSpecies>(oldstyle_collectionspeciesHashByDBName.values());
        out.addAll(collectionspeciesHashByGCAccession.values());
        return out;
    }

    /**
     * Utility method to return only those DBCOllectionSpecies that possess and
     * are indexed by a GCAssemblyAccession(Stem or Chain)
     *
     * @return
     */
    public List<DBCollectionSpecies> getGCAccessionedCollectionSpecies() {
        return new ArrayList<DBCollectionSpecies>(collectionspeciesHashByGCAccession.values());
    }

    /**
     * Utility method to return only those DBCOllectionSpecies that do not
     * possess and are therefore not indexed by a GCAssemblyAccession but bey
     * dbstylename
     *
     * @return
     */
    public List<DBCollectionSpecies> getNamedCollectionSpecies() {
        return new ArrayList<DBCollectionSpecies>(oldstyle_collectionspeciesHashByDBName.values());
    }

    @Override
    public DataSource getDatasourceType() {
        return datasourceType;
    }

    @Override
    public DBDatabase makeDatabase(String db_name) throws ConfigurationException {

        if (db_name == null || db_name.isEmpty()) {
            throw new ConfigurationException("Invalid database name: " + db_name);
        }

        EnsemblDBType t = null;

        for (EnsemblDBType et : EnsemblDBType.getAllDatabaseTypes()) {

            if (et.toString().startsWith("collection")
                    && db_name.contains("_collection_")
                    && db_name.matches(new String(".+_" + et.toString() + "_\\d+.*"))) {
                t = et;
                break;
            } else if (db_name.matches(new String(".+_" + et.toString() + "_\\d+.*"))
                    || (et.toString().startsWith("ensembl_") && db_name.contains(et.toString() + "_"))) {
                t = et;
            }
        }

        if (t == null) {
            throw new ConfigurationException("Unrecognized database type for database name: " + db_name);
        }

        try {
            if (EnsemblDBType.getSingleSpeciesDatabaseTypes().contains(t)) {
                if (t.equals(EnsemblDBType.core)) {
                    return new DBSingleSpeciesCoreDatabase(db_name, t, this);
                } else if (t.equals(EnsemblDBType.variation)) {
                    return new DBSingleSpeciesVariationDatabase(db_name, t, this);
                } else {
                    //haven't implemented othersubtypes yet
                    return new DBSingleSpeciesDatabase(db_name, t, this);
                }
            } else if (EnsemblDBType.getCollectionDatabaseTypes().contains(t)) {
                if (t.equals(EnsemblDBType.collection_core)) {
                    return new DBCollectionCoreDatabase(db_name, t, this);
// to implement in the unlikely event we have collections with variation databases                    
//                } else if (t.equals(EnsemblDBType.collection_variation)) {
//                    return new DBCollectionVariationDatabase(db_name, t, this);
                } else {
                    //haven't implemented other subtypes yet
                    return new DBCollectionDatabase(db_name, t, this);
                }
            } else if (EnsemblDBType.getSpeciesComparisonDatabaseTypes().contains(t)) {
                return new DBComparisonDatabase(db_name, t, this);
            } else {
                //shouldnt hit here
                throw new Exception("Unrecognized database type for database name: " + db_name);
            }
        } catch (Exception e) {
            throw new ConfigurationException(e.getMessage());
        }

    }

    //DAO methods
    private List<String> getDatabases() throws DAOException {

        List<String> outList = null;
        SqlSession session = null;

        try {
            session = sqlMapper.openSession();
            DatabaseMapper dm = session.getMapper(DatabaseMapper.class);
            outList = dm.getAllDatabaseNames();
        } catch (Exception e) {
            throw new DAOException("Failed to interrogate all Database Names", e);
        } finally {
            if (session != null) {
                session.close();
            }
        }

        if (outList == null) {
            /* A null tempList is a perfectly valid outcome to return */
            return new ArrayList<String>();
        }

        // if we want to test parsing of databse names we can add them here..
        /*
         outList.add("ictidomys_tridecemlineatus_variation_68_2");
         outList.add("spermophilus_tridecemlineatus_variation_67_2");
         */
        return outList;
    }

    private DBCollection getCollectionSpeciesProperties(CollectionCoreDatabase database) throws DAOException {

        if (database == null) {
            return null;
        }
        DBCollection collection = null;
        DBCollectionCoreDatabase myDatabase = (DBCollectionCoreDatabase) database;
        String dBName = myDatabase.getdBName();
        String thisSchemaVersion = myDatabase.getSchemaVersion();
        String dbVersion = myDatabase.getDBVersion();
        TreeMap<Integer, DBCollectionSpecies> localSpp = new TreeMap<Integer, DBCollectionSpecies>();
        collection = new DBCollection(myDatabase);

        List<HashMap> results = null;
        SqlSession session = null;

        try {
            session = sqlMapper.openSession();
            DatabaseMapper dm = session.getMapper(DatabaseMapper.class);
            results = dm.getSpeciesFromCollection(dBName);
        } catch (Exception e) {
            throw new DAOException("Failed to interrogate species properties for " + dBName, e);
        } finally {
            if (session != null) {
                session.close();
            }
        }

        if ((results == null) || (results.isEmpty() == true)) {
            /*  return empty collectiont*/
            return collection;
        }

        HashMap<Integer, HashMap<String, String>> buildLevelsHash = new HashMap<Integer, HashMap<String, String>>();

        for (Object o : results) {

            Integer id = null;
            DBCollectionSpecies species = null;

            HashMap m = (HashMap) o;
            m.put("schemaVersion", thisSchemaVersion);
            m.put("dbVersion", dbVersion);

            id = (Integer) m.get("species_id");

            // if we've already made species with this id  for this collection
            //use it
            if (localSpp.containsKey(id)) {
                species = localSpp.get(id);
                //and add this database
                //i think this is redundant - it should already be set...
                species.addDatabase(myDatabase);
            } //we haven't got a species with this id in the collection already
            //- so make one, add it, and use it
            else {
                try {
                    species = new DBCollectionSpecies(myDatabase);
                } catch (ConfigurationException ex) {
                    //what do we do here??
                }
                if (species == null) {
                    continue;
                }
                species.setIDForVersion(id, dbVersion);
                localSpp.put(id, species);
            }

            if (!((String) m.get("meta_key")).endsWith("build.level")) {
                species.setProperty(m);
            } else {

                if (buildLevelsHash.containsKey(id)) {
                    buildLevelsHash.get(id).put((String) m.get("meta_key"), (String) m.get("meta_value"));
                } else {
                    buildLevelsHash.put(id, new HashMap<String, String>());
                    buildLevelsHash.get(id).put((String) m.get("meta_key"), (String) m.get("meta_value"));
                }
            }
        }
        
        //loop through all the species made for this collection database
        //this adds them to one of two maps in the registry - by GCAccession or by db_style_name (production_name)
        for (DBCollectionSpecies s : localSpp.values()) {

            // there may be species without dbstyle names...
            if (s.getDatabaseStyleName() == null
                    || s.getDatabaseStyleName().isEmpty()) {
                s.setDatabaseStyleSpeciesName(s.getSpeciesBinomial().toLowerCase().replace(' ', '_'));
                s.setDatabaseStyleSpeciesName(Integer.parseInt(dbVersion), s.getDatabaseStyleName());
            }

            //this cant be used cos we havent populated the alias look up yet!
            //if (this.registry.getSpeciesByAlias(s.getDatabaseStyleName()) == null) {
            //ensembl have changed the case of the first letter of the dbname between v58 & v59
            //therefore the code below fails to rationalize species between collections 58 and 59
            //i am just going to live with this for the time being bcause the dbname is also used as the comparaname
            //and is also different between 58 and 59
            //hopefully from 59 things will be stable
            //give this to the reg
            //currently we are hashing on the production name ( databasestylename)
            //but i may need to alter this if we want continuity after release 16
            // if this species has a GCAccession - use it for the hashing
            if (s.getAssemblyAccession(dbVersion) != null
                    && !s.getAssemblyAccession(dbVersion).isEmpty()) {

                String accession = s.getAssemblyAccession(dbVersion);
                String stem = accession.split("\\.")[0];
                if (this.getCSpeciesByGCAccessionStem(stem) == null) {
                    //we've made a new species so add it to the registry
                    //but to the hash on GCAccessionStem
                    this.collectionspeciesHashByGCAccession.put(stem, s);
                    //and add it to this collection
                    collection.addSpecies(s);
                } else {
                    //we've already got this species in the registry - so modify it there 
                    DBCollectionSpecies existingSpecies = this.getCSpeciesByGCAccessionStem(stem);

                    Integer existingVersion = existingSpecies.getHighestDBRelease();
                    if (Integer.parseInt(dbVersion) > existingVersion) {
                        //reset any values that should be global for the species
                        existingSpecies.setCommonName(s.getCommonName());
                        existingSpecies.setShortName(s.getShortName());
                        existingSpecies.setSpeciesBinomial(s.getSpeciesBinomial());
                        existingSpecies.setUrlName(s.getUrlName());
                        existingSpecies.setComparaDivision(s.getComparaDivision());
                        existingSpecies.setEnsemblStablePrefix(s.getEnsemblStablePrefix());
                        existingSpecies.setTaxonomyID(s.getTaxonomyID());

                        // the dbstylename is used for the TreeSet ordering - so we have a problem if it changes
                        //so we dont change this here! - but add a finalize step to set the most recent version
                        //existingSpecies.setDatabaseStyleSpeciesName(s.getDatabaseStyleName());
                    }

                    existingSpecies.getAliases().addAll(s.getAliases());
                    existingSpecies.setAssemblyAccession(dbVersion, accession);
                    existingSpecies.setAssemblyName(dbVersion, s.getAssemblyName(dbVersion));
                    existingSpecies.setComparaName(dbVersion, s.getComparaName(dbVersion));
                    existingSpecies.setDatabaseStyleSpeciesName(Integer.parseInt(dbVersion), s.getComparaName(dbVersion));
                    existingSpecies.setIDForVersion(s.getDBSpeciesID(dbVersion), dbVersion);
                    existingSpecies.addDatabase(myDatabase);
                    collection.addSpecies(existingSpecies);
                }
            } //we dont have a GCAccession - so use the db style name....
            else {
                if (this.getSpeciesByEnsemblName(s.getDatabaseStyleName()) == null) {
                    //we've made a new species so add it to the registry
                    //but to the hash on db-syle_name
                    this.oldstyle_collectionspeciesHashByDBName.put(s.getDatabaseStyleName(), s);
                    //and add it to this collection
                    collection.addSpecies(s);
                } else {
                    //we've already got this species in the registry - so modify it there
                    DBCollectionSpecies existingSpecies = (DBCollectionSpecies) this.getSpeciesByEnsemblName(s.getDatabaseStyleName());

                    Integer existingVersion = existingSpecies.getHighestDBRelease();
                    if (Integer.parseInt(dbVersion) > existingVersion) {
                        //reset any values that should be global for the species
                        existingSpecies.setCommonName(s.getCommonName());
                        existingSpecies.setShortName(s.getShortName());
                        existingSpecies.setSpeciesBinomial(s.getSpeciesBinomial());
                        existingSpecies.setUrlName(s.getUrlName());
                        existingSpecies.setComparaDivision(s.getComparaDivision());
                        existingSpecies.setEnsemblStablePrefix(s.getEnsemblStablePrefix());
                        existingSpecies.setTaxonomyID(s.getTaxonomyID());
                        // the dbstylename is used for the TreeSet ordering - so we have a problem if it changes
                        //so we dont change this here! - but add a finalize step
                        //existingSpecies.setDatabaseStyleSpeciesName(s.getDatabaseStyleName());
                    }

                    existingSpecies.getAliases().addAll(s.getAliases());
                    existingSpecies.setAssemblyName(dbVersion, s.getAssemblyName(dbVersion));
                    existingSpecies.setIDForVersion(s.getDBSpeciesID(dbVersion), dbVersion);
                    existingSpecies.setComparaName(dbVersion, s.getComparaName(dbVersion));
                    existingSpecies.setDatabaseStyleSpeciesName(Integer.parseInt(dbVersion), s.getComparaName(dbVersion));
                    existingSpecies.addDatabase(myDatabase);
                    //and add it to this collection
                    collection.addSpecies(existingSpecies);
                }
            }
        }

        myDatabase.setCollection(collection);

        //loop though all the species of the collection
        //to copy the build levels from here to the database
        for (DBSpecies s : collection.getSpecies()) {
            DBCollectionSpecies sp = (DBCollectionSpecies) s;
            if (buildLevelsHash.containsKey(sp.getDBSpeciesID(dbVersion))) {
                myDatabase.setBuildLevels(sp, buildLevelsHash.get(sp.getDBSpeciesID(dbVersion)));
            }
        }

        return collection;
    }

//    //not used yet
//    private List<String> getAllCoreDatabasesForVersion(int dbVersion) throws DAOException {
    //<editor-fold defaultstate="collapsed" desc="comment">
    //
    //        int release = dbVersion;
    //        List<String> outList = null;
    //        SqlSession session = null;
    //
    //        try {
    //            session = sqlMapper.openSession();
    //            DatabaseMapper dm = session.getMapper(DatabaseMapper.class);
    //            outList = dm.getAllCoreDatabaseNames(release);
    //        } catch (Exception e) {
    //            throw new DAOException("Failed to interrogate all core Database Names for release " + release, e);
    //        } finally {
    //            if (session != null) {
    //                session.close();
    //            }
    //        }
    //
    //        if (outList == null || outList.isEmpty()) {
    //            /* A null List is a perfectly valid outcome to return */
    //            return null;
    //        }
    //
    //        return outList;
    //    }
    //</editor-fold>
    private DBSpecies createSpecies(SingleSpeciesCoreDatabase database) throws DAOException {

        DBSpecies spData = new DBSpecies((DBSingleSpeciesCoreDatabase) database);
        Integer dbVersion = database.getIntDBVersion();
        List<HashMap> tempList = null;

        SqlSession session = null;

        try {
            session = sqlMapper.openSession();
            DatabaseMapper dm = session.getMapper(DatabaseMapper.class);
            tempList = dm.getSpeciesProperties(database.getdBName());
        } catch (Exception e) {
            throw new DAOException("Failed to interrogate species properties for " + spData.getDatabaseStyleName(), e);
        } finally {
            if (session != null) {
                session.close();
            }
        }

        //for pre dbVersion 59 releases, the meta table is not guaranteed
        //to hold the species.production_name, species.short_name and
        //species.scientific_name meta_keys
        //pre schemaVersion 59 the compara name was always = Species Binomial
        //post schemaVersion 58 the compara name - production name
        //post schema 67  The following meta keys are retired
        //species.short_name
        //species.ensembl_common_name
        //species.ensembl_alias_name
        //The following meta keys will be added
        //species.url
        //species.display_name
        if (tempList != null) {
            for (HashMap row : tempList) {

                String key = (String) row.get("key");
                String value = (String) row.get("value");

                if (key == null || key.isEmpty() || value == null) {
                    continue;
                }

                //making aliases all lower case now
                if (key.startsWith("species.")
                        && !key.equals("species.division")
                        && !key.equals("species.stable_id_prefix")) {
                    spData.getAliases().add(value.toLowerCase());
                }

                if (key.equals("species.stable_id_prefix")) {
                    spData.setEnsemblStablePrefix(value);
                } else if (key.equals("species.production_name")) {
                    spData.setDatabaseStyleSpeciesName(value);
                    spData.setDatabaseStyleSpeciesName(dbVersion, value);
                } else if (key.equals("species.scientific_name")) {
                    spData.setSpeciesBinomial(value);
                } else if (key.equals("species.common_name")) {
                    spData.setCommonName(value);
                } else if (key.equals("species.taxonomy_id")) {
                    spData.setTaxonomyID(value);
                } else if (key.equals("species.short_name")
                        || key.equals("species.display_name")) {
                    spData.setShortName(value);
                } else if (key.endsWith("build.level")) {
                    database.setBuildLevel(key, value);
                } else if (key.equals("species.division")) {
                    //this is only present in ensemblgenomes
                    spData.setComparaDivision(value);
                } else if (key.equals("species.url")) {
                    //the url name is set for the species from this most recent core db
                    spData.setUrlName(value);
                } else if (key.equals("assembly.name")) {
                    //assembly details are set in the database
                    //as that is where lazy loading is controlled
                    //(for releases other than highest - the values are lazy loaded not filled in here)
                    database.setAssemblyName(value);
                }
                if (key.equals("assembly.accession")) {
                    database.setAssemblyAccession(value);
                }
            }
        }

        if (spData.getSpeciesBinomial().isEmpty()) {
            String s = database.getDbSpeciesName();
            spData.setSpeciesBinomial(s.replaceFirst(s.substring(0, 1), (s.substring(0, 1)).toUpperCase()).replace("_", " "));
            spData.getAliases().add(spData.getSpeciesBinomial().toLowerCase());
        }

        if (spData.getDatabaseStyleName().isEmpty()) {
            spData.setDatabaseStyleSpeciesName(database.getDbSpeciesName());
            //actually not necessary if we are only running create species on the most 
            //recent single species core databases
            spData.setDatabaseStyleSpeciesName(dbVersion, database.getDbSpeciesName());
            spData.getAliases().add(spData.getDatabaseStyleName().toLowerCase());
        }

        if (Integer.parseInt(database.getSchemaVersion()) >= 59) {
            spData.setComparaName(database.getDBVersion(), spData.getDatabaseStyleName());
        } else {
            spData.setComparaName(database.getDBVersion(), spData.getSpeciesBinomial());
        }

        return spData;
    }

    /**
     * should only be called for non current singlespecies core databases
     *
     * @param db
     * @throws DAOException
     */
    @Override
    public void setSpeciesMetadata(CoreDatabase db) throws DAOException {

        if (db == null || db.getdBName() == null || db.getdBName().isEmpty()) {
            return;
        }

        if (db instanceof CollectionDatabase) {

            CollectionCoreDatabase database = (CollectionCoreDatabase) db;
            TreeSet<? extends CollectionSpecies> spp = database.getCollection().getSpecies();
            HashMap<Integer, DBCollectionSpecies> localSpp = new HashMap<Integer, DBCollectionSpecies>();

            for (CollectionSpecies sp : spp) {
                localSpp.put(sp.getDBSpeciesID(database.getDBVersion()), (DBCollectionSpecies) sp);
            }

            String dBName = database.getdBName();
            List<HashMap> results = null;
            SqlSession session = null;

            try {
                session = sqlMapper.openSession();
                DatabaseMapper dm = session.getMapper(DatabaseMapper.class);
                results = dm.getSpeciesFromCollection(dBName);
            } catch (Exception e) {
                throw new DAOException("Failed to set species metadata  for " + dBName, e);
            } finally {
                if (session != null) {
                    session.close();
                }
            }

            if ((results == null) || (results.isEmpty() == true)) {
                return;
            }

            for (Object o : results) {
                Integer id = null;
                DBCollectionSpecies species = null;
                HashMap m = (HashMap) o;
                id = (Integer) m.get("species_id");

                // if we've already made species with this id  for this collection
                //use it
                if (localSpp.containsKey(id)) {
                    species = localSpp.get(id);
                } else {
                    continue;
                }

//                if (m.get("meta_key").equals("assembly.accession")) {
//                    database.setAssemblyAccession(species, (String) m.get("meta_value"));
//                }
//                if (m.get("meta_key").equals("assembly.name")) {
//                    database.setAssemblyName(species, (String) m.get("meta_value"));
//                }
            }
        } else {
            SingleSpeciesCoreDatabase database = (SingleSpeciesCoreDatabase) db;
            String comparaName = "";
            DBSpecies sp = (DBSpecies) database.getSpecies();
            String dBName = database.getdBName();
            List<HashMap> results = null;
            SqlSession session = null;

            try {
                session = sqlMapper.openSession();
                DatabaseMapper dm = session.getMapper(DatabaseMapper.class);
                results = dm.getSpeciesProperties(dBName);
            } catch (Exception e) {
                throw new DAOException("Failed to set species metadata for " + dBName, e);
            } finally {
                if (session != null) {
                    session.close();
                }
            }

            if ((results == null) || (results.isEmpty() == true)) {
                return;
            }

            for (HashMap row : results) {
                String key = (String) row.get("key");
                String value = (String) row.get("value");
                if (key == null || key.isEmpty() || value == null) {
                    continue;
                }
                if (key.equals("species.production_name")) {
                    comparaName = value;
                } else if (key.endsWith("build.level")) {
                    database.setBuildLevel(key, value);
                } else if (key.equals("assembly.name")) {
                    //assembly details are set in the database
                    //as that is where lazy loading is controlled
                    database.setAssemblyName(value);
                } else if (key.equals("assembly.accession")) {
                    database.setAssemblyAccession(value);
                }
            }

            //for pre dbVersion 59 releases, the meta table is not guaranteed
            //to hold the species.production_name, species.short_name and
            //species.scientific_name meta_keys
            //pre schemaVersion 59 the compara name was always = Species Binomial
            //post schemaVersion 58 the compara name - production name
            //post schema 67  The following meta keys are retired
            //species.short_name //species.ensembl_common_name //species.ensembl_alias_name
            //The following meta keys will be added
            //species.url //species.display_name
            if (Integer.parseInt(database.getSchemaVersion()) >= 59) {
                database.setComparaName(comparaName);
            } else {
                database.setComparaName(sp.getSpeciesBinomial());
            }
        }
    }

    /**
     * note by default translation by this default engine is configured to
     * convert non-Methionine initiations to Methionine
     *
     */
    @Override
    public TranscriptionEngine getDefaultTranscriptionEngine() {
        return transcriptionEngines.get(1);
    }

    @Override
    public TranscriptionEngine getTranscriptionEngine(Integer codonTable) {

        /* valid codes are
         * 1 - UNIVERSAL
         * 2 - VERTEBRATE_MITOCHONDRIAL
         * 3 - YEAST_MITOCHONDRIAL
         * 4 - MOLD_MITOCHONDRIAL
         * 5 - INVERTEBRATE_MITOCHONDRIAL
         * 6 - CILIATE_NUCLEAR
         * 9 - ECHINODERM_MITOCHONDRIAL
         * 10 - EUPLOTID_NUCLEAR
         * 11 - BACTERIAL
         * 12 - ALTERNATIVE_YEAST_NUCLEAR
         * 13 - ASCIDIAN_MITOCHONDRIAL
         * 14 - FLATWORM_MITOCHONDRIAL
         * 15 - BLEPHARISMA_MACRONUCLEAR
         * 16 - 2CHLOROPHYCEAN_MITOCHONDRIAL
         * 21 - TREMATODE_MITOCHONDRIAL
         * 23 - SCENEDESMUS_MITOCHONDRIAL 
         */
        //using an invalid code will use the default universal table
        if (transcriptionEngines.containsKey(codonTable)) {
            return transcriptionEngines.get(codonTable);
        }
        TranscriptionEngine.Builder b = new TranscriptionEngine.Builder();
        b.table(codonTable);
        TranscriptionEngine engine = b.build();
        transcriptionEngines.put(codonTable, engine);
        return engine;

    }

    @Override
    public HashMap<String, String> getRenamedDBs() {
        return renamedDBs;
    }

    HashMap<String, Set<DBSpecies>> panComparaSpecies = new HashMap<String, Set<DBSpecies>>();

    @Override
    public boolean isSpeciesInPanHomology(Species querySpecies, String dbVersion) {

        if (querySpecies == null) {
            return false;
        }

        if (dbVersion == null || dbVersion.isEmpty()) {
            if (querySpecies.getMostRecentCoreDatabase() != null) {
                dbVersion = querySpecies.getMostRecentCoreDatabase().getDBVersion();
            }
        }

        if (dbVersion == null || dbVersion.isEmpty() || dbVersion.equals("0")) {
            return false;
        }

        if (!panComparaSpecies.containsKey(dbVersion)) {
            try {
                findPanHomologySpecies(dbVersion);
            } catch (DAOException ex) {
                return false;
            }
        }

        return panComparaSpecies.get(dbVersion).contains(querySpecies);
    }

    public Set<DBSpecies> getPanComparaSpecies(String dbVersion) {

        if (dbVersion == null || dbVersion.isEmpty()) {
            dbVersion = "" + this.highestReleaseVersion;
        }
        if (dbVersion == null || dbVersion.isEmpty() || dbVersion.equals("0")) {
            return null;
        }

        if (!panComparaSpecies.containsKey(dbVersion)) {
            try {
                findPanHomologySpecies(dbVersion);
            } catch (DAOException ex) {
                //dont fail here
            }
        }
        return panComparaSpecies.get(dbVersion);
    }

    private void findPanHomologySpecies(String version) throws DAOException {

        Database db = this.getComparaDatabase(EnsemblComparaDivision.PAN_HOMOLOGY, version);

        if (db == null) {
            panComparaSpecies.put(version, new HashSet<DBSpecies>());
            return;
        }

        HashSet<DBSpecies> spp = new HashSet<DBSpecies>();
        List<HashMap> results = null;
        SqlSession session = null;

        try {
            session = sqlMapper.openSession();
            DatabaseMapper dm = session.getMapper(DatabaseMapper.class);
            results = dm.getPanComparaSpecies(db.getdBName());
        } catch (Exception e) {
            LOGGER.debug("Failed to interrogate species in Pan Compara DB for " + db);
            //throw new DAOException("Failed to interrogate species in Pan Compara DB for " + db, e);
        } finally {
            if (session != null) {
                session.close();
            }
        }

        if ((results == null) || (results.isEmpty() == true)) {
            panComparaSpecies.put(version, new HashSet<DBSpecies>());
            return;
        }

        for (HashMap h : results) {

            String alias = (String) h.get("name");
            //not sure i can use this yet
            String assembly = (String) h.get("assembly");

            DBSpecies sp = null;

            try {
                sp = this.getSpeciesByAlias(alias, version);
                if (sp != null) {
                    spp.add(sp);
                }
            } catch (NonUniqueException ex) {
                LOGGER.debug(ex.getMessage());
                for (Object s : ex.getAllHits()) {
                    LOGGER.debug(((DBSpecies) s).getDatabaseStyleName());
                }
            }
        }
        panComparaSpecies.put(version, spp);
    }

}