//This is a version of PagelMatrixModel that uses Putzer's method for matrix exponentation. //It seems to work correctly but is substantially slower than the released version //PEM 15 May 2006 package mesquite.correl.lib; import java.util.Random; //import org.jacorb.ir.gui.typesystem.remote.IRAlias; import mesquite.categ.lib.CategoricalDistribution; import mesquite.categ.lib.CategoricalState; import mesquite.cont.lib.EigenAnalysis; import mesquite.correl.Pagel94.Pagel94; import mesquite.lib.*; import mesquite.lib.characters.CLikelihoodCalculator; import mesquite.lib.characters.CharacterDistribution; import mesquite.lib.characters.CharacterModel; import mesquite.lib.characters.CharacterState; import mesquite.lib.characters.ProbabilityModel; import mesquite.stochchar.lib.MkModel; import Jama.Matrix; public class PagelMatrixModel extends MultipleProbCategCharModel implements Evaluator { //protected long allStates = 0L; private int assumedMaxState=1; // different model 'behaviors' here, control how parameters are constrained // if qConstrained[i] is true, params[i] is set to params[qBound[i]] unless i=-1 // otherwise qConstrained[i] is set to paramConstant - if this is MesquiteDouble.unassigned then error int [] qMapping; // mapping from known, underparameterized models; domain is set of parameters, range is qList double[] qList; // boolean[] qConstrained; // true if a q value has been constrained by user (currently unimplemented) int [] qBound; // qvalue bound to another qvalue (-1 = not bound) (currently unimplemented) double [] qConstant; // qvalue is set to this constant; (currently unimplemented) // These constants define model 'behaviors' public static final int NOMODEL = 0; // might be useful for flagging when a simpler model isn't available public static final int MODEL2PARAM = 1; public static final int MODEL4PARAM = 10; //Pagel's model with two independent characters public static final int MODEL8PARAM = 20; //Pagel's model with two dependent characters (correlated evolution) public static final int INDEPENDENTCHARACTERSMODEL = MODEL4PARAM; public static final int DEPENDENTCHARACTERSMODEL = MODEL8PARAM; public static final int MODEL7PARAMCONTINGENTCHANGEYFORWARD = 13; public static final int MODEL7PARAMCONTINGENTCHANGEYBACKWARD = 14; public static final int MODEL7PARAMCONTINGENTCHANGEXFORWARD = 15; public static final int MODEL7PARAMCONTINGENTCHANGEXBACKWARD = 16; public static final int MODEL6PARAMCONTINGENTCHANGEY = 17; public static final int MODEL6PARAMCONTINGENTCHANGEX = 18; public static final int MODEL2CHARACTERUSERDEFINED = 19; public static final int MODEL3CHARACTERINDEPENDENT = 100; public static final int MODEL4CHARACTERINDEPENDENT = 1000; private int modelType; //behavior of current model private double [] params; private boolean[] paramUnassigned; private double[][] probMatrix, rateMatrix, eigenVectors, inverseEigenVectors; private double[] eigenValues, imagEigenValues; //may use imagEigenValues with Putzer's method private double[][] tent; private boolean negativeProbabilities = false; private double negativeProbValue = 0.0; //private MesquiteInteger pos = new MesquiteInteger(0); private boolean recalcProbsNeeded = true; private boolean needToPrepareMatrices = true; private Tree workingTree; private CategoricalDistribution observedStates1; private CategoricalDistribution observedStates2; double[][] downProbs; // probability at node N given state s at base double[][][] downProbs2; // probabilty at node N given state s1 for char1 and s2 for char2 at base double[][] savedRootEstimates; // holds the estimates at the root (for simulation priors, etc.) double[] underflowCompDown; // ln of compensation for underflow //double[] empirical; // may be implemented sometime //double[][] empirical2; double[] parametersFromSimplerModel = null; //holds values from 4 parameter model as starting values for 6 p. model PagelMatrixModel intermediate1 = null; //holds 6 parameter models called from 8 parameter model PagelMatrixModel intermediate2 = null; int simplerModelType; private int numStates; long underflowCheckFrequency = 2; //how often to check that not about to underflow; 1 checks every time long underflowCheck = 1; MesquiteNumber minChecker; private MesquiteInteger eightParameterExtraSearch = new MesquiteInteger(5); private double previousBranchLength = MesquiteDouble.unassigned; //private boolean unassigned = true; public static final int SIMPLER_AND_FLIP_FLOP = 2; //conservative and slow public static final int BASE_ON_SIMPLER= 0; public static final int FLIP_FLOP = 1; public static final double FLIP_FLOP_LOW = 0.01; public static final double FLIP_FLOP_HIGH = 10.0; int optimizationMode = BASE_ON_SIMPLER; // Need to check this... static final int SUM = 0; // likelihood is just sum of likelihoods of each state at root (i.e., flat prior) //static final int FIX = 1; static final int SUBROOT_PRIOR = 1; //likelihood has stated prior; this is experimental and not yet available to the user interface static final int SUBROOT_EMPIRICAL = 2; //likelihood uses as prior the empirical frequencies of states in terminal taxa; this is experimental and not yet available to the user interface int rootMode = SUM; //what if anything is done with prior probabilities of states at subroot? Random rng = new Random(System.currentTimeMillis()); double[][] x = null; double[][] identity = null; double[][] p1 = null; // clone from identity? double[][] p2 = null; double[][] p3 = null; double[][] p4 = null; double[][] putzerT1 = null; double[][] putzerT2 = null; double[][] putzerT3 = null; double[] workingValues = null; double[] workingIValues = null; ProgressIndicator prog = null; //when this is non-null, check for abort condition /** * This returns the number of characters implied by the behavior code. It's static so it * can be used as an argument to the superconstructor. * @param behavior specifies what kind of model this will be * @return int specifies number of characters for the indicated behavior */ static int charsFromBehavior(int behavior){ return 2; // add cases when appropriate } /** * Constructs a new model. * @param behavior The behavior (number of parameters, constraints, etc.) the model will implement */ public PagelMatrixModel(String name, Class dataClass, int behavior) { super(name,dataClass,charsFromBehavior(behavior)); this.modelType = behavior; this.params=null; // model's parameters qConstrained=null; // true if a parameter is somehow constrained qBound=null; // parameter bound to another parameter (-1 = not bound) qConstant=null; // parameter is set to this value; tent= null; this.setConstraints(modelType); needToPrepareMatrices = true; recalcProbsNeeded = true; minChecker = new MesquiteNumber(MesquiteDouble.unassigned); if (modelType == MODEL4PARAM) optimizationMode = FLIP_FLOP; else optimizationMode = BASE_ON_SIMPLER; } /** * * @return true if any parameters are free (not bound or assigned) */ boolean anyUnassigned(){ boolean result = false; for(int i=0;imaxState[c] || state<0) return false; else return (CategoricalState.isElement(allStates[c], state)); } /* * recodes a pair of characters into a single state for the transition matrix index */ private int recodeStatePair(int i, int j) { //todo extend beyond binary characters? return (2*i+j); } private int[] pair0 = {0,0}; private int[] pair1 = {0,1}; private int[] pair2 = {1,0}; private int[] pair3 = {1,1}; private int[] statePairFromCode(int code){ switch (code){ case 0: return pair0; case 1: return pair1; case 2: return pair2; case 3: return pair3; } return null; } public double transitionProbability (int beginState[], int endState[], Tree tree, int node){ if ((beginState.length != numChars) || (endState.length != numChars)){ MesquiteMessage.warnProgrammer("Array for beginState or endState don't match models stated number of characters"); return 0; } boolean inStateFlag = true; for (int i=0;i end1,end2 */ private double transitionProbability (int beginState1,int endState1,int beginState2,int endState2,Tree tree, int node){ if (needToPrepareMatrices) prepareMatrices(); double branchLength = tree.getBranchLength(node,1.0); if (recalcProbsNeeded || previousBranchLength != branchLength) recalcProbabilities(branchLength); if (probMatrix ==null) return 0; //begin end reversed as in Asym2 return probMatrix[recodeStatePair(endState1,endState2)][recodeStatePair(beginState1,beginState2)]; } double[][] p, tprobMatrix; /** * Updates the transition probability matrix for a branch with the specified length * @param branchLength the length of the branch */ public void recalcProbabilities(double branchLength){ // if (unassigned) // return; previousBranchLength = branchLength; if (eigenValues == null) return; int evl = eigenValues.length; if (tent == null || tent.length != evl || tent[0].length != evl) { tent = new double[evl][evl]; Double2DArray.zeroArray(tent); } //for (int i=0;i largest) { largest = eigenValues[i]; largestIndex = i; } // check for a conjugate pair int imag1 = -1; int imag2 = -1; int realgroup1 = -1; int realgroup2 = -1; int realgroup3 = -1; for(int i=0;i -1){ eigenValueStatus = COMPLEXPAIR; for(int i= 0; i0 && result[0].length != numColumns)) result = new double[numRows][numColumns]; for(int i=0;i0 && result[0].length != numColumns1)) result = new double[numRows1][numColumns1]; Double2DArray.zeroArray(result); for(int i=0;i0 && result[0].length != numColumns1)) result = new double[numRows1][numColumns1]; for(int i=0;i= 4) { if (prog != null) if (prog.isAborted()) throw new StuckSearchException(); double height = workingTree.tallestPathAboveNode(workingTree.getRoot()); // to stop the optimization from wandering if very high rates if (!sanityChecks(values,limit, height)){ return MesquiteDouble.veryLargeNumber; } negativeProbabilities = false; params = DoubleArray.clone(values); prepareMatrices(); double result = -this.logLikelihoodCalc(workingTree); return result; } else return 0; } int estCount =0; boolean zeroHit = false; /*.................................................................................................................*/ public double logLikelihoodCalc(Tree tree){ if (zeroHit) return -0.001*(++estCount); // to shortcircuit the optimizer wandering around zero with very high rates if (tree == null) { MesquiteMessage.warnProgrammer("Tree passed to logLikelihoodCalc is null"); return(-1*MesquiteDouble.veryLargeNumber); } double likelihood = 0.0; double comp; int root = tree.getRoot(); if (rootMode == SUM){ initProbs2(tree.getNumNodeSpaces(), observedStates1.getMaxState()+1,observedStates2.getMaxState()+1); estCount++; downPass2(root,tree); for (int i = 0; i<=maxState[0]; i++) for (int j = 0; j<=maxState[1];j++) { savedRootEstimates[j][i] = downProbs2[root][j][i]; if (savedRootEstimates[j][i] < 0){ // problem, probably due to bad matrix operation likelihood = 0; return(-1*MesquiteDouble.veryLargeNumber); } else likelihood += savedRootEstimates[j][i]; } } //else // MesquiteMessage.println("Subroot prior and subroot empirical not yet supported"); //} //else if (rootMode == SUBROOT_PRIOR){ // if (tree.getRooted()) // for (int i=0; i -0.00001) { zeroHit = true; } return (logLike); } private void initProbs2(int nodes, int numStates1, int numStates2) { if (numStates1 != numStates2) { if (numStates1 >= numStates2) this.numStates = numStates1; else this.numStates = numStates2; } else if (numStates1 == 1) numStates =2; else numStates = numStates1; if (downProbs2==null || downProbs2.length!=nodes || downProbs2[0].length!=numStates1 || downProbs2[0][0].length!=numStates2){ downProbs2 = new double[nodes][numStates][numStates]; underflowCompDown = new double[nodes]; //empirical2 = new double[numStates][numStates]; savedRootEstimates = new double[numStates][numStates]; } //Double3DArray.zeroArray(downProbs2); for(int i=0;i=0 && obs1 < downProbs2[node].length && !CategoricalState.isUnassigned(observed1) && !CategoricalState.isInapplicable(observed1) && obs2>=0 && obs2 < downProbs2[node][0].length && !CategoricalState.isUnassigned(observed2) && !CategoricalState.isInapplicable(observed2)) { downProbs2[node][obs1][obs2] = 1; } } else { Double2DArray.zeroArray(downProbs2[node]); underflowCompDown[node]=0; for (int d= tree.firstDaughterOfNode(node); tree.nodeExists(d); d = tree.nextSisterOfNode(d)) { downPass2(d,tree); underflowCompDown[node] += underflowCompDown[d]; } for (int i=0;ilimit && allowedEdgeHits--<0) { // too big return false; } if (eParams[i]<=0){ return false; } } return true; } /** * This is really a stub that exists because there is no class for multivariate evaluators * */ public double evaluate(MesquiteDouble param, Object obj) { // TODO Maybe add a class for multivariate evaluators return 0; } //Wayne: this is here so you can use it in Pagel94 - feel free to change the default value public MesquiteInteger getExtraSearch() { return eightParameterExtraSearch; } /* * This just rescales the fixed parameters (as yet unimplemented) by rescale */ private void rescaleFixedParameters(double rescale){ for(int i=0;i= 1) maxState[0] = observedStates1.getMaxState(); else maxState[0] = 1; if (observedStates2.getMaxState()>= 1) maxState[1] = observedStates2.getMaxState(); else maxState[1] = 1; allStates[0] = CategoricalState.span(0,maxState[0]); allStates[1] = CategoricalState.span(0,maxState[1]); double height = 0; double invHeight = MesquiteDouble.unassigned; if (scaleRescale){ //rescales tree to 1.0 height to help optimization, which takes longer with larger numbers MesquiteTree mTree = originalTree.cloneTree(); mTree.setAllUnassignedBranchLengths(1.0, false); height = mTree.tallestPathAboveNode(mTree.getRoot(), 1.0); // to adjust start point depending on how tall is tree if (height != 0){ invHeight = 1.0/height; mTree.scaleAllBranchLengths(invHeight, false); rescaleFixedParameters(height); } workingTree= mTree; } else workingTree = originalTree; Optimizer opt = new Optimizer(this); double best = Double.MAX_VALUE; // **** estParams is the matrix of trial values double estParams[]; // estjust saves the starting values for error reporting double backupEst[] = null; double [] b = null; double next; switch (modelType) { case MODEL8PARAM:{ // first try two versions of a 6-parameter model // no need to reallocate the two 6-p models everytime if (intermediate1 == null) intermediate1 = new PagelMatrixModel("",CategoricalState.class,MODEL6PARAMCONTINGENTCHANGEY); if (intermediate2 == null) intermediate2 = new PagelMatrixModel("",CategoricalState.class,MODEL6PARAMCONTINGENTCHANGEX); PagelMatrixModel model6_cy = intermediate1; PagelMatrixModel model6_cx = intermediate2; estParams = new double[8]; // first a model with changes in Y contingent on changes in X if (parametersFromSimplerModel == null || simplerModelType != MODEL6PARAMCONTINGENTCHANGEY){ model6_cy.setParametersFromSimplerModel(parametersFromSimplerModel,simplerModelType); model6_cy.estimateParameters(workingTree,observedStates1, observedStates2, commandRec); double[] m6_1Params = model6_cy.getParams(); for(int i=0;i<4;i++) estParams[i]=m6_1Params[i]; estParams[4]=m6_1Params[4]; estParams[5]=m6_1Params[0]; estParams[6]=m6_1Params[5]; estParams[7]=m6_1Params[2]; } else { for(int i=0;i<4;i++) estParams[i]=parametersFromSimplerModel[i]; estParams[4]=parametersFromSimplerModel[4]; estParams[5]=parametersFromSimplerModel[0]; estParams[6]=parametersFromSimplerModel[5]; estParams[7]=parametersFromSimplerModel[2]; } allowedEdgeHits = beginningAllowed; //backupEst = (double [])estParams.clone(); next = opt.optimize(estParams, null); // try optimizing from the contingentchangeY best parameters if (!acceptableResults(next, estParams)) { MesquiteMessage.println("Warning: NaN encountered in PagelMatrixModel optimization"); reportUnacceptableValues(next, estParams,backupEst); } else { //shouldn't need to compare best with next here best = next; if (b == null) b = DoubleArray.clone(estParams); else for (int i=0; i< estParams.length; i++) b[i] = estParams[i]; } // now the other 6-parameter model (changes in X contingent on Y) if (parametersFromSimplerModel == null || simplerModelType != MODEL6PARAMCONTINGENTCHANGEX){ model6_cx.setParametersFromSimplerModel(parametersFromSimplerModel,simplerModelType); model6_cx.estimateParameters(workingTree,observedStates1, observedStates2, commandRec); double[] m6_2Params = model6_cx.getParams(); for(int i=0;i<4;i++) estParams[i]=m6_2Params[i]; estParams[4]=m6_2Params[1]; estParams[5]=m6_2Params[4]; estParams[6]=m6_2Params[3]; estParams[7]=m6_2Params[5]; } else { for(int i=0;i<4;i++) estParams[i]=parametersFromSimplerModel[i]; estParams[4]=parametersFromSimplerModel[1]; estParams[5]=parametersFromSimplerModel[4]; estParams[6]=parametersFromSimplerModel[3]; estParams[7]=parametersFromSimplerModel[5]; } allowedEdgeHits = beginningAllowed; //backupEst = (double [])estParams.clone(); next = opt.optimize(estParams, null); // try optimizing from the contingentchangeX best parameters if (!acceptableResults(next, estParams)) { MesquiteMessage.println("Warning: NaN encountered in PagelMatrixModel optimization"); reportUnacceptableValues(next, estParams,backupEst); } else if (next 0) { double [] m6_cy_Params = model6_cy.getParams(); double [] m6_cx_Params = model6_cx.getParams(); for (int i = 0; i< eightParameterExtraSearch.getValue();i++){ for(int j=0;j<4;j++) estParams[j]=Math.abs((m6_cy_Params[j]+m6_cx_Params[j])/2 + (20.0*rng.nextGaussian())*(m6_cx_Params[j]-m6_cy_Params[j])); estParams[4]=Math.abs(1+20.0*rng.nextGaussian())*m6_cy_Params[4]; estParams[5]=Math.abs(1+20.0*rng.nextGaussian())*m6_cx_Params[4]; estParams[6]=Math.abs(1+20.0*rng.nextGaussian())*m6_cy_Params[5]; estParams[7]=Math.abs(1+20.0*rng.nextGaussian())*m6_cx_Params[5]; allowedEdgeHits = beginningAllowed; //backupEst = (double [])estParams.clone(); next = opt.optimize(estParams, null); // try optimizing from the contingentchangeX best parameters if (!acceptableResults(next, estParams)) { MesquiteMessage.warnProgrammer("Warning: NaN encountered in PagelMatrixModel optimization"); reportUnacceptableValues(next, estParams,backupEst); } else if (next probMatrix[0].length) //TODO replace with appropriate test return null; double accumProb = 0; int resultCode = recodeStatePair(maxState[0],maxState[1]); // for (int i=0; i