v5.7.2

improved synteny algorithm

Author: cariann09

java/src/backend/AnchorMain.java

@@ -52,7 +52,8 @@ protected boolean run(Mproject pj1, Mproject pj2) throws Exception {
 			long startTime = Utils.getTimeMem();
 
 			String proj1Name = pj1.getDBName(), proj2Name = pj2.getDBName();
-			plog.msg("\nStart calculating cluster hits for " + proj1Name + " and " + proj2Name);// CAS568 nl
+			if (proj1Name.equals(proj2Name)) plog.msg("\nStart calculating cluster hits for " + proj1Name + " self synteny"); // CAS572 add
+			else plog.msg("\nStart calculating cluster hits for " + proj1Name + " and " + proj2Name);
 
 			String resultDir = Constants.getNameResultsDir(proj1Name, proj2Name); // e.g. data/seq_results/p1_to_p2
 			if ( !Utilities.pathExists(resultDir) ) {
@@ -75,26 +76,26 @@ protected boolean run(Mproject pj1, Mproject pj2) throws Exception {
 				AnchorMain2 an = new AnchorMain2(this);
 				boolean b = an.run(pj1, pj2, dh); if (!b) return false;
 			}
-			if (Cancelled.isCancelled()) {dbc2.close(); return false; } // CAS567 missing return!
+			if (Cancelled.isCancelled()) {dbc2.close(); return false; } 
 
 			saveHitNum();
 			saveAnnoHitCnt();
 
-			/** Run before pseudo; CAS565 moved from DoAlignSyn as does not need blocks**/
+			/** Run before pseudo **/
 			if (pj1.hasGenes() && pj2.hasGenes()) { 
 				AnchorPost collinear = new AnchorPost(pairIdx, pj1, pj2, dbc2, plog);
 				collinear.collinearSets();	
 			}
-			if (Cancelled.isCancelled()) {dbc2.close(); return false; } // CAS567 missing return!
+			if (Cancelled.isCancelled()) {dbc2.close(); return false; } 
 
-			// Numbers pseudo genes; must be after collinear; add CAS565
+			// Numbers pseudo genes; must be after collinear;
 			if (mp.isNumPseudo(Mpair.FILE)) new Pseudo().addPseudo(); 
 
 		/** finish **/	
 			long modDirDate = new File(resultDir).lastModified(); // add for Pair Summary with 'use existing files'
 			mp.setPairProp("pair_align_date", Utils.getDateStr(modDirDate));
 			if (!Constants.VERBOSE) Globals.rclear();
-			Utils.prtMsgTimeDone(plog,  "Finish clustering hits", startTime); // CAS568 change from mem output
+			Utils.prtMsgTimeDone(plog,  "Finish clustering hits", startTime); 
 			return true;
 		}
 		catch (Exception e) {ErrorReport.print(e, "Run load anchors"); return false;}		
@@ -106,7 +107,7 @@ protected boolean run(Mproject pj1, Mproject pj2) throws Exception {
 	 */
 	private boolean saveHitNum() { 
 	try {
-		Globals.rprt("Compute and save hit#"); // CAS568 do not need for Verbose
+		Globals.rprt("Compute and save hit#"); 
 		TreeMap <Integer, String> grpMap1 = mp.mProj1.getGrpIdxMap();
 		TreeMap <Integer, String> grpMap2 = mp.mProj2.getGrpIdxMap();
 
@@ -152,9 +153,9 @@ private boolean saveHitNum() {
 	 * Count number of hits per gene; these include all pairwise projects except self; 
 	 */
 	public void saveAnnoHitCnt() { // public for Mpair.removePairFromDB
-		Globals.rprt("Compute and save gene numHits"); // CAS568 do not need for Verbose
+		Globals.rprt("Compute and save gene numHits"); 
 
-		TreeMap <Integer, String> idxList1 = mp.mProj1.getGrpIdxMap(); // CAS546 was using SyProj.Group
+		TreeMap <Integer, String> idxList1 = mp.mProj1.getGrpIdxMap();
 		TreeMap <Integer, String> idxList2 = mp.mProj2.getGrpIdxMap();
 
 		for (int idx : idxList1.keySet()) if (!saveAnnotHitCnt(idx, idxList1.get(idx))) return;
@@ -185,8 +186,6 @@ public boolean saveAnnotHitCnt(int grpIdx, String grpName) { // public for Mproj
 			}
 			rs.close();
 
-			// CAS568 put in calling Globals.rprt(grpName + " Genes " + geneCntMap.size());
-			
 			PreparedStatement ps = dbc2.prepareStatement("update pseudo_annot set numhits=? where idx=?");
 			for (int idx : geneCntMap.keySet()) {
 				int num = geneCntMap.get(idx);
@@ -249,7 +248,7 @@ protected boolean addPseudoFromFlag() {
 	catch (Exception e) {ErrorReport.print(e, "Add pseudo from flag"); return false;}
 	}
 	/*******************************************************
-	 * Add so can be used in Cluster and Report as regular genes; CAS565
+	 * Add so can be used in Cluster and Report as regular genes;
 	 * Assigns sequentially along chromosome regardless of what the target chromosome is
 	 * 	which allows all numbers to be unique; but it may lead to gaps for a give chrQ-chrT
 	 */
@@ -331,9 +330,6 @@ private boolean addPseudoGrpGrp(int grpIdx, String where, String annoStr, int i,
 			String type = Globals.pseudoType;
 
 			// Must count pseudo too so as to not repeat pseudo numbers
-			// CAS570 would still duplicate pseudo, which worked as in different pairs, but confusing
-			//int genes = dbc2.executeCount("select count(*) from pseudo_annot where (type='gene' or type='pseudo') and grp_idx="+ grpIdx);
-			
 			int genes = dbc2.executeCount("select max(genenum) from pseudo_annot where grp_idx="+ grpIdx);
 			int geneStart = genes; 
 			if (genes==0) geneStart = 1;
@@ -368,14 +364,15 @@ private boolean addPseudoGrpGrp(int grpIdx, String where, String annoStr, int i,
 
 			String dn =  (i==0) ? mProj1.getDisplayName() : mProj2.getDisplayName();
 			String chr = (i==0) ? mProj1.getGrpNameFromIdx(grpIdx) : mProj2.getGrpNameFromIdx(grpIdx);
-		
-			String msg = String.format("Pseudo %-15s   Genes %,6d   Start %,6d ", (dn+" "+chr), genes, geneStart); 
-			if (Constants.VERBOSE && cntGrp==1) Utils.prtIndentNumMsgFile(plog, 1, hitGeneMap.size(), msg);  // CAS568 reduce output
+			String msg = String.format("Pseudo %-20s   Genes %,6d   Start %,6d ", (dn+" "+chr), genes, geneStart); 
+			/* too much output CAS572
+			if (Constants.VERBOSE && cntGrp==1) Utils.prtIndentNumMsgFile(plog, 1, hitGeneMap.size(), msg);  
 			if (Constants.VERBOSE && cntGrp==2) {
 				if (maxGrp>2) msg += "...";
 				Utils.prtIndentNumMsgFile(plog, 1, hitGeneMap.size(), msg);  
 			}
-			else Globals.rprt(String.format("%,5d %s", hitGeneMap.size(),msg));
+			else */
+			Globals.rprt(String.format("%,5d %s", hitGeneMap.size(),msg));
 
 			if (hitGeneMap.size()==0) return true;
 

java/src/backend/Constants.java

@@ -71,9 +71,7 @@ public class Constants {
 public static final String mumSuffix = 		".mum";
 public static final String doneSuffix = 	".done";
 public static final String selfPrefix = 	"self.";
-public static final String finalDir = 		"/final/";
-public static final String anchorFile = 	"anchors.txt";
-public static final String blockFile = 		"block.txt";
+public static final String finalDir = 		"/final/"; // has been discontinued; still exists in order to Remove
 
 public static final String projTo = 		"_to_";
 public static final String faFile =         ".fa";

java/src/backend/DoAlignSynPair.java

@@ -50,12 +50,12 @@ public void run(ManagerFrame frame, DBconn2 dbc2, Mpair mp,  boolean closeWhenDo
 		Mproject mProj2 = mp.mProj2;
 		String dbName1 = mProj1.getDBName();
 		String dbName2 = mProj2.getDBName();
-		String toName =  (mProj1 == mProj2) ? dbName1 + " to self" : dbName1 + " to " + dbName2;
+		String toName =  (mProj1 == mProj2) ? dbName1 : dbName1 + " to " + dbName2; // CAS572 make naming consistent
 
 		FileWriter syFW =   symapLog(mProj1,mProj2);
 		String alignLogDir = buildLogAlignDir(mProj1,mProj2);
 
-		String msg = (mProj1 == mProj2) ? "Synteny "  + dbName1 + " to itself ..." : "Synteny " + toName + " ...";
+		String msg = (mProj1 == mProj2) ? "Self-synteny "  + dbName1 + " ..." : "Synteny " + toName + " ...";
 
 		final ProgressDialog diaLog = new ProgressDialog(this, "Running Synteny", msg, true, syFW); // write version and date
 

java/src/backend/SeqLoadMain.java

@@ -343,7 +343,10 @@ private void uploadSequence(String grp, String fullname, String seq, String file
 			tdbc2.executeUpdate(st);
 		}
 	}
-	catch (Exception e) {ErrorReport.print(e, "Loading sequence - fail loading to database "); };
+	catch (Exception e) {
+		ErrorReport.print(e, "Fail load sequence: " +  projIdx + "','" + grp + "','" + fullname + "'," + order); 
+		Globals.prt("  This can happen if a project was just removed and another added immediately."); // CAS572
+	}
 	}
 	/**************************************************************************/
 	private boolean rtError(String msg) {

java/src/backend/anchor1/AnchorMain1.java

@@ -183,7 +183,7 @@ private boolean scanAlignFiles(File dh) throws Exception {
 		}
 		if (mTotalHits == 0)   return rtError("No good anchors were found");
 		Globals.rclear();
-		if (nFile>1 || !VB) 			Utils.prtNumMsgFile(plog, nHitsScanned, "Total scanned hits");
+		if (nFile>1 || !VB) 			Utils.prtNumMsgFile(plog, nHitsScanned, "Total scanned hits from " + nFile + " files"); // CAS572 add file
 		if (nHitsScanned!=mTotalHits) 	Utils.prtNumMsgFile(plog, mTotalHits, "Total accepted hits");
 		if (mTotalLargeHits > 0 && VB)  Utils.prtNumMsgFile(plog, mTotalLargeHits, "Large hits (> " + Group.FSplitLen + ")  "
 					 															+ mTotalBrokenHits + " Split "); 
@@ -218,7 +218,7 @@ private boolean scanAlignFiles(File dh) throws Exception {
 		}
 		Globals.rclear();
 		if (nFile>1 || !VB) 
-			Utils.prtNumMsg(plog, nHitsScanned, "Total cluster hits   ");
+			Utils.prtNumMsg(plog, nHitsScanned, "Total cluster hits  ");
 
 		if (Constants.PRT_STATS) { 
 			Utils.prtTimeMemUsage(plog, "Complete scan cluster", memTime);
@@ -441,8 +441,9 @@ private Vector<Hit> clusterHits2(Vector<Hit> inHits) throws Exception {
 			AnnotElem qAnno = grp1.getBestOlapAnno(hit.queryHits.start, hit.queryHits.end);   // priority to gene annotElem
 			AnnotElem tAnno = grp2.getBestOlapAnno(hit.targetHits.start, hit.targetHits.end); 
 
-			if (qAnno == null) {rtError("missing query annot!  grp:" + grp1.idStr() + " start:" + hit.queryHits.start + " end:" + hit.queryHits.end); return null;}	
-			if (tAnno == null) {rtError("missing target annot! grp:" + grp2.idStr() + " start:" + hit.targetHits.start + " end:" + hit.targetHits.end); return null;}	
+			// Self-synteny humans failed on this and quit; change to continue... CAS572
+			if (qAnno == null) {Globals.tprt("missing query annot!  grp:" + grp1.idStr() + " start:" + hit.queryHits.start + " end:" + hit.queryHits.end); continue;}	
+			if (tAnno == null) {Globals.tprt("missing target annot! grp:" + grp2.idStr() + " start:" + hit.targetHits.start + " end:" + hit.targetHits.end); continue;}	
 
 			String key = qAnno.mID + "_" + tAnno.mID;
 
@@ -692,7 +693,7 @@ private void saveFilterHits(Vector <Hit> vecHits) throws Exception {
 		}
 		catch (Exception e) {ErrorReport.print(e, "save annot hits"); bSuccess=false;}
 	}
-	private String intArrayToBlockStr(int[] ia) {// CAS569 moved from Util
+	private String intArrayToBlockStr(int[] ia) {
 		String out = "";
 		if (ia != null) {
 			for (int i = 0; i < ia.length; i+=2) {

java/src/backend/anchor2/AnchorMain2.java

@@ -104,21 +104,21 @@ public boolean run(Mproject pj1, Mproject pj2, File dh) throws Exception {
 		prtFileOpen();
 
 	// for each file: read file, build clusters, save results to db
-		processFiles(dh); 					if (!bSuccess) return false; 
+		int nfile = processFiles(dh); 					if (!bSuccess) return false; 	
 
 	// finish
 		prtFileClose();
 		clearTree();
 
 		Globals.rclear();
-		Utils.prtIndentMsgFile(plog, 1, String.format("Final totals    Raw hits %,d", cntRawHits));	
+		Utils.prtIndentMsgFile(plog, 1, String.format("Final totals    Raw hits %,d    Files %,d", cntRawHits, nfile));	
 
-		String msg1 = String.format("Clusters  Both genes %,8d   One gene %,8d   No gene %,8d",
+		String msg1 = String.format("Clusters  Both genes %,9d   One gene %,9d   No gene %,9d", // CAS572 8->9
 				cntG2, cntG1, cntG0);
 		Utils.prtNumMsgFile(plog, cntClusters, msg1);	
 
 		int total = cntG2Fil+cntG1Fil+cntG0Fil;
-		String msg2 = String.format("Filtered  Both genes %,8d   One gene %,8d   No gene %,8d   Pile hits %,d",
+		String msg2 = String.format("Filtered  Both genes %,9d   One gene %,9d   No gene %,9d   Pile hits %,d",
 				total, cntG2Fil, cntG1Fil, cntG0Fil, cntPileFil);
 		Utils.prtNumMsgFile(plog, total, msg2);
 
@@ -271,32 +271,34 @@ private void loadAnno(int X, Mproject mProj, TreeMap <Integer, GeneTree> grpGene
 	/******************************************************
 	 * Step 2: Each file is read and immediately processed and save
 	 */	
-	private void processFiles(File dh) {
+	private int processFiles(File dh) { 
 	try {
+		int nfile=0;
 		File[] fs = dh.listFiles();
 		for (File f : fs) {
 			if (!f.isFile()) continue;
 			fileName = f.getName();
 			if (!fileName.endsWith(Constants.mumSuffix))continue;
-			
+			nfile++;
 			long time = Utils.getTime();
 
 		// load hits into grpPairs
 			readMummer(f);							
 
 		// process all group pairs from file
 			for (GrpPair gp : grpPairMap.values()) {	
-				bSuccess = gp.buildClusters(); 	if (!bSuccess) return;
+				bSuccess = gp.buildClusters(); 	if (!bSuccess) return nfile;
 			}
 
-			if (failCheck()) return;
+			if (failCheck()) return nfile;
 
 			clearFileSpecific();
 
 			if (Arg.TRACE) Utils.prtTimeMemUsage(plog, "   Finish ", time);
 		}
+		return nfile;				// CAS572 add
 	}
-	catch (Exception e) {ErrorReport.print(e, "analyze and save"); bSuccess=false;}
+	catch (Exception e) {ErrorReport.print(e, "analyze and save"); bSuccess=false; return 0;}
 	}
 	/*********************************************************************
 	 // 0        1       2        3       4       5       6		7        8       9       10         11 12   13 14