fix(ENGKNOW-3212): Fix refbases_with_build performance.

Author: gmagnu

gortools/src/main/scala/gorsat/parser/GenomeFunctions.scala

@@ -27,6 +27,7 @@ import gorsat.parser.FunctionTypes.{dFun, iFun, sFun}
 import gorsat.parser.ParseUtilities._
 import gorsat.process.GorJavaUtilities
 import org.gorpipe.exceptions.GorParsingException
+import org.gorpipe.model.gor.iterators.RefSeq
 
 object GenomeFunctions {
   def register(functions: FunctionRegistry): Unit = {
@@ -101,8 +102,12 @@ object GenomeFunctions {
   }
 
   def refBases_with_build(owner: ParseArith, ex1: sFun, ex2: iFun, ex3: iFun, ex4: sFun): sFun = {
+    var refSeq: RefSeq  = null
     cvp => {
-      owner.context.getSession.getProjectContext.createRefSeq(ex4(cvp)).getBases(ex1(cvp), ex2(cvp), ex3(cvp))
+      if (refSeq == null) {
+        refSeq = owner.context.getSession.getProjectContext.createRefSeq(ex4(cvp))
+      }
+      refSeq.getBases(ex1(cvp), ex2(cvp), ex3(cvp))
     }
   }
 

gortools/src/main/scala/gorsat/parser/ParseArith.scala

@@ -995,9 +995,8 @@ class ParseArith(rs: GenomicIterator = null) extends JavaTokenParsers with Seria
     stringIfMatcher |
     genericFunction[sFun](FunctionTypes.StringFun) |
     "REFBASES_WITH_BUILD".ignoreCase ~ "(" ~ sexpr ~ "," ~ iexpr ~ "," ~ iexpr ~ "," ~ myStringLiteralFilename ~ ")" ^^ {
-      case _ ~ "(" ~ ex1 ~ "," ~ ex2 ~ "," ~ ex3 ~ "," ~ ex4 ~ ")" => (line: ColumnValueProvider) => {
-        refBases_with_build(this, ex1, ex2, ex3, (line: ColumnValueProvider) => ex4)(line)
-      }
+      case _ ~ "(" ~ ex1 ~ "," ~ ex2 ~ "," ~ ex3 ~ "," ~ ex4 ~ ")" =>
+        refBases_with_build(this, ex1, ex2, ex3, (line: ColumnValueProvider) => ex4)
     } |
     "GTSTAT".ignoreCase ~ "(" ~ sexpr ~ "," ~ iexpr ~ "," ~ sexpr ~ "," ~ sexpr ~ "," ~ iexpr ~ "," ~ sexpr ~ "," ~ sexpr ~ "," ~ iexpr ~ "," ~ sexpr ~ "," ~ sexpr ~ ")" ^^ {
       case _ ~ "(" ~ ex1 ~ "," ~ ex2 ~ "," ~ ex3 ~ "," ~ ex4 ~ "," ~ ex5 ~ "," ~ ex6 ~ "," ~ ex7 ~ "," ~ ex8 ~ "," ~ ex9 ~ "," ~ ex10 ~ ")" => (line: ColumnValueProvider) => {

gortools/src/test/java/gorsat/parser/UTestGenomicFunctions.java

@@ -110,6 +110,18 @@ public void testRefBasesWithBuild2() {
         TestUtils.assertCalculated("refbases_with_build('chr1', 1010101, 1010103, /private/gorkube-mount/csa/ref/hg19/chromSeq)", "ACC");
     }
 
+    @Ignore("Slow, and not needed for testin.  Uses full build, which is not normally available")
+    @Test
+    public void testRefBasesWithBuildSpeed() {
+        TestUtils.runGorPipeCount("gorrows -p chr1:100000-400000 |  calc b refbases_with_build(chrom, pos, pos, /private/gorkube-mount/csa/ref/hg19/chromSeq)", ".");
+    }
+
+    @Ignore("Slow, and not needed for testin.  Uses full build, which is not normally available")
+    @Test
+    public void testRefBasesSpeed() {
+        TestUtils.runGorPipeCount("gorrows -p chr1:100000-400000 |  calc b refbases(chrom, pos, pos)", "-config", "/private/gorkube-mount/csa/ref/hg19/gor_config.txt");
+    }
+
     @Test
     public void testBamTag() {
         TestUtils.assertCalculated("bamtag('', '')", "NOT_FOUND");

model/src/main/scala/org/gorpipe/model/gor/iterators/RefSeqFromChromSeq.scala

@@ -41,6 +41,13 @@ object RefSeqFromChromSeq {
     new java.util.concurrent.ConcurrentHashMap[String, java.lang.Boolean]
 }
 
+// IMPORTANT NOTE ON THREAD SAFETY:
+// This class is NOT thread safe. It is expected that each thread that needs to access reference sequence data
+// will have its own instance of this class.
+// 1.  The LUFO cache is per se thread safe but it is very small so if used by many threads it will cause a lot
+//     of thrasing and bad performance.
+// 2. The use of the filemap is NOT thread safe and is very likely to cause incorrect reads of the cache data.
+
 class RefSeqFromChromSeq(ipath : String, fileReader : FileReader) extends RefSeq {
   private val GOR_REFSEQ_CACHE_FOLDER = System.getProperty("gor.refseq.cache.folder")
   private val GOR_REFSEQ_CACHE_DOWNLOAD = Option(System.getProperty("gor.refseq.cache.download", "true")).exists(_.toBoolean)