Many conservative changes to make code run with both t4 and t5

.github/workflows/ci.yml

@@ -16,5 +16,9 @@ jobs:
       uses: actions/setup-java@v1
       with:
         java-version: 1.8
+    - name: Setup sbt
+      uses: sbt/setup-sbt@v1
+      with:
+        sbt-version: 1.7.1
     - name: Run tests
-      run: sbt test
+      run: sbt ++test

apps/build.sbt

@@ -7,17 +7,6 @@ resolvers ++= Seq(
 
 libraryDependencies ++= {
   Seq(
-    // Models version 0.1.0 work when LinearLayer.USE_CONCAT == true.
-    "org.clulab"     % "deberta-onnx-model" % "0.1.0",
-    "org.clulab"     % "roberta-onnx-model" % "0.1.0",
-
-    // Models version 0.2.0 work when LinearLayer.USE_CONCAT == false.
-    // Models of different versions cannot be combined into a single project
-    // because the resource names will conflict.  The choice is forced.
-    // "org.clulab"     % "deberta-onnx-model" % "0.2.0",
-    // "org.clulab"     % "electra-onnx-model" % "0.2.0",
-    // "org.clulab"     % "roberta-onnx-model" % "0.2.0",
-
     "org.scalatest" %% "scalatest"          % "3.2.15" % "test"
   )
 }

apps/src/main/scala/org/clulab/scala_transformers/apps/ColumnsToDocument.scala

@@ -0,0 +1,72 @@
+package org.clulab.scala_transformers.apps
+
+import org.slf4j.{Logger, LoggerFactory}
+
+import scala.collection.mutable.ArrayBuffer
+import scala.io.Source
+
+private class ColumnsToDocument
+
+class Document (val sentences: Seq[Sentence])
+
+class Sentence (val words: Seq[String], val labels: Seq[String])
+
+/**
+  * Converts the CoNLLX column-based format to Document
+  */
+object ColumnsToDocument {
+  val logger: Logger = LoggerFactory.getLogger(classOf[ColumnsToDocument])
+
+  def readFromFile(
+    fn: String,
+    wordPos: Int,
+    labelPos: Int
+  ): Document = {
+    val source = Source.fromFile(fn)
+    readFromSource(source, wordPos, labelPos)
+  }
+
+  def readFromSource(
+    source: Source,
+    wordPos: Int,
+    labelPos: Int
+  ): Document = {
+    val words = new ArrayBuffer[String]()
+    val labels = new ArrayBuffer[String]()
+    val sentences = new ArrayBuffer[Sentence]()
+
+    def mkSentence(): Sentence = {
+      val sent = new Sentence(words.toArray.toSeq, labels.toArray.toSeq)
+      words.clear()
+      labels.clear()
+      sent
+    }
+
+    source.getLines().map(_.trim).foreach { l =>
+      if (l.isEmpty) {
+        // end of sentence
+        if (words.nonEmpty) {
+          sentences += mkSentence()
+        }
+      }
+      else {
+        // within the same sentence
+        val bits = l.split("\\s+")
+        if (bits.length < 2)
+          throw new RuntimeException(s"ERROR: invalid line [$l]!")
+
+        words += bits(wordPos)
+        labels += bits(labelPos)
+      }
+    }
+    if (words.nonEmpty) {
+      val sent = mkSentence()
+      sentences += sent
+      //println("words: " + sent.words.mkString(", "))
+      //println("labels: " + sent.entities.get.mkString(", "))
+    }
+    logger.debug(s"Loaded ${sentences.size} sentences.")
+
+    new Document(sentences.toSeq)
+  }
+}

apps/src/main/scala/org/clulab/scala_transformers/apps/TokenClassifierCoNLLApp.scala

@@ -0,0 +1,44 @@
+package org.clulab.scala_transformers.apps
+
+import org.clulab.scala_transformers.encoder.TokenClassifier
+
+import java.io.PrintWriter
+
+/** Processes a file in the CoNLL format and outputs BIO notations in the format expected by the CoNLL scorer */
+object TokenClassifierCoNLLApp extends App {
+  val WORD_POS = 0 // in the column format
+  val LABEL_POS = 1 // in the column format
+  val ANNOTATION_POS = 0 // in the encoder MTL outputs
+
+  val fileName = "../test.txt"
+  val pw = new PrintWriter(fileName + ".conll")
+
+  //val tokenClassifier = TokenClassifier.fromFiles("../answerdotai-ModernBERT-base-mtl/avg_export")
+  //val tokenClassifier = TokenClassifier.fromFiles("../microsoft-deberta-v3-base-mtl/avg_export")
+  // val tokenClassifier = TokenClassifier.fromFiles("../roberta-base-mtl/avg_export")
+  val tokenClassifier = TokenClassifier.fromFiles("../thomas-sounack-BioClinical-ModernBERT-base-mtl/avg_export")
+
+  val goldDoc = ColumnsToDocument.readFromFile(fileName, WORD_POS, LABEL_POS)
+  val sentences = goldDoc.sentences
+  println(s"Read a doc with ${sentences.length} sentences.")
+
+  for(i <- sentences.indices) {
+    val words = sentences(i).words
+    val goldLabels = sentences(i).labels
+
+    val predLabels = tokenClassifier.predict(words)(ANNOTATION_POS)
+
+    println("words: " + words.mkString(", "))
+    println("gold:  " + goldLabels.mkString(", "))
+    println("pred:  " + predLabels.mkString(", "))
+
+    if(goldLabels.length == predLabels.length) {
+      for (j <- words.indices) {
+        pw.println(s"${words(j)} ${goldLabels(j)} ${predLabels(j)}")
+      }
+      pw.println()
+    }
+  }
+
+  pw.close()
+}

apps/src/main/scala/org/clulab/scala_transformers/apps/TokenClassifierExampleApp.scala

@@ -4,7 +4,8 @@ import org.clulab.scala_transformers.encoder.TokenClassifier
 
 object TokenClassifierExampleApp extends App {
   // Choose one of these.
-  val tokenClassifier = TokenClassifier.fromFiles("../microsoft-deberta-v3-base-mtl/avg_export")
+  val tokenClassifier = TokenClassifier.fromFiles("../thomas-sounack-BioClinical-ModernBERT-base-mtl/avg_export")
+  // val tokenClassifier = TokenClassifier.fromFiles("../microsoft-deberta-v3-base-mtl/avg_export")
   //val tokenClassifier = TokenClassifier.fromResources("/org/clulab/scala_transformers/models/microsoft_deberta_v3_base_mtl/avg_export")
   // val tokenClassifier = TokenClassifier.fromFiles("../models/google_electra_small_discriminator_mtl/avg_export")
   // val tokenClassifier = TokenClassifier.fromResources("/org/clulab/scala_transformers/models/google_electra_small_discriminator_mtl/avg_export")
@@ -14,8 +15,8 @@ object TokenClassifierExampleApp extends App {
   //val words = Seq("EU", "rejects", "German", "call", "to", "boycott", "British", "lamb", ".")
   //val words = Seq("John", "Doe", "went", "to", "China", ".")
   //val words = Seq("John", "Doe", "went", "to", "China", ".")
-  //val words = Seq("Ras1", "has", "phosphorylated", "Mek2", ".")
-  val words = Seq("Jack", "Doe", "and", "John", "Doe", "were", "friends", "in", "yyy", "zzz", ".")
+  val words = Seq("Ras1", "has", "phosphorylated", "the", "Mek2", "protein", ".")
+  //val words = Seq("Jack", "Doe", "and", "John", "Doe", "were", "friends", "in", "yyy", "zzz", ".")
 
   println(s"Words: ${words.mkString(", ")}")