don't show bad sentences on annotation

Author: eiennohito

.scalafmt.conf

@@ -3,4 +3,6 @@ align = some
 align.openParenDefnSite = false
 rewrite.rules = [AvoidInfix, SortImports, RedundantParens, PreferCurlyFors]
 rewrite.neverInfix.excludeFilters += "min"
-rewrite.neverInfix.excludeFilters += "max"
+rewrite.neverInfix.excludeFilters += "max"
+rewrite.neverInfix.excludeFilters += "ne"
+rewrite.neverInfix.excludeFilters += "eq"

play/app/code/annotation/AnnotationDb.scala

@@ -11,7 +11,6 @@ import net.codingwell.scalaguice.ScalaModule
 import org.apache.commons.lang3.RandomUtils
 import org.joda.time.DateTime
 import play.api.Configuration
-import play.api.mvc.Result
 import reactivemongo.api.collections.bson.BSONCollection
 import reactivemongo.api.commands.UpdateWriteResult
 import reactivemongo.api.{Cursor, DefaultDB, MongoConnection}
@@ -113,8 +112,8 @@ object BsonPicklers {
 
 object AnnotationToolUser {
   implicit val bsonRw: BSONDocumentHandler[AnnotationToolUser] = Macros.handler[AnnotationToolUser]
-  import prickle._
   import BsonPicklers._
+  import prickle._
 
   implicit val pickler: Pickler[AnnotationToolUser] =
     prickle.Pickler.materializePickler[AnnotationToolUser]
@@ -304,6 +303,29 @@ object SentenceBSON {
 
 class SentenceDbo @Inject()(db: AnnotationDb)(implicit ec: ExecutionContext) extends StrictLogging {
 
+  private val coll = db.db.collection[BSONCollection]("sentences")
+
+  import SentenceBSON._
+
+  def recalcStatuses() = {
+    coll.find(BSONDocument()).cursor[Sentence]().fold(NotUsed) {
+      case (_, doc) =>
+        val newStatus = SentenceUtils.computeStatus(doc)
+        if (newStatus != doc.status) {
+          val search = BSONDocument(
+            "_id" -> doc.id
+          )
+          val upd = BSONDocument(
+            "$set" -> BSONDocument(
+              "status" -> newStatus.value
+            )
+          )
+          coll.update(search, upd)
+        }
+        NotUsed
+    }
+  }
+
   def findAlreadyReported(surfaceComment: String): Future[Option[String]] = {
     val q = BSONDocument(
       "tags" -> "report",
@@ -312,16 +334,12 @@ class SentenceDbo @Inject()(db: AnnotationDb)(implicit ec: ExecutionContext) ext
 
     val proj = BSONDocument("_id" -> 1)
 
-    coll.find(q, proj).one.map {
+    coll.find(q, proj).one[BSONDocument].map {
       case Some(doc) => doc.getAs[String]("_id")
       case _         => None
     }
   }
 
-  private val coll = db.db.collection[BSONCollection]("sentences")
-
-  import SentenceBSON._
-
   def checkExistance(ids: Seq[String]): Future[Set[String]] = {
     if (ids.isEmpty) {
       return Future.successful(Set.empty)
@@ -384,6 +402,10 @@ class SentenceDbo @Inject()(db: AnnotationDb)(implicit ec: ExecutionContext) ext
         doc.merge(
           "status" -> SentenceStatus.WorkRequired.value
         )
+      case "rj" | "rej" =>
+        doc.merge(
+          "status" -> SentenceStatus.Rejected.value
+        )
       case "bad" | "ng" =>
         doc.merge(
           "status" -> BSONDocument(
@@ -460,6 +482,13 @@ class SentenceDbo @Inject()(db: AnnotationDb)(implicit ec: ExecutionContext) ext
           "$ne" -> user._id
         )
       )
+      if (req.query == "") {
+        q = q.merge(
+          "status" -> BSONDocument(
+            "$ne" -> SentenceStatus.Rejected.value
+          )
+        )
+      }
     }
 
     if (req.reviewedBefore.isDefined) {
@@ -659,6 +688,13 @@ class SentenceDbo @Inject()(db: AnnotationDb)(implicit ec: ExecutionContext) ext
 }
 
 object SentenceUtils {
+
+  val badAnnotations = Set(
+    "どうでもいい",
+    "入力：意味不明",
+    "入力：誤字脱字"
+  )
+
   def cleanTags(blocks: Seq[SentenceBlock], allowedFields: Set[String]): Seq[SentenceBlock] = {
     blocks.map { b =>
       b.copy(spans = b.spans.map { s =>
@@ -689,6 +725,7 @@ object SentenceUtils {
     } else {
       val values = anns.map(_.value).distinct
       values match {
+        case vs if vs.exists(badAnnotations.contains)        => SentenceStatus.Rejected
         case Seq(XInt(_))                                    => SentenceStatus.TotalAgreement
         case vs if vs.forall(v => XInt.unapply(v).isDefined) => SentenceStatus.PartialAgreement
         case _                                               => SentenceStatus.WorkRequired

play/app/code/annotation/LogDbo.scala

@@ -0,0 +1,26 @@
+package code.annotation
+
+import javax.inject.Inject
+import reactivemongo.api.collections.bson.BSONCollection
+import reactivemongo.bson.Subtype.GenericBinarySubtype
+import reactivemongo.bson.{BSONBinary, BSONDateTime, BSONDocument, BSONObjectID}
+
+import scala.concurrent.ExecutionContext
+
+class LogDbo @Inject()(
+    db: AnnotationDb
+)(implicit ec: ExecutionContext) {
+
+  private val coll = db.db.collection[BSONCollection]("sentenceActions")
+
+  def log(req: SentenceRequest, user: AnnotationToolUser): Unit = {
+    val doc = BSONDocument(
+      "_id" -> BSONObjectID.generate(),
+      "user" -> user._id,
+      "kind" -> req.request.number,
+      "date" -> BSONDateTime(System.currentTimeMillis()),
+      "blob" -> BSONBinary(req.toByteArray, GenericBinarySubtype)
+    )
+    coll.insert(doc)
+  }
+}

play/app/controllers/SentenceApiController.scala

@@ -1,9 +1,8 @@
 package controllers
 
-import java.time.{Instant, LocalDateTime}
+import java.time.LocalDateTime
 import java.time.format.DateTimeFormatter
 
-import javax.inject.Inject
 import code.annotation._
 import code.transport.lattice.{
   CandidateNode,
@@ -12,6 +11,7 @@ import code.transport.lattice.{
   PartialAnalysisQuery
 }
 import com.typesafe.scalalogging.StrictLogging
+import javax.inject.Inject
 import play.api.mvc.{InjectedController, Result}
 import reactivemongo.bson.BSONObjectID
 import ws.kotonoha.akane.analyzers.jumanpp.wire.lattice.LatticeDump
@@ -21,11 +21,14 @@ import scala.concurrent.{ExecutionContext, Future}
 
 class SentenceApiController @Inject()(
     dbo: SentenceDbo,
+    objLog: LogDbo,
     reportSvc: SentenceReportService
 )(implicit ec: ExecutionContext)
     extends InjectedController {
 
   def doHandle(body: SentenceRequest, user: AnnotationToolUser): Future[Result] = {
+    objLog.log(body, user)
+
     body.request match {
       case SentenceRequest.Request.Sentences(sents) =>
         dbo.getSentences(user, sents).map(sents => Ok(LiftPB(sents)))
@@ -55,6 +58,14 @@ class SentenceApiController @Inject()(
         dbo.saveBadLog(uid, req).map { _ =>
           Ok(LiftPB(Annotation()))
         }
+      case SentenceRequest.Request.RecalcStatuses(_) =>
+        if (user.admin) {
+          dbo.recalcStatuses().map { _ =>
+            Ok(LiftPB(Annotation()))
+          }
+        } else {
+          Future.successful(Forbidden("You are not admin"))
+        }
       case _ => Future.successful(NotImplemented)
     }
   }
@@ -94,13 +105,13 @@ class BlockSeqBuilder {
   private val blocks = new ArrayBuffer[SentenceBlock]()
 
   private def addNode(node: CandidateNode) = {
-    if (nodes.isEmpty || (nodes.last.ne(node))) {
+    if (nodes.isEmpty || nodes.last.ne(node)) {
       nodes += node
     }
   }
 
   private def addPart(part: EditableSentencePart) = {
-    if (parts.isEmpty || (parts.last.ne(part))) {
+    if (parts.isEmpty || parts.last.ne(part)) {
       parts += part
     }
   }

scalajs/src/main/scala/code/annotation/SentenceAnnotation.scala

@@ -666,7 +666,7 @@ case class SentenceAnnotation(apiSvc: ApiService, uid: ObjId, isAdmin: Boolean)
             }
           ),
           <.button(
-            "Reject",
+            "No Candidate",
             ^.onClick --> cback(CommentEdit("Reject", state.comment, state.focus))
           )
         )

scalajs/src/main/scala/code/annotation/SentenceImport.scala

@@ -26,9 +26,14 @@ object SentenceImport {
           }
           .runNow()
       }
-
     }
 
+    def recalcStatuses(): Callback =
+      scope.props.map { api =>
+        val req = SentenceRequest(SentenceRequest.Request.RecalcStatuses(0))
+        api.sentenceCall[Annotation](req)
+      }.void
+
     def close() = Callback {
       if (connection != null) {
         connection.close()
@@ -76,7 +81,12 @@ object SentenceImport {
           "Import"
         ),
         <.h2("Messages"),
-        renderMessages(s.messages)
+        renderMessages(s.messages),
+        <.h2("Controls"),
+        <.button(
+          ^.onClick --> recalcStatuses(),
+          "Recompute Statuses"
+        )
       )
     }
 

shared/src/main/protobuf/sentences.proto

@@ -38,6 +38,7 @@ enum SentenceStatus {
     TotalAgreement = 1;
     PartialAgreement = 2;
     WorkRequired = 3;
+    Rejected = 4;
 }
 
 message Review {
@@ -72,6 +73,7 @@ message SentenceRequest {
         MergeEdits merge = 3;
         ReviewSentence review = 4;
         ReportAllBad bothBad = 5;
+        int32 recalcStatuses = 6;
     }
 }