Removing broken information in brackets

core/src/main/scala/org/dbpedia/extraction/config/Config.scala

@@ -277,7 +277,8 @@ class Config(val configPath: String) extends
       shortAbstractsProperty = this.getProperty("short-abstracts-property", "rdfs:comment").trim,
       longAbstractsProperty = this.getProperty("long-abstracts-property", "abstract").trim,
       shortAbstractMinLength = this.getProperty("short-abstract-min-length", "200").trim.toInt,
-      abstractTags = this.getProperty("abstract-tags", "query,pages,page,extract").trim
+      abstractTags = this.getProperty("abstract-tags", "query,pages,page,extract").trim,
+      removeBrokenBracketsProperty = this.getProperty("remove-broken-brackets", "false").trim
     )
   } match{
     case Success(s) => s
@@ -369,11 +370,12 @@ object Config{
   )
 
   case class AbstractParameters(
-    abstractQuery: String,
-    shortAbstractsProperty: String,
-    longAbstractsProperty: String,
-    shortAbstractMinLength: Int,
-    abstractTags: String
+                                 abstractQuery: String,
+                                 shortAbstractsProperty: String,
+                                 longAbstractsProperty: String,
+                                 shortAbstractMinLength: Int,
+                                 abstractTags: String,
+                                 removeBrokenBracketsProperty: String
   )
 
   case class SlackCredentials(

core/src/main/scala/org/dbpedia/extraction/mappings/AbstractExtractor.scala

@@ -7,7 +7,7 @@ import org.dbpedia.extraction.config.Config
 import org.dbpedia.extraction.config.provenance.DBpediaDatasets
 import org.dbpedia.extraction.ontology.Ontology
 import org.dbpedia.extraction.transform.{Quad, QuadBuilder}
-import org.dbpedia.extraction.util.{Language, MediaWikiConnector}
+import org.dbpedia.extraction.util.{Language, MediaWikiConnector, WikiUtil}
 import org.dbpedia.extraction.wikiparser._
 
 import scala.language.reflectiveCalls
@@ -50,6 +50,8 @@ extends WikiPageExtractor
     //private val apiParametersFormat = "uselang="+language+"&format=xml&action=parse&prop=text&title=%s&text=%s"
   protected val apiParametersFormat = context.configFile.abstractParameters.abstractQuery
 
+  protected val removeBrokenBrackets = context.configFile.abstractParameters.removeBrokenBracketsProperty
+
     // lazy so testing does not need ontology
   protected lazy val shortProperty = context.ontology.properties(context.configFile.abstractParameters.shortAbstractsProperty)
 
@@ -63,7 +65,6 @@ extends WikiPageExtractor
 
   private val mwConnector = new MediaWikiConnector(context.configFile.mediawikiConnection, context.configFile.abstractParameters.abstractTags.split(","))
 
-
     override def extract(pageNode : WikiPage, subjectUri: String): Seq[Quad] =
     {
         //Only extract abstracts for pages from the Main namespace
@@ -79,16 +80,21 @@ extends WikiPageExtractor
         // if(abstractWikiText == "") return Seq.empty
 
         //Retrieve page text
-        val text = mwConnector.retrievePage(pageNode.title, apiParametersFormat, pageNode.isRetry) match{
+        val text = mwConnector.retrievePage(pageNode.title, apiParametersFormat, pageNode.isRetry) match {
           case Some(t) => AbstractExtractor.postProcessExtractedHtml(pageNode.title, replacePatterns(t))
           case None => return Seq.empty
         }
 
+        val modifiedText = removeBrokenBrackets match {
+          case "true" => WikiUtil.removeBrokenBracketsInAbstracts(text)
+          case _ => text
+        }
+
         //Create a short version of the abstract
-        val shortText = short(text)
+        val shortText = short(modifiedText)
 
         //Create statements
-        val quadLong = longQuad(pageNode.uri, text, pageNode.sourceIri)
+        val quadLong = longQuad(pageNode.uri,modifiedText, pageNode.sourceIri)
         val quadShort = shortQuad(pageNode.uri, shortText, pageNode.sourceIri)
 
         if (shortText.isEmpty)
@@ -205,7 +211,7 @@ extends WikiPageExtractor
                 .filter(renderNode)
                 .map(_.toWikiText)
                 .mkString("").trim
-        
+
         // decode HTML entities - the result is plain text
         decodeHtml(text)
     }
@@ -243,6 +249,7 @@ object AbstractExtractor {
 
   val patternsToRemove = List(
     """<div style=[^/]*/>""".r -> " ",
-    """</div>""".r -> " "
+    """</div>""".r -> " ",
+    """<normalized>.*<\/normalized>""".r -> ""
   )
 }

core/src/main/scala/org/dbpedia/extraction/nif/WikipediaNifExtractor.scala

@@ -4,7 +4,7 @@ import org.dbpedia.extraction.config.Config
 import org.dbpedia.extraction.config.provenance.DBpediaDatasets
 import org.dbpedia.extraction.ontology.{Ontology, OntologyProperty, RdfNamespace}
 import org.dbpedia.extraction.transform.{Quad, QuadBuilder}
-import org.dbpedia.extraction.util.{Language, RecordEntry, RecordSeverity}
+import org.dbpedia.extraction.util.{Language, RecordEntry, RecordSeverity, WikiUtil}
 import org.dbpedia.extraction.wikiparser.{Namespace, WikiPage}
 import org.dbpedia.extraction.wikiparser.impl.wikipedia.Namespaces
 import org.jsoup.nodes.{Document, Element, Node}
@@ -67,8 +67,9 @@ class WikipediaNifExtractor(
     */
   override def extendSectionTriples(extractionResults: ExtractedSection, graphIri: String, subjectIri: String): Seq[Quad] = {
     //this is only dbpedia relevant: for singling out long and short abstracts
+
     if (recordAbstracts && extractionResults.section.id == "abstract" && extractionResults.getExtractedLength > 0) {
-      List(longQuad(subjectIri, extractionResults.getExtractedText, graphIri), shortQuad(subjectIri, getShortAbstract(extractionResults), graphIri))
+      List(longQuad(subjectIri, WikiUtil.removeBrokenBracketsInAbstracts(extractionResults.getExtractedText), graphIri), shortQuad(subjectIri, WikiUtil.removeBrokenBracketsInAbstracts(getShortAbstract(extractionResults)), graphIri))
     }
     else
       List()
@@ -219,4 +220,5 @@ class WikipediaNifExtractor(
       test.addAll(doc.select(query))
     test.size() > 0
   }
+
 }

core/src/main/scala/org/dbpedia/extraction/util/MediaWikiConnector.scala

@@ -1,24 +1,25 @@
 package org.dbpedia.extraction.util
 
 import java.io.{InputStream, OutputStreamWriter}
-import java.net.URL
-import javax.xml.ws.WebServiceException
+import java.net.{HttpURLConnection, URL}
+import java.time.temporal.ChronoUnit
 
+import javax.xml.ws.WebServiceException
 import org.dbpedia.extraction.wikiparser.WikiTitle
 import org.dbpedia.util.text.html.{HtmlCoder, XmlCodes}
 
 import scala.io.Source
 import scala.util.{Failure, Success, Try}
 import org.dbpedia.extraction.config.Config.MediaWikiConnection
-
+import org.slf4j.LoggerFactory
 /**
   * The Mediawiki API connector
   * @param connectionConfig - Collection of parameters necessary for API requests (see Config.scala)
   * @param xmlPath - An array of XML tag names leading from the root (usually 'api') to the intended content of the response XML (depending on the request query used)
   */
 class MediaWikiConnector(connectionConfig: MediaWikiConnection, xmlPath: Seq[String]) {
 
-
+  protected val log = LoggerFactory.getLogger(classOf[MediaWikiConnector])
   //protected def apiUrl: URL = new URL(connectionConfig.apiUrl)
   //require(Try{apiUrl.openConnection().connect()} match {case Success(x)=> true case Failure(e) => false}, "can not connect to the apiUrl")
 
@@ -80,6 +81,7 @@ class MediaWikiConnector(connectionConfig: MediaWikiConnection, xmlPath: Seq[Str
       try
       {
         val conn = apiUrl.openConnection
+        val start = java.time.LocalTime.now()
         conn.setDoOutput(true)
         conn.setConnectTimeout(retryFactor * connectMs)
         conn.setReadTimeout(retryFactor * readMs)
@@ -89,8 +91,22 @@ class MediaWikiConnector(connectionConfig: MediaWikiConnection, xmlPath: Seq[Str
         writer.flush()
         writer.close()
 
+        // log URL, POSTparametersifPOST,  HTTP code, time needed, request-time
+        // log.debug(conn.getHeaderFields)
+
+        val inputStream = conn.getInputStream
+        val end = java.time.LocalTime.now()
+        conn match {
+          case connection: HttpURLConnection => {
+            log.debug("Request type: "+ connection.getRequestMethod + "; URL: " + connection.getURL +
+              "; Parameters: " + parameters +"; HTTP code: "+ connection.getHeaderField(null) +
+              "; Request time: "+start+"; Response time: " + end + "; Time needed: " +
+              start.until(end, ChronoUnit.MILLIS))
+          }
+          case _ =>
+        }
         // Read answer
-        return readInAbstract(conn.getInputStream) match{
+        return readInAbstract(inputStream) match{
           case Success(str) => Option(str)
           case Failure(e) => throw e
         }

core/src/main/scala/org/dbpedia/extraction/util/WikiUtil.scala

@@ -167,4 +167,45 @@ object WikiUtil
         result = wikiEmphasisRegex3.replaceAllIn(result, "$1")
         result
     }
+
+  /**
+    this method removes broken information with brackets like (; some info) or ()
+    */
+  def removeBrokenBracketsInAbstracts(text: String): String = {
+    var closeBrackets = 0
+    var result = ""
+    var bracketsWithSemicolon = 0
+    var skipBrackets = 0
+    for (i <- 0 until text.length) {
+      if (text(i) == '(') {
+        if ((i < text.length-1) && (text(i+1) == ';') && bracketsWithSemicolon == 0) {
+          bracketsWithSemicolon = 1
+        }
+        else if (bracketsWithSemicolon > 0) {
+          bracketsWithSemicolon += 1
+        }
+        else if ((i < text.length-1) && (text(i+1) == ')')) {
+          skipBrackets = 2
+        }
+      }
+      else if (text(i) == ')' ) {
+        closeBrackets += 1
+        if (closeBrackets == bracketsWithSemicolon) {
+          bracketsWithSemicolon = 0
+          closeBrackets = 0
+          skipBrackets += 1
+        }
+      }
+      if (bracketsWithSemicolon == 0 && skipBrackets == 0) {
+        // if the previous character was space and the next is also space then we skip it
+        if (!(result.length > 0 && result.last == ' ' && text(i) == ' ' )) {
+          result += text(i)
+        }
+      }
+      if (skipBrackets > 0) {
+        skipBrackets -= 1
+      }
+    }
+    result
+  }
 }

dump/src/main/scala/org/dbpedia/extraction/dump/util/MinidumpDoc.scala

@@ -1,9 +1,14 @@
 package org.dbpedia.extraction.dump.util
 
 import java.io.{File, FileInputStream, PrintWriter}
+import java.util.function.Consumer
+
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream
-import org.apache.jena.query.QueryExecutionFactory
-import org.apache.jena.rdf.model.{Model, ModelFactory}
+import org.apache.jena.query.{QueryExecutionFactory, QuerySolution}
+import org.apache.jena.rdf.model.{Model, ModelFactory, RDFNode, Resource}
+import org.apache.jena.sparql.core.ResultBinding
+import org.apache.jena.sparql.core.Var
+import org.apache.jena.sparql.engine.binding.BindingProject
 
 import scala.collection.mutable
 import scala.collection.mutable.{ArrayBuffer, ListBuffer}
@@ -126,27 +131,42 @@ object MinidumpDoc extends App {
         }
 
         val exec = QueryExecutionFactory.create(queryString.toString(), miniExtraction)
+        val rs = new mutable.LinkedHashSet[String]()
+        exec.execSelect().forEachRemaining(new Consumer[QuerySolution] {
+          override def accept(t: QuerySolution): Unit = rs.add(t.get("t").asResource().getURI)
+        })
 
-        val rs = exec.execSelect()
-
-        while (rs.hasNext) {
-          val qs = rs.next
-          val t = qs.get("t").asResource().getURI
-          if (t.contains(MinidumpDocConfig.dbpediaUriPrefix) ) {
+        for(target <- rs) {
+          if (target.contains(MinidumpDocConfig.dbpediaUriPrefix) ) {
 
-            val englishDbpediaUri = t.replace(MinidumpDocConfig.dbpediaUriPrefix,
+            val englishDbpediaUri = target.replace(MinidumpDocConfig.dbpediaUriPrefix,
               MinidumpDocConfig.englishDbpediaUriPrefix)
 
-            if (minidumpURIs.contains(t) || minidumpURIs.contains(englishDbpediaUri)) {
-              if (!minidumpURIs.contains(t) && minidumpURIs.contains(englishDbpediaUri)) {
+            if (minidumpURIs.contains(target) || minidumpURIs.contains(englishDbpediaUri)) {
+              if (!minidumpURIs.contains(target) && minidumpURIs.contains(englishDbpediaUri)) {
                 saveToMap(englishDbpediaUri, testDef)
               }
               else {
               //  println(s"tests ${testDef.target} on target $t")
-                saveToMap(t, testDef)
+                saveToMap(target, testDef)
               }
             }
           }
+          else {
+            //val citedByURI = "http://dbpedia.org/property/isCitedBy"
+            val newQueryStringGraph = new StringBuilder
+            // maybe it is better to use citedBy URI as a property in the query
+            newQueryStringGraph.append(s"SELECT DISTINCT ?o { <$target> ?p  ?o . }")
+            val exec = QueryExecutionFactory.create(newQueryStringGraph.toString(), miniExtraction)
+            exec.execSelect().forEachRemaining(new Consumer[QuerySolution] {
+              override def accept(t: QuerySolution): Unit = {
+                //println(t.getResource("o").getURI)
+                if (t.get("o").isResource){
+                  rs.add(t.getResource("o").getURI)
+                }
+              }
+            })
+          }
         }
     })
     writeShaclTestsTableToFile()

dump/src/test/resources/extraction-configs/extraction.nif.abstracts.properties

@@ -37,8 +37,8 @@ namespaces=Main
 
 # extractor class names starting with "." are prefixed by "org.dbpedia.extraction.mappings"
 
-extractors=.NifExtractor
-
+extractors=.AbstractExtractor
+remove-broken-brackets=true
 # if ontology and mapping files are not given or do not exist, download info from mappings.dbpedia.org
 # ontology=see universal.properties
 # mappings=see universal.properties

dump/src/test/resources/shacl-tests/properties/dbp_abstract.ttl

@@ -0,0 +1,18 @@
+@base <https://github.com/dbpedia/extraction-framework> .
+@prefix sh: <http://www.w3.org/ns/shacl#>.
+@prefix wgs84: <http://www.w3.org/2003/01/geo/wgs84_pos#>.
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
+@prefix dbr: <http://dbpedia.org/resource/> .
+@prefix dbp: <http://dbpedia.org/property/> .
+@prefix dbo: <http://dbpedia.org/ontology/> .
+@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>.
+@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.
+@prefix prov: <http://www.w3.org/ns/prov#>.
+
+<#en_abstract_validation>
+	a sh:NodeShape  ;
+	sh:targetSubjectsOf <http://dbpedia.org/ontology/abstract> ;
+	sh:property [
+		sh:path <http://dbpedia.org/ontology/abstract>  ;
+        sh:pattern "^((?!\\(\\;).)*$" ;
+	] .