Fix abstract extraction

core/doc/HowTo-release-DBpedia.txt

@@ -22,7 +22,7 @@ release. It might not be complete. Please also consult with the others!
    - Commit the files to the hg repository
    - Don't change the files anymore. The whole extraction should use the same version.
 
- - for AbstractExtractor: insert Wikipedia dumps into a local MySQL database using ...dump.sql.Import.scala
+ - for PlainAbstractExtractor: insert Wikipedia dumps into a local MySQL database using ...dump.sql.Import.scala
    - adjust the LocalSettings.php of mw-modified: specify username+password for the database and the database prefix
    TODO: more in-depth explanations about abstract extraction
 

core/src/main/scala/org/dbpedia/extraction/config/Config.scala

@@ -278,7 +278,7 @@ class Config(val configPath: String) extends
       longAbstractsProperty = this.getProperty("long-abstracts-property", "abstract").trim,
       shortAbstractMinLength = this.getProperty("short-abstract-min-length", "200").trim.toInt,
       abstractTags = this.getProperty("abstract-tags", "query,pages,page,extract").trim,
-      removeBrokenBracketsProperty = this.getProperty("remove-broken-brackets", "false").trim
+      removeBrokenBracketsProperty = this.getProperty("remove-broken-brackets-plain-abstracts", "false").trim
     )
   } match{
     case Success(s) => s
@@ -294,7 +294,8 @@ class Config(val configPath: String) extends
       writeAnchor = this.getProperty("nif-write-anchor", "false").trim.toBoolean,
       writeLinkAnchor = this.getProperty("nif-write-link-anchor", "true").trim.toBoolean,
       abstractsOnly = this.getProperty("nif-extract-abstract-only", "true").trim.toBoolean,
-      cssSelectorMap = this.getClass.getClassLoader.getResource("nifextractionconfig.json") //static config file in core/src/main/resources
+      cssSelectorMap = this.getClass.getClassLoader.getResource("nifextractionconfig.json"), //static config file in core/src/main/resources
+      removeBrokenBracketsProperty = this.getProperty("remove-broken-brackets-html-abstracts", "false").trim.toBoolean
     )
   } match{
     case Success(s) => s
@@ -349,7 +350,8 @@ object Config{
     writeAnchor: Boolean,
     writeLinkAnchor: Boolean,
     abstractsOnly: Boolean,
-    cssSelectorMap: URL
+    cssSelectorMap: URL,
+    removeBrokenBracketsProperty: Boolean
   )
 
   /**

core/src/main/scala/org/dbpedia/extraction/mappings/AbstractExtractorWikipedia.scala → core/src/main/scala/org/dbpedia/extraction/mappings/HtmlAbstractExtractor.scala

@@ -13,7 +13,7 @@ import scala.language.reflectiveCalls
  * Created: 5/19/14 9:21 AM
  */
 
-class AbstractExtractorWikipedia(
+class HtmlAbstractExtractor(
   context : {
     def ontology : Ontology
     def language : Language

core/src/main/scala/org/dbpedia/extraction/mappings/MissingAbstractsExtractor.scala

@@ -54,7 +54,7 @@ extends PageNodeExtractor
 
     private val language = context.language.wikiCode
 
-    private val logger = Logger.getLogger(classOf[AbstractExtractor].getName)
+    private val logger = Logger.getLogger(classOf[PlainAbstractExtractor].getName)
 
     //private val apiParametersFormat = "uselang="+language+"&format=xml&action=parse&prop=text&title=%s&text=%s"
     private val apiParametersFormat = "uselang="+language+"&format=xml&action=query&prop=extracts&exintro=&explaintext=&titles=%s"

core/src/main/scala/org/dbpedia/extraction/mappings/NifExtractor.scala

@@ -14,7 +14,7 @@ import scala.language.reflectiveCalls
 /**
   * Extracts page html.
   *
-  * Based on AbstractExtractor, major difference is the parameter
+  * Based on PlainAbstractExtractor, major difference is the parameter
   * apiParametersFormat = "action=parse&prop=text&section=0&format=xml&page=%s"
   *
   * This class produces all nif related datasets for the abstract as well as the short-, long-abstracts datasets.
@@ -69,7 +69,7 @@ class NifExtractor(
 
 object NifExtractor{
   //TODO check if this function is still relevant
-  //copied from AbstractExtractor
+  //copied from PlainAbstractExtractor
   def postProcessExtractedHtml(pageTitle: WikiTitle, text: String): String =
   {
     val startsWithLowercase =

core/src/main/scala/org/dbpedia/extraction/mappings/AbstractExtractor.scala → core/src/main/scala/org/dbpedia/extraction/mappings/PlainAbstractExtractor.scala

@@ -30,7 +30,7 @@ import scala.language.reflectiveCalls
 
 @deprecated("replaced by NifExtractor.scala: which will extract the whole page content including the abstract", "2016-10")
 @ExtractorAnnotation("abstract extractor")
-class AbstractExtractor(
+class PlainAbstractExtractor(
   context : {
     def ontology : Ontology
     def language : Language
@@ -39,7 +39,7 @@ class AbstractExtractor(
 )
 extends WikiPageExtractor
 {
-  protected val logger = Logger.getLogger(classOf[AbstractExtractor].getName)
+  protected val logger = Logger.getLogger(classOf[PlainAbstractExtractor].getName)
   this.getClass.getClassLoader.getResource("myproperties.properties")
 
 
@@ -81,7 +81,7 @@ extends WikiPageExtractor
 
         //Retrieve page text
         val text = mwConnector.retrievePage(pageNode.title, apiParametersFormat, pageNode.isRetry) match {
-          case Some(t) => AbstractExtractor.postProcessExtractedHtml(pageNode.title, replacePatterns(t))
+          case Some(t) => PlainAbstractExtractor.postProcessExtractedHtml(pageNode.title, replacePatterns(t))
           case None => return Seq.empty
         }
 
@@ -146,7 +146,7 @@ extends WikiPageExtractor
 
     private def replacePatterns(abst: String): String= {
       var ret = abst
-      for ((regex, replacement) <- AbstractExtractor.patternsToRemove) {
+      for ((regex, replacement) <- PlainAbstractExtractor.patternsToRemove) {
         val matches = regex.pattern.matcher(ret)
         if (matches.find()) {
           ret = matches.replaceAll(replacement)
@@ -219,7 +219,7 @@ extends WikiPageExtractor
 
 }
 
-object AbstractExtractor {
+object PlainAbstractExtractor {
 
   //TODO check if this function is still relevant
   def postProcessExtractedHtml(pageTitle: WikiTitle, text: String): String =

core/src/main/scala/org/dbpedia/extraction/nif/WikipediaNifExtractor.scala

@@ -47,6 +47,8 @@ class WikipediaNifExtractor(
   protected val recordAbstracts: Boolean = !context.configFile.nifParameters.isTestRun  //not! will create dbpedia short and long abstracts
   protected val shortAbstractLength: Int = context.configFile.abstractParameters.shortAbstractMinLength
   protected val abstractsOnly: Boolean = context.configFile.nifParameters.abstractsOnly
+  protected val removeBrokenBrackets: Boolean = context.configFile.nifParameters.removeBrokenBracketsProperty
+
   override protected val templateString: String = Namespaces.names(context.language).get(Namespace.Template.code) match {
     case Some(x) => x
     case None => "Template"
@@ -69,7 +71,13 @@ class WikipediaNifExtractor(
     //this is only dbpedia relevant: for singling out long and short abstracts
 
     if (recordAbstracts && extractionResults.section.id == "abstract" && extractionResults.getExtractedLength > 0) {
-      List(longQuad(subjectIri, WikiUtil.removeBrokenBracketsInAbstracts(extractionResults.getExtractedText), graphIri), shortQuad(subjectIri, WikiUtil.removeBrokenBracketsInAbstracts(getShortAbstract(extractionResults)), graphIri))
+      val (cleanLongAbstract, cleanShortAbstract) = if (removeBrokenBrackets) {
+        (WikiUtil.removeBrokenBracketsInAbstracts(extractionResults.getExtractedText),
+          WikiUtil.removeBrokenBracketsInAbstracts(getShortAbstract(extractionResults)))
+      } else {
+        (extractionResults.getExtractedText, getShortAbstract(extractionResults))
+      }
+      List(longQuad(subjectIri, cleanLongAbstract, graphIri), shortQuad(subjectIri, cleanShortAbstract, graphIri))
     }
     else
       List()

core/src/main/scala/org/dbpedia/extraction/util/WikiUtil.scala

@@ -173,12 +173,12 @@ object WikiUtil
     */
   def removeBrokenBracketsInAbstracts(text: String): String = {
     var closeBrackets = 0
-    var result = ""
+    val result = new StringBuilder()
     var bracketsWithSemicolon = 0
     var skipBrackets = 0
     for (i <- 0 until text.length) {
       if (text(i) == '(') {
-        if ((i < text.length-1) && (text(i+1) == ';') && bracketsWithSemicolon == 0) {
+        if ((i < text.length-1) && (text(i+1) == ';' || text(i+1) ==',') && bracketsWithSemicolon == 0) {
           bracketsWithSemicolon = 1
         }
         else if (bracketsWithSemicolon > 0) {
@@ -198,14 +198,14 @@ object WikiUtil
       }
       if (bracketsWithSemicolon == 0 && skipBrackets == 0) {
         // if the previous character was space and the next is also space then we skip it
-        if (!(result.length > 0 && result.last == ' ' && text(i) == ' ' )) {
-          result += text(i)
+        if (!(result.nonEmpty && result.last == ' ' && text(i) == ' ' )) {
+          result.append(text(i))
         }
       }
       if (skipBrackets > 0) {
         skipBrackets -= 1
       }
     }
-    result
+    result.toString().trim
   }
 }

core/src/test/scala/org/dbpedia/extraction/mappings/NifExtractorTest.scala

@@ -57,7 +57,7 @@ class NifExtractorTest extends FunSuite {
 
   private def getHtml(title:WikiTitle): String={
     mwConnector.retrievePage(title, context.configFile.nifParameters.nifQuery) match{
-      case Some(pc) => AbstractExtractor.postProcessExtractedHtml(title, pc)
+      case Some(pc) => PlainAbstractExtractor.postProcessExtractedHtml(title, pc)
       case None => ""
     }
   }

core/src/test/scala/org/dbpedia/extraction/mappings/AbstractExtractorTest.scala → core/src/test/scala/org/dbpedia/extraction/mappings/PlainAbstractExtractorTest.scala

@@ -12,7 +12,7 @@ import scala.io.Source
 import scala.language.reflectiveCalls
 
 @Ignore  // unignore to test; MediaWiki server has to be in place
-class AbstractExtractorTest
+class PlainAbstractExtractorTest
 {
     private val testDataRootDir = new File("core/src/test/resources/org/dbpedia/extraction/mappings")
     private val configFilePath = "extraction-framework/dump/extraction.nif.abstracts.properties"
@@ -47,7 +47,7 @@ class AbstractExtractorTest
         def language = Language.English
         def configFile : Config = new Config(configFilePath)
     }
-    private val extractor = new AbstractExtractor(context)
+    private val extractor = new PlainAbstractExtractor(context)
 
     private val parser = WikiParser.getInstance()
 

dump/extraction.abstracts.properties

@@ -29,7 +29,7 @@ namespaces=Main
 
 # extractor class names starting with "." are prefixed by "org.dbpedia.extraction.mappings"
 
-extractors=.AbstractExtractor
+extractors=.PlainAbstractExtractor
 
 # if ontology and mapping files are not given or do not exist, download info from mappings.dbpedia.org
 # ontology=../ontology.xml see universal.properties

dump/src/test/resources/extraction-configs/extraction.nif.abstracts.properties

@@ -37,8 +37,8 @@ namespaces=Main
 
 # extractor class names starting with "." are prefixed by "org.dbpedia.extraction.mappings"
 
-extractors=.AbstractExtractor
-remove-broken-brackets=true
+extractors=.HtmlAbstractExtractor
+remove-broken-brackets-html-abstracts=true
 # if ontology and mapping files are not given or do not exist, download info from mappings.dbpedia.org
 # ontology=see universal.properties
 # mappings=see universal.properties

dump/src/test/resources/extraction-configs/extraction.plain.abstracts.properties

@@ -0,0 +1,96 @@
+# make sure to fill out the ../core/src/main/resources/universal.properties first and reinstall
+
+# Replace with your Wikipedia dump download directory (should not change over the course of a release)
+base-dir=./target/minidumptest/base
+log-dir=./target/minidumptest/log
+spark-local-dir=./target/minidumptest/spark-local
+spark-master=local[32]
+
+# The log file directory - used to store all log files created in the course of all extractions
+#
+#log-dir= see: ../core/src/main/resources/universal.properties
+
+# WikiPages failed to extract in the first try can be retried with this option (especially interesting when extraction from the mediawiki api)
+retry-failed-pages=false
+
+# Source file. If source file name ends with .gz or .bz2, it is unzipped on the fly.
+# Must exist in the directory xxwiki/yyyymmdd and have the prefix xxwiki-yyyymmdd-
+# where xx is the wiki code and yyyymmdd is the dump date.
+
+# default:
+# source=pages-articles.xml.bz2
+
+# alternatives:
+# source=pages-articles.xml.gz
+# source=pages-articles.xml
+
+# use only directories that contain a 'download-complete' file? Default is false.
+require-download-complete=false
+
+# List of languages or article count ranges, e.g. 'en,de,fr' or '10000-20000' or '10000-', or '@mappings'
+# NOTE sync with minidumps
+#languages=af,als,am,an,arz,ast,azb,ba,bar,bat-smg,bpy,br,bs,bug,cdo,ce,ceb,ckb,cv,fo,fy,gd,he,hsb,ht,ia,ilo,io,is,jv,ka,kn,ku,ky,la,lb,li,lmo,mai,mg,min,ml,mn,mr,mrj,ms,mt,my,mzn,nah,nap,nds,ne,new,nn,no,oc,or,os,pa,pms,pnb,qu,sa,sah,scn,sco,sh,si,simple,sq,su,sw,ta,te,tg,th,tl,tt,uz,vec,wa,xmf,yo,zh-min-nan,zh-yue
+languages=en
+# default namespaces: Main, File, Category, Template
+# we only want abstracts for articles -> only main namespace
+namespaces=Main
+
+# extractor class names starting with "." are prefixed by "org.dbpedia.extraction.mappings"
+
+extractors=.PlainAbstractExtractor
+remove-broken-brackets-plain-abstracts=true
+# if ontology and mapping files are not given or do not exist, download info from mappings.dbpedia.org
+# ontology=see universal.properties
+# mappings=see universal.properties
+
+# Serialization URI policies and file formats. Quick guide:
+# uri-policy keys: uri, generic, xml-safe, reject-long
+# uri-policy position modifiers: -subjects, -predicates, -objects, -datatypes, -contexts
+# uri-policy values: comma-separated languages or '*' for all languages
+# format values: n-triples, n-quads, turtle-triples, turtle-quads, trix-triples, trix-quads
+# See http://git.io/DBpedia-serialization-format-properties for details.
+
+# For backwards compatibility, en uses generic URIs. All others use local IRIs.
+# uri-policy.uri=uri:en; generic:en; xml-safe-predicates:*
+uri-policy.iri=generic:en; xml-safe-predicates:*
+
+# NT is unreadable anyway - might as well use URIs for en
+# format.nt.gz=n-triples;uri-policy.uri
+# format.nq.gz=n-quads;uri-policy.uri
+
+# Turtle is much more readable - use nice IRIs for all languages
+format.ttl.bz2=turtle-triples;uri-policy.iri
+#format.tql.bz2=turtle-quads;uri-policy.iri
+
+
+#the following parameters are for the mediawiki api connection used in nif and abstract extraction
+mwc-apiUrl=https://{{LANG}}.wikipedia.org/w/api.php
+mwc-maxRetries=5
+mwc-connectMs=4000
+mwc-readMs=30000
+mwc-sleepFactor=2000
+
+#parameters specific for the abstract extraction
+abstract-query=&format=xml&action=query&prop=extracts&exintro=&explaintext=&titles=%s
+# the tag path of the XML tags under which the result is expected
+abstract-tags=api,query,pages,page,extract
+# the properties used to specify long- and short abstracts (should not change)
+short-abstracts-property=rdfs:comment
+long-abstracts-property=abstract
+# the short abstract is at least this long
+short-abstract-min-length=200
+
+#parameters specific to the nif extraction
+
+#only extract abstract (not the whole page)
+nif-extract-abstract-only=false
+#the request query string
+nif-query=&format=xml&action=parse&prop=text&page=%s&pageid=%d
+#the xml path of the response
+nif-tags=api,parse,text
+# will leave out the long and short abstract datasets
+nif-isTestRun=false
+# will write all anchor texts for each nif instance
+nif-write-anchor=true
+# write only the anchor text for link instances
+nif-write-link-anchor=true

dump/src/test/scala/org/dbpedia/extraction/dump/ExtractionTest.scala

@@ -2,10 +2,9 @@ package org.dbpedia.extraction.dump
 
 import java.io.File
 import java.util.concurrent.ConcurrentLinkedQueue
-
 import org.apache.commons.io.FileUtils
 import org.dbpedia.extraction.config.Config
-import org.dbpedia.extraction.dump.TestConfig.{date, genericConfig, mappingsConfig, minidumpDir, nifAbstractConfig, sparkSession, wikidataConfig}
+import org.dbpedia.extraction.dump.TestConfig.{classLoader, date, genericConfig, mappingsConfig, minidumpDir, nifAbstractConfig, plainAbstractConfig, sparkSession, wikidataConfig}
 import org.dbpedia.extraction.dump.extract.ConfigLoader
 import org.dbpedia.extraction.dump.tags.ExtractionTestTag
 import org.scalatest.{BeforeAndAfterAll, DoNotDiscover, FunSuite}
@@ -45,9 +44,13 @@ class ExtractionTest extends FunSuite with BeforeAndAfterAll {
     extract(wikidataConfig, jobsRunning)
   }
 
-  test("extract nifAbstract datasets", ExtractionTestTag) {
-    val jobsRunning = new ConcurrentLinkedQueue[Future[Unit]]()
-    extract(nifAbstractConfig, jobsRunning)
+  test("extract abstract datasets", ExtractionTestTag) {
+    val jobsRunning1 = new ConcurrentLinkedQueue[Future[Unit]]()
+    extract(nifAbstractConfig, jobsRunning1)
+    Utils.renameAbstractsDatasetFiles("html")
+    val jobsRunning2 = new ConcurrentLinkedQueue[Future[Unit]]()
+    extract(plainAbstractConfig, jobsRunning2)
+    Utils.renameAbstractsDatasetFiles("plain")
   }
 
   def extractSpark(config: Config, jobsRunning: ConcurrentLinkedQueue[Future[Unit]]): Unit = {

dump/src/test/scala/org/dbpedia/extraction/dump/TestConfig.scala

@@ -17,6 +17,7 @@ object TestConfig {
   val mappingsConfig = new Config(classLoader.getResource("extraction-configs/mappings.extraction.minidump.properties").getFile)
   val genericConfig = new Config(classLoader.getResource("extraction-configs/generic-spark.extraction.minidump.properties").getFile)
   val nifAbstractConfig = new Config(classLoader.getResource("extraction-configs/extraction.nif.abstracts.properties").getFile)
+  val plainAbstractConfig = new Config(classLoader.getResource("extraction-configs/extraction.plain.abstracts.properties").getFile)
   val wikidataConfig = new Config(classLoader.getResource("extraction-configs/wikidata.extraction.properties").getFile)
   val minidumpDir = new File(classLoader.getResource("minidumps").getFile)
 

dump/src/test/scala/org/dbpedia/extraction/dump/Utils.scala

@@ -1,7 +1,8 @@
 package org.dbpedia.extraction.dump
 
-import org.dbpedia.extraction.dump.TestConfig.classLoader
+import org.dbpedia.extraction.dump.TestConfig.{classLoader, date}
 
+import java.io.File
 import java.util.Properties
 
 object Utils {
@@ -42,4 +43,18 @@ object Utils {
       case None => TestConfig.defaultTestGroup
     }
   }
+
+  def renameAbstractsDatasetFiles(datasetName: String): Unit = {
+    val minidumpDir = new File("./target/minidumptest/base")
+    minidumpDir.listFiles().foreach(f => {
+      val longAbstractsFile = new File( s"./target/minidumptest/base/${f.getName}/$date/${f.getName}-$date-long-abstracts.ttl.bz2")
+      if (longAbstractsFile.exists()) {
+        longAbstractsFile.renameTo(new File(s"./target/minidumptest/base/${f.getName}/$date/${f.getName}-$date-long-abstracts-$datasetName.ttl.bz2"))
+      }
+      val shortAbstractsFile = new File( s"./target/minidumptest/base/${f.getName}/$date/${f.getName}-$date-short-abstracts.ttl.bz2")
+      if (shortAbstractsFile.exists()) {
+        shortAbstractsFile.renameTo(new File(s"./target/minidumptest/base/${f.getName}/$date/${f.getName}-$date-short-abstracts-$datasetName.ttl.bz2"))
+      }
+    })
+  }
 }

live/live.default.xml

@@ -294,7 +294,7 @@
                 </produces>
             </extractor>
 
-            <extractor name="org.dbpedia.extraction.mappings.AbstractExtractorWikipedia" status="ACTIVE">
+            <extractor name="org.dbpedia.extraction.mappings.HtmlAbstractExtractor" status="ACTIVE">
                 <produces>
                     <matchPattern type="EXACT">
                         <s></s>