Merge branch 'master' into oersi

Author: fsteeg

.editorconfig

@@ -14,27 +14,28 @@
 # limitations under the License.
 #
 
-root=true
+root = true
 
 [*]
-charset=utf-8
-end_of_line=lf
-insert_final_newline=true
-trim_trailing_whitespace=true
-indent_style=space
-indent_size=4
-continuation_indent_size=8
+charset = utf-8
+end_of_line = lf
+indent_size = 4
+indent_style = space
+insert_final_newline = true
+trim_trailing_whitespace = true
 
-[{*.yml,*.yaml}]
-indent_style=space
-indent_size=2
+[*.bat]
+end_of_line = crlf
+trim_trailing_whitespace = false
 
-[*.gradle]
-indent_style=space
-indent_size=2
+[*.{gradle,yml}]
+indent_size = 2
 
-[*.bat]
-end_of_line=crlf
+[metafacture-io/src/test/resources/org/metafacture/io/compressed.txt]
+insert_final_newline = false
+
+[metamorph/src/test/resources/org/metafacture/metamorph/maps/file-map-test.txt]
+trim_trailing_whitespace = false
 
 [metafacture-runner/src/main/dist/config/java-options.conf]
-end_of_line=crlf
+end_of_line = crlf

.gitattributes

@@ -28,7 +28,7 @@
 *.js text
 *.fom text
 
-# Platform-specific scripts should always use  
+# Platform-specific scripts should always use
 # their native end-of-line markers:
 *.bat text eol=crlf
 *.sh text eol=lf

.gitignore

@@ -22,6 +22,9 @@ target/
 .gradletasknamecache
 build
 
+# Ignore tmp directory
+tmp
+
 # Ignore Eclipse project files:
 .checkstyle
 .pmd
@@ -44,3 +47,10 @@ atlassian-ide-plugin.xml
 
 # Ignore files generated by CI:
 buildbot.keyring
+
+# Ignore VS code/codium config files
+.vscode
+
+# Ignore files with sensitive data
+gradle.properties
+secring.gpg

CONTRIBUTING.md

@@ -2,65 +2,65 @@
 
 [![Build](https://github.com/metafacture/metafacture-core/workflows/Build/badge.svg?branch=master)](https://github.com/metafacture/metafacture-core/actions?query=workflow%3ABuild) [![Quality Gate](https://sonarcloud.io/api/project_badges/measure?project=org.metafacture:metafacture-core&metric=alert_status)](https://sonarcloud.io/dashboard/index/org.metafacture:metafacture-core)
 
-Metafacture is a toolkit for processing semi-structured data with a focus on library metadata. It provides a versatile set of tools for reading, writing and transforming data. Metafacture can be used as a stand-alone application or as a Java library in other applications. The name Metafacture is a portmanteau of the words *meta* data and manu*facture*.
+Metafacture is a toolkit for processing semi-structured data with a focus on library metadata. It provides a versatile set of tools for reading, writing and transforming data. Metafacture can be used as a stand-alone application or as a Java library in other applications. The name Metafacture is a portmanteau of the words *meta*data and manu*facture*.
 
-Metafacture includes a [large number of modules](https://github.com/metafacture/metafacture-documentation/blob/master/flux-commands.md) for operating on semi-structured data. These modules can be combined to build pipelines to perform complex metadata processing tasks. The pipelines can be constructed either in Java code or with the domain-specific language **Flux**. One of the core features of Metafacture is the **Metamorph** module. Metamorph is an xml-based language for specifying transformations of semi-structured data. It can be seamlessly integrated into Java code.
+Metafacture includes a [large number of modules](https://github.com/metafacture/metafacture-documentation/blob/master/flux-commands.md) for operating on semi-structured data. These modules can be combined to build pipelines to perform complex metadata processing tasks. The pipelines can be constructed either in Java code or with the domain-specific language **Flux**. One of the core features of Metafacture is the **Metamorph** module. Metamorph is an XML-based language for specifying transformations of semi-structured data. It can be seamlessly integrated into Java code.
 
 At its heart Metafacture is a framework for implementing modules for metadata processing. This makes Metafacture easily extendable with additional modules. The [plugins and tools page](https://github.com/metafacture/metafacture-core/wiki/Plugins-and-Tools) on the wiki shows supplementary packages and projects which extend Metafacture.
 
-Originally, Metafacture was developed as part of the [Culturegraph](http://culturegraph.org) platform but it is developed independently now and used by others, too: [see who uses Metafacture](https://github.com/metafacture/metafacture-core/wiki/Who-uses-Metafacture).
+Originally, Metafacture was developed as part of the [Culturegraph](http://www.culturegraph.org) platform but it is developed independently now and used by others, too: [see who uses Metafacture](https://github.com/metafacture/metafacture-core/wiki/Who-uses-Metafacture).
 
 # Getting started
 
 You can either use Metafacture as a stand-alone application or include it as a Java library in your own projects.
 
 ## Metafacture as a stand-alone application
- 
-If you are only interested in running Flux scripts without doing any Java programming this is the way to go. The instructions assume that you are using a *nix-like shell.
 
-1. Download the latest distribution package from the [metafacture-core/releases](https://github.com/metafacture/metafacture-core/releases) page. Make sure that you do download a distribution package and _not_ a source code package (the file name should include *-dist*).
+If you are only interested in running Flux scripts without doing any Java programming this is the way to go. The instructions assume that you are using a \*nix-like shell. [See more information in the wiki page about Flux](https://github.com/metafacture/metafacture-core/wiki/Flux-user-guide).
+
+1. Download the latest distribution package from the [release page](https://github.com/metafacture/metafacture-core/releases). Make sure that you do download a distribution package and *not* a source code package (the file name should include `*-dist*`).
 
 2. Extract the downloaded archive:
    ```bash
-   $ tar xzf metafacture-core-VERSION-dist.tar.gz
+   $ tar xzf metafacture-core-$VERSION-dist.tar.gz
    ```
-   This will create a new directory containing a ready-to-use metafacture distribution.
+   This will create a new directory containing a ready-to-use Metafacture distribution.
 3. Change into the newly created directory:
    ```bash
-   $ cd metafacture-core-VERSION
+   $ cd metafacture-core-$VERSION
    ```
 4. Run one of the example scripts:
    ```bash
    $ ./flux.sh examples/read/marc21/read-marc21.flux
    ```
-   This example will print a number of marc21 records on standard out.
+   This example will print a number of MARC 21 records on standard output.
 
-The _examples_ folder contains many more examples which provide a good starting point for learning metafacture. If you have any questions please join our [mailing list](http://lists.dnb.de/mailman/listinfo/metafacture) or use our issue-based discussion forum over at [metafacture-documentation](https://github.com/metafacture/metafacture-documentation).
+The `examples` folder contains many more examples which provide a good starting point for learning Metafacture. If you have any questions please join our [mailing list](http://lists.dnb.de/mailman/listinfo/metafacture) or use our issue-based discussion forum over at [metafacture-documentation](https://github.com/metafacture/metafacture-documentation).
 
 
-## Using Metafacture as a Java libary
+## Using Metafacture as a Java library
 
-If you want use Metafacture in your own Java projects all you need to add some dependencies to your project. As of Metafacture 5 the single metafacture-core package has been replaced with a number of domain-specific packages. You can find the list of packages on [Maven Central](https://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.metafacture%22).
+If you want to use Metafacture in your own Java projects all you need is to add some dependencies to your project. As of Metafacture 5, the single metafacture-core package has been replaced with a number of domain-specific packages. You can find the list of packages on [Maven Central](https://search.maven.org/search?q=g:org.metafacture).
 
 Alternatively, you can simply guess the package names from the top-level folders in the source code repository -- they are the same. For instance, if you want to use Metamorph in your project, simply add the following dependency to your `pom.xml`:
 
 ```xml
 <dependency>
     <groupId>org.metafacture</groupId>
     <artifactId>metamorph</artifactId>
-    <version>VERSION</version>
+    <version>$VERSION</version>
 </dependency>
 ```
 
 or if Gradle is your build tool of choice use:
 
 ```groovy
 dependencies {
-    implementation 'org.metafacture:metamorph:VERSION'
+    implementation 'org.metafacture:metamorph:$VERSION'
 }
 ```
 
-Our integration server automatically publishes successful builds of all branches as snapshot versions on [Sonatype OSS Repository](https://oss.sonatype.org/index.html#nexus-search;quick~metafacture). The version number is derived from the branch name. Snapshot builds from the master branch always have the version "master-SNAPSHOT".
+Occasionally, we publish snapshot builds on [Sonatype OSS Repository](https://oss.sonatype.org/index.html#nexus-search;gav~org.metafacture~~~~). The version number is derived from the branch name. Snapshot builds from the master branch always have the version `master-SNAPSHOT`.
 
 <!--
 TODO: Link to getting started tutorial
@@ -77,13 +77,14 @@ $ git clone https://github.com/metafacture/metafacture-core.git
 $ cd metafacture-core
 ```
 
-2. Invoke the gradle-wrapper to download Gradle and build metafacture-core (on Windows call `gradlew.bat install`):
+2. Invoke the Gradle wrapper to download Gradle and build metafacture-core (on Windows call `gradlew.bat install` instead):
 
 ```bash
 $ ./gradlew install
 ```
 
-The resulting distribution can be found in `metafacture-core/metafacture-runner/build/distributions/`.
+Besides the resulting distribution in `metafacture-core/metafacture-runner/build/distributions/` this also provides builds in your local maven repository.
+
 
 See [Code Quality and Style](https://github.com/metafacture/metafacture-core/wiki/Code-Quality-and-Style) on the wiki for further information on the sources.
 

README.md

@@ -1,5 +1,5 @@
 /*
- * Copyright 2017 Christoph Böhme
+ * Copyright 2017, 2021 Christoph Böhme et al.
  *
  * Licensed under the Apache License, Version 2.0 the "License";
  * you may not use this file except in compliance with the License.
@@ -18,6 +18,7 @@ import java.util.regex.Matcher
 
 plugins {
   id 'org.ajoberstar.grgit' version '2.3.0'
+  id 'org.ec4j.editorconfig' version '0.0.3'
   id 'org.sonarqube' version '2.6.2'
   id 'io.codearte.nexus-staging' version '0.11.0'
 }
@@ -38,16 +39,37 @@ project(':metafacture-runner') {
   apply plugin: 'java'
 }
 
+editorconfig {
+  excludes = [
+    '**/*.beacon',
+    '**/*.bgzf',
+    '**/*.bz2',
+    '**/*.bzip2',
+    '**/*.gzip',
+    '**/*.xz',
+    'gradlew*'
+  ]
+}
+
 subprojects {
   apply plugin: 'signing'
   apply plugin: 'maven'
   apply plugin: 'jacoco'
+  apply plugin: 'checkstyle'
   apply plugin: 'maven-publish'
 
+  check.dependsOn(editorconfigCheck)
+  check.dependsOn(javadoc)
+
   sourceCompatibility = 1.8
   targetCompatibility = 1.8
 
   tasks.withType(JavaCompile) {
+    'all -processing -rawtypes -serial'.split().each {
+        options.compilerArgs << "-Xlint:${it}".toString()
+    }
+
+    options.compilerArgs << '-Werror'
     options.encoding = 'UTF-8'
   }
 
@@ -67,11 +89,33 @@ subprojects {
     group 'build'
   }
 
+  task allDeps(type: DependencyReportTask) {}
+
+  javadoc {
+    options {
+      addBooleanOption 'Xwerror', true
+    }
+  }
+
   artifacts {
     archives sourceJar
     archives javadocJar
   }
 
+  checkstyle {
+    toolVersion '8.30'
+    checkstyleTest.enabled = false
+  }
+
+  test {
+    systemProperties['org.slf4j.simpleLogger.defaultLogLevel'] = 'warn'
+
+    testLogging {
+      showStandardStreams = true
+      exceptionFormat = 'full'
+    }
+  }
+
   signing {
     required {
       scmInfo.isRelease() && gradle.taskGraph.hasTask(tasks.uploadArchives)
@@ -96,16 +140,16 @@ gradle.projectsEvaluated {
     }
 
     def mavenProjectDescription = {
-      name project.mavenName ?: project.name
+      name = project.mavenName ?: project.name
       if (project.description) {
         description project.description
       }
-      url 'https://github.com/culturegraph/metafacture-core'
+      url 'https://github.com/metafacture/metafacture-core'
       inceptionYear '2011'
       developers {
         developer {
           id 'mgeipel'
-          name 'Markus M. Geipel'
+          name = 'Markus M. Geipel'
           email '[email protected]'
           url 'https://github.com/mgeipel'
           roles {
@@ -115,7 +159,7 @@ gradle.projectsEvaluated {
         }
         developer {
           id 'cboehme'
-          name 'Christoph Böhme'
+          name = 'Christoph Böhme'
           email '[email protected]'
           url 'https://github.com/cboehme'
           roles {
@@ -125,18 +169,18 @@ gradle.projectsEvaluated {
         }
       }
       organization {
-        name 'Deutsche Nationalbibliothek'
-        url 'http://dnb.de/'
+        name = 'Metafacture'
+        url 'https://github.com/metafacture'
       }
       licenses {
         license {
-          name 'The Apache License, Version 2.0'
+          name = 'The Apache License, Version 2.0'
           url 'http://www.apache.org/licenses/LICENSE-2.0.txt'
         }
       }
       mailingLists {
         mailingList {
-          name 'Metafacture Mailing List'
+          name = 'Metafacture Mailing List'
           post '[email protected]'
           subscribe '[email protected]'
           unsubscribe '[email protected]'
@@ -154,8 +198,8 @@ gradle.projectsEvaluated {
         url 'https://github.com/metafacture/metafacture-core/issues'
       }
       ciManagement {
-        system 'Travis CI'
-        url 'https://travis-ci.org/metafacture/metafacture-core'
+        system 'Github Actions'
+        url 'https://github.com/metafacture/metafacture-core/actions'
       }
     }
 
@@ -277,7 +321,6 @@ class ScmInfo {
 }
 
 def getScmInfo() {
-  def version = null
   def tag = getGitTag()
   if (tag != null) {
     logger.lifecycle('SCM tag found. Making a release build')
@@ -295,11 +338,11 @@ def getSnapshotVersion() {
     logger.warn('No Git repository found')
     return 'non-scm-build-SNAPSHOT'
   }
-  if (grgit.branch.current().fullName.equals('HEAD')) {
+  if (grgit.branch.current().fullName == 'HEAD') {
     logger.lifecycle('Detached HEAD found')
     return "commit-${grgit.head().id}-SNAPSHOT"
   }
-  if (grgit.branch.current().name.equals('master')) {
+  if (grgit.branch.current().name == 'master') {
     logger.lifecycle('On master branch')
     return 'master-SNAPSHOT'
   }