diff --git a/model-scoring-core/pom.xml b/model-scoring-core/pom.xml index e7ec13d..f364bec 100644 --- a/model-scoring-core/pom.xml +++ b/model-scoring-core/pom.xml @@ -589,5 +589,10 @@ maven-artifact 2.2.1 + + org.apache.hadoop + hadoop-aws + ${tap.hadoop2.version} + \ No newline at end of file diff --git a/model-scoring-core/src/main/resources/reference.conf b/model-scoring-core/src/main/resources/reference.conf index c237fbb..36d1526 100644 --- a/model-scoring-core/src/main/resources/reference.conf +++ b/model-scoring-core/src/main/resources/reference.conf @@ -7,6 +7,13 @@ trustedanalytics.scoring-engine { archive-mar = "" } +#-Dtrustedanalytics.aws.fs.s3a.access.key="ACCESS_KEY" +trustedanalytics.aws { + fs.s3a.proxy.host = "" + fs.s3a.proxy.port = "" + fs.s3a.access.key = "" + fs.s3a.secret.key = "" +} #-Dtrustedanalytics.scoring.port="SOME_PORT" trustedanalytics.scoring { identifier = "ia" diff --git a/model-scoring-core/src/main/scala/org/trustedanalytics/scoring/ScoringEngineHelper.scala b/model-scoring-core/src/main/scala/org/trustedanalytics/scoring/ScoringEngineHelper.scala index ca8211d..ce377a7 100644 --- a/model-scoring-core/src/main/scala/org/trustedanalytics/scoring/ScoringEngineHelper.scala +++ b/model-scoring-core/src/main/scala/org/trustedanalytics/scoring/ScoringEngineHelper.scala @@ -17,14 +17,20 @@ package org.trustedanalytics.scoring import java.io.File +import java.net.URI import java.util.{ArrayList => JArrayList} +import com.typesafe.config.ConfigFactory +import org.apache.commons.io.FileUtils +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.Path import org.slf4j.LoggerFactory import org.trustedanalytics.model.archive.format.ModelArchiveFormat import org.trustedanalytics.scoring.interfaces.Model object ScoringEngineHelper { private val logger = LoggerFactory.getLogger(this.getClass) + val config = ConfigFactory.load(this.getClass.getClassLoader) /** * @@ -39,13 +45,50 @@ object ScoringEngineHelper { model.output().deep == revisedModel.output().deep } + private def getAWSConfig(): Configuration = { + val proxyHost = config.getString("trustedanalytics.aws.fs.s3a.proxy.host") + val proxyPort = config.getString("trustedanalytics.aws.fs.s3a.proxy.port") + val accessKey = config.getString("trustedanalytics.aws.fs.s3a.access.key") + val secretKey = config.getString("trustedanalytics.aws.fs.s3a.secret.key") + + val cfg = new Configuration() + if(proxyHost != "" && proxyPort != "") { + cfg.set("fs.s3a.proxy.host",proxyHost) + cfg.set("fs.s3a.proxy.port",proxyPort) + } + if(accessKey != "") cfg.set("fs.s3a.access.key",accessKey) else throw new Exception("Configuration do not have AWS access key") + if(secretKey != "") cfg.set("fs.s3a.secret.key",secretKey) else throw new Exception("Configuration do not have AWS secret key") + cfg + } def getModel(marFilePath: String): Model = { if (marFilePath != "") { - logger.info("calling ModelArchiveFormat to get the model") - ModelArchiveFormat.read(new File(marFilePath), this.getClass.getClassLoader, None) - } - else { - null + try { + logger.info("Calling ModelArchiveFormat.read() to load the model stored locally") + return ModelArchiveFormat.read(new File(marFilePath), this.getClass.getClassLoader, None) + } catch { + case e: Exception => + logger.info("Unale to load model from local filesystem, trying to load the model from HDFS") + var tempMarFile: File = null + tempMarFile = File.createTempFile("model", ".mar") + try + { + val cfg = getAWSConfig() + val hdfsFileSystem = org.apache.hadoop.fs.FileSystem.get(new URI(marFilePath), cfg) + hdfsFileSystem.copyToLocalFile(false, new Path(marFilePath), new Path(tempMarFile.getAbsolutePath)) + val hdfsMarFilePath = tempMarFile.getAbsolutePath + sys.addShutdownHook(FileUtils.deleteQuietly(tempMarFile)) // Delete temporary directory on exit + return ModelArchiveFormat.read(new File(hdfsMarFilePath), this.getClass.getClassLoader, None) + } catch { + case e: Exception => + logger.info("Unale to load model from HDFS...\n"+e.getMessage) + logger.info("\n"+e.getStackTraceString) + return null + } finally { + FileUtils.deleteQuietly(tempMarFile) + } + } + } else { + return null } } } \ No newline at end of file