From 3a5ec2280e86b31f94925c55cce8fdf3697822aa Mon Sep 17 00:00:00 2001 From: Shanyu Zhao Date: Sat, 27 Feb 2016 19:30:58 -0800 Subject: [PATCH] SLIDER-1090: Speed up SliderClient and AM by reading metainfo from summary file Signed-off-by: Shanyu Zhao --- .../apache/slider/client/SliderClient.java | 38 ++++++++++ .../slider/providers/agent/AgentUtils.java | 60 ++++++++++++--- .../providers/agent/TestAgentUtils.java | 76 +++++++++++++++++++ 3 files changed, 165 insertions(+), 9 deletions(-) create mode 100644 slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentUtils.java diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index cf15dfc47..f6a0887c1 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -25,6 +25,7 @@ import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocatedFileStatus; @@ -179,6 +180,7 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; import java.io.InterruptedIOException; import java.io.PrintStream; @@ -1355,6 +1357,41 @@ private int actionPackageList() throws IOException { return EXIT_SUCCESS; } + private void createSummaryMetainfoFile(Path srcFile, Path destFile, boolean overwrite) + throws IOException { + FileSystem srcFs = srcFile.getFileSystem(getConfig()); + try ( + InputStream inputStreamJson = SliderUtils.getApplicationResourceInputStream( + srcFs, + srcFile, "metainfo.json"); + InputStream inputStreamXml = SliderUtils.getApplicationResourceInputStream( + srcFs, + srcFile, "metainfo.xml"); + ) { + InputStream inputStream = null; + Path summaryFileInFs = null; + if (inputStreamJson != null) { + inputStream = inputStreamJson; + summaryFileInFs = new Path(destFile.getParent(), + destFile.getName()+".metainfo.json"); + log.info("Found JSON metainfo file in package"); + } + else if (inputStreamXml != null) { + inputStream = inputStreamXml; + summaryFileInFs = new Path(destFile.getParent(), + destFile.getName()+".metainfo.xml"); + log.info("Found XML metainfo file in package"); + } + if (inputStream != null) { + try (FSDataOutputStream dataOutputStream = + sliderFileSystem.getFileSystem().create(summaryFileInFs, overwrite)) { + log.info("Creating summary metainfo file"); + IOUtils.copy(inputStream, dataOutputStream); + } + } + } + } + private int actionPackageInstall(ActionPackageArgs actionPackageArgs) throws YarnException, IOException { requireArgumentSet(Arguments.ARG_NAME, actionPackageArgs.name); @@ -1375,6 +1412,7 @@ private int actionPackageInstall(ActionPackageArgs actionPackageArgs) log.info("Installing package {} to {} (overwrite set to {})", srcFile, fileInFs, actionPackageArgs.replacePkg); fs.copyFromLocalFile(false, actionPackageArgs.replacePkg, srcFile, fileInFs); + createSummaryMetainfoFile(srcFile, fileInFs, actionPackageArgs.replacePkg); String destPathWithHomeDir = Path .getPathWithoutSchemeAndAuthority(fileInFs).toString(); diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java index b38dc6e04..333487942 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java @@ -16,6 +16,7 @@ */ package org.apache.slider.providers.agent; +import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.slider.common.tools.SliderFileSystem; @@ -40,30 +41,54 @@ public class AgentUtils { private static final Logger log = LoggerFactory.getLogger(AgentUtils.class); + public static Metainfo getApplicationMetainfoFromSummaryFile( + SliderFileSystem fileSystem, + String metainfoPath, + boolean metainfoForAddon + ) { + FileSystem fs = fileSystem.getFileSystem(); + Path appPathXML = new Path(metainfoPath + ".metainfo.xml"); + Path appPathJson = new Path(metainfoPath + ".metainfo.json"); + try { + FSDataInputStream appStream = null; + if(fs.exists(appPathXML)) { + appStream = fs.open(appPathXML); + return parseMetainfo(appStream, metainfoForAddon, "xml"); + } + else if(fs.exists(appPathJson)) { + appStream = fs.open(appPathJson); + return parseMetainfo(appStream, metainfoForAddon, "json"); + } + } catch (IOException e) { + log.info("Failed to get metainfo from acompany file. {}", e.getMessage()); + } + return null; + } + public static Metainfo getApplicationMetainfo(SliderFileSystem fileSystem, String metainfoPath, boolean metainfoForAddon) throws IOException, BadConfigException { log.info("Reading metainfo at {}", metainfoPath); + Metainfo metainfo = getApplicationMetainfoFromSummaryFile(fileSystem, + metainfoPath, metainfoForAddon); + if(metainfo != null) { + log.info("Got metainfo from summary file"); + return metainfo; + } + FileSystem fs = fileSystem.getFileSystem(); Path appPath = new Path(metainfoPath); - Metainfo metainfo = null; - AbstractMetainfoParser metainfoParser = null; - if (metainfoForAddon) { - metainfoParser = new AddonPackageMetainfoParser(); - } else { - metainfoParser = new MetainfoParser(); - } InputStream metainfoJsonStream = SliderUtils.getApplicationResourceInputStream( fs, appPath, "metainfo.json"); if (metainfoJsonStream == null) { InputStream metainfoXMLStream = SliderUtils.getApplicationResourceInputStream( fs, appPath, "metainfo.xml"); if (metainfoXMLStream != null) { - metainfo = metainfoParser.fromXmlStream(metainfoXMLStream); + metainfo = parseMetainfo(metainfoXMLStream, metainfoForAddon, "xml"); } } else { - metainfo = metainfoParser.fromJsonStream(metainfoJsonStream); + metainfo = parseMetainfo(metainfoJsonStream, metainfoForAddon, "json"); } if (metainfo == null) { @@ -74,6 +99,23 @@ public static Metainfo getApplicationMetainfo(SliderFileSystem fileSystem, return metainfo; } + private static Metainfo parseMetainfo(InputStream stream, + boolean metainfoForAddon, String type) throws IOException { + AbstractMetainfoParser metainfoParser = null; + if (metainfoForAddon) { + metainfoParser = new AddonPackageMetainfoParser(); + } else { + metainfoParser = new MetainfoParser(); + } + if(type.equals("xml")) { + return metainfoParser.fromXmlStream(stream); + } + else if(type.equals("json")) { + return metainfoParser.fromJsonStream(stream); + } + return null; + } + static DefaultConfig getDefaultConfig(SliderFileSystem fileSystem, String appDef, String configFileName) throws IOException { diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentUtils.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentUtils.java new file mode 100644 index 000000000..db8c138de --- /dev/null +++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentUtils.java @@ -0,0 +1,76 @@ +package org.apache.slider.providers.agent; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.slider.common.tools.SliderFileSystem; +import org.apache.slider.providers.agent.application.metadata.Metainfo; +import org.apache.slider.tools.TestUtility; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TestAgentUtils { + protected static final Logger log = + LoggerFactory.getLogger(TestAgentUtils.class); + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + private static final String metainfo_str = "\n" + + " 2.0\n" + + " \n" + + " MYTESTAPPLICATION\n" + + " \n" + + " My Test Application\n" + + " \n" + + " 1.0\n" + + " YARN-APP\n" + + " \n" + + " \n" + + " REST\n" + + " MASTER\n" + + " \n" + + " \n" + + " PYTHON\n" + + " 600\n" + + " \n" + + " \n" + + " \n" + + " \n" + + ""; + + @Test + public void testGetApplicationMetainfo() throws Exception { + String zipFileName = TestUtility.createAppPackage( + folder, + "testpkg", + "test.zip", + "target/test-classes/org/apache/slider/common/tools/test"); + Configuration configuration = new Configuration(); + FileSystem fs = FileSystem.getLocal(configuration); + log.info("fs working dir is {}", fs.getWorkingDirectory().toString()); + SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration); + + // Without accompany metainfo file, read metainfo from the zip file + Metainfo metainfo = AgentUtils.getApplicationMetainfo( + sliderFileSystem, zipFileName, false); + Assert.assertNotNull(metainfo.getApplication()); + Assert.assertEquals("STORM", metainfo.getApplication().getName()); + + // With accompany metainfo file, read metainfo from the accompany file + String acompanyFileName = zipFileName + ".metainfo.xml"; + File f = new File(acompanyFileName); + try (BufferedWriter writer = new BufferedWriter(new FileWriter(f))) { + writer.write(metainfo_str); + } + metainfo = AgentUtils.getApplicationMetainfo( + sliderFileSystem, zipFileName, false); + Assert.assertNotNull(metainfo.getApplication()); + Assert.assertEquals("MYTESTAPPLICATION", metainfo.getApplication().getName()); + } +}