Skip to content
This repository was archived by the owner on Feb 8, 2019. It is now read-only.

SLIDER-1090: Speed up SliderClient and AM by reading metainfo from su… #4

Open
wants to merge 1 commit into
base: develop
Choose a base branch
from
+165 −9
Open

SLIDER-1090: Speed up SliderClient and AM by reading metainfo from su… #4

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions slider-core/src/main/java/org/apache/slider/client/SliderClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
Expand Down Expand Up @@ -179,6 +180,7 @@
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.InterruptedIOException;
import java.io.PrintStream;
Expand Down Expand Up @@ -1355,6 +1357,41 @@ private int actionPackageList() throws IOException {
return EXIT_SUCCESS;
}

private void createSummaryMetainfoFile(Path srcFile, Path destFile, boolean overwrite)
throws IOException {
FileSystem srcFs = srcFile.getFileSystem(getConfig());
try (
InputStream inputStreamJson = SliderUtils.getApplicationResourceInputStream(
srcFs,
srcFile, "metainfo.json");
InputStream inputStreamXml = SliderUtils.getApplicationResourceInputStream(
srcFs,
srcFile, "metainfo.xml");
) {
InputStream inputStream = null;
Path summaryFileInFs = null;
if (inputStreamJson != null) {
inputStream = inputStreamJson;
summaryFileInFs = new Path(destFile.getParent(),
destFile.getName()+".metainfo.json");
log.info("Found JSON metainfo file in package");
}
else if (inputStreamXml != null) {
inputStream = inputStreamXml;
summaryFileInFs = new Path(destFile.getParent(),
destFile.getName()+".metainfo.xml");
log.info("Found XML metainfo file in package");
}
if (inputStream != null) {
try (FSDataOutputStream dataOutputStream =
sliderFileSystem.getFileSystem().create(summaryFileInFs, overwrite)) {
log.info("Creating summary metainfo file");
IOUtils.copy(inputStream, dataOutputStream);
}
}
}
}

private int actionPackageInstall(ActionPackageArgs actionPackageArgs)
throws YarnException, IOException {
requireArgumentSet(Arguments.ARG_NAME, actionPackageArgs.name);
Expand All @@ -1375,6 +1412,7 @@ private int actionPackageInstall(ActionPackageArgs actionPackageArgs)
log.info("Installing package {} to {} (overwrite set to {})", srcFile,
fileInFs, actionPackageArgs.replacePkg);
fs.copyFromLocalFile(false, actionPackageArgs.replacePkg, srcFile, fileInFs);
createSummaryMetainfoFile(srcFile, fileInFs, actionPackageArgs.replacePkg);

String destPathWithHomeDir = Path
.getPathWithoutSchemeAndAuthority(fileInFs).toString();
Expand Down
60 changes: 51 additions & 9 deletions slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
*/
package org.apache.slider.providers.agent;

import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.slider.common.tools.SliderFileSystem;
Expand All @@ -40,30 +41,54 @@
public class AgentUtils {
private static final Logger log = LoggerFactory.getLogger(AgentUtils.class);

public static Metainfo getApplicationMetainfoFromSummaryFile(
SliderFileSystem fileSystem,
String metainfoPath,
boolean metainfoForAddon
) {
FileSystem fs = fileSystem.getFileSystem();
Path appPathXML = new Path(metainfoPath + ".metainfo.xml");
Path appPathJson = new Path(metainfoPath + ".metainfo.json");
try {
FSDataInputStream appStream = null;
if(fs.exists(appPathXML)) {
appStream = fs.open(appPathXML);
return parseMetainfo(appStream, metainfoForAddon, "xml");
}
else if(fs.exists(appPathJson)) {
appStream = fs.open(appPathJson);
return parseMetainfo(appStream, metainfoForAddon, "json");
}
} catch (IOException e) {
log.info("Failed to get metainfo from acompany file. {}", e.getMessage());
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

1.s/acompany/r/summary file
2. include path.
3. add a log@debug with the stack trace

}
return null;
}

public static Metainfo getApplicationMetainfo(SliderFileSystem fileSystem,
String metainfoPath, boolean metainfoForAddon) throws IOException,
BadConfigException {
log.info("Reading metainfo at {}", metainfoPath);
Metainfo metainfo = getApplicationMetainfoFromSummaryFile(fileSystem,
metainfoPath, metainfoForAddon);
if(metainfo != null) {
log.info("Got metainfo from summary file");
return metainfo;
}

FileSystem fs = fileSystem.getFileSystem();
Path appPath = new Path(metainfoPath);

Metainfo metainfo = null;
AbstractMetainfoParser metainfoParser = null;
if (metainfoForAddon) {
metainfoParser = new AddonPackageMetainfoParser();
} else {
metainfoParser = new MetainfoParser();
}
InputStream metainfoJsonStream = SliderUtils.getApplicationResourceInputStream(
fs, appPath, "metainfo.json");
if (metainfoJsonStream == null) {
InputStream metainfoXMLStream = SliderUtils.getApplicationResourceInputStream(
fs, appPath, "metainfo.xml");
if (metainfoXMLStream != null) {
metainfo = metainfoParser.fromXmlStream(metainfoXMLStream);
metainfo = parseMetainfo(metainfoXMLStream, metainfoForAddon, "xml");
}
} else {
metainfo = metainfoParser.fromJsonStream(metainfoJsonStream);
metainfo = parseMetainfo(metainfoJsonStream, metainfoForAddon, "json");
}

if (metainfo == null) {
Expand All @@ -74,6 +99,23 @@ public static Metainfo getApplicationMetainfo(SliderFileSystem fileSystem,
return metainfo;
}

private static Metainfo parseMetainfo(InputStream stream,
boolean metainfoForAddon, String type) throws IOException {
AbstractMetainfoParser metainfoParser = null;
if (metainfoForAddon) {
metainfoParser = new AddonPackageMetainfoParser();
} else {
metainfoParser = new MetainfoParser();
}
if(type.equals("xml")) {
return metainfoParser.fromXmlStream(stream);
}
else if(type.equals("json")) {
return metainfoParser.fromJsonStream(stream);
}
return null;
}

static DefaultConfig getDefaultConfig(SliderFileSystem fileSystem,
String appDef, String configFileName)
throws IOException {
Expand Down
76 changes: 76 additions & 0 deletions slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
package org.apache.slider.providers.agent;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.providers.agent.application.metadata.Metainfo;
import org.apache.slider.tools.TestUtility;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class TestAgentUtils {
protected static final Logger log =
LoggerFactory.getLogger(TestAgentUtils.class);
@Rule
public TemporaryFolder folder = new TemporaryFolder();
private static final String metainfo_str = "<metainfo>\n"
+ " <schemaVersion>2.0</schemaVersion>\n"
+ " <application>\n"
+ " <name>MYTESTAPPLICATION</name>\n"
+ " <comment>\n"
+ " My Test Application\n"
+ " </comment>\n"
+ " <version>1.0</version>\n"
+ " <type>YARN-APP</type>\n"
+ " <components>\n"
+ " <component>\n"
+ " <name>REST</name>\n"
+ " <category>MASTER</category>\n"
+ " <commandScript>\n"
+ " <script>scripts/rest.py</script>\n"
+ " <scriptType>PYTHON</scriptType>\n"
+ " <timeout>600</timeout>\n"
+ " </commandScript>\n"
+ " </component>\n"
+ " </components>\n"
+ " </application>\n"
+ "</metainfo>";

@Test
public void testGetApplicationMetainfo() throws Exception {
String zipFileName = TestUtility.createAppPackage(
folder,
"testpkg",
"test.zip",
"target/test-classes/org/apache/slider/common/tools/test");
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.getLocal(configuration);
log.info("fs working dir is {}", fs.getWorkingDirectory().toString());
SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration);

// Without accompany metainfo file, read metainfo from the zip file
Metainfo metainfo = AgentUtils.getApplicationMetainfo(
sliderFileSystem, zipFileName, false);
Assert.assertNotNull(metainfo.getApplication());
Assert.assertEquals("STORM", metainfo.getApplication().getName());

// With accompany metainfo file, read metainfo from the accompany file
String acompanyFileName = zipFileName + ".metainfo.xml";
File f = new File(acompanyFileName);
try (BufferedWriter writer = new BufferedWriter(new FileWriter(f))) {
writer.write(metainfo_str);
}
metainfo = AgentUtils.getApplicationMetainfo(
sliderFileSystem, zipFileName, false);
Assert.assertNotNull(metainfo.getApplication());
Assert.assertEquals("MYTESTAPPLICATION", metainfo.getApplication().getName());
}
}