public final class SliderUtils extends Object
| Modifier and Type | Class and Description |
|---|---|
static class |
SliderUtils.DelayedHalt
Callable for async/scheduled halt
|
static class |
SliderUtils.OnDemandReportStringifier
This wraps ApplicationReports and generates a string version
iff the toString() operator is invoked
|
| Modifier and Type | Field and Description |
|---|---|
static String |
DOCKER
name of docker program
|
static String |
JAVA_SECURITY_KRB5_KDC |
static String |
JAVA_SECURITY_KRB5_REALM |
static int |
NODE_LIST_LIMIT |
static String |
OPENSSL
name of openssl program
|
static String |
PYTHON
name of python program
|
static String |
WINUTILS
Winutils
|
| Modifier and Type | Method and Description |
|---|---|
static void |
addBuildInfo(Map<String,String> info,
String prefix)
Add the cluster build information; this will include Hadoop details too
|
static String |
appendToURL(String base,
String... paths)
Append a list of paths, inserting "/" signs as appropriate
|
static String |
appendToURL(String base,
String path)
Add a subpath to an existing URL.
|
static void |
applyCommandLineRoleOptsToRoleMap(Map<String,Map<String,String>> clusterRoleMap,
Map<String,Map<String,String>> commandOptions)
Apply a set of command line options to a cluster role map
|
static String |
appReportToString(org.apache.hadoop.yarn.api.records.ApplicationReport r,
String separator) |
static Map<String,org.apache.hadoop.yarn.api.records.ApplicationReport> |
buildApplicationReportMap(List<org.apache.hadoop.yarn.api.records.ApplicationReport> instances,
org.apache.hadoop.yarn.api.records.YarnApplicationState minState,
org.apache.hadoop.yarn.api.records.YarnApplicationState maxState)
Built a (sorted) map of application reports, mapped to the instance name
The list is sorted, and the addition process does not add a report
if there is already one that exists.
|
static ClasspathConstructor |
buildClasspath(String sliderConfDir,
String libdir,
org.apache.hadoop.conf.Configuration config,
SliderFileSystem sliderFileSystem,
boolean usingMiniMRCluster)
Build up the classpath for execution
-behaves very differently on a mini test cluster vs a production
production one.
|
static Map<String,String> |
buildEnvMap(Map<String,String> roleOpts)
Build the environment map from a role option map, finding all entries
beginning with "env.", adding them to a map of (prefix-removed)
env vars
|
static String |
checkCredentialCacheFile()
return a description of whether the current user has created credential
cache files from kerberos servers
|
static String |
checkForRequiredNativeLibraries()
Check for any needed libraries being present.
|
static void |
checkPort(String name,
InetSocketAddress address,
int connectTimeout) |
static void |
checkPort(String hostname,
int port,
int connectTimeout) |
static void |
checkURL(String name,
String url,
int timeout) |
static List<String> |
collectionToStringList(Collection c)
Take a collection, return a list containing the string value of every
element in the collection.
|
static int |
compareTo(long left,
long right)
A compareTo function that converts the result of a long
comparision into the integer that
Comparable
expects. |
static int |
compareTwoLongsReverse(long x,
long y)
Compare two long values for sorting.
|
static String |
containersToString(List<ContainerInformation> containers,
String version,
Set<String> components) |
static String |
containerToString(org.apache.hadoop.yarn.api.records.Container container) |
static int |
copyDirectory(org.apache.hadoop.conf.Configuration conf,
org.apache.hadoop.fs.Path srcDirPath,
org.apache.hadoop.fs.Path destDirPath,
org.apache.hadoop.fs.permission.FsPermission permission)
Copy a directory to a new FS -both paths must be qualified.
|
static org.apache.hadoop.yarn.conf.YarnConfiguration |
createConfiguration()
Create a configuration with Slider-specific tuning.
|
static FilenameFilter |
createJarFilter()
Accept all filenames ending with
.jar |
static org.apache.hadoop.fs.Path |
createLocalPath(File file)
Create a file:// path from a local file
|
static Map<String,Map<String,String>> |
deepClone(Map<String,Map<String,String>> src) |
static void |
deleteDirectoryTree(File dir)
recursive directory delete
|
static ForkedProcessService |
execCommand(String name,
int status,
long timeoutMillis,
org.slf4j.Logger logger,
String outputString,
String... commands)
Execute a command for a test operation
|
static org.apache.hadoop.fs.Path |
extractImagePath(CoreFileSystem fs,
MapOperations internalOptions) |
static String |
extractNodeLabel(org.apache.hadoop.yarn.api.records.NodeReport report)
Get a string node label value from a node report
|
static boolean |
filter(String value,
Set<String> filters)
Filter a string value given a set of filters
|
static boolean |
filter(String value,
String filter)
Filter a string value given a single filter
|
static File |
findContainingJar(Class my_class)
Find a containing JAR
|
static File |
findContainingJarOrFail(Class clazz)
Find a containing JAR
|
static int |
findFreePort(int start,
int limit) |
static void |
forceLogin()
Force an early login: This catches any auth problems early rather than
in RPC operations
|
static String |
getApplicationDefinitionPath(ConfTreeOperations conf)
return the HDFS path where the application package has been uploaded
manually or by using slider client (install package command)
|
static InputStream |
getApplicationResourceInputStream(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path appPath,
String entry) |
static String |
getClientConfigPath()
return the path to the slider-client.xml used by the current running
slider command
|
static String |
getCurrentCommandPath()
return the path to the currently running slider command
|
static org.apache.hadoop.security.UserGroupInformation |
getCurrentUser()
Get the current user -relays to
UserGroupInformation.getCurrentUser()
with any Slider-specific post processing and exception handling |
static String |
getHdpVersion()
Retrieve the HDP version if it is an HDP cluster, or null otherwise
|
static int |
getIntValue(Map<String,String> roleMap,
String key,
int defVal,
int min,
int max)
Get the int value of a role
|
static String |
getJDKInfo()
return the version and path of the JDK invoking the current running
slider command
|
static String |
getKerberosRealm() |
static InetSocketAddress |
getRmAddress(org.apache.hadoop.conf.Configuration conf) |
static InetSocketAddress |
getRmSchedulerAddress(org.apache.hadoop.conf.Configuration conf) |
static String |
getSliderVersion()
Retrieve the version of the current Slider install
|
static Map<String,String> |
getSystemEnv() |
static String |
getSystemEnv(String property) |
static Timer |
haltAM(int status,
String text,
int delay)
trigger a JVM halt with no clean shutdown at all
|
static boolean |
hasAppFinished(org.apache.hadoop.yarn.api.records.ApplicationReport report) |
static boolean |
initProcessSecurity(org.apache.hadoop.conf.Configuration conf)
Turn on security.
|
static String |
instanceDetailsToString(String name,
org.apache.hadoop.yarn.api.records.ApplicationReport report,
List<ContainerInformation> containers,
String version,
Set<String> components,
boolean verbose)
Convert the instance details of an application to a string
|
static boolean |
isAddressDefined(InetSocketAddress address)
probe to see if the address
|
static boolean |
isClusternameValid(String name)
Normalize a cluster name then verify that it is valid
|
static boolean |
isHadoopClusterSecure(org.apache.hadoop.conf.Configuration conf)
Flag to indicate whether the cluster is in secure mode
|
static boolean |
isHdp()
Query to find if it is an HDP cluster
|
static boolean |
isNotEmpty(List l)
Probe for a list existing and not being empty
|
static boolean |
isNotEmpty(Map m)
Probe for a map existing and not being empty
|
static boolean |
isPortAvailable(int port)
See if a port is available for listening on by trying to listen
on it and seeing if that works or fails.
|
static boolean |
isRmSchedulerAddressDefined(org.apache.hadoop.conf.Configuration conf)
probe to see if the RM scheduler is defined
|
static boolean |
isSet(String s) |
static boolean |
isUnset(String s)
Implementation of set-ness, groovy definition of true/false for a string
|
static String |
join(Collection collection,
String separator)
Join an collection of objects with a separator that appears after every
instance in the list -including at the end
|
static String |
join(Collection collection,
String separator,
boolean trailing)
Join an collection of objects with a separator that appears after every
instance in the list -optionally at the end
|
static String |
join(String[] collection,
String separator)
Join an array of strings with a separator that appears after every
instance in the list -including at the end
|
static String |
join(String[] collection,
String separator,
boolean trailing)
Join an array of strings with a separator that appears after every
instance in the list -optionally at the end
|
static String |
joinWithInnerSeparator(String separator,
Object... collection)
Join an array of strings with a separator that appears after every
instance in the list -except at the end
|
static String |
listDir(File dir)
List a directory in the local filesystem
|
static org.apache.hadoop.conf.Configuration |
loadSliderClientXML()
Attempt to load the slider client resource.
|
static String |
mandatoryEnvVariable(String key)
Resolve a mandatory environment variable
|
static boolean |
maybeInitSecurity(org.apache.hadoop.conf.Configuration conf)
Init security if the cluster configuration declares the cluster is secure
|
static void |
maybeVerifyWinUtilsValid()
Strictly verify that windows utils is present.
|
static Map<String,String> |
mergeEntries(Map<String,String> dest,
Iterable<Map.Entry<String,String>> entries)
Merge a set of entries into a map.
|
static Map<String,String> |
mergeMap(Map<String,String> first,
Map<String,String> second)
Merge in one map to another -all entries in the second map are
merged into the first -overwriting any duplicate keys.
|
static <T1,T2> Map<T1,T2> |
mergeMaps(Map<T1,T2> first,
Map<T1,T2> second)
Generic map merge logic
|
static <T1,T2> Map<T1,T2> |
mergeMapsIgnoreDuplicateKeys(Map<T1,T2> first,
Map<T1,T2> second)
Generic map merge logic
|
static boolean |
oldIsClusternameValid(String name) |
static int |
parseAndValidate(String errorKey,
String valS,
int defVal,
int min,
int max)
Parse an int value, replacing it with defval if undefined;
|
static org.apache.hadoop.conf.Configuration |
patchConfiguration(org.apache.hadoop.conf.Configuration conf)
Take an existing conf and patch it for Slider's needs.
|
static String |
propertiesToString(Properties props) |
static void |
putAllJars(Map<String,org.apache.hadoop.yarn.api.records.LocalResource> providerResources,
SliderFileSystem sliderFileSystem,
org.apache.hadoop.fs.Path tempPath,
String libDir,
String srcPath)
Submit a JAR containing and map it
|
static void |
putAmTarGzipAndUpdate(Map<String,org.apache.hadoop.yarn.api.records.LocalResource> providerResources,
SliderFileSystem sliderFileSystem)
Submit the AM tar.gz containing all dependencies and map it
|
static org.apache.hadoop.yarn.api.records.LocalResource |
putJar(Map<String,org.apache.hadoop.yarn.api.records.LocalResource> providerResources,
SliderFileSystem sliderFileSystem,
Class clazz,
org.apache.hadoop.fs.Path tempPath,
String libdir,
String jarName)
Submit a JAR containing a specific class and map it
|
static URL |
registerClientResource()
Register the client resource in
SliderKeys.SLIDER_CLIENT_XML
for Configuration instances. |
static String |
reportToString(org.apache.hadoop.yarn.api.records.ApplicationReport report)
convert an AM report to a string for diagnostics
|
static String |
requestToString(org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest request) |
static File |
requiredFile(String filename,
String role)
A required file
|
static String |
sequenceToString(CharSequence charSequence)
Convert a char sequence to a string.
|
static void |
setInfoTime(Map info,
String keyHumanTime,
String keyMachineTime,
long time)
Set the time for an information (human, machine) timestamp pair of fields.
|
static void |
setRmAddress(org.apache.hadoop.conf.Configuration conf,
String rmAddr) |
static void |
setRmSchedulerAddress(org.apache.hadoop.conf.Configuration conf,
String rmAddr) |
static void |
sortApplicationReport(List<org.apache.hadoop.yarn.api.records.ApplicationReport> instances)
Sorts the given list of application reports
Finished instances are ordered by finished time and running/accepted instances are
ordered by start time
Finally Instance are order by finished instances coming after running instances
|
static void |
sortApplicationsByMostRecent(List<org.apache.hadoop.yarn.api.records.ApplicationReport> instances)
Sorts the given list of application reports, most recently started
or finished instance first.
|
static Map<String,String> |
sortedMap(Map<String,String> source)
Take a map and produce a sorted equivalent
|
static String |
stringify(Throwable t) |
static String |
stringify(org.apache.hadoop.yarn.api.records.URL url)
Convert a YARN URL into a string value of a normal URL
|
static String |
stringifyMap(Map<String,String> map)
Convert a map to a multi-line string for printing
|
static Map<String,String> |
stringMapClone(Map<String,String> src) |
static void |
tarGzipFolder(File srcFolder,
File tarGzipFile,
FilenameFilter filter)
Given a source folder create a tar.gz file
|
static String |
toGMTString(long time)
Convert an epoch time to a GMT time.
|
static Map<String,String> |
toMap(Properties properties)
Convert a properties instance to a string map.
|
static String |
translateTrailingHeapUnit(String heapsize) |
static String |
truncate(String toTruncate,
int maxSize)
Truncate the given string to a maximum length provided
with a pad (...) added to the end if expected size if more than 10.
|
static void |
validateClientConfigFile()
validate if slider-client.xml under the path can be opened
|
static void |
validateClusterName(String clustername)
verify that the supplied cluster name is valid
|
static void |
validateExe(String program,
File exe)
Validate an executable
|
static void |
validateHDFSFile(SliderFileSystem sliderFileSystem,
String pathStr)
validate if a file on HDFS can be open
|
static void |
validateOpenSSLEnv(org.slf4j.Logger logger) |
static void |
validatePythonEnv(org.slf4j.Logger logger) |
static void |
validateSliderClientEnvironment(org.slf4j.Logger logger)
Validate the slider client-side execution environment.
|
static void |
validateSliderServerEnvironment(org.slf4j.Logger logger,
boolean dependencyChecks)
Validate the slider server-side execution environment.
|
static void |
verifyFileExists(File file,
org.slf4j.Logger errorlog)
Verify that a file exists
|
static void |
verifyFileSize(String program,
File exe,
int minFileSize) |
static void |
verifyIsDir(File dir,
org.slf4j.Logger errorlog)
Verify that a path refers to a directory.
|
static void |
verifyIsFile(String program,
File exe) |
static File |
verifyKeytabExists(org.apache.hadoop.conf.Configuration siteConf,
String prop)
Verify that a keytab property is defined and refers to a non-empty file
|
static String |
verifyOptionSet(org.apache.hadoop.conf.Configuration configuration,
String key,
boolean allowEmpty)
verify that a config option is set
|
static void |
verifyPrincipalSet(org.apache.hadoop.conf.Configuration conf,
String principal)
Verify that a Kerberos principal has been set -if not fail
with an error message that actually tells you what is missing
|
static void |
verifyUnixExe(String program,
File exe)
Verify that a Unix exe works
|
static void |
verifyWindowsExe(String program,
File exe)
Look for the windows executable and check it has the right headers.
|
static void |
write(File outfile,
byte[] data,
boolean createParent)
Write bytes to a file
|
static void |
zipFolder(File srcFolder,
File zipFile)
Given a source folder create zipped file
|
public static final String JAVA_SECURITY_KRB5_REALM
public static final String JAVA_SECURITY_KRB5_KDC
public static final String WINUTILS
public static final String OPENSSL
public static final String PYTHON
public static final String DOCKER
public static final int NODE_LIST_LIMIT
public static boolean isUnset(String s)
s - stringpublic static boolean isSet(String s)
public static boolean isNotEmpty(List l)
l - listpublic static boolean isNotEmpty(Map m)
m - mappublic static String translateTrailingHeapUnit(String heapsize) throws BadConfigException
BadConfigExceptionpublic static void deleteDirectoryTree(File dir) throws IOException
dir - dir to deleteIOException - on any problempublic static File findContainingJarOrFail(Class clazz) throws IOException
clazz - class to findIOException - any IO problem, including the class not having a
classloaderFileNotFoundException - if the class did not resolve to a filepublic static File findContainingJar(Class my_class) throws IOException
my_class - class to findIOException - any IO problem, including the class not having a
classloaderpublic static void checkPort(String hostname, int port, int connectTimeout) throws IOException
IOExceptionpublic static void checkPort(String name, InetSocketAddress address, int connectTimeout) throws IOException
IOExceptionpublic static void checkURL(String name, String url, int timeout) throws IOException
IOExceptionpublic static File requiredFile(String filename, String role) throws IOException
role - role of the file (for errors)filename - the filenameExitUtil.ExitException - if the file is missingIOExceptionpublic static boolean isClusternameValid(String name)
name - proposed cluster namepublic static boolean oldIsClusternameValid(String name)
public static int copyDirectory(org.apache.hadoop.conf.Configuration conf,
org.apache.hadoop.fs.Path srcDirPath,
org.apache.hadoop.fs.Path destDirPath,
org.apache.hadoop.fs.permission.FsPermission permission)
throws IOException,
BadClusterStateException
conf - conf filesrcDirPath - src dirdestDirPath - dest dirpermission - permission for the dest directory; null means "default"IOExceptionBadClusterStateExceptionpublic static org.apache.hadoop.yarn.conf.YarnConfiguration createConfiguration()
public static org.apache.hadoop.conf.Configuration patchConfiguration(org.apache.hadoop.conf.Configuration conf)
conf - configurationpublic static List<String> collectionToStringList(Collection c)
c - collectionpublic static String join(Collection collection, String separator)
collection - collection to call toString() on each elementseparator - separator stringpublic static String join(Collection collection, String separator, boolean trailing)
collection - collection to call toString() on each elementseparator - separator stringtrailing - add a trailing entry or notpublic static String join(String[] collection, String separator)
collection - stringsseparator - separator stringpublic static String join(String[] collection, String separator, boolean trailing)
collection - stringsseparator - separator stringtrailing - add a trailing entry or notpublic static String joinWithInnerSeparator(String separator, Object... collection)
collection - stringsseparator - separator stringpublic static String mandatoryEnvVariable(String key) throws BadClusterStateException
key - env varBadClusterStateExceptionpublic static String appReportToString(org.apache.hadoop.yarn.api.records.ApplicationReport r, String separator)
public static String instanceDetailsToString(String name, org.apache.hadoop.yarn.api.records.ApplicationReport report, List<ContainerInformation> containers, String version, Set<String> components, boolean verbose)
name - instance namereport - the application reportverbose - verbose outputpublic static String containersToString(List<ContainerInformation> containers, String version, Set<String> components)
public static boolean filter(String value, String filter)
value - the string value to checkfilter - a single string filterpublic static boolean filter(String value, Set<String> filters)
value - the string value to checkfilters - a set of string filterspublic static void sortApplicationsByMostRecent(List<org.apache.hadoop.yarn.api.records.ApplicationReport> instances)
instances - list of instancespublic static void sortApplicationReport(List<org.apache.hadoop.yarn.api.records.ApplicationReport> instances)
instances - list of instancespublic static Map<String,org.apache.hadoop.yarn.api.records.ApplicationReport> buildApplicationReportMap(List<org.apache.hadoop.yarn.api.records.ApplicationReport> instances, org.apache.hadoop.yarn.api.records.YarnApplicationState minState, org.apache.hadoop.yarn.api.records.YarnApplicationState maxState)
instances - list of intancesminState - minimum YARN state to be includedmaxState - maximum YARN state to be includedpublic static Map<String,String> sortedMap(Map<String,String> source)
source - source mappublic static Map<String,String> toMap(Properties properties)
properties - source property objectpublic static Map<String,String> mergeMap(Map<String,String> first, Map<String,String> second)
first - first map -the updated one.second - the map that is merged inpublic static Map<String,String> mergeEntries(Map<String,String> dest, Iterable<Map.Entry<String,String>> entries)
dest - destinationentries - entriespublic static <T1,T2> Map<T1,T2> mergeMaps(Map<T1,T2> first, Map<T1,T2> second)
T1 - key typeT2 - value typefirst - first mapsecond - second mappublic static <T1,T2> Map<T1,T2> mergeMapsIgnoreDuplicateKeys(Map<T1,T2> first, Map<T1,T2> second)
T1 - key typeT2 - value typefirst - first mapsecond - second mappublic static String stringifyMap(Map<String,String> map)
map - map to stringifypublic static int getIntValue(Map<String,String> roleMap, String key, int defVal, int min, int max) throws BadConfigException
roleMap - map of role key->val entrieskey - key the key to look fordefVal - default value to use if the key is not in the mapmin - min value or -1 for do not checkmax - max value or -1 for do not checkBadConfigException - if the value could not be parsedpublic static int parseAndValidate(String errorKey, String valS, int defVal, int min, int max) throws BadConfigException
errorKey - key to use in exceptionsdefVal - default value to use if the key is not in the mapmin - min value or -1 for do not checkmax - max value or -1 for do not checkBadConfigException - if the value could not be parsedpublic static InetSocketAddress getRmAddress(org.apache.hadoop.conf.Configuration conf)
public static InetSocketAddress getRmSchedulerAddress(org.apache.hadoop.conf.Configuration conf)
public static boolean isRmSchedulerAddressDefined(org.apache.hadoop.conf.Configuration conf)
conf - configpublic static boolean isAddressDefined(InetSocketAddress address)
address - network addresspublic static void setRmAddress(org.apache.hadoop.conf.Configuration conf,
String rmAddr)
public static void setRmSchedulerAddress(org.apache.hadoop.conf.Configuration conf,
String rmAddr)
public static boolean hasAppFinished(org.apache.hadoop.yarn.api.records.ApplicationReport report)
public static String containerToString(org.apache.hadoop.yarn.api.records.Container container)
public static String reportToString(org.apache.hadoop.yarn.api.records.ApplicationReport report)
report - the reportpublic static String stringify(org.apache.hadoop.yarn.api.records.URL url)
url - URLpublic static int findFreePort(int start,
int limit)
public static boolean isPortAvailable(int port)
port - port to listen topublic static Map<String,String> buildEnvMap(Map<String,String> roleOpts)
roleOpts - role options. This can be null, meaning the
role is undefinedpublic static void applyCommandLineRoleOptsToRoleMap(Map<String,Map<String,String>> clusterRoleMap, Map<String,Map<String,String>> commandOptions)
clusterRoleMap - cluster role map to merge ontocommandOptions - command optspublic static void validateClusterName(String clustername) throws BadCommandArgumentsException
clustername - cluster nameBadCommandArgumentsException - if it is invalidpublic static void verifyPrincipalSet(org.apache.hadoop.conf.Configuration conf,
String principal)
throws BadConfigException
conf - configuration to look atprincipal - key of principalBadConfigException - if the key is not setpublic static boolean isHadoopClusterSecure(org.apache.hadoop.conf.Configuration conf)
conf - configuration to look atpublic static boolean maybeInitSecurity(org.apache.hadoop.conf.Configuration conf)
throws IOException,
SliderException
conf - configuration to look atIOException - cluster is secureSliderException - the configuration/process is invalidpublic static boolean initProcessSecurity(org.apache.hadoop.conf.Configuration conf)
throws IOException,
SliderException
conf - configuration to build up securityIOException - IO/Net problemsBadConfigException - the configuration and system state are inconsistentSliderExceptionpublic static void forceLogin()
throws IOException
IOException - if the login failspublic static org.apache.hadoop.yarn.api.records.LocalResource putJar(Map<String,org.apache.hadoop.yarn.api.records.LocalResource> providerResources, SliderFileSystem sliderFileSystem, Class clazz, org.apache.hadoop.fs.Path tempPath, String libdir, String jarName) throws IOException, SliderException
providerResources - provider map to build upsliderFileSystem - remote fsclazz - class to look forlibdir - lib directoryjarName - At the destinationIOException - trouble copying to HDFSSliderExceptionpublic static void putAllJars(Map<String,org.apache.hadoop.yarn.api.records.LocalResource> providerResources, SliderFileSystem sliderFileSystem, org.apache.hadoop.fs.Path tempPath, String libDir, String srcPath) throws IOException, SliderException
providerResources - provider map to build upsliderFileSystem - remote fslibDir - lib directorysrcPath - copy jars fromIOException, - SliderException trouble copying to HDFSIOExceptionSliderExceptionpublic static FilenameFilter createJarFilter()
.jarpublic static void putAmTarGzipAndUpdate(Map<String,org.apache.hadoop.yarn.api.records.LocalResource> providerResources, SliderFileSystem sliderFileSystem) throws IOException, SliderException
providerResources - provider map to build upsliderFileSystem - remote fsIOException, - SliderException trouble copying to HDFSIOExceptionSliderExceptionpublic static Map<String,Map<String,String>> deepClone(Map<String,Map<String,String>> src)
public static String listDir(File dir)
dir - directorypublic static org.apache.hadoop.fs.Path createLocalPath(File file)
file - file to point the pathpublic static org.apache.hadoop.security.UserGroupInformation getCurrentUser()
throws IOException
UserGroupInformation.getCurrentUser()
with any Slider-specific post processing and exception handlingIOException - on a failure to get the credentialspublic static String getKerberosRealm()
public static URL registerClientResource()
SliderKeys.SLIDER_CLIENT_XML
for Configuration instances.public static org.apache.hadoop.conf.Configuration loadSliderClientXML()
public static String sequenceToString(CharSequence charSequence)
charSequence - sourcepublic static ClasspathConstructor buildClasspath(String sliderConfDir, String libdir, org.apache.hadoop.conf.Configuration config, SliderFileSystem sliderFileSystem, boolean usingMiniMRCluster)
sliderConfDir - relative path to the dir containing slider config
options to put on the classpath -or nulllibdir - directory containing the JAR filesconfig - the configurationusingMiniMRCluster - flag to indicate the MiniMR cluster is in use
(and hence the current classpath should be used, not anything built up)public static void verifyIsDir(File dir, org.slf4j.Logger errorlog) throws FileNotFoundException
dir - the directoryerrorlog - log for output on an errorFileNotFoundException - if it is not a directorypublic static void verifyFileExists(File file, org.slf4j.Logger errorlog) throws FileNotFoundException
file - fileerrorlog - log for output on an errorFileNotFoundExceptionpublic static String verifyOptionSet(org.apache.hadoop.conf.Configuration configuration, String key, boolean allowEmpty) throws BadConfigException
configuration - configkey - keyBadConfigException - if the key is missingpublic static File verifyKeytabExists(org.apache.hadoop.conf.Configuration siteConf, String prop) throws BadConfigException
siteConf - configurationprop - property to look forBadConfigException - on a failurepublic static String toGMTString(long time)
time - timestamppublic static void addBuildInfo(Map<String,String> info, String prefix)
info - cluster infoprefix - prefix for the build infopublic static void setInfoTime(Map info, String keyHumanTime, String keyMachineTime, long time)
Date class.info - info fieldskeyHumanTime - name of human time keykeyMachineTime - name of machine timetime - timestamppublic static org.apache.hadoop.fs.Path extractImagePath(CoreFileSystem fs, MapOperations internalOptions) throws SliderException, IOException
SliderExceptionIOExceptionpublic static Timer haltAM(int status, String text, int delay)
status - status code for exittext - text messagedelay - delay in millispublic static String propertiesToString(Properties props)
public static String appendToURL(String base, String path)
base - base path/URLpath - subpathpublic static String appendToURL(String base, String... paths)
base - base path/URLpaths - subpathspublic static String truncate(String toTruncate, int maxSize)
toTruncate - string to truncate; may be nullmaxSize - maximum sizepublic static String extractNodeLabel(org.apache.hadoop.yarn.api.records.NodeReport report)
report - node reportpublic static int compareTo(long left,
long right)
Comparable
expects.left - left sideright - right sidepublic static void zipFolder(File srcFolder, File zipFile) throws IOException
srcFolder - zipFile - IOExceptionpublic static void tarGzipFolder(File srcFolder, File tarGzipFile, FilenameFilter filter) throws IOException
srcFolder - tarGzipFile - IOExceptionpublic static String getHdpVersion()
public static boolean isHdp()
public static String getSliderVersion()
public static InputStream getApplicationResourceInputStream(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path appPath, String entry) throws IOException
IOExceptionpublic static String checkForRequiredNativeLibraries()
public static void maybeVerifyWinUtilsValid()
throws IOException,
SliderException
IOException - on any problem reading the fileFileNotFoundException - if the file is not considered validSliderExceptionpublic static void verifyIsFile(String program, File exe) throws FileNotFoundException
FileNotFoundExceptionpublic static void verifyFileSize(String program, File exe, int minFileSize) throws FileNotFoundException
FileNotFoundExceptionpublic static void verifyWindowsExe(String program, File exe) throws IOException
File.canRead() doesn't work on windows, so the reading
is mandatory.program - program name for errorsexe - executableIOException - IOEpublic static void verifyUnixExe(String program, File exe) throws IOException
program - program name for errorsexe - executableIOException - IOEpublic static void validateExe(String program, File exe) throws IOException
program - program name for errorsexe - program to look atIOExceptionpublic static void write(File outfile, byte[] data, boolean createParent) throws IOException
outfile - output filedata - data to writecreateParent - flag to indicate that the parent dir should
be createdIOException - on any IO problempublic static ForkedProcessService execCommand(String name, int status, long timeoutMillis, org.slf4j.Logger logger, String outputString, String... commands) throws IOException, SliderException
name - name in errorstatus - status code expectedtimeoutMillis - timeout in millis for process to finishlogger - outputString - optional string to grep for (must not span a line)commands - commands @return the processIOException - on any failure.SliderExceptionpublic static void validateSliderClientEnvironment(org.slf4j.Logger logger)
throws IOException,
SliderException
logger - logger to log to on normal executionIOException - on IO failuresSliderException - on validation failurespublic static void validateSliderServerEnvironment(org.slf4j.Logger logger,
boolean dependencyChecks)
throws IOException,
SliderException
logger - logger to log to on normal executiondependencyChecks - flag to indicate checks for agent dependenciesIOException - on IO failuresSliderException - on validation failurespublic static void validateOpenSSLEnv(org.slf4j.Logger logger)
throws IOException,
SliderException
IOExceptionSliderExceptionpublic static void validatePythonEnv(org.slf4j.Logger logger)
throws IOException,
SliderException
IOExceptionSliderExceptionpublic static String getCurrentCommandPath()
NullPointerException - - If the pathname argument is nullSecurityException - - if a security manager exists and its checkPermission method
doesn't allow getting the ProtectionDomainpublic static String getApplicationDefinitionPath(ConfTreeOperations conf) throws BadConfigException
conf - configurationBadConfigExceptionpublic static String getClientConfigPath()
SecurityException - - if a security manager exists and its checkPermission method
denies access to the class loader for the classpublic static void validateClientConfigFile()
throws IOException
IOException - : the file can't be found or openpublic static void validateHDFSFile(SliderFileSystem sliderFileSystem, String pathStr) throws IOException, URISyntaxException
IOException - the file can't be found or openedURISyntaxExceptionpublic static String getJDKInfo()
SecurityException - - if a security manager exists and its checkPropertyAccess
method doesn't allow access to the specified system property.public static String checkCredentialCacheFile() throws IOException, BadConfigException
IOExceptionBadConfigExceptionSecurityException - - if a security manager exists and its checkPropertyAccess
method doesn't allow access to the specified system property.public static int compareTwoLongsReverse(long x,
long y)
x-y
is inapplicablex - x valuey - y valuepublic static String requestToString(org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest request)
Copyright © 2014–2015 The Apache Software Foundation. All rights reserved.