| Modifier and Type | Method and Description |
|---|---|
AbstractClientProvider |
SliderClientAPI.createClientProvider(String provider)
Get the provider for this cluster
|
AbstractClientProvider |
SliderClient.createClientProvider(String provider) |
org.apache.hadoop.fs.Path |
SliderClient.createPathThatMustExist(String uri)
Create a path that must exist in the cluster fs
|
org.apache.hadoop.registry.client.api.RegistryOperations |
SliderClientAPI.getRegistryOperations()
Get the registry binding.
|
org.apache.hadoop.registry.client.api.RegistryOperations |
SliderClient.getRegistryOperations() |
protected void |
SliderClient.initHadoopBinding()
Perform everything needed to init the hadoop binding.
|
Map<String,org.apache.hadoop.registry.client.types.ServiceRecord> |
SliderClient.listRegistryInstances()
List instances in the registry for the current user
|
AggregateConf |
SliderClient.loadInstanceDefinition(String name,
boolean resolved)
Load the instance definition.
|
AggregateConf |
SliderClient.loadInstanceDefinitionUnresolved(String name,
org.apache.hadoop.fs.Path clusterDirectory)
Load the instance definition.
|
AggregateConf |
SliderClient.loadPersistedClusterDescription(String clustername)
Load the persistent cluster description
|
org.apache.hadoop.registry.client.types.ServiceRecord |
SliderClient.lookupServiceRecord(String serviceType,
String id)
Look up a service record of the current user
|
protected void |
SliderClient.persistInstanceDefinition(boolean overwrite,
org.apache.hadoop.fs.Path appconfdir,
InstanceBuilder builder) |
org.apache.hadoop.registry.client.types.ServiceRecord |
SliderClient.resolve(String path)
Look up an instance
|
protected void |
SliderClient.validateInstanceDefinition(AbstractClientProvider provider,
AggregateConf instanceDefinition,
SliderFileSystem fs)
Validate an instance definition against a provider.
|
| Modifier and Type | Method and Description |
|---|---|
abstract void |
CommonArgs.applyAction()
Implementors must implement their action apply routine here
|
void |
ClientArgs.applyAction()
Look at the chosen action and bind it as the core action for the operation.
|
void |
CommonArgs.parse()
Parse routine -includes registering the action-specific argument classes
and postprocess it
|
void |
CommonArgs.postProcess()
validate args via
CommonArgs.validate()
then postprocess the arguments |
| Modifier and Type | Method and Description |
|---|---|
void |
CoreFileSystem.createClusterDirectories(InstancePaths instancePaths)
Create the Slider cluster path for a named cluster and all its subdirs
This is a directory; a mkdirs() operation is executed
to ensure that it is there.
|
org.apache.hadoop.fs.Path |
CoreFileSystem.createClusterDirectories(String clustername,
org.apache.hadoop.conf.Configuration conf)
Create the Slider cluster path for a named cluster and all its subdirs
This is a directory; a mkdirs() operation is executed
to ensure that it is there.
|
org.apache.hadoop.fs.Path |
CoreFileSystem.createPathThatMustExist(String uri)
Create a path that must exist in the cluster fs
|
static ForkedProcessService |
SliderUtils.execCommand(String name,
int status,
long timeoutMillis,
org.slf4j.Logger logger,
String outputString,
String... commands)
Execute a command for a test operation
|
static org.apache.hadoop.fs.Path |
SliderUtils.extractImagePath(CoreFileSystem fs,
MapOperations internalOptions) |
int |
PortScanner.getAvailablePort() |
static boolean |
SliderUtils.initProcessSecurity(org.apache.hadoop.conf.Configuration conf)
Turn on security.
|
org.apache.hadoop.fs.Path |
CoreFileSystem.locateInstanceDefinition(String clustername)
Locate an application conf json in the FS.
|
static boolean |
SliderUtils.maybeInitSecurity(org.apache.hadoop.conf.Configuration conf)
Init security if the cluster configuration declares the cluster is secure
|
static void |
SliderUtils.maybeVerifyWinUtilsValid()
Strictly verify that windows utils is present.
|
static void |
SliderUtils.putAllJars(Map<String,org.apache.hadoop.yarn.api.records.LocalResource> providerResources,
SliderFileSystem sliderFileSystem,
org.apache.hadoop.fs.Path tempPath,
String libDir,
String srcPath)
Submit a JAR containing and map it
|
static void |
SliderUtils.putAmTarGzipAndUpdate(Map<String,org.apache.hadoop.yarn.api.records.LocalResource> providerResources,
SliderFileSystem sliderFileSystem)
Submit the AM tar.gz containing all dependencies and map it
|
static org.apache.hadoop.yarn.api.records.LocalResource |
SliderUtils.putJar(Map<String,org.apache.hadoop.yarn.api.records.LocalResource> providerResources,
SliderFileSystem sliderFileSystem,
Class clazz,
org.apache.hadoop.fs.Path tempPath,
String libdir,
String jarName)
Submit a JAR containing a specific class and map it
|
org.apache.hadoop.yarn.api.records.LocalResource |
CoreFileSystem.submitJarWithClass(Class clazz,
org.apache.hadoop.fs.Path tempPath,
String subdir,
String jarName)
Submit a JAR containing a specific class, returning
the resource to be mapped in
|
static void |
SliderUtils.validateOpenSSLEnv(org.slf4j.Logger logger) |
static void |
SliderUtils.validatePythonEnv(org.slf4j.Logger logger) |
static void |
SliderUtils.validateSliderClientEnvironment(org.slf4j.Logger logger)
Validate the slider client-side execution environment.
|
static void |
SliderUtils.validateSliderServerEnvironment(org.slf4j.Logger logger,
boolean dependencyChecks)
Validate the slider server-side execution environment.
|
void |
CoreFileSystem.verifyClusterDirectoryNonexistent(String clustername,
org.apache.hadoop.fs.Path clusterDirectory)
Verify that the cluster directory is not present
|
void |
CoreFileSystem.verifyClusterSpecExists(String clustername,
org.apache.hadoop.fs.Path clusterSpecPath)
Verify that a cluster specification exists
|
void |
CoreFileSystem.verifyDirectoryNonexistent(org.apache.hadoop.fs.Path clusterDirectory)
Verify that the given directory is not present
|
void |
CoreFileSystem.verifyDirectoryWriteAccess(org.apache.hadoop.fs.Path dirPath)
Verify that a user has write access to a directory.
|
| Modifier and Type | Method and Description |
|---|---|
static AggregateConf |
InstanceIO.loadInstanceDefinitionUnresolved(CoreFileSystem sliderFileSystem,
org.apache.hadoop.fs.Path clusterDirectory)
Load in an instance definition -but do not resolve it
|
void |
InstanceBuilder.persist(org.apache.hadoop.fs.Path appconfdir,
boolean overwrite)
Persist this
|
static void |
InstanceIO.saveInstanceDefinition(CoreFileSystem coreFS,
org.apache.hadoop.fs.Path dir,
AggregateConf instanceDefinition)
Update a persisted instance definition
|
| Modifier and Type | Class and Description |
|---|---|
class |
BadClusterStateException
The system is in a bad state
|
class |
BadCommandArgumentsException |
class |
BadConfigException
An exception to raise on a bad configuration
|
class |
NotFoundException
Whatever was being resolved: it was not found
|
class |
SliderInternalStateException |
class |
TriggerClusterTeardownException
An Exception to be thrown for an explicit "shut down the cluster" operation
raised by the application state or other parts of the AM
|
class |
UnknownApplicationInstanceException |
class |
UsageException
Used to raise a usage exception ...
|
| Modifier and Type | Method and Description |
|---|---|
void |
LockHeldAction.execute()
Execute the action
|
void |
ConfPersister.load(AggregateConf conf)
Load the configuration.
|
void |
ConfPersister.save(AggregateConf conf,
LockHeldAction action)
Save the configuration
|
| Modifier and Type | Method and Description |
|---|---|
static void |
ProviderUtils.addAllDependencyJars(Map<String,org.apache.hadoop.yarn.api.records.LocalResource> providerResources,
SliderFileSystem sliderFileSystem,
org.apache.hadoop.fs.Path tempPath,
String libDir,
String libLocalSrcDir)
Loads all dependency jars from the default path
|
static void |
ProviderUtils.addDependencyJars(Map<String,org.apache.hadoop.yarn.api.records.LocalResource> providerResources,
SliderFileSystem sliderFileSystem,
org.apache.hadoop.fs.Path tempPath,
String libdir,
String[] resources,
Class[] classes)
Add a set of dependencies to the provider resources being built up,
by copying them from the local classpath to the remote one, then
registering them
|
static boolean |
ProviderUtils.addProviderJar(Map<String,org.apache.hadoop.yarn.api.records.LocalResource> providerResources,
Object provider,
String jarName,
SliderFileSystem sliderFileSystem,
org.apache.hadoop.fs.Path tempPath,
String libdir,
boolean miniClusterTestRun)
Add oneself to the classpath.
|
void |
ProviderService.buildContainerLaunchContext(ContainerLauncher containerLauncher,
AggregateConf instanceDefinition,
org.apache.hadoop.yarn.api.records.Container container,
String role,
SliderFileSystem sliderFileSystem,
org.apache.hadoop.fs.Path generatedConfPath,
MapOperations resourceComponent,
MapOperations appComponent,
org.apache.hadoop.fs.Path containerTmpDirPath)
Set up the entire container launch context
|
ForkedProcessService |
AbstractProviderService.buildProcess(String name,
Map<String,String> env,
List<String> commands) |
static SliderProviderFactory |
SliderProviderFactory.createSliderProviderFactory(String application)
Create a provider for a specific application
|
boolean |
ProviderService.exec(AggregateConf instanceDefinition,
File confDir,
Map<String,String> env,
ProviderCompleted execInProgress)
Execute a process in the AM
|
boolean |
AbstractProviderService.exec(AggregateConf instanceDefinition,
File confDir,
Map<String,String> env,
ProviderCompleted execInProgress)
override point to allow a process to start executing in this container
|
Set<String> |
AbstractClientProvider.getApplicationTags(SliderFileSystem fileSystem,
String appDef)
Return a set of application specific string tags.
|
void |
ProviderService.initializeApplicationConfiguration(AggregateConf instanceDefinition,
SliderFileSystem fileSystem)
The application configuration should be initialized here
|
void |
AbstractProviderService.initializeApplicationConfiguration(AggregateConf instanceDefinition,
SliderFileSystem fileSystem)
No-op implementation of this method.
|
void |
AbstractClientProvider.preflightValidateClusterConfiguration(SliderFileSystem sliderFileSystem,
String clustername,
org.apache.hadoop.conf.Configuration configuration,
AggregateConf instanceDefinition,
org.apache.hadoop.fs.Path clusterDirPath,
org.apache.hadoop.fs.Path generatedConfDirPath,
boolean secure)
This is called pre-launch to validate that the cluster specification
is valid.
|
void |
AbstractClientProvider.prepareAMAndConfigForLaunch(SliderFileSystem fileSystem,
org.apache.hadoop.conf.Configuration serviceConf,
AbstractLauncher launcher,
AggregateConf instanceDescription,
org.apache.hadoop.fs.Path snapshotConfDirPath,
org.apache.hadoop.fs.Path generatedConfDirPath,
org.apache.hadoop.conf.Configuration clientConfExtras,
String libdir,
org.apache.hadoop.fs.Path tempPath,
boolean miniClusterTestRun)
Prepare the AM settings for launch
|
void |
AbstractClientProvider.prepareInstanceConfiguration(AggregateConf aggregateConf)
Any provider-side alteration of a configuration can take place here.
|
void |
AbstractClientProvider.processClientOperation(SliderFileSystem fileSystem,
String operation,
File clientInstallPath,
File clientPackage,
org.codehaus.jettison.json.JSONObject config,
String name)
Process client operations for applications such as install, configure
|
protected ForkedProcessService |
AbstractProviderService.queueCommand(String name,
Map<String,String> env,
List<String> commands)
Create a new forked process service with the given
name, environment and command list -then add it as a child
for execution in the sequence.
|
void |
ProviderService.validateApplicationConfiguration(AggregateConf instanceDefinition,
File confDir,
boolean secure)
This is a validation of the application configuration on the AM.
|
void |
AbstractProviderService.validateApplicationConfiguration(AggregateConf instance,
File confDir,
boolean secure)
No-op implementation of this method.
|
void |
ProviderCore.validateInstanceDefinition(AggregateConf instanceDefinition)
Verify that an instance definition is considered valid by the provider
|
void |
AbstractClientProvider.validateInstanceDefinition(AggregateConf instanceDefinition,
SliderFileSystem fs)
Verify that an instance definition is considered valid by the provider
|
| Modifier and Type | Method and Description |
|---|---|
protected void |
AgentProviderService.addGetConfigCommand(String componentName,
String containerId,
HeartBeatResponse response) |
protected void |
AgentProviderService.addGetConfigDockerCommand(String componentName,
String containerId,
HeartBeatResponse response) |
protected void |
AgentProviderService.addInstallCommand(String componentName,
String containerId,
HeartBeatResponse response,
String scriptPath,
ComponentCommand compCmd,
long timeout,
String pkg)
Add install command to the heartbeat response
|
protected void |
AgentProviderService.addInstallDockerCommand(String componentName,
String containerId,
HeartBeatResponse response,
ComponentCommand compCmd,
long timeout) |
protected void |
AgentProviderService.addStartCommand(String componentName,
String containerId,
HeartBeatResponse response,
String scriptPath,
ComponentCommand startCommand,
ComponentCommand stopCommand,
long timeout,
boolean isMarkedAutoRestart) |
protected void |
AgentProviderService.addStartDockerCommand(String componentName,
String containerId,
HeartBeatResponse response,
ComponentCommand startCommand,
long timeout,
boolean isMarkedAutoRestart) |
protected void |
AgentProviderService.addStatusCommand(String componentName,
String containerId,
HeartBeatResponse response,
String scriptPath,
long timeout) |
protected void |
AgentProviderService.addStatusDockerCommand(String componentName,
String containerId,
HeartBeatResponse response,
String scriptPath,
long timeout) |
protected void |
AgentProviderService.addStopCommand(String componentName,
String containerId,
HeartBeatResponse response,
String scriptPath,
long timeout,
boolean isInUpgradeMode) |
protected void |
AgentProviderService.addUpgradeCommand(String componentName,
String containerId,
HeartBeatResponse response,
String scriptPath,
long timeout) |
void |
AgentProviderService.buildContainerLaunchContext(ContainerLauncher launcher,
AggregateConf instanceDefinition,
org.apache.hadoop.yarn.api.records.Container container,
String role,
SliderFileSystem fileSystem,
org.apache.hadoop.fs.Path generatedConfPath,
MapOperations resourceComponent,
MapOperations appComponent,
org.apache.hadoop.fs.Path containerTmpDirPath) |
Set<String> |
AgentClientProvider.getApplicationTags(SliderFileSystem fileSystem,
String appDef) |
protected org.codehaus.jettison.json.JSONObject |
AgentClientProvider.getCommandJson(org.codehaus.jettison.json.JSONObject defaultConfig,
org.codehaus.jettison.json.JSONObject inputConfig,
Metainfo metainfo,
File clientInstallPath,
String name) |
void |
AgentProviderService.initializeApplicationConfiguration(AggregateConf instanceDefinition,
SliderFileSystem fileSystem) |
void |
AgentClientProvider.preflightValidateClusterConfiguration(SliderFileSystem sliderFileSystem,
String clustername,
org.apache.hadoop.conf.Configuration configuration,
AggregateConf instanceDefinition,
org.apache.hadoop.fs.Path clusterDirPath,
org.apache.hadoop.fs.Path generatedConfDirPath,
boolean secure) |
void |
AgentClientProvider.prepareAMAndConfigForLaunch(SliderFileSystem fileSystem,
org.apache.hadoop.conf.Configuration serviceConf,
AbstractLauncher launcher,
AggregateConf instanceDefinition,
org.apache.hadoop.fs.Path snapshotConfDirPath,
org.apache.hadoop.fs.Path generatedConfDirPath,
org.apache.hadoop.conf.Configuration clientConfExtras,
String libdir,
org.apache.hadoop.fs.Path tempPath,
boolean miniClusterTestRun) |
void |
AgentClientProvider.processClientOperation(SliderFileSystem fileSystem,
String operation,
File clientInstallPath,
File appPackage,
org.codehaus.jettison.json.JSONObject config,
String name) |
protected void |
AgentClientProvider.runCommand(File appPkgDir,
File agentPkgDir,
File cmdDir,
String clientScript) |
void |
AgentProviderService.validateInstanceDefinition(AggregateConf instanceDefinition) |
void |
AgentClientProvider.validateInstanceDefinition(AggregateConf instanceDefinition,
SliderFileSystem fs) |
| Modifier and Type | Method and Description |
|---|---|
static void |
Metainfo.checkNonNull(String value,
String field,
String type) |
void |
Metainfo.validate() |
void |
Validate.validate(String version) |
void |
Package.validate(String version) |
void |
OSSpecific.validate(String version) |
void |
OSPackage.validate(String version) |
void |
ExportGroup.validate(String version) |
void |
Export.validate(String version) |
void |
DockerContainerPort.validate(String version) |
void |
DockerContainerMount.validate(String version) |
void |
DockerContainer.validate(String version) |
void |
ConfigFile.validate(String version) |
void |
ComponentsInAddonPackage.validate(String version) |
void |
ComponentCommand.validate(String version) |
void |
Component.validate(String version) |
void |
CommandScript.validate(String version) |
void |
CommandOrder.validate(String version) |
void |
ApplicationPackage.validate(String version) |
void |
Application.validate(String version) |
| Modifier and Type | Method and Description |
|---|---|
void |
SliderAMProviderService.buildContainerLaunchContext(ContainerLauncher containerLauncher,
AggregateConf instanceDefinition,
org.apache.hadoop.yarn.api.records.Container container,
String role,
SliderFileSystem sliderFileSystem,
org.apache.hadoop.fs.Path generatedConfPath,
MapOperations resourceComponent,
MapOperations appComponent,
org.apache.hadoop.fs.Path containerTmpDirPath) |
void |
SliderAMClientProvider.preflightValidateClusterConfiguration(SliderFileSystem sliderFileSystem,
String clustername,
org.apache.hadoop.conf.Configuration configuration,
AggregateConf instanceDefinition,
org.apache.hadoop.fs.Path clusterDirPath,
org.apache.hadoop.fs.Path generatedConfDirPath,
boolean secure) |
void |
SliderAMClientProvider.prepareAMAndConfigForLaunch(SliderFileSystem fileSystem,
org.apache.hadoop.conf.Configuration serviceConf,
AbstractLauncher launcher,
AggregateConf instanceDescription,
org.apache.hadoop.fs.Path snapshotConfDirPath,
org.apache.hadoop.fs.Path generatedConfDirPath,
org.apache.hadoop.conf.Configuration clientConfExtras,
String libdir,
org.apache.hadoop.fs.Path tempPath,
boolean miniClusterTestRun)
The Slider AM sets up all the dependency JARs above slider.jar itself
Prepare the AM settings for launch
|
void |
SliderAMClientProvider.prepareInstanceConfiguration(AggregateConf aggregateConf) |
void |
SliderAMProviderService.validateInstanceDefinition(AggregateConf instanceDefinition) |
void |
SliderAMClientProvider.validateInstanceDefinition(AggregateConf instanceDefinition,
SliderFileSystem fs)
Verify that an instance definition is considered valid by the provider
|
| Modifier and Type | Method and Description |
|---|---|
void |
SliderAppMaster.flexCluster(ConfTree resources)
Implementation of cluster flexing.
|
protected void |
SliderAppMaster.launchProviderService(AggregateConf instanceDefinition,
File confDir)
Launch the provider service
|
protected void |
SliderAppMaster.login(String principal,
File localKeytabFile) |
void |
SliderAppMaster.onUpgradeContainers(ActionUpgradeContainers upgradeContainersRequest)
Signal that containers are being upgraded.
|
void |
SliderAppMaster.providerLostContainer(org.apache.hadoop.yarn.api.records.ContainerId containerId)
report container loss.
|
protected void |
SliderAppMaster.validateLoginUser(org.apache.hadoop.security.UserGroupInformation user)
Ensure that the user is generated from a keytab and has no HDFS delegation
tokens.
|
| Modifier and Type | Method and Description |
|---|---|
static SliderClusterProtocol |
RpcBinder.createProxy(org.apache.hadoop.conf.Configuration conf,
String host,
int port,
org.apache.hadoop.yarn.api.records.Token clientToAMToken,
int rpcTimeout) |
static SliderClusterProtocol |
RpcBinder.getProxy(org.apache.hadoop.conf.Configuration conf,
org.apache.hadoop.yarn.api.records.ApplicationReport application,
int rpcTimeout)
Get a proxy from the application report
|
| Modifier and Type | Method and Description |
|---|---|
File |
SecurityConfiguration.getKeytabFile(AggregateConf instanceDefinition) |
| Constructor and Description |
|---|
SecurityConfiguration(org.apache.hadoop.conf.Configuration configuration,
AggregateConf instanceDefinition,
String clusterName) |
| Modifier and Type | Method and Description |
|---|---|
SecurityStore |
TruststoreGenerator.generate(String hostname,
String containerId,
AggregateConf instanceDefinition,
MapOperations compOps,
String role) |
SecurityStore |
SecurityStoreGenerator.generate(String hostname,
String containerId,
AggregateConf instanceDefinition,
MapOperations compOps,
String role) |
SecurityStore |
KeystoreGenerator.generate(String hostname,
String containerId,
AggregateConf instanceDefinition,
MapOperations compOps,
String role) |
SecurityStore |
CertificateManager.generateContainerKeystore(String hostname,
String requesterId,
String role,
String keystorePass) |
SecurityStore |
CertificateManager.generateContainerTruststore(String containerId,
String role,
String truststorePass) |
static SecurityStore[] |
StoresGenerator.generateSecurityStores(String hostname,
String containerId,
String role,
AggregateConf instanceDefinition,
MapOperations compOps) |
protected String |
AbstractSecurityStoreGenerator.getStorePassword(Map<String,List<String>> credentials,
MapOperations compOps,
String role) |
void |
CertificateManager.initialize(MapOperations compOperations) |
void |
CertificateManager.initialize(MapOperations compOperations,
String hostname,
String containerId,
String appName)
Verify that root certificate exists, generate it otherwise.
|
Copyright © 2014–2015 The Apache Software Foundation. All rights reserved.