org.apache.slider.providers.agent

Class AgentProviderService

java.lang.Object

org.apache.hadoop.service.AbstractService

org.apache.slider.server.services.workflow.WorkflowSequenceService

org.apache.slider.providers.AbstractProviderService

org.apache.slider.providers.agent.AgentProviderService

All Implemented Interfaces:

Closeable, AutoCloseable, org.apache.hadoop.service.Service, org.apache.hadoop.service.ServiceStateChangeListener, SliderKeys, SliderXmlConfKeys, ExitCodeProvider, AgentKeys, ProviderCore, ProviderService, RMOperationHandlerActions, AgentRestOperations, ServiceParent

public class AgentProviderService
extends AbstractProviderService
implements ProviderCore, AgentKeys, SliderKeys, AgentRestOperations

This class implements the server-side logic for application deployment through Slider application package

Nested Class Summary

Nested classes/interfaces inherited from interface org.apache.hadoop.service.Service

org.apache.hadoop.service.Service.STATE

Field Summary

Fields

Modifier and Type	Field and Description
protected static org.slf4j.Logger	log

Fields inherited from class org.apache.slider.providers.AbstractProviderService

amState, amWebAPI, queueAccess, restOps, yarnRegistry

Fields inherited from interface org.apache.slider.providers.agent.AgentKeys

ADDON_DEFINITION_DIR, ADDON_FOR_ALL_COMPONENTS, ADDON_PREFIX, ADDONS, AGENT_CONF, AGENT_CONFIG_FILE, AGENT_INSTALL_DIR, AGENT_INSTANCE_DEBUG_DATA, AGENT_MAIN_SCRIPT, AGENT_MAIN_SCRIPT_ROOT, AGENT_OUT_FILE, AGENT_VERSION, AGENT_VERSION_FILE, APP_DEF, APP_DEFINITION_DIR, APP_HOME, APP_ROOT, ARG_DEBUG, ARG_HOST, ARG_LABEL, ARG_PORT, ARG_SECURED_PORT, ARG_ZOOKEEPER_QUORUM, ARG_ZOOKEEPER_REGISTRY_PATH, CERT_FILE_LOCALIZATION_PATH, CONF_RESOURCE, CONTROLLER_URL, CREATE_DEF_ZK_NODE, HEARTBEAT_MONITOR_INTERVAL, INFRA_RUN_SECURITY_DIR, JAVA_HOME, KEY_AGENT_TWO_WAY_SSL_ENABLED, KEY_CONTAINER_LAUNCH_DELAY, PACKAGE_LIST, PACKAGE_PATH, PROVIDER_AGENT, PYTHON_EXE, ROLE_NODE, RUNAS_USER, SCRIPT_PATH, SERVICE_NAME, SYSTEM_CONFIGS, TEST_RELAX_VERIFICATION, WAIT_HEARTBEAT

Fields inherited from interface org.apache.slider.common.SliderKeys

ADDONS_DIR, AGENT_TAR, AM_DIR_PREFIX, AM_FILTER_NAME, APP_CONTAINER_HEARTBEAT_INTERVAL_SEC, APP_CONTAINER_RELEASE_TIMEOUT, APP_DEF_DIR, APP_TYPE, APP_VERSION, APP_VERSION_UNKNOWN, CLUSTER_DIRECTORY, COMP_KEYSTORE_PASSWORD_ALIAS_DEFAULT, COMP_KEYSTORE_PASSWORD_ALIAS_KEY, COMP_KEYSTORE_PASSWORD_PROPERTY_KEY, COMP_STORES_REQUIRED_KEY, COMP_TRUSTSTORE_PASSWORD_ALIAS_DEFAULT, COMP_TRUSTSTORE_PASSWORD_ALIAS_KEY, COMP_TRUSTSTORE_PASSWORD_PROPERTY_KEY, COMPONENT_AM, CRT_FILE_NAME, CRT_PASS_FILE_NAME, CSR_FILE_NAME, DATA_DIR_NAME, DEFAULT_APP_PKG, DEFAULT_GC_OPTS, DEFAULT_JVM_HEAP, DEFAULT_YARN_MEMORY, GENERATED_CONF_DIR_NAME, GSON_JAR, HADOOP_PROXY_USER, HADOOP_USER_NAME, HDP_VERSION_PROP_NAME, HISTORY_DIR_NAME, HISTORY_FILENAME_CREATION_PATTERN, HISTORY_FILENAME_GLOB_PATTERN, HISTORY_FILENAME_MATCH_PATTERN, HISTORY_FILENAME_PREFIX, HISTORY_FILENAME_SUFFIX, INFRA_DIR_NAME, JCOMMANDER_JAR, JVM_ENABLE_ASSERTIONS, JVM_ENABLE_SYSTEM_ASSERTIONS, JVM_FORCE_IPV4, JVM_JAVA_HEADLESS, KEY_ALLOWED_PORT_RANGE, KEY_AM_ALLOWED_PORT_RANGE, KEY_FILE_NAME, KEYSTORE_FILE_NAME, KEYTAB_DIR, LOCAL_TARBALL_INSTALL_SUBDIR, LOG4J_PROP_FILENAME, LOG4J_SERVER_PROP_FILENAME, PACKAGE_DIRECTORY, PASS_LEN, PROPAGATE_RESOURCE_OPTION, PROPAGATED_CONF_DIR_NAME, PROPERTY_CONF_DIR, PROPERTY_LIB_DIR, PYTHONPATH, ROLE_AM_PRIORITY_INDEX, SECURITY_DIR, SLIDER_BASE_DIRECTORY, SLIDER_CLASSPATH_EXTRA, SLIDER_CLIENT_XML, SLIDER_DEPENDENCY_DIR, SLIDER_DEPENDENCY_DIR_PERMISSIONS, SLIDER_DEPENDENCY_HDP_PARENT_DIR, SLIDER_DEPENDENCY_LOCALIZED_DIR_LINK, SLIDER_DEPENDENCY_TAR_GZ_FILE_EXT, SLIDER_DEPENDENCY_TAR_GZ_FILE_NAME, SLIDER_JAR, SLIDER_JVM_OPTS, SLIDER_PASSPHRASE, SLIDER_SERVER_XML, SLIDER_XML, SNAPSHOT_CONF_DIR_NAME, STDERR_AM, STDOUT_AM, SUBMITTED_CONF_DIR, SYSPROP_LOG_DIR, SYSPROP_LOG4J_CONFIGURATION, TMP_DIR_PREFIX, TMP_LOGDIR_PREFIX, YARN_CONTAINER_PATH

Fields inherited from interface org.apache.slider.common.SliderXmlConfKeys

CLUSTER_DIRECTORY_PERMISSIONS, DATA_DIRECTORY_PERMISSIONS, DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS, DEFAULT_DATA_DIRECTORY_PERMISSIONS, DEFAULT_YARN_QUEUE, DEFAULT_YARN_QUEUE_PRIORITY, DFS_DATANODE_KERBEROS_PRINCIPAL_KEY, DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT, DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_KEY, DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_DEFAULT, DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT, DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY, DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT, DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, DFS_NAMENODE_DU_RESERVED_KEY, DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY, DFS_NAMENODE_KEYTAB_FILE_KEY, FS_DEFAULT_NAME_CLASSIC, HADOOP_HTTP_FILTER_INITIALIZERS, IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH, IPC_CLIENT_RETRY_POLICY_ENABLED_DEFAULT, IPC_CLIENT_RETRY_POLICY_SPEC_DEFAULT, KEY_AM_KEYTAB_LOCAL_PATH, KEY_AM_LAUNCH_ENV, KEY_AM_LOGIN_KEYTAB_NAME, KEY_AM_LOGIN_KEYTAB_REQUIRED, KEY_AM_RESTART_LIMIT, KEY_HDFS_KEYTAB_DIR, KEY_IPC_CLIENT_RETRY_POLICY_ENABLED, KEY_IPC_CLIENT_RETRY_POLICY_SPEC, KEY_KERBEROS_PRINCIPAL, KEY_KEYSTORE_LOCATION, KEY_KEYTAB_PRINCIPAL, KEY_PROTOCOL_ACL, KEY_PROVIDER, KEY_SECURITY_ENABLED, KEY_SLIDER_AM_DEPENDENCY_CHECKS_DISABLED, KEY_SLIDER_BASE_PATH, KEY_TEMPLATE_ORIGIN, KEY_YARN_QUEUE, KEY_YARN_QUEUE_PRIORITY, PREFIX_PROVIDER, PYTHON_EXECUTABLE_PATH, REGISTRY_PATH, REGISTRY_ZK_QUORUM, X_DEV_INSECURE_DEFAULT, X_DEV_INSECURE_REQUIRED, X_DEV_INSECURE_WS

Constructor Summary

Constructors

Constructor and Description
AgentProviderService() Create an instance of AgentProviderService

Method Summary

Methods

Modifier and Type	Method and Description
protected void	addGetConfigCommand(String componentName, String containerId, HeartBeatResponse response)
protected void	addGetConfigDockerCommand(String componentName, String containerId, HeartBeatResponse response)
protected void	addInstallCommand(String componentName, String containerId, HeartBeatResponse response, String scriptPath, ComponentCommand compCmd, long timeout, String pkg) Add install command to the heartbeat response
protected void	addInstallDockerCommand(String componentName, String containerId, HeartBeatResponse response, ComponentCommand compCmd, long timeout)
protected void	addRoleRelatedTokens(Map<String,String> tokens)
protected void	addStartCommand(String componentName, String containerId, HeartBeatResponse response, String scriptPath, ComponentCommand startCommand, ComponentCommand stopCommand, long timeout, boolean isMarkedAutoRestart)
protected void	addStartDockerCommand(String componentName, String containerId, HeartBeatResponse response, ComponentCommand startCommand, long timeout, boolean isMarkedAutoRestart)
protected void	addStatusCommand(String componentName, String containerId, HeartBeatResponse response, String scriptPath, long timeout)
protected void	addStatusDockerCommand(String componentName, String containerId, HeartBeatResponse response, String scriptPath, long timeout)
protected void	addStopCommand(String componentName, String containerId, HeartBeatResponse response, String scriptPath, long timeout, boolean isInUpgradeMode)
protected void	addUpgradeCommand(String componentName, String containerId, HeartBeatResponse response, String scriptPath, long timeout)
void	addUpgradeContainers(Set<String> upgradeContainers)
void	applyInitialRegistryDefinitions(URL amWebURI, URL agentOpsURI, URL agentStatusURI, org.apache.hadoop.registry.client.types.ServiceRecord serviceRecord) Prior to going live -register the initial service registry data
void	buildContainerLaunchContext(ContainerLauncher launcher, AggregateConf instanceDefinition, org.apache.hadoop.yarn.api.records.Container container, String role, SliderFileSystem fileSystem, org.apache.hadoop.fs.Path generatedConfPath, MapOperations resourceComponent, MapOperations appComponent, org.apache.hadoop.fs.Path containerTmpDirPath) Set up the entire container launch context
Map<String,String>	buildMonitorDetails(ClusterDescription clusterDesc) Build a map of data intended for the AM webapp that is specific about this provider.
Map<String,String>	buildProviderStatus() Build the provider status, can be empty
protected boolean	canAnyMasterPublishConfig() Can any master publish config explicitly, if not a random master is used
protected boolean	canPublishConfig(String roleName) Can the role publish configuration
protected void	dereferenceAllConfigs(Map<String,Map<String,String>> configurations)
protected Map<String,String>	getAllocatedPorts()
protected Map<String,String>	getAllocatedPorts(String containerId)
protected Component	getApplicationComponent(String roleName) Return Component based on name
protected List<String>	getApplicationConfigurationTypes()
protected Metainfo	getApplicationMetainfo(SliderFileSystem fileSystem, String appDef)
protected Metainfo	getApplicationMetainfo(SliderFileSystem fileSystem, String appDef, boolean addonPackage)
protected String	getClusterInfoPropertyValue(String name)
protected String	getClusterOptionPropertyValue(String name)
protected Map<String,Map<String,String>>	getComponentInstanceData()
protected Map<String,ComponentInstanceState>	getComponentStatuses()
protected Map<String,List<ExportEntry>>	getCurrentExports(String groupName)
protected Map<String,DefaultConfig>	getDefaultConfigs()
String	getHumanName() Get a human friendly name for web UIs and messages
protected static String	getJDKDir()
protected Map<String,ExportEntry>	getLogFolderExports()
protected Metainfo	getMetaInfo()
protected static String	getPackageListFromApplication(Application application)
protected Map<String,Map<String,ClusterNode>>	getRoleClusterNodeMapping() Get a list of all hosts for all role/container per role
List<ProviderRole>	getRoles()
protected CommandScript	getScriptPathForMasterPackage(String roleName) Extract script path from the application metainfo
protected List<String>	getSystemConfigurationsRequested(ConfTreeOperations appConf)
protected Map<String,ExportEntry>	getWorkFolderExports()
HeartBeatResponse	handleHeartBeat(HeartBeat heartBeat) Handle heartbeat response from agents
RegistrationResponse	handleRegistration(Register registration) Handle registration calls from the agents
void	initializeApplicationConfiguration(AggregateConf instanceDefinition, SliderFileSystem fileSystem) No-op implementation of this method.
protected Map<String,DefaultConfig>	initializeDefaultConfigs(SliderFileSystem fileSystem, String appDef, Metainfo metainfo) Read all default configs
protected boolean	isMarkedAutoRestart(String roleName) Checks if the role is marked auto-restart
protected boolean	isMaster(String roleName) Is the role of type MASTER
boolean	isSupportedRole(String role) Scan through the roles and see if it is supported.
org.apache.hadoop.conf.Configuration	loadProviderConfigurationInformation(File confDir) Load a specific XML configuration file for the provider config
protected void	lostContainer(String label, org.apache.hadoop.yarn.api.records.ContainerId containerId) Lost heartbeat from the container - release it and ask for a replacement (async operation)
void	notifyContainerCompleted(org.apache.hadoop.yarn.api.records.ContainerId containerId) Notify the providers of container completion
protected void	processAllocatedPorts(String fqdn, String roleName, String containerId, Map<String,String> ports)
protected void	processAndPublishComponentSpecificData(Map<String,String> ports, String containerId, String hostFqdn, String componentName) Publish component instance specific data if the component demands it
protected void	processAndPublishComponentSpecificExports(Map<String,String> ports, String containerId, String hostFqdn, String compName) Publish component instance specific data if the component demands it
protected void	publishApplicationInstanceData(String name, String description, Iterable<Map.Entry<String,String>> entries) Publish a named property bag that may contain name-value pairs for app configurations such as hbase-site
protected void	publishConfigAndExportGroups(HeartBeat heartBeat, ComponentInstanceState componentStatus, String componentName) Process return status for component instances
protected void	publishFolderPaths(Map<String,String> folders, String containerId, String componentName, String hostFqdn) Format the folder locations and publish in the registry service
void	rebuildContainerDetails(List<org.apache.hadoop.yarn.api.records.Container> liveContainers, String applicationId, Map<Integer,ProviderRole> providerRoleMap) No-op implementation of this method.
protected void	serviceInit(org.apache.hadoop.conf.Configuration conf)
void	setAppStopInitiated(boolean appStopInitiated)
protected void	setHeartbeatMonitorInterval(int heartbeatMonitorInterval)
void	setInUpgradeMode(boolean inUpgradeMode)
void	validateInstanceDefinition(AggregateConf instanceDefinition) Verify that an instance definition is considered valid by the provider

Methods inherited from class org.apache.slider.providers.AbstractProviderService

addContainerRequest, bind, bindToYarnRegistry, buildEndpointDetails, buildProcess, cancelContainerRequests, cancelSingleRequest, createContainerReleaseSelector, exec, execute, getAgentRestOperations, getAmState, getConf, getExitCode, getFPSFromParentService, getQueueAccess, getYarnRegistry, latestProcess, loadProviderConfigurationInformation, maybeStartCommandSequence, queueCommand, releaseAssignedContainer, setAgentRestOperations, setAmState, validateApplicationConfiguration

Methods inherited from class org.apache.slider.server.services.workflow.WorkflowSequenceService

addService, getActiveService, getPreviousService, getServices, onServiceCompleted, serviceStart, serviceStop, setStopIfNoChildServicesAtStartup, startNextService, stateChanged, toString

Methods inherited from class org.apache.hadoop.service.AbstractService

close, getBlockers, getConfig, getFailureCause, getFailureState, getLifecycleHistory, getName, getServiceState, getStartTime, init, isInState, noteFailure, putBlocker, registerGlobalListener, registerServiceListener, removeBlocker, setConfig, start, stop, unregisterGlobalListener, unregisterServiceListener, waitForServiceToStop

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait

Methods inherited from interface org.apache.slider.providers.ProviderCore

getConf, getName

Methods inherited from interface org.apache.hadoop.service.Service

close, getBlockers, getConfig, getFailureCause, getFailureState, getLifecycleHistory, getName, getServiceState, getStartTime, init, isInState, registerServiceListener, start, stop, unregisterServiceListener, waitForServiceToStop

Field Detail

log

protected static final org.slf4j.Logger log

Constructor Detail

AgentProviderService

public AgentProviderService()

Create an instance of AgentProviderService

Method Detail

getHumanName

public String getHumanName()

Description copied from interface: ProviderService

Get a human friendly name for web UIs and messages

Specified by:

getHumanName in interface ProviderService

Overrides:

getHumanName in class AbstractProviderService

Returns:

a name string. Default is simply the service instance name.

getRoles

public List<ProviderRole> getRoles()

Specified by:

getRoles in interface ProviderCore

serviceInit

protected void serviceInit(org.apache.hadoop.conf.Configuration conf)
                    throws Exception

Overrides:

serviceInit in class org.apache.hadoop.service.AbstractService

Throws:

Exception

loadProviderConfigurationInformation

public org.apache.hadoop.conf.Configuration loadProviderConfigurationInformation(File confDir)
                                                                          throws BadCommandArgumentsException,
                                                                                 IOException

Description copied from interface: ProviderService

Load a specific XML configuration file for the provider config

Specified by:

loadProviderConfigurationInformation in interface ProviderService

Parameters:

confDir - configuration directory

Returns:

a configuration to be included in status

Throws:

BadCommandArgumentsException

IOException

validateInstanceDefinition

public void validateInstanceDefinition(AggregateConf instanceDefinition)
                                throws SliderException

Description copied from interface: ProviderCore

Verify that an instance definition is considered valid by the provider

Specified by:

validateInstanceDefinition in interface ProviderCore

Parameters:

instanceDefinition - instance definition

Throws:

SliderException - if the configuration is not valid

initializeApplicationConfiguration

public void initializeApplicationConfiguration(AggregateConf instanceDefinition,
                                      SliderFileSystem fileSystem)
                                        throws IOException,
                                               SliderException

Description copied from class: AbstractProviderService

No-op implementation of this method.

Specified by:

initializeApplicationConfiguration in interface ProviderService

Overrides:

initializeApplicationConfiguration in class AbstractProviderService

Throws:

IOException

SliderException

buildContainerLaunchContext

public void buildContainerLaunchContext(ContainerLauncher launcher,
                               AggregateConf instanceDefinition,
                               org.apache.hadoop.yarn.api.records.Container container,
                               String role,
                               SliderFileSystem fileSystem,
                               org.apache.hadoop.fs.Path generatedConfPath,
                               MapOperations resourceComponent,
                               MapOperations appComponent,
                               org.apache.hadoop.fs.Path containerTmpDirPath)
                                 throws IOException,
                                        SliderException

Description copied from interface: ProviderService

Set up the entire container launch context

Specified by:

buildContainerLaunchContext in interface ProviderService

Throws:

IOException

SliderException

rebuildContainerDetails

public void rebuildContainerDetails(List<org.apache.hadoop.yarn.api.records.Container> liveContainers,
                           String applicationId,
                           Map<Integer,ProviderRole> providerRoleMap)

Description copied from class: AbstractProviderService

No-op implementation of this method.

Specified by:

rebuildContainerDetails in interface ProviderService

Overrides:

rebuildContainerDetails in class AbstractProviderService

isSupportedRole

public boolean isSupportedRole(String role)

Description copied from class: AbstractProviderService

Scan through the roles and see if it is supported.

Specified by:

isSupportedRole in interface ProviderService

Overrides:

isSupportedRole in class AbstractProviderService

Parameters:

role - role to look for

Returns:

true if the role is known about -and therefore that a launcher thread can be deployed to launch it

handleRegistration

public RegistrationResponse handleRegistration(Register registration)

Handle registration calls from the agents

Specified by:

handleRegistration in interface AgentRestOperations

Parameters:

registration - registration entry

Returns:

response

handleHeartBeat

public HeartBeatResponse handleHeartBeat(HeartBeat heartBeat)

Handle heartbeat response from agents

Specified by:

handleHeartBeat in interface AgentRestOperations

Parameters:

heartBeat - incoming heartbeat from Agent

Returns:

response to send back

processAllocatedPorts

protected void processAllocatedPorts(String fqdn,
                         String roleName,
                         String containerId,
                         Map<String,String> ports)

buildMonitorDetails

public Map<String,String> buildMonitorDetails(ClusterDescription clusterDesc)

Description copied from interface: ProviderService

Build a map of data intended for the AM webapp that is specific about this provider. The key is some text to be displayed, and the value can be a URL that will create an anchor over the key text. If no anchor is needed/desired, insert the key with a null value.

Specified by:

buildMonitorDetails in interface ProviderService

Overrides:

buildMonitorDetails in class AbstractProviderService

Returns:

the details

applyInitialRegistryDefinitions

public void applyInitialRegistryDefinitions(URL amWebURI,
                                   URL agentOpsURI,
                                   URL agentStatusURI,
                                   org.apache.hadoop.registry.client.types.ServiceRecord serviceRecord)
                                     throws IOException

Description copied from interface: ProviderService

Prior to going live -register the initial service registry data

Specified by:

applyInitialRegistryDefinitions in interface ProviderService

Overrides:

applyInitialRegistryDefinitions in class AbstractProviderService

Parameters:

amWebURI - URL to the AM. This may be proxied, so use relative paths

agentOpsURI - URI for agent operations. This will not be proxied

agentStatusURI - URI For agent status. Again: no proxy

serviceRecord - service record to build up

Throws:

IOException

notifyContainerCompleted

public void notifyContainerCompleted(org.apache.hadoop.yarn.api.records.ContainerId containerId)

Description copied from interface: ProviderService

Notify the providers of container completion

Specified by:

notifyContainerCompleted in interface ProviderService

Overrides:

notifyContainerCompleted in class AbstractProviderService

Parameters:

containerId - container that has completed

getLogFolderExports

protected Map<String,ExportEntry> getLogFolderExports()

getWorkFolderExports

protected Map<String,ExportEntry> getWorkFolderExports()

getMetaInfo

protected Metainfo getMetaInfo()

getComponentStatuses

protected Map<String,ComponentInstanceState> getComponentStatuses()

getApplicationMetainfo

protected Metainfo getApplicationMetainfo(SliderFileSystem fileSystem,
                              String appDef,
                              boolean addonPackage)
                                   throws IOException,
                                          BadConfigException

Throws:

IOException

BadConfigException

getApplicationMetainfo

protected Metainfo getApplicationMetainfo(SliderFileSystem fileSystem,
                              String appDef)
                                   throws IOException,
                                          BadConfigException

Throws:

IOException

BadConfigException

setHeartbeatMonitorInterval

protected void setHeartbeatMonitorInterval(int heartbeatMonitorInterval)

setInUpgradeMode

public void setInUpgradeMode(boolean inUpgradeMode)

addUpgradeContainers

public void addUpgradeContainers(Set<String> upgradeContainers)

setAppStopInitiated

public void setAppStopInitiated(boolean appStopInitiated)

initializeDefaultConfigs

protected Map<String,DefaultConfig> initializeDefaultConfigs(SliderFileSystem fileSystem,
                                                 String appDef,
                                                 Metainfo metainfo)
                                                      throws IOException

Read all default configs

Parameters:

fileSystem - fs

appDef - app default path

metainfo - metadata

Returns:

configuration maps

Throws:

IOException

getDefaultConfigs

protected Map<String,DefaultConfig> getDefaultConfigs()

publishApplicationInstanceData

protected void publishApplicationInstanceData(String name,
                                  String description,
                                  Iterable<Map.Entry<String,String>> entries)

Publish a named property bag that may contain name-value pairs for app configurations such as hbase-site

Parameters:

name -

description -

entries -

getRoleClusterNodeMapping

protected Map<String,Map<String,ClusterNode>> getRoleClusterNodeMapping()

Get a list of all hosts for all role/container per role

Returns:

the map of role->node

getClusterInfoPropertyValue

protected String getClusterInfoPropertyValue(String name)

getClusterOptionPropertyValue

protected String getClusterOptionPropertyValue(String name)
                                        throws BadConfigException

Throws:

BadConfigException

lostContainer

protected void lostContainer(String label,
                 org.apache.hadoop.yarn.api.records.ContainerId containerId)

Lost heartbeat from the container - release it and ask for a replacement (async operation)

Parameters:

label -

containerId -

buildProviderStatus

public Map<String,String> buildProviderStatus()

Build the provider status, can be empty

Specified by:

buildProviderStatus in interface ProviderService

Overrides:

buildProviderStatus in class AbstractProviderService

Returns:

the provider status - map of entries to add to the info section

publishFolderPaths

protected void publishFolderPaths(Map<String,String> folders,
                      String containerId,
                      String componentName,
                      String hostFqdn)

Format the folder locations and publish in the registry service

Parameters:

folders -

containerId -

hostFqdn -

componentName -

publishConfigAndExportGroups

protected void publishConfigAndExportGroups(HeartBeat heartBeat,
                                ComponentInstanceState componentStatus,
                                String componentName)

Process return status for component instances

Parameters:

heartBeat -

componentStatus -

getCurrentExports

protected Map<String,List<ExportEntry>> getCurrentExports(String groupName)

processAndPublishComponentSpecificData

protected void processAndPublishComponentSpecificData(Map<String,String> ports,
                                          String containerId,
                                          String hostFqdn,
                                          String componentName)

Publish component instance specific data if the component demands it

processAndPublishComponentSpecificExports

protected void processAndPublishComponentSpecificExports(Map<String,String> ports,
                                             String containerId,
                                             String hostFqdn,
                                             String compName)

Publish component instance specific data if the component demands it

getApplicationComponent

protected Component getApplicationComponent(String roleName)

Return Component based on name

Parameters:

roleName - component name

Returns:

the component entry or null for no match

getScriptPathForMasterPackage

protected CommandScript getScriptPathForMasterPackage(String roleName)

Extract script path from the application metainfo

Parameters:

roleName - component name

Returns:

the script path or null for no match

isMaster

protected boolean isMaster(String roleName)

Is the role of type MASTER

Parameters:

roleName - component name

Returns:

true if the role category is MASTER

canPublishConfig

protected boolean canPublishConfig(String roleName)

Can the role publish configuration

Parameters:

roleName - component name

Returns:

true if it can be pubished

isMarkedAutoRestart

protected boolean isMarkedAutoRestart(String roleName)

Checks if the role is marked auto-restart

Parameters:

roleName - component name

Returns:

true if it is auto-restart

canAnyMasterPublishConfig

protected boolean canAnyMasterPublishConfig()

Can any master publish config explicitly, if not a random master is used

Returns:

true if the condition holds

addInstallCommand

protected void addInstallCommand(String componentName,
                     String containerId,
                     HeartBeatResponse response,
                     String scriptPath,
                     ComponentCommand compCmd,
                     long timeout,
                     String pkg)
                          throws SliderException

Add install command to the heartbeat response

Parameters:

componentName -

containerId -

response -

scriptPath -

pkg - when this field is null, it indicates the command is for the master package; while not null, for the package named by this field

Throws:

SliderException

addInstallDockerCommand

protected void addInstallDockerCommand(String componentName,
                           String containerId,
                           HeartBeatResponse response,
                           ComponentCommand compCmd,
                           long timeout)
                                throws SliderException

Throws:

SliderException

getPackageListFromApplication

protected static String getPackageListFromApplication(Application application)

addStatusCommand

protected void addStatusCommand(String componentName,
                    String containerId,
                    HeartBeatResponse response,
                    String scriptPath,
                    long timeout)
                         throws SliderException

Throws:

SliderException

addStatusDockerCommand

protected void addStatusDockerCommand(String componentName,
                          String containerId,
                          HeartBeatResponse response,
                          String scriptPath,
                          long timeout)
                               throws SliderException

Throws:

SliderException

addGetConfigDockerCommand

protected void addGetConfigDockerCommand(String componentName,
                             String containerId,
                             HeartBeatResponse response)
                                  throws SliderException

Throws:

SliderException

addStartDockerCommand

protected void addStartDockerCommand(String componentName,
                         String containerId,
                         HeartBeatResponse response,
                         ComponentCommand startCommand,
                         long timeout,
                         boolean isMarkedAutoRestart)
                              throws SliderException

Throws:

SliderException

addGetConfigCommand

protected void addGetConfigCommand(String componentName,
                       String containerId,
                       HeartBeatResponse response)
                            throws SliderException

Throws:

SliderException

addStartCommand

protected void addStartCommand(String componentName,
                   String containerId,
                   HeartBeatResponse response,
                   String scriptPath,
                   ComponentCommand startCommand,
                   ComponentCommand stopCommand,
                   long timeout,
                   boolean isMarkedAutoRestart)
                        throws SliderException

Throws:

SliderException

addUpgradeCommand

protected void addUpgradeCommand(String componentName,
                     String containerId,
                     HeartBeatResponse response,
                     String scriptPath,
                     long timeout)
                          throws SliderException

Throws:

SliderException

addStopCommand

protected void addStopCommand(String componentName,
                  String containerId,
                  HeartBeatResponse response,
                  String scriptPath,
                  long timeout,
                  boolean isInUpgradeMode)
                       throws SliderException

Throws:

SliderException

getJDKDir

protected static String getJDKDir()

getAllocatedPorts

protected Map<String,String> getAllocatedPorts()

getComponentInstanceData

protected Map<String,Map<String,String>> getComponentInstanceData()

getAllocatedPorts

protected Map<String,String> getAllocatedPorts(String containerId)

dereferenceAllConfigs

protected void dereferenceAllConfigs(Map<String,Map<String,String>> configurations)

getSystemConfigurationsRequested

protected List<String> getSystemConfigurationsRequested(ConfTreeOperations appConf)

getApplicationConfigurationTypes

protected List<String> getApplicationConfigurationTypes()

addRoleRelatedTokens

protected void addRoleRelatedTokens(Map<String,String> tokens)