public abstract class AbstractYarnClusterDescriptor extends Object implements ClusterDescriptor<org.apache.hadoop.yarn.api.records.ApplicationId>
YarnClusterClient
.Modifier and Type | Field and Description |
---|---|
protected List<File> |
shipFiles
Lazily initialized list of files to ship.
|
Constructor and Description |
---|
AbstractYarnClusterDescriptor(Configuration flinkConfiguration,
org.apache.hadoop.yarn.conf.YarnConfiguration yarnConfiguration,
String configurationDirectory,
org.apache.hadoop.yarn.client.api.YarnClient yarnClient,
boolean sharedYarnClient) |
Modifier and Type | Method and Description |
---|---|
protected void |
addLibFolderToShipFiles(Collection<File> effectiveShipFiles) |
void |
addShipFiles(List<File> shipFiles)
Adds the given files to the list of files to ship.
|
void |
close() |
protected abstract ClusterClient<org.apache.hadoop.yarn.api.records.ApplicationId> |
createYarnClusterClient(AbstractYarnClusterDescriptor descriptor,
int numberTaskManagers,
int slotsPerTaskManager,
org.apache.hadoop.yarn.api.records.ApplicationReport report,
Configuration flinkConfiguration,
boolean perJobCluster)
Creates a YarnClusterClient; may be overriden in tests.
|
protected ClusterClient<org.apache.hadoop.yarn.api.records.ApplicationId> |
deployInternal(ClusterSpecification clusterSpecification,
String applicationName,
String yarnClusterEntrypoint,
JobGraph jobGraph,
boolean detached)
This method will block until the ApplicationMaster/JobManager have been deployed on YARN.
|
ClusterClient<org.apache.hadoop.yarn.api.records.ApplicationId> |
deploySessionCluster(ClusterSpecification clusterSpecification)
Triggers deployment of a cluster.
|
String |
getClusterDescription()
Returns a String containing details about the cluster (NodeManagers, available memory, ...).
|
String |
getDynamicPropertiesEncoded() |
Configuration |
getFlinkConfiguration() |
String |
getNodeLabel() |
org.apache.hadoop.yarn.client.api.YarnClient |
getYarnClient() |
protected abstract String |
getYarnJobClusterEntrypoint()
The class to start the application master with.
|
protected abstract String |
getYarnSessionClusterEntrypoint()
The class to start the application master with.
|
String |
getZookeeperNamespace() |
boolean |
hasUserJarFiles(List<URL> requiredJarFiles)
Returns true if the descriptor has the job jars to include in the classpath.
|
boolean |
isDetachedMode()
Deprecated.
The cluster descriptor should not know about this option.
|
void |
killCluster(org.apache.hadoop.yarn.api.records.ApplicationId applicationId)
Terminates the cluster with the given cluster id.
|
ClusterClient<org.apache.hadoop.yarn.api.records.ApplicationId> |
retrieve(org.apache.hadoop.yarn.api.records.ApplicationId applicationId)
Retrieves an existing Flink Cluster.
|
void |
setDetachedMode(boolean detachedMode)
Deprecated.
The cluster descriptor should not know about this option.
|
void |
setDynamicPropertiesEncoded(String dynamicPropertiesEncoded) |
void |
setLocalJarPath(org.apache.hadoop.fs.Path localJarPath) |
void |
setName(String name) |
void |
setNodeLabel(String nodeLabel) |
void |
setProvidedUserJarFiles(List<URL> userJarFiles)
Sets the user jar which is included in the system classloader of all nodes.
|
void |
setQueue(String queue) |
protected org.apache.hadoop.yarn.api.records.ContainerLaunchContext |
setupApplicationMasterContainer(String yarnClusterEntrypoint,
boolean hasLogback,
boolean hasLog4j,
boolean hasKrb5,
int jobManagerMemoryMb) |
void |
setZookeeperNamespace(String zookeeperNamespace) |
org.apache.hadoop.yarn.api.records.ApplicationReport |
startAppMaster(Configuration configuration,
String applicationName,
String yarnClusterEntrypoint,
JobGraph jobGraph,
org.apache.hadoop.yarn.client.api.YarnClient yarnClient,
org.apache.hadoop.yarn.client.api.YarnClientApplication yarnApplication,
ClusterSpecification clusterSpecification) |
protected ClusterSpecification |
validateClusterResources(ClusterSpecification clusterSpecification,
int yarnMinAllocationMB,
org.apache.hadoop.yarn.api.records.Resource maximumResourceCapability,
org.apache.flink.yarn.AbstractYarnClusterDescriptor.ClusterResourceDescription freeClusterResources) |
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
deployJobCluster
public AbstractYarnClusterDescriptor(Configuration flinkConfiguration, org.apache.hadoop.yarn.conf.YarnConfiguration yarnConfiguration, String configurationDirectory, org.apache.hadoop.yarn.client.api.YarnClient yarnClient, boolean sharedYarnClient)
public org.apache.hadoop.yarn.client.api.YarnClient getYarnClient()
protected abstract String getYarnSessionClusterEntrypoint()
protected abstract String getYarnJobClusterEntrypoint()
public Configuration getFlinkConfiguration()
public void setQueue(String queue)
public void setLocalJarPath(org.apache.hadoop.fs.Path localJarPath)
public void addShipFiles(List<File> shipFiles)
Note that any file matching "flink-dist*.jar" will be excluded from the upload by
uploadAndRegisterFiles(Collection, FileSystem, Path, ApplicationId, List, Map, StringBuilder)
since we upload the Flink uber jar ourselves and do not need to deploy it multiple times.
shipFiles
- files to shippublic void setDynamicPropertiesEncoded(String dynamicPropertiesEncoded)
public boolean hasUserJarFiles(List<URL> requiredJarFiles)
public void setProvidedUserJarFiles(List<URL> userJarFiles)
public String getDynamicPropertiesEncoded()
@Deprecated public void setDetachedMode(boolean detachedMode)
@Deprecated public boolean isDetachedMode()
public String getZookeeperNamespace()
public void setZookeeperNamespace(String zookeeperNamespace)
public String getNodeLabel()
public void setNodeLabel(String nodeLabel)
public void close()
close
in interface AutoCloseable
public ClusterClient<org.apache.hadoop.yarn.api.records.ApplicationId> retrieve(org.apache.hadoop.yarn.api.records.ApplicationId applicationId) throws ClusterRetrieveException
ClusterDescriptor
retrieve
in interface ClusterDescriptor<org.apache.hadoop.yarn.api.records.ApplicationId>
applicationId
- The unique identifier of the running clusterClusterRetrieveException
- if the cluster client could not be retrievedpublic ClusterClient<org.apache.hadoop.yarn.api.records.ApplicationId> deploySessionCluster(ClusterSpecification clusterSpecification) throws ClusterDeploymentException
ClusterDescriptor
deploySessionCluster
in interface ClusterDescriptor<org.apache.hadoop.yarn.api.records.ApplicationId>
clusterSpecification
- Cluster specification defining the cluster to deployClusterDeploymentException
- if the cluster could not be deployedpublic void killCluster(org.apache.hadoop.yarn.api.records.ApplicationId applicationId) throws FlinkException
ClusterDescriptor
killCluster
in interface ClusterDescriptor<org.apache.hadoop.yarn.api.records.ApplicationId>
applicationId
- identifying the cluster to shut downFlinkException
- if the cluster could not be terminatedprotected ClusterClient<org.apache.hadoop.yarn.api.records.ApplicationId> deployInternal(ClusterSpecification clusterSpecification, String applicationName, String yarnClusterEntrypoint, @Nullable JobGraph jobGraph, boolean detached) throws Exception
clusterSpecification
- Initial cluster specification for the Flink cluster to be deployedapplicationName
- name of the Yarn application to startyarnClusterEntrypoint
- Class name of the Yarn cluster entry point.jobGraph
- A job graph which is deployed with the Flink cluster, null
if nonedetached
- True if the cluster should be started in detached modeException
protected ClusterSpecification validateClusterResources(ClusterSpecification clusterSpecification, int yarnMinAllocationMB, org.apache.hadoop.yarn.api.records.Resource maximumResourceCapability, org.apache.flink.yarn.AbstractYarnClusterDescriptor.ClusterResourceDescription freeClusterResources) throws org.apache.flink.yarn.AbstractYarnClusterDescriptor.YarnDeploymentException
org.apache.flink.yarn.AbstractYarnClusterDescriptor.YarnDeploymentException
public org.apache.hadoop.yarn.api.records.ApplicationReport startAppMaster(Configuration configuration, String applicationName, String yarnClusterEntrypoint, JobGraph jobGraph, org.apache.hadoop.yarn.client.api.YarnClient yarnClient, org.apache.hadoop.yarn.client.api.YarnClientApplication yarnApplication, ClusterSpecification clusterSpecification) throws Exception
Exception
public String getClusterDescription()
ClusterDescriptor
getClusterDescription
in interface ClusterDescriptor<org.apache.hadoop.yarn.api.records.ApplicationId>
public void setName(String name)
protected void addLibFolderToShipFiles(Collection<File> effectiveShipFiles)
protected org.apache.hadoop.yarn.api.records.ContainerLaunchContext setupApplicationMasterContainer(String yarnClusterEntrypoint, boolean hasLogback, boolean hasLog4j, boolean hasKrb5, int jobManagerMemoryMb)
protected abstract ClusterClient<org.apache.hadoop.yarn.api.records.ApplicationId> createYarnClusterClient(AbstractYarnClusterDescriptor descriptor, int numberTaskManagers, int slotsPerTaskManager, org.apache.hadoop.yarn.api.records.ApplicationReport report, Configuration flinkConfiguration, boolean perJobCluster) throws Exception
Exception
Copyright © 2014–2020 The Apache Software Foundation. All rights reserved.