public abstract class AbstractYarnClusterDescriptor extends Object implements ClusterDescriptor<YarnClusterClient>
YarnClusterClient
.Modifier and Type | Field and Description |
---|---|
protected List<File> |
shipFiles
Lazily initialized list of files to ship.
|
Constructor and Description |
---|
AbstractYarnClusterDescriptor(Configuration flinkConfiguration,
String configurationDirectory) |
Modifier and Type | Method and Description |
---|---|
protected void |
addLibFolderToShipFiles(Collection<File> effectiveShipFiles) |
void |
addShipFiles(List<File> shipFiles)
Adds the given files to the list of files to ship.
|
protected YarnClusterClient |
createYarnClusterClient(AbstractYarnClusterDescriptor descriptor,
int numberTaskManagers,
int slotsPerTaskManager,
org.apache.hadoop.yarn.client.api.YarnClient yarnClient,
org.apache.hadoop.yarn.api.records.ApplicationReport report,
Configuration flinkConfiguration,
boolean perJobCluster)
Creates a YarnClusterClient; may be overriden in tests.
|
protected YarnClusterClient |
deployInternal(ClusterSpecification clusterSpecification,
String yarnClusterEntrypoint,
JobGraph jobGraph)
This method will block until the ApplicationMaster/JobManager have been
deployed on YARN.
|
YarnClusterClient |
deployJobCluster(ClusterSpecification clusterSpecification,
JobGraph jobGraph)
Deploys a per-job cluster with the given job on the cluster.
|
YarnClusterClient |
deploySessionCluster(ClusterSpecification clusterSpecification)
Triggers deployment of a cluster.
|
String |
getClusterDescription()
Returns a String containing details about the cluster (NodeManagers, available memory, ...).
|
String |
getDynamicPropertiesEncoded() |
Configuration |
getFlinkConfiguration() |
protected org.apache.hadoop.yarn.client.api.YarnClient |
getYarnClient()
Gets a Hadoop Yarn client.
|
protected abstract String |
getYarnJobClusterEntrypoint() |
protected abstract String |
getYarnSessionClusterEntrypoint()
The class to bootstrap the application master of the Yarn cluster (runs main method).
|
String |
getZookeeperNamespace() |
boolean |
hasUserJarFiles(List<URL> requiredJarFiles)
Returns true if the descriptor has the job jars to include in the classpath.
|
boolean |
isDetachedMode() |
YarnClusterClient |
retrieve(String applicationID)
Retrieves an existing Flink Cluster.
|
void |
setDetachedMode(boolean detachedMode) |
void |
setDynamicPropertiesEncoded(String dynamicPropertiesEncoded) |
void |
setLocalJarPath(org.apache.hadoop.fs.Path localJarPath) |
void |
setName(String name) |
void |
setProvidedUserJarFiles(List<URL> userJarFiles)
Sets the user jar which is included in the system classloader of all nodes.
|
void |
setQueue(String queue) |
protected org.apache.hadoop.yarn.api.records.ContainerLaunchContext |
setupApplicationMasterContainer(String yarnClusterEntrypoint,
boolean hasLogback,
boolean hasLog4j,
boolean hasKrb5,
int jobManagerMemoryMb) |
void |
setZookeeperNamespace(String zookeeperNamespace) |
org.apache.hadoop.yarn.api.records.ApplicationReport |
startAppMaster(String yarnClusterEntrypoint,
JobGraph jobGraph,
org.apache.hadoop.yarn.client.api.YarnClient yarnClient,
org.apache.hadoop.yarn.client.api.YarnClientApplication yarnApplication,
ClusterSpecification clusterSpecification) |
protected ClusterSpecification |
validateClusterResources(ClusterSpecification clusterSpecification,
int yarnMinAllocationMB,
org.apache.hadoop.yarn.api.records.Resource maximumResourceCapability,
org.apache.flink.yarn.AbstractYarnClusterDescriptor.ClusterResourceDescription freeClusterResources) |
public AbstractYarnClusterDescriptor(Configuration flinkConfiguration, String configurationDirectory)
protected abstract String getYarnSessionClusterEntrypoint()
protected abstract String getYarnJobClusterEntrypoint()
public Configuration getFlinkConfiguration()
public void setQueue(String queue)
public void setLocalJarPath(org.apache.hadoop.fs.Path localJarPath)
public void addShipFiles(List<File> shipFiles)
Note that any file matching "flink-dist*.jar" will be excluded from the upload by
uploadAndRegisterFiles(Collection, FileSystem, Path, ApplicationId, List, Map, StringBuilder)
since we upload the Flink uber jar ourselves and do not need to deploy it multiple times.
shipFiles
- files to shippublic void setDynamicPropertiesEncoded(String dynamicPropertiesEncoded)
public boolean hasUserJarFiles(List<URL> requiredJarFiles)
public void setProvidedUserJarFiles(List<URL> userJarFiles)
public String getDynamicPropertiesEncoded()
public void setDetachedMode(boolean detachedMode)
public boolean isDetachedMode()
public String getZookeeperNamespace()
public void setZookeeperNamespace(String zookeeperNamespace)
protected org.apache.hadoop.yarn.client.api.YarnClient getYarnClient()
public YarnClusterClient retrieve(String applicationID)
ClusterDescriptor
retrieve
in interface ClusterDescriptor<YarnClusterClient>
applicationID
- The unique application identifier of the running clusterpublic YarnClusterClient deploySessionCluster(ClusterSpecification clusterSpecification)
ClusterDescriptor
deploySessionCluster
in interface ClusterDescriptor<YarnClusterClient>
clusterSpecification
- Cluster specification defining the cluster to deploypublic YarnClusterClient deployJobCluster(ClusterSpecification clusterSpecification, JobGraph jobGraph)
ClusterDescriptor
deployJobCluster
in interface ClusterDescriptor<YarnClusterClient>
clusterSpecification
- Initial cluster specification with which the Flink cluster is launchedjobGraph
- JobGraph with which the job cluster is startedprotected YarnClusterClient deployInternal(ClusterSpecification clusterSpecification, String yarnClusterEntrypoint, @Nullable JobGraph jobGraph) throws Exception
clusterSpecification
- Initial cluster specification for the to be deployed Flink clusterjobGraph
- A job graph which is deployed with the Flink cluster, null if noneException
protected ClusterSpecification validateClusterResources(ClusterSpecification clusterSpecification, int yarnMinAllocationMB, org.apache.hadoop.yarn.api.records.Resource maximumResourceCapability, org.apache.flink.yarn.AbstractYarnClusterDescriptor.ClusterResourceDescription freeClusterResources) throws org.apache.flink.yarn.AbstractYarnClusterDescriptor.YarnDeploymentException
org.apache.flink.yarn.AbstractYarnClusterDescriptor.YarnDeploymentException
public org.apache.hadoop.yarn.api.records.ApplicationReport startAppMaster(String yarnClusterEntrypoint, JobGraph jobGraph, org.apache.hadoop.yarn.client.api.YarnClient yarnClient, org.apache.hadoop.yarn.client.api.YarnClientApplication yarnApplication, ClusterSpecification clusterSpecification) throws Exception
Exception
public String getClusterDescription()
ClusterDescriptor
getClusterDescription
in interface ClusterDescriptor<YarnClusterClient>
public void setName(String name)
protected void addLibFolderToShipFiles(Collection<File> effectiveShipFiles)
protected org.apache.hadoop.yarn.api.records.ContainerLaunchContext setupApplicationMasterContainer(String yarnClusterEntrypoint, boolean hasLogback, boolean hasLog4j, boolean hasKrb5, int jobManagerMemoryMb)
protected YarnClusterClient createYarnClusterClient(AbstractYarnClusterDescriptor descriptor, int numberTaskManagers, int slotsPerTaskManager, org.apache.hadoop.yarn.client.api.YarnClient yarnClient, org.apache.hadoop.yarn.api.records.ApplicationReport report, Configuration flinkConfiguration, boolean perJobCluster) throws Exception
Exception
Copyright © 2014–2018 The Apache Software Foundation. All rights reserved.