@ -14,12 +14,12 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License . * limitations under the License .
* / * /
package org.apache.dolphinscheduler.common.task.spark ; package org.apache.dolphinscheduler.common.task.spark ;
import org.apache.dolphinscheduler.common.enums.ProgramType ; import org.apache.dolphinscheduler.common.enums.ProgramType ;
import org.apache.dolphinscheduler.common.process.ResourceInfo ; import org.apache.dolphinscheduler.common.process.ResourceInfo ;
import org.apache.dolphinscheduler.common.task.AbstractParameters ; import org.apache.dolphinscheduler.common.task.AbstractParameters ;
import org.apache.dolphinscheduler.common.utils.CollectionUtils ;
import java.util.ArrayList ; import java.util.ArrayList ;
import java.util.List ; import java.util.List ;
@ -29,203 +29,214 @@ import java.util.List;
* / * /
public class SparkParameters extends AbstractParameters { public class SparkParameters extends AbstractParameters {
/ * * / * *
* major jar * main jar
* / * /
private ResourceInfo mainJar ; private ResourceInfo mainJar ;
/ * * / * *
* major class
* main class
* / * /
private String mainClass ; private String mainClass ;
/ * * / * *
* deploy mode * deploy mode
* / * /
private String deployMode ; private String deployMode ;
/ * * / * *
* arguments * arguments
* / * /
private String mainArgs ; private String mainArgs ;
/ * * / * *
* driver - cores Number of cores used by the driver , only in cluster mode * driver - cores Number of cores used by the driver , only in cluster mode
* / * /
private int driverCores ; private int driverCores ;
/ * * / * *
* driver - memory Memory for driver * driver - memory Memory for driver
* / * /
private String driverMemory ; private String driverMemory ;
/ * * / * *
* num - executors Number of executors to launch * num - executors Number of executors to launch
* / * /
private int numExecutors ; private int numExecutors ;
/ * * / * *
* executor - cores Number of cores per executor * executor - cores Number of cores per executor
* / * /
private int executorCores ; private int executorCores ;
/ * * / * *
* Memory per executor * Memory per executor
* / * /
private String executorMemory ; private String executorMemory ;
/ * * / * *
* resource list * app name
* / * /
private List < ResourceInfo > resourceList = new ArrayList < > ( ) ; private String appName ;
/ * * / * *
* The YARN queue to submit to * The YARN queue to submit to
* / * /
private String queue ; private String queue ;
/ * * / * *
* other arguments * other arguments
* / * /
private String others ; private String others ;
/ * * / * *
* program type * program type
* 0 JAVA , 1 SCALA , 2 PYTHON * 0 JAVA , 1 SCALA , 2 PYTHON
* / * /
private ProgramType programType ; private ProgramType programType ;
/ * * / * *
* spark version * spark version
* / * /
private String sparkVersion ; private String sparkVersion ;
public ResourceInfo getMainJar ( ) { / * *
return mainJar ; * resource list
} * /
private List < ResourceInfo > resourceList = new ArrayList < > ( ) ;
public void setMainJar ( ResourceInfo mainJar ) {
this . mainJar = mainJar ; public ResourceInfo getMainJar ( ) {
} return mainJar ;
}
public String getMainClass ( ) {
return mainClass ; public void setMainJar ( ResourceInfo mainJar ) {
} this . mainJar = mainJar ;
}
public void setMainClass ( String mainClass ) {
this . mainClass = mainClass ; public String getMainClass ( ) {
} return mainClass ;
}
public String getDeployMode ( ) {
return deployMode ; public void setMainClass ( String mainClass ) {
} this . mainClass = mainClass ;
}
public void setDeployMode ( String deployMode ) {
this . deployMode = deployMode ; public String getDeployMode ( ) {
} return deployMode ;
}
public String getMainArgs ( ) {
return mainArgs ; public void setDeployMode ( String deployMode ) {
} this . deployMode = deployMode ;
}
public void setMainArgs ( String mainArgs ) {
this . mainArgs = mainArgs ; public String getMainArgs ( ) {
} return mainArgs ;
}
public int getDriverCores ( ) {
return driverCores ; public void setMainArgs ( String mainArgs ) {
} this . mainArgs = mainArgs ;
}
public void setDriverCores ( int driverCores ) {
this . driverCores = driverCores ; public int getDriverCores ( ) {
} return driverCores ;
}
public String getDriverMemory ( ) {
return driverMemory ; public void setDriverCores ( int driverCores ) {
} this . driverCores = driverCores ;
}
public void setDriverMemory ( String driverMemory ) {
this . driverMemory = driverMemory ; public String getDriverMemory ( ) {
} return driverMemory ;
}
public void setDriverMemory ( String driverMemory ) {
this . driverMemory = driverMemory ;
}
public int getNumExecutors ( ) { public int getNumExecutors ( ) {
return numExecutors ; return numExecutors ;
} }
public void setNumExecutors ( int numExecutors ) { public void setNumExecutors ( int numExecutors ) {
this . numExecutors = numExecutors ; this . numExecutors = numExecutors ;
} }
public int getExecutorCores ( ) { public int getExecutorCores ( ) {
return executorCores ; return executorCores ;
} }
public void setExecutorCores ( int executorCores ) {
this . executorCores = executorCores ;
}
public String getExecutorMemory ( ) { public void setExecutorCores ( int executorCores ) {
return executorMemory ; this . executorCores = executorCores ;
} }
public void setExecutorMemory ( String executorMemory ) { public String getExecutorMemory ( ) {
this . executorMemory = executorMemory ; return executorMemory ;
} }
public void setExecutorMemory ( String executorMemory ) {
this . executorMemory = executorMemory ;
}
public String getQueue ( ) { public String getAppName ( ) {
return queue ; return appName ;
} }
public void setAppName ( String appName ) {
this . appName = appName ;
}
public void setQueue ( String queue ) { public String getQueue ( ) {
this . queue = queue ; return queue ;
} }
public List < ResourceInfo > getResourceList ( ) { public void setQueue ( String queue ) {
return resourceList ; this . queue = queue ;
} }
public void setResourceList ( List < ResourceInfo > resourceList ) { public String getOthers ( ) {
this . resourceList = resourceList ; return others ;
} }
public String getOthers ( ) { public void setOthers ( String others ) {
return others ; this . others = others ;
} }
public void setOthers ( String others ) { public List < ResourceInfo > getResourceList ( ) {
this . others = others ; return resourceList ;
} }
public ProgramType getProgramType ( ) { public void setResourceList ( List < ResourceInfo > resourceList ) {
return programType ; this . resourceList = resourceList ;
} }
public void setProgramType ( ProgramType programType ) { public ProgramType getProgramType ( ) {
this . programType = programType ; return programType ;
} }
public String getSparkVersion ( ) { public void setProgramType ( ProgramType programType ) {
return sparkVersion ; this . programType = programType ;
} }
public void setSparkVersion ( String sparkVersion ) { public String getSparkVersion ( ) {
this . sparkVersion = sparkVersion ; return sparkVersion ;
} }
@Override public void setSparkVersion ( String sparkVersion ) {
public boolean checkParameters ( ) { this . sparkVersion = sparkVersion ;
return mainJar ! = null & & programType ! = null ; }
}
@Override @Override
public List < ResourceInfo > getResourceFilesList ( ) { public boolean checkParameters ( ) {
if ( mainJar ! = null & & ! resourceList . contains ( mainJar ) ) { return mainJar ! = null & & programType ! = null ;
resourceList . add ( mainJar ) ;
} }
return resourceList ;
}
@Override
public List < ResourceInfo > getResourceFilesList ( ) {
if ( mainJar ! = null & & ! resourceList . contains ( mainJar ) ) {
resourceList . add ( mainJar ) ;
}
return resourceList ;
}
} }
lenboo
4 years ago