Requirements

• JDK 1.8
• Scala 2.12.18
• Spark-3.4.0（other spark version of piflow.jar should be built with code）
• Hadoop-3.3.0（other hadoop version of piflow.jar should be with code）

config.properties

  spark.master=yarn
  spark.deploy.mode=cluster
  
  #hdfs default file system
  fs.defaultFS=hdfs://10.0.85.83:9000
  
  #yarn resourcemanager hostname
  yarn.resourcemanager.hostname=10.0.85.83
  
  #if you want to use hive, set hive metastore uris
  #hive.metastore.uris=thrift://10.0.85.83:9083
  
  #show data in log, set 0 if you do not show the logs
  data.show=10

  #monitor the throughput of flow
  monitor.throughput=true

  #server port
  server.port=8001

  #h2db port
  h2.port=50001

  #if you want to mount python component,please set hdfs params
  #hdfs.cluster=master:127.0.0.1;slave1:127.0.0.2
  #hdfs.web.url=master:50070

Command

  ./start.sh
  ./stop.sh
  ./restart.sh
  ./status.sh