vivekmurugesan-striim/spark-feature-engineering

# spark-feature-engineering 为欺诈检测 ML 训练生成特征的 Spark 数据处理 # 运行方式 spark-submit --master local[*] --driver-memory 4g --executor-memory 4g --class "com.test.feature.engg.SparkDriver" target/fraud-detection-ml-featureengg-1.0-SNAPSHOT-jar-with-dependencies.jar /home/ec2-user/ml-data/FraudulentData/RawData /home/ec2-user/ml-data/FraudulentData/FraudTransactionMapping output >target/out.txt 2>target/err.txt & # 运行交易特征生成 spark-submit --master local[*] --driver-memory 4g --executor-memory 4g --class "com.test.feature.engg.TransactionFeatureGenerator" target/fraud-detection-ml-featureengg-1.0-SNAPSHOT-jar-with-dependencies.jar /home/ec2-user/ml-data/FraudulentData/RawData output >target/out.txt 2>target/err.txt & # 运行交易特征生成 spark-submit --master local[*] --driver-memory 4g --executor-memory 4g --class "com.test.feature.engg.CustomerFeatureGenerator" target/fraud-detection-ml-featureengg-1.0-SNAPSHOT-jar-with-dependencies.jar /home/ec2-user/ml-data/FraudulentData/RawData /home/ec2-user/ml-data/FraudulentData/FraudTransactionMapping output >target/out.txt 2>target/err.txt & # 过滤交易特征 spark-submit --master local[*] --driver-memory 4g --executor-memory 4g --class "com.test.feature.engg.TransactionRecordFilter" target/fraud-detection-ml-featureengg-1.0-SNAPSHOT-jar-with-dependencies.jar /home/ec2-user/ml-data/data output >target/out.txt 2>target/err.txt & # 过滤欺诈标志 spark-submit --master local[*] --driver-memory 4g --executor-memory 4g --class "com.test.feature.engg.RandomFradulentDataFilter" target/fraud-detection-ml-featureengg-1.0-SNAPSHOT-jar-with-dependencies.jar /home/ec2-user/ml-data/data output >target/out.txt 2>target/err.txt & # 生成基于规则的欺诈标记 spark-submit --master local[*] --driver-memory 4g --executor-memory 4g --class "com.test.feature.engg.RuleBasedFraudTransMarker" target/fraud-detection-ml-featureengg-1.0-SNAPSHOT-jar-with-dependencies.jar /home/ec2-user/ml-data/FraudulentData/RawData output >target/out.txt 2>target/err.txt & # 运行训练记录生成器 spark-submit --master local[*] --driver-memory 4g --executor-memory 4g --class "com.test.feature.engg.TrainingRecordGenerator" target/fraud-detection-ml-featureengg-1.0-SNAPSHOT-jar-with-dependencies.jar /home/ec2-user/ml-data/features output >target/out.txt 2>target/err.txt & # 触发模型训练 spark-submit --master local[*] --driver-memory 8g --executor-memory 8g --conf spark.memory.fraction=0.7 --conf spark.memory.storageFraction=0.3 --conf spark.network.timeout=18000s --conf spark.executor.heartbeatInterval=60s --conf spark.sql.shuffle.partitions=800 --conf spark.driver.maxResultSize=4g --conf spark.shuffle.service.enabled=true --conf spark.dynamicAllocation.enabled=true --conf spark.sql.broadcastTimeout=18000 --class "com.test.ml.training.FraudDetectionMLLogisticRegressionClassifier" target/fraud-detection-ml-featureengg-1.0-SNAPSHOT-jar-with-dependencies.jar /home/ec2-user/ml-data/training-features output >target/out.txt 2>target/err.txt & # 触发模型推理 java -cp target/fraud-detection-ml-featureengg-1.0-SNAPSHOT-jar-with-dependencies.jar com.test.ml.inference.ONNXModelInference /home/ec2-user/ml-data/PythonONNXExporter/model.onnx

标签：Apex, CNCF毕业项目, JS文件枚举, Spark, 反欺诈, 域名枚举, 大数据, 机器学习, 特征工程, 目录扫描