From c5a46a98e04134f9f3f16bf7eada6b8aeaad8533 Mon Sep 17 00:00:00 2001 From: dev_xulongjin Date: Mon, 21 Apr 2025 16:23:46 +0800 Subject: [PATCH] =?UTF-8?q?refactor(spark-lesson):=20=E5=AE=9E=E9=AA=8C?= =?UTF-8?q?=E4=BA=94=E9=83=A8=E5=88=86=E5=86=85=E5=AE=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/scala/date_20250415/RDDAverageScore.scala | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/spark-lesson/src/main/scala/date_20250415/RDDAverageScore.scala b/spark-lesson/src/main/scala/date_20250415/RDDAverageScore.scala index 8a09672..e0682ed 100755 --- a/spark-lesson/src/main/scala/date_20250415/RDDAverageScore.scala +++ b/spark-lesson/src/main/scala/date_20250415/RDDAverageScore.scala @@ -17,9 +17,9 @@ object RDDAverageScore { val sc = new SparkContext(conf) // 读取每个科目的成绩文件 - val mathRDD: RDD[String] = sc.textFile("hdfs://192.168.182.100:9000/data/math.txt") - val chineseRDD: RDD[String] = sc.textFile("hdfs://192.168.182.100:9000/data/chinese.txt") - val englishRDD: RDD[String] = sc.textFile("hdfs://192.168.182.100:9000/data/english.txt") + val mathRDD: RDD[String] = sc.textFile("hdfs://hadoop102:8020/input/data/math.txt") + val chineseRDD: RDD[String] = sc.textFile("hdfs://hadoop102:8020/input/data/chinese.txt") + val englishRDD: RDD[String] = sc.textFile("hdfs://hadoop102:8020/input/data/english.txt") // 处理每个科目RDD，将每一行数据转化为 (姓名, 分数) 二元组 val mathScores: RDD[(String, Int)] = mathRDD.map(line => { @@ -41,7 +41,8 @@ object RDDAverageScore { val allScores: RDD[(String, Int)] = mathScores.union(chineseScores).union(englishScores) // 对每个学生的所有分数进行聚合 (姓名, (总分, 科目数)) - val studentScores: RDD[(String, (Int, Int))] = allScores.补充 + val studentScores: RDD[(String, (Int, Int))] = allScores + .map( x => (x._1,(x._2,1))) .reduceByKey { case ((sum1, count1), (sum2, count2)) => (sum1 + sum2, count1 + count2) }