spark source code debug

直接github下载spark的源码
\spark-2.4.2\core\src\test\scala\org\apache\spark\SortShuffleSuite.scala

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
test("mytest") {
 sc = new SparkContext("local", "test", conf)
 val dfsFilename = "c:/temp/abc.txt"
 val readFileRDD = sc.textFile(dfsFilename)
 val wcounts1 = readFileRDD.flatMap(line => line.split(" "))
 wcounts1.collect().foreach(println)
 }

test("mytest2") {
 sc = new SparkContext("local", "test", conf)
 val dfsFilename = "c:/temp/abc.txt"
 val readFileRDD = sc.textFile(dfsFilename)
 val wcounts1 = readFileRDD.flatMap(line => line.split(" ")).filter(w => (w == "Humpty") || (w == "Dumpty")).map(word => (word, 1)).reduceByKey(_ + _)
 wcounts1.collect.foreach(println)
 }

最方便的就是源码中找到一个sparkcontext创建好的，然后加上自己的代码就好了