1
// 查看 Dataset 中的内容
2
scala> textFile.show
3
+--------------------+
4
|               value|
5
+--------------------+
6
|      # Apache Spark|
7
|                    |
8
|Spark is a fast a...|
9
|high-level APIs i...|
10
|supports general ...|
11
|rich set of highe...|
12
|MLlib for machine...|
13
|and Spark Streami...|
14
...
15
only showing top 20 rows
16

17
// 如果你想要获取全部信息的话
18
scala> textFile.collect.foreach(println)
19

20
// 查看 Dataset 的一些描述信息
21
scala> textFile.describe().show()
22
+-------+--------------------+
23
|summary|               value|
24
+-------+--------------------+
25
|  count|                 105|
26
|   mean|                null|
27
| stddev|                null|
28
|    min|                    |
29
|    max|will run the Pi e...|
30
+-------+--------------------+

1
// DataSet 中的items个数，在此即文件的行数
2
scala> textFile.count()
3
res5: Long = 105
4

5
// 取前四行，返回 Array
6
// 等同于 `textFile.take(4)`
7
scala> textFile.head(4)
8
res6: Array[String] = Array(# Apache Spark, "", Spark is a fast and general cluster computing system for Big Data. It provides, high-level APIs in Scala, Java, Python, and R, and an optimized engine that)
9

10
// 取前四行并且转化为 List
11
// 等同于 `textFile.take(4).toList`
12
scala> textFile.takeAsList(4)
13
res8: java.util.List[String] = [# Apache Spark, , Spark is a fast and general cluster computing system for Big Data. It provides, high-level APIs in Scala, Java, Python, and R, and an optimized engine that]

1
scala> textFile.map(line => line.split(" ").size)
2
res14: org.apache.spark.sql.Dataset[Int] = [value: int]

1
scala> textFile.filter(line => line.split(" ").size > 10).count()
2
res20: Long = 22

1
var counter = 0
2
var rdd = sc.parallelize(Array(1, 2, 3, 4, 5))
3

4
rdd.foreach(x => counter += x)
5

6
println(counter)

1
var rdd = sc.parallelize(Array(1, 2, 3, 4, 5))
2
var pairs = rdd.map(x => (if (x > 3) 10 else 1, x))
3
pairs.foreach(println)
4

5
// 打印出来数据如下
6
// (1,1)
7
// (1,2)
8
// (1,3)
9
// (10,4)
10
// (10,5)
11

12
pairs.keys.foreach(println)
13
// 1
14
// 1
15
// 1
16
// 10
17
// 10

1
{"name":"Michael"}
2
{"name":"Andy", "age":30}
3
{"name":"Justin", "age":19}

1
scala> val df = spark.read.json("examples/src/main/resources/people.json")
2
df: org.apache.spark.sql.DataFrame = [age: bigint, name: string]
3

4
scala> df.show
5
+----+-------+
6
| age|   name|
7
+----+-------+
8
|null|Michael|
9
|  30|   Andy|
10
|  19| Justin|
11
+----+-------+
12

13
// 打印 schema 信息
14
scala> df.printSchema
15
root
16
 |-- age: long (nullable = true)
17
 |-- name: string (nullable = true)
18

19
scala> df.select("name").show
20
+-------+
21
|   name|
22
+-------+
23
|Michael|
24
|   Andy|
25
| Justin|
26
+-------+
27

28
scala> df.select($"name", $"age" + 100).show()
29
+-------+-----------+
30
|   name|(age + 100)|
31
+-------+-----------+
32
|Michael|       null|
33
|   Andy|        130|
34
| Justin|        119|
35
+-------+-----------+
36

37
scala> df.filter($"age" > 21).show()
38
+---+----+
39
|age|name|
40
+---+----+
41
| 30|Andy|
42
+---+----+
43

44
// 把 df 作为一个 `global_temp` 的 sql table
45
scala> df.createGlobalTempView("people")
46
19/04/19 17:10:30 WARN ObjectStore: Version information not found in metastore. hive.metastore.schema.verification is not enabled so recording the schema version 1.2.0
47
19/04/19 17:10:31 WARN ObjectStore: Failed to get database default, returning NoSuchObjectException
48
19/04/19 17:10:32 WARN ObjectStore: Failed to get database global_temp, returning NoSuchObjectException
49

50
scala> spark.sql("SELECT count(*) FROM global_temp.people").show
51
+--------+
52
|count(1)|
53
+--------+
54
|       3|
55
+--------+
56

57
scala> spark.sql("SELECT * FROM global_temp.people").show
58
+----+-------+
59
| age|   name|
60
+----+-------+
61
|null|Michael|
62
|  30|   Andy|
63
|  19| Justin|
64
+----+-------+