spark单机搭建

# _*_ coding:utf-8 _*_
from __future__ import print_function

from pyspark.sql import SparkSession
from pyspark.sql import Row

def json_dataset_example(spark):
    sc = spark.sparkContext

    #读取json串
    path = "/home/hadoop/spark-2.2.0-bin-hadoop2.7/mydemo/employees.json"
    peopleDF = spark.read.json(path)

    peopleDF.printSchema()

    peopleDF.createOrReplaceTempView("employees")

    teenagerNamesDF = spark.sql("SELECT name FROM employees WHERE salary BETWEEN 3500 AND 4500")
    teenagerNamesDF.show()

    #直接字符串
    jsonStrings = [‘{"name":"Yin","address":{"city":"Columbus","state":"Ohio"}}‘]
    otherPeopleRDD = sc.parallelize(jsonStrings)
    otherPeople = spark.read.json(otherPeopleRDD)
    otherPeople.show()

if __name__ == "__main__":
    spark = SparkSession         .builder         .appName("myPeople demo")         .getOrCreate()
    json_dataset_example(spark)
    spark.stop()