Скачать или смотреть Session 10 - PySpark select transformation

select Transformation

/FileStore/emp_data.csv

1. Select transformation in PySpark takes column name as input.

from pyspark.sql.types import *

emp_schema = StructType([
StructField("emp_id",IntegerType()),
StructField("emp_name",StringType()),
StructField("emp_salary",LongType())
])

emp_df = spark.read\
.format("csv")\
.schema(emp_schema)\
.option("header",True)\
.load("/FileStore/emp_data.csv")

from pyspark.sql import functions as F

emp_df1 = emp_df.select(F.col("emp_id"),F.col("emp_salary"))

emp_df2 = emp_df.withColumn("emp_salary",F.expr("emp_salary*1.06"))

emp_df3 = emp_df.select(F.col("emp_id"),F.col("emp_name"),F.expr("emp_salary*1.06 AS emp_salary"))

emp_df4 = emp_df.select(F.col("emp_id"),F.col("emp_name"),F.expr("emp_salary*1.06").alias("emp_salary"))

emp_df5 = emp_df.withColumn("new_salary",F.expr("emp_salary*1.06"))

emp_df6 = emp_df.select(F.col("emp_id"),F.col("emp_name"),F.col("emp_salary"),F.expr("emp_salary*1.06").alias("new_salary"))

emp_df.createOrReplaceTempView("employee")

spark.sql("SELECT emp_id,emp_name,emp_salary*1.06 AS emp_salary FROM employee").display()

spark.sql("SELECT emp_id,emp_name,emp_salary,emp_salary*1.06 AS new_salary FROM employee").display()

#pyspark #apachespark #learningvideos #learning #dataframes #sparksql