Spark Interview Scenario 2 - Zeyobron Analytics - +917395899448

🔴 Full Code Below


package pack

import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.{SparkSession, Row}
import org.apache.spark.sql.types._
import org.apache.spark.sql.functions._

object obj {


def main(args:Array[String]):Unit={

println("===Hello====")

val conf = new SparkConf().setAppName("first").setMaster("local[*]").set("spark.driver.host","localhost")
.set("spark.driver.allowMultipleContexts", "true")

val sc = new SparkContext(conf)

sc.setLogLevel("ERROR")

val spark = SparkSession.builder.getOrCreate()

import spark.implicits._



val data = Seq(
("A", "AA"),
("B", "BB"),
("C", "CC"),
("AA", "AAA"),
("BB", "BBB"),
("CC", "CCC")
)


val df = data.toDF("child", "parent")

df.show()



val df1 = df

val df2 = df.withColumnRenamed("child","child1")
.withColumnRenamed("parent","parent1")



df1.show()

df2.show()


val joineddf = df1.join(df2, df1("parent")=== df2("child1"),"inner")
.drop("child1")
.withColumnRenamed("parent1","GrandParent")


joineddf.show()



df1.createOrReplaceTempView("df1")
df2.createOrReplaceTempView("df2")


spark.sql("""

select a.child,a.parent,b.parent1 as GrandParent


from df1 a join df2 b


on a.parent = b.child1

""").show()



}

}