package date_20250429

import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{Row, SparkSession}
import org.apache.spark.sql.types._
import scala.collection.mutable.ListBuffer


object sparksqlkeywords {
    def main(args: Array[String]): Unit = {
        val spark = SparkSession.builder()
          .appName("")
          .master("local[*]")
          .getOrCreate()

        val linesRDD: RDD[String] = spark.sparkContext
          .textFile("hdfs://hadoop102:8020/input/data_7/keywords.txt")

        val tupleRDD: RDD[((String, String), String)] = linesRDD.map(line => {
            val date = line.split(",")(0)
            val user = line.split(",")(1)
            val keyword = line.split(",")(2)

            ((date, keyword), user)

        })

        val groupedRDD: RDD[((String, String), Iterable[String])] = tupleRDD.groupByKey()

        val uvRDD: RDD[((String, String), Int)] = groupedRDD.map(line => {
            val dateAndKeyword: (String, String) = line._1

            val users: Iterator[String] = line._2.iterator

            val distinctUsers = new ListBuffer[String]()
            while (users.hasNext) {
                val user = users.next
                if (!distinctUsers.contains(user)) {
                    distinctUsers += user
                }

            }

            val uv = distinctUsers.size

            (dateAndKeyword, uv)
        })

        val rowRDD: RDD[Row] = uvRDD.map(line => {
            Row(
                line._1._1,
                line._1._2,
                line._2.toInt
            )
        })
        val structType: StructType = StructType(Array(
            StructField("date", StringType, true),
            StructField("keyword", StringType, true),
            StructField("uv", IntegerType, true)
        ))

        val df = spark.createDataFrame(rowRDD,structType)

        df.createTempView("date_keyword_uv")

        spark.sql(
            """
              |SELECT * FROM date_keyword_uv
              |""".stripMargin).show()

        spark.close()
    }
}