如何解决添加父列名称作为前缀以避免歧义
检查以下代码。如果存在重复的密钥,它将生成含糊不清的数据帧。我们应该如何修改代码以添加父列名称作为前缀。
添加了另一列包含json数据。
scala> val df = Seq(
(77,"email1","""{"key1":38,"key3":39}""","""{"name":"aaa","age":10}"""),(78,"email2","key4":39}""","""{"name":"bbb","age":20}"""),(178,"email21","""{"key1":"when string","key4":36,"key6":"test","key10":false }""","""{"name":"ccc","age":30}"""),(179,"email8","""{"sub1":"qwerty","sub2":["42"]}""","""{"name":"ddd","age":40}"""),(180,"sub2":["42","56","test"]}""","""{"name":"eee","age":50}""")
).toDF("id","name","colJson","personInfo")
scala> df.printSchema
root
|-- id: integer (nullable = false)
|-- name: string (nullable = true)
|-- colJson: string (nullable = true)
|-- personInfo: string (nullable = true)
scala> df.show(false)
+---+-------+---------------------------------------------------------------+-----------------------+
|id |name |colJson |personInfo |
+---+-------+---------------------------------------------------------------+-----------------------+
|77 |email1 |{"key1":38,"key3":39} |{"name":"aaa","age":10}|
|78 |email2 |{"key1":38,"key4":39} |{"name":"bbb","age":20}|
|178|email21|{"key1":"when string","key10":false }|{"name":"ccc","age":30}|
|179|email8 |{"sub1":"qwerty","sub2":["42"]} |{"name":"ddd","age":40}|
|180|email8 |{"sub1":"qwerty","test"]} |{"name":"eee","age":50}|
+---+-------+---------------------------------------------------------------+-----------------------+
创建了 fromJson 隐式函数,您可以将多个列传递给此&它将解析并从json中提取列。
scala> :paste
// Entering paste mode (ctrl-D to finish)
import org.apache.spark.sql.{Column,DataFrame,Row}
import org.apache.spark.sql.functions.from_json
implicit class DFHelper(inDF: DataFrame) {
import inDF.sparkSession.implicits._
def fromJson(columns:Column*):DataFrame = {
val schemas = columns.map(column => (column,inDF.sparkSession.read.json(inDF.select(column).as[String]).schema))
val mdf = schemas.foldLeft(inDF)((df,schema) => {
df.withColumn(schema._1.toString(),from_json(schema._1,schema._2))
})
mdf.selectExpr(mdf.schema.map(c => if(c.dataType.typeName =="struct") s"${c.name}.*" else c.name):_*)
}
}
// Exiting paste mode,now interpreting.
import org.apache.spark.sql.{Column,Row}
import org.apache.spark.sql.functions.from_json
defined class DFHelper
scala> df.fromJson($"colJson",$"personInfo").show(false)
+---+-------+-----------+-----+----+----+----+------+--------------+---+----+
|id |name |key1 |key10|key3|key4|key6|sub1 |sub2 |age|name|
+---+-------+-----------+-----+----+----+----+------+--------------+---+----+
|77 |email1 |38 |null |39 |null|null|null |null |10 |aaa |
|78 |email2 |38 |null |null|39 |null|null |null |20 |bbb |
|178|email21|when string|false|null|36 |test|null |null |30 |ccc |
|179|email8 |null |null |null|null|null|qwerty|[42] |40 |ddd |
|180|email8 |null |null |null|null|null|qwerty|[42,56,test]|50 |eee |
+---+-------+-----------+-----+----+----+----+------+--------------+---+----+
scala> df.fromJson($"colJson",$"personInfo").printSchema()
root
|-- id: integer (nullable = false)
|-- name: string (nullable = true)
|-- key1: string (nullable = true)
|-- key10: boolean (nullable = true)
|-- key3: long (nullable = true)
|-- key4: long (nullable = true)
|-- key6: string (nullable = true)
|-- sub1: string (nullable = true)
|-- sub2: array (nullable = true)
| |-- element: string (containsNull = true)
|-- age: long (nullable = true)
|-- name: string (nullable = true)
解决方法
试试这个-
<form action = "s{{url_for('success')}}" methods="POST">
<form action = "{{url_for('success')}}" method="POST">
df.show(false)
df.printSchema()
/**
* +---+-------+---------------------------------------------------------------+-----------------------+
* |id |name |colJson |personInfo |
* +---+-------+---------------------------------------------------------------+-----------------------+
* |77 |email1 |{"key1":38,"key3":39} |{"name":"aaa","age":10}|
* |78 |email2 |{"key1":38,"key4":39} |{"name":"bbb","age":20}|
* |178|email21|{"key1":"when string","key4":36,"key6":"test","key10":false }|{"name":"ccc","age":30}|
* |179|email8 |{"sub1":"qwerty","sub2":["42"]} |{"name":"ddd","age":40}|
* |180|email8 |{"sub1":"qwerty","sub2":["42","56","test"]} |{"name":"eee","age":50}|
* +---+-------+---------------------------------------------------------------+-----------------------+
*
* root
* |-- id: integer (nullable = false)
* |-- name: string (nullable = true)
* |-- colJson: string (nullable = true)
* |-- personInfo: string (nullable = true)
*
* @param inDF
*/
版权声明:本文内容由互联网用户自发贡献,该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务,不拥有所有权,不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容, 请发送邮件至 dio@foxmail.com 举报,一经查实,本站将立刻删除。