如何解决无法使用 Kylo Catalog 解析数据源:com.thinkbiganalytics.spark.file.metadata:{}
class Script (engine: com.thinkbiganalytics.spark.repl.ScriptEngine) extends com.thinkbiganalytics.spark.repl.Script (engine) {
override def eval (): Any = {
class Transform (sqlContext: org.apache.spark.sql.SQLContext,sparkContextService: com.thinkbiganalytics.spark.SparkContextService) extends com.thinkbiganalytics.spark.metadata.TransformScript20(sqlContext,sparkContextService) {
override def dataFrame: org.apache.spark.sql.DataFrame = {import org.apache.spark.sql.functions.{concat,lit,concat_ws,collect_list,split,size,col,when}
import com.thinkbiganalytics.kylo.catalog._
var listBuffer = new scala.collection.mutable.ListBuffer[org.apache.spark.sql.DataFrame]()
var kyloClientBuilder = KyloCatalog.builder()
var kyloClient = kyloClientBuilder.build()
var kyloClientReader = kyloClient.read.format("com.thinkbiganalytics.spark.file.metadata")
listBuffer += kyloClientReader.load("file:/opt/kylo_backup/read/test.csv")
val dataFrameList = listBuffer.toList
var unionedFileMetadataDf : org.apache.spark.sql.DataFrame = null
dataFrameList.foreach({ df1 =>
if(unionedFileMetadataDf == null){
unionedFileMetadataDf = df1
}
else {
unionedFileMetadataDf = unionedFileMetadataDf.unionAll(df1)
}
})
var fileMetadataDf = unionedFileMetadataDf.select(col("mimeType"),col("encoding"),when(col("properties")("headerCount").isNotNull,col("properties")("headerCount")).otherwise(lit("0")).as("headerCount"),when(col("properties")("delimiter").isNotNull,col("properties")("delimiter")).otherwise(lit("")).as("delimiter"),when(col("properties")("rowTag").isNotNull,col("properties")("rowTag")).otherwise(lit("")).as("rowTag"),col("properties").as("properties"),col("resource"))
var df = fileMetadataDf
df
}
}
new Transform(sqlContext,sparkContextService).run()
}
def sparkContextService (): com.thinkbiganalytics.spark.SparkContextService = getValue("sparkContextService")
}
new Script(engine).run()
正在从 kylo wrangler 运行此程序
无法使用 Kylo Catalog 解析数据源:com.thinkbiganalytics.spark.file.metadata:{} java.lang.ClassNotFoundException: com.thinkbiganalytics.spark.file.metadata
有人知道这个 com.thinkbiganalytics.spark.file.metadata 类吗?
版权声明:本文内容由互联网用户自发贡献,该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务,不拥有所有权,不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容, 请发送邮件至 dio@foxmail.com 举报,一经查实,本站将立刻删除。