如何解决连接到蜂巢时出现德鲁伊异常
我对Dremio非常陌生,我正在尝试将Dremio(独立版本)与现有的Hive 2.3.3(在Hadoop 2.7之上)设置集成。
奇怪的是,虽然我可以访问几个表,但由于遇到以下错误,所以无法访问其他表:
master:8020 failed on connection exception: java.net.ConnectException
这是我的设置:
我已将以下hive-site.xml和core-site.xml文件复制到dremio root / conf文件夹中。
hive-site.xml
<xmp>
<configuration>
<property>
<name>hive.execution.engine</name>
<value>spark</value>
</property>
<property>
<name>javax.jdo.option.ConnectionURL</name>
<value>jdbc:mysql://data5:3306/hive_metastore?createDatabaseIfNotExist=true&characterEncoding=UTF-8&useSSL=false</value>
</property>
<property>
<name>javax.jdo.option.ConnectionDriverName</name>
<value>com.mysql.jdbc.Driver</value>
</property>
<property>
<name>javax.jdo.option.ConnectionUserName</name>
<value>someuser</value>
<description>user name for connecting to mysql server</description>
</property>
<property>
<name>javax.jdo.option.ConnectionPassword</name>
<value>testpassword</value>
<description>password for connecting to mysql server</description>
</property>
<property>
<name>spark.master</name>
<value>yarn</value>
</property>
<property>
<name>spark.submit.deployMode</name>
<value>cluster</value>
</property>
<property>
<name>spark.home</name>
<value>/usr/local/spark-2.4.4-bin-without-hadoop</value>
</property>
<property>
<name>spark.executor.memory</name>
<value>2g</value>
</property>
<property>
<name>spark.executor.memoryOverhead</name>
<value>512M</value>
</property>
<property>
<name>spark.yarn.executor.memoryOverhead</name>
<value>512M</value>
</property>
<property>
<name>spark.executor.cores</name>
<value>1</value>
</property>
<property>
<name>spark.executor.instances</name>
<value>3</value>
</property>
<property>
<name>spark.debug.maxToStringFields</name>
<value>200</value>
</property>
<!-- <property> <name>spark.dynamicAllocation.enabled</name> <value>true</value> </property> -->
<property>
<name>spark.driver.memory</name>
<value>1g</value>
</property>
<property>
<name>spark.yarn.driver.memoryOverhead</name>
<value>256M</value>
</property>
<property>
<name>spark.yarn.jars</name>
<value>hdfs://master:9000/user/spark/jars/*</value>
</property>
<property>
<name>hive.exec.reducers.bytes.per.reducer</name>
<value>256000000</value>
</property>
<property>
<name>hive.exec.reducers.max</name>
<value>100</value>
</property>
<property>
<name>mapreduce.job.reducers</name>
<value>2</value>
</property>
<property>
<name>hive.spark.client.connect.timeout</name>
<value>30000ms</value>
</property>
<property>
<name>hive.spark.client.server.connect.timeout</name>
<value>30000ms</value>
</property>
<property>
<name>spark.yarn.dist.files</name>
<value>/usr/local/apache-hive-2.3.3-bin/conf/hive-site.xml</value>
</property>
<!-- thrift setup -->
<property>
<name>hive.server2.transport.mode</name>
<value>http</value>
</property>
<property>
<name>hive.server2.thrift.http.port</name>
<value>10000</value>
</property>
<property>
<name>hive.server2.thrift.http.max.worker.threads</name>
<value>500</value>
</property>
<property>
<name>hive.server2.thrift.http.min.worker.threads</name>
<value>10</value>
</property>
<property>
<name>hive.server2.thrift.http.path</name>
<value>cliservice</value>
</property>
<property>
<name>hive.server2.enable.doAs</name>
<value>false</value>
</property>
<property>
<name>hive.metastore.uris</name>
<value>thrift://data6:9083</value>
</property>
<property>
<name>hive.exec.dynamic.partition.mode</name>
<value>nonstrict</value>
</property>
<!-- Transaction properties. Disabled for now. <property> <name>hive.support.concurrency</name> <value>true</value> </property> <property> <name>hive.enforce.bucketing</name> <value>true</value> </property> <property> <name>hive.txn.manager</name>
<value>org.apache.hadoop.hive.ql.lockmgr.DbTxnManager</value> </property> <property> <name>hive.compactor.initiator.on</name> <value>true</value> </property> -->
<property>
<name>hive.server2.authentication</name>
<value>NONE</value>
</property>
<property>
<name>hive.metastore.event.db.notification.api.auth</name>
<value>false</value>
</property>
<property>
<name>hive.server2.active.passive.ha.enable</name>
<value>true</value>
</property>
<property>
<name>hive.server2.limit.connections.per.user</name>
<value>50</value>
</property>
<property>
<name>hive.spark.client.rpc.max.size</name>
<value>1262485504</value>
</property>
<property>
<name>hive.server2.thrift.min.worker.threads</name>
<value>5</value>
</property>
<property>
<name>hive.server2.thrift.max.worker.threads</name>
<value>500</value>
</property>
<property>
<name>hive.mapjoin.hybridgrace.hashtable</name>
<value>false</value>
</property>
<property>
<name>hive.spark.job.monitor.timeout</name>
<value>60000</value>
</property>
<property>
<name>hive.spark.client.server.connect.timeout</name>
<value>60000</value>
</property>
<property>
<name>auto.convert.join</name>
<value>false</value>
</property>
<property>
<name>hive.vectorized.execution</name>
<value>false</value>
</property>
<!-- <property> <name>hive.vectorized.execution.enabled</name> <value>true</value> </property> -->
<property>
<name>hive.cbo.enable</name>
<value>true</value>
</property>
<property>
<name>hive.exec.max.dynamic.partitions.pernode</name>
<value>20000</value>
</property>
<property>
<name>hive.exec.max.dynamic.partitions</name>
<value>20000</value>
</property>
<property>
<name>hive.exec.dynamic.partition</name>
<value>true</value>
</property>
<property>
<name>hive.exec.dynamic.partition.mode</name>
<value>nonstrict</value>
</property>
<property>
<name>hive.enforce.bucketing</name>
<value>false</value>
</property>
<property>
<name>hive.enforce.sorting</name>
<value>false</value>
</property>
<property>
<name>optimize.sort.dynamic.partitionining</name>
<value>true</value>
</property>
<property>
<name>spark.sql.sources.bucketing.enabled</name>
<value>true</value>
</property>
<property>
<name>hive.mapjoin.optimized.hashtable</name>
<value>false</value>
</property>
</configuration>
</xmp>
core-site.xml
<xmp>
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<property>
<name>fs.default.name</name>
<value>hdfs://master:9000</value>
</property>
<property>
<name>fs.defaultFS</name>
<value>hdfs://master:9000</value>
</property>
<property>
<name>dfs.permissions</name>
<value>false</value>
</property>
<property>
<name>hadoop.proxyuser.hadoopuser.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.hadoopuser.hosts</name>
<value>*</value>
</property>
<property>
<name>dfs.webhdfs.enabled</name>
<value>true</value>
</property>
<property>
<name>hadoop.proxyuser.hue.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.hue.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.dremio.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.dremio.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.dremio.users</name>
<value>*</value>
</property>
</configuration>
</xmp>
关于什么可能导致该错误的任何建议?
版权声明:本文内容由互联网用户自发贡献,该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务,不拥有所有权,不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容, 请发送邮件至 dio@foxmail.com 举报,一经查实,本站将立刻删除。