hadoop 数据仓库大数据权限【大数据】Ranger-2.3.0 Hive插件安装

coding技术猿数据库 2024-05-16 4 0

组件规划

clusterhostnameip机型组件abc-clusterspark-3110.253.128.31armRanger-adminspark-3210.253.128.32mysqlspark-3310.253.128.33hivemetastore/ hiveserver2

HIVE 插件安装

解压

tar -xvzf ranger-2.3.0-hdfs-plugin.tar.gz

修改配置

cd ranger-2.3.0-hive-plugin/

vim install.properties

POLICY_MGR_URL=http://10.253.128.31:6080

# This is the repository name created within policy manager

# Example:

# REPOSITORY_NAME=hivedev

REPOSITORY_NAME=hive_repo

# Hive installation directory

# Example:

# COMPONENT_INSTALL_DIR_NAME=/var/local/apache-hive-2.1.0-bin

COMPONENT_INSTALL_DIR_NAME=/data/apps/hive-3.1.2

# AUDIT configuration with V3 properties

# Enable audit logs to Solr

#Example

#XAAUDIT.SOLR.ENABLE=true

#XAAUDIT.SOLR.URL=http://localhost:6083/solr/ranger_audits

#XAAUDIT.SOLR.ZOOKEEPER=

#XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/hive/audit/solr/spool

XAAUDIT.SOLR.ENABLE=false

XAAUDIT.SOLR.URL=NONE

XAAUDIT.SOLR.USER=NONE

XAAUDIT.SOLR.PASSWORD=NONE

XAAUDIT.SOLR.ZOOKEEPER=NONE

XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/hive/audit/solr/spool

# Enable audit logs to ElasticSearch

#Example

#XAAUDIT.ELASTICSEARCH.ENABLE=true

#XAAUDIT.ELASTICSEARCH.URL=localhost

#XAAUDIT.ELASTICSEARCH.INDEX=audit

XAAUDIT.ELASTICSEARCH.ENABLE=false

XAAUDIT.ELASTICSEARCH.URL=NONE

XAAUDIT.ELASTICSEARCH.USER=NONE

XAAUDIT.ELASTICSEARCH.PASSWORD=NONE

XAAUDIT.ELASTICSEARCH.INDEX=NONE

XAAUDIT.ELASTICSEARCH.PORT=NONE

XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE

# Enable audit logs to HDFS

#Example

#XAAUDIT.HDFS.ENABLE=true

#XAAUDIT.HDFS.HDFS_DIR=hdfs://node-1.example.com:8020/ranger/audit

# If using Azure Blob Storage

#XAAUDIT.HDFS.HDFS_DIR=wasb[s]://@.blob.core.windows.net/

#XAAUDIT.HDFS.HDFS_DIR=wasb://ranger_audit_container@my-azure-account.blob.core.windows.net/ranger/audit

#XAAUDIT.HDFS.FILE_SPOOL_DIR=/var/log/hive/audit/hdfs/spool

XAAUDIT.HDFS.ENABLE=false

XAAUDIT.HDFS.HDFS_DIR=hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit

XAAUDIT.HDFS.FILE_SPOOL_DIR=/var/log/hive/audit/hdfs/spool

# Following additional propertis are needed When auditing to Azure Blob Storage via HDFS

# Get these values from your /etc/hadoop/conf/core-site.xml

#XAAUDIT.HDFS.HDFS_DIR=wasb[s]://@.blob.core.windows.net/

XAAUDIT.HDFS.AZURE_ACCOUNTNAME=__REPLACE_AZURE_ACCOUNT_NAME

XAAUDIT.HDFS.AZURE_ACCOUNTKEY=__REPLACE_AZURE_ACCOUNT_KEY

XAAUDIT.HDFS.AZURE_SHELL_KEY_PROVIDER=__REPLACE_AZURE_SHELL_KEY_PROVIDER

XAAUDIT.HDFS.AZURE_ACCOUNTKEY_PROVIDER=__REPLACE_AZURE_ACCOUNT_KEY_PROVIDER

#Log4j Audit Provider

XAAUDIT.LOG4J.ENABLE=false

XAAUDIT.LOG4J.IS_ASYNC=false

XAAUDIT.LOG4J.ASYNC.MAX.QUEUE.SIZE=10240

XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS=30000

XAAUDIT.LOG4J.DESTINATION.LOG4J=true

XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER=xaaudit

# Enable audit logs to Amazon CloudWatch Logs

#Example

#XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=true

#XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=ranger_audits

#XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM={instance_id}

#XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=/var/log/hive/audit/amazon_cloudwatch/spool

XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false

XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE

XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE

XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE

XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE

# End of V3 properties

# Audit to HDFS Configuration

# If XAAUDIT.HDFS.IS_ENABLED is set to true, please replace tokens

# that start with __REPLACE__ with appropriate values

# XAAUDIT.HDFS.IS_ENABLED=true

# XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd%

# XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=__REPLACE__LOG_DIR/hive/audit/%app-type%

# XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=__REPLACE__LOG_DIR/hive/audit/archive/%app-type%

# Example:

# XAAUDIT.HDFS.IS_ENABLED=true

# XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://namenode.example.com:8020/ranger/audit/%app-type%/%time:yyyyMMdd%

# XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/var/log/hive/audit/%app-type%

# XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/var/log/hive/audit/archive/%app-type%

XAAUDIT.HDFS.IS_ENABLED=false

XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd%

XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=__REPLACE__LOG_DIR/hive/audit/%app-type%

XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=__REPLACE__LOG_DIR/hive/audit/archive/%app-type%

XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log

XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900

XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400

XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60

XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log

XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60

XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600

XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10

#Solr Audit Provider

XAAUDIT.SOLR.IS_ENABLED=false

XAAUDIT.SOLR.MAX_QUEUE_SIZE=1

XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000

XAAUDIT.SOLR.SOLR_URL=http://localhost:6083/solr/ranger_audits

# End of V2 properties

# SSL Client Certificate Information

# Example:

# SSL_KEYSTORE_FILE_PATH=/etc/hive/conf/ranger-plugin-keystore.jks

# SSL_KEYSTORE_PASSWORD=none

# SSL_TRUSTSTORE_FILE_PATH=/etc/hive/conf/ranger-plugin-truststore.jks

# SSL_TRUSTSTORE_PASSWORD=none

# You do not need use SSL between agent and security admin tool, please leave these sample value as it is.

SSL_KEYSTORE_FILE_PATH=/etc/hive/conf/ranger-plugin-keystore.jks

SSL_KEYSTORE_PASSWORD=myKeyFilePassword

SSL_TRUSTSTORE_FILE_PATH=/etc/hive/conf/ranger-plugin-truststore.jks

SSL_TRUSTSTORE_PASSWORD=changeit

# Should Hive GRANT/REVOKE update XA policies?

# Example:

# UPDATE_XAPOLICIES_ON_GRANT_REVOKE=true

# UPDATE_XAPOLICIES_ON_GRANT_REVOKE=false

UPDATE_XAPOLICIES_ON_GRANT_REVOKE=true

# Custom component user

# CUSTOM_COMPONENT_USER=

# keep blank if component user is default

CUSTOM_USER=root

# Custom component group

# CUSTOM_COMPONENT_GROUP=

# keep blank if component group is default

CUSTOM_GROUP=root

因为 hive 机器不在当前 ranger 机器上，所以需要手动创建 COMPONENT_INSTALL_DIR_NAME=/data/apps/hive-3.1.2 指定的目录

mkdir -p /data/apps/hive-3.1.2/conf

安装插件

./enable-hive-plugin.sh

重启Ranger服务

ranger-admin restart

复制 enable-hive-plugin.sh 生成的hive的jar包和配置

scp -r /data/apps/ranger-2.3.0-hive-plugin/lib/ranger-hive-plugin-impl spark-33:/data/apps/hive-3.1.3/lib/

scp -r /data/apps/ranger-2.3.0-hive-plugin/lib/ranger-hive-plugin-shim-2.3.0.jar spark-33:/data/apps/hive-3.1.3/lib/

scp -r /data/apps/ranger-2.3.0-hive-plugin/lib/ranger-plugin-classloader-2.3.0.jar spark-33:/data/apps/hive-3.1.3/lib/

scp -r /data/apps/ranger-2.3.0-hive-plugin/conf/* spark-33:/data/apps/hive-3.1.3/conf/

重启 hive 组件

ssh spark-33

bin/hive --service metastore &

bin/hive --service hiveserver2 &

在Ranger管理界面配置hive策略

Service Name: hive_repo jdbc.driverClassName: org.apache.hive.jdbc.HiveDriver jdbc.url: jdbc:hive2://10.253.128.33:10000/default

hiveserver2拉取的策略缓存目录

cat /etc/ranger/hive_repo/policycache/

至此，hive-ranger插件安装完成

希望对正在查看文章的您有所帮助，记得关注、评论、收藏，谢谢您

参考链接

评论可见，请评论后查看内容，谢谢！！！

您阅读本篇文章共花了：

Hive hadoop 数据仓库 ranger 大数据权限

本文由用户于 2024-05-16 发布在金钥匙，如有疑问，请联系我们。
本文链接：https://www.51969.com/post/18879813.html

金钥匙

hadoop 数据仓库大数据权限【大数据】Ranger-2.3.0 Hive插件安装

hadoop 数据仓库 Hive UDF 函数

数据库数据仓库常用的数据集成ETL工具有哪些？

发表评论取消回复

金钥匙

hadoop 数据仓库 大数据权限 【大数据】Ranger-2.3.0 Hive插件安装

hadoop 数据仓库 Hive UDF 函数

数据库 数据仓库 常用的数据集成ETL工具有哪些？

相关文章

发表评论取消回复

hadoop 数据仓库大数据权限【大数据】Ranger-2.3.0 Hive插件安装

数据库数据仓库常用的数据集成ETL工具有哪些？