RANGER-4883: fix failure in audit-to-HDFS from Kafka and Knox plugins

dev-support/ranger-docker/docker-compose.ranger-kafka.yml

@@ -15,7 +15,8 @@ services:
     volumes:
       - ./dist/keytabs/ranger-kafka:/etc/keytabs
       - ./scripts/kdc/krb5.conf:/etc/krb5.conf:ro
-      - ./scripts/hadoop/core-site.xml:/home/ranger/scripts/core-site.xml
+      - ./scripts/hadoop/core-site.xml:/home/ranger/scripts/core-site.xml:ro
+      - ./scripts/hadoop/hdfs-client-site.xml:/home/ranger/scripts/hdfs-site.xml:ro
       - ./scripts/kafka/kafka-server-jaas.conf:/home/ranger/scripts/kafka-server-jaas.conf
       - ./dist/version:/home/ranger/dist/version:ro
       - ./scripts/kafka/ranger-kafka-plugin-install.properties:/opt/ranger/ranger-kafka-plugin/install.properties

dev-support/ranger-docker/docker-compose.ranger-knox.yml

@@ -15,6 +15,8 @@ services:
     volumes:
       - ./dist/keytabs/ranger-knox:/etc/keytabs
       - ./scripts/kdc/krb5.conf:/etc/krb5.conf:ro
+      - ./scripts/hadoop/core-site.xml:/home/ranger/scripts/core-site.xml:ro
+      - ./scripts/hadoop/hdfs-client-site.xml:/home/ranger/scripts/hdfs-site.xml:ro
       - ./dist/version:/home/ranger/dist/version:ro
       - ./scripts/knox/ranger-knox-sandbox.xml:/opt/knox/conf/topologies/sandbox.xml
       - ./scripts/knox/ranger-knox-plugin-install.properties:/opt/ranger/ranger-knox-plugin/install.properties

dev-support/ranger-docker/scripts/hadoop/hdfs-client-site.xml

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<configuration>
+  <property>
+    <name>dfs.namenode.kerberos.principal</name>
+    <value>nn/ranger-hadoop.rangernw@EXAMPLE.COM</value>
+  </property>
+  <property>
+    <name>dfs.datanode.kerberos.principal</name>
+    <value>dn/ranger-hadoop.rangernw@EXAMPLE.COM</value>
+  </property>
+</configuration>

dev-support/ranger-docker/scripts/kafka/ranger-kafka-setup.sh

@@ -29,6 +29,7 @@ then
 fi
 
 cp ${RANGER_SCRIPTS}/core-site.xml          ${KAFKA_HOME}/config/
+cp ${RANGER_SCRIPTS}/hdfs-site.xml          ${KAFKA_HOME}/config/
 cp ${RANGER_SCRIPTS}/kafka-server-jaas.conf ${KAFKA_HOME}/config/
 
 chown -R kafka:hadoop /opt/kafka/

dev-support/ranger-docker/scripts/knox/ranger-knox-plugin-install.properties

@@ -17,6 +17,12 @@ POLICY_MGR_URL=http://ranger:6080
 REPOSITORY_NAME=dev_knox
 COMPONENT_INSTALL_DIR_NAME=/opt/knox
 
+UGI_INITIALIZE=true
+UGI_LOGIN_TYPE=keytab
+UGI_KEYTAB_PRINCIPAL=knox/ranger-knox.rangernw@EXAMPLE.COM
+UGI_KEYTAB_FILE=/etc/keytabs/knox.keytab
+UGI_JAAS_APPCONFIG=unknown
+
 CUSTOM_USER=knox
 CUSTOM_GROUP=knox
 

dev-support/ranger-docker/scripts/knox/ranger-knox-sandbox.xml

@@ -64,7 +64,7 @@
 
         <provider>
             <role>authorization</role>
-            <name>AclsAuthz</name>
+            <name>XASecurePDPKnox</name>
             <enabled>true</enabled>
         </provider>
 

dev-support/ranger-docker/scripts/knox/ranger-knox-setup.sh

@@ -28,6 +28,9 @@ then
   ${RANGER_SCRIPTS}/wait_for_testusers_keytab.sh
 fi
 
+cp ${RANGER_SCRIPTS}/core-site.xml /opt/knox/conf/
+cp ${RANGER_SCRIPTS}/hdfs-site.xml /opt/knox/conf/
+
 chown -R knox:knox /opt/knox/
 
 mkdir -p /opt/knox/logs

distro/src/main/assembly/knox-agent.xml

@@ -69,6 +69,7 @@
           <include>com.google.protobuf:protobuf-java:jar:${protobuf-java.version}</include>
           <include>org.apache.hadoop:hadoop-client-api:jar:${hadoop.version}</include>
           <include>org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version}</include>
+          <include>org.apache.hadoop:hadoop-hdfs-client:jar:${hadoop.version}</include>
           <include>com.fasterxml.jackson.core:jackson-annotations:jar:${fasterxml.jackson.version}</include>
           <include>com.fasterxml.jackson.core:jackson-core:jar:${fasterxml.jackson.version}</include>
           <include>com.fasterxml.jackson.core:jackson-databind:jar:${fasterxml.jackson.version}</include>

distro/src/main/assembly/plugin-kafka.xml

@@ -59,6 +59,7 @@
 					<include>org.apache.hadoop:hadoop-auth:jar:${hadoop.version}</include>
 					<include>org.apache.hadoop:hadoop-client-api:jar:${hadoop.version}</include>
 					<include>org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version}</include>
+					<include>org.apache.hadoop:hadoop-hdfs-client:jar:${hadoop.version}</include>
 					<include>com.google.code.gson:gson</include>
 					<include>org.eclipse.jetty:jetty-client:jar:${jetty-client.version}</include>
 					<include>commons-collections:commons-collections</include>

knox-agent/conf/ranger-knox-security-changes.cfg

@@ -23,4 +23,9 @@ ranger.plugin.knox.policy.rest.ssl.config.file %COMPONENT_INSTALL_DIR_NAME%/conf
 ranger.plugin.knox.policy.pollIntervalMs       30000                                                       mod create-if-not-exists
 ranger.plugin.knox.policy.cache.dir            %POLICY_CACHE_FILE_PATH%                                    mod create-if-not-exists
 ranger.plugin.knox.policy.rest.client.connection.timeoutMs 120000								   		   mod create-if-not-exists
-ranger.plugin.knox.policy.rest.client.read.timeoutMs	   30000								  		   mod create-if-not-exists
+ranger.plugin.knox.policy.rest.client.read.timeoutMs	   30000								  		   mod create-if-not-exists
+ranger.plugin.knox.ugi.initialize       %UGI_INITIALIZE%       mod create-if-not-exists
+ranger.plugin.knox.ugi.login.type       %UGI_LOGIN_TYPE%       mod create-if-not-exists
+ranger.plugin.knox.ugi.keytab.principal %UGI_KEYTAB_PRINCIPAL% mod create-if-not-exists
+ranger.plugin.knox.ugi.keytab.file      %UGI_KEYTAB_FILE%      mod create-if-not-exists
+ranger.plugin.knox.ugi.jaas.appconfig   %UGI_JAAS_APPCONFIG%   mod create-if-not-exists