diff --git a/dev-support/ranger-docker/Dockerfile.ranger-tagsync b/dev-support/ranger-docker/Dockerfile.ranger-tagsync index 9c28ee0c696..db50778d733 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-tagsync +++ b/dev-support/ranger-docker/Dockerfile.ranger-tagsync @@ -27,7 +27,7 @@ RUN tar xvfz /home/ranger/dist/ranger-${TAGSYNC_VERSION}-tagsync.tar.gz --direct ln -s ${RANGER_HOME}/ranger-${TAGSYNC_VERSION}-tagsync ${RANGER_HOME}/tagsync && \ rm -f /home/ranger/dist/ranger-${TAGSYNC_VERSION}-tagsync.tar.gz && \ rm -f ${RANGER_HOME}/tagsync/install.properties && \ - mkdir -p /opt/ranger/tagsync/data /var/run/ranger /var/log/ranger/tagsync /etc/ranger && \ + mkdir -p /opt/ranger/tagsync/data /var/run/ranger /var/log/ranger/tagsync /etc/ranger/tagsync/conf && \ cp -f ${RANGER_SCRIPTS}/ranger-tagsync-install.properties ${RANGER_HOME}/tagsync/install.properties && \ mkdir /etc/init.d || true && \ mkdir /etc/rc2.d || true && \ diff --git a/dev-support/ranger-docker/docker-compose.ranger-tagsync.yml b/dev-support/ranger-docker/docker-compose.ranger-tagsync.yml index 05c6a4d06cf..f7070ca15bd 100644 --- a/dev-support/ranger-docker/docker-compose.ranger-tagsync.yml +++ b/dev-support/ranger-docker/docker-compose.ranger-tagsync.yml @@ -22,6 +22,9 @@ services: tty: true networks: - ranger + ports: + - "8180:8180" + - "8185:8185" depends_on: ranger: condition: service_started diff --git a/dev-support/ranger-docker/scripts/tagsync/ranger-tagsync-install.properties b/dev-support/ranger-docker/scripts/tagsync/ranger-tagsync-install.properties index c34d74ee7a9..83d2eb249f8 100755 --- a/dev-support/ranger-docker/scripts/tagsync/ranger-tagsync-install.properties +++ b/dev-support/ranger-docker/scripts/tagsync/ranger-tagsync-install.properties @@ -125,3 +125,16 @@ JVM_METRICS_FILEPATH= #frequency for jvm metrics to be updated # default value : 10000 milliseconds JVM_METRICS_FREQUENCY_TIME_IN_MILLIS= + +TAGSYNC_HTTP_PORT = 8180 +TAGSYNC_HTTPS_PORT = 8183 + +TAGSYNC_KEYSTORE_FILE_ALIAS = +TAGSYNC_KEYSTORE_CRED_ALIAS = sslkeystore +TAGSYNC_TRUSTSTORE_ALIAS = sslTrustStore +TAGSYNC_CLIENT_AUTH = want +TAGSYNC_SERVICE_HOST = ranger-tagsync.example.com +TAGSYNC_CRED_PROVIDER_PATH = +TAGSYNC_KEYSTORE_FILE_TYPE = +TAGSYNC_TRUSTSTORE_FILE_TYPE = +TAGSYNC_SHUTDOWN_PORT = 8185 diff --git a/dev-support/ranger-docker/scripts/tagsync/ranger-tagsync.sh b/dev-support/ranger-docker/scripts/tagsync/ranger-tagsync.sh index dcb01cb541f..b98848ce81a 100755 --- a/dev-support/ranger-docker/scripts/tagsync/ranger-tagsync.sh +++ b/dev-support/ranger-docker/scripts/tagsync/ranger-tagsync.sh @@ -47,7 +47,7 @@ fi cd ${RANGER_HOME}/tagsync && ./ranger-tagsync-services.sh start -RANGER_TAGSYNC_PID=`ps -ef | grep -v grep | grep -i "org.apache.ranger.tagsync.process.TagSynchronizer" | awk '{print $2}'` +RANGER_TAGSYNC_PID=`ps -ef | grep -v grep | grep -i "org.apache.ranger.tagsync.server.RangerTagSyncServer" | awk '{print $2}'` # prevent the container from exiting if [ -z "$RANGER_TAGSYNC_PID" ] diff --git a/distro/src/main/assembly/tagsync.xml b/distro/src/main/assembly/tagsync.xml index f919f18d125..5c44c015e24 100644 --- a/distro/src/main/assembly/tagsync.xml +++ b/distro/src/main/assembly/tagsync.xml @@ -37,7 +37,7 @@ 644 - lib + ews/lib false ch.qos.logback:logback-classic:jar:${logback.version} @@ -91,6 +91,8 @@ org.apache.ranger:ranger-plugins-common org.apache.ranger:ranger-util org.apache.ranger:ugsync-util + org.apache.tomcat.embed:tomcat-embed* + org.apache.tomcat:tomcat-annotations-api* org.apache.zookeeper:zookeeper-jute:jar:${zookeeper.version} org.apache.zookeeper:zookeeper:jar:${zookeeper.version} org.codehaus.woodstox:stax2-api:jar:${codehaus.woodstox.stax2api.version} @@ -102,11 +104,23 @@ org.glassfish.hk2:hk2-runlevel org.glassfish.hk2:hk2-utils org.glassfish.hk2:hk2 + org.glassfish.jersey.containers:jersey-container-servlet + org.glassfish.jersey.containers:jersey-container-servlet-core org.glassfish.jersey.core:jersey-client org.glassfish.jersey.core:jersey-common + org.glassfish.jersey.core:jersey-server org.glassfish.jersey.ext:jersey-entity-filtering org.glassfish.jersey.inject:jersey-hk2 org.glassfish.jersey.media:jersey-media-json-jackson + org.springframework.security:spring-security-config + org.springframework.security:spring-security-web + org.springframework:spring-aop + org.springframework:spring-beans + org.springframework:spring-context + org.springframework:spring-context-support + org.springframework:spring-core + org.springframework:spring-expression + org.springframework:spring-web org.scala-lang:scala-library:jar:${scala.version} org.slf4j:log4j-over-slf4j:jar:${slf4j.version} org.slf4j:slf4j-api:jar:${slf4j.version} @@ -115,6 +129,17 @@ + + true + + org.apache.ranger:ranger-tagsync + + + ews/webapp + false + true + + diff --git a/tagsync/conf/templates/installprop2xml.properties b/tagsync/conf/templates/installprop2xml.properties index aea6537ee15..29fe1a4591a 100644 --- a/tagsync/conf/templates/installprop2xml.properties +++ b/tagsync/conf/templates/installprop2xml.properties @@ -13,60 +13,51 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - - -TAGSYNC_ENABLED = ranger.tagsync.enabled - -TAG_DEST_RANGER_ENDPOINT = ranger.tagsync.dest.ranger.endpoint -TAG_DEST_RANGER_SSL_CONFIG_FILENAME = ranger.tagsync.dest.ranger.ssl.config.filename - -TAG_SOURCE_ATLAS_ENABLED = ranger.tagsync.source.atlas - -TAG_SOURCE_ATLAS_KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR = atlas.kafka.offsets.topic.replication.factor -TAG_SOURCE_ATLAS_KAFKA_BOOTSTRAP_SERVERS = atlas.kafka.bootstrap.servers -TAG_SOURCE_ATLAS_KAFKA_ENTITIES_GROUP_ID = atlas.kafka.entities.group.id - -TAG_SOURCE_ATLAS_KAFKA_SERVICE_NAME = atlas.kafka.sasl.kerberos.service.name -TAG_SOURCE_ATLAS_KAFKA_SECURITY_PROTOCOL = atlas.kafka.security.protocol - -TAG_SOURCE_ATLAS_KERBEROS_PRINCIPAL = atlas.jaas.kafkaClient.option.principal -TAG_SOURCE_ATLAS_KERBEROS_KEYTAB = atlas.jaas.kafkaClient.option.keyTab - -TAG_SOURCE_ATLASREST_ENABLED = ranger.tagsync.source.atlasrest - -TAG_SOURCE_ATLASREST_ENDPOINT = ranger.tagsync.source.atlasrest.endpoint -TAG_SOURCE_ATLASREST_DOWNLOAD_INTERVAL_IN_MILLIS = ranger.tagsync.source.atlasrest.download.interval.millis - -TAG_SOURCE_ATLASREST_USERNAME = ranger.tagsync.source.atlasrest.username -TAG_SOURCE_ATLASREST_PASSWORD = ranger.tagsync.source.atlasrest.password - -TAG_SOURCE_FILE_ENABLED = ranger.tagsync.source.file - -TAG_SOURCE_FILE_FILENAME = ranger.tagsync.source.file.filename -TAG_SOURCE_FILE_CHECK_INTERVAL_IN_MILLIS = ranger.tagsync.source.file.check.interval.millis - -TAGSYNC_ATLAS_TO_RANGER_SERVICE_MAPPING = ranger.tagsync.atlas.to.ranger.service.mapping -TAGSYNC_ATLAS_CUSTOM_RESOURCE_MAPPERS = ranger.tagsync.atlas.custom.resource.mappers - -TAGSYNC_KEYSTORE_FILENAME = ranger.tagsync.keystore.filename -TAG_SOURCE_ATLASREST_KEYSTORE_FILENAME = ranger.tagsync.source.atlasrest.keystore.filename -TAG_SOURCE_ATLASREST_SSL_CONFIG_FILENAME = ranger.tagsync.source.atlasrest.ssl.config.filename - - -unix_user = unix_user -unix_group = unix_group - -logdir = ranger.tagsync.logdir - -tagsync_principal = ranger.tagsync.kerberos.principal -tagsync_keytab = ranger.tagsync.kerberos.keytab - - +TAGSYNC_ENABLED=ranger.tagsync.enabled +TAG_DEST_RANGER_ENDPOINT=ranger.tagsync.dest.ranger.endpoint +TAG_DEST_RANGER_SSL_CONFIG_FILENAME=ranger.tagsync.dest.ranger.ssl.config.filename +TAG_SOURCE_ATLAS_ENABLED=ranger.tagsync.source.atlas +TAG_SOURCE_ATLAS_KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=atlas.kafka.offsets.topic.replication.factor +TAG_SOURCE_ATLAS_KAFKA_BOOTSTRAP_SERVERS=atlas.kafka.bootstrap.servers +TAG_SOURCE_ATLAS_KAFKA_ENTITIES_GROUP_ID=atlas.kafka.entities.group.id +TAG_SOURCE_ATLAS_KAFKA_SERVICE_NAME=atlas.kafka.sasl.kerberos.service.name +TAG_SOURCE_ATLAS_KAFKA_SECURITY_PROTOCOL=atlas.kafka.security.protocol +TAG_SOURCE_ATLAS_KERBEROS_PRINCIPAL=atlas.jaas.kafkaClient.option.principal +TAG_SOURCE_ATLAS_KERBEROS_KEYTAB=atlas.jaas.kafkaClient.option.keyTab +TAG_SOURCE_ATLASREST_ENABLED=ranger.tagsync.source.atlasrest +TAG_SOURCE_ATLASREST_ENDPOINT=ranger.tagsync.source.atlasrest.endpoint +TAG_SOURCE_ATLASREST_DOWNLOAD_INTERVAL_IN_MILLIS=ranger.tagsync.source.atlasrest.download.interval.millis +TAG_SOURCE_ATLASREST_USERNAME=ranger.tagsync.source.atlasrest.username +TAG_SOURCE_ATLASREST_PASSWORD=ranger.tagsync.source.atlasrest.password +TAG_SOURCE_FILE_ENABLED=ranger.tagsync.source.file +TAG_SOURCE_FILE_FILENAME=ranger.tagsync.source.file.filename +TAG_SOURCE_FILE_CHECK_INTERVAL_IN_MILLIS=ranger.tagsync.source.file.check.interval.millis +TAGSYNC_ATLAS_TO_RANGER_SERVICE_MAPPING=ranger.tagsync.atlas.to.ranger.service.mapping +TAGSYNC_ATLAS_CUSTOM_RESOURCE_MAPPERS=ranger.tagsync.atlas.custom.resource.mappers +TAGSYNC_KEYSTORE_FILENAME=ranger.tagsync.keystore.filename +TAG_SOURCE_ATLASREST_KEYSTORE_FILENAME=ranger.tagsync.source.atlasrest.keystore.filename +TAG_SOURCE_ATLASREST_SSL_CONFIG_FILENAME=ranger.tagsync.source.atlasrest.ssl.config.filename +unix_user=unix_user +unix_group=unix_group +logdir=ranger.tagsync.logdir +tagsync_principal=ranger.tagsync.kerberos.principal +tagsync_keytab=ranger.tagsync.kerberos.keytab # TODO - What property in ranger-tagsync-site.xml should hadoop_conf map to?? -hadoop_conf = hadoop_conf - +hadoop_conf=hadoop_conf #JVM metrics related property JVM_METRICS_ENABLED=ranger.tagsync.metrics.enabled JVM_METRICS_FILENAME=ranger.tagsync.metrics.filename JVM_METRICS_FILEPATH=ranger.tagsync.metrics.filepath JVM_METRICS_FREQUENCY_TIME_IN_MILLIS=ranger.tagsync.metrics.frequencytimeinmillis +# Ports +TAGSYNC_HTTP_PORT=ranger.tagsync.service.http.port +TAGSYNC_HTTPS_PORT=ranger.tagsync.service.https.port +TAGSYNC_KEYSTORE_FILE_ALIAS=ranger.tagsync.service.https.attrib.keystore.keyalias +TAGSYNC_KEYSTORE_CRED_ALIAS=ranger.tagsync.service.https.attrib.keystore.credential.alias +TAGSYNC_TRUSTSTORE_ALIAS=ranger.tagsync.truststore.alias +TAGSYNC_CLIENT_AUTH=ranger.tagsync.service.https.attrib.client.auth +TAGSYNC_SERVICE_HOST=ranger.tagsync.service.host +TAGSYNC_CRED_PROVIDER_PATH=ranger.tagsync.credential.provider.path +TAGSYNC_KEYSTORE_FILE_TYPE=ranger.tagsync.keystore.file.type +TAGSYNC_TRUSTSTORE_FILE_TYPE=ranger.tagsync.truststore.file.type +TAGSYNC_SHUTDOWN_PORT=ranger.tagsync.service.shutdown.port diff --git a/tagsync/pom.xml b/tagsync/pom.xml index 5d8ca7a39d5..7a45207ef10 100644 --- a/tagsync/pom.xml +++ b/tagsync/pom.xml @@ -322,6 +322,11 @@ ranger-common-ha ${project.version} + + org.apache.ranger + ranger-metrics + ${project.version} + org.apache.ranger ranger-plugins-common @@ -342,6 +347,31 @@ ranger-util ${project.version} + + org.apache.tomcat + tomcat-annotations-api + ${tomcat.embed.version} + + + org.apache.tomcat.embed + tomcat-embed-core + ${tomcat.embed.version} + + + org.apache.tomcat.embed + tomcat-embed-el + ${tomcat.embed.version} + + + org.apache.tomcat.embed + tomcat-embed-jasper + ${tomcat.embed.version} + + + org.apache.tomcat.embed + tomcat-embed-websocket + ${tomcat.embed.version} + org.apache.zookeeper zookeeper @@ -426,6 +456,54 @@ log4j-over-slf4j ${slf4j.version} + + org.springframework + spring-beans + ${springframework.version} + + + org.springframework + spring-jcl + + + + + org.springframework + spring-context + ${springframework.version} + + + org.springframework + spring-context-support + ${springframework.version} + + + org.springframework + spring-web + ${springframework.version} + + + org.springframework.security + spring-security-config + ${springframework.security.version} + + + org.springframework + * + + + + + org.springframework.security + spring-security-web + ${springframework.security.version} + + + org.springframework + * + + + org.glassfish.hk2 class-model @@ -475,6 +553,12 @@ + + + true + src/main/resources + + org.apache.maven.plugins @@ -487,6 +571,46 @@ + + org.apache.maven.plugins + maven-war-plugin + ${maven-war-plugin.version} + + + prepare + + exploded + + prepare-package + + WEB-INF/classes/ranger-tagsync-site.xml, + WEB-INF/lib/jersey-client-1.*.jar, + WEB-INF/lib/jersey-core-1.*.jar, + WEB-INF/lib/jersey-json-1.*.jar, + WEB-INF/lib/jersey-multipart-1.*.jar, + WEB-INF/lib/jersey-server-1.*.jar, + WEB-INF/lib/jersey-servlet-1.*.jar + + + + default-war + + war + + package + + WEB-INF/classes/ranger-tagsync-site.xml, + WEB-INF/lib/jersey-client-1.*.jar, + WEB-INF/lib/jersey-core-1.*.jar, + WEB-INF/lib/jersey-json-1.*.jar, + WEB-INF/lib/jersey-multipart-1.*.jar, + WEB-INF/lib/jersey-server-1.*.jar, + WEB-INF/lib/jersey-servlet-1.*.jar + ${project.build.directory}/${project.build.finalName} + + + + diff --git a/tagsync/scripts/install.properties b/tagsync/scripts/install.properties index 278df0ff80d..51711241726 100644 --- a/tagsync/scripts/install.properties +++ b/tagsync/scripts/install.properties @@ -20,36 +20,25 @@ # # # URL for tag destination - Ranger -TAG_DEST_RANGER_ENDPOINT = http://localhost:6080 - +TAG_DEST_RANGER_ENDPOINT=http://localhost:6080 # SSL config file name for HTTPS messages to tag destination - Ranger -TAG_DEST_RANGER_SSL_CONFIG_FILENAME = - -TAG_SOURCE_ATLAS_ENABLED = true - +TAG_DEST_RANGER_SSL_CONFIG_FILENAME= +TAG_SOURCE_ATLAS_ENABLED=true # Kafka settings for Atlas notification consumer (bootstrap.servers only; no ZooKeeper) -TAG_SOURCE_ATLAS_KAFKA_BOOTSTRAP_SERVERS = localhost:6667 -TAG_SOURCE_ATLAS_KAFKA_ENTITIES_GROUP_ID = ranger_entities_consumer - -TAG_SOURCE_ATLAS_KAFKA_SERVICE_NAME = kafka -TAG_SOURCE_ATLAS_KAFKA_SECURITY_PROTOCOL = PLAINTEXTSASL - -TAG_SOURCE_ATLAS_KERBEROS_PRINCIPAL = -TAG_SOURCE_ATLAS_KERBEROS_KEYTAB = - -TAG_SOURCE_ATLASREST_ENABLED = false - -TAG_SOURCE_ATLASREST_ENDPOINT = http://localhost:21000 -TAG_SOURCE_ATLASREST_DOWNLOAD_INTERVAL_IN_MILLIS = 900000 - -TAG_SOURCE_ATLASREST_USERNAME = -TAG_SOURCE_ATLASREST_PASSWORD = - -TAG_SOURCE_FILE_ENABLED = false - -TAG_SOURCE_FILE_FILENAME = /etc/ranger/data/tags.json -TAG_SOURCE_FILE_CHECK_INTERVAL_IN_MILLIS = 60000 - +TAG_SOURCE_ATLAS_KAFKA_BOOTSTRAP_SERVERS=localhost:6667 +TAG_SOURCE_ATLAS_KAFKA_ENTITIES_GROUP_ID=ranger_entities_consumer +TAG_SOURCE_ATLAS_KAFKA_SERVICE_NAME=kafka +TAG_SOURCE_ATLAS_KAFKA_SECURITY_PROTOCOL=PLAINTEXTSASL +TAG_SOURCE_ATLAS_KERBEROS_PRINCIPAL= +TAG_SOURCE_ATLAS_KERBEROS_KEYTAB= +TAG_SOURCE_ATLASREST_ENABLED=false +TAG_SOURCE_ATLASREST_ENDPOINT=http://localhost:21000 +TAG_SOURCE_ATLASREST_DOWNLOAD_INTERVAL_IN_MILLIS=900000 +TAG_SOURCE_ATLASREST_USERNAME= +TAG_SOURCE_ATLASREST_PASSWORD= +TAG_SOURCE_FILE_ENABLED=false +TAG_SOURCE_FILE_FILENAME=/etc/ranger/data/tags.json +TAG_SOURCE_FILE_CHECK_INTERVAL_IN_MILLIS=60000 # Mapping from Atlas hive cluster-name to Ranger service-name # this needs to be in format clusterName,componentType,serviceName;clusterName2,componentType2,serviceName2 # Note that there are no blanks anywhere in the value-string @@ -66,64 +55,45 @@ TAG_SOURCE_FILE_CHECK_INTERVAL_IN_MILLIS = 60000 # # TAGSYNC_ATLAS_TO_RANGER_SERVICE_MAPPING = clusterName,hive,rangerServiceName # - TAGSYNC_ATLAS_TO_RANGER_SERVICE_MAPPING= - # A comma separated list of custom mapper class names which convert Atlas entities to # RangerServiceResource structures are specified here. If there are no custom mappers, # then it can be left blank - TAGSYNC_ATLAS_CUSTOM_RESOURCE_MAPPERS= - # # The file where all credential is kept in cryptic format # -TAGSYNC_KEYSTORE_FILENAME = /etc/ranger/tagsync/conf/rangertagsync.jceks - +TAGSYNC_KEYSTORE_FILENAME=/etc/ranger/tagsync/conf/rangertagsync.jceks # File where Atlas credentials is kept in cryptic format - -TAG_SOURCE_ATLASREST_KEYSTORE_FILENAME = /etc/ranger/tagsync/conf/atlasuser.jceks - +TAG_SOURCE_ATLASREST_KEYSTORE_FILENAME=/etc/ranger/tagsync/conf/atlasuser.jceks # SSL config file name for HTTPS messages to tag source - Atlas-REST -TAG_SOURCE_ATLASREST_SSL_CONFIG_FILENAME = - +TAG_SOURCE_ATLASREST_SSL_CONFIG_FILENAME= #User and group for the tagsync process unix_user=ranger unix_group=ranger - #change password of rangerTagsync user. Please note that this password should be as per rangerTagsync user in ranger rangerTagsync_password= - # Logs are stored in logdir -logdir = log - +logdir=log #PID file path TAGSYNC_PID_DIR_PATH=/var/run/ranger - #Set to run in kerberos environment -is_secure = false +is_secure=false tagsync_principal= tagsync_keytab= - - - hadoop_conf=/etc/hadoop/conf - # if you want to enable or disable jvm metrics for tagsync process # valid values: true, false # any value other than true would be treated as false # default value: false # if the value is false, jvm metrics is not created JVM_METRICS_ENABLED= - # filename of jvm metrics created for tagsync process # default value: ranger_tagsync_metric.json JVM_METRICS_FILENAME= - #file directory for jvm metrics # default value : logdir JVM_METRICS_FILEPATH= - #frequency for jvm metrics to be updated # default value : 10000 milliseconds JVM_METRICS_FREQUENCY_TIME_IN_MILLIS= diff --git a/tagsync/scripts/ranger-tagsync-services.sh b/tagsync/scripts/ranger-tagsync-services.sh index ef75644eccd..ad67dbabb07 100755 --- a/tagsync/scripts/ranger-tagsync-services.sh +++ b/tagsync/scripts/ranger-tagsync-services.sh @@ -15,6 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +set -x if [[ -z $1 ]]; then echo "No argument provided.." echo "Usage: $0 {start | stop | restart | version}" @@ -77,7 +78,8 @@ if [ "${action}" == "START" ]; then chmod 777 $RANGER_TAGSYNC_LOG_DIR fi - cp="${cdir}/conf:${cdir}/dist/*:${cdir}/lib/*:${RANGER_TAGSYNC_HADOOP_CONF_DIR}/*" + RANGER_TAGSYNC_WEBAPP=${cdir}/ews/webapp + cp="${cdir}/conf:${RANGER_TAGSYNC_WEBAPP}/WEB-INF/classes:${RANGER_TAGSYNC_WEBAPP}/WEB-INF/lib/*:${cdir}/ews/lib/*:${RANGER_TAGSYNC_HADOOP_CONF_DIR}/*:${JAVA_HOME}/lib/*" if [ -f "$pidf" ] ; then pid=`cat $pidf` @@ -98,7 +100,7 @@ if [ "${action}" == "START" ]; then export TAGSYNC_CONF_DIR SLEEP_TIME_AFTER_START=5 - nohup java -Dproc_rangertagsync ${JAVA_OPTS} -Datlas.conf="${TAGSYNC_CONF_DIR}" -Dlogdir="${RANGER_TAGSYNC_LOG_DIR}" -Dlogback.configurationFile=file:${TAGSYNC_CONF_DIR}/logback.xml -Duser=${USER} -Dhostname=${HOSTNAME} -cp "${cp}" org.apache.ranger.tagsync.process.TagSynchronizer > ${RANGER_TAGSYNC_LOG_DIR}/tagsync.out 2>&1 & + nohup java -Dproc_rangertagsync ${JAVA_OPTS} -Dlogback.configurationFile=file:${TAGSYNC_CONF_DIR}/logback.xml -Dlogdir="${RANGER_TAGSYNC_LOG_DIR}" -Dranger.tagsync.log.dir="${RANGER_TAGSYNC_LOG_DIR}" -Duser="${TAGSYNC_LOG_USER}" -Dhostname="${HOSTNAME}" -Dservername=rangertagsync -Dranger.tagsync.home=`pwd` -Dranger.tagsync.webapp.dir="${RANGER_TAGSYNC_WEBAPP}" -Dcatalina.base=${cdir}/ews -cp "${cp}" org.apache.ranger.tagsync.server.RangerTagSyncServer > ${RANGER_TAGSYNC_LOG_DIR}/catalina.out 2>&1 & VALUE_OF_PID=$! echo "Starting Apache Ranger Tagsync Service" sleep $SLEEP_TIME_AFTER_START diff --git a/tagsync/scripts/setup.py b/tagsync/scripts/setup.py index c7ca4b3b0be..737cff54e13 100755 --- a/tagsync/scripts/setup.py +++ b/tagsync/scripts/setup.py @@ -208,7 +208,7 @@ def updatePropertyInJCKSFile(jcksFileName,propName,value): fn = jcksFileName if (value == ''): value = ' ' - cmd = "java -cp './lib/*' %s create '%s' -value '%s' -provider jceks://file%s 2>&1" % (credUpdateClassName,propName,value,fn) + cmd = "java -cp './ews/lib/*' %s create '%s' -value '%s' -provider jceks://file%s 2>&1" % (credUpdateClassName,propName,value,fn) ret = os.system(cmd) if (ret != 0): print("ERROR: Unable to update the JCKSFile (%s) for aliasName (%s)" % (fn,propName)) diff --git a/tagsync/scripts/updatetagadminpassword.py b/tagsync/scripts/updatetagadminpassword.py index ff0cce13f65..6163bab39c8 100644 --- a/tagsync/scripts/updatetagadminpassword.py +++ b/tagsync/scripts/updatetagadminpassword.py @@ -170,7 +170,7 @@ def main(argv): if KEYSTORE_FILENAME != "" or USERNAME != "" or PASSWORD != "": log("[I] Storing password for " + ENDPOINT + " user " + USERNAME + " in credential store:","info") - cmd="%s -cp lib/* org.apache.ranger.credentialapi.buildks create %s -value %s -provider jceks://file%s" %(JAVA_BIN,ALIAS,PASSWORD,KEYSTORE_FILENAME) + cmd="%s -cp ews/lib/* org.apache.ranger.credentialapi.buildks create %s -value %s -provider jceks://file%s" %(JAVA_BIN,ALIAS,PASSWORD,KEYSTORE_FILENAME) ret=subprocess.call(shlex.split(cmd)) if ret == 0: cmd="chown %s:%s %s" %(unix_user,unix_group,KEYSTORE_FILENAME) diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/metrics/TagSyncMetricsWrapper.java b/tagsync/src/main/java/org/apache/ranger/tagsync/metrics/TagSyncMetricsWrapper.java new file mode 100644 index 00000000000..e51144790cd --- /dev/null +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/metrics/TagSyncMetricsWrapper.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.tagsync.metrics; + +import org.apache.ranger.metrics.RangerMetricsSystemWrapper; +import org.apache.ranger.metrics.wrapper.RangerMetricsSourceWrapper; +import org.apache.ranger.tagsync.metrics.source.RangerTagSyncMetricsSourceTags; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import javax.annotation.PostConstruct; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +@Service +public class TagSyncMetricsWrapper { + private static final Logger LOG = LoggerFactory.getLogger(TagSyncMetricsWrapper.class); + + private static final String context = "tagsync"; + private final RangerMetricsSystemWrapper rangerMetricsSystemWrapper = new RangerMetricsSystemWrapper(); + @Autowired + private RangerTagSyncMetricsSourceTags rangerTagSyncMetricsSourceTags; + + @PostConstruct + public void init() { + if (LOG.isDebugEnabled()) { + LOG.debug("===>> TagSyncMetricsWrapper.init()"); + } + + // Source + List sourceWrappers = new ArrayList<>(); + sourceWrappers.add(new RangerMetricsSourceWrapper("TagsyncSource", "Ranger TAGSYNC metrics", context, + rangerTagSyncMetricsSourceTags)); + + rangerMetricsSystemWrapper.init(context, sourceWrappers, Collections.emptyList()); + + if (LOG.isDebugEnabled()) { + LOG.debug("<<=== TagSyncMetricsWrapper.init()"); + } + } + + public String getRangerMetricsInPrometheusFormat() throws Exception { + return rangerMetricsSystemWrapper.getRangerMetricsInPrometheusFormat(); + } + + public Map> getRangerMetricsInJsonFormat() { + return rangerMetricsSystemWrapper.getRangerMetrics(); + } +} diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/metrics/source/RangerTagSyncMetricsSourceBase.java b/tagsync/src/main/java/org/apache/ranger/tagsync/metrics/source/RangerTagSyncMetricsSourceBase.java new file mode 100644 index 00000000000..a0f52a6ff13 --- /dev/null +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/metrics/source/RangerTagSyncMetricsSourceBase.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.tagsync.metrics.source; + +import org.apache.hadoop.metrics2.MetricsCollector; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.ranger.metrics.RangerMetricsInfo; +import org.apache.ranger.metrics.source.RangerMetricsSource; + +import java.util.HashMap; +import java.util.Map; + +public abstract class RangerTagSyncMetricsSourceBase extends RangerMetricsSource { + Map metricsMap = new HashMap<>(); + private final String context; + private final String record; + + public RangerTagSyncMetricsSourceBase(String context, String record) { + this.context = context; + this.record = record; + } + + @Override + protected void update(MetricsCollector collector, boolean all) { + MetricsRecordBuilder builder = collector.addRecord(this.record).setContext(this.context); + + for (String key : metricsMap.keySet()) { + builder.addGauge(new RangerMetricsInfo(key, ""), metricsMap.get(key)); + } + } +} diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/metrics/source/RangerTagSyncMetricsSourceTags.java b/tagsync/src/main/java/org/apache/ranger/tagsync/metrics/source/RangerTagSyncMetricsSourceTags.java new file mode 100644 index 00000000000..721e8cf0d81 --- /dev/null +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/metrics/source/RangerTagSyncMetricsSourceTags.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.tagsync.metrics.source; + +import org.springframework.stereotype.Component; + +import java.util.concurrent.atomic.AtomicLong; + +@Component +public class RangerTagSyncMetricsSourceTags extends RangerTagSyncMetricsSourceBase { + private static final AtomicLong totalUploads = new AtomicLong(0L); + private static final AtomicLong totalEvents = new AtomicLong(0L); + private static Long totalTimeEvents = 0L; + private static Long totalTimeUploads = 0L; + + public RangerTagSyncMetricsSourceTags() { + super("tagsync", "Tagsync"); + } + + public static Long getTotalTimeUploads() { + return totalTimeUploads; + } + + public static void updateTotalUploadsTime(Long totalTime) { + totalTimeUploads += totalTime; + } + + public static Long getTotalTimeEvents() { + return totalTimeEvents; + } + + public static void updateTotalEventsTime(Long totalTime) { + totalTimeEvents += totalTime; + } + + public static Long getTotalEvents() { + return totalEvents.get(); + } + + public static void updateTotalEvents(long events) { + totalEvents.addAndGet(events); + } + + public static Long getTotalUploads() { + return totalUploads.get(); + } + + public static void updateTotalUploads() { + totalUploads.incrementAndGet(); + } + + @Override + protected void refresh() { + metricsMap.put("TotalUploads", getTotalUploads()); + metricsMap.put("TotalEvents", getTotalEvents()); + metricsMap.put("TotalTimeForEvents", getTotalTimeEvents()); + metricsMap.put("TotalTimeForUploads", getTotalTimeUploads()); + } +} diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/model/AbstractTagSource.java b/tagsync/src/main/java/org/apache/ranger/tagsync/model/AbstractTagSource.java index cba69860d26..647433f2c83 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/model/AbstractTagSource.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/model/AbstractTagSource.java @@ -21,6 +21,7 @@ import org.apache.ranger.authorization.utils.JsonUtils; import org.apache.ranger.plugin.util.ServiceTags; +import org.apache.ranger.tagsync.metrics.source.RangerTagSyncMetricsSourceTags; import org.apache.ranger.tagsync.process.TagSyncConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -70,6 +71,12 @@ protected void updateSink(final ServiceTags toUpload) throws Exception { if (LOG.isDebugEnabled()) { LOG.debug("Uploaded serviceTags={}", JsonUtils.objectToJson(uploaded)); } + if (!uploaded.getTags().isEmpty()) { + RangerTagSyncMetricsSourceTags.updateTotalUploads(); + if (LOG.isDebugEnabled()) { + LOG.debug("current total_upload count = " + RangerTagSyncMetricsSourceTags.getTotalUploads()); + } + } } } catch (Exception exception) { LOG.error("Failed to upload serviceTags: {}", JsonUtils.objectToJson(toUpload)); diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/process/RangerTagSyncStarter.java b/tagsync/src/main/java/org/apache/ranger/tagsync/process/RangerTagSyncStarter.java new file mode 100644 index 00000000000..5e74299ed15 --- /dev/null +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/process/RangerTagSyncStarter.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.tagsync.process; + +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; + +@Component +public class RangerTagSyncStarter { + @PostConstruct + public void init() { + TagSynchronizerthread tagSynchronizerthread = new TagSynchronizerthread(); + + tagSynchronizerthread.setName("TagSync_Thread"); + tagSynchronizerthread.setDaemon(true); + tagSynchronizerthread.start(); + } + + static class TagSynchronizerthread extends Thread { + @Override + public void run() { + TagSynchronizer.main(null); + } + } +} diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java index b6fe7be3f59..915b0e15a5b 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java @@ -41,65 +41,65 @@ import java.util.Properties; public class TagSyncConfig extends Configuration { + public static final String TAGSYNC_ENABLED_PROP = "ranger.tagsync.enabled"; + public static final String TAGSYNC_LOGDIR_PROP = "ranger.tagsync.logdir"; + public static final String TAGSYNC_FILESOURCE_FILENAME_PROP = "ranger.tagsync.source.file.filename"; + public static final String TAGSYNC_RANGER_COOKIE_ENABLED_PROP = "ranger.tagsync.cookie.enabled"; + public static final String TAGSYNC_TAGADMIN_COOKIE_NAME_PROP = "ranger.tagsync.dest.ranger.session.cookie.name"; + public static final int DEFAULT_TAGSYNC_ATLASREST_SOURCE_ENTITIES_BATCH_SIZE = 10000; + public static final String TAGSYNC_KERBEROS_IDENTITY = "tagsync.kerberos.identity"; + public static final String TAGSYNC_SERVER_HA_ENABLED_PARAM = "ranger-tagsync.server.ha.enabled"; private static final Logger LOG = LoggerFactory.getLogger(TagSyncConfig.class); - - public static final String TAGSYNC_ENABLED_PROP = "ranger.tagsync.enabled"; - public static final String TAGSYNC_LOGDIR_PROP = "ranger.tagsync.logdir"; - public static final String TAGSYNC_FILESOURCE_FILENAME_PROP = "ranger.tagsync.source.file.filename"; - public static final String TAGSYNC_RANGER_COOKIE_ENABLED_PROP = "ranger.tagsync.cookie.enabled"; - public static final String TAGSYNC_TAGADMIN_COOKIE_NAME_PROP = "ranger.tagsync.dest.ranger.session.cookie.name"; - public static final int DEFAULT_TAGSYNC_ATLASREST_SOURCE_ENTITIES_BATCH_SIZE = 10000; - public static final String TAGSYNC_KERBEROS_IDENTITY = "tagsync.kerberos.identity"; - public static final String TAGSYNC_SERVER_HA_ENABLED_PARAM = "ranger-tagsync.server.ha.enabled"; - private static final String CONFIG_FILE = "ranger-tagsync-site.xml"; - private static final String DEFAULT_CONFIG_FILE = "ranger-tagsync-default.xml"; - private static final String CORE_SITE_FILE = "core-site.xml"; - private static final String TAGSYNC_TAGADMIN_REST_URL_PROP = "ranger.tagsync.dest.ranger.endpoint"; - private static final String TAGSYNC_TAGADMIN_REST_SSL_CONFIG_FILE_PROP = "ranger.tagsync.dest.ranger.ssl.config.filename"; - private static final String TAGSYNC_SINK_CLASS_PROP = "ranger.tagsync.dest.ranger.impl.class"; - private static final String TAGSYNC_DEST_RANGER_PASSWORD_ALIAS = "tagadmin.user.password"; - private static final String TAGSYNC_SOURCE_ATLASREST_PASSWORD_ALIAS = "atlas.user.password"; - private static final String TAGSYNC_TAGADMIN_USERNAME_PROP = "ranger.tagsync.dest.ranger.username"; - private static final String TAGSYNC_ATLASREST_USERNAME_PROP = "ranger.tagsync.source.atlasrest.username"; - private static final String TAGSYNC_TAGADMIN_PASSWORD_PROP = "ranger.tagsync.dest.ranger.password"; - private static final String TAGSYNC_ATLASREST_PASSWORD_PROP = "ranger.tagsync.source.atlasrest.password"; - private static final String TAGSYNC_TAGADMIN_CONNECTION_CHECK_INTERVAL_PROP = "ranger.tagsync.dest.ranger.connection.check.interval"; - private static final String TAGSYNC_SOURCE_ATLAS_CUSTOM_RESOURCE_MAPPERS_PROP = "ranger.tagsync.atlas.custom.resource.mappers"; - private static final String TAGSYNC_ATLASSOURCE_ENDPOINT_PROP = "ranger.tagsync.source.atlasrest.endpoint"; - private static final String TAGSYNC_ATLAS_REST_SOURCE_DOWNLOAD_INTERVAL_PROP = "ranger.tagsync.source.atlasrest.download.interval.millis"; - private static final String TAGSYNC_ATLAS_REST_SSL_CONFIG_FILE_PROP = "ranger.tagsync.source.atlasrest.ssl.config.filename"; - private static final String TAGSYNC_FILESOURCE_MOD_TIME_CHECK_INTERVAL_PROP = "ranger.tagsync.source.file.check.interval.millis"; - private static final String TAGSYNC_KEYSTORE_TYPE_PROP = "ranger.keystore.file.type"; - private static final String TAGSYNC_TAGADMIN_KEYSTORE_PROP = "ranger.tagsync.keystore.filename"; - private static final String TAGSYNC_ATLASREST_KEYSTORE_PROP = "ranger.tagsync.source.atlasrest.keystore.filename"; - private static final String TAGSYNC_SOURCE_RETRY_INITIALIZATION_INTERVAL_PROP = "ranger.tagsync.source.retry.initialization.interval.millis"; - private static final String DEFAULT_TAGADMIN_USERNAME = "rangertagsync"; - private static final String DEFAULT_ATLASREST_USERNAME = "admin"; - private static final String DEFAULT_ATLASREST_PASSWORD = "admin"; - private static final int DEFAULT_TAGSYNC_TAGADMIN_CONNECTION_CHECK_INTERVAL = 15000; - private static final long DEFAULT_TAGSYNC_ATLASREST_SOURCE_DOWNLOAD_INTERVAL = 900000; - private static final long DEFAULT_TAGSYNC_FILESOURCE_MOD_TIME_CHECK_INTERVAL = 60000; - private static final long DEFAULT_TAGSYNC_SOURCE_RETRY_INITIALIZATION_INTERVAL = 10000; - private static final long DEFAULT_TAGSYNC_HA_PASSIVE_SLEEP_INTERVAL = 5000; - private static final String AUTH_TYPE = "hadoop.security.authentication"; - private static final String NAME_RULES = "hadoop.security.auth_to_local"; - private static final String TAGSYNC_KERBEROS_PRICIPAL = "ranger.tagsync.kerberos.principal"; - private static final String TAGSYNC_KERBEROS_KEYTAB = "ranger.tagsync.kerberos.keytab"; - private static final String TAGSYNC_METRICS_FILEPATH = "ranger.tagsync.metrics.filepath"; - private static final String DEFAULT_TAGSYNC_METRICS_FILEPATH = "/tmp/"; - private static final String TAGSYNC_METRICS_FILENAME = "ranger.tagsync.metrics.filename"; - private static final String DEFAULT_TAGSYNC_METRICS_FILENAME = "ranger_tagsync_metric.json"; - private static final String TAGSYNC_METRICS_FREQUENCY_TIME_IN_MILLIS_PARAM = "ranger.tagsync.metrics.frequencytimeinmillis"; - private static final long DEFAULT_TAGSYNC_METRICS_FREQUENCY__TIME_IN_MILLIS = 10000L; - private static final String TAGSYNC_METRICS_ENABLED_PROP = "ranger.tagsync.metrics.enabled"; - private static final int DEFAULT_TAGSYNC_SINK_MAX_BATCH_SIZE = 1; - private static final String TAGSYNC_SINK_MAX_BATCH_SIZE_PROP = "ranger.tagsync.dest.ranger.max.batch.size"; - private static final String TAGSYNC_ATLASREST_SOURCE_ENTITIES_BATCH_SIZE = "ranger.tagsync.source.atlasrest.entities.batch.size"; + private static final String CONFIG_FILE = "ranger-tagsync-site.xml"; + private static final String DEFAULT_CONFIG_FILE = "ranger-tagsync-default.xml"; + private static final String CORE_SITE_FILE = "core-site.xml"; + private static final String TAGSYNC_TAGADMIN_REST_URL_PROP = "ranger.tagsync.dest.ranger.endpoint"; + private static final String TAGSYNC_TAGADMIN_REST_SSL_CONFIG_FILE_PROP = "ranger.tagsync.dest.ranger.ssl.config.filename"; + private static final String TAGSYNC_SINK_CLASS_PROP = "ranger.tagsync.dest.ranger.impl.class"; + private static final String TAGSYNC_DEST_RANGER_PASSWORD_ALIAS = "tagadmin.user.password"; + private static final String TAGSYNC_SOURCE_ATLASREST_PASSWORD_ALIAS = "atlas.user.password"; + private static final String TAGSYNC_TAGADMIN_USERNAME_PROP = "ranger.tagsync.dest.ranger.username"; + private static final String TAGSYNC_ATLASREST_USERNAME_PROP = "ranger.tagsync.source.atlasrest.username"; + private static final String TAGSYNC_TAGADMIN_PASSWORD_PROP = "ranger.tagsync.dest.ranger.password"; + private static final String TAGSYNC_ATLASREST_PASSWORD_PROP = "ranger.tagsync.source.atlasrest.password"; + private static final String TAGSYNC_TAGADMIN_CONNECTION_CHECK_INTERVAL_PROP = "ranger.tagsync.dest.ranger.connection.check.interval"; + private static final String TAGSYNC_SOURCE_ATLAS_CUSTOM_RESOURCE_MAPPERS_PROP = "ranger.tagsync.atlas.custom.resource.mappers"; + private static final String TAGSYNC_ATLASSOURCE_ENDPOINT_PROP = "ranger.tagsync.source.atlasrest.endpoint"; + private static final String TAGSYNC_ATLAS_REST_SOURCE_DOWNLOAD_INTERVAL_PROP = "ranger.tagsync.source.atlasrest.download.interval.millis"; + private static final String TAGSYNC_ATLAS_REST_SSL_CONFIG_FILE_PROP = "ranger.tagsync.source.atlasrest.ssl.config.filename"; + private static final String TAGSYNC_FILESOURCE_MOD_TIME_CHECK_INTERVAL_PROP = "ranger.tagsync.source.file.check.interval.millis"; + private static final String TAGSYNC_KEYSTORE_TYPE_PROP = "ranger.keystore.file.type"; + private static final String TAGSYNC_TAGADMIN_KEYSTORE_PROP = "ranger.tagsync.keystore.filename"; + private static final String TAGSYNC_ATLASREST_KEYSTORE_PROP = "ranger.tagsync.source.atlasrest.keystore.filename"; + private static final String TAGSYNC_SOURCE_RETRY_INITIALIZATION_INTERVAL_PROP = "ranger.tagsync.source.retry.initialization.interval.millis"; + private static final String DEFAULT_TAGADMIN_USERNAME = "rangertagsync"; + private static final String DEFAULT_ATLASREST_USERNAME = "admin"; + private static final String DEFAULT_ATLASREST_PASSWORD = "admin"; + private static final int DEFAULT_TAGSYNC_TAGADMIN_CONNECTION_CHECK_INTERVAL = 15000; + private static final long DEFAULT_TAGSYNC_ATLASREST_SOURCE_DOWNLOAD_INTERVAL = 900000; + private static final long DEFAULT_TAGSYNC_FILESOURCE_MOD_TIME_CHECK_INTERVAL = 60000; + private static final long DEFAULT_TAGSYNC_SOURCE_RETRY_INITIALIZATION_INTERVAL = 10000; + private static final long DEFAULT_TAGSYNC_HA_PASSIVE_SLEEP_INTERVAL = 5000; + private static final String AUTH_TYPE = "hadoop.security.authentication"; + private static final String NAME_RULES = "hadoop.security.auth_to_local"; + private static final String TAGSYNC_KERBEROS_PRICIPAL = "ranger.tagsync.kerberos.principal"; + private static final String TAGSYNC_KERBEROS_KEYTAB = "ranger.tagsync.kerberos.keytab"; + private static final String TAGSYNC_METRICS_FILEPATH = "ranger.tagsync.metrics.filepath"; + private static final String DEFAULT_TAGSYNC_METRICS_FILEPATH = "/tmp/"; + private static final String TAGSYNC_METRICS_FILENAME = "ranger.tagsync.metrics.filename"; + private static final String DEFAULT_TAGSYNC_METRICS_FILENAME = "ranger_tagsync_metric.json"; + private static final String TAGSYNC_METRICS_FREQUENCY_TIME_IN_MILLIS_PARAM = "ranger.tagsync.metrics.frequencytimeinmillis"; + private static final long DEFAULT_TAGSYNC_METRICS_FREQUENCY__TIME_IN_MILLIS = 10000L; + private static final String TAGSYNC_METRICS_ENABLED_PROP = "ranger.tagsync.metrics.enabled"; + private static final int DEFAULT_TAGSYNC_SINK_MAX_BATCH_SIZE = 1; + private static final String TAGSYNC_SINK_MAX_BATCH_SIZE_PROP = "ranger.tagsync.dest.ranger.max.batch.size"; + private static final String TAGSYNC_ATLASREST_SOURCE_ENTITIES_BATCH_SIZE = "ranger.tagsync.source.atlasrest.entities.batch.size"; + private static final String SSL_POLICYMGR_CONFIG_FILE = "ranger-tagsync-policymgr-ssl.xml"; private static TagSyncConfig instance; private static String localHostname; - private Properties props; + private Properties props; private TagSyncConfig() { super(false); @@ -450,7 +450,7 @@ public static String getTagsyncKerberosIdentity(Properties prop) { } public static int getSinkMaxBatchSize(Properties prop) { - int ret = DEFAULT_TAGSYNC_SINK_MAX_BATCH_SIZE; + int ret = DEFAULT_TAGSYNC_SINK_MAX_BATCH_SIZE; String maxBatchSizeStr = prop.getProperty(TAGSYNC_SINK_MAX_BATCH_SIZE_PROP); if (StringUtils.isNotEmpty(maxBatchSizeStr)) { @@ -490,10 +490,8 @@ public Properties getProperties() { @Override public String toString() { - StringBuilder sb = new StringBuilder(); - - sb.append("DEFAULT_CONFIG_FILE=").append(DEFAULT_CONFIG_FILE).append(", ") - .append("CONFIG_FILE=").append(CONFIG_FILE).append("\n\n"); + String sb = "DEFAULT_CONFIG_FILE=" + DEFAULT_CONFIG_FILE + ", " + + "CONFIG_FILE=" + CONFIG_FILE + "\n\n"; return sb + super.toString(); } @@ -549,6 +547,7 @@ private void init() { readConfigFile(CORE_SITE_FILE); readConfigFile(DEFAULT_CONFIG_FILE); readConfigFile(CONFIG_FILE); + readConfigFile(SSL_POLICYMGR_CONFIG_FILE); props = getProps(); @@ -563,6 +562,12 @@ private void init() { props.setProperty(propertyName, systemPropertyValue); } } + + if (null != props.getProperty("xasecure.policymgr.clientssl.keystore.credential.file") && + null != props.getProperty("xasecure.policymgr.clientssl.keystore")) { + props.setProperty("ranger.tagsync.credential.provider.path", props.getProperty("xasecure.policymgr.clientssl.keystore.credential.file")); + props.setProperty("ranger.tagsync.https.attrib.keystore.file", props.getProperty("xasecure.policymgr.clientssl.keystore")); + } } private void readConfigFile(String fileName) { diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/rest/MetricsREST.java b/tagsync/src/main/java/org/apache/ranger/tagsync/rest/MetricsREST.java new file mode 100644 index 00000000000..c53ea7159fd --- /dev/null +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/rest/MetricsREST.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.tagsync.rest; + +import org.apache.ranger.plugin.model.RangerMetrics; +import org.apache.ranger.plugin.util.RangerMetricsUtil; +import org.apache.ranger.tagsync.metrics.TagSyncMetricsWrapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; + +import java.lang.management.ManagementFactory; +import java.lang.management.RuntimeMXBean; +import java.util.LinkedHashMap; +import java.util.Map; + +@Path("metrics") +@Component +public class MetricsREST { + private static final Logger LOG = LoggerFactory.getLogger(MetricsREST.class); + + private static final RuntimeMXBean RUNTIME = ManagementFactory.getRuntimeMXBean(); + private static final String JVM_MACHINE_ACTUAL_NAME = RUNTIME.getVmName(); + private static final String VERSION = RUNTIME.getVmVersion(); + private static final String JVM_MACHINE_REPRESENTATION_NAME = RUNTIME.getName(); + private static final String JVM_VENDOR_NAME = RUNTIME.getVmVendor(); + + RangerMetricsUtil jvmMetricUtil = new RangerMetricsUtil(); + + @Autowired + private TagSyncMetricsWrapper tagSyncMetricsWrapper; + + @GET + @Path("/status") + @Produces("application/json") + public RangerMetrics getStatus() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> MetricsREST.getStatus()"); + } + LOG.info("==> MetricsREST.getStatus()"); + Map jvm = new LinkedHashMap<>(); + Map vmDetails = new LinkedHashMap<>(); + vmDetails.put("JVM Machine Actual Name", JVM_MACHINE_ACTUAL_NAME); + vmDetails.put("version", VERSION); + vmDetails.put("JVM Machine Representation Name", JVM_MACHINE_REPRESENTATION_NAME); + vmDetails.put("Up time of JVM", RUNTIME.getUptime()); + vmDetails.put("JVM Vendor Name", JVM_VENDOR_NAME); + vmDetails.putAll(jvmMetricUtil.getValues()); + jvm.put("jvm", vmDetails); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== MetricsREST.getStatus() {}", jvm); + } + + return new RangerMetrics(jvm); + } + + @GET + @Path("/prometheus") + @Produces(MediaType.TEXT_PLAIN) + public String getMetricsPrometheus() { + if (LOG.isDebugEnabled()) { + LOG.debug("===>> MetricsREST.getMetricsPrometheus()"); + } + String ret = ""; + try { + ret = tagSyncMetricsWrapper.getRangerMetricsInPrometheusFormat(); + } catch (Exception e) { + LOG.error("MetricsREST.getMetricsPrometheus(): Exception occured while getting metric.", e); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<<=== MetricsREST.getMetricsPrometheus() {}" + ret); + } + return ret; + } + + @GET + @Path("/json") + @Produces({"application/json", "application/xml"}) + public Map> getMetricsJson() { + if (LOG.isDebugEnabled()) { + LOG.debug("===>> MetricsREST.getMetricsJson()"); + } + + Map> ret = null; + try { + ret = tagSyncMetricsWrapper.getRangerMetricsInJsonFormat(); + } catch (Exception e) { + LOG.error("MetricsREST.getMetricsJson(): Exception occured while getting metric.", e); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<<=== MetricsREST.getMetricsJson() {}" + ret); + } + return ret; + } +} diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/security/TagsyncAuthEntryPoint.java b/tagsync/src/main/java/org/apache/ranger/tagsync/security/TagsyncAuthEntryPoint.java new file mode 100644 index 00000000000..68ae69f39c7 --- /dev/null +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/security/TagsyncAuthEntryPoint.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.tagsync.security; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.security.core.AuthenticationException; +import org.springframework.security.web.authentication.LoginUrlAuthenticationEntryPoint; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import java.io.IOException; + +public class TagsyncAuthEntryPoint extends LoginUrlAuthenticationEntryPoint { + private static final Logger LOG = LoggerFactory.getLogger(TagsyncAuthEntryPoint.class); + + public TagsyncAuthEntryPoint(String loginFormUrl) { + super(loginFormUrl); + } + + @Override + public void commence(HttpServletRequest request, HttpServletResponse response, AuthenticationException authException) throws IOException, ServletException { + // any task for entry filter + response.setHeader("X-Frame-Options", "DENY"); + + if (LOG.isDebugEnabled()) { + LOG.debug("commence() X-Requested-With=[{}]", request.getHeader("X-Requested-With")); + LOG.debug("requestURL : [{}]", (request.getRequestURL() != null) ? request.getRequestURL().toString() : ""); + } + + // calling parent, after validations are passed. + super.commence(request, response, authException); + } +} diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/server/EmbeddedServer.java b/tagsync/src/main/java/org/apache/ranger/tagsync/server/EmbeddedServer.java new file mode 100644 index 00000000000..55c5723ebdc --- /dev/null +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/server/EmbeddedServer.java @@ -0,0 +1,733 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.tagsync.server; + +import org.apache.catalina.Context; +import org.apache.catalina.LifecycleException; +import org.apache.catalina.WebResourceRoot; +import org.apache.catalina.connector.Connector; +import org.apache.catalina.core.StandardContext; +import org.apache.catalina.startup.Tomcat; +import org.apache.catalina.valves.AccessLogValve; +import org.apache.catalina.valves.ErrorReportValve; +import org.apache.catalina.webresources.StandardRoot; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.SecureClientLogin; +import org.apache.hadoop.security.alias.BouncyCastleFipsKeyStoreProvider; +import org.apache.hadoop.security.alias.CredentialProvider; +import org.apache.hadoop.security.alias.CredentialProviderFactory; +import org.apache.hadoop.security.alias.JavaKeyStoreProvider; +import org.apache.hadoop.security.alias.LocalBouncyCastleFipsKeyStoreProvider; +import org.apache.ranger.authorization.hadoop.utils.RangerCredentialProvider; +import org.apache.tomcat.util.net.SSLHostConfig; +import org.apache.tomcat.util.scan.StandardJarScanner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.KeyManager; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; +import javax.security.auth.Subject; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.security.KeyManagementException; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.PrivilegedAction; +import java.security.SecureRandom; +import java.security.UnrecoverableKeyException; +import java.security.cert.CertificateException; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +public class EmbeddedServer { + public static final String RANGER_KERBEROS_KEYTAB = "kerberos.keytab"; + public static final String RANGER_KERBEROS_PRINCIPAL = "kerberos.principal"; + public static final String HADOOP_KERBEROS_NAME_RULES = "hadoop.security.auth_to_local"; + public static final String HADOOP_AUTHENTICATION_TYPE = "hadoop.security.authentication"; + public static final String HADOOP_AUTH_TYPE_KERBEROS = "kerberos"; + public static final String DEFAULT_NAME_RULE = "DEFAULT"; + public static final String KEYSTORE_FILE_TYPE_DEFAULT = KeyStore.getDefaultType(); + public static final String TRUSTSTORE_FILE_TYPE_DEFAULT = KeyStore.getDefaultType(); + private static final Logger LOG = LoggerFactory.getLogger(EmbeddedServer.class); + private static final String RANGER_SSL_CONTEXT_ALGO_TYPE = "TLS"; + private static final String RANGER_SSL_KEYMANAGER_ALGO_TYPE = KeyManagerFactory.getDefaultAlgorithm(); + private static final String RANGER_SSL_TRUSTMANAGER_ALGO_TYPE = TrustManagerFactory.getDefaultAlgorithm(); + private static final String ACCESS_LOG_PREFIX = "accesslog.prefix"; + private static final String ACCESS_LOG_DATE_FORMAT = "accesslog.dateformat"; + private static final String ACCESS_LOG_PATTERN = "accesslog.pattern"; + private static final String ACCESS_LOG_ROTATE_MAX_DAYS = "accesslog.rotate.max.days"; + private static final int DEFAULT_HTTP_PORT = 8180; + private static final int DEFAULT_HTTPS_PORT = 8183; + private static final int DEFAULT_SHUTDOWN_PORT = 8185; + private static final String DEFAULT_SHUTDOWN_COMMAND = "SHUTDOWN"; + private static final String DEFAULT_WEBAPPS_ROOT_FOLDER = "webapps"; + private static final String DEFAULT_ENABLED_PROTOCOLS = "TLSv1.2"; + private static final String DEFAULT_SSL_PROTOCOL = "TLS"; + private final Configuration configuration; + private final String appName; + private final String configPrefix; + private String keyStoreFileType; + private String trustStoreFileType; + + public EmbeddedServer(Configuration configuration, String appName, String configPrefix) { + LOG.info("==> EmbeddedServer(appName={}, configPrefix={})", appName, configPrefix); + + this.configuration = configuration; + this.appName = appName; + this.configPrefix = configPrefix; + + LOG.info("<== EmbeddedServer(appName={}, configPrefix={}", appName, configPrefix); + } + + public static void main(String[] args) { + Configuration config = new Configuration(); + String appName = "ranger-tagsync"; + String configPrefix = "ranger.tagsync."; + + new EmbeddedServer(config, appName, configPrefix).start(); + } + + @SuppressWarnings("deprecation") + public void start() { + LOG.info("==> EmbeddedServer.start(appName={})", appName); + + SSLContext sslContext = getSSLContext(); + + if (sslContext != null) { + SSLContext.setDefault(sslContext); + } + + final Tomcat server = new Tomcat(); + + String logDir = getConfig("log.dir"); + String hostName = getConfig("service.host"); + int serverPort = getIntConfig("service.http.port", DEFAULT_HTTP_PORT); + int sslPort = getIntConfig("service.https.port", DEFAULT_HTTPS_PORT); + int shutdownPort = getIntConfig("service.shutdown.port", DEFAULT_SHUTDOWN_PORT); + String shutdownCommand = getConfig("service.shutdown.command", DEFAULT_SHUTDOWN_COMMAND); + boolean isHttpsEnabled = getBooleanConfig("service.https.attrib.ssl.enabled", false); + boolean ajpEnabled = getBooleanConfig("ajp.enabled", false); + keyStoreFileType = getConfig("ranger.tagsync.keystore.file.type", KEYSTORE_FILE_TYPE_DEFAULT); + trustStoreFileType = getConfig("ranger.tagsync.truststore.file.type", TRUSTSTORE_FILE_TYPE_DEFAULT); + + server.setHostname(hostName); + server.setPort(serverPort); + server.getServer().setPort(shutdownPort); + server.getServer().setShutdown(shutdownCommand); + if (ajpEnabled) { + Connector ajpConnector = new Connector("org.apache.coyote.ajp.AjpNioProtocol"); + + ajpConnector.setPort(serverPort); + ajpConnector.setProperty("protocol", "AJP/1.3"); + + server.getService().addConnector(ajpConnector); + + // Making this as a default connector + server.setConnector(ajpConnector); + + LOG.info("Created AJP Connector"); + } else if (isHttpsEnabled && sslPort > 0) { + String clientAuth = getConfig("service.https.attrib.clientAuth", "false"); + String providerPath = getConfig("credential.provider.path"); + String keyAlias = getConfig("service.https.attrib.keystore.credential.alias", "keyStoreCredentialAlias"); + String keystorePass = null; + String enabledProtocols = getConfig("service.https.attrib.ssl.enabled.protocols", DEFAULT_ENABLED_PROTOCOLS); + String ciphers = getConfig("tomcat.ciphers"); + + if (StringUtils.equalsIgnoreCase(clientAuth, "false")) { + clientAuth = getConfig("service.https.attrib.client.auth", "want"); + } + + if (providerPath != null && keyAlias != null) { + keystorePass = getDecryptedString(providerPath.trim(), keyAlias.trim(), keyStoreFileType); + + if (StringUtils.isBlank(keystorePass) || StringUtils.equalsIgnoreCase(keystorePass.trim(), "none")) { + keystorePass = getConfig("service.https.attrib.keystore.pass"); + } + } + + Connector ssl = new Connector(); + String sslKeystoreKeyAlias = getConfig("service.https.attrib.keystore.keyalias", ""); + ssl.setPort(sslPort); + ssl.setSecure(true); + ssl.setScheme("https"); + ssl.setAttribute("SSLEnabled", "true"); + ssl.setAttribute("sslProtocol", getConfig("service.https.attrib.ssl.protocol", DEFAULT_SSL_PROTOCOL)); + ssl.setAttribute("clientAuth", clientAuth); + if (StringUtils.isNotBlank(sslKeystoreKeyAlias)) { + ssl.setAttribute("keyAlias", sslKeystoreKeyAlias); + } + ssl.setAttribute("keystorePass", keystorePass); + ssl.setAttribute("keystoreFile", getKeystoreFile()); + ssl.setAttribute("sslEnabledProtocols", enabledProtocols); + ssl.setAttribute("keystoreType", keyStoreFileType); + ssl.setAttribute("truststoreType", trustStoreFileType); + + if (StringUtils.isNotBlank(ciphers)) { + ssl.setAttribute("ciphers", ciphers); + SSLHostConfig[] configs = ssl.findSslHostConfigs(); + if (configs != null) { + for (SSLHostConfig hostConfig : configs) { + if (hostConfig != null) { + hostConfig.setCipherSuites(ciphers); + } + } + } + } + server.getService().addConnector(ssl); + + // + // Making this as a default connector + // + server.setConnector(ssl); + } + updateHttpConnectorAttribConfig(server); + + File logDirectory = new File(logDir); + + if (!logDirectory.exists()) { + logDirectory.mkdirs(); + } + + String logPattern = getConfig(ACCESS_LOG_PATTERN, "%h %l %u %t \"%r\" %s %b"); + + AccessLogValve valve = new AccessLogValve(); + + valve.setRotatable(true); + valve.setAsyncSupported(true); + valve.setBuffered(false); + valve.setEnabled(true); + valve.setPrefix(getConfig(ACCESS_LOG_PREFIX, "access_log-" + hostName + "-")); + valve.setFileDateFormat(getConfig(ACCESS_LOG_DATE_FORMAT, "yyyy-MM-dd.HH")); + valve.setDirectory(logDirectory.getAbsolutePath()); + valve.setSuffix(".log"); + valve.setPattern(logPattern); + valve.setMaxDays(getIntConfig(ACCESS_LOG_ROTATE_MAX_DAYS, 15)); + + server.getHost().getPipeline().addValve(valve); + + ErrorReportValve errorReportValve = new ErrorReportValve(); + boolean showServerinfo = getBooleanConfig("ranger.valve.errorreportvalve.showserverinfo", false); + boolean showReport = getBooleanConfig("ranger.valve.errorreportvalve.showreport", false); + + errorReportValve.setShowServerInfo(showServerinfo); + errorReportValve.setShowReport(showReport); + + server.getHost().getPipeline().addValve(errorReportValve); + + try { + String webappDir = getConfig("webapp.dir"); + LOG.info("==> EmbeddedServer.start(webappDir={})", webappDir); + if (StringUtils.isBlank(webappDir)) { + LOG.error("Tomcat Server failed to start: {}.webapp.dir is not set", configPrefix); + + System.exit(1); + } + + String webContextName = getConfig("contextName", ""); + + if (StringUtils.isBlank(webContextName)) { + webContextName = ""; + } else if (!webContextName.startsWith("/")) { + LOG.info("Context Name [{}] is being loaded as [ /{}]", webContextName, webContextName); + + webContextName = "/" + webContextName; + } + + File wad = new File(webappDir); + + if (wad.isDirectory()) { + LOG.info("Webapp dir={}, webAppName={}", webappDir, webContextName); + } else if (wad.isFile()) { + File webAppDir = new File(DEFAULT_WEBAPPS_ROOT_FOLDER); + + if (!webAppDir.exists()) { + webAppDir.mkdirs(); + } + + LOG.info("Webapp file={}, webAppName={}", webappDir, webContextName); + } + + LOG.info("Adding webapp [{}] = path [{}] .....", webContextName, webappDir); + + Context webappCtx = server.addWebapp(webContextName, new File(webappDir).getAbsolutePath()); + if (webappCtx instanceof StandardContext) { + boolean allowLinking = getBooleanConfig("allow.linking", true); + StandardContext standardContext = (StandardContext) webappCtx; + String workDirPath = getConfig("tomcat.work.dir", ""); + if (!workDirPath.isEmpty() && new File(workDirPath).exists()) { + standardContext.setWorkDir(workDirPath); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Skipping to set tomcat server work directory {} as it is blank or directory does not exist.", workDirPath); + } + } + WebResourceRoot resRoot = new StandardRoot(standardContext); + webappCtx.setResources(resRoot); + webappCtx.getResources().setAllowLinking(allowLinking); + StandardJarScanner scanner = new StandardJarScanner(); + scanner.setScanManifest(false); + webappCtx.setJarScanner(scanner); + if (LOG.isDebugEnabled()) { + LOG.debug("Tomcat Configuration - allowLinking=[{}]", allowLinking); + } + } else { + LOG.error("Tomcat Context [{}] is either NULL OR it's NOT instanceof StandardContext", webappCtx); + } + + webappCtx.init(); + + LOG.info("Finished init of webapp [{}] = path [{}].", webContextName, webappDir); + } catch (LifecycleException lce) { + LOG.error("Tomcat Server failed to start webapp", lce); + + lce.printStackTrace(); + } + + String authenticationType = getConfig(HADOOP_AUTHENTICATION_TYPE); + + if (StringUtils.equalsIgnoreCase(authenticationType, HADOOP_AUTH_TYPE_KERBEROS)) { + String keyTab = getConfig(RANGER_KERBEROS_KEYTAB); + String principal = null; + String nameRules = getConfig(HADOOP_KERBEROS_NAME_RULES, DEFAULT_NAME_RULE); + + try { + principal = SecureClientLogin.getPrincipal(getConfig(RANGER_KERBEROS_PRINCIPAL), hostName); + } catch (IOException excp) { + LOG.warn("Failed to get principal {}", excp); + } + + if (SecureClientLogin.isKerberosCredentialExists(principal, keyTab)) { + try { + LOG.info("Logging in with Kerberos: principal={}, keytab={}, nameRules={}", principal, keyTab, nameRules); + + Subject sub = SecureClientLogin.loginUserFromKeytab(principal, keyTab, nameRules); + + Subject.doAs(sub, new PrivilegedAction() { + @Override + public Void run() { + LOG.info("Starting Server using kerberos credential"); + + startServer(server); + + return null; + } + }); + } catch (Exception excp) { + LOG.error("Tomcat Server failed to start", excp); + } + } else { + LOG.warn("Kerberos principal={} not found in keytab={}. Starting server in non-kerberos mode", principal, keyTab); + + startServer(server); + } + } else { + LOG.info("Starting server in non-kerberos mode"); + + startServer(server); + } + + LOG.info("<== EmbeddedServer.start(appName={})", appName); + } + + public String getDecryptedString(String crendentialProviderPath, String alias, String storeType) { + String ret = null; + + if (StringUtils.isNotBlank(crendentialProviderPath) && StringUtils.isNotBlank(alias)) { + try { + Configuration conf = new Configuration(); + String prefixJceks = JavaKeyStoreProvider.SCHEME_NAME + "://file"; + String prefixLocalJceks = "localjceks://file"; + String prefixBcfks = BouncyCastleFipsKeyStoreProvider.SCHEME_NAME + "://file"; + String prefixLocalBcfks = LocalBouncyCastleFipsKeyStoreProvider.SCHEME_NAME + "://file"; + + if (StringUtils.startsWithIgnoreCase(crendentialProviderPath, prefixJceks) || StringUtils.startsWithIgnoreCase(crendentialProviderPath, prefixLocalJceks) || StringUtils.startsWithIgnoreCase(crendentialProviderPath, prefixBcfks) || StringUtils.startsWithIgnoreCase(crendentialProviderPath, prefixLocalBcfks)) { + conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, crendentialProviderPath); + } else { + if (crendentialProviderPath.startsWith("/")) { + if (StringUtils.equalsIgnoreCase(storeType, "bcfks")) { + conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, prefixLocalBcfks + crendentialProviderPath); + } else { + conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, prefixJceks + crendentialProviderPath); + } + } else { + conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, prefixJceks + "/" + crendentialProviderPath); + } + } + + List providers = CredentialProviderFactory.getProviders(conf); + + if (LOG.isDebugEnabled()) { + LOG.debug("credentialProviderPath={}, alias={}, storeType={}", crendentialProviderPath, alias, storeType); + LOG.debug("List of CredentialProvider = {}", providers); + } + + for (CredentialProvider provider : providers) { + //System.out.println("Credential Provider :" + provider); + List aliasesList = provider.getAliases(); + + if (CollectionUtils.isNotEmpty(aliasesList) && aliasesList.contains(alias)) { + CredentialProvider.CredentialEntry credEntry = provider.getCredentialEntry(alias); + char[] pass = credEntry.getCredential(); + + if (pass != null && pass.length > 0) { + ret = String.valueOf(pass); + + break; + } + } + } + } catch (Exception ex) { + LOG.error("CredentialReader failed while decrypting provided string. Reason: {}", ex.toString()); + + ret = null; + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("getDecryptedString() : ret = {}", ret); + } + + return ret; + } + + protected long getLongConfig(String key, long defaultValue) { + long ret = defaultValue; + String retStr = getConfig(key); + + try { + if (retStr != null) { + ret = Long.parseLong(retStr); + } + } catch (Exception err) { + LOG.warn("{} can't be parsed to long. Reason: {}", retStr, err.toString()); + } + + return ret; + } + + private void startServer(final Tomcat server) { + LOG.info("==> EmbeddedServer.startServer(appName={})", appName); + + try { + server.start(); + + server.getServer().await(); + + shutdownServer(); + } catch (LifecycleException e) { + LOG.error("Tomcat Server failed to start", e); + + e.printStackTrace(); + } catch (Exception e) { + LOG.error("Tomcat Server failed to start", e); + + e.printStackTrace(); + } + + LOG.info("<== EmbeddedServer.startServer(appName={})", appName); + } + + private void shutdownServer() { + LOG.info("==> EmbeddedServer.shutdownServer(appName={})", appName); + + int timeWaitForShutdownInSeconds = getIntConfig("service.waitTimeForForceShutdownInSeconds", 0); + + if (timeWaitForShutdownInSeconds > 0) { + long endTime = System.currentTimeMillis() + (timeWaitForShutdownInSeconds * 1000L); + + LOG.info("Will wait for all threads to shutdown gracefully. Final shutdown Time: {}", new Date(endTime)); + + while (System.currentTimeMillis() < endTime) { + int activeCount = Thread.activeCount(); + + if (activeCount == 0) { + LOG.info("Number of active threads = {}", activeCount); + + break; + } else { + LOG.info("Number of active threads = {}. Waiting for all threads to shutdown ...", activeCount); + + try { + Thread.sleep(5000L); + } catch (InterruptedException e) { + LOG.warn("shutdownServer process is interrupted with exception", e); + + break; + } + } + } + } + + LOG.info("Shutting down the Server."); + + LOG.info("<== EmbeddedServer.shutdownServer(appName={})", appName); + + System.exit(0); + } + + @SuppressWarnings("deprecation") + private void updateHttpConnectorAttribConfig(Tomcat server) { + server.getConnector().setAllowTrace(getBooleanConfig("service.http.connector.attrib.allowTrace", false)); + server.getConnector().setAsyncTimeout(getLongConfig("service.http.connector.attrib.asyncTimeout", 10000)); + server.getConnector().setEnableLookups(getBooleanConfig("service.http.connector.attrib.enableLookups", false)); + server.getConnector().setMaxParameterCount(getIntConfig("service.http.connector.attrib.maxParameterCount", 10000)); + server.getConnector().setMaxPostSize(getIntConfig("service.http.connector.attrib.maxPostSize", 2097152)); + server.getConnector().setMaxSavePostSize(getIntConfig("service.http.connector.attrib.maxSavePostSize", 4096)); + server.getConnector().setParseBodyMethods(getConfig("service.http.connector.attrib.methods", "POST")); + server.getConnector().setURIEncoding(getConfig("service.http.connector.attrib.URIEncoding", "UTF-8")); + server.getConnector().setProperty("acceptCount", getConfig("service.http.connector.attrib.acceptCount", "1024")); + server.getConnector().setXpoweredBy(false); + server.getConnector().setAttribute("server", getConfig("servername", "Apache Ranger TAGSYNC")); + server.getConnector().setProperty("sendReasonPhrase", getConfig(configPrefix + "service.http.connector.property.sendReasonPhrase", "true")); + + for (Iterator> iterator = configuration.iterator(); iterator.hasNext(); ) { + Map.Entry entry = iterator.next(); + String key = entry.getKey(); + String value = entry.getValue(); + + if (key != null && key.startsWith(configPrefix + "service.http.connector.property.")) { + String property = key.replace(configPrefix + "service.http.connector.property.", ""); + + server.getConnector().setProperty(property, value); + + LOG.info("{}:{}", property, server.getConnector().getProperty(property)); + } + } + } + + private SSLContext getSSLContext() { + SSLContext ret = null; + KeyManager[] kmList = getKeyManagers(); + TrustManager[] tmList = getTrustManagers(); + + if (tmList != null) { + try { + ret = SSLContext.getInstance(RANGER_SSL_CONTEXT_ALGO_TYPE); + + ret.init(kmList, tmList, new SecureRandom()); + } catch (NoSuchAlgorithmException e) { + LOG.error("SSL algorithm is not available in the environment", e); + } catch (KeyManagementException e) { + LOG.error("Unable to initials the SSLContext", e); + } + } + + return ret; + } + + private KeyManager[] getKeyManagers() { + KeyManager[] ret = null; + String keyStoreFile = getConfig("keystore.file"); + String keyStoreAlias = getConfig("keystore.alias"); + String credentialProviderPath = getConfig("credential.provider.path"); + String keyStoreFilepwd = getCredential(credentialProviderPath, keyStoreAlias); + + if (StringUtils.isNotBlank(keyStoreFile) && StringUtils.isNotBlank(keyStoreFilepwd)) { + InputStream in = null; + + try { + in = getFileInputStream(keyStoreFile); + + if (in != null) { + KeyStore keyStore = KeyStore.getInstance(keyStoreFileType); + + keyStore.load(in, keyStoreFilepwd.toCharArray()); + + KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(RANGER_SSL_KEYMANAGER_ALGO_TYPE); + + keyManagerFactory.init(keyStore, keyStoreFilepwd.toCharArray()); + + ret = keyManagerFactory.getKeyManagers(); + } else { + LOG.error("Unable to obtain keystore from file [{}]", keyStoreFile); + } + } catch (KeyStoreException e) { + LOG.error("Unable to obtain from KeyStore", e); + } catch (NoSuchAlgorithmException e) { + LOG.error("SSL algorithm is NOT available in the environment", e); + } catch (CertificateException e) { + LOG.error("Unable to obtain the requested certification", e); + } catch (FileNotFoundException e) { + LOG.error("Unable to find the necessary SSL Keystore Files", e); + } catch (IOException e) { + LOG.error("Unable to read the necessary SSL Keystore Files", e); + } catch (UnrecoverableKeyException e) { + LOG.error("Unable to recover the key from keystore", e); + } finally { + close(in, keyStoreFile); + } + } + + return ret; + } + + private TrustManager[] getTrustManagers() { + TrustManager[] ret = null; + String truststoreFile = getConfig("truststore.file"); + String truststoreAlias = getConfig("truststore.alias"); + String credentialProviderPath = getConfig("credential.provider.path"); + String trustStoreFilepwd = getCredential(credentialProviderPath, truststoreAlias); + + if (StringUtils.isNotBlank(truststoreFile) && StringUtils.isNotBlank(trustStoreFilepwd)) { + InputStream in = null; + + try { + in = getFileInputStream(truststoreFile); + + if (in != null) { + KeyStore trustStore = KeyStore.getInstance(trustStoreFileType); + + trustStore.load(in, trustStoreFilepwd.toCharArray()); + + TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(RANGER_SSL_TRUSTMANAGER_ALGO_TYPE); + + trustManagerFactory.init(trustStore); + + ret = trustManagerFactory.getTrustManagers(); + } else { + LOG.error("Unable to obtain truststore from file [{}]", truststoreFile); + } + } catch (KeyStoreException e) { + LOG.error("Unable to obtain from KeyStore", e); + } catch (NoSuchAlgorithmException e) { + LOG.error("SSL algorithm is NOT available in the environment", e); + } catch (CertificateException e) { + LOG.error("Unable to obtain the requested certification", e); + } catch (FileNotFoundException e) { + LOG.error("Unable to find the necessary SSL TrustStore File:{}", truststoreFile, e); + } catch (IOException e) { + LOG.error("Unable to read the necessary SSL TrustStore Files:{}", truststoreFile, e); + } finally { + close(in, truststoreFile); + } + } + + return ret; + } + + private String getKeystoreFile() { + String ret = getConfig("service.https.attrib.keystore.file"); + + if (StringUtils.isBlank(ret)) { + // new property not configured, lets use the old property + ret = getConfig("https.attrib.keystore.file"); + } + + return ret; + } + + private String getCredential(String url, String alias) { + return RangerCredentialProvider.getInstance().getCredentialString(url, alias); + } + + private String getConfig(String key) { + String propertyWithPrefix = configPrefix + key; + String value = configuration.get(propertyWithPrefix); + + if (value == null) { // config-with-prefix not found; look at System properties + value = System.getProperty(propertyWithPrefix); + + if (value == null) { // config-with-prefix not found in System properties; look for config-without-prefix + value = configuration.get(key); + + if (value == null) { // config-without-prefix not found; look at System properties + value = System.getProperty(key); + } + } + } + + return value; + } + + private String getConfig(String key, String defaultValue) { + String ret = getConfig(key); + + if (ret == null) { + ret = defaultValue; + } + + return ret; + } + + private int getIntConfig(String key, int defaultValue) { + int ret = defaultValue; + String strValue = getConfig(key); + + try { + if (strValue != null) { + ret = Integer.parseInt(strValue); + } + } catch (Exception err) { + LOG.warn("{} can't be parsed to int. Reason: {}", strValue, err.toString()); + } + + return ret; + } + + private boolean getBooleanConfig(String key, boolean defaultValue) { + boolean ret = defaultValue; + String strValue = getConfig(key); + + if (StringUtils.isNotBlank(strValue)) { + ret = Boolean.valueOf(strValue); + } + + return ret; + } + + private InputStream getFileInputStream(String fileName) throws IOException { + InputStream ret = null; + + if (StringUtils.isNotEmpty(fileName)) { + File f = new File(fileName); + + if (f.exists()) { + ret = new FileInputStream(f); + } else { + ret = ClassLoader.getSystemResourceAsStream(fileName); + } + } + + return ret; + } + + private void close(InputStream str, String filename) { + if (str != null) { + try { + str.close(); + } catch (IOException excp) { + LOG.error("Error while closing file: [{}]", filename, excp); + } + } + } +} diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/server/RangerTagSyncServer.java b/tagsync/src/main/java/org/apache/ranger/tagsync/server/RangerTagSyncServer.java new file mode 100644 index 00000000000..ef6ab291c4f --- /dev/null +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/server/RangerTagSyncServer.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.tagsync.server; + +import org.apache.ranger.tagsync.process.TagSyncConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public final class RangerTagSyncServer { + private static final Logger LOG = LoggerFactory.getLogger(RangerTagSyncServer.class); + + private RangerTagSyncServer() { + //to stop instantiation + } + + public static void main(String[] args) { + LOG.info("==>> RangerTagSyncServer.main()"); + + TagSyncConfig config = TagSyncConfig.getInstance(); + try { + EmbeddedServer server = new EmbeddedServer(config, "ranger-tagsync", "ranger.tagsync."); + server.start(); + } catch (Throwable e) { + LOG.error("Failed to initialize embedded server due to: ", e); + System.exit(1); + } + } +} diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java index b8756fdf681..030e22b883e 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java @@ -39,14 +39,12 @@ import java.util.Map; public final class AtlasNotificationMapper { - private static final Logger LOG = LoggerFactory.getLogger(AtlasNotificationMapper.class); - - private AtlasNotificationMapper() {} - + private static final Logger LOG = LoggerFactory.getLogger(AtlasNotificationMapper.class); private static final int REPORTING_INTERVAL_FOR_UNHANDLED_ENTITYTYPE_IN_MILLIS = 5 * 60 * 1000; // 5 minutes - private static final Map unhandledEventTypes = new HashMap<>(); + private AtlasNotificationMapper() {} + public static void logUnhandledEntityNotification(EntityNotificationWrapper entityNotification) { boolean skipLogging = entityNotification.getIsEntityCreateOp() && entityNotification.getIsEmptyClassifications(); @@ -58,8 +56,7 @@ public static void logUnhandledEntityNotification(EntityNotificationWrapper enti if (entityTypeName != null) { Long timeInMillis = unhandledEventTypes.get(entityTypeName); long currentTimeInMillis = System.currentTimeMillis(); - if (timeInMillis == null || - (currentTimeInMillis - timeInMillis) >= REPORTING_INTERVAL_FOR_UNHANDLED_ENTITYTYPE_IN_MILLIS) { + if (timeInMillis == null || (currentTimeInMillis - timeInMillis) >= REPORTING_INTERVAL_FOR_UNHANDLED_ENTITYTYPE_IN_MILLIS) { unhandledEventTypes.put(entityTypeName, currentTimeInMillis); loggingNeeded = true; } @@ -126,6 +123,7 @@ public static boolean isNotificationHandled(EntityNotificationWrapper entityNoti } if (!ret) { LOG.debug("Notification : [{}] will NOT be processed.", entityNotification); + AtlasTagSource.isTagAssociated = false; } } @@ -220,6 +218,7 @@ private static ServiceTags buildServiceTags(RangerAtlasEntityWithTags entityWith } } else { LOG.debug("Entity {} does not have any tags associated with it", entityWithTags); + AtlasTagSource.isTagAssociated = false; } ret.getResourceToTagIds().put(serviceResource.getId(), tagIds); diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasTagSource.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasTagSource.java index 95019a98ff5..b03127273ad 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasTagSource.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasTagSource.java @@ -30,9 +30,11 @@ import org.apache.commons.collections.MapUtils; import org.apache.commons.configuration.Configuration; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.util.StopWatch; import org.apache.kafka.common.TopicPartition; import org.apache.ranger.authorization.utils.JsonUtils; import org.apache.ranger.plugin.util.ServiceTags; +import org.apache.ranger.tagsync.metrics.source.RangerTagSyncMetricsSourceTags; import org.apache.ranger.tagsync.model.AbstractTagSource; import org.apache.ranger.tagsync.process.TagSyncConfig; import org.apache.ranger.tagsync.source.atlasrest.RangerAtlasEntityWithTags; @@ -44,6 +46,7 @@ import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.concurrent.TimeUnit; public class AtlasTagSource extends AbstractTagSource { private static final Logger LOG = LoggerFactory.getLogger(AtlasTagSource.class); @@ -52,6 +55,7 @@ public class AtlasTagSource extends AbstractTagSource { public static final String TAGSYNC_ATLAS_KAFKA_ENDPOINTS = "atlas.kafka.bootstrap.servers"; public static final String TAGSYNC_ATLAS_CONSUMER_GROUP = "atlas.kafka.entities.group.id"; public static final int MAX_WAIT_TIME_IN_MILLIS = 1000; + protected static boolean isTagAssociated = true; private int maxBatchSize; private ConsumerRunnable consumerTask; @@ -199,8 +203,10 @@ public void run() { while (true) { if (TagSyncConfig.isTagSyncServiceActive()) { LOG.debug("==> ConsumerRunnable.run() is running as server is active"); + StopWatch eventStopWatch = null; try { List> newMessages = consumer.receive(MAX_WAIT_TIME_IN_MILLIS); + eventStopWatch = new StopWatch().start(); if (newMessages.isEmpty()) { LOG.debug("AtlasTagSource.ConsumerRunnable.run: no message from NotificationConsumer within {} milliseconds", MAX_WAIT_TIME_IN_MILLIS); @@ -209,6 +215,8 @@ public void run() { buildAndUploadServiceTags(); } } else { + isTagAssociated = true; + RangerTagSyncMetricsSourceTags.updateTotalEvents(Long.valueOf(newMessages.size())); for (AtlasKafkaMessage message : newMessages) { EntityNotification notification = message != null ? message.getMessage() : null; @@ -245,6 +253,12 @@ public void run() { buildAndUploadServiceTags(); } } + eventStopWatch.stop(); + long timeElapsed = TimeUnit.MILLISECONDS.convert(eventStopWatch.now(), TimeUnit.NANOSECONDS); + RangerTagSyncMetricsSourceTags.updateTotalEventsTime(timeElapsed); + if (AtlasTagSource.isTagAssociated) { + RangerTagSyncMetricsSourceTags.updateTotalUploadsTime(timeElapsed); + } if (lastUnhandledMessage != null) { commitToKafka(lastUnhandledMessage); lastUnhandledMessage = null; @@ -259,6 +273,10 @@ public void run() { LOG.error("Returning from thread. May cause process to be up but not processing events!!"); return; } + } finally { + if (eventStopWatch != null) { + eventStopWatch.close(); + } } } else { try { diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java index c2547a39fd9..7c628f6e32c 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java @@ -35,9 +35,11 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.util.StopWatch; import org.apache.ranger.authorization.utils.JsonUtils; import org.apache.ranger.plugin.model.RangerValiditySchedule; import org.apache.ranger.plugin.util.ServiceTags; +import org.apache.ranger.tagsync.metrics.source.RangerTagSyncMetricsSourceTags; import org.apache.ranger.tagsync.model.AbstractTagSource; import org.apache.ranger.tagsync.model.TagSink; import org.apache.ranger.tagsync.process.TagSyncConfig; @@ -59,6 +61,7 @@ import java.util.Properties; import java.util.Set; import java.util.TimeZone; +import java.util.concurrent.TimeUnit; public class AtlasRESTTagSource extends AbstractTagSource implements Runnable { private static final Logger LOG = LoggerFactory.getLogger(AtlasRESTTagSource.class); @@ -71,11 +74,11 @@ public class AtlasRESTTagSource extends AbstractTagSource implements Runnable { return dateFormat; }); - private long sleepTimeBetweenCycleInMillis; + private long sleepTimeBetweenCycleInMillis; private AtlasRESTHttpClient atlasRestClient; - private boolean isKerberized; - private String[] userNamePassword; - private int entitiesBatchSize = TagSyncConfig.DEFAULT_TAGSYNC_ATLASREST_SOURCE_ENTITIES_BATCH_SIZE; + private boolean isKerberized; + private String[] userNamePassword; + private int entitiesBatchSize = TagSyncConfig.DEFAULT_TAGSYNC_ATLASREST_SOURCE_ENTITIES_BATCH_SIZE; private Thread myThread; @@ -174,6 +177,7 @@ public void run() { LOG.debug("==> AtlasRESTTagSource.run()"); while (true) { + StopWatch eventStopWatchRest = new StopWatch().start(); try { if (TagSyncConfig.isTagSyncServiceActive()) { LOG.debug("==> AtlasRESTTagSource.run() is running as server is Active"); @@ -183,6 +187,11 @@ public void run() { LOG.debug("==> This server is running passive mode"); } LOG.debug("Sleeping for [{}] milliSeconds", sleepTimeBetweenCycleInMillis); + eventStopWatchRest.stop(); + long timeElapsed = TimeUnit.MILLISECONDS.convert( + eventStopWatchRest.now(), TimeUnit.NANOSECONDS); + RangerTagSyncMetricsSourceTags.updateTotalUploadsTime( + timeElapsed); Thread.sleep(sleepTimeBetweenCycleInMillis); } catch (InterruptedException exception) { @@ -191,6 +200,10 @@ public void run() { } catch (Exception e) { LOG.error("Caught exception", e); return; + } finally { + if (eventStopWatchRest != null) { + eventStopWatchRest.close(); + } } } } @@ -213,7 +226,7 @@ public void synchUp() throws Exception { String serviceTagsString = JsonUtils.objectToJson(entry.getValue()); LOG.debug("serviceTags={}", serviceTagsString); } catch (Exception e) { - LOG.error("An error occurred while conveting serviceTags to string", e); + LOG.error("An error occurred while converting serviceTags to string", e); } } updateSink(entry.getValue()); @@ -288,7 +301,7 @@ private List getAtlasActiveEntities() { if (CollectionUtils.isNotEmpty(entityHeaders)) { nextStartIndex += entityHeaders.size(); - isMoreData = true; + isMoreData = true; for (AtlasEntityHeader header : entityHeaders) { if (!header.getStatus().equals(AtlasEntity.Status.ACTIVE)) { diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/file/FileTagSource.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/file/FileTagSource.java index e9f920c80be..d27441043c9 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/file/FileTagSource.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/file/FileTagSource.java @@ -21,8 +21,10 @@ import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.util.StopWatch; import org.apache.ranger.authorization.utils.JsonUtils; import org.apache.ranger.plugin.util.ServiceTags; +import org.apache.ranger.tagsync.metrics.source.RangerTagSyncMetricsSourceTags; import org.apache.ranger.tagsync.model.AbstractTagSource; import org.apache.ranger.tagsync.model.TagSink; import org.apache.ranger.tagsync.process.TagSyncConfig; @@ -42,6 +44,7 @@ import java.nio.charset.StandardCharsets; import java.util.Date; import java.util.Properties; +import java.util.concurrent.TimeUnit; public class FileTagSource extends AbstractTagSource implements Runnable { private static final Logger LOG = LoggerFactory.getLogger(FileTagSource.class); @@ -203,17 +206,23 @@ public void run() { LOG.debug("==> FileTagSource.run()"); while (true) { + StopWatch eventStopWatchFile = new StopWatch().start(); try { if (TagSyncConfig.isTagSyncServiceActive()) { LOG.debug("==> FileTagSource is running as server is active"); synchUp(); } + eventStopWatchFile.stop(); + long timeElapsed = TimeUnit.MILLISECONDS.convert(eventStopWatchFile.now(), TimeUnit.NANOSECONDS); + RangerTagSyncMetricsSourceTags.updateTotalUploadsTime(timeElapsed); } catch (Exception e) { LOG.error("Caught exception..", e); } finally { LOG.debug("Sleeping for [{}] milliSeconds", fileModTimeCheckIntervalInMs); - + if (eventStopWatchFile != null) { + eventStopWatchFile.close(); + } try { Thread.sleep(fileModTimeCheckIntervalInMs); } catch (InterruptedException exception) { diff --git a/tagsync/src/main/resources/hadoop-metrics2.properties b/tagsync/src/main/resources/hadoop-metrics2.properties new file mode 100644 index 00000000000..9899c102153 --- /dev/null +++ b/tagsync/src/main/resources/hadoop-metrics2.properties @@ -0,0 +1,64 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# syntax: [prefix].[source|sink].[instance].[options] +# See javadoc of package-info.java for org.apache.hadoop.metrics2 for details + +# syntax: [prefix].[source|sink].[instance].[options] +# See javadoc of package-info.java for org.apache.hadoop.metrics2 for details + +# default sampling period, in seconds +*.period=30 +#*.periodMillis=100 + +####### File Sink >>>>>> CONF +# prefix=* && context=* +#*.sink.file.class=org.apache.hadoop.metrics2.sink.FileSink +#*.sink.file.filename=${ranger.tagsync.log.dir}/${ranger.tagsync.metrics.file}-all-metrics.out + +# prefix=tagsync && context=* +#tagsync.sink.file.filename=${ranger.tagsync.log.dir}/${ranger.tagsync.metrics.file}-metrics.out + +# prefix=tagsync && context=auth +#tagsync.sink.file_auth.class=org.apache.hadoop.metrics2.sink.FileSink +#tagsync.sink.file_auth.context=auth +#tagsync.sink.file_auth.filename=${ranger.tagsync.log.dir}/${ranger.tagsync.metrics.file}-auth-metrics.out + +## prefix=tagsync && context=global +#tagsync.sink.file_global.class=org.apache.hadoop.metrics2.sink.FileSink +#tagsync.sink.file_global.context=global +#tagsync.sink.file_global.filename=${ranger.tagsync.log.dir}/${ranger.tagsync.metrics.file}-global-metrics.out + +## prefix=tagsync && context=adls +#tagsync.sink.file_adls.class=org.apache.hadoop.metrics2.sink.FileSink +#tagsync.sink.file_adls.context=adls +#tagsync.sink.file_adls.filename=${ranger.tagsync.log.dir}/${ranger.tagsync.metrics.file}-adls-metrics.out +####### File Sink <<<<<< CONF + +####### Graphite Sink >>>>>> CONF +## prefix=* && context=* +#*.sink.graphite.class=org.apache.hadoop.metrics2.sink.GraphiteSink +#*.sink.graphite.server_host=127.0.0.1 +#*.sink.graphite.server_port=2003 +#*.sink.graphite.metrics_prefix=tagsync + +## prefix=tagsync && context=auth +#tagsync.sink.graphite_tagsynccntx.class=org.apache.hadoop.metrics2.sink.GraphiteSink +#tagsync.sink.graphite_tagsynccntx.server_host=127.0.0.1 +#tagsync.sink.graphite_tagsynccntx.server_port=2003 +#tagsync.sink.graphite_tagsynccntx.metrics_prefix=tagsync +#tagsync.sink.graphite_tagsynccntx.context=auth +####### Graphite Sink <<<<<< CONF diff --git a/tagsync/src/main/resources/ranger-tagsync-default.xml b/tagsync/src/main/resources/ranger-tagsync-default.xml index 5a16f2546c8..fe25d64e20a 100644 --- a/tagsync/src/main/resources/ranger-tagsync-default.xml +++ b/tagsync/src/main/resources/ranger-tagsync-default.xml @@ -45,4 +45,14 @@ ranger.tagsync.dest.ranger.max.batch.size 1 + + + + ranger.valve.errorreportvalve.showserverinfo + false + + + ranger.valve.errorreportvalve.showreport + false + diff --git a/tagsync/src/main/webapp/WEB-INF/applicationContext.xml b/tagsync/src/main/webapp/WEB-INF/applicationContext.xml new file mode 100644 index 00000000000..f183c5f18ed --- /dev/null +++ b/tagsync/src/main/webapp/WEB-INF/applicationContext.xml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/tagsync/src/main/webapp/WEB-INF/security/spring-security.xml b/tagsync/src/main/webapp/WEB-INF/security/spring-security.xml new file mode 100644 index 00000000000..76f09c7773d --- /dev/null +++ b/tagsync/src/main/webapp/WEB-INF/security/spring-security.xml @@ -0,0 +1,56 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tagsync/src/main/webapp/WEB-INF/web.xml b/tagsync/src/main/webapp/WEB-INF/web.xml new file mode 100644 index 00000000000..92813370c13 --- /dev/null +++ b/tagsync/src/main/webapp/WEB-INF/web.xml @@ -0,0 +1,81 @@ + + + + + Apache Ranger - Tag Synchronizer + Apache Ranger - Tag Synchronizer + + + + contextConfigLocation + + /WEB-INF/applicationContext.xml + /WEB-INF/security/spring-security.xml + + + + + springSecurityFilterChain + org.springframework.web.filter.DelegatingFilterProxy + + + + springSecurityFilterChain + /* + + + + org.springframework.web.context.ContextLoaderListener + + + + org.springframework.web.context.request.RequestContextListener + + + + tagsync-apis + org.glassfish.jersey.servlet.ServletContainer + + jersey.config.server.provider.packages + org.apache.ranger.tagsync.rest + + + jersey.config.server.provider.classnames + + com.fasterxml.jackson.jaxrs.json.JacksonJaxbJsonProvider, + org.glassfish.jersey.server.spring.SpringComponentProvider + + + + + jersey.config.disableAutoDiscovery + true + + + + jersey.config.server.provider.scanning.recursive + false + + 1 + + + + tagsync-apis + /api/* + + diff --git a/tagsync/src/main/webapp/index.jsp b/tagsync/src/main/webapp/index.jsp new file mode 100644 index 00000000000..d4b59a4b49e --- /dev/null +++ b/tagsync/src/main/webapp/index.jsp @@ -0,0 +1,29 @@ + + +
+

This is secured!

+

+ Hello +

+ +
+ + +
+
+