Ranger 0.5

.gitignore

@@ -8,5 +8,5 @@
 /target/
 winpkg/target
 .DS_Store
-.gitignore
 .idea
+**/target/

NOTICE.txt

@@ -1,36 +1,7 @@
 Apache Ranger
-Copyright 2011-2013 The Apache Software Foundation
+Copyright 2014-2016 The Apache Software Foundation
 
 This product includes software developed at The Apache Software
 Foundation (http://www.apache.org/).
 
 
-This product includes jQuery (http://jquery.org - MIT license), Copyright © 2014, John Resig.  
-This product includes jQuery UI (http://jqueryui.com - MIT license), Copyright © 2013 jQuery Foundation.  
-This product includes Backbone (http://backbonejs.org - MIT license), Copyright © 2010-2014 Jeremy Ashkenas, DocumentCloud.
-This product includes underscore (http:underscorejs.org - MIT license), Copyright © 2009-2014 Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors.  
-This product includes Backbone.Marionette (http://marionettejs.com/ - MIT license).  
-This product includes Backbone.Wreqr (http://marionettejs.com/ - MIT license) Copyright ©,2012 Derick Bailey, Muted Solutions, LLC.
-This product includes Backbone.BabySitter (http://marionettejs.com/ - MIT license), Copyright ©2013 Derick Bailey, Muted Solutions, LLC.  
-This product includes Backbone fetch cache (https://github.com/mrappleton/backbone-fetch-cache - MIT license), Copyright © 2012-2013 Andrew Appleton.  
-This product includes Backbone-forms (https://github.com/powmedia/backbone-forms - MIT license), Copyright © 2013 Charles Davison.
-This product includes Backbone localStorage Adapter v1.1.7 (http://documentup.com/jeromegn/backbone.localStorage - MIT license), Copyright © 2010 Jerome Gravel-Niquet.
-This product includes Backbone.BootstrapModal (https://github.com/powmedia/backbone.bootstrap-modal - MIT license), Copyright © 2013 Charles Davison.  
-This product includes X-editable - v1.5.0 (http://vitalets.github.io/x-editable/ - MIT license), Copyright © 2013 Vitaliy Potapov;
-This product includes Bootstrap Datepicker (http://www.eyecon.ro/bootstrap-datepicker - Apache License - 2.0), Copyright 2012 Stefan Petre.  
-This product includes bootstrap-transition.js v2.3.2 (http://twbs.github.com/bootstrap/javascript.html#transitions - Apache License - 2.0), Copyright 2013 Twitter, Inc.  
-This product includes backgrid 0.3.5 (http://backgridjs.com/ - MIT license), Copyright © 2014 Jimmy Yuen Ho Wong and contributors.
-This product includes backgrid-paginator (http://github.com/wyuenho/backgrid - MIT license), Copyright © 2013 Jimmy Yuen Ho Wong and contributors.  This product includes backgrid-filter (http://github.com/wyuenho/backgrid - MIT license), Copyright © 2013 Jimmy Yuen Ho Wong and contributors.  
-This product includes backbone-pageable 1.3.2 (http://github.com/wyuenho/backbone-pageable - MIT license), Copyright © 2013 Jimmy Yuen Ho Wong.
-This product includes jQuery Cookie Plugin v1.4.0 (https://github.com/carhartl/jquery-cookie - MIT license), Copyright 2013 Klaus Hartl  
-This product includes jQuery Toggles v2.0.4 (http://simontabor.com/labs/toggles - MIT license), Copyright 2013 Simon Tabor.  
-This product includes jQuery UI Tag-it! (http://aehlke.github.com/tag-it/ - MIT license), Copyright 2011, Levy Carneiro Jr.
-This product includes Select2 3.4.3 (http://ivaynberg.github.io/select2/ - Apache License - 2.0), Copyright 2012 Igor Vaynberg.  
-This product includes bootbox.js v3.3.0 (http://bootboxjs.com/ - MIT license), Copyright © 2011-2014 by Nick Payne.  
-This product includes jQuery Pines Notify (pnotify) Plugin 1.2.0 (http://sciactive.com/pnotify/- MPL), Copyright © 2009-2012 Hunter Perrin.
-This product includes moment.js (http://moment.js - MIT license), Copyright © 2011-2014 Tim Wood, Iskren Chernev, Moment.js contributors.  
-This product includes VisualSearch.js 0.4.0 (http://documentcloud.github.com/visualsearch - MIT license), © 2011 Samuel Clay, @samuelclay, DocumentCloud Inc.  
-This product includes Globalize (http://github.com/jquery/globalize - MIT license), Copyright Software Freedom Conservancy, Inc.
-This product includes RequireJS 2.1.8 (http://github.com/jrburke/requirejs - New BSD or MIT licensed), Copyright © 2010-2012, The Dojo Foundation All Rights Reserved. 
-This product includes Handlebars.js (http://handlebarsjs.com/ - MIT license), Copyright © 2011-2014 by Yehuda Katz.  
-This product includes Require.js Handlebars Plugin (https://github.com/SlexAxton/require-handlebars-plugin - WTFPL + requirejs + handlebars), Copyright © 2011-2014 by Yehuda Katz.

agents-audit/pom.xml

@@ -17,7 +17,6 @@
 -->
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
-  <groupId>security_plugins.ranger-plugins-audit</groupId>
   <artifactId>ranger-plugins-audit</artifactId>
   <name>Audit Component</name>
   <description>Auth Audit</description>
@@ -28,12 +27,12 @@
   <parent>
      <groupId>org.apache.ranger</groupId>
      <artifactId>ranger</artifactId>
-     <version>0.5.0</version>
+     <version>0.5.4-SNAPSHOT</version>
      <relativePath>..</relativePath>
   </parent>
   <dependencies>
     <dependency>
-	<groupId>security_plugins.ranger-plugins-cred</groupId>
+	<groupId>org.apache.ranger</groupId>
 	<artifactId>ranger-plugins-cred</artifactId>
         <version>${project.version}</version>
     </dependency>

agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java

@@ -33,6 +33,7 @@
 import javax.persistence.Persistence;
 
 import org.apache.ranger.audit.dao.DaoManager;
+import org.apache.ranger.audit.entity.AuthzAuditEventDbObj;
 import org.apache.ranger.audit.model.AuditEventBase;
 import org.apache.ranger.audit.provider.MiscUtil;
 
@@ -63,9 +64,11 @@ public DBAuditDestination() {
 	public void init(Properties props, String propPrefix) {
 		logger.info("init() called");
 		super.init(props, propPrefix);
-
 		// Initial connect
 		connect();
+
+		// initialize the database related classes
+		AuthzAuditEventDbObj.init(props);
 	}
 
 	/*

agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java

@@ -23,12 +23,9 @@
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.net.URI;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Date;
-import java.util.List;
-import java.util.Properties;
+import java.util.*;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -99,7 +96,7 @@ public void init(Properties prop, String propPrefix) {
 		logger.info("logFolder=" + logFolder + ", destName=" + getName());
 		logger.info("logFileNameFormat=" + logFileNameFormat + ", destName="
 				+ getName());
-
+		logger.info("config=" + configProps.toString());
 		initDone = true;
 	}
 
@@ -127,7 +124,15 @@ synchronized public boolean logJSON(Collection<String> events) {
 			for (String event : events) {
 				out.println(event);
 			}
-			out.flush();
+			// flush and check the stream for errors
+			if (out.checkError()) {
+				// In theory, this count may NOT be accurate as part of the messages may have been successfully written.
+				// However, in practice, since client does buffering, either all of none would succeed.
+				addDeferredCount(events.size());
+				out.close();
+				logWriter = null;
+				return false;
+			}
 		} catch (Throwable t) {
 			addDeferredCount(events.size());
 			logError("Error writing to log file.", t);
@@ -207,7 +212,7 @@ synchronized private PrintWriter getLogFileStream() throws Throwable {
 					currentTime.getTime());
 			String parentFolder = MiscUtil.replaceTokens(logFolder,
 					currentTime.getTime());
-			Configuration conf = new Configuration();
+			Configuration conf = createConfiguration();
 
 			String fullPath = parentFolder
 					+ org.apache.hadoop.fs.Path.SEPARATOR + fileName;
@@ -216,7 +221,7 @@ synchronized private PrintWriter getLogFileStream() throws Throwable {
 			FileSystem fileSystem = FileSystem.get(uri, conf);
 
 			Path hdfPath = new Path(fullPath);
-			logger.info("Checking whether log file exists. hdfPath=" + fullPath);
+			logger.info("Checking whether log file exists. hdfPath=" + fullPath + ", UGI=" + MiscUtil.getUGILoginUser());
 			int i = 0;
 			while (fileSystem.exists(hdfPath)) {
 				i++;
@@ -243,6 +248,22 @@ synchronized private PrintWriter getLogFileStream() throws Throwable {
 		return logWriter;
 	}
 
+	Configuration createConfiguration() {
+		Configuration conf = new Configuration();
+		for (Map.Entry<String, String> entry : configProps.entrySet()) {
+			String key = entry.getKey();
+			String value = entry.getValue();
+			// for ease of install config file may contain properties with empty value, skip those
+			if (StringUtils.isNotEmpty(value)) {
+				conf.set(key, value);
+			}
+			logger.info("Adding property to HDFS config: " + key + " => " + value);
+		}
+
+		logger.info("Returning HDFS Filesystem Config: " + conf.toString());
+		return conf;
+	}
+
 	private void createParents(Path pathLogfile, FileSystem fileSystem)
 			throws Throwable {
 		logger.info("Creating parent folder for " + pathLogfile);

agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java

@@ -43,7 +43,9 @@ public class SolrAuditDestination extends AuditDestination {
 
 	public static final String PROP_SOLR_URLS = "urls";
 	public static final String PROP_SOLR_ZK = "zookeepers";
+	public static final String PROP_SOLR_COLLECTION = "collection";
 
+	public static final String DEFAULT_COLLECTION_NAME = "ranger_audits";
 	SolrClient solrClient = null;
 
 	public SolrAuditDestination() {
@@ -55,7 +57,7 @@ public void init(Properties props, String propPrefix) {
 		super.init(props, propPrefix);
 		connect();
 	}
-	
+
 	@Override
 	public void stop() {
 		super.stop();
@@ -67,13 +69,12 @@ synchronized void connect() {
 			if (solrClient == null) {
 				String urls = MiscUtil.getStringProperty(props, propPrefix
 						+ "." + PROP_SOLR_URLS);
-				if( urls != null) {
+				if (urls != null) {
 					urls = urls.trim();
 				}
 				if (urls != null && urls.equalsIgnoreCase("NONE")) {
 					urls = null;
 				}
-
 
 				List<String> solrURLs = new ArrayList<String>();
 				String zkHosts = null;
@@ -84,11 +85,32 @@ synchronized void connect() {
 					zkHosts = null;
 				}
 
-				try {
-					if (zkHosts != null && !zkHosts.isEmpty()) {
+				String collectionName = MiscUtil.getStringProperty(props,
+						propPrefix + "." + PROP_SOLR_COLLECTION);
+				if (collectionName == null
+						|| collectionName.equalsIgnoreCase("none")) {
+					collectionName = DEFAULT_COLLECTION_NAME;
+				}
+
+				LOG.info("Solr zkHosts=" + zkHosts + ", solrURLs=" + urls
+						+ ", collectionName=" + collectionName);
+
+				if (zkHosts != null && !zkHosts.isEmpty()) {
+					LOG.info("Connecting to solr cloud using zkHosts="
+							+ zkHosts);
+					try {
 						// Instantiate
-						solrClient = new CloudSolrClient(zkHosts);
-					} else if (solrURLs != null && !solrURLs.isEmpty()) {
+						CloudSolrClient solrCloudClient = new CloudSolrClient(
+								zkHosts);
+						solrCloudClient.setDefaultCollection(collectionName);
+						solrClient = solrCloudClient;
+					} catch (Throwable t) {
+						LOG.fatal("Can't connect to Solr server. ZooKeepers="
+								+ zkHosts, t);
+					}
+				} else if (solrURLs != null && !solrURLs.isEmpty()) {
+					try {
+						LOG.info("Connecting to Solr using URLs=" + solrURLs);
 						LBHttpSolrClient lbSolrClient = new LBHttpSolrClient(
 								solrURLs.get(0));
 						lbSolrClient.setConnectionTimeout(1000);
@@ -97,10 +119,10 @@ synchronized void connect() {
 							lbSolrClient.addSolrServer(solrURLs.get(i));
 						}
 						solrClient = lbSolrClient;
+					} catch (Throwable t) {
+						LOG.fatal("Can't connect to Solr server. URL="
+								+ solrURLs, t);
 					}
-				} catch (Throwable t) {
-					LOG.fatal("Can't connect to Solr server. URL=" + solrURLs,
-							t);
 				}
 			}
 		}
@@ -111,7 +133,7 @@ public boolean log(Collection<AuditEventBase> events) {
 		try {
 			logStatusIfRequired();
 			addTotalCount(events.size());
-			
+
 			if (solrClient == null) {
 				connect();
 				if (solrClient == null) {