From ca197a4d89507138925aadd8f3887693e2870cd4 Mon Sep 17 00:00:00 2001 From: frbattid Date: Wed, 27 Jan 2016 10:29:18 +0100 Subject: [PATCH 1/7] ADD OAuth2-based authentication provider for Hive --- CHANGES_NEXT_RELEASE | 1 + cosmos-hive-auth-provider/pom.xml | 43 +++++ .../hive/authprovider/HttpClientFactory.java | 152 ++++++++++++++++++ .../OAuth2AuthenticationProviderImpl.java | 102 ++++++++++++ .../OAuth2AuthenticationProviderImplTest.java | 88 ++++++++++ 5 files changed, 386 insertions(+) create mode 100644 cosmos-hive-auth-provider/pom.xml create mode 100644 cosmos-hive-auth-provider/src/main/java/com/telefonica/iot/cosmos/hive/authprovider/HttpClientFactory.java create mode 100644 cosmos-hive-auth-provider/src/main/java/com/telefonica/iot/cosmos/hive/authprovider/OAuth2AuthenticationProviderImpl.java create mode 100644 cosmos-hive-auth-provider/src/test/java/com/telefonica/iot/cosmos/hive/authprovider/OAuth2AuthenticationProviderImplTest.java diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index acaf37a..6c003b7 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -17,3 +17,4 @@ - [cosmos-auth] [HARDENING] Force the usage of TLSv1 instead of SSLv3 (#107) - [cosmos-gui] [BUG] /new_password route now provisions the new password (#134) - [doc] [HARDENING] Document how Cosmos has been deployed in FIWARE Lab (#141) +- [cosmos-hive-auth-provider] [FEATURE] Add an OAuth2-based authentication provider for Hive (#57) diff --git a/cosmos-hive-auth-provider/pom.xml b/cosmos-hive-auth-provider/pom.xml new file mode 100644 index 0000000..d13878e --- /dev/null +++ b/cosmos-hive-auth-provider/pom.xml @@ -0,0 +1,43 @@ + + + 4.0.0 + com.telefonica.iot + cosmos-hive-auth-provider + 0.0.0-SNAPSHOT + jar + + + UTF-8 + 1.7 + 1.7 + + + + + org.apache.hive + hive-service + 0.13.0 + + + org.apache.httpcomponents + httpclient + 4.2.1 + + + log4j + log4j + 1.2.17 + + + com.googlecode.json-simple + json-simple + 1.1 + + + junit + junit + 4.11 + test + + + \ No newline at end of file diff --git a/cosmos-hive-auth-provider/src/main/java/com/telefonica/iot/cosmos/hive/authprovider/HttpClientFactory.java b/cosmos-hive-auth-provider/src/main/java/com/telefonica/iot/cosmos/hive/authprovider/HttpClientFactory.java new file mode 100644 index 0000000..2ef442e --- /dev/null +++ b/cosmos-hive-auth-provider/src/main/java/com/telefonica/iot/cosmos/hive/authprovider/HttpClientFactory.java @@ -0,0 +1,152 @@ +/** + * Copyright 2016 Telefonica Investigación y Desarrollo, S.A.U + * + * This file is part of fiware-cosmos (FI-WARE project). + * + * fiware-cosmos is free software: you can redistribute it and/or modify it under the terms of the GNU Affero + * General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your + * option) any later version. + * fiware-cosmos is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the + * implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License + * for more details. + * + * You should have received a copy of the GNU Affero General Public License along with fiware-cosmos. If not, see + * http://www.gnu.org/licenses/. + * + * For those usages not covered by the GNU Affero General Public License please contact with iot_support at tid dot es + */ +package com.telefonica.iot.cosmos.hive.authprovider; + +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.security.cert.X509Certificate; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; +import org.apache.http.conn.ssl.SSLSocketFactory; +import org.apache.http.conn.scheme.Scheme; +import org.apache.http.conn.scheme.SchemeRegistry; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.impl.conn.PoolingClientConnectionManager; +import org.apache.log4j.Logger; + +/** + * + * @author frb + * + * This is a convenience class in order the clients do not have to import stuff from org.apache.http and deal with its + * details. It implements a Http client factory. + * + */ +public class HttpClientFactory { + + private static final Logger LOGGER = Logger.getLogger(HttpClientFactory.class); + public static final int MAX_CONNS = 500; + public static final int MAX_CONNS_PER_ROUTE = 100; + private static PoolingClientConnectionManager connectionsManager; + private static PoolingClientConnectionManager sslConnectionsManager; + + /** + * Constructor. + * @param ssl True if SSL connections are desired. False otherwise. + */ + public HttpClientFactory(boolean ssl) { + // create the appropriate connections manager + if (ssl) { + sslConnectionsManager = new PoolingClientConnectionManager(getSSLSchemeRegistry()); + sslConnectionsManager.setMaxTotal(MAX_CONNS); + sslConnectionsManager.setDefaultMaxPerRoute(MAX_CONNS_PER_ROUTE); + } else { + connectionsManager = new PoolingClientConnectionManager(); + connectionsManager.setMaxTotal(MAX_CONNS); + connectionsManager.setDefaultMaxPerRoute(MAX_CONNS_PER_ROUTE); + } // if else + + LOGGER.info("Setting max total connections (" + MAX_CONNS + ")"); + LOGGER.info("Setting default max connections per route (" + MAX_CONNS_PER_ROUTE + ")"); + } // HttpClientFactory + + /** + * Gets a HTTP client. + * @param ssl True if SSL connections are desired. False otherwise + * @return A http client obtained from the (SSL) Connections Manager. + */ + public DefaultHttpClient getHttpClient(boolean ssl) { + DefaultHttpClient httpClient; + + if (ssl) { + httpClient = new DefaultHttpClient(sslConnectionsManager); + } else { + httpClient = new DefaultHttpClient(connectionsManager); + } // if else + + return httpClient; + } // getHttpClient + + /** + * Gets the number of leased connections for this connections manager. This is not really used within the code, but + * could be used for debugging purposes. + * @param ssl + * @return + */ + public int getLeasedConnections(boolean ssl) { + if (ssl) { + return sslConnectionsManager.getTotalStats().getLeased(); + } else { + return connectionsManager.getTotalStats().getLeased(); + } // if else + } // getLeasedConnections + + /** + * Gets a SSL SchemeRegistry object accepting all the X509 certificates by default. + * @return A SSL SchemeRegistry object. + */ + private SchemeRegistry getSSLSchemeRegistry() { + // http://stackoverflow.com/questions/2703161/how-to-ignore-ssl-certificate-errors-in-apache-httpclient-4-0 + + SSLContext sslContext; + + try { + sslContext = SSLContext.getInstance("SSL"); + } catch (NoSuchAlgorithmException e) { + LOGGER.fatal("Fatal error (SSL cannot be used, no such algorithm. Details=" + e.getMessage() + ")"); + return null; + } // try catch // try catch + + try { + // set up a TrustManager that trusts everything + sslContext.init(null, new TrustManager[] { + new X509TrustManager() { + @Override + public X509Certificate[] getAcceptedIssuers() { + return null; + } // getAcceptedIssuers + + @Override + public void checkClientTrusted(X509Certificate[] certs, String authType) { + } // getAcceptedIssuers + + @Override + public void checkServerTrusted(X509Certificate[] certs, String authType) { + } // checkServerTrusted + } + }, new SecureRandom()); + } catch (KeyManagementException e) { + LOGGER.fatal("Fatal error (Cannot ignore SSL certificates. Details=" + e.getMessage() + ")"); + return null; + } // try catch // try catch + + if (sslContext == null) { + LOGGER.fatal("Fatal error (Cannot ignore SSL certificates, SSL context is null)"); + return null; + } // if + + SSLSocketFactory sf = new SSLSocketFactory(sslContext, SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER); + Scheme httpsScheme = new Scheme("https", 443, sf); + SchemeRegistry schemeRegistry = new SchemeRegistry(); + schemeRegistry.register(httpsScheme); + return schemeRegistry; + } // getSSLSchemeRegistry + +} // HttpClientFactory diff --git a/cosmos-hive-auth-provider/src/main/java/com/telefonica/iot/cosmos/hive/authprovider/OAuth2AuthenticationProviderImpl.java b/cosmos-hive-auth-provider/src/main/java/com/telefonica/iot/cosmos/hive/authprovider/OAuth2AuthenticationProviderImpl.java new file mode 100644 index 0000000..3a06b1e --- /dev/null +++ b/cosmos-hive-auth-provider/src/main/java/com/telefonica/iot/cosmos/hive/authprovider/OAuth2AuthenticationProviderImpl.java @@ -0,0 +1,102 @@ +/** + * Copyright 2016 Telefonica Investigación y Desarrollo, S.A.U + * + * This file is part of fiware-cosmos (FI-WARE project). + * + * fiware-cosmos is free software: you can redistribute it and/or modify it under the terms of the GNU Affero + * General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your + * option) any later version. + * fiware-cosmos is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the + * implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License + * for more details. + * + * You should have received a copy of the GNU Affero General Public License along with fiware-cosmos. If not, see + * http://www.gnu.org/licenses/. + * + * For those usages not covered by the GNU Affero General Public License please contact with iot_support at tid dot es + */ +package com.telefonica.iot.cosmos.hive.authprovider; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import javax.security.sasl.AuthenticationException; +import org.apache.hive.service.auth.PasswdAuthenticationProvider; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpRequestBase; +import org.apache.log4j.Logger; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; + +/** + * Custom implementation of the org.apache.hive.service.auth.PasswdAuthenticationProvider interface. + * + * @author frb + */ +public class OAuth2AuthenticationProviderImpl implements PasswdAuthenticationProvider { + + private static final Logger LOGGER = Logger.getLogger(HttpClientFactory.class); + private final HttpClientFactory httpClientFactory; + + /** + * Constructor. + */ + public OAuth2AuthenticationProviderImpl() { + httpClientFactory = new HttpClientFactory(true); + } // OAuth2AuthenticationProviderImpl + + @Override + public void Authenticate(String user, String password) throws AuthenticationException { + // create the Http client + HttpClient httpClient = httpClientFactory.getHttpClient(true); + + // create the request + String url = "https://account.lab.fiware.org/user?access_token=" + password; + HttpRequestBase request = new HttpGet(url); + + // do the request + HttpResponse httpRes = null; + + try { + httpRes = httpClient.execute(request); + LOGGER.debug("Doing request: " + request.toString()); + } catch (IOException e) { + throw new AuthenticationException(e.getMessage()); + } // try catch + + // get the input streamResponse + String streamResponse = ""; + + try { + BufferedReader reader = new BufferedReader(new InputStreamReader(httpRes.getEntity().getContent())); + streamResponse = reader.readLine(); + LOGGER.debug("Response received: " + streamResponse); + } catch (IOException e) { + throw new AuthenticationException(e.getMessage()); + } // try catch + + // parse the input streamResponse as a Json + JSONObject jsonResponse = null; + + try { + JSONParser jsonParser = new JSONParser(); + jsonResponse = (JSONObject) jsonParser.parse(streamResponse); + } catch (ParseException e) { + throw new AuthenticationException(e.getMessage()); + } // try catch + + // check if the given token does not exist + if (jsonResponse.containsKey("error")) { + throw new AuthenticationException("The given token does not exist"); + } // if + + // check if the obtained user id matches the given user + if (jsonResponse.containsKey("id") && !jsonResponse.get("id").equals(user)) { + throw new AuthenticationException("The given token does not match the given user"); + } // if + } // Authenticate + +} // OAuth2AuthenticationProviderImpl diff --git a/cosmos-hive-auth-provider/src/test/java/com/telefonica/iot/cosmos/hive/authprovider/OAuth2AuthenticationProviderImplTest.java b/cosmos-hive-auth-provider/src/test/java/com/telefonica/iot/cosmos/hive/authprovider/OAuth2AuthenticationProviderImplTest.java new file mode 100644 index 0000000..cf7d97b --- /dev/null +++ b/cosmos-hive-auth-provider/src/test/java/com/telefonica/iot/cosmos/hive/authprovider/OAuth2AuthenticationProviderImplTest.java @@ -0,0 +1,88 @@ +/** + * Copyright 2016 Telefonica Investigación y Desarrollo, S.A.U + * + * This file is part of fiware-cosmos (FI-WARE project). + * + * fiware-cosmos is free software: you can redistribute it and/or modify it under the terms of the GNU Affero + * General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your + * option) any later version. + * fiware-cosmos is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the + * implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License + * for more details. + * + * You should have received a copy of the GNU Affero General Public License along with fiware-cosmos. If not, see + * http://www.gnu.org/licenses/. + * + * For those usages not covered by the GNU Affero General Public License please contact with iot_support at tid dot es + */ +package com.telefonica.iot.cosmos.hive.authprovider; + +import javax.security.sasl.AuthenticationException; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import org.junit.Test; + +/** + * Suite of tests for OAuth2AuthenticationProviderImpl. + * + * @author frb + */ +public class OAuth2AuthenticationProviderImplTest { + + private final String matchingUser = "frb"; + private final String notMatchingUser = "other"; + private final String matchingToken = "8azH35GGPdapmgVCODLOtxy52f6a8o"; + private final String notMatchingToken = "000000000000000000000000000000"; + + /** + * Tests OAuth2AuthenticationProviderImpl.Authenticate when the given user and token match. + */ + @Test + public void testAuthenticateSuccess() { + System.out.println("Testing OAuth2AuthenticationProviderImpl.Authenticate, it must success"); + + try { + OAuth2AuthenticationProviderImpl oapi = new OAuth2AuthenticationProviderImpl(); + oapi.Authenticate(matchingUser, matchingToken); + } catch (AuthenticationException e) { + fail(e.getMessage()); + } finally { + assertTrue(true); + } // try catch finally + } // testAuthenticateSuccess + + /** + * Tests OAuth2AuthenticationProviderImpl.Authenticate when the given user and token don't match. + */ + @Test + public void testAuthenticateFail() { + System.out.println("Testing OAuth2AuthenticationProviderImpl.Authenticate, " + + "it must fail because the user and the token don't match"); + + try { + OAuth2AuthenticationProviderImpl oapi = new OAuth2AuthenticationProviderImpl(); + oapi.Authenticate(notMatchingUser, matchingToken); + assertTrue(false); + } catch (AuthenticationException e) { + assertTrue(true); + } // try catch + } // testAuthenticateFail + + /** + * Tests OAuth2AuthenticationProviderImpl.Authenticate when the given token does not exist. + */ + @Test + public void testAuthenticateUnexistentToken() { + System.out.println("Testing OAuth2AuthenticationProviderImpl.Authenticate, " + + "it must fail because the token does not exist"); + + try { + OAuth2AuthenticationProviderImpl oapi = new OAuth2AuthenticationProviderImpl(); + oapi.Authenticate(matchingUser, notMatchingToken); + assertTrue(false); + } catch (AuthenticationException e) { + assertTrue(true); + } // try catch + } // testAuthenticateUnexistentToken + +} // OAuth2AuthenticationProviderImplTest From 6d6501fe47cdbb0a55abea484883430331c2a459 Mon Sep 17 00:00:00 2001 From: frbattid Date: Wed, 27 Jan 2016 10:34:45 +0100 Subject: [PATCH 2/7] ADD final line feed in pom.xml --- cosmos-hive-auth-provider/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cosmos-hive-auth-provider/pom.xml b/cosmos-hive-auth-provider/pom.xml index d13878e..0a8813e 100644 --- a/cosmos-hive-auth-provider/pom.xml +++ b/cosmos-hive-auth-provider/pom.xml @@ -40,4 +40,4 @@ test - \ No newline at end of file + From 07e4717773cd635b3dbe60c512e64bb8d2aefcba Mon Sep 17 00:00:00 2001 From: frbattid Date: Wed, 27 Jan 2016 11:10:36 +0100 Subject: [PATCH 3/7] FIX a typo --- .../hive/authprovider/OAuth2AuthenticationProviderImplTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cosmos-hive-auth-provider/src/test/java/com/telefonica/iot/cosmos/hive/authprovider/OAuth2AuthenticationProviderImplTest.java b/cosmos-hive-auth-provider/src/test/java/com/telefonica/iot/cosmos/hive/authprovider/OAuth2AuthenticationProviderImplTest.java index cf7d97b..784013e 100644 --- a/cosmos-hive-auth-provider/src/test/java/com/telefonica/iot/cosmos/hive/authprovider/OAuth2AuthenticationProviderImplTest.java +++ b/cosmos-hive-auth-provider/src/test/java/com/telefonica/iot/cosmos/hive/authprovider/OAuth2AuthenticationProviderImplTest.java @@ -31,7 +31,7 @@ public class OAuth2AuthenticationProviderImplTest { private final String matchingUser = "frb"; private final String notMatchingUser = "other"; - private final String matchingToken = "8azH35GGPdapmgVCODLOtxy52f6a8o"; + private final String matchingToken = "111111111111111111111111111111"; private final String notMatchingToken = "000000000000000000000000000000"; /** From a7fe14d6141644c7061aaf1d9a442e5c6ba1cc8d Mon Sep 17 00:00:00 2001 From: frbattid Date: Wed, 27 Jan 2016 16:14:04 +0100 Subject: [PATCH 4/7] FIX the httpclient version and add a successful log --- cosmos-hive-auth-provider/pom.xml | 20 ++++++++++++++++++- .../OAuth2AuthenticationProviderImpl.java | 9 +++++++-- .../OAuth2AuthenticationProviderImplTest.java | 2 +- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/cosmos-hive-auth-provider/pom.xml b/cosmos-hive-auth-provider/pom.xml index 0a8813e..562c8fe 100644 --- a/cosmos-hive-auth-provider/pom.xml +++ b/cosmos-hive-auth-provider/pom.xml @@ -21,7 +21,7 @@ org.apache.httpcomponents httpclient - 4.2.1 + 4.2.5 log4j @@ -40,4 +40,22 @@ test + + + + + maven-assembly-plugin + + + + com.telefonica.iot.cosmos.hive.authprovider.OAuth2AuthenticationProviderImpl + + + + jar-with-dependencies + + + + + diff --git a/cosmos-hive-auth-provider/src/main/java/com/telefonica/iot/cosmos/hive/authprovider/OAuth2AuthenticationProviderImpl.java b/cosmos-hive-auth-provider/src/main/java/com/telefonica/iot/cosmos/hive/authprovider/OAuth2AuthenticationProviderImpl.java index 3a06b1e..efe5985 100644 --- a/cosmos-hive-auth-provider/src/main/java/com/telefonica/iot/cosmos/hive/authprovider/OAuth2AuthenticationProviderImpl.java +++ b/cosmos-hive-auth-provider/src/main/java/com/telefonica/iot/cosmos/hive/authprovider/OAuth2AuthenticationProviderImpl.java @@ -49,12 +49,12 @@ public OAuth2AuthenticationProviderImpl() { } // OAuth2AuthenticationProviderImpl @Override - public void Authenticate(String user, String password) throws AuthenticationException { + public void Authenticate(String user, String token) throws AuthenticationException { // create the Http client HttpClient httpClient = httpClientFactory.getHttpClient(true); // create the request - String url = "https://account.lab.fiware.org/user?access_token=" + password; + String url = "https://account.lab.fiware.org/user?access_token=" + token; HttpRequestBase request = new HttpGet(url); // do the request @@ -97,6 +97,11 @@ public void Authenticate(String user, String password) throws AuthenticationExce if (jsonResponse.containsKey("id") && !jsonResponse.get("id").equals(user)) { throw new AuthenticationException("The given token does not match the given user"); } // if + + // release the connection + request.releaseConnection(); + + LOGGER.debug("User " + user + " authenticated"); } // Authenticate } // OAuth2AuthenticationProviderImpl diff --git a/cosmos-hive-auth-provider/src/test/java/com/telefonica/iot/cosmos/hive/authprovider/OAuth2AuthenticationProviderImplTest.java b/cosmos-hive-auth-provider/src/test/java/com/telefonica/iot/cosmos/hive/authprovider/OAuth2AuthenticationProviderImplTest.java index 784013e..d6b4764 100644 --- a/cosmos-hive-auth-provider/src/test/java/com/telefonica/iot/cosmos/hive/authprovider/OAuth2AuthenticationProviderImplTest.java +++ b/cosmos-hive-auth-provider/src/test/java/com/telefonica/iot/cosmos/hive/authprovider/OAuth2AuthenticationProviderImplTest.java @@ -31,7 +31,7 @@ public class OAuth2AuthenticationProviderImplTest { private final String matchingUser = "frb"; private final String notMatchingUser = "other"; - private final String matchingToken = "111111111111111111111111111111"; + private final String matchingToken = "111111111111111111111111111111"; private final String notMatchingToken = "000000000000000000000000000000"; /** From 75bd13a090b5ba907e04c558175c8df1118592bd Mon Sep 17 00:00:00 2001 From: frbattid Date: Thu, 28 Jan 2016 11:34:16 +0100 Subject: [PATCH 5/7] ADD documentation about the OAuth2-based auth provider for Hive --- README.md | 2 + cosmos-hive-auth-provider/README.md | 256 ++++++++++++++++++++++++++++ 2 files changed, 258 insertions(+) create mode 100644 cosmos-hive-auth-provider/README.md diff --git a/README.md b/README.md index 2f7abec..6ff3a61 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ Cosmos comprises several different sub-projects: * A set of administration tools such as HDFS data copiers and much more, under [cosmos-admin](/cosmos-admin) folder. * An OAuth2 tokens generator, under [cosmos-auth](cosmos-auth/) folder. * A web portal for users and accounts management, running MapReduce jobs and doing I/O of big data, under [cosmos-gui](cosmos-gui/) folder. +* A custom authentication provider for Hive, under [cosmos-hive-auth-provider](cosmos-hive-auth-provider). ##Reporting issues and contact information There are several channels suited for reporting issues and asking for doubts in general. Each one depends on the nature of the question: @@ -21,5 +22,6 @@ There are several channels suited for reporting issues and asking for doubts in * [francisco.romerobueno@telefonica.com](mailto:francisco.romerobueno@telefonica.com) **[Main contributor]** * [fermin.galanmarquez@telefonica.com](mailto:fermin.galanmarquez@telefonica.com) **[Contributor]** * [german.torodelvalle@telefonica.com](german.torodelvalle@telefonica.com) **[Contributor]** + * [pablo.coellovillalba@telefonica.com](pablo.coellovillalba@telefonica.com) **[Contributor]** **NOTE**: Please try to avoid personaly emailing the contributors unless they ask for it. In fact, if you send a private email you will probably receive an automatic response enforcing you to use [stackoverflow.com](stackoverflow.com) or [fiware-tech-help@lists.fi-ware.org](mailto:fiware-tech-help@lists.fi-ware.org). This is because using the mentioned methods will create a public database of knowledge that can be useful for future users; private email is just private and cannot be shared. \ No newline at end of file diff --git a/cosmos-hive-auth-provider/README.md b/cosmos-hive-auth-provider/README.md new file mode 100644 index 0000000..fc452f7 --- /dev/null +++ b/cosmos-hive-auth-provider/README.md @@ -0,0 +1,256 @@ +#OAuth2-based authentication provider for Cosmos +* [What is cosmos-hive-auth-provider](#section1) +* [Installation](#section2) + * [Prerequisites](#section2.1) + * [Building](#section2.2) + * [Jar copying](#section2.3) + * [Unit tests](#section2.4) +* [Configuration](#section3) +* [Running](#section4) +* [Usage](#section5) +* [Administration](#section6) +* [Reporting issues and contact information](#section7) + +##What is cosmos-hive-auth-provider +cosmos-hive-auth-provider is a custom authentication provider for [Hive](https://hive.apache.org/). Hive natively provides many ways of implementing authentication, e.g. [Kerberos, PAM or LDAP](https://cwiki.apache.org/confluence/display/Hive/Setting+Up+HiveServer2#SettingUpHiveServer2-Authentication/SecurityConfiguration), but it also allows for configuring custom mechanisms, like this one. + +By using cosmos-hive-auth-provider users will be able to authenticate by means or their [OAuth2](https://www.google.es/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&ved=0ahUKEwjOj8WZpszKAhXGtBoKHTCvBLMQFgggMAA&url=http%3A%2F%2Foauth.net%2F2%2F&usg=AFQjCNG58oSRksGnoIWIhfWYBB1sg_YGng&sig2=9Sux2Pq4TJwspuoLrFoFrQ) token, generated by a OAuth2 Tokens Generator (a third party) handled by any trusted Identity Manager (for instance, [FIWARE Lab one](https://account.lab.fiware.org/)). + +The advantage regarding the way this library has been implemented is that any user-and-password-based Hive client will continue working; simply, the password configuration parameter takes the token value. + +[Top](#top) + +##Installation +###Prerequisites +[`git`](https://git-scm.com/) tool and [Maven](https://maven.apache.org/) must be installed in order to download and build the cosmos-hive-auth-provider library, respectively. It is not the goal of this document to provide detailed installation instructions about the mentioned tools. + +Of course, this library has no sense if no [Hive](https://hive.apache.org/) service is deployed in a [Hadoop](http://hadoop.apache.org/) cluster. + +[Top](#top) + +###Building +Start by cloning the `fiware-cosmos` repository at Github: + + $ git clone https://github.com/telefonicaid/fiware-cosmos.git + +That will create a `fiware-cosmos` folder. Then, the library is built by runing the following `mvn` command under `fiware-cosmos/cosmos-hive-auth-provider`: + +``` +$ cd fiware-cosmos/cosmos-hive-auth-provider +$ mvn clean compile assembly:single +[INFO] Scanning for projects... +[INFO] +[INFO] ------------------------------------------------------------------------ +[INFO] Building cosmos-hive-auth-provider 0.0.0-SNAPSHOT +[INFO] ------------------------------------------------------------------------ +[INFO] +[INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ cosmos-hive-auth-provider --- +[INFO] Deleting /home/fiware-portal/fiware-cosmos/cosmos-hive-auth-provider/target +[INFO] +[INFO] --- maven-resources-plugin:2.6:resources (default-resources) @ cosmos-hive-auth-provider --- +[INFO] Using 'UTF-8' encoding to copy filtered resources. +[INFO] skip non existing resourceDirectory /home/fiware-portal/fiware-cosmos/cosmos-hive-auth-provider/src/main/resources +[INFO] +[INFO] --- maven-compiler-plugin:3.1:compile (default-compile) @ cosmos-hive-auth-provider --- +[INFO] Changes detected - recompiling the module! +[INFO] Compiling 2 source files to /home/fiware-portal/fiware-cosmos/cosmos-hive-auth-provider/target/classes +[INFO] +[INFO] --- maven-assembly-plugin:2.2-beta-5:single (default-cli) @ cosmos-hive-auth-provider --- +... +... +[INFO] META-INF/MANIFEST.MF already added, skipping +[INFO] META-INF/ already added, skipping +[INFO] META-INF/MANIFEST.MF already added, skipping +[INFO] org/ already added, skipping +[INFO] org/json/ already added, skipping +[INFO] ------------------------------------------------------------------------ +[INFO] BUILD SUCCESS +[INFO] ------------------------------------------------------------------------ +[INFO] Total time: 13.144 s +[INFO] Finished at: 2016-01-27T17:44:07+01:00 +[INFO] Final Memory: 103M/1993M +[INFO] ------------------------------------------------------------------------ +``` + +[Top](#top) + +###Jar copying +The cosmos-hive-auth-provider jar containing the `OAuth2AuthenticationProviderImpl` class must be copied into one folder within the Hive classpath, for instance: + +``` +$ ls /usr/lib/hive/lib/ | grep cosmos +cosmos-hive-auth-provider-0.0.0-SNAPSHOT-jar-with-dependencies.jar +``` + +[Top](#top) + +###Unit tests +The unit tests are run by invoking this parameterized `mvn test` command: + +``` +$ mvn test -DmatchingToken=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +[INFO] Scanning for projects... +[INFO] +[INFO] ------------------------------------------------------------------------ +[INFO] Building cosmos-hive-auth-provider 0.0.0-SNAPSHOT +[INFO] ------------------------------------------------------------------------ +[INFO] +[INFO] --- maven-resources-plugin:2.6:resources (default-resources) @ cosmos-hive-auth-provider --- +[INFO] Using 'UTF-8' encoding to copy filtered resources. +[INFO] skip non existing resourceDirectory /home/fiware-portal/fiware-cosmos/cosmos-hive-auth-provider/src/main/resources +[INFO] +[INFO] --- maven-compiler-plugin:3.1:compile (default-compile) @ cosmos-hive-auth-provider --- +[INFO] Nothing to compile - all classes are up to date +[INFO] +[INFO] --- maven-resources-plugin:2.6:testResources (default-testResources) @ cosmos-hive-auth-provider --- +[INFO] Using 'UTF-8' encoding to copy filtered resources. +[INFO] skip non existing resourceDirectory /home/fiware-portal/fiware-cosmos/cosmos-hive-auth-provider/src/test/resources +[INFO] +[INFO] --- maven-compiler-plugin:3.1:testCompile (default-testCompile) @ cosmos-hive-auth-provider --- +[INFO] Changes detected - recompiling the module! +[INFO] Compiling 1 source file to /home/fiware-portal/fiware-cosmos/cosmos-hive-auth-provider/target/test-classes +[INFO] +[INFO] --- maven-surefire-plugin:2.12.4:test (default-test) @ cosmos-hive-auth-provider --- +[INFO] Surefire report directory: /home/fiware-portal/fiware-cosmos/cosmos-hive-auth-provider/target/surefire-reports + +------------------------------------------------------- + T E S T S +------------------------------------------------------- +Running com.telefonica.iot.cosmos.hive.authprovider.OAuth2AuthenticationProviderImplTest +Testing OAuth2AuthenticationProviderImpl.Authenticate, it must fail because the token does not exist +log4j:WARN No appenders could be found for logger (com.telefonica.iot.cosmos.hive.authprovider.HttpClientFactory). +log4j:WARN Please initialize the log4j system properly. +log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info. +Testing OAuth2AuthenticationProviderImpl.Authenticate, it must success +Testing OAuth2AuthenticationProviderImpl.Authenticate, it must fail because the user and the token don't match +Tests run: 3, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 1.256 sec + +Results : + +Tests run: 3, Failures: 0, Errors: 0, Skipped: 0 + +[INFO] ------------------------------------------------------------------------ +[INFO] BUILD SUCCESS +[INFO] ------------------------------------------------------------------------ +[INFO] Total time: 3.110 s +[INFO] Finished at: 2016-01-28T10:59:10+01:00 +[INFO] Final Memory: 23M/481M +[INFO] ------------------------------------------------------------------------ +``` + +[Top](#top) + +##Configuration +Nothing has to be configured regarding cosmos-hive-auth-provider itself, but the Hive service must be configured in order to use it. + +The following properties must be added to `hive-site.xml` in order to enable a custom authentication provider, i.e. cosmos-hive-auth-provider and its `OAuth2AuthenticationProviderImpl` class: + +``` + + hive.server2.authentication + CUSTOM + + + + hive.server2.custom.authentication.class + com.telefonica.iot.cosmos.hive.authprovider.OAuth2AuthenticationProviderImpl + +``` + +This other property must be modified in order to enable impersonation (on the contrary, all the queries are executed by the user `hive` instead of the real end user): + +``` + + hive.server2.enable.doAs + true + +``` + +[Top](#top) + +##Running +This is a library directly used by Hive. In order HiveServer2 knows about it, restart the service from your cluster manager (e.g. Ambari) or from the command line: + + $ (sudo) service hive-server2 restart + +[Top](#top) + +##Usage +The comsos-hive-auth-provider library is used when a HiveServer2 client connects and pass a user and a OAuth2 token. For instance, let's assume we are using any of Hive clients given with [Cygnus](https://github.com/telefonicaid/fiware-cygnus/tree/master/resources/hiveclients) tool: + +``` +$ pwd +/home/centos/fiware-cygnus/resources/hiveclients/java/hiveserver2-client +$ mvn exec:java -Dexec.args="computing.cosmos.lab.fiware.org 10000 default frb xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" +[INFO] Scanning for projects... +[INFO] +[INFO] ------------------------------------------------------------------------ +[INFO] Building hiveserver2-client 0.0.0-SNAPSHOT +[INFO] ------------------------------------------------------------------------ +[INFO] +[INFO] >>> exec-maven-plugin:1.2.1:java (default-cli) > validate @ hiveserver2-client >>> +[INFO] +[INFO] <<< exec-maven-plugin:1.2.1:java (default-cli) < validate @ hiveserver2-client <<< +[INFO] +[INFO] --- exec-maven-plugin:1.2.1:java (default-cli) @ hiveserver2-client --- +Connecting to jdbc:hive2://computing.cosmos.lab.fiware.org:10000/default?user=frb&password=XXXXXXXXXX +remotehive> show tables; +frb_test +remotehive> describe frb_test; +name,string +job,string +age,int +remotehive> select * from frb_test; +han,smuggler,35 +luke,jedi,32 +leia,princess,28 +r2d2,robot,15 +remotehive> +`` + +[Top](#top) + +##Administration +HiveServer2 traces are usually logged within `/var/log/hive/hiveserver2.log`. There, you can find the traces regarding cosmos-hive-auth-provider, for instance, if everything goes well: + +``` +2016-01-27 15:59:17,587 INFO [pool-5-thread-4]: authprovider.HttpClientFactory (OAuth2AuthenticationProviderImpl.java:Authenticate(67)) - Doing request: GET https://account.lab.fiware.org/user?access_token=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx HTTP/1.1 +2016-01-27 15:59:17,594 INFO [pool-5-thread-4]: authprovider.HttpClientFactory (OAuth2AuthenticationProviderImpl.java:Authenticate(78)) - Response received: {"organizations": [], "displayName": "frb", "roles": [{"name": "provider", "id": "106"}], "app_id": "8556cc76154f41b3b43d7b31f0699982", "email": "frb@tid.es", "id": "frb"} +2016-01-27 15:59:17,667 INFO [pool-5-thread-4]: thrift.ThriftCLIService (ThriftCLIService.java:OpenSession(188)) - Client protocol version: HIVE_CLI_SERVICE_PROTOCOL_V6 +2016-01-27 15:59:17,676 INFO [pool-5-thread-4]: hive.metastore (HiveMetaStoreClient.java:open(297)) - Trying to connect to metastore with URI thrift://dev-fiwr-bignode-11.hi.inet:9083 +2016-01-27 15:59:17,677 INFO [pool-5-thread-4]: hive.metastore (HiveMetaStoreClient.java:open(385)) - Connected to metastore. +``` + +If the token does not exist, this is an example of relevant traces: + +``` +2016-01-27 16:10:10,196 INFO [pool-5-thread-28]: authprovider.HttpClientFactory (OAuth2AuthenticationProviderImpl.java:Authenticate(67)) - Doing request: GET https://account.lab.fiware.org/user?access_token=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx HTTP/1.1 +2016-01-27 16:10:10,197 INFO [pool-5-thread-28]: authprovider.HttpClientFactory (OAuth2AuthenticationProviderImpl.java:Authenticate(78)) - Response received: {"error": {"message": "Access Token xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx not found", "code": 404, "title": "Not Found"}} +2016-01-27 16:10:10,197 ERROR [pool-5-thread-28]: transport.TSaslTransport (TSaslTransport.java:open(296)) - SASL negotiation failure +``` + +If the token exists but does not match the given user, then something like this is logged: + +``` +2016-01-27 16:12:11,520 INFO [pool-5-thread-32]: authprovider.HttpClientFactory (OAuth2AuthenticationProviderImpl.java:Authenticate(67)) - Doing request: GET https://account.lab.fiware.org/user?access_token=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx HTTP/1.1 +2016-01-27 16:12:11,521 INFO [pool-5-thread-32]: authprovider.HttpClientFactory (OAuth2AuthenticationProviderImpl.java:Authenticate(78)) - Response received: {"organizations": [], "displayName": "frb", "roles": [{"name": "provider", "id": "106"}], "app_id": "8556cc76154f41b3b43d7b31f0699982", "email": "frb@tid.es", "id": "frb"} +2016-01-27 16:12:11,521 ERROR [pool-5-thread-32]: transport.TSaslTransport (TSaslTransport.java:open(296)) - SASL negotiation failure +javax.security.sasl.SaslException: Error validating the login [Caused by javax.security.sasl.AuthenticationException: The given token does not match the given user] +``` + +[Top](#top) + +##Reporting issues and contact information +There are several channels suited for reporting issues and asking for doubts in general. Each one depends on the nature of the question: + +* Use [stackoverflow.com](http://stackoverflow.com) for specific questions about this software. Typically, these will be related to installation problems, errors and bugs. Development questions when forking the code are welcome as well. Use the `fiware-cosmos` tag. +* Use [ask.fiware.org](https://ask.fiware.org/questions/) for general questions about FIWARE, e.g. how many cities are using FIWARE, how can I join the accelarator program, etc. Even for general questions about this software, for instance, use cases or architectures you want to discuss. +* Personal email: + * [francisco.romerobueno@telefonica.com](mailto:francisco.romerobueno@telefonica.com) **[Main contributor]** + * [fermin.galanmarquez@telefonica.com](mailto:fermin.galanmarquez@telefonica.com) **[Contributor]** + * [german.torodelvalle@telefonica.com](mailto:german.torodelvalle@telefonica.com) **[Contributor]** + * [pablo.coellovillalba@telefonica.com](mailto:pablo.coellovillalba@telefonica.com) **[Contributor]** + +**NOTE**: Please try to avoid personaly emailing the contributors unless they ask for it. In fact, if you send a private email you will probably receive an automatic response enforcing you to use [stackoverflow.com](stackoverflow.com) or [ask.fiware.org](https://ask.fiware.org/questions/). This is because using the mentioned methods will create a public database of knowledge that can be useful for future users; private email is just private and cannot be shared. + +[Top](#top) \ No newline at end of file From c573fde5f8cf5c39558319ba63186dbe3fc3ad71 Mon Sep 17 00:00:00 2001 From: frbattid Date: Thu, 28 Jan 2016 11:34:37 +0100 Subject: [PATCH 6/7] ADD the valid token as a parameter in the tests --- .../authprovider/OAuth2AuthenticationProviderImplTest.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cosmos-hive-auth-provider/src/test/java/com/telefonica/iot/cosmos/hive/authprovider/OAuth2AuthenticationProviderImplTest.java b/cosmos-hive-auth-provider/src/test/java/com/telefonica/iot/cosmos/hive/authprovider/OAuth2AuthenticationProviderImplTest.java index d6b4764..235d8c0 100644 --- a/cosmos-hive-auth-provider/src/test/java/com/telefonica/iot/cosmos/hive/authprovider/OAuth2AuthenticationProviderImplTest.java +++ b/cosmos-hive-auth-provider/src/test/java/com/telefonica/iot/cosmos/hive/authprovider/OAuth2AuthenticationProviderImplTest.java @@ -31,9 +31,13 @@ public class OAuth2AuthenticationProviderImplTest { private final String matchingUser = "frb"; private final String notMatchingUser = "other"; - private final String matchingToken = "111111111111111111111111111111"; + private static String matchingToken; private final String notMatchingToken = "000000000000000000000000000000"; + static { + matchingToken = System.getProperty("token"); + } // static + /** * Tests OAuth2AuthenticationProviderImpl.Authenticate when the given user and token match. */ From d1d58af81d5ebac867b616e24ff8e14408d0835f Mon Sep 17 00:00:00 2001 From: frbattid Date: Thu, 28 Jan 2016 13:33:58 +0100 Subject: [PATCH 7/7] ADD references to cosmos-hive-auth-provider from the Cosmos install and admin manual --- .../batch/oauth2_protected_apis.md | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/doc/manuals/installation_and_administration_manual/batch/oauth2_protected_apis.md b/doc/manuals/installation_and_administration_manual/batch/oauth2_protected_apis.md index 8928aa3..f2952da 100644 --- a/doc/manuals/installation_and_administration_manual/batch/oauth2_protected_apis.md +++ b/doc/manuals/installation_and_administration_manual/batch/oauth2_protected_apis.md @@ -1,14 +1,16 @@ -#OAuth2-protected REST APIs +#OAuth2-protected APIs Content:
-* [Introduction](#section1) -* [Installation](#section2) -* [Configuration](#section3) -* [Running](#section4) -* [Administration](#section5) +* [OAuth2-protected REST APIs](#section1) + * [Installation](#section1.1) + * [Configuration](#section1.2) + * [Running](#section1.3) + * [Administration](#section1.4) +* [Other APIs protected with OAuth2](#section2) + * [HiveServer2](#section2.1) -##Introduction +##OAuth2-protected REST APIs Many tools from Hadoop Ecosystem, and others added by Cosmos Ecosystem, expose REST APIs. These APIs are not usually secured in terms of authentication nor authorization. Even in the case they provide any means of authenticating/authorizing the users, the mechanisms for doing so may be very heterogeneous. @@ -19,7 +21,7 @@ Generator](http://github.com/telefonicaid/fiware-cosmos/tree/develop/cosmos-auth [Top](#top) -##Installation +###Installation Please refer to this [Installation and Administration Guide](http://forge.fiware.org/plugins/mediawiki/wiki/fiware/index.php/PEP_Proxy_-_Wilma_-_Installation_and_Administration_Guide) in order to install Wilma PEP Proxy. @@ -27,7 +29,7 @@ Please refer to the [installation section](http://github.com/telefonicaid/fiware [Top](#top) -##Configuration +###Configuration Please refer to this [Installation and Administration Guide](http://forge.fiware.org/plugins/mediawiki/wiki/fiware/index.php/PEP_Proxy_-_Wilma_-_Installation_and_Administration_Guide) in order to configure Wilma PEP Proxy. @@ -35,7 +37,7 @@ Please refer to the [configuration section](http://github.com/telefonicaid/fiwar [Top](#top) -##Running +###Running Please refer to this [Installation and Administration Guide](http://forge.fiware.org/plugins/mediawiki/wiki/fiware/index.php/PEP_Proxy_-_Wilma_-_Installation_and_Administration_Guide) in order to run Wilma PEP Proxy. @@ -43,10 +45,16 @@ Please refer to the [running section](http://github.com/telefonicaid/fiware-cosm [Top](#top) -##Administration +###Administration Please refer to this [Installation and Administration Guide](http://forge.fiware.org/plugins/mediawiki/wiki/fiware/index.php/PEP_Proxy_-_Wilma_-_Installation_and_Administration_Guide) in order to administrate Wilma PEP Proxy. Please refer to the [administration section](http://github.com/telefonicaid/fiware-cosmos/blob/develop/cosmos-auth/README.md#administration) of the README in Github in order to configure the OAuth2 Tokens Generator. -[Top](#top) \ No newline at end of file +[Top](#top) + +##Other APIs protected with OAuth2 +###HiveServer2 +There exist the possibility to configure custom authentication providers for Hive and its server, HiveServer2. In this sense, we have developed [cosmos-hive-auth-provider](https://github.com/telefonicaid/fiware-cosmos/tree/master/cosmos-hive-auth-provider), a OAuth2-based authentication provider. Please, check the [README](https://github.com/telefonicaid/fiware-cosmos/blob/master/cosmos-hive-auth-provider/README.md) in order to learn how to install, configure, run and use it. + +[Top](#top)