Get project ID from JSON credentials file

docs/gcloud-config.rst

@@ -0,0 +1,60 @@
+Configuration
+*************
+
+Overview
+========
+
+Use service client objects to configure your applications.
+
+For example:
+
+.. code-block:: python
+
+    >>> from gcloud import bigquery
+    >>> client = bigquery.Client()
+
+When creating a client in this way, the project ID will be determined by
+searching these locations in the following order.
+
+* GCLOUD_PROJECT environment variable
+* GOOGLE_APPLICATION_CREDENTIALS JSON file
+* Default service configuration path from
+  ``$ gcloud beta auth application-default login``.
+* Google App Engine application ID
+* Google Compute Engine project ID (from metadata server)
+
+You can override the detection of your default project by setting the
+ ``project`` parameter when creating client objects.
+
+.. code-block:: python
+
+    >>> from gcloud import bigquery
+    >>> client = bigquery.Client(project='my-project')
+
+You can see what project ID a client is referencing by accessing the ``project``
+property on the client object.
+
+.. code-block:: python
+
+    >>> client.project
+    u'my-project'
+
+Authentication
+==============
+
+The authentication credentials can be implicitly determined from the
+environment or directly. See :doc:`gcloud-auth`.
+
+Logging in via ``gcloud beta auth application-default login`` will
+automatically configure a JSON key file with your default project ID and
+credentials.
+
+Setting the ``GOOGLE_APPLICATION_CREDENTIALS`` and ``GCLOUD_PROJECT``
+environment variables will override the automatically configured credentials.
+
+You can change your default project ID to ``my-new-default-project`` by
+using the ``gcloud`` CLI tool to change the configuration.
+
+.. code-block:: bash
+
+    $ gcloud config set project my-new-default-project

docs/index.rst

@@ -4,6 +4,7 @@
   :caption: gcloud
 
   gcloud-api
+  gcloud-config
   gcloud-auth
 
 .. toctree::

gcloud/_helpers.py

@@ -18,6 +18,7 @@
 
 import calendar
 import datetime
+import json
 import os
 import re
 import socket
@@ -27,8 +28,10 @@
 from google.protobuf import timestamp_pb2
 import six
 from six.moves.http_client import HTTPConnection
+from six.moves import configparser
 
 from gcloud.environment_vars import PROJECT
+from gcloud.environment_vars import CREDENTIALS
 
 try:
     from google.appengine.api import app_identity
@@ -48,6 +51,7 @@
     (?P<nanos>\d{1,9})                       # nanoseconds, maybe truncated
     Z                                        # Zulu
 """, re.VERBOSE)
+DEFAULT_CONFIGURATION_PATH = '~/.config/gcloud/configurations/config_default'
 
 
 class _LocalStack(Local):
@@ -129,13 +133,13 @@ def _ensure_tuple_or_list(arg_name, tuple_or_list):
     This effectively reduces the iterable types allowed to a very short
     whitelist: list and tuple.
 
-    :type arg_name: string
+    :type arg_name: str
     :param arg_name: Name of argument to use in error message.
 
-    :type tuple_or_list: sequence of string
+    :type tuple_or_list: sequence of str
     :param tuple_or_list: Sequence to be verified.
 
-    :rtype: list of string
+    :rtype: list of str
     :returns: The ``tuple_or_list`` passed in cast to a ``list``.
     :raises: class:`TypeError` if the ``tuple_or_list`` is not a tuple or
              list.
@@ -149,7 +153,7 @@ def _ensure_tuple_or_list(arg_name, tuple_or_list):
 def _app_engine_id():
     """Gets the App Engine application ID if it can be inferred.
 
-    :rtype: string or ``NoneType``
+    :rtype: str or ``NoneType``
     :returns: App Engine application ID if running in App Engine,
               else ``None``.
     """
@@ -159,6 +163,39 @@ def _app_engine_id():
     return app_identity.get_application_id()
 
 
+def _file_project_id():
+    """Gets the project id from the credentials file if one is available.
+
+    :rtype: str or ``NoneType``
+    :returns: Project-ID from JSON credentials file if value exists,
+              else ``None``.
+    """
+    credentials_file_path = os.getenv(CREDENTIALS)
+    if credentials_file_path:
+        with open(credentials_file_path, 'rb') as credentials_file:
+            credentials_json = credentials_file.read()
+            credentials = json.loads(credentials_json.decode('utf-8'))
+            return credentials.get('project_id')
+
+
+def _default_service_project_id():
+    """Retrieves the project ID from the gcloud command line tool.
+
+    Files that cannot be opened with configparser are silently ignored; this is
+    designed so that you can specify a list of potential configuration file
+    locations.
+
+    :rtype: str or ``NoneType``
+    :returns: Project-ID from default configuration file else ``None``
+    """
+    full_config_path = os.path.expanduser(DEFAULT_CONFIGURATION_PATH)
+    config = configparser.RawConfigParser()
+    config.read(full_config_path)
+
+    if config.has_section('core'):
+        return config.get('core', 'project')
+
+
 def _compute_engine_id():
     """Gets the Compute Engine project ID if it can be inferred.
 
@@ -172,7 +209,7 @@ def _compute_engine_id():
     See https://github.com/google/oauth2client/issues/93 for context about
     DNS latency.
 
-    :rtype: string or ``NoneType``
+    :rtype: str or ``NoneType``
     :returns: Compute Engine project ID if the metadata service is available,
               else ``None``.
     """
@@ -204,18 +241,27 @@ def _determine_default_project(project=None):
     implicit environments are:
 
     * GCLOUD_PROJECT environment variable
+    * GOOGLE_APPLICATION_CREDENTIALS JSON file
+    * Get default service project from
+      ``$ gcloud beta auth application-default login``
     * Google App Engine application ID
     * Google Compute Engine project ID (from metadata server)
 
-    :type project: string
+    :type project: str
     :param project: Optional. The project name to use as default.
 
-    :rtype: string or ``NoneType``
+    :rtype: str or ``NoneType``
     :returns: Default project if it can be determined.
     """
     if project is None:
         project = _get_production_project()
 
+    if project is None:
+        project = _file_project_id()
+
+    if project is None:
+        project = _default_service_project_id()
+
     if project is None:
         project = _app_engine_id()
 
@@ -231,7 +277,7 @@ def _millis(when):
     :type when: :class:`datetime.datetime`
     :param when: the datetime to convert
 
-    :rtype: integer
+    :rtype: int
     :returns: milliseconds since epoch for ``when``
     """
     micros = _microseconds_from_datetime(when)
@@ -256,7 +302,7 @@ def _microseconds_from_datetime(value):
     :type value: :class:`datetime.datetime`
     :param value: The timestamp to convert.
 
-    :rtype: integer
+    :rtype: int
     :returns: The timestamp, in microseconds.
     """
     if not value.tzinfo:
@@ -273,7 +319,7 @@ def _millis_from_datetime(value):
     :type value: :class:`datetime.datetime`, or None
     :param value: the timestamp
 
-    :rtype: integer, or ``NoneType``
+    :rtype: int, or ``NoneType``
     :returns: the timestamp, in milliseconds, or None
     """
     if value is not None:
@@ -430,20 +476,20 @@ def _datetime_to_pb_timestamp(when):
 def _name_from_project_path(path, project, template):
     """Validate a URI path and get the leaf object's name.
 
-    :type path: string
+    :type path: str
     :param path: URI path containing the name.
 
-    :type project: string or NoneType
+    :type project: str or NoneType
     :param project: The project associated with the request. It is
                     included for validation purposes.  If passed as None,
                     disables validation.
 
-    :type template: string
+    :type template: str
     :param template: Template regex describing the expected form of the path.
                      The regex must have two named groups, 'project' and
                      'name'.
 
-    :rtype: string
+    :rtype: str
     :returns: Name parsed from ``path``.
     :raises: :class:`ValueError` if the ``path`` is ill-formed or if
              the project from the ``path`` does not agree with the

gcloud/test__helpers.py

@@ -148,6 +148,85 @@ def test_value_set(self):
             self.assertEqual(dataset_id, APP_ENGINE_ID)
 
 
+class Test__get_credentials_file_project_id(unittest2.TestCase):
+
+    def _callFUT(self):
+        from gcloud._helpers import _file_project_id
+        return _file_project_id()
+
+    def setUp(self):
+        import os
+        self.old_env = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS')
+
+    def tearDown(self):
+        import os
+        if (not self.old_env and
+                'GOOGLE_APPLICATION_CREDENTIALS' in os.environ):
+            del os.environ['GOOGLE_APPLICATION_CREDENTIALS']
+
+    def test_success(self):
+        import os
+        import tempfile
+        with tempfile.NamedTemporaryFile(mode='w') as credential_file:
+            credential_file.write('{"project_id": "test-project-id"}')
+            credential_file.seek(0)
+            os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = credential_file.name
+
+            self.assertEqual('test-project-id', self._callFUT())
+
+    def test_no_environment(self):
+        self.assertEqual(None, self._callFUT())
+
+
+class Test__get_default_service_project_id(unittest2.TestCase):
+    temp_config_path = None
+    config_path = '.config/gcloud/configurations/'
+    config_file = 'config_default'
+
+    def setUp(self):
+        import tempfile
+        import os
+        self.temp_config_path = tempfile.gettempdir()
+
+        conf_path = os.path.join(self.temp_config_path, self.config_path)
+        os.makedirs(conf_path)
+        full_config_path = os.path.join(conf_path, self.config_file)
+
+        self.temp_config_file = full_config_path
+
+        with open(full_config_path, 'w') as conf_file:
+            conf_file.write('[core]\nproject = test-project-id')
+
+    def tearDown(self):
+        import os
+        import shutil
+
+        if self.temp_config_path:
+            shutil.rmtree(os.path.join(self.temp_config_path,
+                                       '.config'))
+
+    def callFUT(self, project_id=None):
+        from gcloud._helpers import _default_service_project_id
+        import os
+
+        def mock_expanduser(path=''):
+            if project_id and path.startswith('~'):
+                return self.temp_config_file
+            return ''
+
+        from gcloud._testing import _Monkey
+        with _Monkey(os.path, expanduser=mock_expanduser):
+            return _default_service_project_id()
+
+    def test_read_from_cli_info(self):
+        project_id = self.callFUT('test-project-id')
+        self.assertEqual('test-project-id', project_id)
+
+    def test_info_value_not_present(self):
+        project_id = self.callFUT()
+        self.assertEqual(None, project_id)
+
+
 class Test__compute_engine_id(unittest2.TestCase):
 
     def _callFUT(self):
@@ -219,7 +298,7 @@ def _callFUT(self, project=None):
         return _determine_default_project(project=project)
 
     def _determine_default_helper(self, prod=None, gae=None, gce=None,
-                                  project=None):
+                                  file_id=None, srv_id=None, project=None):
         from gcloud._testing import _Monkey
         from gcloud import _helpers
 
@@ -229,6 +308,14 @@ def prod_mock():
             _callers.append('prod_mock')
             return prod
 
+        def file_id_mock():
+            _callers.append('file_id_mock')
+            return file_id
+
+        def srv_id_mock():
+            _callers.append('srv_id_mock')
+            return srv_id
+
         def gae_mock():
             _callers.append('gae_mock')
             return gae
@@ -239,6 +326,8 @@ def gce_mock():
 
         patched_methods = {
             '_get_production_project': prod_mock,
+            '_file_project_id': file_id_mock,
+            '_default_service_project_id': srv_id_mock,
             '_app_engine_id': gae_mock,
             '_compute_engine_id': gce_mock,
         }
@@ -251,7 +340,8 @@ def gce_mock():
     def test_no_value(self):
         project, callers = self._determine_default_helper()
         self.assertEqual(project, None)
-        self.assertEqual(callers, ['prod_mock', 'gae_mock', 'gce_mock'])
+        self.assertEqual(callers, ['prod_mock', 'file_id_mock', 'srv_id_mock',
+                                   'gae_mock', 'gce_mock'])
 
     def test_explicit(self):
         PROJECT = object()
@@ -269,13 +359,15 @@ def test_gae(self):
         PROJECT = object()
         project, callers = self._determine_default_helper(gae=PROJECT)
         self.assertEqual(project, PROJECT)
-        self.assertEqual(callers, ['prod_mock', 'gae_mock'])
+        self.assertEqual(callers, ['prod_mock', 'file_id_mock',
+                                   'srv_id_mock', 'gae_mock'])
 
     def test_gce(self):
         PROJECT = object()
         project, callers = self._determine_default_helper(gce=PROJECT)
         self.assertEqual(project, PROJECT)
-        self.assertEqual(callers, ['prod_mock', 'gae_mock', 'gce_mock'])
+        self.assertEqual(callers, ['prod_mock', 'file_id_mock', 'srv_id_mock',
+                                   'gae_mock', 'gce_mock'])
 
 
 class Test__millis(unittest2.TestCase):