Simplified Cloud AI API integration. (#932)

* Cloud Translation, Natural language, Vision,
Video intelligence GCP integrations.

dev.Dockerfile

@@ -19,4 +19,5 @@ ADD patches/kaggle_gcp.py /root/.local/lib/python3.7/site-packages/kaggle_gcp.py
 ADD patches/kaggle_secrets.py /root/.local/lib/python3.7/site-packages/kaggle_secrets.py
 ADD patches/kaggle_session.py /root/.local/lib/python3.7/site-packages/kaggle_session.py
 ADD patches/kaggle_web_client.py /root/.local/lib/python3.7/site-packages/kaggle_web_client.py
-ADD patches/kaggle_datasets.py /root/.local/lib/python3.7/site-packages/kaggle_datasets.py
+ADD patches/kaggle_datasets.py /root/.local/lib/python3.7/site-packages/kaggle_datasets.py
+ADD patches/sitecustomize.py /root/.local/lib/python3.7/site-packages/sitecustomize.py

patches/kaggle_gcp.py

@@ -42,20 +42,9 @@ def has_bigquery(self):
  def has_gcs(self):
  return GcpTarget.GCS in self.integrations
 
- def has_automl(self):
- return GcpTarget.AUTOML in self.integrations
-
- def has_translation(self):
- return GcpTarget.TRANSLATION in self.integrations
-
- def has_natural_language(self):
- return GcpTarget.NATURAL_LANGUAGE in self.integrations
-
- def has_video_intelligence(self):
- return GcpTarget.VIDEO_INTELLIGENCE in self.integrations
-
- def has_vision(self):
- return GcpTarget.VISION in self.integrations
+ def has_cloudai(self):
+ return GcpTarget.CLOUDAI in self.integrations or \
+ GcpTarget.AUTOML in self.integrations
 
 class KaggleKernelCredentials(credentials.Credentials):
  """Custom Credentials used to authenticate using the Kernel's connected OAuth account.
@@ -74,16 +63,8 @@ def refresh(self, request):
  self.token, self.expiry = client.get_bigquery_access_token()
  elif self.target == GcpTarget.GCS:
  self.token, self.expiry = client._get_gcs_access_token()
- elif self.target == GcpTarget.AUTOML:
- self.token, self.expiry = client._get_automl_access_token()
- elif self.target == GcpTarget.TRANSLATION:
- self.token, self.expiry = client._get_translation_access_token()
- elif self.target == GcpTarget.NATURAL_LANGUAGE:
- self.token, self.expiry = client._get_natural_language_access_token()
- elif self.target == GcpTarget.VIDEO_INTELLIGENCE:
- self.token, self.expiry = client._get_video_intelligence_access_token()
- elif self.target == GcpTarget.VISION:
- self.token, self.expiry = client._get_vision_access_token()
+ elif self.target == GcpTarget.CLOUDAI:
+ self.token, self.expiry = client._get_cloudai_access_token()
  except ConnectionError as e:
  Log.error(f"Connection error trying to refresh access token: {e}")
  print("There was a connection error trying to fetch the access token. "
@@ -262,12 +243,12 @@ def init_automl():
  return
 
  from kaggle_gcp import get_integrations
- if not get_integrations().has_automl():
+ if not get_integrations().has_cloudai():
  return
 
  from kaggle_secrets import GcpTarget
  from kaggle_gcp import KaggleKernelCredentials
- kaggle_kernel_credentials = KaggleKernelCredentials(target=GcpTarget.AUTOML)
+ kaggle_kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI)
 
  # Patch the 2 GA clients: AutoMlClient and PreditionServiceClient
  monkeypatch_client(automl.AutoMlClient, kaggle_kernel_credentials)
@@ -293,10 +274,10 @@ def init_translation_v2():
  return translate_v2
 
  from kaggle_gcp import get_integrations
- if not get_integrations().has_translation():
+ if not get_integrations().has_cloudai():
  return translate_v2
  from kaggle_secrets import GcpTarget
- kernel_credentials = KaggleKernelCredentials(target=GcpTarget.TRANSLATION)
+ kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI)
  monkeypatch_client(translate_v2.Client, kernel_credentials)
  return translate_v2
 
@@ -307,10 +288,10 @@ def init_translation_v3():
  return translate_v3
 
  from kaggle_gcp import get_integrations
- if not get_integrations().has_translation():
+ if not get_integrations().has_cloudai():
  return translate_v3
  from kaggle_secrets import GcpTarget
- kernel_credentials = KaggleKernelCredentials(target=GcpTarget.TRANSLATION)
+ kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI)
  monkeypatch_client(translate_v3.TranslationServiceClient, kernel_credentials)
  return translate_v3
 
@@ -320,11 +301,11 @@ def init_natural_language():
  return language
 
  from kaggle_gcp import get_integrations
- if not get_integrations().has_natural_language():
+ if not get_integrations().has_cloudai():
  return language
 
  from kaggle_secrets import GcpTarget
- kernel_credentials = KaggleKernelCredentials(target=GcpTarget.NATURAL_LANGUAGE)
+ kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI)
  monkeypatch_client(language.LanguageServiceClient, kernel_credentials) 
  monkeypatch_client(language.LanguageServiceAsyncClient, kernel_credentials)
  return language
@@ -335,11 +316,11 @@ def init_video_intelligence():
  return videointelligence
 
  from kaggle_gcp import get_integrations
- if not get_integrations().has_video_intelligence():
+ if not get_integrations().has_cloudai():
  return videointelligence
 
  from kaggle_secrets import GcpTarget
- kernel_credentials = KaggleKernelCredentials(target=GcpTarget.VIDEO_INTELLIGENCE)
+ kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI)
  monkeypatch_client(
  videointelligence.VideoIntelligenceServiceClient,
  kernel_credentials)
@@ -354,11 +335,11 @@ def init_vision():
  return vision
 
  from kaggle_gcp import get_integrations
- if not get_integrations().has_vision():
+ if not get_integrations().has_cloudai():
  return vision
 
  from kaggle_secrets import GcpTarget
- kernel_credentials = KaggleKernelCredentials(target=GcpTarget.VISION)
+ kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI)
  monkeypatch_client(vision.ImageAnnotatorClient, kernel_credentials)
  monkeypatch_client(vision.ImageAnnotatorAsyncClient, kernel_credentials)
  return vision

patches/kaggle_secrets.py

@@ -23,11 +23,9 @@ class GcpTarget(Enum):
  """Enum class to store GCP targets."""
  BIGQUERY = (1, "BigQuery")
  GCS = (2, "Google Cloud Storage")
+ # Old name, should remove later.
  AUTOML = (3, "Cloud AutoML")
- TRANSLATION = (4, "Cloud Translation")
- NATURAL_LANGUAGE = (5, "Cloud Natural Language")
- VIDEO_INTELLIGENCE = (6, "Cloud Video Intelligence")
- VISION = (7, "Cloud Vision")
+ CLOUDAI = (3, "Google Cloud AI Platform")
 
  def __init__(self, target, service):
  self._target = target
@@ -155,20 +153,8 @@ def _write_gsutil_credentials_file(self, credentials) -> str:
  def _get_gcs_access_token(self) -> Tuple[str, Optional[datetime]]:
  return self._get_access_token(GcpTarget.GCS)
 
- def _get_automl_access_token(self) -> Tuple[str, Optional[datetime]]:
- return self._get_access_token(GcpTarget.AUTOML)
-
- def _get_translation_access_token(self) -> Tuple[str, Optional[datetime]]:
- return self._get_access_token(GcpTarget.TRANSLATION)
-
- def _get_natural_language_access_token(self) -> Tuple[str, Optional[datetime]]:
- return self._get_access_token(GcpTarget.NATURAL_LANGUAGE)
-
- def _get_video_intelligence_access_token(self) -> Tuple[str, Optional[datetime]]:
- return self._get_access_token(GcpTarget.VIDEO_INTELLIGENCE)
-
- def _get_vision_access_token(self) -> Tuple[str, Optional[datetime]]:
- return self._get_access_token(GcpTarget.VISION)
+ def _get_cloudai_access_token(self) -> Tuple[str, Optional[datetime]]:
+ return self._get_access_token(GcpTarget.CLOUDAI)
 
  def _get_access_token(self, target: GcpTarget) -> Tuple[str, Optional[datetime]]:
  request_body = {

tests/test_automl.py

@@ -27,7 +27,7 @@ def test_user_provided_credentials(self):
  credentials = _make_credentials()
  env = EnvironmentVarGuard()
  env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
- env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML')
+ env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
  with env:
  init_automl()
  client = automl.AutoMlClient(credentials=credentials)
@@ -48,7 +48,7 @@ def test_tables_client_credentials(self):
  credentials = _make_credentials()
  env = EnvironmentVarGuard()
  env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
- env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML')
+ env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
  with env:
  init_automl()
  tables_client = automl_v1beta1.TablesClient(credentials=credentials)
@@ -58,7 +58,7 @@ def test_tables_client_credentials(self):
  def test_default_credentials_automl_client(self):
  env = EnvironmentVarGuard()
  env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
- env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML')
+ env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
  with env:
  init_automl()
  automl_client = automl.AutoMlClient()
@@ -70,7 +70,7 @@ def test_default_credentials_automl_client(self):
  def test_default_credentials_automl_v1beta1_client(self):
  env = EnvironmentVarGuard()
  env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
- env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML')
+ env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
  with env:
  init_automl()
  automl_client = automl_v1beta1.AutoMlClient()
@@ -82,7 +82,7 @@ def test_default_credentials_automl_v1beta1_client(self):
  def test_default_credentials_tables_client(self):
  env = EnvironmentVarGuard()
  env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
- env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML')
+ env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
  with env:
  init_automl()
  tables_client = automl_v1beta1.TablesClient()
@@ -94,7 +94,7 @@ def test_default_credentials_tables_client(self):
  def test_default_credentials_prediction_client(self):
  env = EnvironmentVarGuard()
  env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
- env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML')
+ env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
  with env:
  prediction_client = automl.PredictionServiceClient()
  self.assertIsNotNone(prediction_client.credentials)
@@ -105,7 +105,7 @@ def test_default_credentials_prediction_client(self):
  def test_default_credentials_prediction_v1beta1_client(self):
  env = EnvironmentVarGuard()
  env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
- env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML')
+ env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
  with env:
  prediction_client = automl_v1beta1.PredictionServiceClient()
  self.assertIsNotNone(prediction_client.credentials)
@@ -115,9 +115,23 @@ def test_default_credentials_prediction_v1beta1_client(self):
  def test_monkeypatching_idempotent(self):
  env = EnvironmentVarGuard()
  env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
- env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML')
+ env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
  with env:
  client1 = automl.AutoMlClient.__init__
  init_automl()
  client2 = automl.AutoMlClient.__init__
  self.assertEqual(client1, client2)
+
+ @patch("google.cloud.automl_v1beta1.PredictionServiceClient", new=FakeClient)
+ def test_legacy_AUTOML_variable_v1beta1_client(self):
+ """
+ Tests previous KAGGLE_KERNEL_INTEGRATIONS="AUTOML" environment setting
+ """
+ env = EnvironmentVarGuard()
+ env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
+ env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML')
+ with env:
+ prediction_client = automl_v1beta1.PredictionServiceClient()
+ self.assertIsNotNone(prediction_client.credentials)
+ self.assertIsInstance(prediction_client.credentials, KaggleKernelCredentials)
+ self.assertTrue(prediction_client._connection.user_agent.startswith("kaggle-gcp-client/1.0"))

tests/test_natural_language.py

@@ -26,7 +26,7 @@ def __init__(self, user_agent):
  def test_default_credentials(self):
  env = EnvironmentVarGuard()
  env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
- env.set('KAGGLE_KERNEL_INTEGRATIONS', 'NATURAL_LANGUAGE')
+ env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
  with env:
  init_natural_language()
  client = language.LanguageServiceClient()
@@ -38,7 +38,7 @@ def test_user_provided_credentials(self):
  credentials = _make_credentials()
  env = EnvironmentVarGuard()
  env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
- env.set('KAGGLE_KERNEL_INTEGRATIONS', 'NATURAL_LANGUAGE')
+ env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
  with env:
  init_natural_language()
  client = language.LanguageServiceClient(credentials=credentials)
@@ -49,7 +49,7 @@ def test_user_provided_credentials(self):
  def test_monkeypatching_succeed(self):
  env = EnvironmentVarGuard()
  env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
- env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION')
+ env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
  with env:
  init_natural_language()
  client = language.LanguageServiceClient.__init__
@@ -58,7 +58,7 @@ def test_monkeypatching_succeed(self):
  def test_monkeypatching_idempotent(self):
  env = EnvironmentVarGuard()
  env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
- env.set('KAGGLE_KERNEL_INTEGRATIONS', 'NATURAL_LANGUAGE')
+ env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
  with env:
  init_natural_language()
  client1 = language.LanguageServiceClient.__init__

tests/test_tensorflow_credentials.py

@@ -13,7 +13,7 @@ def test_set_tensorflow_credential(self, mock_configure_gcs):
  credential = '{"client_id":"fake_client_id",' \
  '"client_secret":"fake_client_secret",' \
  '"refresh_token":"not a refresh token",' \
- '"type":"authorized_user"}';
+ '"type":"authorized_user"}'
 
  env = EnvironmentVarGuard()
  env.set('HOME', '/tmp')
@@ -22,7 +22,7 @@ def test_set_tensorflow_credential(self, mock_configure_gcs):
  # These need to be set to make UserSecretsClient happy, but aren't
  # pertinent to this test.
  env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
- env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML')
+ env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
 
  user_secrets = UserSecretsClient()
  user_secrets.set_tensorflow_credential(credential)

tests/test_translation.py

@@ -28,7 +28,7 @@ def __init__(self, user_agent):
  def test_default_credentials_v2(self):
  env = EnvironmentVarGuard()
  env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
- env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION')
+ env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
  with env:
  init_translation_v2()
  client = translate_v2.Client()
@@ -41,7 +41,7 @@ def test_user_provided_credentials_v2(self):
  credentials = _make_credentials()
  env = EnvironmentVarGuard()
  env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
- env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION')
+ env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
  with env:
  init_translation_v2()
  client = translate_v2.Client(credentials=credentials)
@@ -52,7 +52,7 @@ def test_user_provided_credentials_v2(self):
  def test_default_credentials_v3(self):
  env = EnvironmentVarGuard()
  env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
- env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION')
+ env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
  with env:
  init_translation_v3()
  client = translate.TranslationServiceClient()
@@ -65,7 +65,7 @@ def test_user_provided_credentials_v3(self):
  credentials = _make_credentials()
  env = EnvironmentVarGuard()
  env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
- env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION')
+ env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
  with env:
  init_translation_v3()
  client = translate.TranslationServiceClient(credentials=credentials)
@@ -76,7 +76,7 @@ def test_user_provided_credentials_v3(self):
  def test_monkeypatching_succeed(self):
  env = EnvironmentVarGuard()
  env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
- env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION')
+ env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
  with env:
  init_translation_v2()
  init_translation_v3()
@@ -90,7 +90,7 @@ def test_monkeypatching_succeed(self):
  def test_monkeypatching_idempotent(self):
  env = EnvironmentVarGuard()
  env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
- env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION')
+ env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
  with env:
  init_translation_v2()
  init_translation_v3()
@@ -116,7 +116,7 @@ def test_client_credential_uniqueness_v3(self):
  credentials = _make_credentials()
  env = EnvironmentVarGuard()
  env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
- env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION')
+ env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
  with env:
  init_translation_v3()
  client1 = translate.TranslationServiceClient()