[BB2-2336] Upgrade Python from 3.7 to 3.8 (LTS) for BB2 web server (#…

…1113)

* bb2 server changes per python 3.8 upgrade.

* apps/dot_ext/migrations/0005_alter_application_client_secret.py

* consolidate migration.

* fix custom migrate on application model client_secret copy over.

* fix lint

* added tar ball for 2 dependencies.

* add crytpo whl for amazon linux2.

* add alt whl files.

* whl update for new relic

* remove py37 newrelic whl

* Add client_secret_plain to app admin

---------

Co-authored-by: JAMES FUQIAN <[email protected]>
Co-authored-by: Don Seymour <[email protected]>
Co-authored-by: Dave Tisza <[email protected]>

Dockerfile

@@ -1,6 +1,5 @@
-FROM python:3.7.13
+FROM python:3.8
 ENV PYTHONUNBUFFERED 1
-# ENV PYTHONDEVMODE 1
 RUN useradd -m -s /bin/bash DEV
 USER DEV
 ADD . /code
@@ -9,5 +8,7 @@ RUN python -m venv /tmp/venv
 RUN . /tmp/venv/bin/activate
 ENV PATH="/tmp/venv/bin:${PATH}"
 RUN pip install --upgrade pip
-RUN pip install pip-tools
-RUN make reqs-install-dev
+RUN pip install --upgrade pip-tools
+RUN pip install --upgrade setuptools
+RUN pip install -r requirements/requirements.dev.txt --no-index --find-links ./vendor/
+

Dockerfiles/Dockerfile.selenium-jenkins-python38

@@ -0,0 +1,18 @@
+FROM python:3.8
+# For build CBC Jenkins job ECR image
+ENV PYTHONUNBUFFERED 1
+
+RUN mkdir /code
+ADD . /code/
+WORKDIR /code
+
+RUN pip install --upgrade pip
+RUN apt-get update && apt-get install -yq git unzip curl
+# Install Chrome for Selenium
+RUN curl https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb -o /chrome.deb \
+ && dpkg -i /chrome.deb || apt-get install -yf \
+ && rm /chrome.deb
+# Install chromedriver for Selenium
+RUN wget -O /tmp/chromedriver.zip https://chromedriver.storage.googleapis.com/`curl -sS chromedriver.storage.googleapis.com/LATEST_RELEASE`/chromedriver_linux64.zip \
+ && unzip /tmp/chromedriver.zip chromedriver -d /usr/local/bin/ \
+ && chmod +x /usr/local/bin/chromedriver

Jenkinsfiles/Jenkinsfile.cbc-run-multi-pr-checks-w-selenium

@@ -1,8 +1,8 @@
 pipeline {
  agent {
  kubernetes {
- defaultContainer "bb2-cbc-build-selenium-django32"
- yamlFile "Jenkinsfiles/cbc-pod-deployment-config-w-selenium.yaml"
+ defaultContainer "bb2-cbc-build-selenium-python38"
+ yamlFile "Jenkinsfiles/cbc-pod-deployment-config-w-selenium-p38.yaml"
  }
  }
 

Jenkinsfiles/cbc-pod-deployment-config-w-selenium-p38.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Pod
+spec:
+ containers:
+ - name: bb2-cbc-build-selenium-python38
+ image: "public.ecr.aws/f5g8o1y9/bb2-cbc-build-selenium-python38:latest"
+ tty: true
+ command: ["tail", "-f"]
+ imagePullPolicy: Always
+ nodeSelector:
+ Agents: true

apps/accounts/utils.py

@@ -48,7 +48,7 @@ def validate_activation_key(activation_key):
  msg = LINK_EXPIRED_MSG
  else:
  msg = ACCT_HAS_ISSUE_MSG
- except(ActivationKey.DoesNotExist):
+ except (ActivationKey.DoesNotExist):
  # The key does not exist, corner case: a fabricated url with a fake activation key
  msg = ACCT_HAS_ISSUE_MSG
 

apps/authorization/migrations/0003_auto_20230513_0718.py

@@ -0,0 +1,37 @@
+# Generated by Django 3.2.18 on 2023-05-13 07:18
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ migrations.swappable_dependency(settings.OAUTH2_PROVIDER_APPLICATION_MODEL),
+ ('authorization', '0002_auto_20221121_1516'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='archiveddataaccessgrant',
+ name='application',
+ field=models.ForeignKey(db_constraint=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.OAUTH2_PROVIDER_APPLICATION_MODEL),
+ ),
+ migrations.AlterField(
+ model_name='archiveddataaccessgrant',
+ name='beneficiary',
+ field=models.ForeignKey(db_constraint=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
+ ),
+ migrations.AlterField(
+ model_name='dataaccessgrant',
+ name='application',
+ field=models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, to=settings.OAUTH2_PROVIDER_APPLICATION_MODEL),
+ ),
+ migrations.AlterField(
+ model_name='dataaccessgrant',
+ name='beneficiary',
+ field=models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
+ ),
+ ]

apps/authorization/tests/test_data_access_grant.py

@@ -227,14 +227,14 @@ def test_delete_authenticated_user_grant(self):
  self.assertEqual(response.status_code, 403)
 
  # 5. Create authentication headers but pass bad patient_id and expect 404
- auth = self._create_authorization_header(application.client_id, application.client_secret)
+ auth = self._create_authorization_header(application.client_id, application.client_secret_plain)
  response = self.client.post('/v1/o/expire_authenticated_user/{0}/'.format("-2014000000832"),
  HTTP_AUTHORIZATION=auth,
  )
  self.assertEqual(response.status_code, 404)
 
  # 6. Create authentication headers and expect success
- auth = self._create_authorization_header(application.client_id, application.client_secret)
+ auth = self._create_authorization_header(application.client_id, application.client_secret_plain)
  response = self.client.post('/v1/o/expire_authenticated_user/{0}/'.format("-20140000008325"),
  HTTP_AUTHORIZATION=auth,
  )

apps/authorization/tests/test_data_access_grant_permissions.py

@@ -243,7 +243,7 @@ def _assert_call_token_refresh_endpoint(
  "refresh_token": refresh_token,
  "redirect_uri": application.redirect_uris,
  "client_id": application.client_id,
- "client_secret": application.client_secret,
+ "client_secret": application.client_secret_plain,
  }
  response = self.client.post("/v1/o/token/", data=refresh_post_data)
  content = json.loads(response.content)

apps/creds/templates/get_creds.html

@@ -47,8 +47,8 @@ <h2 class="ds-u-margin-bottom--2">App Credentials</h2>
  <label for"id-{{ application.client_id }}">Client ID</label>
  <input type="password" value="{{ client_id }}" id="id-{{ client_id }}"
  onclick="copyCredential(this.id)" />
- <label for"secret-{{ application.client_secret }}">Client Secret</label>
- <input type="password" value="{{ client_secret }}" id="secret-{{ client_id }}"
+ <label for"secret-{{ application.client_secret_plain }}">Client Secret</label>
+ <input type="password" value="{{ client_secret_plain }}" id="secret-{{ client_id }}"
  onclick="copyCredential(this.id)" />
  </form>
  </div>

apps/dot_ext/admin.py

@@ -56,6 +56,7 @@ class Meta:
  "user",
  "client_type",
  "authorization_grant_type",
+ "client_secret_plain",
  "client_secret",
  "name",
  "skip_authorization",

apps/dot_ext/migrations/0005_alter_application_client_secret.py

@@ -0,0 +1,28 @@
+# Generated by Django 3.2.18 on 2023-05-13 07:18
+
+from django.db import migrations, models
+import oauth2_provider.generators
+import oauth2_provider.models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('dot_ext', '0004_auto_20221117_2012'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='application',
+ name='client_secret_plain',
+ field=models.CharField(blank=True, default='', max_length=255),
+ ),
+ migrations.RunSQL(
+ "UPDATE dot_ext_application SET client_secret_plain=client_secret"
+ ),
+ migrations.AlterField(
+ model_name='application',
+ name='client_secret',
+ field=oauth2_provider.models.ClientSecretField(blank=True, db_index=True, default=oauth2_provider.generators.generate_client_secret, help_text='Hashed on Save. Copy it now if this is a new secret.', max_length=255),
+ ),
+ ]

apps/dot_ext/models.py

@@ -42,6 +42,11 @@ class Application(AbstractApplication):
  updated = models.DateTimeField(auto_now=True)
  op_tos_uri = models.CharField(default=settings.TOS_URI, blank=True, max_length=512)
  op_policy_uri = models.CharField(default="", blank=True, max_length=512)
+ # oauth2_provider upgraded and there is a breaking change on Application.client_secret field
+ # see migration file 0005_alter_application_client_secret.py
+ # field added to save client_secret in plain text before Application.save()
+ # where the client_secret is hashed ireversible
+ client_secret_plain = models.CharField(default="", blank=True, max_length=255)
 
  # client_uri is depreciated but will continued to be referenced until it can be removed safely
  client_uri = models.URLField(
@@ -280,17 +285,26 @@ def save(self, *args, **kwargs):
  except Application.DoesNotExist:
  # new app
  pass
+ self.copy_client_secret()
  super().save(*args, **kwargs)
  if app_type_changed:
  log_dict.update({"application_saved_and_grants_deleted": "Yes"})
  logger.info(log_dict)
  else:
+ self.copy_client_secret()
  super().save(*args, **kwargs)
 
  # dedicated save for high frequency used first / last active timestamp updates
  def save_without_validate(self, *args, **kwargs):
  super().save(*args, **kwargs)
 
+ # per oauth2_provider release note: copy client_secret before
+ # it gets hashed
+ def copy_client_secret(self, *args, **kwargs):
+ if self.client_secret is not None and len(self.client_secret) == 128:
+ # make sure it is a generated hash
+ self.client_secret_plain = self.client_secret
+
 
 class ApplicationLabel(models.Model):
  name = models.CharField(max_length=255, unique=True)

apps/dot_ext/templates/oauth2_provider/application_detail.html

@@ -49,8 +49,8 @@ <h2 class="ds-u-margin-bottom--2">App Credentials</h2>
  <a class="app-credentials-toggle" id="{{ application.client_id }}" tabindex="0">Show/Hide Credentials</a>
  <label for"id-{{ application.client_id }}">Client ID</label>
  <input type="password" value="{{ application.client_id }}" id="id-{{ application.client_id }}" onclick="copyCredential(this.id)" />
- <label for"secret-{{ application.client_secret }}">Client Secret</label>
- <input type="password" value="{{ application.client_secret }}" id="secret-{{ application.client_id }}" onclick="copyCredential(this.id)" />
+ <label for"secret-{{ application.client_secret_plain }}">Client Secret</label>
+ <input type="password" value="{{ application.client_secret_plain }}" id="secret-{{ application.client_id }}" onclick="copyCredential(this.id)" />
  </form>
  </div>
 

apps/dot_ext/tests/test_authentication.py

@@ -66,7 +66,7 @@ def _create_authentication_header(self, username):
 
  @unittest.skipUnless(rest_framework_installed, "djangorestframework not installed")
  def test_authentication_allow(self):
- auth = self._create_authorization_header(self.application.client_id, self.application.client_secret)
+ auth = self._create_authorization_header(self.application.client_id, self.application.client_secret_plain)
  response = self.client.get("/oauth2-test/",
  HTTP_AUTHORIZATION=auth,
  HTTP_X_AUTHENTICATION=self._create_authentication_header(self.test_username))
@@ -81,27 +81,27 @@ def test_authentication_denied(self):
  self.assertEqual(response.status_code, 403)
 
  def test_user_dne(self):
- auth = self._create_authorization_header(self.application.client_id, self.application.client_secret)
+ auth = self._create_authorization_header(self.application.client_id, self.application.client_secret_plain)
  response = self.client.get("/oauth2-test/",
  HTTP_AUTHORIZATION=auth,
  HTTP_X_AUTHENTICATION=self._create_authentication_header('bogus'))
  self.assertEqual(response.status_code, 404)
 
  def test_no_authentication(self):
- auth = self._create_authorization_header(self.application.client_id, self.application.client_secret)
+ auth = self._create_authorization_header(self.application.client_id, self.application.client_secret_plain)
  response = self.client.get("/oauth2-test/",
  HTTP_AUTHORIZATION=auth)
  self.assertEqual(response.status_code, 403)
 
  def test_bad_authentication(self):
- auth = self._create_authorization_header(self.application.client_id, self.application.client_secret)
+ auth = self._create_authorization_header(self.application.client_id, self.application.client_secret_plain)
  response = self.client.get("/oauth2-test/",
  HTTP_AUTHORIZATION=auth,
  HTTP_X_AUTHENTICATION="thisisabadheader")
  self.assertEqual(response.status_code, 404)
 
  def test_unknown_authentication(self):
- auth = self._create_authorization_header(self.application.client_id, self.application.client_secret)
+ auth = self._create_authorization_header(self.application.client_id, self.application.client_secret_plain)
  response = self.client.get("/oauth2-test/",
  HTTP_AUTHORIZATION=auth,
  HTTP_X_AUTHENTICATION="UUID thisisabadheader")

apps/dot_ext/tests/test_authorization.py

@@ -168,7 +168,7 @@ def test_post_with_valid_non_standard_scheme_granttype_authcode_clienttype_confi
  self.assertEqual(response.status_code, 401)
 
  # Setup application's client_secret
- token_request_data.update({'client_secret': application.client_secret})
+ token_request_data.update({'client_secret': application.client_secret_plain})
 
  # Test that using a BAD code_verifier has a bad request response
  token_request_data.update({'code_verifier': 'test1234567bad9verifier23456789123456789123456789'})
@@ -250,7 +250,7 @@ def test_refresh_token(self):
  'code': authorization_code,
  'redirect_uri': redirect_uri,
  'client_id': application.client_id,
- 'client_secret': application.client_secret,
+ 'client_secret': application.client_secret_plain,
  }
  c = Client()
  response = c.post('/v1/o/token/', data=token_request_data)
@@ -263,7 +263,7 @@ def test_refresh_token(self):
  'refresh_token': refresh_tkn,
  'redirect_uri': redirect_uri,
  'client_id': application.client_id,
- 'client_secret': application.client_secret,
+ 'client_secret': application.client_secret_plain,
  }
  response = self.client.post(reverse('oauth2_provider:token'), data=refresh_request_data)
  self.assertEqual(response.status_code, 200)
@@ -305,7 +305,7 @@ def test_refresh_with_expired_token(self):
  'code': authorization_code,
  'redirect_uri': redirect_uri,
  'client_id': application.client_id,
- 'client_secret': application.client_secret,
+ 'client_secret': application.client_secret_plain,
  }
  c = Client()
  response = c.post('/v1/o/token/', data=token_request_data)
@@ -320,7 +320,7 @@ def test_refresh_with_expired_token(self):
  'refresh_token': refresh_tkn,
  'redirect_uri': redirect_uri,
  'client_id': application.client_id,
- 'client_secret': application.client_secret,
+ 'client_secret': application.client_secret_plain,
  }
  response = self.client.post(reverse('oauth2_provider:token'), data=refresh_request_data)
  self.assertEqual(response.status_code, 400)
@@ -361,7 +361,7 @@ def test_refresh_with_revoked_token(self):
  'code': authorization_code,
  'redirect_uri': redirect_uri,
  'client_id': application.client_id,
- 'client_secret': application.client_secret,
+ 'client_secret': application.client_secret_plain,
  }
  c = Client()
  response = c.post('/v1/o/token/', data=token_request_data)
@@ -371,7 +371,7 @@ def test_refresh_with_revoked_token(self):
  revoke_request_data = {
  'token': tkn,
  'client_id': application.client_id,
- 'client_secret': application.client_secret,
+ 'client_secret': application.client_secret_plain,
  }
  rev_response = c.post('/v1/o/revoke_token/', data=revoke_request_data)
  self.assertEqual(rev_response.status_code, 200)
@@ -384,7 +384,7 @@ def test_refresh_with_revoked_token(self):
  'refresh_token': refresh_tkn,
  'redirect_uri': redirect_uri,
  'client_id': application.client_id,
- 'client_secret': application.client_secret,
+ 'client_secret': application.client_secret_plain,
  }
  response = self.client.post(reverse('oauth2_provider:token'), data=refresh_request_data)
  self.assertEqual(response.status_code, 400)
@@ -427,7 +427,7 @@ def test_application_delete_after_auth(self):
  'code': authorization_code,
  'redirect_uri': redirect_uri,
  'client_id': application.client_id,
- 'client_secret': application.client_secret,
+ 'client_secret': application.client_secret_plain,
  }
  c = Client()
  response = c.post('/v1/o/token/', data=token_request_data)
@@ -483,7 +483,7 @@ def test_user_delete_after_auth(self):
  'code': authorization_code,
  'redirect_uri': redirect_uri,
  'client_id': application.client_id,
- 'client_secret': application.client_secret,
+ 'client_secret': application.client_secret_plain,
  }
  c = Client()
  response = c.post('/v1/o/token/', data=token_request_data)
@@ -543,7 +543,7 @@ def test_revoked_token_on_inactive_app(self):
  'code': authorization_code,
  'redirect_uri': redirect_uri,
  'client_id': application.client_id,
- 'client_secret': application.client_secret,
+ 'client_secret': application.client_secret_plain,
  }
  c = Client()
  response = c.post('/v1/o/token/', data=token_request_data)
@@ -553,7 +553,7 @@ def test_revoked_token_on_inactive_app(self):
  revoke_request_data = {
  'token': tkn,
  'client_id': application.client_id,
- 'client_secret': application.client_secret,
+ 'client_secret': application.client_secret_plain,
  }
  # set app to inactive before revoke
  application.active = False
@@ -612,7 +612,7 @@ def test_introspect_token_on_inactive_app(self):
  'code': authorization_code,
  'redirect_uri': redirect_uri,
  'client_id': application.client_id,
- 'client_secret': application.client_secret,
+ 'client_secret': application.client_secret_plain,
  }
  c = Client()
  response = c.post('/v1/o/token/', data=token_request_data)
@@ -622,7 +622,7 @@ def test_introspect_token_on_inactive_app(self):
  introspect_request_data = {
  'token': tkn,
  'client_id': application.client_id,
- 'client_secret': application.client_secret,
+ 'client_secret': application.client_secret_plain,
  }
 
  auth_headers = {'Authorization': 'Bearer %s' % tkn}