[BB2-1283, BB2-1313] Django Upgrade 2.2.28 to 3.2.13 Python37 (#1027)

* first cut p37 django upgrade

* missed whl

* pip upgrade in CI docker.

* remove source dist for cryptography-37.0.2.tar.gz

* store the custom migration sql scripts to be handy on the target EC2

* pip upgrade on CI docker venv.

* pip install cryptography==37.0.2

* fix LGTM encoding warning in source code text.

* manually provision cryptography-37.0.2-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl(the binary used on CI docker) in vendor folder.

* added debugging and tracing..

* added debugging and tracing..

* fix dotenv import ...

* clean up.

* bump django to 3.2.14 to address C vuln CVE-2022-34265

* bump django_storages from 1.7.1 to 1.12.3 to resolve django.util.six reference.

* restore base settings DEBUG flag setting.

* change Django DEBUG flag default to False - to be safer.

Dockerfile.selenium

@@ -2,11 +2,13 @@ FROM selenium/standalone-chrome-debug
 
 ENV PYTHONUNBUFFERED 1
 USER root
-RUN apt-get update && apt-get install -yq python3.8 python3-pip git
-RUN pip3 install --upgrade pip
-RUN pip3 install selenium psycopg2-binary==2.8.6 pyyaml==5.4.1 Pillow==9.0.1
+# libpq-dev: ubuntu dev lib for psypsycopg2 sdist build
+RUN apt-get update && apt-get install -yq python3.7 python3-pip git libpq-dev libffi-dev
 RUN mkdir /code
 ADD . /code/
 WORKDIR /code
+RUN ln -s /usr/bin/python3 /usr/local/bin/python
+RUN pip install --upgrade pip
+RUN pip install selenium
+RUN pip install pyyaml==6.0 Pillow==9.0.1
 RUN make reqs-install-dev
-RUN ln -s /usr/bin/python3 /usr/local/bin/python

Dockerfiles/Dockerfile.selenium-jenkins

@@ -7,6 +7,8 @@ RUN mkdir /code
 ADD . /code/
 WORKDIR /code
 
+RUN pip install --upgrade pip
+
 RUN apt-get update && apt-get install -yq git unzip curl
 
 # Install Chrome for Selenium

Jenkinsfiles/Jenkinsfile.cbc-run-multi-pr-checks

@@ -36,6 +36,7 @@ pipeline {
  sh """
  python -m venv venv
  . venv/bin/activate
+ pip install --upgrade pip
  make reqs-install-dev
  """
  }

Jenkinsfiles/Jenkinsfile.cbc-run-multi-pr-checks-w-selenium

@@ -1,7 +1,7 @@
 pipeline {
  agent {
  kubernetes {
- defaultContainer "bb2-cbc-build-selenium"
+ defaultContainer "bb2-cbc-build-selenium-django32"
  yamlFile "Jenkinsfiles/cbc-pod-deployment-config-w-selenium.yaml"
  }
  }
@@ -55,8 +55,9 @@ pipeline {
  sh """
  python -m venv venv
  . venv/bin/activate
- make reqs-install-dev
  pip install selenium
+ pip install cryptography==37.0.2
+ make reqs-install-dev
  """
  }
  }

Jenkinsfiles/cbc-pod-deployment-config-w-selenium.yaml

@@ -2,8 +2,8 @@ apiVersion: v1
 kind: Pod
 spec:
  containers:
- - name: bb2-cbc-build-selenium
- image: "public.ecr.aws/f5g8o1y9/bb2-cbc-build-selenium:latest"
+ - name: bb2-cbc-build-selenium-django32
+ image: "public.ecr.aws/f5g8o1y9/bb2-cbc-build-selenium-django32:latest"
  tty: true
  command: ["tail", "-f"]
  imagePullPolicy: Always

apps/accounts/tests/test_case_ins_username.py

@@ -1,7 +1,9 @@
+from django.contrib.auth.models import User
+from django.http import HttpRequest
 from django.test import TestCase
 from django.test.client import Client
-from django.contrib.auth.models import User
 from django.urls import reverse
+
 from ..models import UserProfile
 
 
@@ -22,13 +24,15 @@ def setUp(self):
  self.client = Client()
 
  def test_page_loads(self):
- self.client.login(username="[email protected]", password="foobar")
+ request = HttpRequest()
+ self.client.login(request=request, username="[email protected]", password="foobar")
  url = reverse('account_settings')
  response = self.client.get(url, follow=True)
  self.assertEqual(response.status_code, 200)
 
  def test_all_fields_required(self):
- self.client.login(username="[email protected]", password="foobar")
+ request = HttpRequest()
+ self.client.login(request=request, username="[email protected]", password="foobar")
  url = reverse('account_settings')
  form_data = {'username': '',
  'email': '[email protected]',
@@ -41,7 +45,8 @@ def test_all_fields_required(self):
  self.assertContains(response, 'This field is required.')
 
  def test_username_forced_to_lower(self):
- self.client.login(username="[email protected]", password="foobar")
+ request = HttpRequest()
+ self.client.login(request=request, username="[email protected]", password="foobar")
  url = reverse('account_settings')
  form_data = {'username': '[email protected]',
  'email': '[email protected]',

apps/accounts/tests/test_create_account.py

@@ -42,6 +42,7 @@ def test_valid_account_create(self):
  'last_name': 'Rubble',
  'identification_choice': str(ident_choice.pk),
  }
+
  response = self.client.post(self.url, form_data, follow=True)
 
  self.assertEqual(response.status_code, 200)

apps/accounts/tests/test_enduser_cannot_reach_standard_admin_login.py

@@ -1,3 +1,4 @@
+from django.http import HttpRequest
 from django.test import TestCase
 from django.contrib.auth.models import User
 from django.test.client import Client
@@ -27,7 +28,8 @@ def setUp(self):
  user = User.objects.get(username='fred')
  UserProfile.objects.create(user=user)
  self.client = Client()
- self.client.login(username='fred', password='bedrocks')
+ request = HttpRequest()
+ self.client.login(request=request, username='fred', password='bedrocks')
  self.url = reverse('admin:login')
 
  def test_standard_user_cannot_reach_admin_login(self):

apps/accounts/tests/test_login.py

@@ -67,6 +67,11 @@ def test_valid_login_case_insensitive_username(self):
  def test_invalid_login(self):
  """
  Invalid user cannot login
+ Django upgrade 3.2:
+ Now AXES gives warning as shown below:
+ AXES: New login failure by
+ {username: "fred", ip_address: "127.0.0.1", user_agent: "<unknown>", path_info: "/v1/accounts/login"}.
+ Created new record in the database
  """
  form_data = {'username': 'fred', 'password': 'dino'}
  response = self.client.post(self.url, form_data, follow=True)

apps/accounts/tests/test_password_reset_while_authenticated.py

@@ -1,5 +1,6 @@
 import time
 from django.contrib.auth.models import User
+from django.http import HttpRequest
 from django.test import TestCase
 from django.test.client import Client
 from django.urls import reverse
@@ -50,7 +51,8 @@ def setUp(self):
 
  @override_switch('login', active=True)
  def test_page_loads(self):
- self.client.login(username="fred", password="foobarfoobarfoobar")
+ request = HttpRequest()
+ self.client.login(request=request, username="fred", password="foobarfoobarfoobar")
  url = reverse('password_change')
  response = self.client.get(url, follow=True)
  self.assertEqual(response.status_code, 200)
@@ -63,7 +65,8 @@ def test_page_requires_authentication(self):
 
  @override_switch('login', active=True)
  def test_password_ischanged(self):
- self.client.login(username="fred", password="foobarfoobarfoobar")
+ request = HttpRequest()
+ self.client.login(request=request, username="fred", password="foobarfoobarfoobar")
  url = reverse('password_change')
  form_data = {'old_password': 'foobarfoobarfoobar',
  'new_password1': 'IchangedTHEpassword#123',
@@ -78,7 +81,8 @@ def test_password_ischanged(self):
 
  @override_switch('login', active=True)
  def test_password_change_complexity_and_min_age_validation(self):
- self.client.login(username="fred", password="foobarfoobarfoobar")
+ request = HttpRequest()
+ self.client.login(request=request, username="fred", password="foobarfoobarfoobar")
  url = reverse('password_change')
  # sleep 3 sec to let min password age of 3 sec elapse
  time.sleep(3)
@@ -115,7 +119,8 @@ def test_password_change_complexity_and_min_age_validation(self):
 
  @override_switch('login', active=True)
  def test_password_change_reuse_validation(self):
- self.client.login(username="fred", password="foobarfoobarfoobar")
+ request = HttpRequest()
+ self.client.login(request=request, username="fred", password="foobarfoobarfoobar")
  url = reverse('password_change')
 
  # first password change

apps/accounts/tests/test_user_is_developer.py

@@ -1,6 +1,7 @@
+from django.contrib.auth.models import Group, User
+from django.http import HttpRequest
 from django.test import TestCase
 from django.test.client import Client
-from django.contrib.auth.models import Group, User
 from django.urls import reverse
 from waffle.testutils import override_switch
 
@@ -27,7 +28,8 @@ def setUp(self):
 
  @override_switch('show_testclient_link', active=True)
  def test_developer_can_register_apps(self):
- self.client.login(username="fred", password="foobar")
+ request = HttpRequest()
+ self.client.login(request=request, username="fred", password="foobar")
  response = self.client.get(self.url, follow=True)
  self.assertEqual(response.status_code, 200)
  self.assertContains(response, 'Logout')

apps/accounts/urls.py

@@ -46,7 +46,7 @@
  template_name='registration/password_forgot_reset_done.html')),
  name='password_reset_done'),
 
- url(r'^password-reset-confirm/(?P<uidb64>[0-9A-Za-z_\-]+)/(?P<token>[0-9A-Za-z]{1,13}-[0-9A-Za-z]{1,20})/$',
+ url(r'^password-reset-confirm/(?P<uidb64>[0-9A-Za-z_\-]+)/(?P<token>[0-9A-Za-z]{1,13}-[0-9A-Za-z]{1,50})/$',
  waffle_switch('login')(PasswordResetConfirmView.as_view(
  template_name='registration/password_forgot_reset_confirm_form.html')),
  name='password_reset_confirm'),

apps/accounts/v2/urls.py

@@ -46,7 +46,7 @@
  template_name='registration/password_forgot_reset_done.html')),
  name='password_reset_done_v2'),
 
- url(r'^password-reset-confirm/(?P<uidb64>[0-9A-Za-z_\-]+)/(?P<token>[0-9A-Za-z]{1,13}-[0-9A-Za-z]{1,20})/$',
+ url(r'^password-reset-confirm/(?P<uidb64>[0-9A-Za-z_\-]+)/(?P<token>[0-9A-Za-z]{1,13}-[0-9A-Za-z]{1,50})/$',
  waffle_switch('login')(PasswordResetConfirmView.as_view(
  template_name='registration/password_forgot_reset_confirm_form.html')),
  name='password_reset_confirm_v2'),

apps/authorization/migrations/0001_squashed_0005_auto_20190227_1737.py

@@ -23,7 +23,7 @@ class Migration(migrations.Migration):
  ('dot_ext', '__first__'),
  migrations.swappable_dependency(settings.AUTH_USER_MODEL),
  ('dot_ext', '0001_squashed_0025_auto_20210513_1812'),
- ('oauth2_provider', '0006_auto_20171214_2232'),
+ ('oauth2_provider', '0004_auto_20200902_2022'),
  ]
 
  operations = [

apps/authorization/migrations/0003_auto_20181203_1843.py

@@ -14,7 +14,7 @@ class Migration(migrations.Migration):
  atomic = False
 
  dependencies = [
- ('oauth2_provider', '0006_auto_20171214_2232'),
+ ('oauth2_provider', '0004_auto_20200902_2022'),
  ('dot_ext', '0013_auto_20181221_2114'),
  ('authorization', '0002_auto_20181203_1542'),
  ]

apps/authorization/models.py

@@ -178,9 +178,23 @@ def get_grant_bene_counts(application=None):
  synthetic_archived_queryset
  )
 
+ # Django 3.2.13 upgrade: seems need to re-write the query to work around?
+ # django.db.utils.NotSupportedError: Calling QuerySet.distinct() after union() is not supported
+ # and below is the quote from Django doc:
+ #
+ # union()
+ # union(*other_qs, all=False)
+ # Uses SQL UNION operator to combine the results of two or more QuerySets. For example:
+ #
+ # >>> qs1.union(qs2, qs3)
+ # The UNION operator selects only distinct values by default. To allow duplicate values, use the all=True argument.
+
+ # counts_returned[
+ # "grant_and_archived_real_deduped"
+ # ] = real_union_queryset.distinct().count()
  counts_returned[
  "grant_and_archived_real_deduped"
- ] = real_union_queryset.distinct().count()
+ ] = real_union_queryset.count()
  counts_returned[
  "grant_and_archived_synthetic_deduped"
  ] = synthetic_union_queryset.count()

apps/authorization/tests/test_data_access_grant.py

@@ -1,6 +1,8 @@
+from django.http import HttpRequest
 from django.utils import timezone
 from datetime import timedelta
-from oauth2_provider.compat import parse_qs, urlparse
+# from oauth2_provider.compat import parse_qs, urlparse
+from urllib.parse import parse_qs, urlparse
 from oauth2_provider.models import (
  get_application_model,
  get_access_token_model,
@@ -32,7 +34,8 @@ def test_creation_on_approval(self):
  application.scope.add(capability_a, capability_b)
 
  # user logs in
- self.client.login(username='anna', password='123456')
+ request = HttpRequest()
+ self.client.login(request=request, username='anna', password='123456')
 
  payload = {
  'client_id': application.client_id,
@@ -81,7 +84,8 @@ def test_no_action_on_reapproval(self):
  application.scope.add(capability_a, capability_b)
 
  # user logs in
- self.client.login(username='anna', password='123456')
+ request = HttpRequest()
+ self.client.login(request=request, username='anna', password='123456')
 
  payload = {
  'client_id': application.client_id,
@@ -225,7 +229,8 @@ def test_permission_deny_on_app_or_org_disabled(self):
  application.active = False
  application.save()
  # user logs in
- self.client.login(username='anna', password='123456')
+ request = HttpRequest()
+ self.client.login(request=request, username='anna', password='123456')
 
  payload = {
  'client_id': application.client_id,

apps/bb2_tools/apps.py

@@ -3,5 +3,5 @@
 
 class BB2ToolsConfig(AppConfig):
  name = 'apps.bb2_tools'
- label = 'BlueButton_2.0_TOOLS'
+ label = 'bb2_tools'
  verbose_name = "Blue Button 2.0 Tools"

apps/bb2_tools/migrations/0001_initial.py

@@ -9,7 +9,7 @@ class Migration(migrations.Migration):
 
  dependencies = [
  ('accounts', '0001_squashed_0041_auto_20210324_1543'),
- ('oauth2_provider', '0006_auto_20171214_2232'),
+ ('oauth2_provider', '0004_auto_20200902_2022'),
  ('bluebutton', '0001_squashed_0006_auto_20210513_1812'),
  ('dot_ext', '0001_squashed_0025_auto_20210513_1812'),
  ]

apps/capabilities/models.py

@@ -3,7 +3,8 @@
 
 from django.db import models
 from django.contrib.auth.models import Group
-from django.utils.lru_cache import lru_cache
+# from django.utils.lru_cache import lru_cache
+from functools import lru_cache
 from django.db.models import CASCADE
 
 

apps/dot_ext/migrations/0001_initial.py

@@ -4,9 +4,15 @@
 
 from django.conf import settings
 from django.db import migrations, models
+from oauth2_provider.validators import RedirectURIValidator
+
 import django.db.models.deletion
 import oauth2_provider.generators
-import oauth2_provider.validators
+
+
+def check_redirect_uri(value):
+ validator = RedirectURIValidator()
+ validator(value)
 
 
 class Migration(migrations.Migration):
@@ -24,7 +30,7 @@ class Migration(migrations.Migration):
  fields=[
  ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
  ('client_id', models.CharField(db_index=True, default=oauth2_provider.generators.generate_client_id, max_length=100, unique=True)),
- ('redirect_uris', models.TextField(blank=True, help_text='Allowed URIs list, space separated', validators=[oauth2_provider.validators.validate_uris])),
+ ('redirect_uris', models.TextField(blank=True, help_text='Allowed URIs list, space separated', validators=[check_redirect_uri])),
  ('client_type', models.CharField(choices=[('confidential', 'Confidential'), ('public', 'Public')], max_length=32)),
  ('authorization_grant_type', models.CharField(choices=[('authorization-code', 'Authorization code'), ('implicit', 'Implicit'), ('password', 'Resource owner password-based'), ('client-credentials', 'Client credentials')], max_length=32)),
  ('client_secret', models.CharField(blank=True, db_index=True, default=oauth2_provider.generators.generate_client_secret, max_length=255)),

apps/dot_ext/migrations/0003_auto_20220422_2147.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.13 on 2022-04-22 21:47
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('dot_ext', '0002_authflowuuidcopy'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='application',
+ name='algorithm',
+ field=models.CharField(blank=True, choices=[('', 'No OIDC support'), ('RS256', 'RSA with SHA-2 256'), ('HS256', 'HMAC with SHA-2 256')], default='', max_length=5),
+ ),
+ migrations.AlterField(
+ model_name='application',
+ name='authorization_grant_type',
+ field=models.CharField(choices=[('authorization-code', 'Authorization code'), ('implicit', 'Implicit'), ('password', 'Resource owner password-based'), ('client-credentials', 'Client credentials'), ('openid-hybrid', 'OpenID connect hybrid')], max_length=32),
+ ),
+ ]

apps/dot_ext/signals.py

@@ -4,8 +4,8 @@
 from django.db.models.signals import post_save, pre_save
 from oauth2_provider.models import get_application_model, get_access_token_model
 from libs.mail import Mailer
-from .models import ArchivedToken
 from libs.decorators import waffle_function_switch
+from .models import ArchivedToken
 
 import apps.logging.request_logger as bb2logging