Dependency bump + minor code updates (log2timeline#367)

* Bump requirements

* Fix gcp_logging monkey patching

* Fix default args

* Adjust e2e tests to create analysis VM

* Fix test

dftimewolf/lib/collectors/gcloud.py

@@ -89,8 +89,8 @@ def SetUp(self,
  remote_instance_name=None,
  disk_names=None,
  all_disks=False,
- image_project=None,
- image_family=None):
+ image_project='ubuntu-os-cloud',
+ image_family='ubuntu-1804-lts'):
  """Sets up a Google Cloud Platform(GCP) collector.
 
  This method creates and starts an analysis VM in the analysis project and

dftimewolf/lib/collectors/gcp_logging.py

@@ -6,6 +6,8 @@
 from google.api_core import exceptions as google_api_exceptions
 from google.auth import exceptions as google_auth_exceptions
 from google.cloud import logging
+from google.cloud.logging_v2 import entries
+
 from googleapiclient.errors import HttpError
 
 from dftimewolf.lib import module
@@ -20,12 +22,12 @@
 # https://github.com/googleapis/google-cloud-python/issues/7918
 def _CustomToAPIRepr(self):
  """API repr (JSON format) for entry."""
- info = super(logging.entries.ProtobufEntry, self).to_api_repr()
+ info = super(entries.ProtobufEntry, self).to_api_repr()
  info['protoPayload'] = self.payload
  return info
 
 
-logging.entries.ProtobufEntry.to_api_repr = _CustomToAPIRepr
+entries.ProtobufEntry.to_api_repr = _CustomToAPIRepr
 
 
 class GCPLogsCollector(module.BaseModule):

requirements-dev.txt

@@ -1,86 +1,86 @@
 -i https://pypi.org/simple
-adal==1.2.4
+adal==1.2.5
 altair==4.1.0; python_version >= '3.6'
 astroid==2.4.2; python_version >= '3.5'
-attrs==20.2.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-azure-common==1.1.25
-azure-core==1.8.2
-azure-identity==1.4.1
-azure-mgmt-compute==17.0.0
-azure-mgmt-core==1.2.1
+attrs==20.3.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+azure-common==1.1.26
+azure-core==1.9.0
+azure-identity==1.5.0
+azure-mgmt-compute==18.0.0
+azure-mgmt-core==1.2.2
 azure-mgmt-monitor==1.0.1
 azure-mgmt-network==16.0.0
 azure-mgmt-resource==15.0.0
 azure-mgmt-storage==16.0.0
-azure-storage-blob==12.5.0
+azure-storage-blob==12.6.0
 beautifulsoup4==4.9.3
-boto3==1.15.18
-botocore==1.18.18
+boto3==1.16.24
+botocore==1.19.24
 cachetools==4.1.1; python_version ~= '3.5'
-certifi==2020.6.20
-cffi==1.14.3
+certifi==2020.11.8
+cffi==1.14.4
 chardet==3.0.4
 click==7.1.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 colorlog==2.10.0
 coverage==5.3
 cryptography==2.9.2
-ecdsa==0.16.0; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
+ecdsa==0.16.1; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'
 entrypoints==0.3; python_version >= '2.7'
 filelock==3.0.12
-google-api-core[grpc]==1.22.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-google-api-python-client==1.12.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+google-api-core[grpc]==1.23.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+google-api-python-client==1.12.8; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 google-auth-httplib2==0.0.4
-google-auth-oauthlib==0.4.1
-google-auth==1.22.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+google-auth-oauthlib==0.4.2; python_version >= '3.6'
+google-auth==1.23.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 google-cloud-core==1.4.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-google-cloud-datastore==1.15.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-google-cloud-logging==1.15.1
-google-cloud-pubsub==2.1.0; python_version >= '3.6'
-google-cloud-storage==1.31.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+google-cloud-datastore==2.0.0; python_version >= '3.6'
+google-cloud-logging==2.0.0
+google-cloud-pubsub==1.7.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+google-cloud-storage==1.33.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
 google-crc32c==1.0.0; python_version >= '3.5'
 google-resumable-media==1.1.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 googleapis-common-protos[grpc]==1.52.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 grpc-google-iam-v1==0.12.3
-grpcio==1.32.0
+grpcio==1.33.2
 grr-api-client==3.4.2.post4
 grr-response-proto==3.4.2.post4
 httplib2==0.17.4
 idna==2.10
-importlib-metadata==2.0.0; python_version < '3.8'
+importlib-metadata==3.1.0; python_version < '3.8'
 isodate==0.6.0
 isort==5.6.4; python_version >= '3.6' and python_version < '4.0'
 jinja2==2.11.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-jmespath==0.10.0; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
+jmespath==0.10.0; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'
 jsonschema==3.2.0
 lazy-object-proxy==1.4.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-libcloudforensics==20201016
-libcst==0.3.13; python_version >= '3.6'
+libcloudforensics==20201124
+libcst==0.3.14; python_version >= '3.6'
 markupsafe==1.1.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 mccabe==0.6.1
 mock==4.0.2
-msal-extensions==0.2.2
-msal==1.5.0
+msal-extensions==0.3.0
+msal==1.6.0
 msrest==0.6.19
 msrestazure==0.6.4
 mypy-extensions==0.4.3
-numpy==1.19.2; python_version >= '3.6'
+numpy==1.19.4; python_version >= '3.6'
 oauthlib==3.1.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-pandas==1.1.3
+pandas==1.1.4
 portalocker==1.7.1; platform_system != 'Windows'
-proto-plus==1.10.2
+proto-plus==1.11.0
 protobuf==3.12.2
 psq==0.8.0
 pyasn1-modules==0.2.8
 pyasn1==0.4.8
 pycparser==2.20; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-pycryptodome==3.9.8; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
+pycryptodome==3.9.9; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
 pyjwt[crypto]==1.7.1
 pylint==2.6.0
 pyopenssl==19.1.0
 pypdf2==1.26.0
 pyrsistent==0.17.3; python_version >= '3.5'
-python-dateutil==2.8.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-pytz==2020.1
+python-dateutil==2.8.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'
+pytz==2020.4
 pyyaml==5.3.1
 requests-oauthlib==1.3.0
 requests==2.23.0
@@ -89,17 +89,17 @@ s3transfer==0.3.3
 six==1.15.0
 soupsieve==2.0.1; python_version >= '3.0'
 sshpubkeys==3.1.0
-timesketch-api-client==20201015
+timesketch-api-client==20201117
 timesketch-import-client==20200910
-toml==0.10.1
+toml==0.10.2; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'
 toolz==0.11.1; python_version >= '3.5'
-turbinia==20190819.16
-typed-ast==1.4.1; python_version < '3.8' and implementation_name == 'cpython'
-typing-extensions==3.7.4.3; python_version < '3.8'
+turbinia==20190819.19
+typed-ast==1.4.1; implementation_name == 'cpython' and python_version < '3.8'
+typing-extensions==3.7.4.3
 typing-inspect==0.6.0
 uritemplate==3.0.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-urllib3[secure]==1.25.10; python_version != '3.4'
+urllib3[secure]==1.25.11; python_version != '3.4'
 werkzeug==0.16.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 wrapt==1.12.1
 xlrd==1.2.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-zipp==3.3.1; python_version >= '3.6'
+zipp==3.4.0; python_version >= '3.6'

requirements.txt

@@ -1,79 +1,79 @@
 -i https://pypi.org/simple
-adal==1.2.4
+adal==1.2.5
 altair==4.1.0; python_version >= '3.6'
-attrs==20.2.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-azure-common==1.1.25
-azure-core==1.8.2
-azure-identity==1.4.1
-azure-mgmt-compute==17.0.0
-azure-mgmt-core==1.2.1
+attrs==20.3.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+azure-common==1.1.26
+azure-core==1.9.0
+azure-identity==1.5.0
+azure-mgmt-compute==18.0.0
+azure-mgmt-core==1.2.2
 azure-mgmt-monitor==1.0.1
 azure-mgmt-network==16.0.0
 azure-mgmt-resource==15.0.0
 azure-mgmt-storage==16.0.0
-azure-storage-blob==12.5.0
+azure-storage-blob==12.6.0
 beautifulsoup4==4.9.3
-boto3==1.15.18
-botocore==1.18.18
+boto3==1.16.24
+botocore==1.19.24
 cachetools==4.1.1; python_version ~= '3.5'
-certifi==2020.6.20
-cffi==1.14.3
+certifi==2020.11.8
+cffi==1.14.4
 chardet==3.0.4
 click==7.1.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 colorlog==2.10.0
 cryptography==2.9.2
-ecdsa==0.16.0; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
+ecdsa==0.16.1; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'
 entrypoints==0.3; python_version >= '2.7'
 filelock==3.0.12
-google-api-core[grpc]==1.22.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-google-api-python-client==1.12.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+google-api-core[grpc]==1.23.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+google-api-python-client==1.12.8; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 google-auth-httplib2==0.0.4
-google-auth-oauthlib==0.4.1
-google-auth==1.22.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+google-auth-oauthlib==0.4.2; python_version >= '3.6'
+google-auth==1.23.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 google-cloud-core==1.4.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-google-cloud-datastore==1.15.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-google-cloud-logging==1.15.1
-google-cloud-pubsub==2.1.0; python_version >= '3.6'
-google-cloud-storage==1.31.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+google-cloud-datastore==2.0.0; python_version >= '3.6'
+google-cloud-logging==2.0.0
+google-cloud-pubsub==1.7.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+google-cloud-storage==1.33.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
 google-crc32c==1.0.0; python_version >= '3.5'
 google-resumable-media==1.1.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 googleapis-common-protos[grpc]==1.52.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 grpc-google-iam-v1==0.12.3
-grpcio==1.32.0
+grpcio==1.33.2
 grr-api-client==3.4.2.post4
 grr-response-proto==3.4.2.post4
 httplib2==0.17.4
 idna==2.10
-importlib-metadata==2.0.0; python_version < '3.8'
+importlib-metadata==3.1.0; python_version < '3.8'
 isodate==0.6.0
 jinja2==2.11.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-jmespath==0.10.0; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
+jmespath==0.10.0; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'
 jsonschema==3.2.0
-libcloudforensics==20201016
-libcst==0.3.13; python_version >= '3.6'
+libcloudforensics==20201124
+libcst==0.3.14; python_version >= '3.6'
 markupsafe==1.1.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-msal-extensions==0.2.2
-msal==1.5.0
+msal-extensions==0.3.0
+msal==1.6.0
 msrest==0.6.19
 msrestazure==0.6.4
 mypy-extensions==0.4.3
-numpy==1.19.2; python_version >= '3.6'
+numpy==1.19.4; python_version >= '3.6'
 oauthlib==3.1.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-pandas==1.1.3
+pandas==1.1.4
 portalocker==1.7.1; platform_system != 'Windows'
-proto-plus==1.10.2
+proto-plus==1.11.0
 protobuf==3.12.2
 psq==0.8.0
 pyasn1-modules==0.2.8
 pyasn1==0.4.8
 pycparser==2.20; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-pycryptodome==3.9.8; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
+pycryptodome==3.9.9; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
 pyjwt[crypto]==1.7.1
 pyopenssl==19.1.0
 pypdf2==1.26.0
 pyrsistent==0.17.3; python_version >= '3.5'
-python-dateutil==2.8.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-pytz==2020.1
+python-dateutil==2.8.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'
+pytz==2020.4
 pyyaml==5.3.1
 requests-oauthlib==1.3.0
 requests==2.23.0
@@ -82,14 +82,14 @@ s3transfer==0.3.3
 six==1.15.0
 soupsieve==2.0.1; python_version >= '3.0'
 sshpubkeys==3.1.0
-timesketch-api-client==20201015
+timesketch-api-client==20201117
 timesketch-import-client==20200910
 toolz==0.11.1; python_version >= '3.5'
-turbinia==20190819.16
-typing-extensions==3.7.4.3; python_version < '3.8'
+turbinia==20190819.19
+typing-extensions==3.7.4.3
 typing-inspect==0.6.0
 uritemplate==3.0.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-urllib3[secure]==1.25.10; python_version != '3.4'
+urllib3[secure]==1.25.11; python_version != '3.4'
 werkzeug==0.16.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 xlrd==1.2.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-zipp==3.3.1; python_version >= '3.6'
+zipp==3.4.0; python_version >= '3.6'

tests/lib/collectors/gcloud.py

@@ -94,8 +94,8 @@ def testSetUp(self,
  42.0,
  'pd-standard',
  16,
- image_family=None,
- image_project=None
+ image_project='ubuntu-os-cloud',
+ image_family='ubuntu-1804-lts'
  )
  mock_AddLabels.assert_has_calls(
  [mock.call({'incident_id': 'fake_incident_id'})])

tests/lib/collectors/gcloud_e2e.py

@@ -97,6 +97,7 @@ def test_end_to_end_boot_disk(self):
  self.project_id,
  self.incident_id,
  self.zone,
+ True,
  42.0,
  'pd-standard',
  16,
@@ -148,6 +149,7 @@ def test_end_to_end_other_disk(self):
  self.project_id,
  self.incident_id,
  self.zone,
+ True,
  42.0,
  'pd-standard',
  16,