Clean up lint failures (#74)

.github/workflows/lint.yml

@@ -16,10 +16,7 @@ name: Lint Code Base
 #############################
 on:
  push:
- branches-ignore: [main]
- # Remove the line above to run when pushing to master
  pull_request:
- branches: [main]
 
 ###############
 # Set the Job #
@@ -56,4 +53,7 @@ jobs:
  VALIDATE_BASH: false
  VALIDATE_TERRAFORM_TERRASCAN: false
  VALIDATE_MARKDOWN: false
+ VALIDATE_HTML: false
+ VALIDATE_JSCPD: false
+ VALIDATE_PYTHON_ISORT: false
  FILTER_REGEX_EXCLUDE: monitoring/.*

.github/workflows/terraform_plan.yml

@@ -25,7 +25,6 @@ jobs:
  valid_workflow: ${{ steps.validate_workflow.outputs.valid_workflow }}
  steps:
  - uses: actions/checkout@v2
- - uses: psf/black@ae2c0758c9e61a385df9700dc9c231bf54887041 #v22.3.0
  - name: Install and configure Poetry
  uses: snok/install-poetry@2bf112a0f6979928eb6b011f39700db589c5961e #v1
  with:
@@ -45,7 +44,7 @@ jobs:
  run: |
  cd scripts
  output=$(poetry run python validate.py)
- echo "::set-output name=valid_workflow::$output"
+ echo "valid_workflow=$output" >> "$GITHUB_OUTPUT"
 
  plan:
  env:

.github/workflows/test.yml

@@ -6,11 +6,6 @@ on:
  - main
  paths:
  - terraform/tvm_bot/**
- pull_request_target:
- branches:
- - main
- paths:
- - terraform/tvm_bot/**
 
 jobs:
  test:

jenkins/jenkins-jobs/update.sh

@@ -1,8 +1,9 @@
 #!/bin/bash
 git clone https://opendev.org/jjb/jenkins-job-builder.git
-cd jenkins-job-builder
+cd jenkins-job-builder || exit 1
 virtualenv .venv
+# shellcheck disable=SC1091
 source .venv/bin/activate
 pip install -r test-requirements.txt -e .
-python jenkins_jobs/__main__.py --user tvm-bot --password $1 --conf "/home/ubuntu/jenkins-jobs/$2/jenkins_jobs.ini" update "/home/ubuntu/jenkins-jobs/$2"
+python jenkins_jobs/__main__.py --user tvm-bot --password "$1" --conf "/home/ubuntu/jenkins-jobs/$2/jenkins_jobs.ini" update "/home/ubuntu/jenkins-jobs/$2"
 

jenkins/readme.md

@@ -1,10 +1,10 @@
-## Components
+# Components
 
-### Docker
+## Docker
 
 The 'jenkins' docker image contains the build tooling for the docker image that runs the head node. There is also a container for a Jenkins agent (for the documentation stage of the TVM pipeline)
 
-### Glossary of terms
+## Glossary of terms
 | Term | Meaning | Notes |
 --- | --- | ---
 | Head (node) | This is the primary Jenkins server/process, it schedules jobs | |

jenkins/scripts/common.sh

@@ -1,13 +1,15 @@
 #!/bin/bash
+# shellcheck disable=SC2129
 
-echo "ENVIRONMENT=prod" >> $GITHUB_ENV
-echo "STATE_PATH=tvm-ci-prod" >> $GITHUB_ENV
-echo "OAUTH_CLIENT_ID=${OAUTH_CLIENT_ID_PROD}" >> $GITHUB_ENV
-echo "OAUTH_CLIENT_SECRET=${OAUTH_CLIENT_SECRET_PROD}" >> $GITHUB_ENV
+echo "ENVIRONMENT=prod" >> "$GITHUB_ENV"
+echo "STATE_PATH=tvm-ci-prod" >> "$GITHUB_ENV"
+echo "OAUTH_CLIENT_ID=${OAUTH_CLIENT_ID_PROD}" >> "$GITHUB_ENV"
+echo "OAUTH_CLIENT_SECRET=${OAUTH_CLIENT_SECRET_PROD}" >> "$GITHUB_ENV"
 
-echo 'JENKINS_PRIV_KEY<<EOF' >> $GITHUB_ENV
-echo "$JENKINS_PRIV_KEY_PROD" >> $GITHUB_ENV
-echo 'EOF' >> $GITHUB_ENV
-export JENKINS_PRIV_KEY=$JENKINS_PRIV_KEY_PROD
+echo 'JENKINS_PRIV_KEY<<EOF' >> "$GITHUB_ENV"
+echo "$JENKINS_PRIV_KEY_PROD" >> "$GITHUB_ENV"
+echo 'EOF' >> "$GITHUB_ENV"
+export JENKINS_PRIV_KEY="$JENKINS_PRIV_KEY_PROD"
 
-echo "JENKINS_PUB_KEY=$(ssh-keygen -yf /dev/stdin <<<$JENKINS_PRIV_KEY)" >> $GITHUB_ENV
+# shellcheck disable=SC2086
+echo "JENKINS_PUB_KEY=$(ssh-keygen -yf /dev/stdin <<<$JENKINS_PRIV_KEY)" >> "$GITHUB_ENV"

jenkins/scripts/set_trust_on_jobs.py

@@ -1,14 +1,18 @@
+import argparse
 import os
+
 import jenkins
-import argparse
 
 USER = os.environ["JENKINS_USER"]
 PW = os.environ["JENKINS_PW"]
 
 
 def add_fork_trust_plugin(xml: str) -> str:
  nobody = '<trust class="org.jenkinsci.plugins.github_branch_source.ForkPullRequestDiscoveryTrait$TrustNobody"/>'
- some_people = '<trust class="org.jenkinsci.plugins.githubScmTraitNotificationContext.ForkPullRequestDiscoveryTrait$TrustSomePeople" plugin="[email protected]"/>'
+ some_people = (
+ '<trust class="org.jenkinsci.plugins.githubScmTraitNotificationContext.'
+ 'ForkPullRequestDiscoveryTrait$TrustSomePeople" plugin="[email protected]"/>'
+ )
  return xml.replace(nobody, some_people)
 
 
@@ -28,7 +32,7 @@ def test():
  )
  parser.parse_args()
 
- server = jenkins.Jenkins("https://ci.tlcpack.ai", username=USER, password=PW)
+ server = jenkins.Jenkins("https://ci.tlcpack.ai", username=USER, password=PW) # type: ignore[attr-defined]
  jobs = server.get_jobs()
 
  to_update = []

lint_local.py

@@ -0,0 +1,120 @@
+import subprocess
+import argparse
+from pathlib import Path
+
+env = {
+ "SUPPRESS_POSSUM": "true",
+ "RUN_LOCAL": "true",
+ "USE_FIND_ALGORITHM": "true",
+ "LOG_LEVEL": "VERBOSE",
+}
+
+validators = {
+ "VALIDATE_ANSIBLE",
+ "VALIDATE_ARM",
+ "VALIDATE_BASH",
+ "VALIDATE_BASH_EXEC",
+ "VALIDATE_CPP",
+ "VALIDATE_CLANG_FORMAT",
+ "VALIDATE_CLOJURE",
+ "VALIDATE_CLOUDFORMATION",
+ "VALIDATE_COFFEESCRIPT",
+ "VALIDATE_CSHARP",
+ "VALIDATE_CSS",
+ "VALIDATE_DART",
+ "VALIDATE_DOCKERFILE_HADOLINT",
+ "VALIDATE_EDITORCONFIG",
+ "VALIDATE_ENV",
+ "VALIDATE_GHERKIN",
+ "VALIDATE_GITHUB_ACTIONS",
+ "VALIDATE_GITLEAKS",
+ "VALIDATE_GO",
+ "VALIDATE_GOOGLE_JAVA_FORMAT",
+ "VALIDATE_GROOVY",
+ "VALIDATE_HTML",
+ "VALIDATE_JAVA",
+ "VALIDATE_JAVASCRIPT_ES",
+ "VALIDATE_JAVASCRIPT_STANDARD",
+ "VALIDATE_JSCPD",
+ "VALIDATE_JSON",
+ "VALIDATE_JSX",
+ "VALIDATE_KOTLIN",
+ "VALIDATE_KOTLIN_ANDROID",
+ "VALIDATE_KUBERNETES_KUBECONFORM",
+ "VALIDATE_LATEX",
+ "VALIDATE_LUA",
+ "VALIDATE_MARKDOWN",
+ "VALIDATE_NATURAL_LANGUAGE",
+ "VALIDATE_OPENAPI",
+ "VALIDATE_PERL",
+ "VALIDATE_PHP",
+ "VALIDATE_PHP_BUILTIN",
+ "VALIDATE_PHP_PHPCS",
+ "VALIDATE_PHP_PHPSTAN",
+ "VALIDATE_PHP_PSALM",
+ "VALIDATE_POWERSHELL",
+ "VALIDATE_PROTOBUF",
+ "VALIDATE_PYTHON",
+ "VALIDATE_PYTHON_BLACK",
+ "VALIDATE_PYTHON_FLAKE8",
+ "VALIDATE_PYTHON_ISORT",
+ "VALIDATE_PYTHON_MYPY",
+ "VALIDATE_PYTHON_PYLINT",
+ "VALIDATE_R",
+ "VALIDATE_RAKU",
+ "VALIDATE_RUBY",
+ "VALIDATE_RUST_2015",
+ "VALIDATE_RUST_2018",
+ "VALIDATE_RUST_2021",
+ "VALIDATE_RUST_CLIPPY",
+ "VALIDATE_SCALAFMT",
+ "VALIDATE_SHELL_SHFMT",
+ "VALIDATE_SNAKEMAKE_LINT",
+ "VALIDATE_SNAKEMAKE_SNAKEFMT",
+ "VALIDATE_STATES",
+ "VALIDATE_SQL",
+ "VALIDATE_SQLFLUFF",
+ "VALIDATE_TEKTON",
+ "VALIDATE_TERRAFORM_FMT",
+ "VALIDATE_TERRAFORM_TERRASCAN",
+ "VALIDATE_TERRAFORM_TFLINT",
+ "VALIDATE_TERRAGRUNT",
+ "VALIDATE_TSX",
+ "VALIDATE_TYPESCRIPT_ES",
+ "VALIDATE_TYPESCRIPT_STANDARD",
+ "VALIDATE_XML",
+ "VALIDATE_YAML",
+}
+
+# To fix
+# BASH:[14]
+# DOCKERFILE_HADOLINT:[2]
+# PYTHON_PYLINT:[4]
+# PYTHON_FLAKE8:[17]
+# PYTHON_ISORT:[25]
+# PYTHON_MYPY:[13]
+# TERRAFORM_TFLINT:[8]
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--dir", default=".")
+ parser.add_argument("--image", default="github/super-linter:latest")
+ parser.add_argument("-l", "--lint", default="")
+ args = parser.parse_args()
+ volume = Path(args.dir).resolve()
+ cmd = ["docker", "run", "-v", f"{volume}:/tmp/lint"]
+
+ linters = [x.strip() for x in args.lint.split(",") if x.strip() != ""]
+
+ for key, value in env.items():
+ cmd.append("-e")
+ cmd.append(f"{key}={value}")
+
+ for validator in validators:
+ if not any(linter.lower() in validator.lower() for linter in linters):
+ cmd.append("-e")
+ cmd.append(f"{validator}=false")
+
+ cmd.append(args.image)
+ p = subprocess.run(cmd)
+ exit(p.returncode)

monitoring/images/Dockerfile

@@ -3,6 +3,6 @@ FROM python:3.9.12
 WORKDIR /opt/fetcher
 COPY *.py requirements.txt ./
 
-RUN python3 -m pip install -r requirements.txt
+RUN python3 -m pip install --no-cache-dir -r requirements.txt
 
-CMD python3 forward.py --forever --wait-minutes 15
+CMD ["python3", "forward.py", "--forever", "--wait-minutes", "15"]

monitoring/images/Dockerfile.tests

@@ -3,6 +3,6 @@ FROM python:3.9.12
 WORKDIR /opt/fetcher
 COPY *.py requirements.txt ./
 
-RUN python3 -m pip install -r requirements.txt
+RUN python3 -m pip install --no-cache-dir -r requirements.txt
 
-CMD python3 tests_fetcher.py --forever --wait-minutes 15
+CMD ["python3", "tests_fetcher.py", "--forever", "--wait-minutes", "15"]

monitoring/images/tests_fetcher.py

@@ -14,6 +14,7 @@
 from net import *
 from typing import *
 from forward import *
+from forward import Branch, blue, init, init_log
 import forward
 from urllib.parse import unquote
 

terraform/main.tf

@@ -8,6 +8,10 @@ terraform {
  required_version = ">= 1.1.9, < 2.0.0"
  required_providers {
  aws = "~> 3.73.0"
+ archive = {
+ source = "hashicorp/archive"
+ version = "2.3.0"
+ }
  }
 
  backend "s3" {

terraform/modules/autoscaler/main.tf

@@ -43,6 +43,7 @@ resource "aws_launch_template" "autoscaler" {
  instance_type = var.agent_instance_type
 }
 
+# tflint-ignore: terraform_required_providers
 resource "aws_autoscaling_group" "agents" {
  name = var.autoscaler_name
  min_size = var.min_size

terraform/modules/autoscaler/variables.tf

@@ -1,7 +1,9 @@
 variable "jenkins_pub_key" {
+ type = string
 }
 
 variable "executor_access_pub_keys" {
+ type = list(string)
 }
 
 variable "global_access_key_name" {
@@ -10,10 +12,12 @@ variable "global_access_key_name" {
 }
 
 variable "autoscaler_name" {
+ type = string
 }
 
 variable "agent_instance_type" {
  default = "c4.2xlarge"
+ type = string
  description = "The instance type runners will be created on."
 }
 

terraform/modules/head-node/main.tf

@@ -101,6 +101,7 @@ resource "aws_eip" "static" {
  }
 }
 
+# tflint-ignore: terraform_required_providers
 resource "aws_instance" "jenkins_head_node" {
  #Older jenkins-stock-agent AMI
  ami = "ami-056c71c88d13476dd"

terraform/modules/head-node/variables.tf

@@ -1,25 +1,31 @@
 variable "environment" {
+ type = string
 }
 
 variable "instance_type" {
+ type = string
 }
 
 variable "ebs_vol_size" {
  description = "The size of the persistent volume which stores runtime data"
+ type = number
 }
 
 variable "ebs_jobs_vol_size" {
  description = "The size of the persistent volume which stores job histories"
+ type = number
 }
 
 variable "vpc_id" {
+ type = string
 }
 
 variable "subnet_ids" {
  type = list(string)
 }
 
 variable "global_access_key_name" {
+ type = string
  description = "The name of the key used to access all executors for SSH authentication"
 }
 
@@ -33,10 +39,10 @@ variable "subject_alternative_names" {
  default = []
 }
 
-variable "disk_size" {
- type = number
- default = 100
-}
+# variable "disk_size" {
+#  type = number
+#  default = 100
+# }
 
 variable "internal_load_balancer" {
  type = bool

terraform/modules/persistent-agents/main.tf

@@ -50,6 +50,7 @@ resource "aws_launch_template" "template" {
  key_name = "tvm_ci_creds"
 }
 
+# tflint-ignore: terraform_required_providers
 resource "aws_instance" "instances" {
  count = var.replicas
  tags = {