Merge branch 'main' into rbPoly

github · Feb 13, 2023 · 634087b · 634087b
2 parents 80d05c0 + f07c598
commit 634087b
Show file tree

Hide file tree

Showing 871 changed files with 114,873 additions and 10,913 deletions.
diff --git a/.github/actions/find-latest-bundle/action.yml b/.github/actions/find-latest-bundle/action.yml
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -5,6 +5,11 @@ updates:
  schedule:
  interval: "daily"
 
+ - package-ecosystem: "cargo"
+ directory: "ql"
+ schedule:
+ interval: "daily"
+
  - package-ecosystem: "github-actions"
  directory: "/"
  schedule:

diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -28,9 +28,9 @@ jobs:
 
  steps:
  - name: Setup dotnet
- uses: actions/setup-dotnet@v2
+ uses: actions/setup-dotnet@v3
  with:
- dotnet-version: 6.0.202
+ dotnet-version: 7.0.102
 
  - name: Checkout repository
  uses: actions/checkout@v3

diff --git a/.github/workflows/csharp-qltest.yml b/.github/workflows/csharp-qltest.yml
@@ -77,10 +77,10 @@ jobs:
  - name: Setup dotnet
  uses: actions/setup-dotnet@v3
  with:
- dotnet-version: 6.0.202
+ dotnet-version: 7.0.102
  - name: Extractor unit tests
  run: |
- dotnet test -p:RuntimeFrameworkVersion=6.0.4 "${{ github.workspace }}/csharp/extractor/Semmle.Util.Tests"
- dotnet test -p:RuntimeFrameworkVersion=6.0.4 "${{ github.workspace }}/csharp/extractor/Semmle.Extraction.Tests"
- dotnet test -p:RuntimeFrameworkVersion=6.0.4 "${{ github.workspace }}/csharp/autobuilder/Semmle.Autobuild.CSharp.Tests"
- dotnet test -p:RuntimeFrameworkVersion=6.0.4 "${{ github.workspace }}/cpp/autobuilder/Semmle.Autobuild.Cpp.Tests"
+ dotnet test -p:RuntimeFrameworkVersion=7.0.2 "${{ github.workspace }}/csharp/extractor/Semmle.Util.Tests"
+ dotnet test -p:RuntimeFrameworkVersion=7.0.2 "${{ github.workspace }}/csharp/extractor/Semmle.Extraction.Tests"
+ dotnet test -p:RuntimeFrameworkVersion=7.0.2 "${{ github.workspace }}/csharp/autobuilder/Semmle.Autobuild.CSharp.Tests"
+ dotnet test -p:RuntimeFrameworkVersion=7.0.2 "${{ github.workspace }}/cpp/autobuilder/Semmle.Autobuild.Cpp.Tests"
diff --git a/.github/workflows/ql-for-ql-build.yml b/.github/workflows/ql-for-ql-build.yml
@@ -22,144 +22,54 @@ jobs:
  steps:
  ### Build the queries ###
  - uses: actions/checkout@v3
- - name: Find latest bundle
- id: find-latest-bundle
- uses: ./.github/actions/find-latest-bundle
  - name: Find codeql
  id: find-codeql
- uses: github/codeql-action/init@45955cb1830b640e2c1603ad72ad542a49d47b96
+ uses: github/codeql-action/init@v2
  with:
  languages: javascript # does not matter
- tools: ${{ steps.find-latest-bundle.outputs.url }}
- - name: Get CodeQL version
- id: get-codeql-version
- run: |
- echo "version=$("${CODEQL}" --version | head -n 1 | rev | cut -d " " -f 1 | rev)" >> $GITHUB_OUTPUT
- shell: bash
- env:
- CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
  - uses: ./.github/actions/os-version
  id: os_version
- - name: Cache entire pack
- id: cache-pack
- uses: actions/cache@v3
- with:
- path: ${{ runner.temp }}/pack
- key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-pack-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/**/qlpack.yml') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}--${{ hashFiles('.github/workflows/ql-for-ql-build.yml') }}
- - name: Cache queries
- if: steps.cache-pack.outputs.cache-hit != 'true'
- id: cache-queries
- uses: actions/cache@v3
- with:
- path: ${{ runner.temp }}/queries
- key: queries-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/**/qlpack.yml') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}--${{ hashFiles('.github/workflows/ql-for-ql-build.yml') }}
- - name: Build query pack
- if: steps.cache-queries.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
- run: |
- cd ql/ql/src
- "${CODEQL}" pack create -j 16
- mv .codeql/pack/codeql/ql/0.0.0 ${{ runner.temp }}/queries
- env:
- CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
- - name: Move cache queries to pack
- if: steps.cache-pack.outputs.cache-hit != 'true'
- run: |
- cp -r ${{ runner.temp }}/queries ${{ runner.temp }}/pack
- env:
- CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-
  ### Build the extractor ###
  - name: Cache entire extractor
- if: steps.cache-pack.outputs.cache-hit != 'true'
  id: cache-extractor
  uses: actions/cache@v3
  with:
- path: |
- ql/target/release/ql-autobuilder
- ql/target/release/ql-autobuilder.exe
- ql/target/release/ql-extractor
- ql/target/release/ql-extractor.exe
+ path: ql/extractor-pack/
  key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-extractor-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}
  - name: Cache cargo
- if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
+ if: steps.cache-extractor.outputs.cache-hit != 'true'
  uses: actions/cache@v3
  with:
  path: |
  ~/.cargo/registry
  ~/.cargo/git
  ql/target
  key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-rust-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
- - name: Check formatting
- if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
- run: cd ql; cargo fmt --all -- --check
- - name: Build
- if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
- run: cd ql; cargo build --verbose
- - name: Run tests
- if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
- run: cd ql; cargo test --verbose
  - name: Release build
- if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
- run: cd ql; cargo build --release
- - name: Generate dbscheme
- if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
- run: ql/target/release/ql-generator --dbscheme ql/ql/src/ql.dbscheme --library ql/ql/src/codeql_ql/ast/internal/TreeSitter.qll
-
- ### Package the queries and extractor ###
- - name: Package pack
- if: steps.cache-pack.outputs.cache-hit != 'true'
- run: |
- cp -r ql/codeql-extractor.yml ql/tools ql/ql/src/ql.dbscheme.stats ${PACK}/
- mkdir -p ${PACK}/tools/linux64
- cp ql/target/release/ql-autobuilder ${PACK}/tools/linux64/autobuilder
- cp ql/target/release/ql-extractor ${PACK}/tools/linux64/extractor
- chmod +x ${PACK}/tools/linux64/autobuilder
- chmod +x ${PACK}/tools/linux64/extractor
- env:
- PACK: ${{ runner.temp }}/pack
-
- ### Run the analysis ###
- - name: Hack codeql-action options
- run: |
- JSON=$(jq -nc --arg pack "${PACK}" '.database."run-queries"=["--search-path", $pack] | .resolve.queries=["--search-path", $pack] | .resolve.extractor=["--search-path", $pack] | .resolve.languages=["--search-path", $pack] | .database.init=["--search-path", $pack]')
- echo "CODEQL_ACTION_EXTRA_OPTIONS=${JSON}" >> ${GITHUB_ENV}
- env:
- PACK: ${{ runner.temp }}/pack
-
- - name: Create CodeQL config file
- run: |
- echo "paths-ignore:" >> ${CONF}
- echo " - ql/ql/test" >> ${CONF}
- echo " - \"*/ql/lib/upgrades/\"" >> ${CONF}
- echo "disable-default-queries: true" >> ${CONF}
- echo "queries:" >> ${CONF}
- echo " - uses: ./ql/ql/src/codeql-suites/ql-code-scanning.qls" >> ${CONF}
- echo "Config file: "
- cat ${CONF}
+ if: steps.cache-extractor.outputs.cache-hit != 'true'
+ run: cd ql; ./scripts/create-extractor-pack.sh 
  env:
- CONF: ./ql-for-ql-config.yml
- - name: Initialize CodeQL
- uses: github/codeql-action/init@45955cb1830b640e2c1603ad72ad542a49d47b96
- with:
- languages: ql
- db-location: ${{ runner.temp }}/db
- config-file: ./ql-for-ql-config.yml
- tools: ${{ steps.find-latest-bundle.outputs.url }}
- - name: Move pack queries
+ GH_TOKEN: ${{ github.token }} 
+ - name: Cache compilation cache
+ id: query-cache
+ uses: ./.github/actions/cache-query-compilation
+ with: 
+ key: run-ql-for-ql
+ - name: Make database and analyze
  run: |
- cp -r ${PACK}/queries ql/ql/src
- env:
- PACK: ${{ runner.temp }}/pack
-
- - name: Perform CodeQL Analysis
- uses: github/codeql-action/analyze@45955cb1830b640e2c1603ad72ad542a49d47b96
+ ${CODEQL} database create -l=ql --search-path ql/extractor-pack ${DB}
+ ${CODEQL} database analyze -j0 --format=sarif-latest --output=ql-for-ql.sarif ${DB} ql/ql/src/codeql-suites/ql-code-scanning.qls --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+ env: 
+ CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
+ DB: ${{ runner.temp }}/DB
+ LGTM_INDEX_FILTERS: |
+ exclude:ql/ql/test
+ exclude:*/ql/lib/upgrades/
+ - name: Upload sarif to code-scanning
+ uses: github/codeql-action/upload-sarif@v2
  with:
- category: "ql-for-ql"
- - name: Copy sarif file to CWD
- run: cp ../results/ql.sarif ./ql-for-ql.sarif
- - name: Fixup the $scema in sarif # Until https://github.com/microsoft/sarif-vscode-extension/pull/436/ is part in a stable release
- run: |
- sed -i 's/\$schema.*/\$schema": "https:\/\/raw.githubusercontent.com\/oasis-tcs\/sarif-spec\/master\/Schemata\/sarif-schema-2.1.0",/' ql-for-ql.sarif
+ sarif_file: ql-for-ql.sarif
+ category: ql-for-ql
  - name: Sarif as artifact
  uses: actions/upload-artifact@v3
  with:

diff --git a/.github/workflows/ql-for-ql-dataset_measure.yml b/.github/workflows/ql-for-ql-dataset_measure.yml
@@ -25,7 +25,7 @@ jobs:
 
  - name: Find codeql
  id: find-codeql
- uses: github/codeql-action/init@45955cb1830b640e2c1603ad72ad542a49d47b96
+ uses: github/codeql-action/init@v2
  with:
  languages: javascript # does not matter
  - uses: ./.github/actions/os-version

diff --git a/.github/workflows/ql-for-ql-tests.yml b/.github/workflows/ql-for-ql-tests.yml
@@ -6,11 +6,13 @@ on:
  paths:
  - "ql/**"
  - codeql-workspace.yml
+ - .github/workflows/ql-for-ql-tests.yml
  pull_request:
  branches: [main]
  paths:
  - "ql/**"
  - codeql-workspace.yml
+ - .github/workflows/ql-for-ql-tests.yml
 
 env:
  CARGO_TERM_COLOR: always
@@ -22,7 +24,7 @@ jobs:
  - uses: actions/checkout@v3
  - name: Find codeql
  id: find-codeql
- uses: github/codeql-action/init@45955cb1830b640e2c1603ad72ad542a49d47b96
+ uses: github/codeql-action/init@v2
  with:
  languages: javascript # does not matter
  - uses: ./.github/actions/os-version
@@ -34,6 +36,8 @@ jobs:
  ~/.cargo/git
  ql/target
  key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-qltest-cargo-${{ hashFiles('ql/rust-toolchain.toml', 'ql/**/Cargo.lock') }}
+ - name: Check formatting
+ run: cd ql; cargo fmt --all -- --check
  - name: Build extractor
  run: |
  cd ql;
@@ -65,7 +69,7 @@ jobs:
  echo "/usr/local/opt/gnu-tar/libexec/gnubin" >> $GITHUB_PATH
  - name: Find codeql
  id: find-codeql
- uses: github/codeql-action/init@77a8d2d10c0b403a8b4aadbd223dc489ecd22683
+ uses: github/codeql-action/init@v2
  with:
  languages: javascript # does not matter
  - uses: ./.github/actions/os-version

diff --git a/config/identical-files.json b/config/identical-files.json
@@ -34,9 +34,9 @@
  "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl2.qll",
  "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl3.qll",
  "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl4.qll",
+ "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplForRegExp.qll",
  "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl.qll",
  "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl2.qll",
- "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForRegExp.qll",
  "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForHttpClientLibraries.qll",
  "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForPathname.qll",
  "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImpl.qll"

diff --git a/cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/Semmle.Autobuild.Cpp.Tests.csproj b/cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/Semmle.Autobuild.Cpp.Tests.csproj
@@ -2,7 +2,7 @@
 
  <PropertyGroup>
  <OutputType>Exe</OutputType>
- <TargetFramework>net6.0</TargetFramework>
+ <TargetFramework>net7.0</TargetFramework>
  <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
  <RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
  <Nullable>enable</Nullable>

diff --git a/cpp/autobuilder/Semmle.Autobuild.Cpp/Semmle.Autobuild.Cpp.csproj b/cpp/autobuilder/Semmle.Autobuild.Cpp/Semmle.Autobuild.Cpp.csproj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
  <PropertyGroup>
- <TargetFramework>net6.0</TargetFramework>
+ <TargetFramework>net7.0</TargetFramework>
  <AssemblyName>Semmle.Autobuild.Cpp</AssemblyName>
  <RootNamespace>Semmle.Autobuild.Cpp</RootNamespace>
  <ApplicationIcon />

diff --git a/cpp/ql/lib/CHANGELOG.md b/cpp/ql/lib/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.5.2
+
+No user-facing changes.
+
 ## 0.5.1
 
 No user-facing changes.

diff --git a/cpp/ql/lib/change-notes/released/0.5.2.md b/cpp/ql/lib/change-notes/released/0.5.2.md
@@ -0,0 +1,3 @@
+## 0.5.2
+
+No user-facing changes.
diff --git a/cpp/ql/lib/codeql-pack.release.yml b/cpp/ql/lib/codeql-pack.release.yml
@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.5.1
+lastReleaseVersion: 0.5.2