Refactor PR checks

.gitattributes

@@ -1,4 +1,5 @@
 lib/*.js linguist-generated=true
+.github/workflows/__* linguist-generated=true
 
 # Reduce incidence of needless merge conflicts on CHANGELOG.md
 # The man page at

.github/prepare-test/action.yml

@@ -0,0 +1,39 @@
+name: "Prepare test"
+description: Performs some preparation to run tests
+inputs:
+  version:
+    required: true
+outputs:
+  tools-url:
+    value: ${{ steps.get-url.outputs.tools-url }}
+runs:
+  using: composite
+  steps:
+    - name: Move codeql-action
+      shell: bash
+      run: |
+        mkdir ../action
+        mv * .github ../action/
+        mv ../action/tests/multi-language-repo/{*,.github} .
+        mv ../action/.github/workflows .github
+    - id: get-url
+      name: Determine URL
+      shell: bash
+      run: |
+        if [[ ${{ inputs.version }} == "nightly-latest" ]]; then
+          export LATEST=`gh release list --repo dsp-testing/codeql-cli-nightlies -L 1 | cut -f 3`
+          echo "::set-output name=tools-url::https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/$LATEST/codeql-bundle.tar.gz"
+        elif [[ ${{ inputs.version }} == *"nightly"* ]]; then
+          export VERSION=`echo ${{ inputs.version }} | sed -e 's/^.*\-//'`
+          echo "::set-output name=tools-url::https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/codeql-bundle-$VERSION-manual/codeql-bundle.tar.gz"
+        elif [[ ${{ inputs.version }} == *"stable"* ]]; then
+          export VERSION=`echo ${{ inputs.version }} | sed -e 's/^.*\-//'`
+          echo "Hello $VERSION"
+          echo "::set-output name=tools-url::https://github.com/github/codeql-action/releases/download/codeql-bundle-$VERSION/codeql-bundle.tar.gz"
+        elif [[ ${{ inputs.version }} == "latest" ]]; then
+          echo "::set-output name=tools-url::latest"
+        elif [[ ${{ inputs.version }} == "cached" ]]; then
+          echo "::set-output name=tools-url::"
+        else
+          echo "::error Unrecognized version specified!"
+        fi

.github/workflows/__go-custom-queries.yml

@@ -0,0 +1,60 @@
+# Warning: This file is generated automatically, and should not be modified.
+# Instead, please modify the template in the pr-checks directory and run:
+#     pip install ruamel.yaml && python3 sync.py
+# to regenerate this file.
+
+name: 'PR Check - Go: Custom queries'
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  GO111MODULE: auto
+on:
+  push:
+    branches:
+    - main
+    - v1
+  pull_request:
+    types:
+    - opened
+    - synchronize
+    - reopened
+    - ready_for_review
+  workflow_dispatch: {}
+jobs:
+  go-custom-queries:
+    strategy:
+      matrix:
+        version:
+        - stable-20201028
+        - stable-20210319
+        - stable-20210809
+        - cached
+        - latest
+        - nightly-latest
+        os:
+        - ubuntu-latest
+        - macos-latest
+        - windows-latest
+    name: 'Go: Custom queries'
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v2
+    - name: Prepare test
+      id: prepare-test
+      uses: ./.github/prepare-test
+      with:
+        version: ${{ matrix.version }}
+    - uses: actions/setup-go@v2
+      with:
+        go-version: ^1.13.1
+    - uses: ./../action/init
+      with:
+        languages: go
+        config-file: ./.github/codeql/custom-queries.yml
+        tools: ${{ steps.prepare-test.outputs.tools-url }}
+    - name: Build code
+      shell: bash
+      run: ./build.sh
+    - uses: ./../action/analyze
+      env:
+        TEST_MODE: true

.github/workflows/__go-custom-tracing-autobuild.yml

@@ -0,0 +1,63 @@
+# Warning: This file is generated automatically, and should not be modified.
+# Instead, please modify the template in the pr-checks directory and run:
+#     pip install ruamel.yaml && python3 sync.py
+# to regenerate this file.
+
+name: 'PR Check - Go: Autobuild custom tracing'
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  GO111MODULE: auto
+on:
+  push:
+    branches:
+    - main
+    - v1
+  pull_request:
+    types:
+    - opened
+    - synchronize
+    - reopened
+    - ready_for_review
+  workflow_dispatch: {}
+jobs:
+  go-custom-tracing-autobuild:
+    strategy:
+      matrix:
+        version:
+        - stable-20201028
+        - stable-20210319
+        - stable-20210809
+        - cached
+        - latest
+        - nightly-latest
+        os: [ubuntu-latest, macos-latest]
+    name: 'Go: Autobuild custom tracing'
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v2
+    - name: Prepare test
+      id: prepare-test
+      uses: ./.github/prepare-test
+      with:
+        version: ${{ matrix.version }}
+    - uses: actions/setup-go@v2
+      with:
+        go-version: ^1.13.1
+    - uses: ./../action/init
+      with:
+        languages: go
+        tools: ${{ steps.prepare-test.outputs.tools-url }}
+    - uses: ./../action/autobuild
+    - uses: ./../action/analyze
+      env:
+        TEST_MODE: true
+    - shell: bash
+      run: |
+        cd "$RUNNER_TEMP/codeql_databases"
+        if [[ ! -d go ]]; then
+          echo "Did not find a Go database"
+          exit 1
+        fi
+    env:
+      CODEQL_EXTRACTOR_GO_BUILD_TRACING: 'true'

.github/workflows/__go-custom-tracing.yml

@@ -0,0 +1,61 @@
+# Warning: This file is generated automatically, and should not be modified.
+# Instead, please modify the template in the pr-checks directory and run:
+#     pip install ruamel.yaml && python3 sync.py
+# to regenerate this file.
+
+name: 'PR Check - Go: Custom tracing'
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  GO111MODULE: auto
+on:
+  push:
+    branches:
+    - main
+    - v1
+  pull_request:
+    types:
+    - opened
+    - synchronize
+    - reopened
+    - ready_for_review
+  workflow_dispatch: {}
+jobs:
+  go-custom-tracing:
+    strategy:
+      matrix:
+        version:
+        - stable-20201028
+        - stable-20210319
+        - stable-20210809
+        - cached
+        - latest
+        - nightly-latest
+        os:
+        - ubuntu-latest
+        - macos-latest
+        - windows-latest
+    name: 'Go: Custom tracing'
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v2
+    - name: Prepare test
+      id: prepare-test
+      uses: ./.github/prepare-test
+      with:
+        version: ${{ matrix.version }}
+    - uses: actions/setup-go@v2
+      with:
+        go-version: ^1.13.1
+    - uses: ./../action/init
+      with:
+        languages: go
+        tools: ${{ steps.prepare-test.outputs.tools-url }}
+    - name: Build code
+      shell: bash
+      run: go build main.go
+    - uses: ./../action/analyze
+      env:
+        TEST_MODE: true
+    env:
+      CODEQL_EXTRACTOR_GO_BUILD_TRACING: 'true'

.github/workflows/__javascript-source-root.yml

@@ -0,0 +1,59 @@
+# Warning: This file is generated automatically, and should not be modified.
+# Instead, please modify the template in the pr-checks directory and run:
+#     pip install ruamel.yaml && python3 sync.py
+# to regenerate this file.
+
+name: PR Check - Custom source root
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  GO111MODULE: auto
+on:
+  push:
+    branches:
+    - main
+    - v1
+  pull_request:
+    types:
+    - opened
+    - synchronize
+    - reopened
+    - ready_for_review
+  workflow_dispatch: {}
+jobs:
+  javascript-source-root:
+    strategy:
+      matrix:
+        version: [latest, cached, nightly-latest] # This feature is not compatible with old CLIs
+        os: [ubuntu-latest]
+    name: Custom source root
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v2
+    - name: Prepare test
+      id: prepare-test
+      uses: ./.github/prepare-test
+      with:
+        version: ${{ matrix.version }}
+    - name: Move codeql-action
+      shell: bash
+      run: |
+        mkdir ../new-source-root
+        mv * ../new-source-root
+    - uses: ./../action/init
+      with:
+        languages: javascript
+        source-root: ../new-source-root
+        tools: ${{ steps.prepare-test.outputs.tools-url }}
+    - uses: ./../action/analyze
+      with:
+        skip-queries: true
+        upload: false
+    - name: Assert database exists
+      shell: bash
+      run: |
+        cd "$RUNNER_TEMP/codeql_databases"
+        if [[ ! -d javascript ]]; then
+          echo "Did not find a JavaScript database"
+          exit 1
+        fi

.github/workflows/__multi-language-autodetect.yml

@@ -0,0 +1,86 @@
+# Warning: This file is generated automatically, and should not be modified.
+# Instead, please modify the template in the pr-checks directory and run:
+#     pip install ruamel.yaml && python3 sync.py
+# to regenerate this file.
+
+name: PR Check - Multi-language repository
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  GO111MODULE: auto
+on:
+  push:
+    branches:
+    - main
+    - v1
+  pull_request:
+    types:
+    - opened
+    - synchronize
+    - reopened
+    - ready_for_review
+  workflow_dispatch: {}
+jobs:
+  multi-language-autodetect:
+    strategy:
+      matrix:
+        version:
+        - stable-20201028
+        - stable-20210319
+        - stable-20210809
+        - cached
+        - latest
+        - nightly-latest
+        os: [ubuntu-latest, macos-latest]
+    name: Multi-language repository
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v2
+    - name: Prepare test
+      id: prepare-test
+      uses: ./.github/prepare-test
+      with:
+        version: ${{ matrix.version }}
+    - uses: ./../action/init
+      with:
+        db-location: ${{ runner.temp }}/customDbLocation
+        tools: ${{ steps.prepare-test.outputs.tools-url }}
+    - name: Build code
+      shell: bash
+      run: ./build.sh
+    - uses: ./../action/analyze
+      id: analysis
+      env:
+        TEST_MODE: true
+    - shell: bash
+      run: |
+        CPP_DB=${{ fromJson(steps.analysis.outputs.db-locations).cpp }}
+        if [[ ! -d $CPP_DB ]] || [[ ! $CPP_DB == ${{ runner.temp }}/customDbLocation/* ]]; then
+          echo "Did not create a database for CPP, or created it in the wrong location."
+          exit 1
+        fi
+        CSHARP_DB=${{ fromJson(steps.analysis.outputs.db-locations).csharp }}
+        if [[ ! -d $CSHARP_DB ]] || [[ ! $CSHARP_DB == ${{ runner.temp }}/customDbLocation/* ]]; then
+          echo "Did not create a database for C Sharp, or created it in the wrong location."
+          exit 1
+        fi
+        GO_DB=${{ fromJson(steps.analysis.outputs.db-locations).go }}
+        if [[ ! -d $GO_DB ]] || [[ ! $GO_DB == ${{ runner.temp }}/customDbLocation/* ]]; then
+          echo "Did not create a database for Go, or created it in the wrong location."
+          exit 1
+        fi
+        JAVA_DB=${{ fromJson(steps.analysis.outputs.db-locations).java }}
+        if [[ ! -d $JAVA_DB ]] || [[ ! $JAVA_DB == ${{ runner.temp }}/customDbLocation/* ]]; then
+          echo "Did not create a database for Java, or created it in the wrong location."
+          exit 1
+        fi
+        JAVASCRIPT_DB=${{ fromJson(steps.analysis.outputs.db-locations).javascript }}
+        if [[ ! -d $JAVASCRIPT_DB ]] || [[ ! $JAVASCRIPT_DB == ${{ runner.temp }}/customDbLocation/* ]]; then
+          echo "Did not create a database for Javascript, or created it in the wrong location."
+          exit 1
+        fi
+        PYTHON_DB=${{ fromJson(steps.analysis.outputs.db-locations).python }}
+        if [[ ! -d $PYTHON_DB ]] || [[ ! $PYTHON_DB == ${{ runner.temp }}/customDbLocation/* ]]; then
+          echo "Did not create a database for Python, or created it in the wrong location."
+          exit 1
+        fi

.github/workflows/__packaging-config-inputs-js.yml

@@ -0,0 +1,65 @@
+# Warning: This file is generated automatically, and should not be modified.
+# Instead, please modify the template in the pr-checks directory and run:
+#     pip install ruamel.yaml && python3 sync.py
+# to regenerate this file.
+
+name: 'PR Check - Packaging: Config and input'
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  GO111MODULE: auto
+on:
+  push:
+    branches:
+    - main
+    - v1
+  pull_request:
+    types:
+    - opened
+    - synchronize
+    - reopened
+    - ready_for_review
+  workflow_dispatch: {}
+jobs:
+  packaging-config-inputs-js:
+    strategy:
+      matrix:
+        version: [nightly-20210831] # This CLI version is known to work with package used in this test
+        os: [ubuntu-latest, macos-latest]
+    name: 'Packaging: Config and input'
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v2
+    - name: Prepare test
+      id: prepare-test
+      uses: ./.github/prepare-test
+      with:
+        version: ${{ matrix.version }}
+    - uses: ./../action/init
+      with:
+        config-file: .github/codeql/codeql-config-packaging3.yml
+        packs: +dsp-testing/codeql-pack1@0.1.0
+        languages: javascript
+        tools: ${{ steps.prepare-test.outputs.tools-url }}
+    - name: Build code
+      shell: bash
+      run: ./build.sh
+    - uses: ./../action/analyze
+      with:
+        output: ${{ runner.temp }}/results
+      env:
+        TEST_MODE: true
+    - name: Assert Results
+      shell: bash
+      run: |
+        cd "$RUNNER_TEMP/results"
+        # We should have 3 hits from these rules
+        EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/two-block"
+
+        # use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
+        RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n" " " | xargs)"
+        echo "Found matching rules '$RULES'"
+        if [ "$RULES" != "$EXPECTED_RULES" ]; then
+          echo "Did not match expected rules '$EXPECTED_RULES'."
+          exit 1
+        fi

.github/workflows/__packaging-config-js.yml

@@ -0,0 +1,64 @@
+# Warning: This file is generated automatically, and should not be modified.
+# Instead, please modify the template in the pr-checks directory and run:
+#     pip install ruamel.yaml && python3 sync.py
+# to regenerate this file.
+
+name: 'PR Check - Packaging: Config file'
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  GO111MODULE: auto
+on:
+  push:
+    branches:
+    - main
+    - v1
+  pull_request:
+    types:
+    - opened
+    - synchronize
+    - reopened
+    - ready_for_review
+  workflow_dispatch: {}
+jobs:
+  packaging-config-js:
+    strategy:
+      matrix:
+        version: [nightly-20210831] # This CLI version is known to work with package used in this test
+        os: [ubuntu-latest, macos-latest]
+    name: 'Packaging: Config file'
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v2
+    - name: Prepare test
+      id: prepare-test
+      uses: ./.github/prepare-test
+      with:
+        version: ${{ matrix.version }}
+    - uses: ./../action/init
+      with:
+        config-file: .github/codeql/codeql-config-packaging.yml
+        languages: javascript
+        tools: ${{ steps.prepare-test.outputs.tools-url }}
+    - name: Build code
+      shell: bash
+      run: ./build.sh
+    - uses: ./../action/analyze
+      with:
+        output: ${{ runner.temp }}/results
+      env:
+        TEST_MODE: true
+    - name: Assert Results
+      shell: bash
+      run: |
+        cd "$RUNNER_TEMP/results"
+        # We should have 3 hits from these rules
+        EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/two-block"
+
+        # use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
+        RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n" " " | xargs)"
+        echo "Found matching rules '$RULES'"
+        if [ "$RULES" != "$EXPECTED_RULES" ]; then
+          echo "Did not match expected rules '$EXPECTED_RULES'."
+          exit 1
+        fi

.github/workflows/__packaging-inputs-js.yml

@@ -0,0 +1,65 @@
+# Warning: This file is generated automatically, and should not be modified.
+# Instead, please modify the template in the pr-checks directory and run:
+#     pip install ruamel.yaml && python3 sync.py
+# to regenerate this file.
+
+name: 'PR Check - Packaging: Action input'
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  GO111MODULE: auto
+on:
+  push:
+    branches:
+    - main
+    - v1
+  pull_request:
+    types:
+    - opened
+    - synchronize
+    - reopened
+    - ready_for_review
+  workflow_dispatch: {}
+jobs:
+  packaging-inputs-js:
+    strategy:
+      matrix:
+        version: [nightly-20210831] # This CLI version is known to work with package used in this test
+        os: [ubuntu-latest, macos-latest]
+    name: 'Packaging: Action input'
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v2
+    - name: Prepare test
+      id: prepare-test
+      uses: ./.github/prepare-test
+      with:
+        version: ${{ matrix.version }}
+    - uses: ./../action/init
+      with:
+        config-file: .github/codeql/codeql-config-packaging2.yml
+        languages: javascript
+        packs: dsp-testing/codeql-pack1@0.1.0, dsp-testing/codeql-pack2
+        tools: ${{ steps.prepare-test.outputs.tools-url }}
+    - name: Build code
+      shell: bash
+      run: ./build.sh
+    - uses: ./../action/analyze
+      with:
+        output: ${{ runner.temp }}/results
+      env:
+        TEST_MODE: true
+    - name: Assert Results
+      shell: bash
+      run: |
+        cd "$RUNNER_TEMP/results"
+        # We should have 3 hits from these rules
+        EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/two-block"
+
+        # use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
+        RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n" " " | xargs)"
+        echo "Found matching rules '$RULES'"
+        if [ "$RULES" != "$EXPECTED_RULES" ]; then
+          echo "Did not match expected rules '$EXPECTED_RULES'."
+          exit 1
+        fi

.github/workflows/__remote-config.yml

@@ -0,0 +1,58 @@
+# Warning: This file is generated automatically, and should not be modified.
+# Instead, please modify the template in the pr-checks directory and run:
+#     pip install ruamel.yaml && python3 sync.py
+# to regenerate this file.
+
+name: PR Check - Remote config file
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  GO111MODULE: auto
+on:
+  push:
+    branches:
+    - main
+    - v1
+  pull_request:
+    types:
+    - opened
+    - synchronize
+    - reopened
+    - ready_for_review
+  workflow_dispatch: {}
+jobs:
+  remote-config:
+    strategy:
+      matrix:
+        version:
+        - stable-20201028
+        - stable-20210319
+        - stable-20210809
+        - cached
+        - latest
+        - nightly-latest
+        os:
+        - ubuntu-latest
+        - macos-latest
+        - windows-latest
+    name: Remote config file
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v2
+    - name: Prepare test
+      id: prepare-test
+      uses: ./.github/prepare-test
+      with:
+        version: ${{ matrix.version }}
+    - uses: ./../action/init
+      with:
+        tools: ${{ steps.prepare-test.outputs.tools-url }}
+        languages: cpp,csharp,java,javascript,python
+        config-file: github/codeql-action/tests/multi-language-repo/.github/codeql/custom-queries.yml@${{
+          github.sha }}
+    - name: Build code
+      shell: bash
+      run: ./build.sh
+    - uses: ./../action/analyze
+      env:
+        TEST_MODE: true

.github/workflows/__rubocop-multi-language.yml

@@ -0,0 +1,65 @@
+# Warning: This file is generated automatically, and should not be modified.
+# Instead, please modify the template in the pr-checks directory and run:
+#     pip install ruamel.yaml && python3 sync.py
+# to regenerate this file.
+
+name: PR Check - RuboCop multi-language
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  GO111MODULE: auto
+on:
+  push:
+    branches:
+    - main
+    - v1
+  pull_request:
+    types:
+    - opened
+    - synchronize
+    - reopened
+    - ready_for_review
+  workflow_dispatch: {}
+jobs:
+  rubocop-multi-language:
+    strategy:
+      matrix:
+        version:
+        - stable-20201028
+        - stable-20210319
+        - stable-20210809
+        - cached
+        - latest
+        - nightly-latest
+        os: [ubuntu-latest]
+    name: RuboCop multi-language
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v2
+    - name: Prepare test
+      id: prepare-test
+      uses: ./.github/prepare-test
+      with:
+        version: ${{ matrix.version }}
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 2.6
+    - name: Install Code Scanning integration
+      shell: bash
+      run: bundle add code-scanning-rubocop --version 0.3.0 --skip-install
+    - name: Install dependencies
+      shell: bash
+      run: bundle install
+    - name: RuboCop run
+      shell: bash
+      run: |
+        bash -c "
+          bundle exec rubocop --require code_scanning --format CodeScanning::SarifFormatter -o rubocop.sarif
+          [[ $? -ne 2 ]]
+        "
+    - uses: ./../action/upload-sarif
+      with:
+        sarif_file: rubocop.sarif
+      env:
+        TEST_MODE: true

.github/workflows/__split-workflow.yml

@@ -0,0 +1,78 @@
+# Warning: This file is generated automatically, and should not be modified.
+# Instead, please modify the template in the pr-checks directory and run:
+#     pip install ruamel.yaml && python3 sync.py
+# to regenerate this file.
+
+name: PR Check - Split workflow
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  GO111MODULE: auto
+on:
+  push:
+    branches:
+    - main
+    - v1
+  pull_request:
+    types:
+    - opened
+    - synchronize
+    - reopened
+    - ready_for_review
+  workflow_dispatch: {}
+jobs:
+  split-workflow:
+    strategy:
+      matrix:
+        version: [nightly-20210831] # This CLI version is known to work with package used in this test
+        os: [ubuntu-latest, macos-latest]
+    name: Split workflow
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v2
+    - name: Prepare test
+      id: prepare-test
+      uses: ./.github/prepare-test
+      with:
+        version: ${{ matrix.version }}
+    - uses: ./../action/init
+      with:
+        config-file: .github/codeql/codeql-config-packaging3.yml
+        packs: +dsp-testing/codeql-pack1@0.1.0
+        languages: javascript
+        tools: ${{ steps.prepare-test.outputs.tools-url }}
+    - name: Build code
+      shell: bash
+      run: ./build.sh
+    - uses: ./../action/analyze
+      with:
+        skip-queries: true
+        output: ${{ runner.temp }}/results
+      env:
+        TEST_MODE: true
+    - name: Assert No Results
+      shell: bash
+      run: |
+        if [ "$(ls -A $RUNNER_TEMP/results)" ]; then
+          echo "Expected results directory to be empty after skipping query execution!"
+          exit 1
+        fi
+    - uses: ./../action/analyze
+      with:
+        output: ${{ runner.temp }}/results
+      env:
+        TEST_MODE: true
+    - name: Assert Results
+      shell: bash
+      run: |
+        cd "$RUNNER_TEMP/results"
+        # We should have 3 hits from these rules
+        EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/two-block"
+
+        # use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
+        RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n" " " | xargs)"
+        echo "Found matching rules '$RULES'"
+        if [ "$RULES" != "$EXPECTED_RULES" ]; then
+          echo "Did not match expected rules '$EXPECTED_RULES'."
+          exit 1
+        fi

.github/workflows/__test-local-codeql.yml

@@ -0,0 +1,52 @@
+# Warning: This file is generated automatically, and should not be modified.
+# Instead, please modify the template in the pr-checks directory and run:
+#     pip install ruamel.yaml && python3 sync.py
+# to regenerate this file.
+
+name: PR Check - Local CodeQL bundle
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  GO111MODULE: auto
+on:
+  push:
+    branches:
+    - main
+    - v1
+  pull_request:
+    types:
+    - opened
+    - synchronize
+    - reopened
+    - ready_for_review
+  workflow_dispatch: {}
+jobs:
+  test-local-codeql:
+    strategy:
+      matrix:
+        version: [nightly-latest]
+        os: [ubuntu-latest]
+    name: Local CodeQL bundle
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v2
+    - name: Prepare test
+      id: prepare-test
+      uses: ./.github/prepare-test
+      with:
+        version: ${{ matrix.version }}
+    - name: Fetch a CodeQL bundle
+      shell: bash
+      env:
+        CODEQL_URL: ${{ steps.prepare-test.outputs.tools-url }}
+      run: |
+        wget "$CODEQL_URL"
+    - uses: ./../action/init
+      with:
+        tools: ./codeql-bundle.tar.gz
+    - name: Build code
+      shell: bash
+      run: ./build.sh
+    - uses: ./../action/analyze
+      env:
+        TEST_MODE: true

.github/workflows/__test-proxy.yml

@@ -0,0 +1,54 @@
+# Warning: This file is generated automatically, and should not be modified.
+# Instead, please modify the template in the pr-checks directory and run:
+#     pip install ruamel.yaml && python3 sync.py
+# to regenerate this file.
+
+name: PR Check - Proxy test
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  GO111MODULE: auto
+on:
+  push:
+    branches:
+    - main
+    - v1
+  pull_request:
+    types:
+    - opened
+    - synchronize
+    - reopened
+    - ready_for_review
+  workflow_dispatch: {}
+jobs:
+  test-proxy:
+    strategy:
+      matrix:
+        version: [latest]
+        os: [ubuntu-latest]
+    name: Proxy test
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v2
+    - name: Prepare test
+      id: prepare-test
+      uses: ./.github/prepare-test
+      with:
+        version: ${{ matrix.version }}
+    - uses: ./../action/init
+      with:
+        languages: javascript
+        tools: ${{ steps.prepare-test.outputs.tools-url }}
+    - uses: ./../action/analyze
+      env:
+        TEST_MODE: true
+    env:
+      https_proxy: http://squid-proxy:3128
+    container:
+      image: ubuntu:18.04
+      options: --dns 127.0.0.1
+    services:
+      squid-proxy:
+        image: datadog/squid:latest
+        ports:
+        - 3128:3128

.github/workflows/__test-ruby.yml

@@ -0,0 +1,55 @@
+# Warning: This file is generated automatically, and should not be modified.
+# Instead, please modify the template in the pr-checks directory and run:
+#     pip install ruamel.yaml && python3 sync.py
+# to regenerate this file.
+
+name: PR Check - Ruby analysis
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  GO111MODULE: auto
+on:
+  push:
+    branches:
+    - main
+    - v1
+  pull_request:
+    types:
+    - opened
+    - synchronize
+    - reopened
+    - ready_for_review
+  workflow_dispatch: {}
+jobs:
+  test-ruby:
+    strategy:
+      matrix:
+        version: [latest, cached, nightly-latest]
+        os: [ubuntu-latest, macos-latest]
+    name: Ruby analysis
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v2
+    - name: Prepare test
+      id: prepare-test
+      uses: ./.github/prepare-test
+      with:
+        version: ${{ matrix.version }}
+    - uses: ./../action/init
+      with:
+        languages: ruby
+        tools: ${{ steps.prepare-test.outputs.tools-url }}
+    - uses: ./../action/analyze
+      id: analysis
+      env:
+        TEST_MODE: true
+    - name: Check database
+      shell: bash
+      run: |
+        RUBY_DB="${{ fromJson(steps.analysis.outputs.db-locations).ruby }}"
+        if [[ ! -d "$RUBY_DB" ]]; then
+          echo "Did not create a database for Ruby."
+          exit 1
+        fi
+    env:
+      CODEQL_ENABLE_EXPERIMENTAL_FEATURES: 'true'

.github/workflows/script/verify-pr-checks.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -eu
+
+# Sanity check that repo is clean to start with
+if [ ! -z "$(git status --porcelain)" ]; then
+    # If we get a fail here then this workflow needs attention...
+    >&2 echo "Failed: Repo should be clean before testing!"
+    exit 1
+fi
+
+# Wipe the generated PR checks in case there are extra unnecessary files in there
+rm -rf .github/workflows/__*
+
+# Generate the PR checks
+cd pr-checks && python3 sync.py
+
+# Check that repo is still clean
+if [ ! -z "$(git status --porcelain)" ]; then
+    # If we get a fail here then the PR needs attention
+    git diff
+    git status
+    >&2 echo "Failed: PR checks are not up to date. Run 'cd pr-checks && python3 sync.py' to update"
+    exit 1
+fi
+echo "Success: PR checks are up to date"

pr-checks/checks/go-custom-queries.yml

@@ -0,0 +1,17 @@
+name: "Go: Custom queries"
+description: "Checks that Go works in conjunction with a config file specifying custom queries"
+steps:
+  - uses: actions/setup-go@v2
+    with:
+      go-version: "^1.13.1"
+  - uses: ./../action/init
+    with:
+      languages: go
+      config-file: ./.github/codeql/custom-queries.yml
+      tools: ${{ steps.prepare-test.outputs.tools-url }}
+  - name: Build code
+    shell: bash
+    run: ./build.sh
+  - uses: ./../action/analyze
+    env:
+      TEST_MODE: true

pr-checks/checks/go-custom-tracing-autobuild.yml

@@ -0,0 +1,24 @@
+name: "Go: Autobuild custom tracing"
+description: "Checks that Go tracing works in conjunction with the autobuilder"
+os: ["ubuntu-latest", "macos-latest"]
+env:
+  CODEQL_EXTRACTOR_GO_BUILD_TRACING: "true"
+steps:
+  - uses: actions/setup-go@v2
+    with:
+      go-version: "^1.13.1"
+  - uses: ./../action/init
+    with:
+      languages: go
+      tools: ${{ steps.prepare-test.outputs.tools-url }}
+  - uses: ./../action/autobuild
+  - uses: ./../action/analyze
+    env:
+      TEST_MODE: true
+  - shell: bash
+    run: |
+      cd "$RUNNER_TEMP/codeql_databases"
+      if [[ ! -d go ]]; then
+        echo "Did not find a Go database"
+        exit 1
+      fi

pr-checks/checks/go-custom-tracing.yml

@@ -0,0 +1,18 @@
+name: "Go: Custom tracing"
+description: "Checks that Go tracing works"
+env:
+  CODEQL_EXTRACTOR_GO_BUILD_TRACING: "true"
+steps:
+  - uses: actions/setup-go@v2
+    with:
+      go-version: "^1.13.1"
+  - uses: ./../action/init
+    with:
+      languages: go
+      tools: ${{ steps.prepare-test.outputs.tools-url }}
+  - name: Build code
+    shell: bash
+    run: go build main.go
+  - uses: ./../action/analyze
+    env:
+      TEST_MODE: true

pr-checks/checks/javascript-source-root.yml

@@ -0,0 +1,27 @@
+name: "Custom source root"
+description: "Checks that the argument specifying a non-default source root works"
+versions: ["latest", "cached", "nightly-latest"] # This feature is not compatible with old CLIs
+os: ["ubuntu-latest"]
+steps:
+  - name: Move codeql-action
+    shell: bash
+    run: |
+      mkdir ../new-source-root
+      mv * ../new-source-root
+  - uses: ./../action/init
+    with:
+      languages: javascript
+      source-root: ../new-source-root
+      tools: ${{ steps.prepare-test.outputs.tools-url }}
+  - uses: ./../action/analyze
+    with:
+      skip-queries: true
+      upload: false
+  - name: Assert database exists
+    shell: bash
+    run: |
+      cd "$RUNNER_TEMP/codeql_databases"
+      if [[ ! -d javascript ]]; then
+        echo "Did not find a JavaScript database"
+        exit 1
+      fi

pr-checks/checks/multi-language-autodetect.yml

@@ -0,0 +1,47 @@
+name: "Multi-language repository"
+description: "An end-to-end integration test of a multi-language repository using automatic language detection"
+os: ["ubuntu-latest", "macos-latest"]
+steps:
+  - uses: ./../action/init
+    with:
+      db-location: "${{ runner.temp }}/customDbLocation"
+      tools: ${{ steps.prepare-test.outputs.tools-url }}
+  - name: Build code
+    shell: bash
+    run: ./build.sh
+  - uses: ./../action/analyze
+    id: analysis
+    env:
+      TEST_MODE: true
+  - shell: bash
+    run: |
+      CPP_DB=${{ fromJson(steps.analysis.outputs.db-locations).cpp }}
+      if [[ ! -d $CPP_DB ]] || [[ ! $CPP_DB == ${{ runner.temp }}/customDbLocation/* ]]; then
+        echo "Did not create a database for CPP, or created it in the wrong location."
+        exit 1
+      fi
+      CSHARP_DB=${{ fromJson(steps.analysis.outputs.db-locations).csharp }}
+      if [[ ! -d $CSHARP_DB ]] || [[ ! $CSHARP_DB == ${{ runner.temp }}/customDbLocation/* ]]; then
+        echo "Did not create a database for C Sharp, or created it in the wrong location."
+        exit 1
+      fi
+      GO_DB=${{ fromJson(steps.analysis.outputs.db-locations).go }}
+      if [[ ! -d $GO_DB ]] || [[ ! $GO_DB == ${{ runner.temp }}/customDbLocation/* ]]; then
+        echo "Did not create a database for Go, or created it in the wrong location."
+        exit 1
+      fi
+      JAVA_DB=${{ fromJson(steps.analysis.outputs.db-locations).java }}
+      if [[ ! -d $JAVA_DB ]] || [[ ! $JAVA_DB == ${{ runner.temp }}/customDbLocation/* ]]; then
+        echo "Did not create a database for Java, or created it in the wrong location."
+        exit 1
+      fi
+      JAVASCRIPT_DB=${{ fromJson(steps.analysis.outputs.db-locations).javascript }}
+      if [[ ! -d $JAVASCRIPT_DB ]] || [[ ! $JAVASCRIPT_DB == ${{ runner.temp }}/customDbLocation/* ]]; then
+        echo "Did not create a database for Javascript, or created it in the wrong location."
+        exit 1
+      fi
+      PYTHON_DB=${{ fromJson(steps.analysis.outputs.db-locations).python }}
+      if [[ ! -d $PYTHON_DB ]] || [[ ! $PYTHON_DB == ${{ runner.temp }}/customDbLocation/* ]]; then
+        echo "Did not create a database for Python, or created it in the wrong location."
+        exit 1
+      fi

pr-checks/checks/packaging-config-inputs-js.yml

@@ -0,0 +1,33 @@
+name: "Packaging: Config and input"
+description: "Checks that specifying packages using a combination of a config file and input to the Action works"
+versions: ["nightly-20210831"] # This CLI version is known to work with package used in this test
+os: ["ubuntu-latest", "macos-latest"]
+steps:
+  - uses: ./../action/init
+    with:
+      config-file: ".github/codeql/codeql-config-packaging3.yml"
+      packs: +dsp-testing/codeql-pack1@0.1.0
+      languages: javascript
+      tools: ${{ steps.prepare-test.outputs.tools-url }}
+  - name: Build code
+    shell: bash
+    run: ./build.sh
+  - uses: ./../action/analyze
+    with:
+      output: "${{ runner.temp }}/results"
+    env:
+      TEST_MODE: true
+  - name: Assert Results
+    shell: bash
+    run: |
+      cd "$RUNNER_TEMP/results"
+      # We should have 3 hits from these rules
+      EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/two-block"
+
+      # use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
+      RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n" " " | xargs)"
+      echo "Found matching rules '$RULES'"
+      if [ "$RULES" != "$EXPECTED_RULES" ]; then
+        echo "Did not match expected rules '$EXPECTED_RULES'."
+        exit 1
+      fi

pr-checks/checks/packaging-config-js.yml

@@ -0,0 +1,32 @@
+name: "Packaging: Config file"
+description: "Checks that specifying packages using only a config file works"
+versions: ["nightly-20210831"] # This CLI version is known to work with package used in this test
+os: ["ubuntu-latest", "macos-latest"]
+steps:
+  - uses: ./../action/init
+    with:
+      config-file: ".github/codeql/codeql-config-packaging.yml"
+      languages: javascript
+      tools: ${{ steps.prepare-test.outputs.tools-url }}
+  - name: Build code
+    shell: bash
+    run: ./build.sh
+  - uses: ./../action/analyze
+    with:
+      output: "${{ runner.temp }}/results"
+    env:
+      TEST_MODE: true
+  - name: Assert Results
+    shell: bash
+    run: |
+      cd "$RUNNER_TEMP/results"
+      # We should have 3 hits from these rules
+      EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/two-block"
+
+      # use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
+      RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n" " " | xargs)"
+      echo "Found matching rules '$RULES'"
+      if [ "$RULES" != "$EXPECTED_RULES" ]; then
+        echo "Did not match expected rules '$EXPECTED_RULES'."
+        exit 1
+      fi

pr-checks/checks/packaging-inputs-js.yml

@@ -0,0 +1,33 @@
+name: "Packaging: Action input"
+description: "Checks that specifying packages using the input to the Action works"
+versions: ["nightly-20210831"] # This CLI version is known to work with package used in this test
+os: ["ubuntu-latest", "macos-latest"]
+steps:
+  - uses: ./../action/init
+    with:
+      config-file: ".github/codeql/codeql-config-packaging2.yml"
+      languages: javascript
+      packs: dsp-testing/codeql-pack1@0.1.0, dsp-testing/codeql-pack2
+      tools: ${{ steps.prepare-test.outputs.tools-url }}
+  - name: Build code
+    shell: bash
+    run: ./build.sh
+  - uses: ./../action/analyze
+    with:
+      output: "${{ runner.temp }}/results"
+    env:
+      TEST_MODE: true
+  - name: Assert Results
+    shell: bash
+    run: |
+      cd "$RUNNER_TEMP/results"
+      # We should have 3 hits from these rules
+      EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/two-block"
+
+      # use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
+      RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n" " " | xargs)"
+      echo "Found matching rules '$RULES'"
+      if [ "$RULES" != "$EXPECTED_RULES" ]; then
+        echo "Did not match expected rules '$EXPECTED_RULES'."
+        exit 1
+      fi

pr-checks/checks/remote-config.yml

@@ -0,0 +1,14 @@
+name: "Remote config file"
+description: "Checks that specifying packages using only a config file works"
+steps:
+  - uses: ./../action/init
+    with:
+      tools: ${{ steps.prepare-test.outputs.tools-url }}
+      languages: cpp,csharp,java,javascript,python
+      config-file: github/codeql-action/tests/multi-language-repo/.github/codeql/custom-queries.yml@${{ github.sha }}
+  - name: Build code
+    shell: bash
+    run: ./build.sh
+  - uses: ./../action/analyze
+    env:
+      TEST_MODE: true

pr-checks/checks/rubocop-multi-language.yml

@@ -0,0 +1,26 @@
+name: "RuboCop multi-language"
+description: "Tests using RuboCop to analyze a multi-language repository and then using the CodeQL Action to upload the resulting SARIF"
+os: ["ubuntu-latest"]
+steps:
+  - name: Set up Ruby
+    uses: ruby/setup-ruby@v1
+    with:
+      ruby-version: 2.6
+  - name: Install Code Scanning integration
+    shell: bash
+    run: bundle add code-scanning-rubocop --version 0.3.0 --skip-install
+  - name: Install dependencies
+    shell: bash
+    run: bundle install
+  - name: RuboCop run
+    shell: bash
+    run: |
+      bash -c "
+        bundle exec rubocop --require code_scanning --format CodeScanning::SarifFormatter -o rubocop.sarif
+        [[ $? -ne 2 ]]
+      "
+  - uses: ./../action/upload-sarif
+    with:
+      sarif_file: rubocop.sarif
+    env:
+      TEST_MODE: true

pr-checks/checks/split-workflow.yml

@@ -0,0 +1,46 @@
+name: "Split workflow"
+description: "Tests a split-up workflow in which we first build a database and later analyze it"
+versions: ["nightly-20210831"] # This CLI version is known to work with package used in this test
+os: ["ubuntu-latest", "macos-latest"]
+steps:
+  - uses: ./../action/init
+    with:
+      config-file: ".github/codeql/codeql-config-packaging3.yml"
+      packs: +dsp-testing/codeql-pack1@0.1.0
+      languages: javascript
+      tools: ${{ steps.prepare-test.outputs.tools-url }}
+  - name: Build code
+    shell: bash
+    run: ./build.sh
+  - uses: ./../action/analyze
+    with:
+      skip-queries: true
+      output: "${{ runner.temp }}/results"
+    env:
+      TEST_MODE: true
+  - name: Assert No Results
+    shell: bash
+    run: |
+      if [ "$(ls -A $RUNNER_TEMP/results)" ]; then
+        echo "Expected results directory to be empty after skipping query execution!"
+        exit 1
+      fi
+  - uses: ./../action/analyze
+    with:
+      output: "${{ runner.temp }}/results"
+    env:
+      TEST_MODE: true
+  - name: Assert Results
+    shell: bash
+    run: |
+      cd "$RUNNER_TEMP/results"
+      # We should have 3 hits from these rules
+      EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/two-block"
+
+      # use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
+      RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n" " " | xargs)"
+      echo "Found matching rules '$RULES'"
+      if [ "$RULES" != "$EXPECTED_RULES" ]; then
+        echo "Did not match expected rules '$EXPECTED_RULES'."
+        exit 1
+      fi

pr-checks/checks/test-local-codeql.yml

@@ -0,0 +1,20 @@
+name: "Local CodeQL bundle"
+description: "Tests using a CodeQL bundle from a local file rather than a URL"
+versions: ["nightly-latest"]
+os: ["ubuntu-latest"]
+steps:
+  - name: Fetch a CodeQL bundle
+    shell: bash
+    env:
+      CODEQL_URL: ${{ steps.prepare-test.outputs.tools-url }}
+    run: |
+      wget "$CODEQL_URL"
+  - uses: ./../action/init
+    with:
+      tools: ./codeql-bundle.tar.gz
+  - name: Build code
+    shell: bash
+    run: ./build.sh
+  - uses: ./../action/analyze
+    env:
+      TEST_MODE: true

pr-checks/checks/test-proxy.yml

@@ -0,0 +1,22 @@
+name: "Proxy test"
+description: "Tests using a proxy specified by the https_proxy environment variable"
+versions: ["latest"]
+os: ["ubuntu-latest"]
+container:
+  image: ubuntu:18.04
+  options: --dns 127.0.0.1
+services:
+  squid-proxy:
+    image: datadog/squid:latest
+    ports:
+      - 3128:3128
+env:
+  https_proxy: http://squid-proxy:3128
+steps:
+  - uses: ./../action/init
+    with:
+      languages: javascript
+      tools: ${{ steps.prepare-test.outputs.tools-url }}
+  - uses: ./../action/analyze
+    env:
+      TEST_MODE: true

pr-checks/checks/test-ruby.yml

@@ -0,0 +1,23 @@
+name: "Ruby analysis"
+description: "Tests creation of a Ruby database"
+versions: ["latest", "cached", "nightly-latest"]
+os: ["ubuntu-latest", "macos-latest"]
+env:
+  CODEQL_ENABLE_EXPERIMENTAL_FEATURES: "true"
+steps:
+  - uses: ./../action/init
+    with:
+      languages: ruby
+      tools: ${{ steps.prepare-test.outputs.tools-url }}
+  - uses: ./../action/analyze
+    id: analysis
+    env:
+      TEST_MODE: true
+  - name: Check database
+    shell: bash
+    run: |
+      RUBY_DB="${{ fromJson(steps.analysis.outputs.db-locations).ruby }}"
+      if [[ ! -d "$RUBY_DB" ]]; then
+        echo "Did not create a database for Ruby."
+        exit 1
+      fi

pr-checks/sync.py

@@ -0,0 +1,104 @@
+import ruamel.yaml
+import os
+
+defaultTestVersions = [
+    # The oldest supported CodeQL version: 2.3.1. If bumping, update `CODEQL_MINIMUM_VERSION` in `codeql.ts`
+    "stable-20201028",
+    # The last CodeQL release in the 2.4 series: 2.4.6.
+    "stable-20210319",
+    # The last CodeQL release in the 2.5 series: 2.5.9.
+    "stable-20210809",
+    # The version of CodeQL currently in the toolcache. Typically either the latest release or the one before.
+    "cached",
+    # The latest release of CodeQL.
+    "latest",
+    # A nightly build directly from the our private repo, built in the last 24 hours.
+    "nightly-latest"
+]
+defaultOperatingSystems = ["ubuntu-latest", "macos-latest", "windows-latest"]
+header = """# Warning: This file is generated automatically, and should not be modified.
+# Instead, please modify the template in the pr-checks directory and run:
+#     pip install ruamel.yaml && python3 sync.py
+# to regenerate this file.
+
+"""
+
+
+class NonAliasingRTRepresenter(ruamel.yaml.representer.RoundTripRepresenter):
+    def ignore_aliases(self, data):
+        return True
+
+
+def writeHeader(checkStream):
+    checkStream.write(header)
+
+
+yaml = ruamel.yaml.YAML()
+yaml.Representer = NonAliasingRTRepresenter
+allJobs = {}
+for file in os.listdir('checks'):
+    with open(f"checks/{file}", 'r') as checkStream:
+        checkSpecification = yaml.load(checkStream)
+
+    versions = defaultTestVersions
+    if 'versions' in checkSpecification:
+        versions = checkSpecification['versions']
+    operatingSystems = defaultOperatingSystems
+    if 'os' in checkSpecification:
+        operatingSystems = checkSpecification['os']
+
+    steps = [
+        {
+            'name': 'Check out repository',
+            'uses': 'actions/checkout@v2'
+        },
+        {
+            'name': 'Prepare test',
+            'id': 'prepare-test',
+            'uses': './.github/prepare-test',
+            'with': {
+                'version': '${{ matrix.version }}'
+            }
+        }
+    ]
+    steps.extend(checkSpecification['steps'])
+
+    checkJob = {
+        'strategy': {
+            'matrix': {
+                'version': versions,
+                'os': operatingSystems
+            }
+        },
+        'name': checkSpecification['name'],
+        'runs-on': '${{ matrix.os }}',
+        'steps': steps
+    }
+
+    for key in ["env", "container", "services"]:
+        if key in checkSpecification:
+            checkJob[key] = checkSpecification[key]
+
+    checkName = file[:len(file) - 4]
+
+    with open(f"../.github/workflows/__{checkName}.yml", 'w') as output_stream:
+        writeHeader(output_stream)
+        yaml.dump({
+            'name': f"PR Check - {checkSpecification['name']}",
+            'env': {
+                'GITHUB_TOKEN': '${{ secrets.GITHUB_TOKEN }}',
+                'GO111MODULE': 'auto',
+            },
+            'on': {
+                'push': {
+                    'branches': ['main', 'v1']
+                },
+                'pull_request': {
+                    'types': ["opened", "synchronize", "reopened", "ready_for_review"]
+                },
+                'workflow_dispatch': {}
+            },
+            'jobs': {
+                checkName: checkJob
+            }
+        }, output_stream)