Universum 0.19.20

.github/workflows/postcommit-check.yml

@@ -0,0 +1,40 @@
+name: Universum check
+on:
+  push
+
+jobs:
+  universum_postcommit:
+    name: Universum postcommit
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Setup python 3.8
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.8
+
+    - name: Install dependency
+      run: pip install universum[test]
+
+    - name: Universum
+      run:
+        python -u -m universum
+        --fail-unsuccessful
+        --vcs-type="git"
+        --git-repo "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY"
+        --git-refspec "$GITHUB_REF_NAME"
+        --no-archive
+        --no-diff
+
+    - name: Publish Test Results
+      uses: EnricoMi/publish-unit-test-result-action@v1
+      if: always()
+      with:
+        files: artifacts/*.xml
+
+    - name: Collect artifacts
+      uses: actions/upload-artifact@v4
+      if: ${{ always() }}
+      with:
+        name: artifacts
+        path: artifacts

.github/workflows/precommit-check.yml

@@ -35,7 +35,7 @@ jobs:
         files: artifacts/*.xml
 
     - name: Collect artifacts
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       if: ${{ always() }}
       with:
         name: artifacts

.github/workflows/telegram-bot.yml

@@ -40,42 +40,44 @@ jobs:
       run: |
 
         if [[ ! -z "${{ github.event.pull_request }}" && "${{ github.event.action }}" == "opened" ]]; then
-          ESCAPED_NAME=`echo -e "${{ env.PR_NAME }}" | sed 's/\&/\&amp;/g' | sed 's/</\&lt;/g' | sed 's/>/\&gt;/g'`
-          TEXT=`echo -e "<b>${{ env.PR_AUTHOR }}</b> created new <a href=\"${{ env.PR_URL }}\">PR#${{ env.PR_NUMBER }}</a> '$ESCAPED_NAME' to branch '${{ env.PR_BASE }}'"`
+          ESCAPED_NAME=`echo -e "$PR_NAME" | sed 's/\&/\&amp;/g' | sed 's/</\&lt;/g' | sed 's/>/\&gt;/g'`
+          TEXT=`echo -e "<b>"$PR_AUTHOR"</b> created new <a href=\"$PR_URL\">PR#"$PR_NUMBER"</a> '"$ESCAPED_NAME"' to branch '"$PR_BASE"'"`
         elif [[ ! -z "${{ github.event.pull_request }}" && "${{ github.event.action }}" == "synchronize" ]]; then
-          TEXT=`echo -e "<b>${{ env.PR_AUTHOR }}</b> updated <a href=\"${{ env.PR_URL }}\">PR#${{ env.PR_NUMBER }}</a>"`
+          TEXT=`echo -e "<b>"$PR_AUTHOR"</b> updated <a href=\"$PR_URL\">PR#"$PR_NUMBER"</a>"`
         elif [[ ! -z "${{ github.event.pull_request }}" && "${{ github.event.action }}" == "closed" && "${{ github.event.pull_request.merged }}" == "true" ]]; then
-          TEXT=`echo -e "<b>${{ env.PR_MERGED }}</b> merged <a href=\"${{ env.PR_URL }}\">PR#${{ env.PR_NUMBER }}</a> to branch <b>'${{ env.PR_BASE }}'</b>"`
+          TEXT=`echo -e "<b>"$PR_MERGED"</b> merged <a href=\"$PR_URL\">PR#"$PR_NUMBER"</a> to branch <b>'"$PR_BASE"'</b>"`
         elif [[ ! -z "${{ github.event.pull_request }}" && "${{ github.event.action }}" == "closed" ]]; then
-          TEXT=`echo -e "<b>${{ env.PR_AUTHOR }}</b> closed <a href=\"${{ env.PR_URL }}\">PR#${{ env.PR_NUMBER }}</a>"`
+          TEXT=`echo -e "<b>"$PR_AUTHOR"</b> closed <a href=\"$PR_URL\">PR#"$PR_NUMBER"</a>"`
 
         elif [[ ! -z "${{ github.event.comment }}" ]]; then
-          ESCAPED_TEXT=`echo -e "${{ env.COMMENT_BODY }}"| sed 's/\&/\&amp;/g' | sed 's/</\&lt;/g' | sed 's/>/\&gt;/g'`
+          ESCAPED_TEXT=`echo -e "$COMMENT_BODY"| sed 's/\&/\&amp;/g' | sed 's/</\&lt;/g' | sed 's/>/\&gt;/g'`
           if [[ ! -z "${{ github.event.pull_request }}" ]]; then
-            TEXT=`echo -e "<b>${{ env.COMMENT_AUTHOR }}</b> posted the following comment to file <i>${{ env.COMMENT_FILE }}</i> in <a href=\"${{ env.COMMENT_URL }}\">PR#${{ env.PR_NUMBER }}</a>:\n<i>$ESCAPED_TEXT</i>"`
+            TEXT=`echo -e "<b>"$COMMENT_AUTHOR"</b> posted the following comment to file <i>"$COMMENT_FILE"</i> in <a href=\"$COMMENT_URL\">PR#"$PR_NUMBER"</a>:\n<i>"$ESCAPED_TEXT"</i>"`
           else
-            TEXT=`echo -e "<b>${{ env.COMMENT_AUTHOR }}</b> posted the following comment to issue <a href=\"${{ env.COMMENT_URL }}\">#${{ env.COMMENT_NUMBER }}</a>:\n<i>$ESCAPED_TEXT</i>"`
+            TEXT=`echo -e "<b>"$COMMENT_AUTHOR"</b> posted the following comment to issue <a href=\"$COMMENT_URL\">#"$COMMENT_NUMBER"</a>:\n<i>"$ESCAPED_TEXT"</i>"`
           fi
 
-        elif [[ ! -z "${{ github.event.review }}" && "${{ env.REVIEW_STATE }}" == "changes_requested" ]]; then
-          TEXT=`echo -e "<b>${{ env.REVIEW_AUTHOR }}</b> requested changes for <a href=\"${{ env.PR_URL }}\">PR#${{ env.PR_NUMBER }}</a>"`
-        elif [[ ! -z "${{ github.event.review }}" && "${{ env.REVIEW_STATE }}" == "commented" && ! -z "${{ env.REVIEW_COMMENT }}" ]]; then
-          ESCAPED_TEXT=`echo -e "${{ env.REVIEW_COMMENT }}"| sed 's/\&/\&amp;/g' | sed 's/</\&lt;/g' | sed 's/>/\&gt;/g'`
-          TEXT=`echo -e "<b>${{ env.REVIEW_AUTHOR }}</b> posted the following comment to <a href=\"${{ env.PR_URL }}\">PR#${{ env.PR_NUMBER }}</a>:\n<i>$ESCAPED_TEXT</i>"`
-        elif [[ ! -z "${{ github.event.review }}" && "${{ env.REVIEW_STATE }}" != "commented" ]]; then
-          TEXT=`echo -e "<b>${{ env.REVIEW_AUTHOR }}</b> ${{ env.REVIEW_STATE }} <a href=\"${{ env.PR_URL }}\">PR#${{ env.PR_NUMBER }}</a>"`
+        elif [[ ! -z "${{ github.event.review }}" && "$REVIEW_STATE" == "changes_requested" ]]; then
+          TEXT=`echo -e "<b>"$REVIEW_AUTHOR"</b> requested changes for <a href=\"$PR_URL\">PR#"$PR_NUMBER"</a>"`
+        elif [[ ! -z "${{ github.event.review }}" && "$REVIEW_STATE" == "commented" && ! -z "$REVIEW_COMMENT" ]]; then
+          ESCAPED_TEXT=`echo -e "$REVIEW_COMMENT"| sed 's/\&/\&amp;/g' | sed 's/</\&lt;/g' | sed 's/>/\&gt;/g'`
+          TEXT=`echo -e "<b>"$REVIEW_AUTHOR"</b> posted the following comment to <a href=\"$PR_URL\">PR#"$PR_NUMBER"</a>:\n<i>"$ESCAPED_TEXT"</i>"`
+        elif [[ ! -z "${{ github.event.review }}" && "$REVIEW_STATE" != "commented" ]]; then
+          TEXT=`echo -e "<b>"$REVIEW_AUTHOR"</b> "$REVIEW_STATE" <a href=\"$PR_URL\">PR#"$PR_NUMBER"</a>"`
         elif [[ -z "${{ github.event.review }}" && "${{ github.event.action }}" == "submitted" ]]; then
-          TEXT=`echo -e "Due to GitHub Actions bug we cannot identify, who approved <a href=\"${{ env.PR_URL }}\">PR#${{ env.PR_NUMBER }}</a>"`
+          TEXT=`echo -e "Due to GitHub Actions bug we cannot identify, who approved <a href=\"$PR_URL\">PR#"$PR_NUMBER"</a>"`
 
         elif [[ ! -z "${{ github.event.workflow_run }}" && "${{ github.event.workflow_run.conclusion }}" == "success" ]]; then
-          TEXT=`echo -e "<a href=\"${{ env.UNIVERSUM_LOG }}\">Universum run for branch '${{ env.UNIVERSUM_BRANCH }}'</a> <b>SUCCEDED</b>; commit ${{ env.UNIVERUM_COMMIT }} "`
+          ESCAPED_TEXT=`echo -e "$UNIVERSUM_BRANCH"| sed 's/\&/\&amp;/g' | sed 's/</\&lt;/g' | sed 's/>/\&gt;/g'`
+          TEXT=`echo -e "<a href=\"$UNIVERSUM_LOG\">Universum run for branch "$ESCAPED_TEXT"</a> <b>SUCCEDED</b>; commit "$UNIVERUM_COMMIT" "`
         elif [[ ! -z "${{ github.event.workflow_run }}" && "${{ github.event.workflow_run.conclusion }}" == "failure" ]]; then
-          TEXT=`echo -e "<a href=\"${{ env.UNIVERSUM_LOG }}\">Universum run for branch '${{ env.UNIVERSUM_BRANCH }}'</a> <b>FAILED</b>; commit ${{ env.UNIVERUM_COMMIT }} "`
+          ESCAPED_TEXT=`echo -e "$UNIVERSUM_BRANCH"| sed 's/\&/\&amp;/g' | sed 's/</\&lt;/g' | sed 's/>/\&gt;/g'`
+          TEXT=`echo -e "<a href=\"$UNIVERSUM_LOG\">Universum run for branch "$ESCAPED_TEXT"</a> <b>FAILED</b>; commit "$UNIVERUM_COMMIT" "`
         fi
 
-        if [[ ! -z $TEXT ]]; then
+        if [[ ! -z "$TEXT" ]]; then
           curl --get --data-urlencode "chat_id=${{ secrets.TELEGRAM_CHAT_ID }}" --data-urlencode "disable_web_page_preview=True" \
-          --data-urlencode "text=$TEXT" --data-urlencode "parse_mode=HTML" $URL
+          --data-urlencode "text=$TEXT" --data-urlencode "parse_mode=HTML" "$URL"
         fi
 
       env:

CHANGELOG.rst

@@ -1,6 +1,18 @@
 Change log
 ==========
 
+0.19.20 (2024-10-31)
+--------------------
+
+Bug fixes
+~~~~~~~~~
+
+* **report:** parse data from sarif as URI, instead of treating it as string
+* **code_report:** create parent folders for target file for clang_format
+* **analyzers:** use recursive glob setting
+* **doc:** added dependencies to setup.py and to README
+
+
 0.19.19 (2023-11-22)
 --------------------
 

README.md

@@ -40,14 +40,18 @@ and then use the commands listed below. Please note we use `venv` to properly se
 python interpreter version and to isolate development environment from the system.
 
 Prerequisites:
-1. Install all of the VCS extras as described in the [Universum installation manual](
+1. Make sure the libssl-dev and libcrypto++-dev packages are available in your environment.
+    ```bash
+    sudo apt install libssl-dev libcrypto++-dev
+    ```
+2. Install all of the VCS extras as described in the [Universum installation manual](
    https://universum.readthedocs.io/en/latest/install.html#vcs-related-extras),
    (including installation of Git and P4 CLI)
-2. Install Docker (`docker-ce`, `docker-ce-cli`) as described in the [official installation manual](
+3. Install Docker (`docker-ce`, `docker-ce-cli`) as described in the [official installation manual](
    https://docs.docker.com/engine/installation/linux/ubuntu/#install-using-the-repository)
 
    * Also add current user to 'docker' group (use `sudo usermod -a -G docker $USER` and then relogin)
-3. Install Mozilla WebDriver: `sudo apt install firefox-geckodriver`
+4. Install Mozilla WebDriver: `sudo apt install firefox-geckodriver`
 
 Further commands:
 ```bash

setup.py

@@ -59,7 +59,8 @@ def readme():
             'types-requests',
             'selenium==3.141',
             'urllib3==1.26.15',  # This is required for selenium-3.141 to work correctly
-            'types-PyYAML==6.0'
+            'types-PyYAML==6.0',
+            'wheel'
         ]
     },
     package_data={'': ['*.css', '*.js']}

tests/test_code_report.py

@@ -169,6 +169,46 @@ def finalize(self) -> str:
 }
 """
 
+sarif_report_uri = """
+{
+  "$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json",
+  "version": "2.1.0",
+  "runs": [
+    {
+      "tool": {
+        "driver": {
+          "name": "Checkstyle",
+          "semanticVersion": "8.43",
+          "version": "8.43"
+        }
+      },
+      "results": [
+        {
+          "level": "warning",
+          "locations": [
+            {
+              "physicalLocation": {
+                "artifactLocation": {
+                  "uri": "file:///my_path/my_file"
+                },
+                "region": {
+                  "startColumn": 1,
+                  "startLine": 1
+                }
+              }
+            }
+          ],
+          "message": {
+            "text": "Error!"
+          },
+          "ruleId": "testRule"
+        }
+      ]
+    }
+  ]
+}
+"""
+
 config_uncrustify = """
 code_width = 120
 input_tab_size = 2
@@ -189,6 +229,7 @@ def finalize(self) -> str:
     [[sarif_report_minimal], True],
     [[sarif_report], False],
     [[sarif_report_split_uri], False],
+    [[sarif_report_uri], False],
     [[json_report_minimal, sarif_report_minimal], True],
     [[json_report, sarif_report], False],
     [[json_report_minimal, sarif_report], False],
@@ -199,6 +240,7 @@ def finalize(self) -> str:
     'sarif_no_issues',
     'sarif_issues_found',
     'sarif_split_uri_issues_found',
+    'sarif_uri',
     'both_tested_no_issues',
     'both_tested_issues_in_both',
     'both_tested_issues_in_sarif',
@@ -405,3 +447,20 @@ def test_code_report_extended_arg_search_embedded(tmp_path: pathlib.Path, stdout
 
     env.run()
     stdout_checker.assert_absent_calls_with_param("${CODE_REPORT_FILE}")
+
+
+def test_clang_format_analyzer_with_subfolder(runner_with_analyzers: UniversumRunner):
+    root = runner_with_analyzers.local.root_directory
+    source_file = root / "subdir" / "source_file"
+    source_file.parent.mkdir(parents=True, exist_ok=True)
+    source_file.write_text(source_code_c)
+    common_args = [
+        "--result-file", "${CODE_REPORT_FILE}",
+        "--files", "subdir/source_file"
+    ]
+
+    (root / ".clang-format").write_text(config_clang_format)
+    log = runner_with_analyzers.run(ConfigData().add_analyzer("clang_format", common_args).finalize())
+
+    assert not re.findall(r'No such file or directory', log), f"'No such file or directory' is found in '{log}'"
+    assert re.findall(log_fail, log), f"'{log_fail}' is not found in '{log}'"

tests/test_p4_exception_handling.py

@@ -91,12 +91,8 @@ def test_p4_print_exception_in_sync(perforce_environment: P4TestEnvironment, std
     stdout_checker.assert_has_calls_with_param(text)
 
 
-def test_p4_print_exception_wrong_shelve(perforce_environment: P4TestEnvironment, stdout_checker: FuzzyCallChecker):
+def test_p4_print_exception_already_committed_shelve(perforce_environment: P4TestEnvironment, stdout_checker: FuzzyCallChecker):
     cl = perforce_environment.vcs_client.make_a_change()
     perforce_environment.settings.PerforceMainVcs.shelve_cls = [cl]
-
-    # This is not the 'already committed' case of Swarm review, so it actually should fail
     perforce_environment.run(expect_failure=True)
-    stdout_checker.assert_has_calls_with_param(
-        f"Errors during command execution( \"p4 unshelve -s {cl} -f\" )")
-    stdout_checker.assert_has_calls_with_param(f"[Error]: 'Change {cl} is already committed.'")
+    stdout_checker.assert_has_calls_with_param(f"Change {cl} is already committed.")

tests/test_poll.py

@@ -59,7 +59,7 @@ def test_p4_error_command_line_wrong_port(stdout_checker: FuzzyCallChecker,
                             "-p4d", perforce_workspace.depot,
                             "-jtu", "https://localhost/?%s"])
     assert result != 0
-    stdout_checker.assert_has_calls_with_param("TCP connect to 127.0.0.1:1024 failed.")
+    stdout_checker.assert_has_calls_with_param("Connect to server failed; check $P4PORT")
 
 
 def test_git_error_command_line_wrong_port(stdout_checker: FuzzyCallChecker,

universum/__init__.py

@@ -1,2 +1,2 @@
 __title__ = "Universum"
-__version__ = "0.19.19"
+__version__ = "0.19.20"

universum/analyzers/clang_format.py

@@ -39,6 +39,7 @@ def main(settings: argparse.Namespace) -> List[utils.ReportData]:
         cmd = [settings.executable, src_file_absolute]
         _add_style_param_if_present(cmd, settings)
         output, _ = utils.run_for_output(cmd)
+        target_file_absolute.parent.mkdir(parents=True, exist_ok=True)
         with open(target_file_absolute, "w", encoding="utf-8") as output_file:
             output_file.write(output)
 

universum/analyzers/utils.py

@@ -120,7 +120,7 @@ def expand_files_argument(settings: argparse.Namespace) -> None:
     # TODO: subclass argparse.Action
     result: Set[str] = set()
     for pattern in settings.file_list:
-        file_list: List[str] = glob.glob(pattern)
+        file_list: List[str] = glob.glob(pattern, recursive=True)
         if not file_list:
             sys.stderr.write(f"Warning: no files found for input pattern {pattern}\n")
         else:

universum/modules/code_report_collector.py

@@ -65,17 +65,17 @@ def _process_one_sarif_issue(self, issue, root_uri_base_paths, who) -> None:
                 if location_data.get('address'):
                     continue  # binary artifact can't be processed
                 raise ValueError("Unexpected lack of artifactLocation tag")
+            uri = artifact_data.get('uri')
+            if not uri:
+                raise ValueError("Unexpected lack of uri tag")
+            path = urllib.parse.unquote(urllib.parse.urlparse(uri).path)
             if artifact_data.get('uriBaseId'):
-                uri = artifact_data.get('uri')
-                if not uri:
-                    raise ValueError("Unexpected lack of uri tag")
+                # means path is relative, need to make absolute
                 uri_base_id = artifact_data.get('uriBaseId', '')
                 root_base_path = root_uri_base_paths.get(uri_base_id, '')
                 if uri_base_id and not root_base_path:
                     raise ValueError(f"Unexpected lack of 'originalUriBaseIds' value for {uri_base_id}")
-                path = str(Path(root_base_path, urllib.parse.unquote(uri)))
-            else:
-                path = urllib.parse.unquote(artifact_data.get('uri', ''))
+                path = str(Path(root_base_path) / path)
             region_data = location_data.get('region')
             if not region_data:
                 continue  # TODO: cover this case as comment to the file as a whole