Merge pull request #555 from victordrose/feature/azure-ci-pipeline

Feature/azure ci pipeline

documentation/azure-devops/setup-ci-pipeline.asciidoc

@@ -0,0 +1,78 @@
+:provider: Azure DevOps
+:pipeline_type: pipeline
+:trigger_sentence: This pipeline will be configured to be triggered every time there is a commit to the Azure DevOps repository
+:pipeline_type2: pipeline
+:path_provider: azure-devops
+:openBrowserFlag: -w
+= Setting up a CI {pipeline_type} on {provider}
+
+In this section we will create a Continuous Integration {pipeline_type} for compiling project code. {trigger_sentence}, regardless of which branch it is made on.
+
+The creation of the {pipeline_type2} will follow the project workflow, so a new branch named `feature/ci-pipeline` will be created and the YAML file for the {pipeline_type} will be pushed to it.
+
+Then, a Pull Request (PR) will be created in order to merge the new branch into the appropriate branch (provided in `-b` flag). The PR will be automatically merged if the repository policies are met. If the merge is not possible, either the PR URL will be shown as output, or it will be opened in your web browser if using `-w` flag.
+
+The script located at `/scripts/pipelines/{path_provider}/pipeline_generator.sh` will automatically create this new branch, create a build {pipeline_type} based on a YAML template appropriate for the project programming language or framework, create the Pull Request and, if it is possible, merge this new branch into the specified branch.
+
+== Prerequisites
+
+This script will commit and push the corresponding YAML template into your repository, so please be sure your local repository is up-to-date (i.e you have pulled latest changes with `git pull`).
+
+== Creating the {pipeline_type} using provided script
+
+=== Usage
+[subs=attributes+]
+```
+pipeline_generator.sh \
+  -c <config file path> \
+  -n <{pipeline_type} name> \
+  -l <language or framework> \
+  -d <project local path> \
+  [-t <target-directory>] \
+  [-b <branch>] \
+  [-w]
+```
+
+NOTE: The config file for the build {pipeline_type} is located at `/scripts/pipelines/{path_provider}/templates/ci/ci-pipeline.cfg`.
+
+=== Flags
+[subs=attributes+]
+```
+-c, --config-file                    [Required] Configuration file containing {pipeline_type} definition.
+-n, --pipeline-name                  [Required] Name that will be set to the {pipeline_type}.
+-l, --language or framework          [Required] Language or framework of the project.
+--sonar-url <sonarqube url>
+--sonar-token <sonarqube token>
+-d, --local-directory                [Required] Local directory of your project.
+-t, --target-directory                Target directory of build process. Takes precedence over the language/framework default one.
+-b, --target-branch                   Name of the branch to which the Pull Request will target. PR is not created if the flag is not provided.
+-w                                    Open the Pull Request on the web browser if it cannot be automatically merged. Requires -b flag.
+```
+
+=== Examples
+
+==== Quarkus project
+
+===== Quarkus native project
+[subs=attributes+]
+```
+./pipeline_generator.sh -c ./templates/ci/ci-pipeline.cfg -n quarkus-project-build -l quarkus --sonar-url http://1.2.3.4:9000 --sonar-token 6ce6663b63fc02881c6ea4c7cBa6563b8247a04e -d C:/Users/$USERNAME/Desktop/quarkus-project -b develop {openBrowserFlag}
+```
+
+===== Quarkus project using JVM
+[subs=attributes+]
+```
+./pipeline_generator.sh -c ./templates/ci/ci-pipeline.cfg -n quarkus-project-build -l quarkus-jvm --sonar-url http://1.2.3.4:9000 --sonar-token 6ce6663b63fc02881c6ea4c7cBa6563b8247a04e -d C:/Users/$USERNAME/Desktop/quarkus-project -b develop {openBrowserFlag}
+```
+
+==== Node.js project
+[subs=attributes+]
+```
+./pipeline_generator.sh -c ./templates/ci/ci-pipeline.cfg -n node-project-build -l node --sonar-url http://1.2.3.4:9000 --sonar-token 6ce6663b63fc02881c6ea4c7cBa6563b8247a04e -d C:/Users/$USERNAME/Desktop/node-project -b develop {openBrowserFlag}
+```
+
+==== Angular project
+[subs=attributes+]
+```
+./pipeline_generator.sh -c ./templates/ci/ci-pipeline.cfg -n angular-project-build -l angular --sonar-url http://1.2.3.4:9000 --sonar-token 6ce6663b63fc02881c6ea4c7cBa6563b8247a04e -d C:/Users/$USERNAME/Desktop/angular-project -b develop {openBrowserFlag}
+```

documentation/src/azure-devops/setup-ci-pipeline.asciidoc

@@ -0,0 +1,7 @@
+:provider: Azure DevOps
+:pipeline_type: pipeline
+:trigger_sentence: This pipeline will be configured to be triggered every time there is a commit to the Azure DevOps repository
+:pipeline_type2: pipeline
+:path_provider: azure-devops
+:openBrowserFlag: -w
+include::../common_templates/setup-ci-pipeline.asciidoc[]

documentation/src/common_templates/setup-ci-pipeline.asciidoc

@@ -0,0 +1,114 @@
+= Setting up a ci {pipeline_type} on {provider}
+
+In this section we will create a Continuous Integration {pipeline_type} for compiling project code. {trigger_sentence}, regardless of which branch it is made on.
+
+The creation of the {pipeline_type2} will follow the project workflow, so a new branch named `feature/ci-pipeline` will be created and the YAML file for the {pipeline_type} will be pushed to it.
+
+ifndef::no-PR-or-MR[]
+Then, a Pull Request (PR) will be created in order to merge the new branch into the appropriate branch (provided in `-b` flag). The PR will be automatically merged if the repository policies are met. If the merge is not possible, either the PR URL will be shown as output, or it will be opened in your web browser if using `-w` flag.
+
+endif::[]
+ifdef::no-PR-or-MR[]
+Then, the new branch will be merged into the appropriate branch (provided in `-b` flag).
+
+endif::[]
+The script located at `/scripts/pipelines/{path_provider}/pipeline_generator.sh` will automatically create this new branch, create a ci {pipeline_type} based on a YAML template appropriate for the project programming language or framework, create the Pull Request and, if it is possible, merge this new branch into the specified branch.
+
+ifdef::extra_sentence_ci[]
+{extra_sentence_ci}
+
+endif::[]
+== Prerequisites
+
+This script will commit and push the corresponding YAML template into your repository, so please be sure your local repository is up-to-date (i.e you have pulled latest changes with `git pull`).
+
+== Creating the {pipeline_type} using provided script
+
+=== Usage
+[subs=attributes+]
+```
+pipeline_generator.sh \
+  -c <config file path> \
+  -n <{pipeline_type} name> \
+  -l <language or framework> \
+  -d <project local path> \
+ifeval::["{provider}" == "Google Cloud"]
+  [--language-version <version>] \
+  [--registry-location <artifact registry>] \
+endif::[]
+  [-t <target-directory>] \
+  [-b <branch>] \
+ifndef::no-PR-or-MR[  [-w]]
+ifeval::["{provider}" == "Google Cloud"]
+  [-m <machine type for {pipeline_type} runner>]
+  [--env-vars <env vars list>]
+  [--secret-vars <secret vars list>]
+endif::[]
+```
+
+NOTE: The config file for the build {pipeline_type} is located at `/scripts/pipelines/{path_provider}/templates/build/build-pipeline.cfg`.
+
+=== Flags
+[subs=attributes+]
+```
+-c, --config-file        [Required] Configuration file containing {pipeline_type} definition.
+-n, --pipeline-name      [Required] Name that will be set to the {pipeline_type}.
+-l, --language           [Required] Language or framework of the project.
+--sonar-url <sonarqube url>
+--sonar-token <sonarqube token>
+-d, --local-directory    [Required] Local directory of your project.
+ifeval::["{provider}" == "Google Cloud"]
+    --language-version   [Required, if Flutter or Python] Language or framework version.
+    --registry-location  [Required, if Flutter] Artifact registry location.
+endif::[]
+-t, --target-directory              Target directory of build process. Takes precedence over the language/framework default one.
+-b, --target-branch                 Name of the branch to which the Pull Request will target. PR is not created if the flag is not provided.
+ifndef::no-PR-or-MR[-w                                  Open the Pull Request on the web browser if it cannot be automatically merged. Requires -b flag.]
+ifeval::["{provider}" == "Google Cloud"]
+-m, --machine-type                  Machine type for {pipeline_type} runner. Accepted values: E2_HIGHCPU_8, E2_HIGHCPU_32, N1_HIGHCPU_8, N1_HIGHCPU_32.
+    --env-vars                      List of environment variables to be made available in pipeline. Syntax: "var1=val1 var2=val2 ...".
+    --secret-vars                   List of environment variables (saved as secrets in Secret Manager) to be made available in pipeline. Syntax: "var1=val1 var2=val2 ...".
+endif::[]
+```
+
+=== Examples
+
+==== Quarkus project
+
+===== Quarkus native project
+[subs=attributes+]
+```
+./pipeline_generator.sh -c ./templates/ci/ci-pipeline.cfg -n quarkus-project-build -l quarkus --sonar-url http://1.2.3.4:9000 --sonar-token 6ce6663b63fc02881c6ea4c7cBa6563b8247a04e -d C:/Users/$USERNAME/Desktop/quarkus-project -b develop -w
+```
+
+===== Quarkus project using JVM
+[subs=attributes+]
+```
+./pipeline_generator.sh -c ./templates/ci/ci-pipeline.cfg -n quarkus-project-build -l quarkus-jvm --sonar-url http://1.2.3.4:9000 --sonar-token 6ce6663b63fc02881c6ea4c7cBa6563b8247a04e -d C:/Users/$USERNAME/Desktop/quarkus-project -b develop -w
+```
+
+==== Node.js project
+[subs=attributes+]
+```
+./pipeline_generator.sh -c ./templates/ci/ci-pipeline.cfg -n node-project-build -l node --sonar-url http://1.2.3.4:9000 --sonar-token 6ce6663b63fc02881c6ea4c7cBa6563b8247a04e -d C:/Users/$USERNAME/Desktop/node-project -b develop -w
+```
+
+==== Angular project
+[subs=attributes+]
+```
+./pipeline_generator.sh -c ./templates/ci/ci-pipeline.cfg -n angular-project-build -l angular --sonar-url http://1.2.3.4:9000 --sonar-token 6ce6663b63fc02881c6ea4c7cBa6563b8247a04e -d C:/Users/$USERNAME/Desktop/angular-project -b develop -w
+```
+ifeval::["{provider}" == "Google Cloud"]
+
+==== Python project
+[subs=attributes+]
+```
+./pipeline_generator.sh -c ./templates/ci/ci-pipeline.cfg -n python-project-build -l python --sonar-url http://1.2.3.4:9000 --sonar-token 6ce6663b63fc02881c6ea4c7cBa6563b8247a04e -d C:/Users/$USERNAME/Desktop/python-project -b develop --language-version 3.10 {openBrowserFlag}
+```
+
+==== Flutter project
+[subs=attributes+]
+```
+./pipeline_generator.sh -c ./templates/ci/ci-pipeline.cfg -n flutter-project-build -l flutter --sonar-url http://1.2.3.4:9000 --sonar-token 6ce6663b63fc02881c6ea4c7cBa6563b8247a04e -d C:/Users/$USERNAME/Desktop/flutter-project -b develop --language-version 3.3.4 --registry-location europe {openBrowserFlag}
+```
+endif::[]

scripts/pipelines/azure-devops/pipeline_generator.sh

@@ -99,7 +99,7 @@ function createPR {
         echo -e "${green}Creating a Pull Request..."
         echo -ne ${white}
         # Create the Pull Request to merge into the specified branch.
-        pr=$(az repos  pr create --source-branch ${sourceBranch} --target-branch $targetBranch --title "Pipeline" --auto-complete true)
+        pr=$(az repos  pr create --source-branch ${sourceBranch} --target-branch $targetBranch --title "Pipeline" --auto-complete true --project "$projectName" --repository "$repoName")
 
         # Obtain the PR id.
         id=$(echo "$pr" | python -c "import sys, json; print(json.load(sys.stdin)['pullRequestId'])")

scripts/pipelines/azure-devops/templates/ci/ci-pipeline.cfg

@@ -0,0 +1,43 @@
+# Mandatory flags.
+mandatoryFlags="$pipelineName,$localDirectory,$language,$sonarUrl,$sonarToken"
+# Path to the templates.
+templatesPath="scripts/pipelines/azure-devops/templates/ci"
+# Path to the common templates folder
+commonTemplatesPipelinePath=("scripts/pipelines/common/templates/build" "scripts/pipelines/common/templates/test" "scripts/pipelines/common/templates/quality")
+# YAML file name.
+yamlFile="ci-pipeline.yml"
+# Script name.
+scriptFile=("build.sh" "test.sh" "quality.sh")
+# Source branch.
+sourceBranch="feature/ci-pipeline"
+
+
+# Function that copies the script to build, test and check quality of the application.
+function copyScript {
+    # Copy the build script.
+    cp "${hangarPath}/${commonTemplatesPipelinePath[0]}/${language}-${scriptFile[0]}" "${localDirectory}/${scriptFilePath}/${scriptFile[0]}"
+    # Copy the test script.
+    cp "${hangarPath}/${commonTemplatesPipelinePath[1]}/${language}-${scriptFile[1]}" "${localDirectory}/${scriptFilePath}/${scriptFile[1]}"
+    # Check if the language is quarkus.
+    if [[ $language == "quarkus"* ]]
+    then
+        language="quarkus"
+    fi
+    # Copy the quality script.
+    cp "${hangarPath}/${commonTemplatesPipelinePath[2]}/${language}-${scriptFile[2]}.template" "${localDirectory}/${scriptFilePath}/${scriptFile[2]}"
+}
+
+# Function that adds the variables to be used in the pipeline.
+function addPipelineVariables {
+
+    # Add sonar credentials.
+    az pipelines variable create --name "sonarUrl" --pipeline-name "$pipelineName" --value "${sonarUrl}"
+    az pipelines variable create --name "sonarToken" --pipeline-name "$pipelineName" --value "${sonarToken}" --secret true
+    # if the user did not specify a custom target-directory
+    # we default to the language specific defaults
+    if test -z $targetDirectory
+    then
+        setTargetDirectory
+    fi
+    az pipelines variable create --name "targetPath" --pipeline-name "${pipelineName}" --value "${targetDirectory}"
+}

scripts/pipelines/azure-devops/templates/ci/ci-pipeline.yml.template

@@ -0,0 +1,62 @@
+trigger:
+  branches:
+   include:
+   - '*'
+
+parameters:
+  - name: artifactPath
+    displayName: Build Artifact Path
+    default: $(targetPath)
+    type: string
+  - name: sonarUrl
+    displayName: Sonar URL
+    default: $(sonarUrl)
+    type: string
+  - name: sonarToken
+    displayName: Sonar Token
+    default: $(sonarToken)
+    type: string 
+
+variables:
+  projectPath: $(Build.Repository.LocalPath)
+
+pool:
+  vmImage: ubuntu-latest
+
+steps:
+- task: Bash@3
+  displayName: "Build"
+  inputs:
+    filePath: '$(Build.Repository.LocalPath)/.pipelines/scripts/build.sh'
+
+- task: Bash@3
+  displayName: "Archiving build artifact to the ArtifactStagingDirectory"
+  inputs:
+    targetType: inline
+    script: 'tar -cvf $(Build.ArtifactStagingDirectory)/BuildOutput.tar -C $(Build.Repository.LocalPath)/${{ parameters.artifactPath }} .'
+
+- publish: $(Build.ArtifactStagingDirectory)/BuildOutput.tar
+  displayName: "Publish Artifact"
+  artifact: BuildOutput
+
+- task: Bash@3
+  displayName: "Test"
+  inputs:
+    filePath: '$(Build.Repository.LocalPath)/.pipelines/scripts/test.sh'
+
+- task: PublishTestResults@2
+  displayName: "Publish Test Results"
+  inputs:
+    testResultsFormat: 'JUnit'
+    testResultsFiles: '**/TEST-*.xml'
+
+- task: Bash@3
+  displayName: "SonarQube Analysis"
+  inputs:
+    filePath: '$(Build.Repository.LocalPath)/.pipelines/scripts/quality.sh'
+  env:
+    PROJECT_PATH: $(projectPath)
+    SONAR_URL: ${{ parameters.sonarUrl }}
+    SONAR_TOKEN: ${{ parameters.sonarToken }}
+
+

scripts/pipelines/common/templates/build/python-build.sh

@@ -9,11 +9,11 @@ export PIP_NO_CACHE_DIR=1
 export POETRY_VERSION=1.0.5
 
 pip install "poetry==$POETRY_VERSION"
-python -m venv /venv
-poetry export --without-hashes -f requirements.txt | /venv/bin/pip install -r /dev/stdin
+python -m venv ./venv
+poetry export --without-hashes -f requirements.txt | ./venv/bin/pip install -r /dev/stdin
 # shellcheck source=/dev/null
-. /venv/bin/activate
+. ./venv/bin/activate
 mypy .
-poetry build && /venv/bin/pip install dist/*.whl
+poetry build && ./venv/bin/pip install dist/*.whl
 deactivate
-mv /venv ./venv
+#mv /venv ./venv

scripts/pipelines/common/templates/test/python-test.sh

@@ -1,9 +1,9 @@
 #!/bin/bash
 set -e
 # shellcheck source=/dev/null
-mv venv /venv
+#mv venv ./venv
 # shellcheck source=/dev/null
-. /venv/bin/activate
+. ./venv/bin/activate
 coverage run -m unittest
 coverage report
-coverage xml
+coverage xml