azure-pipelines.yml

#####################################################################
# PUMA Azure run.
# Checks out all the various codes (source, config, web), runs the
# pipeline then commits the HTML files back to GitHub to display at
# https://ollybutters.github.io/puma/
# Set to run regularly so the citations are updated as mandated by
# Scopus.
#
#####################################################################


#########################################################################################
# Need to define all the GH repos and their access tokens, see:
# https://docs.microsoft.com/en-us/azure/devops/pipelines/library/service-endpoints?view=azure-devops&tabs=yaml
resources:
  repositories:
  # The actual source code
  - repository: puma_source
    type: github
    endpoint: puma_gh_deployment
    name: OllyButters/puma
    ref: refs/tags/v2.9
  
  # Where all the configs are (private repo)
  - repository: puma_configs
    type: github
    endpoint: puma_gh_deployment
    name: OllyButters/puma_configs
    ref: master

  # The gh-pages branch of puma_Web repo
  - repository: puma_web
    type: github
    endpoint: puma_gh_deployment
    name: datashield/papers
    ref: gh-pages

  # Cache repo (private repo)
  - repository: puma_cache
    type: github
    endpoint: puma_gh_deployment
    name: OllyButters/puma_cache
    ref: cached_data


#########################################################################################
# Don't start a run when code is committed. You can do this manually in Azure. 
trigger: none


#########################################################################################
# Do run on a schedule though.
schedules:
  - cron: "32 1 * * 0"
    displayName: Weekly build
    branches:
     include:
      - main
    always: true

jobs:
- job: RUN_PUMA
  timeoutInMinutes: 120
  pool:
    vmImage: 'ubuntu-22.04'
  strategy:
    matrix:
      ${{ if or(eq(variables['Build.CronSchedule.DisplayName'], 'Weekly build'), eq(variables['Build.Reason'], 'manual')) }}:
        DATASHIELD:
          config_file_name: 'config_DataSHIELD.ini'
          project_short_name: 'datashield'
    maxParallel: 2

  steps:
    #####################################################################################
    # Checkout the source code to a subfolder.
    # This may give an error in the logs like:
    # [warning]Unable move and reuse existing repository to required location
    # This is an Azure bug - https://github.com/microsoft/azure-pipelines-yaml/issues/403
  - checkout: puma_source
    path: 'puma'

  - checkout: puma_configs
    path: 'configs'

  - checkout: puma_web
    path: 'web'
    persistCredentials: true

  - checkout: puma_cache
    path: 'cache'
    persistCredentials: true

    #####################################################################################
    # Clear scopus cache files if they are old. Need to do it here as when checked out
    # from github they have a modification date of now (according to the OS).
  - bash: |

      echo "Time now:"
      echo $(date +%s)

      let allowed_dif=6*24*60*60
      
      files=$(project_short_name)/raw/scopus/*

      num_files=$(ls ${files} | wc -l)   
    
      if [ "$num_files" -gt "0" ]
      then
        for thisFile in ${files}
        do
          echo "----------"
          echo ${thisFile}
          
          fileAge=$(git log -1 --pretty="format:%at" ${thisFile})
          echo ${fileAge}
          echo $(date -d @${fileAge})
          
          now=$(date +%s)
    
          let diff=now-fileAge

          echo ${diff}
    
          if [ "${diff}" -gt "${allowed_dif}" ]
          then
            echo "Too old, deleting!"
            rm ${thisFile}
          fi
        done
      fi

      # zotero
      let allowed_dif=7*24*60*60
      
      files=$(project_short_name)/raw/zotero/*

      num_files=$(ls ${files} | wc -l)   
    
      if [ "$num_files" -gt "0" ]
      then
        for thisFile in ${files}
        do
          echo ${thisFile}
          fileAge=$(git log -1 --pretty="format:%at" ${thisFile})
    
          now=$(date +%s)
    
          let diff=now-fileAge
    
          if [ "${diff}" -gt "${allowed_dif}" ]
          then
            echo "Too old, deleting!"
            rm ${thisFile}
          fi
        done
      fi

    workingDirectory: $(Pipeline.Workspace)/cache
    displayName: 'Clear old cache files'
    condition: succeeded()



    #####################################################################################
    # Install all the Python dependencies from the requirements.txt file.
  - bash: |

      cd source
      sudo -H pip3 install -r requirements.txt
  
    workingDirectory: $(Pipeline.Workspace)/puma
    displayName: 'Install Python deps'
    condition: succeeded()



    #####################################################################################
    # Some minor set up, then run the code. The pipeline assumes config.ini if no name specified.
  - bash: |

      # Soft link to the checked out cache dir.
      ln -s $(Pipeline.Workspace)/cache cache

      # Soft link to the config file from the configs repo
      cd config
      ln -s $(Pipeline.Workspace)/configs/$(config_file_name) config.ini

      # Soft link to the about DataSHIELD html file
      ln -s $(Pipeline.Workspace)/configs/about_DataSHIELD.html about_DataSHIELD.html


      cd ../source
      sudo ./papers.py

    workingDirectory: $(Pipeline.Workspace)/puma
    displayName: 'Run the pipeline'
    condition: succeeded()


    #####################################################################################
    # Prettify the output HTML (i.e. put on new lines and indent)
  - bash: |

      sudo apt install tidy -y

      OIFS="$IFS"
      IFS=$'\n'

      for f in `find . -type f -name "*.html"`
      do
        echo "$f" 
        sudo tidy --quiet true --indent true --indent-spaces 4 --wrap 0 --tidy-mark false -m "$(Pipeline.Workspace)/puma/html/$(project_short_name)/$f"
      done

      IFS="$OIFS"
    
    workingDirectory: $(Pipeline.Workspace)/puma/html/$(project_short_name)
    displayName: 'Prettify HTML'
    condition: succeeded()


    #####################################################################################
    # Commit the generated HTML files to the gh-pages. This makes them available
    # (after some caching time) at https://ollybutters.github.io/puma/$(project_short_name)
  - bash: |

      # Git needs some config set to be able to push to a repo. 
      git config --global user.email "you@example.com"
      git config --global user.name "Azure pipeline"
      
      # Checkout again - stuff may have changed in this branch from elsewhere
      git fetch --all
      git checkout gh-pages    
      git pull

      cp -r $(Pipeline.Workspace)/puma/html/$(project_short_name)/* .

      git add .
      git commit -m "Automatically updating web files for $(project_short_name)"
      git push

    workingDirectory: $(Pipeline.Workspace)/web
    displayName: 'Commit web files'
    condition: succeeded()


    #####################################################################################
    # Update cache. Always do this so if it fails mid way through all is not lost.
  - bash: |

      # Git needs some config set to be able to push to a repo. 
      git config --global user.email "you@example.com"
      git config --global user.name "Azure pipeline"
      
      # Checkout again - stuff may have changed from elsewhere
      git fetch --all
      git checkout cached_data
      git pull

      git add .
      git commit -m "Automatically updating cache files for $(project_short_name)"
      git push

    workingDirectory: $(Pipeline.Workspace)/cache
    displayName: 'Commit cache files'
    condition: always()



    #####################################################################################
    # Output the log file for debug
  - bash: |
      
      ls
  
      du -h $(project_short_name).log

      cat $(project_short_name).log
      
    workingDirectory: $(Pipeline.Workspace)/puma/logs
    displayName: 'Print logfile'
    condition: always()


    #####################################################################################
    # Output some diagnostics in case something went wrong.
  - bash: |

      echo -e "\n#############################"
      echo -e "python /: ######################"
      python3 --version

      echo -e "\n#############################"
      echo -e "python modules/: ######################"
      pip3 freeze

      echo -e "\n#############################"
      echo -e "ls /: ######################"
      ls $(Pipeline.Workspace)
    
      echo -e "\n#############################"
      echo -e "lscpu: ######################"
      lscpu
      
      echo -e "\n#############################"
      echo -e "memory: #####################"
      free -m
      
      echo -e "\n#############################"
      echo -e "env: ########################"
      env

      sudo apt install tree -y
      pwd
      echo -e "\n#############################"
      echo -e "File tree: ##################"
      tree $(Pipeline.Workspace)

    displayName: 'Diagnostics'
    condition: always()