From 6b9b27c788bc6959fa1e5c367e7a27329171e885 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 18 May 2019 12:06:47 +0700 Subject: [PATCH] tox.ini: Create This replaces Travis CI 'single image' with one VM for each set of bears with similar dependencies, using tox to select the sets. And it replaces AppVeyor CI to use FudgeCI to be a managable 'single image' VM that includes most of the dependencies needed by bears, and uses tox to select the sets of bears. The tox setup allows being used with or without --sitepackages, and the AppVeyor CI does use --sitepackages and other tricks to allow .ps1 finer control over the process of installing python dependencies. This uses tox-backticks to allow multiple selectors to be mapped to multiple tests that pytest will run. It uses pytest-cov-threshold to allow custom thresholds for each source file, allowing each job to deselect the source files that need coverage depending on the selected tests for the job. --- .ci/Export-NUnitXml.psm1 | 107 ++ .ci/Fudge.ps1 | 448 +++++ .ci/FudgeCI.ps1 | 183 ++ .ci/FudgeGenerateFake.ps1 | 42 + .ci/FudgePostInstall.ps1 | 52 + .ci/Modules/FudgeTools.psm1 | 1896 +++++++++++++++++++++ .ci/PSLint.ps1 | 94 + .ci/PSScriptAnalyzerSettings.psd1 | 120 ++ .ci/PrepareAVVM.ps1 | 308 ++++ .ci/appveyor.yml | 224 ++- .ci/choco.config | 59 + .ci/constants.ps1 | 9 + .ci/deps.ActivePerl-packages.ps1 | 9 + .ci/deps.ActivePerl.ps1 | 11 + .ci/deps.alex.sh | 7 + .ci/deps.apt.sh | 74 - .ci/deps.apt_get.sh | 7 + .ci/deps.bakalint.sh | 11 + .ci/deps.cabal.sh | 2 + .ci/deps.composer-packages.ps1 | 9 + .ci/deps.composer.sh | 5 + .ci/deps.elm.sh | 15 + .ci/deps.flawfinder.sh | 16 + .ci/deps.generic.sh | 12 + .ci/deps.ghc-mod.sh | 6 + .ci/deps.ghc-packages.ps1 | 7 + .ci/deps.go.sh | 2 + .ci/deps.golang-packages.ps1 | 10 + .ci/deps.golang.ps1 | 18 + .ci/deps.infer.sh | 6 + .ci/deps.java.sh | 25 +- .ci/deps.julia.jl | 7 + .ci/deps.julia.sh | 7 + .ci/deps.lua.sh | 5 + .ci/deps.minimal.sh | 1 + .ci/deps.node_js.sh | 7 + .ci/deps.nodejs-packages.ps1 | 33 + .ci/deps.npm.sh | 1 + .ci/deps.objective_c.sh | 1 + .ci/deps.opam.sh | 2 + .ci/deps.perl.sh | 1 + .ci/deps.php.ps1 | 123 ++ .ci/deps.php.sh | 1 + .ci/deps.pmd.sh | 11 + .ci/deps.pyenv.sh | 19 + .ci/deps.python-packages.ps1 | 191 +++ .ci/deps.python.ps1 | 44 + .ci/deps.python27.sh | 6 + .ci/deps.python36.sh | 28 + .ci/deps.r.sh | 10 - .ci/deps.ruby-packages.ps1 | 30 + .ci/deps.ruby.ps1 | 11 + .ci/deps.ruby.sh | 15 + .ci/deps.sh | 60 - .ci/deps.tailor.sh | 14 + .ci/generate_coverage_thresholds.py | 35 + .ci/get_bears.py | 55 + .ci/get_codecov_tags.py | 18 + .ci/get_cov_args.py | 20 + .ci/get_tests.py | 156 ++ .ci/refreshenv.sh | 29 + .ci/run_with_env.cmd | 88 - .ci/store_env_in_registry.py | 85 + .ci/travis_extra_globals.sh | 48 + .ci/travis_init.ps1 | 5 + .coafile | 4 +- .moban.dt/bears-appveyor.yml.jj2 | 10 + .moban.dt/bears-test-requirements.txt.jj2 | 7 + .moban.dt/bears-travis.yml.jj2 | 251 +++ .moban.dt/travis-manual-matrix.yml | 210 +++ .moban.yaml | 138 +- .travis.yml | 631 +++++-- Fudgefile | 111 ++ Makefile.PL | 6 + bear-requirements.yaml | 405 +++-- composer.json | 6 + package.json | 2 +- setup.cfg | 1 + test-requirements.txt | 7 + tox.ini | 114 ++ 80 files changed, 6303 insertions(+), 561 deletions(-) create mode 100644 .ci/Export-NUnitXml.psm1 create mode 100644 .ci/Fudge.ps1 create mode 100644 .ci/FudgeCI.ps1 create mode 100644 .ci/FudgeGenerateFake.ps1 create mode 100644 .ci/FudgePostInstall.ps1 create mode 100644 .ci/Modules/FudgeTools.psm1 create mode 100644 .ci/PSLint.ps1 create mode 100644 .ci/PSScriptAnalyzerSettings.psd1 create mode 100644 .ci/PrepareAVVM.ps1 create mode 100644 .ci/choco.config create mode 100644 .ci/constants.ps1 create mode 100644 .ci/deps.ActivePerl-packages.ps1 create mode 100644 .ci/deps.ActivePerl.ps1 create mode 100644 .ci/deps.alex.sh delete mode 100755 .ci/deps.apt.sh create mode 100755 .ci/deps.apt_get.sh create mode 100755 .ci/deps.bakalint.sh create mode 100644 .ci/deps.composer-packages.ps1 create mode 100755 .ci/deps.composer.sh create mode 100755 .ci/deps.elm.sh create mode 100755 .ci/deps.flawfinder.sh create mode 100755 .ci/deps.generic.sh create mode 100755 .ci/deps.ghc-mod.sh create mode 100644 .ci/deps.ghc-packages.ps1 create mode 100644 .ci/deps.golang-packages.ps1 create mode 100644 .ci/deps.golang.ps1 create mode 100755 .ci/deps.infer.sh create mode 100644 .ci/deps.julia.jl create mode 100755 .ci/deps.julia.sh create mode 100755 .ci/deps.lua.sh create mode 120000 .ci/deps.minimal.sh create mode 100755 .ci/deps.node_js.sh create mode 100644 .ci/deps.nodejs-packages.ps1 create mode 100755 .ci/deps.npm.sh create mode 120000 .ci/deps.objective_c.sh create mode 120000 .ci/deps.perl.sh create mode 100644 .ci/deps.php.ps1 create mode 120000 .ci/deps.php.sh create mode 100755 .ci/deps.pmd.sh create mode 100755 .ci/deps.pyenv.sh create mode 100644 .ci/deps.python-packages.ps1 create mode 100644 .ci/deps.python.ps1 create mode 100755 .ci/deps.python27.sh create mode 100755 .ci/deps.python36.sh delete mode 100755 .ci/deps.r.sh create mode 100644 .ci/deps.ruby-packages.ps1 create mode 100644 .ci/deps.ruby.ps1 create mode 100755 .ci/deps.ruby.sh delete mode 100755 .ci/deps.sh create mode 100755 .ci/deps.tailor.sh create mode 100755 .ci/generate_coverage_thresholds.py create mode 100755 .ci/get_bears.py create mode 100755 .ci/get_codecov_tags.py create mode 100755 .ci/get_cov_args.py create mode 100755 .ci/get_tests.py create mode 100644 .ci/refreshenv.sh delete mode 100644 .ci/run_with_env.cmd create mode 100755 .ci/store_env_in_registry.py create mode 100644 .ci/travis_extra_globals.sh create mode 100644 .ci/travis_init.ps1 create mode 100644 .moban.dt/bears-appveyor.yml.jj2 create mode 100644 .moban.dt/bears-travis.yml.jj2 create mode 100644 .moban.dt/travis-manual-matrix.yml create mode 100644 Fudgefile create mode 100644 Makefile.PL create mode 100644 composer.json create mode 100644 tox.ini diff --git a/.ci/Export-NUnitXml.psm1 b/.ci/Export-NUnitXml.psm1 new file mode 100644 index 0000000000..dc73bdfdd9 --- /dev/null +++ b/.ci/Export-NUnitXml.psm1 @@ -0,0 +1,107 @@ +Function Export-NUnitXml { +<# +.SYNOPSIS + Takes results from PSScriptAnalyzer and exports them as a Pester test results file (NUnitXml format). + +.DESCRIPTION + Takes results from PSScriptAnalyzer and exports them as a Pester test results file (NUnit XML schema). + Because the generated file in NUnit-compatible, it can be consumed and published by most continuous integration tools. +#> + [CmdletBinding()] + Param ( + [Parameter(Mandatory, Position=0)] + [AllowNull()] + [Microsoft.Windows.PowerShell.ScriptAnalyzer.Generic.DiagnosticRecord[]]$ScriptAnalyzerResult, + + [Parameter(Mandatory, Position=1)] + [string]$Path + ) + + $TotalNumber = If ($ScriptAnalyzerResult) { $ScriptAnalyzerResult.Count -as [string] } Else { '1' } + $FailedNumber = If ($ScriptAnalyzerResult) { $ScriptAnalyzerResult.Count -as [string] } Else { '0' } + $Now = Get-Date + $FormattedDate = Get-Date $Now -Format 'yyyy-MM-dd' + $FormattedTime = Get-Date $Now -Format 'T' + $User = $env:USERNAME + $MachineName = $env:COMPUTERNAME + $Cwd = $pwd.Path + $UserDomain = $env:USERDOMAIN + $OS = Get-CimInstance -ClassName Win32_OperatingSystem + $Platform = $OS.Caption + $OSVersion = $OS.Version + $ClrVersion = $PSVersionTable.CLRVersion.ToString() + $CurrentCulture = (Get-Culture).Name + $UICulture = (Get-UICulture).Name + + Switch ($ScriptAnalyzerResult) { + $Null { $TestResult = 'Success'; $TestSuccess = 'True'; Break} + Default { $TestResult = 'Failure'; $TestSuccess = 'False'} + } + + $Header = @" + + + + + + + + `n +"@ + + $Footer = @" + + + + + +"@ + + If ( -not($ScriptAnalyzerResult) ) { + + $TestDescription = 'All PowerShell files pass the specified PSScriptAnalyzer rules' + $TestName = "PSScriptAnalyzer.{0}" -f $TestDescription + + $Body = @" + `n +"@ + } + Else { # $ScriptAnalyzerResult is not null + $Body = [string]::Empty + Foreach ( $Result in $ScriptAnalyzerResult ) { + + $TestDescription = "Rule name : $($Result.RuleName)" + $TestName = "PSScriptAnalyzer.{0} - {1} - Line {2}" -f $TestDescription, $($Result.ScriptName), $($Result.Line.ToString()) + + # Need to Escape these otherwise we can end up with an invalid XML if the Stacktrace has non XML friendly chars like &, etc + $Line = [System.Security.SecurityElement]::Escape($Result.Line) + $ScriptPath = [System.Security.SecurityElement]::Escape($Result.ScriptPath) + $Text = [System.Security.SecurityElement]::Escape($Result.Extent.Text) + $Severity = [System.Security.SecurityElement]::Escape($Result.Severity) + + $TestCase = @" + + + $($Result.Message) + at line: $($Line) in $($ScriptPath) + $($Line): $($Text) + Rule severity : $($Severity) + + + `n +"@ + + $Body += $TestCase + } + } + $OutputXml = $Header + $Body + $Footer + + # Checking our output is a well formed XML document + Try { + $XmlCheck = [xml]$OutputXml + } + Catch { + Throw "There was an problem when attempting to cast the output to XML : $($_.Exception.Message)" + } + $OutputXml | Out-File -FilePath $Path -Encoding utf8 -Force +} diff --git a/.ci/Fudge.ps1 b/.ci/Fudge.ps1 new file mode 100644 index 0000000000..df4d407c29 --- /dev/null +++ b/.ci/Fudge.ps1 @@ -0,0 +1,448 @@ +<# + .SYNOPSIS + Fudge is a tool to help you manage and version control Chocolatey packages required for environments to function + + .DESCRIPTION + Fudge is a tool to help you manage and version control Chocolatey packages required for environments to function. + This is done via a Fudgefile which allows you to specify packages (and their versions) to install. You can also + specify dev-specific packages (like git, or fiddler) + + You are also able to define pre/post install/upgrade/downgrade/uninstall scripts for additional required functionality + + Furthermore, Fudge has a section to allow you to specify multiple nuspec files and pack the one you need + + .PARAMETER Action + The action that Fudge should undertake + Actions: install, upgrade, downgrade, uninstall, reinstall, pack, list, search, new, delete, prune, clean, rebuild, + which, help, renew, add, remove + [Alias: -a] + + .PARAMETER Key + The key represents a package/nuspec name in the Fudgefile + [Actions: install, upgrade, downgrade, uninstall, reinstall, pack, new, which, renew, add, remove] + [Alias: -k] + + .PARAMETER FudgefilePath + This will override looking for a default 'Fudgefile' at the root of the current path, and allow you to specify + other files instead. This allows you to have multiple Fudgefiles + [Actions: install, upgrade, downgrade, uninstall, reinstall, pack, list, new, delete, prune, rebuild, renew, add, remove] + [Default: ./Fudgefile] + [Alias: -fp] + + .PARAMETER Limit + This argument only applies for the 'search' action. It will limit the amount of packages returned when searching + If 0 is supplied, the full list is returned + [Actions: search] + [Default: 10] + [Alias: -l] + + .PARAMETER Source + Passing this argument will allow you to specify custom source locations to get/download packages for Chocolatey. + This allows you to install packages from local directories, or from custom Chocolatey servers. Passing this will + also override the source specified in any Fudgefiles + [Default: Chocolatey's server] + [Actions: install, upgrade, downgrade, reinstall, search, rebuild, add] + [Alias: -s] + + .PARAMETER Parameters + This argument allows you to pass parameters to a chocolatey package, as if you were using "--params" on choco. + For install/upgrade/downgrade/uninstall/reinstall, this argument only works when "-Adhoc" is also supplied + [Default: Empty] + [Actions: install, upgrade, downgrade, uninstall, reinstall, add] + [Alias: -p] + + .PARAMETER Arguments + This argument allows you to pass extra arguments to a chocolatey, such as "--x86" or "--ignore-checksum" + For install/upgrade/downgrade/uninstall/reinstall, this argument only works when "-Adhoc" is also supplied + [Default: Empty] + [Actions: install, upgrade, downgrade, uninstall, reinstall, add] + [Alias: -args] + + .PARAMETER Dev + Switch parameter, if supplied will also action upon the devPackages in the Fudgefile + [Actions: install, upgrade, downgrade, uninstall, reinstall, list, delete, prune, rebuild, add, remove] + [Alias: -d] + + .PARAMETER DevOnly + Switch parameter, if supplied will only action upon the devPackages in the Fudgefile + [Actions: install, upgrade, downgrade, uninstall, reinstall, list, delete, prune, rebuild] + [Alias: -do] + + .PARAMETER Install + Switch parameter, if supplied will install packages after creating a new Fudgefile + [Actions: new, renew, add] + [Alias: -i] + + .PARAMETER Uninstall + Switch parameter, if supplied will uninstall packages before deleting a Fudgefile + [Actions: delete, renew, remove] + [Alias: -u] + + .PARAMETER Adhoc + Switch parameter, if supplied will install software from Chocolatey whether or not + the package is in the Fudgefile + [Actions: install, upgrade, downgrade, uninstall, reinstall] + [Alias: -ad] + + .PARAMETER Version + Switch parameter, if supplied will just display the current version of Fudge installed + [Alias: -v] + + .PARAMETER Help + Switch parameter, if supplied will just display help output + [Alias: -h] + + .EXAMPLE + fudge install + + .EXAMPLE + fudge install -d # to also install devPackages (-do will only install devPackages) + + .EXAMPLE + fudge install git -ad # installs git dispite not being in the Fudgefile + + .EXAMPLE + fudge pack website + + .EXAMPLE + fudge list + + .EXAMPLE + fudge search checksum +#> +param ( + [Alias('a')] + [string] + $Action, + + [Alias('k')] + [string] + $Key, + + [Alias('fp')] + [string] + $FudgefilePath, + + [Alias('l')] + [int] + $Limit = 10, + + [Alias('s')] + [string] + $Source, + + [Alias('p')] + [string] + $Parameters, + + [Alias('args')] + [string] + $Arguments, + + [Alias('d')] + [switch] + $Dev, + + [Alias('do')] + [switch] + $DevOnly, + + [Alias('i')] + [switch] + $Install, + + [Alias('u')] + [switch] + $Uninstall, + + [Alias('v')] + [switch] + $Version, + + [Alias('h')] + [switch] + $Help, + + [Alias('ad')] + [switch] + $Adhoc +) + +# ensure if there's an error, we stop +$ErrorActionPreference = 'Stop' + + +# Import required modules +$root = Split-Path -Parent -Path $MyInvocation.MyCommand.Path +Import-Module "$($root)\Modules\FudgeTools.psm1" -Force -ErrorAction Stop + + +# output the version +$ver = 'v$version$' +Write-Success "Fudge $($ver)" + +# if we were only after the version, just return +if ($Version -or (@('v', 'version') -icontains $Action)) +{ + return +} + + +# if action is just to display Help, show it and return +if ($Help -or (@('h', 'help') -icontains $Action)) +{ + Write-Host "`nUsage: fudge " + Write-Host "`nWhere is one of:" + Write-Host " add, clean, delete, downgrade, help, install, list, new, pack," + Write-Host " prune, rebuild, reinstall, remove, renew, search, uninstall," + Write-Host " upgrade, version, which" + Write-Host "" + return +} + + +try +{ + # start timer + $timer = [DateTime]::UtcNow + + + # ensure we have a valid action + $packageActions = @('install', 'upgrade', 'uninstall', 'reinstall', 'list', 'rebuild', 'downgrade', 'add', 'remove') + $maintainActions = @('prune') + $packingActions = @('pack') + $miscActions = @('search', 'clean', 'which') + $newActions = @('new') + $alterActions = @('delete', 'renew') + + $actions = ($packageActions + $maintainActions + $packingActions + $miscActions + $newActions + $alterActions) + if ((Test-Empty $Action) -or $actions -inotcontains $Action) { + Write-Fail "Unrecognised action supplied '$($Action)', should be either: $($actions -join ', ')" + return + } + + + # actions that require chocolatey + $isChocoAction = (@('which', 'add', 'remove', 'delete') -inotcontains $Action) + if (!$isChocoAction -and ($Install -or $Uninstall)) { + $isChocoAction = $true + } + + + # if adhoc was supplied for an invalid action + if ($Adhoc -and @('install', 'uninstall', 'upgrade', 'downgrade', 'reinstall') -inotcontains $Action) { + Write-Fail "Adhoc supplied for invalid action: $($Action)" + return + } + + # if adhoc supplied with no package name, fail + if ($Adhoc -and [string]::IsNullOrWhiteSpace($Key)) { + Write-Fail "No package name supplied for adhoc $($Action)" + return + } + + + # if -devOnly is passed, set -dev to true + if ($DevOnly) { + $Dev = $true + } + + + # get the Fudgefile path, if adhoc is supplied set to empty + $FudgefilePath = Get-FudgefilePath $FudgefilePath -Adhoc:$Adhoc + + + # ensure that the Fudgefile exists (for certain actions), and deserialise it + if (($packageActions + $maintainActions + $packingActions + $alterActions) -icontains $Action) + { + $config = $null + + # if adhoc is supplied, we don't need to get the content + if (!$Adhoc) { + if (!(Test-Path $FudgefilePath)) { + Write-Fail "Path to Fudgefile does not exist: $($FudgefilePath)" + return + } + + $config = Get-FudgefileContent $FudgefilePath + } + + # if we have a custom source in the config and no CLI source, set the source + if ((Test-Empty $Source) -and ($null -ne $config) -and !(Test-Empty $config.source)) { + $Source = $config.source + } + } + + # ensure that the Fudgefile doesn't exist + elseif ($newActions -icontains $Action) + { + if (Test-Path $FudgefilePath) { + Write-Fail "Path to Fudgefile already exists: $($FudgefilePath)" + return + } + } + + + # if there are no packages to install or nuspecs to pack, just return + if ($null -ne $config) + { + # check nuspecs + if ($packingActions -icontains $Action) + { + if (Test-Empty $config.pack) { + Write-Notice "There are no nuspecs to $($Action)" + return + } + + if (![string]::IsNullOrWhiteSpace($Key) -and [string]::IsNullOrWhiteSpace($config.pack.$Key)) { + Write-Notice "Fudgefile does not contain a nuspec pack file for '$($Key)'" + return + } + } + + # check packages + elseif ($packageActions -icontains $Action) + { + if ((Test-Empty $config.packages) -and (!$Dev -or ($Dev -and (Test-Empty $config.devPackages)))) { + Write-Notice "There are no packages to $($Action)" + return + } + + if ($DevOnly -and (Test-Empty $config.devPackages)) { + Write-Notice "There are no devPackages to $($Action)" + return + } + } + } + + + # check to see if chocolatey is installed + if ($isChocoAction) { + $isChocoInstalled = Test-Chocolatey + } + + + # check if the console is elevated (only needs to be done for certain actions) + $isAdminAction = @('list', 'search', 'new', 'delete', 'renew', 'which', 'add', 'remove', 'pack') -inotcontains $Action + $actionNeedsAdmin = (@('delete', 'remove') -icontains $Action -and $Uninstall) -or (@('new', 'renew', 'add') -icontains $Action -and $Install) + + if (((!$isChocoInstalled -and $isChocoAction) -or $isAdminAction -or $actionNeedsAdmin) -and !(Test-AdminUser)) + { + Write-Notice 'Must be running with administrator privileges for Fudge to fully function' + return + } + + + # if chocolatey isn't installed, install it + if (!$isChocoInstalled -and $isChocoAction) { + Install-Chocolatey + } + + + # if we are using a global custom source, output it for info + if (!(Test-Empty $Source)) { + Write-Notice "Source: $($Source)" + } + + Write-Host ([string]::Empty) + + + # retrieve a local list of what's currently installed + if ($isChocoAction) { + $localList = Get-ChocolateyLocalList + } + + + # invoke chocolatey based on the action required + switch ($Action) + { + {($_ -ieq 'install') -or ($_ -ieq 'uninstall') -or ($_ -ieq 'upgrade') -or ($_ -ieq 'downgrade')} + { + Invoke-ChocolateyAction -Action $Action -Key $Key -Source $Source -Config $config -LocalList $localList ` + -Parameters $Parameters -Arguments $Arguments -Dev:$Dev -DevOnly:$DevOnly -Adhoc:$Adhoc + } + + {($_ -ieq 'reinstall')} + { + Invoke-ChocolateyAction -Action 'uninstall' -Key $Key -Source $Source -Config $config -LocalList $localList ` + -Parameters $Parameters -Arguments $Arguments -Dev:$Dev -DevOnly:$DevOnly -Adhoc:$Adhoc + + Invoke-ChocolateyAction -Action 'install' -Key $Key -Source $Source -Config $config -LocalList $localList ` + -Parameters $Parameters -Arguments $Arguments -Dev:$Dev -DevOnly:$DevOnly -Adhoc:$Adhoc + } + + {($_ -ieq 'pack')} + { + Invoke-ChocolateyAction -Action 'pack' -Key $Key -Config $config + } + + {($_ -ieq 'list')} + { + Invoke-FudgeLocalDetails -Config $config -Key $Key -LocalList $localList -Dev:$Dev -DevOnly:$DevOnly + } + + {($_ -ieq 'search')} + { + Invoke-Search -Key $Key -Limit $Limit -Source $Source -LocalList $localList + } + + {($_ -ieq 'new')} + { + New-Fudgefile -Path $FudgefilePath -Key $Key -LocalList $localList -Install:$Install -Dev:$Dev -DevOnly:$DevOnly + } + + {($_ -ieq 'renew')} + { + Restore-Fudgefile -Path $FudgefilePath -Key $Key -LocalList $localList -Install:$Install -Uninstall:$Uninstall -Dev:$Dev -DevOnly:$DevOnly + } + + {($_ -ieq 'delete')} + { + Remove-Fudgefile -Path $FudgefilePath -Uninstall:$Uninstall -Dev:$Dev -DevOnly:$DevOnly + } + + {($_ -ieq 'prune')} + { + Invoke-FudgePrune -Config $config -LocalList $localList -Dev:$Dev -DevOnly:$DevOnly + } + + {($_ -ieq 'clean')} + { + Invoke-FudgeClean -LocalList $localList + } + + {($_ -ieq 'add')} + { + Invoke-FudgeAdd -Path $FudgefilePath -Key $Key -Source $Source -Config $config -LocalList $localList ` + -Parameters $Parameters -Arguments $Arguments -Dev:$Dev -Install:$Install + } + + {($_ -ieq 'remove')} + { + Invoke-FudgeRemove -Path $FudgefilePath -Key $Key -Config $config -LocalList $localList ` + -Parameters $Parameters -Arguments $Arguments -Dev:$Dev -Uninstall:$Uninstall + } + + {($_ -ieq 'which')} + { + Invoke-FudgeWhich -Key $Key + } + + {($_ -ieq 'rebuild')} + { + Invoke-FudgeClean -LocalList $localList + Invoke-ChocolateyAction -Action 'install' -Key $Key -Source $Source -Config $config -Dev:$Dev -DevOnly:$DevOnly + } + + default + { + Write-Fail "Action not recognised: $($_)" + } + } +} +finally +{ + # output duration, and cleanup + Write-Details "`nDuration: $(([DateTime]::UtcNow - $timer).ToString())" + Remove-Module -Name 'FudgeTools' -ErrorAction SilentlyContinue | Out-Null +} \ No newline at end of file diff --git a/.ci/FudgeCI.ps1 b/.ci/FudgeCI.ps1 new file mode 100644 index 0000000000..8f834ce475 --- /dev/null +++ b/.ci/FudgeCI.ps1 @@ -0,0 +1,183 @@ +if (!($env:FudgeCI)) { + if (Test-Path 'assets/fudge/FudgeCI.ps1') { + $env:FudgeCI = 'assets/fudge/' + } + elseif (Test-Path '.ci/FudgeCI.ps1') { + $env:FudgeCI = '.ci/' + } +} + +. $env:FudgeCI/PrepareAVVM.ps1 + +Set-StrictMode -Version latest + +function Initialize-MinGW { + # TODO: Handle versions other than 8.1.0 + Move-Item C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64 C:\MinGW81-x64 +} + +function Initialize-AppVeyorPreinstalledProduct { + param( + [array] + $packages + ) + + foreach ($pkg in $packages) { + $name = $pkg.Name + + $product = $pkg.AppVeyor_ID + if ($product -eq $true) { + Write-Information "No version choices available for AppVeyor $name" + continue + } + + $version = $pkg.Version + + $version_parts = ($version.Split('.')) + + if (!($env:AppVeyor)) { + Write-Verbose "AppVeyor not set; skipping $product $version_parts" + continue + } + + if ($product -eq 'jdk') { + # 8 -> 1.8.0 + $version = "1." + $version_parts[0] + ".0" + } + elseif ($product -eq 'MinGW') { + Initialize-MinGW + } + elseif ($product -eq 'miniconda') { + # TODO improve translation of real miniconda versions + # into AppVeyor versions which are the python version + if ($version -eq '4.5.12') { + $version = '3.7' + } + + if ($version[0] -eq '2') { + Initialize-Miniconda27 + } + } + + # Allow the installed version of python to be over + if ($product -eq 'python') { + if ($env:PYTHON_VERSION) { + $version = $env:PYTHON_VERSION + } + } + + Add-Product $product $version $env:PLATFORM + if (Test-Path "C:\avvm\$product\$version\$env:PLATFORM") { + Install-Product $product $version $env:PLATFORM + } + elseif (Test-Path "C:\avvm\$product\$version") { + if ($env:PLATFORM -eq 'x86') { + $platform = 'x64' + } + else { + $platform = 'x86' + } + Install-Product $product $version $platform + } + } +} + +function Initialize-AppVeyorFakeChocoPackage { + param( + [array] + $packages + ) + + New-Item -ItemType Directory -Force ($env:FudgeCI + '\\nuspecs\\') > $null + + Remove-Item "$env:FudgeCI/nuspecs/*.nupkg" -Force > $null + + foreach ($pkg in $packages) { + . $env:FudgeCI/FudgeGenerateFake.ps1 + + GenerateFakeNuspec $pkg.Name $pkg.Version + + $name = $pkg.Name + + $pkg.Source = "$env:FudgeCI/nuspecs/" + + $filename = "$env:FudgeCI/nuspecs/$name.nuspec" + + Invoke-Chocolatey -Action pack -Package $pkg.Name -Version $filename + } +} + +function Get-Preinstalled { + try { + if ($config) { } + } + catch { + $config = Get-Content Fudgefile | + ConvertFrom-Json + } + + $appveyor_preinstalled = New-Object System.Collections.ArrayList + + foreach ($pkg in $config.packages) { + try { + if ($pkg.AppVeyor_ID) { + $appveyor_preinstalled.Add($pkg) > $null + } + } + catch { + continue + } + } + + $appveyor_preinstalled +} + +function Initialize-AppVeyorVM { + $appveyor_preinstalled = Get-Preinstalled + + if (!($env:AppVeyor)) { + Write-Notice "Not running on AppVeyor; skipping" + return + } + + Initialize-AppVeyorProductVersion + + Initialize-AppVeyorPreinstalledProduct $appveyor_preinstalled +} + +function Invoke-FudgeAppVeyor { + $appveyor_preinstalled = Get-Preinstalled + + if (!($env:AppVeyor)) { + Write-Notice "Not running on AppVeyor; skipping" + return + } + + Initialize-AppVeyorFakeChocoPackage $appveyor_preinstalled +} + +function Repair-Config { + foreach ($pkg in $config.packages) { + if (!($pkg.Source)) { + $pkg.Source = $config.Source + } + } +} + +function Invoke-FudgeCI { + if (!($env:CI)) { + Fix-Config + + Write-Notice "Not running on CI; skipping" + return + } + + Invoke-FudgeAppVeyor + + Repair-Config +} + +$old_EAP = $ErrorActionPreference +$ErrorActionPreference = 'SilentlyContinue'; +Export-ModuleMember -Function Prepare-AppVeyor-AVVM, Invoke-FudgeCI +$ErrorActionPreference = $old_EAP; diff --git a/.ci/FudgeGenerateFake.ps1 b/.ci/FudgeGenerateFake.ps1 new file mode 100644 index 0000000000..ac85e407f2 --- /dev/null +++ b/.ci/FudgeGenerateFake.ps1 @@ -0,0 +1,42 @@ +Set-StrictMode -Version latest + +$template = @' + + + + {name} + {version} + {name} {version} + AppVeyor + Fake generated {name} package to fulfil dependencies. + + + + + +'@ + +function GenerateFakeNuspec { + param( + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] + $name, + + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] + $version + ) + + $content = $template -replace '{name}', $name + $content = $content -replace '{version}', $version + + $nuspec = ($env:FudgeCI + '\nuspecs\' + $name + '.nuspec') + + Set-Content $nuspec $content + + Write-Output "Created $nuspec" +} + +Export-ModuleMember -Function GenerateFakeNuspec diff --git a/.ci/FudgePostInstall.ps1 b/.ci/FudgePostInstall.ps1 new file mode 100644 index 0000000000..c8df753663 --- /dev/null +++ b/.ci/FudgePostInstall.ps1 @@ -0,0 +1,52 @@ +. $env:ChocolateyInstall\helpers\functions\Write-FunctionCallLogMessage.ps1 +. $env:ChocolateyInstall\helpers\functions\Get-EnvironmentVariable.ps1 +. $env:ChocolateyInstall\helpers\functions\Get-EnvironmentVariableNames.ps1 +. $env:ChocolateyInstall\helpers\functions\Start-ChocolateyProcessAsAdmin.ps1 +. $env:ChocolateyInstall\helpers\functions\Set-EnvironmentVariable.ps1 +. $env:ChocolateyInstall\helpers\functions\Set-PowerShellExitCode.ps1 +. $env:ChocolateyInstall\helpers\functions\Update-SessionEnvironment.ps1 +. $env:ChocolateyInstall\helpers\functions\Write-FunctionCallLogMessage.ps1 +. $env:ChocolateyInstall\helpers\functions\Install-ChocolateyPath.ps1 + +Set-StrictMode -Version latest + +$deps_base = $env:FudgeCI + +function Invoke-PostInstall { + choco list --local-only + + Update-SessionEnvironment + + Write-Host "PATH = $env:PATH" + + foreach ($pkg in $config.Packages) { + $name = $pkg.Name + + $glob = "$deps_base/deps.$name.ps1" + + if (Test-Path $glob) { + Write-Host "Running post-install for $name" + + . $glob + Complete-Install + } + } + + Update-SessionEnvironment + + Write-Host "PATH = $env:PATH" + + foreach ($pkg in $config.Packages) { + $name = $pkg.Name + + $glob = "$deps_base/deps.$name-packages.ps1" + if (Test-Path $glob) { + Write-Host "Running $name package installation" + + . $glob + Invoke-ExtraInstallation + } + } +} + +Export-ModuleMember -Function Invoke-PostInstall diff --git a/.ci/Modules/FudgeTools.psm1 b/.ci/Modules/FudgeTools.psm1 new file mode 100644 index 0000000000..4c51f2f95d --- /dev/null +++ b/.ci/Modules/FudgeTools.psm1 @@ -0,0 +1,1896 @@ + +function Write-Success +{ + param ( + [Parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + [string] + $Message, + + [switch] + $NoNewLine + ) + + Write-Host $Message -NoNewline:$NoNewLine -ForegroundColor Green +} + +function Write-Information +{ + param ( + [Parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + [string] + $Message, + + [switch] + $NoNewLine + ) + + Write-Host $Message -NoNewline:$NoNewLine -ForegroundColor Magenta +} + + +function Write-Details +{ + param ( + [Parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + [string] + $Message, + + [switch] + $NoNewLine + ) + + Write-Host $Message -NoNewline:$NoNewLine -ForegroundColor Cyan +} + + +function Write-Notice +{ + param ( + [Parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + [string] + $Message, + + [switch] + $NoNewLine + ) + + Write-Host $Message -NoNewline:$NoNewLine -ForegroundColor Yellow +} + + +function Write-Fail +{ + param ( + [Parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + [string] + $Message, + + [switch] + $NoNewLine + ) + + Write-Host $Message -NoNewline:$NoNewLine -ForegroundColor Red +} + + +# compares two versions, and returns a value specifying if they're equal or different +# -1: installed version is behind +# 0: installed version is latest +# 1: installed version is ahead +function Compare-Versions +{ + param ( + [Parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + [string] + $Installed, + + [Parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + [string] + $NewVersion + ) + + if (Test-VersionPassedIsLatest $NewVersion) { + return -1 + } + + $i_parts = $Installed -split '\.' + $o_parts = $NewVersion -split '\.' + + $count = $i_parts.Length + if ($o_parts.Length -gt $count) { + $count = $o_parts.Length + } + + for ($i = 0; $i -lt $count; $i++) + { + if ($i_parts[$i] -eq $o_parts[$i]) { + continue + } + + if ($i_parts[$i] -lt $o_parts[$i]) { + return -1 + } + else { + return 1 + } + } + + return 0 +} + + +# takes a package about to be installed, and checks if the version needs to be installed, +# or just a downgrade/upgrade of what's already installed +function Get-InstallAction +{ + param ( + [string] + $Package, + + [string] + $Version, + + $LocalList + ) + + # if it's not installed, it needs installing + $current = $LocalList[$Package] + if (Test-Empty $current) { + return 'install' + } + + # compare the version, and determine if we need to upgrade/downgrade + $action = Compare-Versions -Installed $current -NewVersion $Version + + switch ($action) + { + 1 { return 'downgrade' } + -1 { return 'upgrade' } + default { return 'install' } + } +} + + +# return the package name from a supplied key +function Get-PackageFromKey +{ + param ( + [string] + $Key + ) + + return ($Key -isplit '@')[0].Trim() +} + + +# return the version from a supplied key +function Get-VersionFromKey +{ + param ( + [string] + $Key + ) + + $parts = ($Key -isplit '@') + if ($parts.Length -le 1) { + return 'latest' + } + + return $parts[1].Trim() +} + + +# returns the levenshtein distance between two strings +function Get-Levenshtein +{ + param ( + [string] + $Value1, + + [string] + $Value2 + ) + + $len1 = $Value1.Length + $len2 = $Value2.Length + + if ($len1 -eq 0) { return $len2 } + if ($len2 -eq 0) { return $len1 } + + $Value1 = $Value1.ToLowerInvariant() + $Value2 = $Value2.ToLowerInvariant() + + $dist = New-Object -Type 'int[,]' -Arg ($len1 + 1), ($len2 + 1) + + 0..$len1 | ForEach-Object { $dist[$_, 0] = $_ } + 0..$len2 | ForEach-Object { $dist[0, $_] = $_ } + + $cost = 0 + + for ($i = 1; $i -le $len1; $i++) + { + for ($j = 1; $j -le $len2; $j++) + { + $cost = 1 + if ($Value2[$j - 1] -ceq $Value1[$i - 1]) + { + $cost = 0 + } + + $tempmin = [System.Math]::Min(([int]$dist[($i - 1), $j] + 1), ([int]$dist[$i, ($j - 1)] + 1)) + $dist[$i, $j] = [System.Math]::Min($tempmin, ([int]$dist[($i - 1), ($j - 1)] + $cost)) + } + } + + # the actual distance is stored in the bottom right cell + return $dist[$len1, $len2]; +} + + +# safeguards a string, by return empty or a default if empty +function Format-SafeguardString +{ + param ( + [string] + $Value, + + [string] + $Default = $null + ) + + if (!(Test-Empty $Value)) { + return $Value + } + + $Value = [string]::Empty + if (!(Test-Empty $Default)) { + $Value = $Default + } + + return $Value +} + + +# checks to see if a passed version is to use the latest version +function Test-VersionPassedIsLatest +{ + param ( + [string] + $Version + ) + + return ((Test-Empty $Version) -or ($Version.Trim() -ieq 'latest')) +} + + +# checks to see if a passed path is a valid nuspec (path, xml, and content) +function Test-Nuspec +{ + param ( + [string] + $Path + ) + + if (!(Test-NuspecPath $Path)) + { + Write-Fail "Path to nuspec file doesn't exist or is invalid: $($Path)" + return $false + } + + if (!(Test-XmlContent $Path)) + { + Write-Fail "Nuspec file fails to parse as a valid XML document: $($Path)" + return $false + } + + $nuspecData = Get-XmlContent $Path + + if (!(Test-NuspecContent $nuspecData)) + { + Write-Fail "Nuspec file is missing the package/metadata XML sections: $($Path)" + return $false + } + + return $true +} + + +# checks to see if a passed path is a valid nuspec file path +function Test-NuspecPath +{ + param ( + [string] + $Path + ) + + # ensure a path was passed + if ([string]::IsNullOrWhiteSpace($Path)) + { + return $false + } + + # ensure path is exists, or is not just a directory path + if (!(Test-Path $Path) -or (Test-PathDirectory $Path)) + { + return $false + } + + return $true +} + + +# checks to see if the file at passed path is a valid XML file +function Test-XmlContent +{ + param ( + [string] + $Path + ) + + # fail if the path doesn't exist + if ([string]::IsNullOrWhiteSpace($Path) -or !(Test-Path $Path)) + { + return $false + } + + # ensure the content parses as xml + try + { + [xml](Get-Content $Path) | Out-Null + return $true + } + catch [exception] + { + return $false + } +} + + +# checks to see if the passed XML content is a valid nuspec file +function Test-NuspecContent +{ + param ( + [xml] + $Content + ) + + return ($Content -ne $null -and $Content.package -ne $null -and $Content.package.metadata -ne $null) +} + + +# returns the XML content of the file at the passed path +function Get-XmlContent +{ + param ( + [string] + $Path + ) + + if (!(Test-XmlContent $Path)) + { + return $null + } + + return [xml](Get-Content $Path) +} + + +# checks to see if a passed path is a directory +function Test-PathDirectory +{ + param ( + [string] + $Path + ) + + if ([string]::IsNullOrWhiteSpace($Path) -or !(Test-Path $Path)) + { + return $false + } + + return ((Get-Item $Path) -is [System.IO.DirectoryInfo]) +} + + +# returns a path to a fidgefile based on a passed path +function Get-FudgefilePath +{ + param ( + [string] + $Path, + + [switch] + $Adhoc + ) + + if ($Adhoc) + { + return [string]::Empty + } + + $rootpath = './Fudgefile' + if (![string]::IsNullOrWhiteSpace($Path)) + { + if ((Test-Path $Path) -and (Test-PathDirectory $Path)) + { + $rootpath = Join-Path $Path 'Fudgefile' + } + else + { + $rootpath = $Path + } + } + + return $rootpath +} + + +# returns the content of a passed fudgefile path +function Get-FudgefileContent +{ + param ( + [Parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + [string] + $Path + ) + + if (!(Test-Path $Path)) { + throw "Path to Fudgefile does not exist: $($Path)" + } + + try { + $config = Get-Content -Path $Path -Raw | ConvertFrom-Json + } + catch { + throw "Failed to parse the Fudgefile at: $($Path)`n$($_.Exception)" + } + + return $config +} + + +# removes an existing fudgefile, and if passed attempting to uninstall the packages +function Remove-Fudgefile +{ + param ( + [Parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + [string] + $Path, + + $LocalList, + + [switch] + $Uninstall, + + [switch] + $Dev, + + [switch] + $DevOnly + ) + + # ensure the path actually exists + if (!(Test-Path $Path)) + { + Write-Fail "Path to Fudgefile does not exist: $($Path)" + return + } + + # uninstall packages first, if requested + if ($Uninstall) + { + $config = Get-FudgefileContent $Path + Invoke-ChocolateyAction -Action 'uninstall' -Key $null -Config $config -LocalList $LocalList -Dev:$Dev -DevOnly:$DevOnly + } + + # remove the fudgefile + Write-Information "> Deleting Fudgefile" -NoNewLine + Remove-Item -Path $Path -Force -Confirm:$false | Out-Null + Write-Success " > deleted" + Write-Details " > $($Path)" +} + + +# restores the packages of an existing fudgefile (either empty, from nuspec, or from local) +function Restore-Fudgefile +{ + param ( + [Parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + [string] + $Path, + + [string] + $Key, + + $LocalList, + + [switch] + $Install, + + [switch] + $Uninstall, + + [switch] + $Dev, + + [switch] + $DevOnly + ) + + # retrieve the content of the current fudgefile + $content = Get-FudgefileContent $Path + + # check to see if we're restoring this file using local packages + if ($Key -ieq 'local') + { + if (Test-Empty $LocalList) + { + Write-Fail "No packages installed locally to renew Fudgefile" + return + } + + Write-Information "> Renewing Fudgefile using $($LocalList.Count) local packages" -NoNewLine + } + + # check if there are any nuspecs in the pack section + elseif ($Key -ieq 'nuspec') + { + if (Test-Empty $content.pack) + { + Write-Fail "No nuspecs in pack section to renew Fudgefile" + return + } + + Write-Information "> Renewing Fudgefile using nuspecs" -NoNewLine + } + + # else if it's not empty, we don't know what they mean + elseif (!(Test-Empty $Key)) + { + Write-Fail "Renew command not recognised: $($Key). Should be blank, or either local/nuspec" + return + } + + # else we're renewing to an empty file + else + { + Write-Information "> Renewing Fudgefile" -NoNewLine + } + + # first, uninstall the packages, if requested + if ($Uninstall) + { + Invoke-ChocolateyAction -Action 'uninstall' -Key $null -Config $content -LocalList $LocalList -Dev:$Dev -DevOnly:$DevOnly + } + + # remove all current packages + if (!$DevOnly) + { + $content.packages = @() + } + + if ($Dev) + { + $content.devPackages = @() + } + + # insert packages from any nuspecs in the pack section + if ($Key -ieq 'nuspec') + { + $nuspecs = $content.pack.psobject.properties.name + $nuspecs | ForEach-Object { + $content = Add-PackagesFromNuspec -Content $content -Path $content.pack.$_ -Dev:$Dev -DevOnly:$DevOnly + } + } + + # insert any data from the local packages + if ($Key -ieq 'local') + { + $content = Add-PackagesFromLocal -Content $content -LocalList $LocalList -DevOnly:$DevOnly + } + + # save contents as json + $content | ConvertTo-Json | Out-File -FilePath $Path -Encoding utf8 -Force + Write-Success " > renewed" + Write-Details " > $($Path)" + + # now install the packages + if ($Install) + { + Invoke-ChocolateyAction -Action 'install' -Key $null -Config $content -LocalList $LocalList -Dev:$Dev -DevOnly:$DevOnly + } +} + + +# create a new empty fudgefile, or a new one from a nuspec file +function New-Fudgefile +{ + param ( + [Parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + [string] + $Path, + + [string] + $Key, + + $LocalList, + + [switch] + $Install, + + [switch] + $Dev, + + [switch] + $DevOnly + ) + + # check to see if we're making this file using local packages + if ($Key -ieq 'local') + { + if (Test-Empty $LocalList) + { + Write-Fail "No packages installed locally to create new Fudgefile" + return + } + + Write-Information "> Creating new Fudgefile using $($LocalList.Count) local packages" -NoNewLine + } + + # if the key is passed, ensure it's a valid nuspec file + elseif (!(Test-Empty $Key)) + { + if (!(Test-Nuspec $Key)) + { + return + } + + $nuspecName = Split-Path -Leaf -Path $Key + Write-Information "> Creating new Fudgefile using $($nuspecName)" -NoNewLine + } + + # else it's just an empty file + else + { + Write-Information "> Creating new Fudgefile" -NoNewLine + } + + # setup the empty fudgefile + $content = @{ + 'scripts' = @{ + 'pre' = @{ 'install'= ''; 'uninstall'= ''; 'upgrade'= ''; 'downgrade' = ''; 'pack'= ''; }; + 'post' = @{ 'install'= ''; 'uninstall'= ''; 'upgrade'= ''; 'downgrade' = ''; 'pack'= ''; }; + }; + 'packages' = @(); + 'devPackages' = @(); + 'pack' = @{}; + } + + # insert any data from the nuspec + if (!(Test-Empty $nuspecName)) + { + $content = Add-PackagesFromNuspec -Content $content -Path $Key -Dev:$Dev -DevOnly:$DevOnly + $name = [System.IO.Path]::GetFileNameWithoutExtension($nuspecName) + $content.pack[$name] = $Key + } + + # insert any data from the local packages + if ($Key -ieq 'local') + { + $content = Add-PackagesFromLocal -Content $content -LocalList $LocalList -DevOnly:$DevOnly + } + + # save contents as json + $content | ConvertTo-Json | Out-File -FilePath $Path -Encoding utf8 -Force + Write-Success " > created" + Write-Details " > $($Path)" + + # now install the packages + if ($Install) + { + $config = Get-FudgefileContent $Path + Invoke-ChocolateyAction -Action 'install' -Key $null -Config $config -LocalList $LocalList -Dev:$Dev -DevOnly:$DevOnly + } +} + + +# adds packages to a fudgefile from local packages +function Add-PackagesFromLocal +{ + param ( + [Parameter(Mandatory=$true)] + [ValidateNotNull()] + $Content, + + [Parameter(Mandatory=$true)] + [ValidateNotNull()] + $LocalList, + + [switch] + $DevOnly + ) + + $LocalList.Keys | ForEach-Object { + $package = @{ + 'name' = $_; + 'version' = $LocalList[$_]; + } + + if ($DevOnly) + { + $Content.devPackages += $package + } + else + { + $Content.packages += $package + } + } + + return $Content +} + + +# adds packages to a fudgefile from a nuspec file +function Add-PackagesFromNuspec +{ + param ( + [Parameter(Mandatory=$true)] + [ValidateNotNull()] + $Content, + + [Parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $Path, + + [switch] + $Dev, + + [switch] + $DevOnly + ) + + # get the content from the nuspec + $data = Get-XmlContent -Path $Path + $metadata = $data.package.metadata + + # if we have any dependencies, add them as packages + if (!$DevOnly -and $metadata.dependencies -ne $null) + { + $metadata.dependencies.dependency | ForEach-Object { + $version = Format-SafeguardString $_.version 'latest' + $Content.packages += @{ + 'name' = $_.id; + 'version' = $version; + } + } + } + + # if we have any devDependencies, add them as devPackages + if ($Dev -and $metadata.devDependencies -ne $null) + { + $metadata.devDependencies.dependency | ForEach-Object { + $version = Format-SafeguardString $_.version 'latest' + $Content.devPackages += @{ + 'name' = $_.id; + 'version' = $version; + } + } + } + + return $Content +} + + +# checks to see if the user has administrator privileges +function Test-AdminUser +{ + if ($PSVersionTable.Platform -ieq 'Unix') + { + Write-Notice 'Windows Admin check bypassed on Unix' + return $true + } + + try + { + $principal = New-Object System.Security.Principal.WindowsPrincipal([System.Security.Principal.WindowsIdentity]::GetCurrent()) + + if ($principal -eq $null) + { + return $false + } + + return $principal.IsInRole([System.Security.Principal.WindowsBuiltInRole]::Administrator) + } + catch [exception] + { + Write-Fail 'Error checking user administrator privileges' + Write-Fail $_.Exception.Message + return $false + } +} + + +# checks to see if the passed value is empty +function Test-Empty +{ + param ( + $Value + ) + + if ($Value -eq $null) + { + return $true + } + + if ($Value.GetType().Name -ieq 'string') + { + return [string]::IsNullOrWhiteSpace($Value) + } + + $type = $Value.GetType().BaseType.Name.ToLowerInvariant() + switch ($type) + { + 'valuetype' + { + return $false + } + + 'array' + { + return (($Value | Measure-Object).Count -eq 0 -or $Value.Count -eq 0) + } + } + + return ([string]::IsNullOrWhiteSpace($Value) -or ($Value | Measure-Object).Count -eq 0 -or $Value.Count -eq 0) +} + + +# check to see if chocolatey is installed on the current machine +function Test-Chocolatey +{ + try { + $output = Invoke-Expression -Command 'choco -v' + Write-Details "Chocolatey v$($output)" + return $true + } + catch { + return $false + } +} + + +# installs chocolatey +function Install-Chocolatey +{ + Write-Notice "Installing Chocolatey" + + $policies = @('Unrestricted', 'ByPass', 'AllSigned') + if ($policies -inotcontains (Get-ExecutionPolicy)) { + Set-ExecutionPolicy Bypass -Force + } + + Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) | Out-Null + Write-Success "Chocolatey installed`n" +} + + +# invoke scripts for pre/post actions +function Invoke-Script +{ + param ( + [Parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + [string] + $Action, + + [string] + $Stage, + + $Scripts + ) + + # if there is no stage, return + if (Test-Empty $Stage) { + return + } + + # if there are no scripts, return + if ((Test-Empty $Scripts) -or (Test-Empty $Scripts.$Stage)) { + return + } + + $script = $Scripts.$Stage.$Action + if (Test-Empty $script) { + return + } + + # run the script + Invoke-Expression -Command $script +} + + +# cycle through the passed packages, actioning upon them +function Start-ActionPackages +{ + param ( + [Parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + [string] + $Action, + + [string] + $Key, + + [string] + $Source, + + [array] + $Packages, + + [string] + $Arguments, + + $LocalList + ) + + # if there are no packages to deal with, return + if (Test-Empty $Packages) { + return + } + + # have we installed anything + $installed = $false + $haveKey = (![string]::IsNullOrWhiteSpace($Key)) + + # loop through each of the packages, and call chocolatey on them + foreach ($pkg in $Packages) + { + # skip if we forcing action on a specific package + if ($haveKey -and ($pkg.name -ine $Key)) { + continue + } + + # use a package specific custom source + if (![string]::IsNullOrWhiteSpace($pkg.source)) { + $Source = $pkg.source + } + + # force a specific action on the package + $_action = $Action + if (!(Test-Empty $pkg.action)) { + $_action = $pkg.action + } + + $installed = $true + + Invoke-Chocolatey -Action $_action -Package $pkg.name -Version $pkg.version ` + -Source $Source -Parameters $pkg.params -Arguments "$($pkg.args) $($Arguments)".Trim() -LocalList $LocalList + } + + # if we didn't install anything, and we have a key - say it isn't present in file + if (!$installed -and $haveKey) { + Write-Notice "Package not found in Fudgefile: $($Key)" + } +} + + +# cycle through the passed nuspecs for packing via nuget +function Start-ActionPack +{ + param ( + [Parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + [string] + $Action, + + [string] + $Key, + + $Nuspecs + ) + + # if there are no nuspecs to deal with, return + if (Test-Empty $Nuspecs) { + return + } + + # have we packed anything + $packed = $false + $haveKey = (![string]::IsNullOrWhiteSpace($Key)) + + # loop through each of the nuspecs, and call chocolatey on them for packing + foreach ($pkg in $Nuspecs.psobject.properties.name) + { + if ($haveKey -and ($pkg -ine $Key)) { + continue + } + + $packed = $true + Invoke-Chocolatey -Action $Action -Package $pkg -Version ($Nuspecs.$pkg) + } + + # if we didn't pavk anything, and we have a key - say it isn't present in file + if (!$packed -and $haveKey) { + Write-Notice "Nuspec not found in Fudgefile: $($Key)" + } +} + + +# invokes a chocolatey action, which also runs the pre/post scripts +function Invoke-ChocolateyAction +{ + param ( + [Parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + [string] + $Action, + + [string] + $Key, + + [string] + $Source, + + $Config, + + $LocalList, + + [string] + $Parameters, + + [string] + $Arguments, + + [switch] + $Dev, + + [switch] + $DevOnly, + + [switch] + $Adhoc + ) + + # ensure the config object exists, unless adhoc is supplied + if (!$Adhoc -and $null -eq $Config) { + throw "Invalid Fudge configuration supplied" + } + + # if we're running as adhoc, just call chocolatey directly + if ($Adhoc) + { + $pkg = Get-PackageFromKey -Key $Key + $vsn = Get-VersionFromKey -Key $Key + Invoke-Chocolatey -Action $Action -Package $pkg -Source $Source -Version $vsn -LocalList $LocalList -Parameters $Parameters -Arguments $Arguments + } + else + { + # invoke pre-script for current action + Invoke-Script -Action $Action -Stage 'pre' -Scripts $Config.scripts + + # invoke chocolatey based on the action + switch ($Action.ToLowerInvariant()) { + 'pack' { + # run pack on supplied nuspecs + Start-ActionPack -Action $Action -Key $Key -Nuspecs $Config.pack + } + + default { + # install normal packages, unless flagged as dev only + if (!$DevOnly) { + Start-ActionPackages -Action $Action -Key $Key -Packages $Config.packages -Source $Source -LocalList $LocalList -Arguments $Arguments + } + + # install dev packages + if ($Dev) { + Start-ActionPackages -Action $Action -Key $Key -Packages $Config.devPackages -Source $Source -LocalList $LocalList -Arguments $Arguments + } + } + } + + # invoke post-script for current action + Invoke-Script -Action $Action -Stage 'post' -Scripts $Config.scripts + } +} + + +# add the core chocolatey and fudge packages to a package map +function Add-CoreChocoPackages +{ + param ( + [Parameter(Mandatory=$true)] + [ValidateNotNull()] + $Packages + ) + + $Packages['chocolatey'] = 'latest' + $Packages['chocolatey-core.extension'] = 'latest' + $Packages['fudge'] = 'latest' + + return $Packages +} + + +# pruning the current machine's packages with what's in the fudgefile +function Invoke-FudgePrune +{ + param ( + $Config, + + $LocalList, + + [switch] + $Dev, + + [switch] + $DevOnly + ) + + # package map for what to leave installed + $packages = @{} + $pruned = $false + + if (!$DevOnly) + { + $Config.packages | ForEach-Object { $packages[$_.name] = $_.version } + } + + if ($Dev) + { + $Config.devPackages | ForEach-Object { $packages[$_.name] = $_.version } + } + + # add core chocolatey packages (and fudge) + $packages = Add-CoreChocoPackages $packages + + # loop through each local package, and remove if not in fudgefile + $LocalList.Keys | ForEach-Object { + if (!$packages.ContainsKey($_)) + { + $pruned = $true + Invoke-Chocolatey -Action 'uninstall' -Package $_ + } + } + + if(!$pruned) + { + Write-Notice 'Nothing to prune' + } +} + + +# cleaning a machine of all packages currently installed +function Invoke-FudgeClean +{ + param ( + $LocalList + ) + + # package map for what to leave installed + $packages = @{} + $cleaned = $false + + # add core chocolatey packages (and fudge) + $packages = Add-CoreChocoPackages $packages + + # loop through each local package, and remove it + $LocalList.Keys | ForEach-Object { + if (!$packages.ContainsKey($_)) + { + $cleaned = $true + Invoke-Chocolatey -Action 'uninstall' -Package $_ + } + } + + if(!$cleaned) + { + Write-Notice 'Nothing to clean' + } +} + + +# add a package to a fudgefile +function Invoke-FudgeAdd +{ + param ( + [Parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + [string] + $Path, + + [Parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + [string] + $Key, + + [string] + $Source, + + $Config, + + $LocalList, + + [string] + $Parameters, + + [string] + $Arguments, + + [switch] + $Dev, + + [switch] + $Install + ) + + # get name and version from key + $KeyName = Get-PackageFromKey -Key $Key + $KeyVer = Get-VersionFromKey -Key $Key + + # ensure package isn't already present + $pkg_count = ($config.packages | Where-Object { $_.name -ieq $KeyName } | Measure-Object).Count + $dev_count = ($config.devPackages | Where-Object { $_.name -ieq $KeyName } | Measure-Object).Count + + if ($pkg_count -ne 0 -or $dev_count -ne 0) + { + Write-Notice "The package '$($KeyName)' already exists in the Fudgefile" + return + } + + # attempt to install if specified + if ($Install) + { + Invoke-Chocolatey -Action 'install' -Package $KeyName -Source $Source -Version $KeyVer -LocalList $LocalList -Parameters $Parameters -Arguments $Arguments + } + + Write-Information "> Adding $($KeyName)@$($KeyVer) to Fudgefile" -NoNewLine + + # create package json object + $obj = @{ 'name' = $KeyName; } + + if (!(Test-Empty $KeyVer)) + { + $obj['version'] = $KeyVer + } + + if (!(Test-Empty $Source) -and ($Source -ine $Config.source)) + { + $obj['source'] = $Source + } + + # add to config + if ($Dev) + { + [array]$Config.devPackages += $obj + } + else + { + [array]$Config.packages += $obj + } + + # save new config back + $Config | ConvertTo-Json | Out-File -FilePath $Path -Encoding utf8 -Force + Write-Success " > added" + Write-Details " > $($Path)" +} + + +# removes a package from a fudgefile +function Invoke-FudgeRemove +{ + param ( + [Parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + [string] + $Path, + + [Parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + [string] + $Key, + + $Config, + + $LocalList, + + [string] + $Parameters, + + [string] + $Arguments, + + [switch] + $Dev, + + [switch] + $Uninstall + ) + + # get name and version from key + $KeyName = Get-PackageFromKey -Key $Key + + # ensure package isn't already removed + $pkg_count = ($config.packages | Where-Object { $_.name -ieq $KeyName } | Measure-Object).Count + $dev_count = ($config.devPackages | Where-Object { $_.name -ieq $KeyName } | Measure-Object).Count + + if ($pkg_count -eq 0 -and $dev_count -eq 0) + { + Write-Notice "The package '$($KeyName)' is not present in the Fudgefile" + return + } + + # attempt to uninstall if specified + if ($Uninstall) + { + Invoke-Chocolatey -Action 'uninstall' -Package $KeyName -LocalList $LocalList -Parameters $Parameters -Arguments $Arguments + } + + Write-Information "> Removing $($KeyName) from Fudgefile" -NoNewLine + + # remove to config + if ($Dev) + { + $Config.devPackages = ($Config.devPackages | Where-Object { $_.name -ine $KeyName }) + } + else + { + $Config.packages = ($Config.packages | Where-Object { $_.name -ine $KeyName }) + } + + # save new config back + $Config | ConvertTo-Json | Out-File -FilePath $Path -Encoding utf8 -Force + Write-Success " > removed" + Write-Details " > $($Path)" +} + + +# returns the path for where a command is located +function Invoke-FudgeWhich +{ + param ( + [string] + $Key + ) + + if (Test-Empty $Key) + { + Write-Notice 'No command passed to find' + } + else + { + $path = (Get-Command -Name $Key -ErrorAction SilentlyContinue | Select-Object -ExpandProperty Definition) + if (!(Test-Empty $path)) + { + Write-Host "> $($path)" + } + else + { + Write-Notice "Command not found: $($Key)" + } + } +} + + +# returns a source argument for chocolatey +function Format-ChocolateySource +{ + param ( + [string] + $Source + ) + + if (Test-Empty $Source) { + return [string]::Empty + } + + return "-s '$($Source)'" +} + + +# returns a params argument for chocolatey +function Format-ChocolateyParams +{ + param ( + [string] + $Parameters + ) + + if (Test-Empty $Parameters) { + return [string]::Empty + } + + return "--params='$($Parameters)'" +} + + +# returns a version argument for chocolatey +function Format-ChocolateyVersion +{ + param ( + [string] + $Version + ) + + # if the version is latest, return empty + if (Test-VersionPassedIsLatest -Version $Version) { + return [string]::Empty + } + + # return the specific version to install + return "--version $($Version)" +} + + +# returns the list of search returns from chocolatey +function Get-ChocolateySearchList +{ + param ( + [string] + $Key, + + [string] + $Source + ) + + # set the source if we have one + $Source = Format-ChocolateySource $Source + + $list = (choco search $Key $Source) + if (!$?) { + Write-Fail "$($list)" + throw 'Failed to retrieve search results from Chocolatey' + } + + return (Format-ChocolateyList -List $list) +} + + +# returns a list of packages installed localled +function Get-ChocolateyLocalList +{ + $list = (choco list -lo) + if (!$?) { + Write-Fail "$($list)" + throw 'Failed to retrieve local list of installed packages' + } + + return (Format-ChocolateyList -List $list) +} + + +# formats list/search results from chocolatey into a hash table +function Format-ChocolateyList +{ + param ( + [string[]] + $List + ) + + $map = @{} + + $List | ForEach-Object { + $row = $_ -ireplace ' Downloads cached for licensed users', '' + if ($row -imatch '^(?.*?)\s+(?[\d\.]+(\s+\[Approved\]){0,1}(\s+-\s+Possibly broken){0,1}).*?$') + { + $map[$Matches['name']] = $Matches['version'] + } + } + + return $map +} + +# get chocolatey search results, filtered by fudge +function Format-ChocolateySearchResults +{ + param ( + [Parameter()] + [string] + $Key, + + [Parameter()] + [int] + $Limit, + + [Parameter()] + [string] + $Source + ) + + # get search results from chocolatey + $results = Get-ChocolateySearchList -Key $Key -Source $Source + $OrganisedResults = @() + $count = 0 + + # if limit is 0, set to total results returned + if ($Limit -eq 0) { + $Limit = ($results | Measure-Object).Count + } + + # perfect match result + if ($results.ContainsKey($Key)) { + $count++ + $OrganisedResults += @{ 'Name' = $Key; 'Version' = $results[$Key]; } + $results.Remove($Key) + } + + # starts with for sub-packages + if ($count -lt $Limit) + { + $results.Clone().Keys | ForEach-Object { + if ($_.StartsWith("$($Key).")) { + $count++ + $OrganisedResults += @{ 'Name' = $_; 'Version' = $results[$_]; } + $results.Remove($_) + } + } + } + + # starts with for other packages + if ($count -lt $Limit) + { + $results.Clone().Keys | ForEach-Object { + if ($_.StartsWith($Key)) { + $count++ + $OrganisedResults += @{ 'Name' = $_; 'Version' = $results[$_]; } + $results.Remove($_) + } + } + } + + # contains the key + if ($count -lt $Limit) + { + $results.Clone().Keys | ForEach-Object { + if ($_.Contains($Key)) { + $count++ + $OrganisedResults += @{ 'Name' = $_; 'Version' = $results[$_]; } + $results.Remove($_) + } + } + } + + # levenshtein for remaining packages + if ($count -lt $Limit) + { + $leven = @() + + $results.Keys | ForEach-Object { + $leven += @{ 'Name' = $_; 'Version' = $results[$_]; 'Dist' = (Get-Levenshtein $Key $_) } + } + + $leven | Sort-Object { $_.Dist } | ForEach-Object { + $OrganisedResults += @{ 'Name' = $_.name; 'Version' = $_.Version; } + } + } + + return @($OrganisedResults) +} + + +# invokes fudge to search chocolatey for a package and display the results +function Invoke-Search +{ + param ( + [string] + $Key, + + [int] + $Limit, + + [string] + $Source, + + $LocalList + ) + + # validate the key/package name supplied + if ([string]::IsNullOrWhiteSpace($Key)) { + Write-Notice 'No search key provided' + return + } + + # validate the limit supplied + if ($Limit -lt 0) { + Write-Notice "Limit for searching must be 0 or greater, found: $($Limit)" + return + } + + # get search results from chocolatey + $results = Format-ChocolateySearchResults -Key $Key -Limit $Limit -Source $Source + + # display the search results + $results | Select-Object -First $Limit | ForEach-Object { + if ($LocalList.ContainsKey($_.Name)) + { + ($_.Version -imatch '^(?[\d\.]+).*?$') | Out-Null + + if ($Matches['version'] -eq $LocalList[$_.Name]) { + Write-Success ("{0,-30} {1,-40} (installed: {2})" -f $_.Name, $_.Version, $LocalList[$_.Name]) + } + else { + Write-Notice ("{0,-30} {1,-40} (installed: {2})" -f $_.Name, $_.Version, $LocalList[$_.Name]) + } + } + else { + Write-Host ("{0,-30} {1,-30}" -f $_.Name, $_.Version) + } + } + + # display the total + $total = ($results | Measure-Object).Count + Write-Notice "$($total) package(s) found" +} + + +# invokes fudge to display details of local packages +function Invoke-FudgeLocalDetails +{ + param ( + $Config, + + [string] + $Key, + + $LocalList, + + [switch] + $Dev, + + [switch] + $DevOnly + ) + + # maps for filtering packages + $installed = @{} + $updating = @{} + $missing = @{} + + # package map + $packages = @{} + + if (!$DevOnly) { + $Config.packages | ForEach-Object { $packages[$_.name] = $_.version } + } + + if ($Dev) { + $Config.devPackages | ForEach-Object { $packages[$_.name] = $_.version } + } + + # loop through packages + $packages.Keys | ForEach-Object { + if ([string]::IsNullOrWhiteSpace($Key) -or $_ -ieq $Key) + { + $version = $packages[$_] + + if ($LocalList.ContainsKey($_)) + { + if ($LocalList[$_] -ieq $version -or (Test-VersionPassedIsLatest $version)) { + $installed[$_] = $version + } + else { + $updating[$_] = $version + } + } + else { + $missing[$_] = $version + } + } + } + + if (![string]::IsNullOrWhiteSpace($Key) -and (Test-Empty $installed) -and (Test-Empty $updating) -and (Test-Empty $missing)) + { + $missing[$Key] = 'Not in Fudgefile' + } + + # output the details + Write-Host "Package details from Fudgefile:" + $installed.Keys | Sort-Object | ForEach-Object { Write-Success ("{0,-30} {1,-20} (installed: {2})" -f $_, $installed[$_], $LocalList[$_]) } + $updating.Keys | Sort-Object | ForEach-Object { Write-Notice ("{0,-30} {1,-20} (installed: {2})" -f $_, $updating[$_], $LocalList[$_]) } + $missing.Keys | Sort-Object | ForEach-Object { Write-Fail ("{0,-30} {1,-20}" -f $_, $missing[$_]) } +} + +function Get-ChocolateyLatestVersion +{ + param ( + [Parameter()] + [string] + $Package, + + [Parameter()] + [string] + $Source + ) + + $result = (Format-ChocolateySearchResults -Key $Package -Limit 1 -Source $Source | Select-Object -First 1) + if (!(Test-Empty $result)) { + return ($result.Version -split '\s+')[0] + } + + return 'latest' +} + +# invoke chocolate for the specific action +function Invoke-Chocolatey +{ + param ( + [Parameter()] + [string] + $Action, + + [Parameter()] + [string] + $Package, + + [Parameter()] + [string] + $Version, + + [Parameter()] + [string] + $Source, + + [Parameter()] + [string] + $Parameters, + + [Parameter()] + [string] + $Arguments, + + [Parameter()] + $LocalList + ) + + # if there was no package passed, return + if (Test-Empty $Package) { + return + } + + # set the source from which to install/upgrade/downgrade packages + $SourceArg = Format-ChocolateySource $Source + + # set the parameters to pass + $ParametersArg = Format-ChocolateyParams $Parameters + + $Arguments += ' --allow-unofficial' + + # if the version is latest, attempt to get the real current version + $latest = Test-VersionPassedIsLatest -Version $Version + if ($latest) { + $Version = Get-ChocolateyLatestVersion -Package $Package -Source $Source + } + + $Version = Format-SafeguardString $Version 'latest' + + # build a version string, so if latest we can show the latest version + $VersionStr = "$($Version)" + if ($latest) { + $VersionStr = "latest [$($Version)]" + } + + # set the version arg to pass + $VersionArg = Format-ChocolateyVersion $Version + + # if action is 'install', do we need to install, or upgrade/downgrade based on version? + if (($Action -ieq 'install') -and ($null -ne $LocalList)) { + $Action = Get-InstallAction -Package $Package -Version $Version -LocalList $LocalList + } + + # attempt 3 times - mostly do to help prevent timeouts + foreach ($i in 1..3) + { + $success = $true + + # run chocolatey for appropriate action + switch ($Action.ToLowerInvariant()) + { + 'install' + { + if ($i -eq 1) { + Write-Information "> Installing $($Package) ($($VersionStr))" -NoNewLine + } + $output = Invoke-Expression "choco install $($Package) $($VersionArg) -y $($SourceArg) $($ParametersArg) $($Arguments)" + } + + 'upgrade' + { + if ($i -eq 1) { + Write-Information "> Upgrading $($Package) to ($($VersionStr))" -NoNewLine + } + $output = Invoke-Expression "choco upgrade $($Package) $($VersionArg) -y $($SourceArg) $($ParametersArg) $($Arguments)" + } + + 'downgrade' + { + if ($i -eq 1) { + Write-Information "> Downgrading $($Package) to ($($VersionStr))" -NoNewLine + } + $output = Invoke-Expression "choco upgrade $($Package) $($VersionArg) -y $($SourceArg) $($ParametersArg) $($Arguments) --allow-downgrade" + } + + 'uninstall' + { + if ($i -eq 1) { + Write-Information "> Uninstalling $($Package)" -NoNewLine + } + $output = Invoke-Expression "choco uninstall $($Package) -y -x $($ParametersArg) $($Arguments)" + } + + 'pack' + { + Write-Information "> Packing $($Package)" -NoNewLine + $path = Split-Path -Parent -Path $Version + $name = Split-Path -Leaf -Path $Version + + try { + Push-Location $path + $output = Invoke-Expression "choco pack $($name) $($Arguments)" + } + finally { + Pop-Location + } + } + } + + # determine if we failed, or if the exit code indicates a reboot is needed, + # or we've timed out and need to try + $lastcode = $LASTEXITCODE + if (!$? -or (@(0, 3010) -notcontains $lastcode)) + { + # flag as failure + $success = $false + + # check if the package timed-out, so we can retry again + if ($output -ilike '*the operation has timed out*') { + Start-Sleep -Seconds 5 + continue + } + + # double check that the error isn't a false positive + switch ($Action) + { + {($_ -ieq 'uninstall')} + { + $success = ($output -ilike '*has been successfully uninstalled*' -or $output -ilike '*Cannot uninstall a non-existent package*') + } + + {($_ -ieq 'install') -or ($_ -ieq 'upgrade') -or ($_ -ieq 'downgrade')} + { + $success = ($output -ilike '*has been successfully installed*' -or $output -ilike '*has been installed*') + } + } + } + + # if we get here, break out + break + } + + # if not successful, output the error details + if (!$success) { + Write-Fail " > failed" + Write-Notice "`n`n$($output)`n" + throw "Failed to $($Action) package: $($Package)" + } + + $actionVerb = ("$($Action)ed" -ireplace 'eed$', 'ed') + + if ($output -ilike '*exit code 3010*' -or $lastcode -eq 3010) + { + Write-Success " > $($actionVerb)" -NoNewLine + Write-Notice " > Reboot Required" + } + else { + Write-Success " > $($actionVerb)" + } +} \ No newline at end of file diff --git a/.ci/PSLint.ps1 b/.ci/PSLint.ps1 new file mode 100644 index 0000000000..e989bb044c --- /dev/null +++ b/.ci/PSLint.ps1 @@ -0,0 +1,94 @@ +Set-StrictMode -Version Latest + +$ErrorActionPreference = 'Stop' + +Import-Module -Name 'PsScriptAnalyzer' -Force +$base = $global:PSScriptRoot + +$Utf8NoBomEncoding = New-Object System.Text.UTF8Encoding $False + +function Test-Skip-File { + param ( + [Parameter(Mandatory)] + [string] + $filename + ) + + # Ignore imported files + # https://github.com/pypa/virtualenv/issues/1371 + # https://github.com/ogrisel/python-appveyor-demo/issues/55 + # https://github.com/MathieuBuisson/PowerShell-DevOps/issues/6 + return ( + $filename -eq 'Export-NUnitXml.psm1' -or + $filename -eq 'Fudge.ps1' -or + $filename -eq 'FudgeTools.psm1' -or + $filename -eq 'install.ps1' -or + $filename -eq 'activate.ps1' -or + $filename.EndsWith('.jj2')) +} + +$Utf8NoBomEncoding = New-Object System.Text.UTF8Encoding $False + +$UNIXEOL = "`n" + +function ReformatFile { + param ( + [Parameter(Mandatory)] + [string] + $filename, + + [Parameter(Mandatory)] + [string] + $eol + ) + $orig = Get-Content -Raw $filename + $content = Invoke-Formatter -ScriptDefinition $orig -Settings $base\PSScriptAnalyzerSettings.psd1 -ErrorAction Stop + + if ($content) { + $content = $content -split "`r?`n" + if (!($content[-1])) { + $content = $content[0..($content.length - 2)] + } + $content = (($content -join $eol) + $eol) + [System.IO.File]::WriteAllText($filename, $content, $Utf8NoBomEncoding) + } +} + +[Microsoft.Windows.PowerShell.ScriptAnalyzer.Generic.DiagnosticRecord[]]$ScriptAnalyzerResult = $null + +Get-ChildItem -Recurse -Force '*.ps*' | ForEach-Object { + $path = $_.FullName + + if ( $path.Contains('.git') -or (Test-Skip-File($_.Name)) ) { + Write-Output "Skipping $path" + } + else { + $NewResult = Invoke-ScriptAnalyzer -IncludeDefaultRules -Setting $base\PSScriptAnalyzerSettings.psd1 -Path $path + if ($ScriptAnalyzerResult) { + $ScriptAnalyzerResult += $NewResult + } + else { + $ScriptAnalyzerResult = $NewResult + } + + ReformatFile $path $UNIXEOL + } +} + +if ($env:APPVEYOR) { + Import-Module "$base\Export-NUnitXml.psm1" + Export-NUnitXml -ScriptAnalyzerResult $ScriptAnalyzerResult -Path ".\ScriptAnalyzerResult.xml" + (New-Object 'System.Net.WebClient').UploadFile( + "https://ci.appveyor.com/api/testresults/nunit/$($env:APPVEYOR_JOB_ID)", + (Resolve-Path .\ScriptAnalyzerResult.xml) + ) +} + +git diff + +if ($ScriptAnalyzerResult) { + Write-Output $ScriptAnalyzerResult + + # Failing the build + Throw 'Build failed because there was one or more PSScriptAnalyzer violation. See test results for more information.' +} diff --git a/.ci/PSScriptAnalyzerSettings.psd1 b/.ci/PSScriptAnalyzerSettings.psd1 new file mode 100644 index 0000000000..1199c79dad --- /dev/null +++ b/.ci/PSScriptAnalyzerSettings.psd1 @@ -0,0 +1,120 @@ +@{ + Severity = @('Error', 'Warning') + + IncludeDefaultRules = $true + + # List all rules here, otherwise they are excluded. + # Some are inactive because they need to be configured before usable. + + IncludeRules = @( + 'PSAlignAssignmentStatement' + 'PSAvoidAlias' + 'PSAvoidAssignmentToAutomaticVariable' + 'PSAvoidDefaultTrueValueSwitchParameter' + 'PSAvoidDefaultValueForMandatoryParameter' + 'PSAvoidEmptyCatchBlock' + 'PSAvoidGlobalAliases' + 'PSAvoidGlobalFunctions' + 'PSAvoidGlobalVars' + 'PSAvoidInvokingEmptyMembers' + 'PSAvoidNullOrEmptyHelpMessageAttribute' + 'PSAvoidPositionalParameters' + 'PSAvoidReservedCharInCmdlet' + 'PSAvoidReservedParams' + 'PSAvoidShouldContinueWithoutForce' + 'PSAvoidTrailingWhitespace' + 'PSAvoidUserNameAndPasswordParams' + 'PSAvoidUsingComputerNameHardcoded' + 'PSAvoidUsingConvertToSecureStringWithPlainText' + 'PSAvoidUsingDeprecatedManifestFields' + 'PSAvoidUsingInvokeExpression' + 'PSAvoidUsingPlainTextForPassword' + 'PSAvoidUsingWMICmdlet' + # Easier to use inside Fudge + # 'PSAvoidUsingWriteHost' + 'PSDscExamplesPresent' + 'PSDscTestsPresent' + 'PSMisleadingBacktick' + 'PSMissingModuleManifestField' + 'PSPlaceCloseBrace' + 'PSPlaceOpenBrace' + 'PSPossibleIncorrectComparisonWithNull' + 'PSPossibleIncorrectUsageOfAssignmentOperator' + 'PSPossibleIncorrectUsageOfRedirectionOperator' + 'PSProvideCommentHelp' + 'PSReturnCorrectTypesForDSCFunctions' + 'PSUseApprovedVerbs' + # Not suitable for other OS + # 'PSUseBOMForUnicodeEncodedFile' + 'PSUseCmdletCorrectly' + 'PSUseCompatibleCmdlets' + 'PSUseConsistentIndentation' + 'PSUseConsistentWhitespace' + 'PSUseCorrectCasing' + 'PSUseDeclaredVarsMoreThanAssignments' + 'PSUseIdenticalMandatoryParametersDSC' + 'PSUseIdenticalParametersDSC' + 'PSUseLiteralInitializerForHashtable' + 'PSUseOutputTypeCorrectly' + 'PSUsePSCredentialType' + 'PSUseShouldProcessCorrectly' + 'PSUseShouldProcessForStateChangingFunctions' + 'PSUseSingularNouns' + 'PSUseStandardDSCFunctionsInResource' + 'PSUseSupportsShouldProcess' + 'PSUseToExportFieldsInManifest' + 'PSUseUTF8EncodingForHelpFile' + 'PSUseVerboseMessageInDSCResource' + # Compatibility subdirectory + 'PSUseCompatibleCommands' + 'PSUseCompatibleSyntax' + 'PSUseCompatibleTypes' + ) + + # The Rules here are mostly manually imported from + # https://github.com/PowerShell/PSScriptAnalyzer/blob/master/Engine/Settings/CodeFormatting.psd1 + # except `PSUseConsistentWhitespace.CheckOperator` is disabled as incompatible + # https://github.com/PowerShell/PSScriptAnalyzer/issues/769 + + Rules = @{ + PSPlaceOpenBrace = @{ + Enable = $true + OnSameLine = $true + NewLineAfter = $true + IgnoreOneLineBlock = $true + } + + PSPlaceCloseBrace = @{ + Enable = $true + NewLineAfter = $true + IgnoreOneLineBlock = $true + NoEmptyLineBefore = $false + } + + PSUseConsistentIndentation = @{ + Enable = $true + Kind = 'space' + PipelineIndentation = 'IncreaseIndentationForFirstPipeline' + IndentationSize = 4 + } + + PSUseConsistentWhitespace = @{ + Enable = $true + CheckInnerBrace = $true + CheckOpenBrace = $true + CheckOpenParen = $true + CheckOperator = $true + CheckPipe = $true + CheckSeparator = $true + } + + PSAlignAssignmentStatement = @{ + Enable = $true + CheckHashtable = $false + } + + PSUseCorrectCasing = @{ + Enable = $true + } + } +} diff --git a/.ci/PrepareAVVM.ps1 b/.ci/PrepareAVVM.ps1 new file mode 100644 index 0000000000..65cfd5fc4a --- /dev/null +++ b/.ci/PrepareAVVM.ps1 @@ -0,0 +1,308 @@ +Set-StrictMode -Version latest + +$PACKAGES_ROOT = "$env:SYSTEMDRIVE\avvm" +$REGISTRY_ROOT = 'HKLM:\Software\AppVeyor\VersionManager' + +# GetInstalledProductVersion and Get-Version are copied from AVVM.ps1 +function GetInstalledProductVersion ($product) { + $productRegPath = "$REGISTRY_ROOT\$product" + if (Test-Path $productRegPath) { + $ver = Get-ItemProperty -Path $productRegPath + @{ + Product = $product + version = $ver.version + Platform = $ver.Platform + } + } +} + +function Get-Version ([string]$str) { + $versionDigits = $str.Split('.') + $version = @{ + major = -1 + minor = -1 + build = -1 + revision = -1 + number = 0 + value = $null + } + + $version.value = $str + + if ($versionDigits -and $versionDigits.Length -gt 0) { + $version.major = [int]$versionDigits[0] + } + if ($versionDigits.Length -gt 1) { + $version.minor = [int]$versionDigits[1] + } + if ($versionDigits.Length -gt 2) { + $version.build = [int]$versionDigits[2] + } + if ($versionDigits.Length -gt 3) { + $version.revision = [int]$versionDigits[3] + } + + for ($i = 0; $i -lt $versionDigits.Length; $i++) { + $version.number += [long]$versionDigits[$i] -shl 16 * (3 - $i) + } + + return $version +} + +function SetInstalledProductVersion { + param( + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] + $product, + + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] + $version, + + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] + $platform + ) + + $productRegPath = "$REGISTRY_ROOT\$product" + New-Item $productRegPath -Force | + Out-Null + + New-ItemProperty -Path $productRegPath -Name Version -PropertyType String -Value $version -Force | + Out-Null + + New-ItemProperty -Path $productRegPath -Name Platform -PropertyType String -Value $platform -Force | + Out-Null + + Write-Output "Creating $PACKAGES_ROOT\$product\$version\$platform" + + if (!(Test-Path "$PACKAGES_ROOT\$product\$version\$platform")) { + New-Item -ItemType Directory "$PACKAGES_ROOT\$product\$version\$platform" -Force > $null + } + + if (!(Test-Path "$PACKAGES_ROOT\$product\$version\$platform")) { + throw "Something went wrong" + } +} + +function Add-Product { + param( + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] + $product, + + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] + $version, + + [string] + $platform + ) + + $name = $product + + if (!$platform) { + $platform = $env:platform + } + + $installed = GetInstalledProductVersion $Product + + $in_program_files = $false + + $version_parts = Get-Version $version + + if (($product -eq 'jdk') -or ($product -eq 'node')) { + $in_program_files = $true + + if (($product -eq 'jdk') -and ($version_parts.minor -gt 8)) { + # 1.9.0 -> 9 + $shortver = $version_parts.minor + } + else { + $shortver = $version + } + + if ($platform -eq 'x86') { + $dir_name = "Program Files (x86)" + } + else { + $dir_name = "Program Files" + } + if ($product -eq 'jdk') { + $dir_name = "$dir_name\Java" + } + $dir_name = "$dir_name\$name" + } + else { + $shortver = "{0}{1}" -f ($version_parts.major, $version_parts.minor) + $dir_name = $name + } + + + if (Test-Path "$PACKAGES_ROOT\$name\$version\") { + Write-Output "$PACKAGES_ROOT\$name\$version exists; skipping" + + if ($product -eq 'node') { + $base = 'https://appveyordownloads.blob.core.windows.net/avvm' + $versions_content = (New-Object Net.WebClient).DownloadString("$base/$name-versions.txt") + Set-Content "$PACKAGES_ROOT\$name-versions.txt" $versions_content + return + } + } + + if ($installed) { + $current_version = $installed.version + if ((Get-Version $current_version).number -gt $version_parts.number) { + $versions_content = "$current_version +$version +lts:$version +stable:$version +current:$current_version +" + } + else { + $versions_content = "$version +$current_version +lts:$version +stable:$version +current:$current_version +" + } + } + else { + $versions_content = "$version +lts:$version +stable:$version +" + } + Set-Content "$PACKAGES_ROOT\$name-versions.txt" $versions_content + + Write-Output "Wrote $PACKAGES_ROOT\$name-versions.txt" + + if ($product -eq 'MinGW') { + if (Test-Path C:\avvm\MinGW) { + # This is only needed for 5.3.0 x86, only vs2015 image + Write-Output "Deleting pre-existing C:\avvm\MinGW ..." + Remove-Item C:\avvm\MinGW + } + } + + if ($installed) { + if ($installed.version -eq $version) { + if ($installed.Platform -eq $platform) { + Write-Output "$product $version $env:platform already set up" + return; + } + } + } + + New-Item -ItemType Directory "$PACKAGES_ROOT\$name" -Force > $null + + New-Item -ItemType Directory "$PACKAGES_ROOT\$name\$version" -Force > $null + + Write-Verbose "Looking for C:\$dir_name$shortver .." + + if (!(Test-Path "C:\$dir_name$shortver")) { + if (!(Test-Path "C:\$dir_name$shortver-x64")) { + throw "Cant find $dir_name$shortver or C:\$dir_name$shortver-x64" + } + + # Use x64 if x86 not available + $platform = 'x64' + } + + New-Item -ItemType Directory "$PACKAGES_ROOT\$name\$version\$platform" -Force > $null + + if ($in_program_files) { + $dir = "C:\$dir_name$shortver" + } + else { + Write-Output "Looking for C:\$name$shortver-x64 .." + + $dir = '' + if (Test-Path "C:\$dir_name$shortver-x64") { + if ($platform -eq "x64") { + $dir = "C:\$dir_name$shortver-x64" + } + else { + $dir = "C:\$dir_name$shortver" + } + } + + # TODO: Re-arrange to look only for the needed platform + if (!$dir) { + Write-Output "Looking for C:\$dir_name$shortver-x86 .." + } + + if ((!($dir)) -and (Test-Path "C:\$dir_name$shortver-x86")) { + if ($platform -eq "x86") { + $dir = "C:\$dir_name$shortver-x86" + } + else { + $dir = "C:\$dir_name$shortver" + } + } + } + + if (!($dir)) { + throw 'couldnt find x86/x64 directories for $name' + } + + if (!(Test-Path $dir)) { + throw "Cant find $dir" + } + + Write-Output "Coping $dir to $PACKAGES_ROOT\$name\$version\$platform\$name ..." + + Move-Item $dir "$PACKAGES_ROOT\$name\$version\$platform\$name" + + $files_content = ('$files = @{ "' + $name + '" = "C:\' + $name + '" }') + $files_path = "$PACKAGES_ROOT\$name\$version\$platform\files.ps1" + + Set-Content $files_path $files_content + + Write-Output "Wrote $files_path" +} + +function Initialize-Miniconda27 { + # The algorithm above to prepare products for switching depends on a version + # in the directory name, which works for all cases except for Miniconda27. + # Use this if you need Miniconda27. + Write-Output "Moving C:\Miniconda(-x64) to C:\Miniconda27(-x64)" + Move-Item C:\Miniconda C:\Miniconda27 + Move-Item C:\Miniconda-x64 C:\Miniconda27-x64 +} + +function Initialize-AppVeyorProductVersion { + # TODO: Only set up default versions for products which are needed + + # This tells Install-Product to load product versions from $PACKAGES_ROOT + $env:AVVM_DOWNLOAD_URL = '../../avvm/' + Set-ItemProperty -Path 'HKCU:\Environment' -Name 'AVVM_DOWNLOAD_URL' -Value $env:AVVM_DOWNLOAD_URL + + # node already set to 8.x + SetInstalledProductVersion go 1.12.3 x64 + + # todo: Handle Python27 and Ruby193 + + SetInstalledProductVersion miniconda 2.7.15 x86 + SetInstalledProductVersion miniconda3 3.7.0 x86 + + SetInstalledProductVersion jdk 1.6.0 x86 + SetInstalledProductVersion perl 5.20.1 x86 + # /C/MinGW is only set on vs2013 and vs2015 images, and it isnt desirable + # SetInstalledProductVersion MinGW 5.3.0 x86 + + # hg 4.1.1 is pre-installed, but it does not need to be replaced, + # and is instead reported as 5.0 +} + +$old_EAP = $ErrorActionPreference +$ErrorActionPreference = 'SilentlyContinue'; +Export-ModuleMember -Function Initialize-AppVeyorProductVersion, Add-Product, Initialize-Miniconda27 +$ErrorActionPreference = $old_EAP; diff --git a/.ci/appveyor.yml b/.ci/appveyor.yml index ea56a828c7..61bdfe907a 100644 --- a/.ci/appveyor.yml +++ b/.ci/appveyor.yml @@ -1,100 +1,184 @@ +image: Visual Studio 2015 + environment: global: - # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the - # /E:ON and /V:ON options are not enabled in the batch script intepreter - # See: http://stackoverflow.com/a/13751649/163740 - CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\.ci\\run_with_env.cmd" PIP_CACHE_DIR: C:\pip_cache + PIP_DISABLE_PIP_VERSION_CHECK: 1 + # Needed if pip uninstall is used + PIP_YES: 1 + YARN_GPG: 'no' + FudgeCI: $(APPVEYOR_BUILD_FOLDER)\.ci + NUGET_EXE_NO_PROMPT: 1 + NUGET_HTTP_CACHE_PATH: C:\nuget_http_cache + CHOCO_CACHE_DIR: C:\choco_cache + MSYS_ROOT: C:\msys64 + MSYS_BIN: $(MSYS_ROOT)\usr\bin + JAVA_HOME: C:\jdk + MSSDK_ROOT: C:\Program Files\Microsoft SDKs + VS_ROOT: C:\Program Files (x86)\Microsoft Visual Studio + VIRTUALENV_NO_DOWNLOAD: 1 + VIRTUALENV_NO_PIP: 1 + VIRTUALENV_NO_SETUPTOOLS: 1 + # Uncomment to debug tox venv problems + # VIRTUALENV_VERBOSE: 1 + GOPATH: C:\gopath + # A problem with store_env_in_registry.py means this needs to be explicit + PSModulePath: >- + $(PSModulePath);C:\Program Files (x86)\WindowsPowerShell\Modules; + PATH: >- + C:\python;C:\python\Scripts;$(PATH);$(MSYS_BIN); + C:\tools\fudge\tools; + C:\ruby\bin; + C:\jdk\bin; + $(GOPATH)\bin; + $(APPVEYOR_BUILD_FOLDER)\node_modules\.bin; + $(APPVEYOR_BUILD_FOLDER)\vendor\bin; + JL_PKG: CoalaBears + JULIA_PROJECT: '@.' + TOX_FEATURES: check-noskip-codecov + BEAR_LIST: astyle cppcheck xmllint + TOX_TEST_SELECTORS: pip-noreqs-npm-gem-go-perl-php-java8-adhoc + TOXENV: py$(PYTHON_MINOR_NODOTS)-$(TOX_TEST_SELECTORS)-$(TOX_FEATURES)-win matrix: - - PYTHON: "C:\\Python34" - PYTHON_VERSION: "3.4.4" - PYTHON_ARCH: "32" - ruby_version: "24" + - PYTHON_VERSION: 3.6 + PYTHON_MINOR_NODOTS: 36 + - PYTHON_VERSION: 3.5 + PYTHON_MINOR_NODOTS: 35 + - PYTHON_VERSION: 3.4 + PYTHON_MINOR_NODOTS: 34 - - PYTHON: "C:\\Python34-x64" - PYTHON_VERSION: "3.4.4" - PYTHON_ARCH: "64" - ruby_version: "24-x64" +platform: + - x86 + - x64 cache: + - C:\nuget_http_cache + - C:\choco_cache - "C:\\pip_cache" - "node_modules" - "C:\\Users\\appveyor\\AppData\\Local\\coala-bears\\coala-bears" - "C:\\Users\\appveyor\\AppData\\Roaming\\nltk_data" + - "%LOCALAPPDATA%\\Composer" branches: except: - /^sils\/.*/ +# This forces unix style line endings in the clone, which is necessary to +# avoid warning regarding EOLs when running git diff on Windows +init: git config --global core.autocrlf false + install: - # Prepend newly installed Python to the PATH of this build (this cannot be - # done from inside the powershell script as it would require to restart - # the parent CMD process). - - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - - "SET PATH=C:\\Program\ Files\\Java\\jdk1.7.0\\bin;%PATH%" + # Show initial state + - powershell -c "$PSVersionTable" + # Uncomment to debug + # printenv + - python --version + - "python -c \"import struct; print(struct.calcsize('P') * 8)\"" + - python -m pip --version + - python -c "import setuptools; print(setuptools.__version__)" + - node --version + - which npm + - npm --version + - npm config get prefix + - ruby --version + - bundler --version + - go version + - which java + - java -version + - which javac + - javac -version + # JAVA_HOME set above is not populated yet + - echo %JAVA_HOME% + - ls %JAVA_HOME%/bin/java & exit 0 + - which perl + - perl --version + - which gcc & exit 0 + - gcc --version & exit 0 + - which cl & exit 0 + + # Stores environment in registry, with minor tweaks + - python .ci/store_env_in_registry.py + - refreshenv + + # Set up AppVeyor product versions, and install dummy choco entries for them + - ps: . .ci/FudgeCI.ps1; Initialize-AppVeyorVM + - refreshenv + - echo %PATH% + # Avoid tools finding and using MinGW + - mv C:\MinGW %TEMP% + # TODO: Avoid tools finding and using Visual Studio + + # Show updated SOE; versions should be as defined in top of the Fudgefile + - python --version + - "python -c \"import struct; print(struct.calcsize('P') * 8)\"" + - node --version + - which npm + - npm --version + - npm config get prefix + - go version + - ruby --version + - which java + - java -version + - which javac + - javac -version + - ls %JAVA_HOME%/bin/java + - perl --version + - which ppm + - ppm --version + - which gcc & exit 0 + - gcc --version & exit 0 + + - "%MSYS_BIN%\\date.exe" + - cp .ci/choco.config + %ChocolateyInstall%\config\chocolatey.config + # Install remainer of the Fudgefile with chocolatey using Fudge + - ps: . .ci/Fudge.ps1 install + - refreshenv + - echo %PATH% - # language-tool needs the registry tweaked here since it determines the java - # version wrong (since appveyor has both, 1.7 and 1.8 in x86 and x64). - - "SET KEY_NAME=HKLM\\Software\\JavaSoft\\Java Runtime Environment" - - "REG add \"%KEY_NAME%\" /v CurrentVersion /t REG_SZ /d 1.7 /f" + - ps: if ($env:APPVEYOR_JOB_NUMBER -eq 1) { . .ci/PSLint.ps1 } # Check that we have the expected version and architecture for Python - "python --version" - "python -c \"import struct; print(struct.calcsize('P') * 8)\"" - - "%CMD_IN_ENV% python -m pip install --upgrade setuptools==21 pip==9" - - "%CMD_IN_ENV% python -m pip install -r test-requirements.txt \ - -r requirements.txt -r docs-requirements.txt" - - ps: "Install-Product node ''" # Use latest node v5.x.x - - "npm config set loglevel warn" - - "sed -i '/alex/d' package.json" - - "sed -i '/coffeelint/d' package.json" - - "sed -i '/csscomb/d' package.json" - - "sed -i '/docker/d' package.json" - - "sed -i '/elm/d' package.json" - - "sed -i '/gherkin/d' package.json" - - "sed -i '/jshint/d' package.json" - - "sed -i '/remark/d' package.json" - - "sed -i '/postcss/d' package.json" - - "sed -i '/sass/d' package.json" - - "sed -i '/textlint/d' package.json" - - "sed -i '/tslint/d' package.json" - - "npm install" - - "SET PATH=node_modules\\.bin;%PATH%" - - "DEL node_modules\\.bin\\eslint*" - - # Commands for Ruby - - "sed -i '/sqlint/d' Gemfile" - - "SET PATH=C:\\Ruby%ruby_version%\\bin;%PATH%" - - "bundle install" + # Confirm other versions + - node --version + - which npm + - npm --version + - npm config get prefix + - go version + - ruby --version + - perl --version + - which java + - java -version + - which javac + - javac -version + - ls %JAVA_HOME%/bin/java + # Newly installed versions + - which composer & exit 0 + - composer --version & exit 0 + - ppm --version & exit 0 + + + - "%MSYS_BIN%\\date.exe" build: false # Not a C# project, build stuff at the test step instead. test_script: - # Force DOS format, as Checkstyle configs enable NewlineAtEndOfFile, - # which defaults to CRLF on Windows, and Appveyor CI ignores .gitattributes - # http://help.appveyor.com/discussions/problems/5687-gitattributes-changes-dont-have-any-effect - - unix2dos tests/java/test_files/CheckstyleGood.java - # Clang DLLs x64 were nowadays installed, but the x64 version hangs, so we - # exclude according tests. See https://github.com/appveyor/ci/issues/495 and - # https://github.com/appveyor/ci/issues/688 - - > - "%CMD_IN_ENV% python -m pytest - --cov -k "not ClangASTPrintBear and not ClangCloneDetectionBear and - not ClangComplexityBear and not ClangCountVectorCreator and - not ClangCountingConditions and not RuboCopBearTest and - not CSVLintBearTest" - - "%CMD_IN_ENV% python setup.py install" - - "%CMD_IN_ENV% python -m pip install \ - git+https://github.com/coala/coala" - - sed -i '/ShellCheckBear/d' .coafile - - "%CMD_IN_ENV% coala --ci" - -on_success: - - codecov - -on_failure: - - codecov + - python -m pip --version + - python -c "import setuptools; print(setuptools.__version__)" + + - "sed -i 's/^envlist.*$/envlist: %TOXENV%/' tox.ini" + - python -m tox --sitepackages + + - python setup.py install + + + - rm -rf .tox + + - coala --ci matrix: fast_finish: true diff --git a/.ci/choco.config b/.ci/choco.config new file mode 100644 index 0000000000..026715ec07 --- /dev/null +++ b/.ci/choco.config @@ -0,0 +1,59 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/.ci/constants.ps1 b/.ci/constants.ps1 new file mode 100644 index 0000000000..f8979db466 --- /dev/null +++ b/.ci/constants.ps1 @@ -0,0 +1,9 @@ + +New-Variable -Scope global -Name project_name -Value 'coala-bears' +New-Variable -Scope global -Name pip_version -Value '9.0.1' +New-Variable -Scope global -Name setuptools_version -Value '21.2.2' + +$old_EAP = $ErrorActionPreference +$ErrorActionPreference = 'SilentlyContinue' +Export-ModuleMember -Variable name, pip_version, setuptools_version +$ErrorActionPreference = $old_EAP diff --git a/.ci/deps.ActivePerl-packages.ps1 b/.ci/deps.ActivePerl-packages.ps1 new file mode 100644 index 0000000000..dd3e21058c --- /dev/null +++ b/.ci/deps.ActivePerl-packages.ps1 @@ -0,0 +1,9 @@ +function Install-Perl-Modules { + cpanm --quiet --installdeps --with-develop --notest . + + Remove-Item -Force MYMETA.yml -ErrorAction Ignore +} + +function Invoke-ExtraInstallation { + Install-Perl-Modules +} diff --git a/.ci/deps.ActivePerl.ps1 b/.ci/deps.ActivePerl.ps1 new file mode 100644 index 0000000000..4464207b19 --- /dev/null +++ b/.ci/deps.ActivePerl.ps1 @@ -0,0 +1,11 @@ +Set-StrictMode -Version latest + +function Install-PPM-cpanm { + ppm install App-cpanminus +} + +function Complete-Install { + Install-PPM-cpanm +} + +Export-ModuleMember -Function Install-PPM-cpanm, Complete-Install diff --git a/.ci/deps.alex.sh b/.ci/deps.alex.sh new file mode 100644 index 0000000000..37e47144c5 --- /dev/null +++ b/.ci/deps.alex.sh @@ -0,0 +1,7 @@ +ALEX=$(which alex || true) +# Delete 'alex' if it is not in a node_modules directory, +# which means it is ghc-alex. +if [[ -n "$ALEX" && "${ALEX/node_modules/}" == "${ALEX}" ]]; then + echo "Removing $ALEX" + sudo rm -rf $ALEX +fi diff --git a/.ci/deps.apt.sh b/.ci/deps.apt.sh deleted file mode 100755 index ac2a6920e2..0000000000 --- a/.ci/deps.apt.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/bash - -set -e -set -x - -# This script is no longer used by Travis CI. -# Any related aspects can be removed if beneficial. - -# apt-get commands -export DEBIAN_FRONTEND=noninteractive - -deps="libclang1-3.4 astyle indent mono-mcs chktex r-base julia golang-go luarocks verilator cppcheck flawfinder devscripts mercurial" -deps_infer="m4 opam" - -case $CIRCLE_BUILD_IMAGE in - "ubuntu-12.04") - USE_PPAS="true" - # The Circle provided Go is too old - sudo mv /usr/local/go /usr/local/circleci-go - ;; - "ubuntu-14.04") - # Use xenial, needed to replace outdated julia provided by Circle CI - ADD_APT_UBUNTU_RELEASE=xenial - # Work around lack of systemd on trusty, which xenial's lxc-common expects - echo '#!/bin/sh' | sudo tee /usr/bin/systemd-detect-virt > /dev/null - sudo chmod a+x /usr/bin/systemd-detect-virt - - # The non-apt go provided by Circle CI is acceptable - deps=${deps/golang-go/} - # Add packages which are already in the precise image - deps="$deps g++-4.9 libxml2-utils php-cli php7.0-cli php-codesniffer" - # gfortran on CircleCI precise is 4.6 and R irlba compiles ok, - # but for reasons unknown it fails on trusty without gfortran-4.9 - deps="$deps gfortran-4.9" - # Add extra infer deps - deps_infer="$deps_infer ocaml camlp4" - # opam install --deps-only --yes infer fails with - # Fatal error: - # Stack overflow - # aspcud is an external dependency resolver, and is the recommended - # solution: https://github.com/ocaml/opam/issues/2507 - deps_infer="$deps_infer aspcud" - ;; -esac - -if [ -n "$ADD_APT_UBUNTU_RELEASE" ]; then - echo "deb http://archive.ubuntu.com/ubuntu/ $ADD_APT_UBUNTU_RELEASE main universe" | sudo tee -a /etc/apt/sources.list.d/$ADD_APT_UBUNTU_RELEASE.list > /dev/null -fi - -if [ "$USE_PPAS" = "true" ]; then - sudo add-apt-repository -y ppa:cs50/ppa - sudo add-apt-repository -y ppa:marutter/rdev - sudo add-apt-repository -y ppa:staticfloat/juliareleases - sudo add-apt-repository -y ppa:staticfloat/julia-deps - sudo add-apt-repository -y ppa:ondrej/golang - sudo add-apt-repository -y ppa:avsm/ppa -elif [ -n "$USE_PPAS" ]; then - for ppa in $USE_PPAS; do - sudo add-apt-repository -y ppa:$ppa - done -fi - -deps_perl="libperl-critic-perl" - -sudo apt-get -y update -sudo apt-get -y --no-install-recommends install $deps $deps_perl $deps_infer - -# On Trusty, g++ & gfortran 4.9 need activating for R lintr dependency irlba. -ls -al /usr/bin/gcc* /usr/bin/g++* /usr/bin/gfortran* || true -if [[ "$CIRCLE_BUILD_IMAGE" == "ubuntu-14.04" ]]; then - sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.9 20 - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.9 20 - sudo update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-4.9 20 -fi diff --git a/.ci/deps.apt_get.sh b/.ci/deps.apt_get.sh new file mode 100755 index 0000000000..faac6a05a8 --- /dev/null +++ b/.ci/deps.apt_get.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +set -e -x + +if [ -f /usr/bin/flawfinder ]; then + sh .ci/deps.flawfinder.sh +fi diff --git a/.ci/deps.bakalint.sh b/.ci/deps.bakalint.sh new file mode 100755 index 0000000000..363e822eae --- /dev/null +++ b/.ci/deps.bakalint.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -e -x + +# VHDL Bakalint Installation +if [ ! -e ~/.local/bin/bakalint.pl ]; then + BAKALINT_VERSION=0.4.0 + wget "http://downloads.sourceforge.net/project/fpgalibre/bakalint/0.4.0/bakalint-0.4.0.tar.gz?r=https%3A%2F%2Fsourceforge.net%2Fprojects%2Ffpgalibre%2Ffiles%2Fbakalint%2F0.4.0%2F&ts=1461844926&use_mirror=netcologne" -O ~/bl.tar.gz + tar xf ~/bl.tar.gz -C ~/ + mv ~/bakalint-$BAKALINT_VERSION/bakalint.pl ~/.local/bin/ +fi diff --git a/.ci/deps.cabal.sh b/.ci/deps.cabal.sh index b8980b7741..d103d91f2d 100755 --- a/.ci/deps.cabal.sh +++ b/.ci/deps.cabal.sh @@ -7,6 +7,8 @@ cabal --version cabal update +cabal install happy + cabal install --only-dependencies --avoid-reinstalls # Force ghc-mod to resolve its Cabal version diff --git a/.ci/deps.composer-packages.ps1 b/.ci/deps.composer-packages.ps1 new file mode 100644 index 0000000000..9a99c0d5b2 --- /dev/null +++ b/.ci/deps.composer-packages.ps1 @@ -0,0 +1,9 @@ +function Install-Composer-DependList { + $composer_phar = "C:\ProgramData\ComposerSetup\bin\composer.phar" + + php.exe $composer_phar install +} + +function Invoke-ExtraInstallation { + Install-Composer-DependList +} diff --git a/.ci/deps.composer.sh b/.ci/deps.composer.sh new file mode 100755 index 0000000000..61deea413a --- /dev/null +++ b/.ci/deps.composer.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +set -e -x + +composer install diff --git a/.ci/deps.elm.sh b/.ci/deps.elm.sh new file mode 100755 index 0000000000..e60bf614d0 --- /dev/null +++ b/.ci/deps.elm.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e -x + +# elm-format Installation +if [ ! -e ~/.local/bin/elm-format ]; then + curl -fsSL -o elm-format.tgz https://github.com/avh4/elm-format/releases/download/0.5.2-alpha/elm-format-0.17-0.5.2-alpha-linux-x64.tgz + tar -xvzf elm-format.tgz -C ~/.local/bin/ +fi + +if [ "$TRAVIS_ELM_VERSION" = "0.18.0" ]; then + touch elm-package.json +else + touch elm.json +fi diff --git a/.ci/deps.flawfinder.sh b/.ci/deps.flawfinder.sh new file mode 100755 index 0000000000..32df5db717 --- /dev/null +++ b/.ci/deps.flawfinder.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +set -e -x + +# Change environment for flawfinder from python to python2 +if [ ! -e ~/.local/bin/flawfinder ]; then + cp /usr/bin/flawfinder ~/.local/bin/flawfinder + sed -i '1s/.*/#!\/usr\/bin\/env python2/' ~/.local/bin/flawfinder + chmod +x ~/.local/bin/flawfinder +fi + +.ci/deps.python27.sh + +head ~/.local/bin/flawfinder + +~/.local/bin/flawfinder || true diff --git a/.ci/deps.generic.sh b/.ci/deps.generic.sh new file mode 100755 index 0000000000..e0cbeb80bd --- /dev/null +++ b/.ci/deps.generic.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +set -e -x + +# TODO implement DISABLE_BEARS here +if [ -n "$BEARS" ]; then + for item in $BEARS $BEAR_LIST; do + if [ -f ".ci/deps.$item.sh" ]; then + bash -e -x ".ci/deps.$item.sh" + fi + done +fi diff --git a/.ci/deps.ghc-mod.sh b/.ci/deps.ghc-mod.sh new file mode 100755 index 0000000000..42ef9fa42d --- /dev/null +++ b/.ci/deps.ghc-mod.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +set -e -x +# This file causes ghc-mod to try to install them all +# which fails https://github.com/coala/coala-bears/issues/1384 +rm coala-bears.cabal diff --git a/.ci/deps.ghc-packages.ps1 b/.ci/deps.ghc-packages.ps1 new file mode 100644 index 0000000000..f590fcdf1e --- /dev/null +++ b/.ci/deps.ghc-packages.ps1 @@ -0,0 +1,7 @@ +function Install-Cabal-Deps { + cabal install --only-dependencies --avoid-reinstalls +} + +function Invoke-ExtraInstallation { + Install-Cabal-Deps +} diff --git a/.ci/deps.go.sh b/.ci/deps.go.sh index f42575be59..7d0b582e1e 100755 --- a/.ci/deps.go.sh +++ b/.ci/deps.go.sh @@ -7,3 +7,5 @@ go get -u github.com/alecthomas/gometalinter gometalinter --install go get -u github.com/BurntSushi/toml/cmd/tomlv + +go get -u sourcegraph.com/sqs/goreturns diff --git a/.ci/deps.golang-packages.ps1 b/.ci/deps.golang-packages.ps1 new file mode 100644 index 0000000000..26d102032d --- /dev/null +++ b/.ci/deps.golang-packages.ps1 @@ -0,0 +1,10 @@ +function Install-GoMetalinter-Linters { + gometalinter.v2.exe '--install' +} + +function Invoke-ExtraInstallation { + Install-GoMetalinter-Linters + + go get -u github.com/BurntSushi/toml/cmd/tomlv + go get -u sourcegraph.com/sqs/goreturns +} diff --git a/.ci/deps.golang.ps1 b/.ci/deps.golang.ps1 new file mode 100644 index 0000000000..5adf27ab5b --- /dev/null +++ b/.ci/deps.golang.ps1 @@ -0,0 +1,18 @@ +Set-StrictMode -Version latest + +function Install-GoPM { + go.exe get -u github.com/gpmgo/gopm + go.exe install github.com/gpmgo/gopm +} + +function Install-GoMetaLinter { + go.exe get -u gopkg.in/alecthomas/gometalinter.v2 +} + +function Complete-Install { + Install-GoMetaLinter + + Install-GoPM +} + +Export-ModuleMember -Function Install-GoPM, Install-GoMetaLinter, Complete-Install diff --git a/.ci/deps.infer.sh b/.ci/deps.infer.sh new file mode 100755 index 0000000000..ff2f3d2102 --- /dev/null +++ b/.ci/deps.infer.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +set -e -x + +INFER_URL="https://github.com/facebook/infer/releases/download/v$INFER_VERSION/infer-linux64-v$INFER_VERSION.tar.xz" +curl -sSL "$INFER_URL" | tar -C ~/ -xJ diff --git a/.ci/deps.java.sh b/.ci/deps.java.sh index 236b136d76..7e8d8799d0 100755 --- a/.ci/deps.java.sh +++ b/.ci/deps.java.sh @@ -1,22 +1,17 @@ -#!/bin/sh +#!/bin/bash set -e set -x -# PMD commands -PMD_VERSION=5.4.1 -if [ ! -e ~/.local/bin/pmd ]; then - wget -nc -O ~/pmd.zip https://github.com/pmd/pmd/releases/download/pmd_releases%2F5.4.1/pmd-bin-5.4.1.zip - unzip ~/pmd.zip -d ~/ - cp -r ~/pmd-bin-$PMD_VERSION/* ~/.local/ +if [ -n "$TRAVIS_JDK_VERSION" ]; then + jdk_version=${TRAVIS_JDK_VERSION#openjdk} + jdk_version=${jdk_version#oraclejdk} fi -# Tailor (Swift) commands -# Comment out the hardcoded PREFIX, so we can put it into ~/.local -if [ ! -e ~/.local/tailor/tailor-latest ]; then - curl -fsSL -o install.sh https://tailor.sh/install.sh - sed -i 's/read -r CONTINUE < \/dev\/tty/CONTINUE=y/;;s/^PREFIX.*/# PREFIX=""/;' install.sh - PREFIX=$HOME/.local bash ./install.sh - # Provide a constant path for the executable - ln -s ~/.local/tailor/tailor-* ~/.local/tailor/tailor-latest +if [ -z "$jdk_version" ] || [ $jdk_version -eq 8 ]; then + .ci/deps.tailor.sh +fi + +if [ -z "$(which pmd || true)" ]; then + .ci/deps.pmd.sh fi diff --git a/.ci/deps.julia.jl b/.ci/deps.julia.jl new file mode 100644 index 0000000000..ec2e28fa51 --- /dev/null +++ b/.ci/deps.julia.jl @@ -0,0 +1,7 @@ +if VERSION < v"0.7.0-DEV.5183" + Pkg.clone(pwd(), ENV["JL_PKG"]) + Pkg.build(ENV["JL_PKG"]) +else + using Pkg + Pkg.build() +end diff --git a/.ci/deps.julia.sh b/.ci/deps.julia.sh new file mode 100755 index 0000000000..3608be2e59 --- /dev/null +++ b/.ci/deps.julia.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +set -e -x + +# Normal package management not possible due to +# https://github.com/tonyhffong/Lint.jl/issues/254 +julia -e 'using Pkg; Pkg.add(PackageSpec(url="https://github.com/tonyhffong/Lint.jl", rev="v0.6.0"))' diff --git a/.ci/deps.lua.sh b/.ci/deps.lua.sh new file mode 100755 index 0000000000..7f7ab70029 --- /dev/null +++ b/.ci/deps.lua.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +set -e -x + +luarocks install --local --deps-mode=none luacheck diff --git a/.ci/deps.minimal.sh b/.ci/deps.minimal.sh new file mode 120000 index 0000000000..e78c42d4d0 --- /dev/null +++ b/.ci/deps.minimal.sh @@ -0,0 +1 @@ +deps.generic.sh \ No newline at end of file diff --git a/.ci/deps.node_js.sh b/.ci/deps.node_js.sh new file mode 100755 index 0000000000..81fe68945e --- /dev/null +++ b/.ci/deps.node_js.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e -x + +source .ci/deps.alex.sh + +source .ci/deps.elm.sh diff --git a/.ci/deps.nodejs-packages.ps1 b/.ci/deps.nodejs-packages.ps1 new file mode 100644 index 0000000000..5db9b8d665 --- /dev/null +++ b/.ci/deps.nodejs-packages.ps1 @@ -0,0 +1,33 @@ +function Install-Node-Packages { + npm config set loglevel warn + + # pnpm setting + npm config set reporter silent + + cp -force package.json package.json.bak + + # elm-platform should be added to Fudgefile + # https://github.com/coala/coala-bears/issues/2924 + sed -i '/elm/d' package.json + + # textlint-plugin-asciidoc-loose requires a compiler + # and should be replaced with textlint-plugin-asciidoctor + sed -i '/textlint-plugin-asciidoc-loose/d' package.json + + # If gyp fails, use npm config python to help locate Python 2.7 + + $old_EAP = $ErrorActionPreference + $ErrorActionPreference = 'Continue' + + npm install + + $ErrorActionPreference = $old_EAP + + mv -force package.json.bak package.json + + npm config set reporter default +} + +function Invoke-ExtraInstallation { + Install-Node-Packages +} diff --git a/.ci/deps.npm.sh b/.ci/deps.npm.sh new file mode 100755 index 0000000000..5fdd18ffd2 --- /dev/null +++ b/.ci/deps.npm.sh @@ -0,0 +1 @@ +npm install diff --git a/.ci/deps.objective_c.sh b/.ci/deps.objective_c.sh new file mode 120000 index 0000000000..41a87f2ce0 --- /dev/null +++ b/.ci/deps.objective_c.sh @@ -0,0 +1 @@ +deps.tailor.sh \ No newline at end of file diff --git a/.ci/deps.opam.sh b/.ci/deps.opam.sh index a3f1daa3c6..c4a7a076e5 100755 --- a/.ci/deps.opam.sh +++ b/.ci/deps.opam.sh @@ -3,6 +3,8 @@ set -e set -x +.ci/deps.python27.sh + # Infer commands if [ ! -e ~/infer-linux64-v0.7.0/infer/bin ]; then wget -nc -O ~/infer.tar.xz https://github.com/facebook/infer/releases/download/v0.7.0/infer-linux64-v0.7.0.tar.xz diff --git a/.ci/deps.perl.sh b/.ci/deps.perl.sh new file mode 120000 index 0000000000..5fdd9fa4b3 --- /dev/null +++ b/.ci/deps.perl.sh @@ -0,0 +1 @@ +deps.bakalint.sh \ No newline at end of file diff --git a/.ci/deps.php.ps1 b/.ci/deps.php.ps1 new file mode 100644 index 0000000000..1991fc9314 --- /dev/null +++ b/.ci/deps.php.ps1 @@ -0,0 +1,123 @@ +Set-StrictMode -Version latest + +function Get-PHP-Root { + $PHP_ROOT = '' + Get-ChildItem -Directory 'C:\tools\' -Filter 'php*' | + ForEach-Object { + $PHP_ROOT = $_.FullName.ToString() + + Write-Host "Setting PHP_ROOT='$PHP_ROOT'" + + $env:PHP_ROOT = $PHP_ROOT + Set-ItemProperty -Path 'HKCU:\Environment' -Name 'PHP_ROOT' -Value $PHP_ROOT + } + + if ($PHP_ROOT) { + return $PHP_ROOT + } + + throw ('php not found in ' + $list) +} + +# This is not needed with the recent choco php packages +function Initialize-PHP-Ini { + param ( + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] + $PHP_ROOT + ) + + $PHP_INI = ($PHP_ROOT + '\php.ini') + + Write-Host 'Creating '$PHP_INI + + cp ($PHP_INI + '-production') $PHP_INI + sed -i 's/;date.timezone =.*/date.timezone=UTC/' $PHP_INI + + $list = (Get-ChildItem -Recurse $PHP_ROOT | + Out-String) + + Write-Verbose ('php dir ' + $list) + + Write-Host 'Enabling PHP openssl ...' + + $openssl_dll = '' + + Get-ChildItem $PHP_ROOT -Recurse -filter '*openssl*.dll' | + ForEach-Object { + $openssl_dll = $_.FullName + + Write-Host ' found '$openssl_dll + } + + if (! $openssl_dll) { + Write-Host ' not found' + + throw ('openssl not found in ' + $list) + } + + sed -i 's/;extension=openssl/extension=openssl/' $PHP_INI + + $dir = Split-Path -Path $openssl_dll + + Write-Host 'Setting extension directory: '$dir + + (Get-Content $PHP_INI) | + ForEach-Object { + $_ -replace ';extension_dir *=.*', ('extension_dir="' + $dir + '"') + } | + Set-Content $PHP_INI + + grep '^extension' $PHP_INI +} + +function Install-PEAR { + param( + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] + $PHP_ROOT + ) + + $PHP_INI = "$PHP_ROOT\php.ini" + + grep '^extension' $PHP_INI + + $PHP = "$PHP_ROOT\php.exe" + + Write-Output 'Installing PEAR' + + $pear_install_url = 'http://pear.php.net/install-pear-nozlib.phar' + $phar = $env:TMP + '\install-pear.phar' + + curl -o $phar $pear_install_url + + php.exe $phar -b $PHP_ROOT -d $PHP_ROOT -p $PHP +} + +function Sync-PEAR { + param( + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] + $PHP_ROOT + ) + + Write-Output 'Updating PEAR channel pear.php.net' + + php.exe $PHP_ROOT\pearcmd.php channel-update pear.php.net +} + +function Complete-Install { + $PHP_ROOT = Get-PHP-Root + + Write-Host "PHP_ROOT = $PHP_ROOT" + + $env:PATH = ($env:PATH + ';' + $PHP_ROOT) + + Install-PEAR $PHP_ROOT + Sync-PEAR $PHP_ROOT +} + +Export-ModuleMember -Function Get-PHP-Root, Initialize-PHP-Ini, Install-PEAR, Sync-PEAR diff --git a/.ci/deps.php.sh b/.ci/deps.php.sh new file mode 120000 index 0000000000..1133214314 --- /dev/null +++ b/.ci/deps.php.sh @@ -0,0 +1 @@ +deps.composer.sh \ No newline at end of file diff --git a/.ci/deps.pmd.sh b/.ci/deps.pmd.sh new file mode 100755 index 0000000000..bca89db899 --- /dev/null +++ b/.ci/deps.pmd.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +set -e -x + +# PMD commands +PMD_VERSION=5.4.1 +if [ ! -e ~/.local/bin/pmd ]; then + wget -nc -O ~/pmd.zip https://github.com/pmd/pmd/releases/download/pmd_releases%2F5.4.1/pmd-bin-5.4.1.zip + unzip ~/pmd.zip -d ~/ + cp -r ~/pmd-bin-$PMD_VERSION/* ~/.local/ +fi diff --git a/.ci/deps.pyenv.sh b/.ci/deps.pyenv.sh new file mode 100755 index 0000000000..96092d9f7e --- /dev/null +++ b/.ci/deps.pyenv.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e -x + +if [ -z "$(which pyenv)" ]; then + # This is only useful if sources + export PYENV_ROOT="$HOME/.pyenv"; + export PATH="$PYENV_ROOT/bin:$PATH"; + if [ ! -f "$PYENV_ROOT/bin/pyenv" ]; then + if [ -d "$PYENV_ROOT" ]; then + rm -rf "$PYENV_ROOT" + fi + git clone https://github.com/pyenv/pyenv.git $PYENV_ROOT; + if [ -d ~/local/bin ]; then + rm ~/local/bin/pyenv + (cd ~/local/bin && ln -s $PYENV_ROOT/bin/pyenv .) + fi + fi +fi diff --git a/.ci/deps.python-packages.ps1 b/.ci/deps.python-packages.ps1 new file mode 100644 index 0000000000..7055fe1a52 --- /dev/null +++ b/.ci/deps.python-packages.ps1 @@ -0,0 +1,191 @@ +$ci_directory = $env:FudgeCI + +if (!($ci_directory)) { + $ci_directory = '.ci' +} + +. $ci_directory/constants.ps1 + +function Checkpoint-Pip-Constraints { + python -m pip freeze --all > constraints.txt +} + +function Install-Pip-Requirement { + param ( + [parameter(Mandatory, ValueFromPipeline)] + [string] + $requirement + ) + + if ($requirement.EndsWith('.txt')) { + python -m pip install --constraint constraints.txt --constraint test-requirements.txt -r $requirement + } + else { + python -m pip install --constraint constraints.txt --constraint test-requirements.txt $requirement.Split() + } +} + +function Install-Binary-Packages { + # Install lxml needed by for coala-bears as a wheel as libxml2 and libxslt + # headers and library files are not available, and STATIC_DEPS=true often + # results in linter problems due to different VS compilers and MS runtimes. + # Also use cffi wheel to avoid need for VS compilers + python -m pip install --prefer-binary cffi lxml + # pycparser not a wheel, but ensure it is installable before proceeding + # https://github.com/eliben/pycparser/issues/251 + python -m pip --verbose install pycparser +} + +function Install-coala { + param ( + [string] + $stop_at + ) + + python -m pip install -U six pip==$pip_version setuptools==$setuptools_version + + if (!(Test-Path constraints.txt)) { + if ($stop_at -eq 'coala-bears') { + cp bear-requirements.txt constraints.txt + } + elseif (Test-Path 'requirements.txt') { + cp requirements.txt constraints.txt + } + else { + Checkpoint-Pip-Constraints + } + + # pip bales on encountering VCS or other imprecise requirements + sed -Ei '/(git|hg)+/d' constraints.txt + } + + if (!($stop_at -eq 'PyPrint')) { + Write-Output "Installing PyPrint" + Install-Pip-Requirement 'git+https://gitlab.com/coala/PyPrint#egg=PyPrint' + + if (!($stop_at -eq 'coala_utils')) { + Write-Output "Installing coala_utils" + + Checkpoint-Pip-Constraints + + Install-Pip-Requirement 'git+https://gitlab.com/coala/coala-utils#egg=coala-utils' + + if (!($stop_at -eq 'dependency-management')) { + Write-Output "Installing sarge with Windows support" + + Install-Pip-Requirement 'hg+https://bitbucket.org/jayvdb/sarge@win-reg-lookup#egg=sarge' + + if (!(Test-Path $env:TEMP/pm-master)) { + $PM_URL = "https://gitlab.com/coala/package_manager.git/" + git clone $PM_URL $env:TEMP/pm-master + } + rm $env:TEMP/pm-master/test-requirements.txt + rm $env:TEMP/pm-master/requirements.txt + touch $env:TEMP/pm-master/test-requirements.txt + touch $env:TEMP/pm-master/requirements.txt + + Install-Pip-Requirement "$env:TEMP/pm-master" + + if (!($stop_at -eq 'coala')) { + Write-Output "Installing coala" + + Checkpoint-Pip-Constraints + + Install-Pip-Requirement 'git+https://github.com/coala/coala#egg=coala' + + if (!($stop_at -eq 'coala-bears')) { + Write-Output "Installing coala-bears" + + Checkpoint-Pip-Constraints + + Install-Pip-Requirement 'git+https://github.com/coala/coala-bears#egg=coala-bears' + } + } + } + } + } +} + +function Install-Project-Dependency-Packages { + Write-Output "Installing dependencies of $project_name" + Install-coala $project_name +} + +function Install-Project { + if (Test-Path 'requirements.txt') { + Write-Output "Installing $project_name requirements.txt" + + Install-Pip-Requirement 'requirements.txt' + } + + if (Test-Path 'setup.py') { + Write-Output "Installing $project_name setup.py" + Install-Pip-Requirement '.' + } + + # coala-bears has an ignore.txt for optional dependencies that ordinary + # users may be unable to install. They are needed to reach 100% coverage. + if (Test-Path 'ignore.txt') { + Install-Pip-Requirement 'ignore.txt' + } +} + +function Install-Test-Packages { + if (Test-Path docs-requirements.txt) { + Write-Output "Installing docs-requirements.txt" + + Checkpoint-Pip-Constraints + + Install-Pip-Requirement 'docs-requirements.txt' + } + + Checkpoint-Pip-Constraints + + Write-Output "Installing test-requirements.txt" + + Install-Pip-Requirement 'test-requirements.txt' + Install-Pip-Requirement 'pytest-spec' + + if ($project_name -eq 'coala-bears') { + Write-Output "Installing tox" + # Avoid previous cache entry for setuptools, as it + # causes a deserialisation error + python -m pip install -U --no-cache-dir setuptools + + Checkpoint-Pip-Constraints + + # tox 3.13 uses pluggy 0.12.0 which is incompatible with a pytest 3.6.4 + Install-Pip-Requirement 'tox~=3.12.0 tox-backticks' + } +} + +function Invoke-ExtraInstallation { + + $old_pip_check_flag = 0 + if ($env:PIP_DISABLE_PIP_VERSION_CHECK) { + $old_pip_check_flag = 1 + } + $env:PIP_DISABLE_PIP_VERSION_CHECK = 1 + + Install-Binary-Packages + + if (!($env:PYTHON_VERSION -eq '2.7')) { + Install-Project-Dependency-Packages + } + + Install-Project + + Install-Test-Packages + + if (Test-Path constraints.txt) { + Move-Item constraints.txt $env:TEMP -Force + } + + if (!$old_pip_check_flag) { + Remove-Item Env:\PIP_DISABLE_PIP_VERSION_CHECK + } +} + +$ErrorActionPreference = 'SilentlyContinue'; +Export-ModuleMember -Function Invoke-ExtraInstallation -ErrorAction:Ignore +$ErrorActionPreference = 'Continue'; diff --git a/.ci/deps.python.ps1 b/.ci/deps.python.ps1 new file mode 100644 index 0000000000..e8f63683dc --- /dev/null +++ b/.ci/deps.python.ps1 @@ -0,0 +1,44 @@ +Set-StrictMode -Version latest + +function Add-EnvPythonVersion { + if ($env:TRAVIS -and $env:TRAVIS_PYTHON_VERSION) { + $env:PYTHON_VERSION = $env:TRAVIS_PYTHON_VERSION + } + else { + $env:PYTHON_VERSION = python -c 'import sys; print(sys.version[0:3])' + } +} + +function Add-EnvPythonMinorDotless { + if (!($env:PYTHON_MINOR_NODOTS)) { + $python_minor = $env:PYTHON_VERSION.Substring(0, 3) + + $env:PYTHON_MINOR_NODOTS = $python_minor -replace '.', '' + } +} + +function Add-PATHPythonRoaming { + $roaming_home = ( + $env:APPDATA + '/Python/Python' + $env:PYTHON_MINOR_NODOTS) + + Install-ChocolateyPath -PathToInstall $roaming_home + Install-ChocolateyPath -PathToInstall ($roaming_home + '/Scripts') +} + +function Add-EnvPipNonEagerUpgradeStrategy { + $env:PIP_UPGRADE_STRATEGY = 'only-if-needed' + + Set-ItemProperty -Path 'HKCU:\Environment' -Name 'PIP_UPGRADE_STRATEGY' -Value $env:PIP_UPGRADE_STRATEGY +} + +function Complete-Install { + Add-EnvPythonVersion + + Add-EnvPythonMinorDotless + + Add-EnvPipNonEagerUpgradeStrategy + + Add-PATHPythonRoaming +} + +Export-ModuleMember -Function Complete-Install diff --git a/.ci/deps.python27.sh b/.ci/deps.python27.sh new file mode 100755 index 0000000000..865c43f396 --- /dev/null +++ b/.ci/deps.python27.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +set -e -x + +pyenv global 2.7 $(pyenv versions --bare | fgrep '3.6' --max-count 1) +hash -r diff --git a/.ci/deps.python36.sh b/.ci/deps.python36.sh new file mode 100755 index 0000000000..5e264bf5b9 --- /dev/null +++ b/.ci/deps.python36.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -e -x + +source .ci/deps.pyenv.sh + +PYTHON36_BIN="$(which python3.6 || true)" +if [ -n "$PYTHON36_BIN" ]; then + if [ ! -d "$PYENV_ROOT/plugins/pyenv-register" ]; then + # https://github.com/doloopwhile/pyenv-register/pull/3 + git clone https://github.com/garyp/pyenv-register.git \ + "$PYENV_ROOT/plugins/pyenv-register" + fi + + pyenv register -f "$PYTHON36_BIN" || true +fi + +PYTHON36_VERSION=$(pyenv versions --bare | fgrep '3.6' --max-count 1 || true) + +if [ -z "$PYTHON36_VERSION" ]; then + PYTHON36_VERSION=3.6.3 + + pyenv install "$PYTHON36_VERSION"; +fi + +pyenv global "$PYTHON36_VERSION" + +hash -r diff --git a/.ci/deps.r.sh b/.ci/deps.r.sh deleted file mode 100755 index 8e32ff8057..0000000000 --- a/.ci/deps.r.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -set -e -set -x - -# R commands -echo '.libPaths( c( "'"$R_LIB_USER"'", .libPaths()) )' >> .Rprofile -echo 'options(repos=structure(c(CRAN="http://cran.rstudio.com")))' >> .Rprofile -R -q -e 'install.packages("lintr")' -R -q -e 'install.packages("formatR")' diff --git a/.ci/deps.ruby-packages.ps1 b/.ci/deps.ruby-packages.ps1 new file mode 100644 index 0000000000..6c987cf0bf --- /dev/null +++ b/.ci/deps.ruby-packages.ps1 @@ -0,0 +1,30 @@ +function Install-Gemfile { + cp -force Gemfile Gemfile.orig + cp -force Gemfile Gemfile.win + + # Unbuildable on Windows + sed -i '/sqlint/d' Gemfile.win + # https://github.com/coala/coala-bears/issues/2909 + sed -i '/csvlint/d' Gemfile.win + + # pusher-client 0.4.0 doesnt depend on json, which requires + # a compiler and the GMP library + Write-Output 'gem "pusher-client", "~>0.4.0", require: false' | + Out-File -FilePath Gemfile.win -Append -Encoding ascii + + cp -force Gemfile.win Gemfile + + # The build crawls if DevKit is included in the PATH + $old_PATH = $env:PATH + $env:PATH = ($env:ChocolateyToolsLocation + '\DevKit2\bin;' + $env:PATH) + + bundle install + + $env:PATH = $old_PATH + + mv -force Gemfile.orig Gemfile +} + +function Invoke-ExtraInstallation { + Install-Gemfile +} diff --git a/.ci/deps.ruby.ps1 b/.ci/deps.ruby.ps1 new file mode 100644 index 0000000000..429f970475 --- /dev/null +++ b/.ci/deps.ruby.ps1 @@ -0,0 +1,11 @@ +Set-StrictMode -Version latest + +function Install-Bundler { + gem install bundler +} + +function Complete-Install { + Install-Bundler +} + +Export-ModuleMember -Function Complete-Install diff --git a/.ci/deps.ruby.sh b/.ci/deps.ruby.sh new file mode 100755 index 0000000000..8454830ed7 --- /dev/null +++ b/.ci/deps.ruby.sh @@ -0,0 +1,15 @@ +# Remove Ruby directive from Gemfile as we test many versions +sed -i.bak '/^ruby/d' Gemfile + +if [ "$TRAVIS_RUBY_VERSION" != "2.1" ]; then + gem update --system +fi + +# Install maximum version of bundler for each Ruby version +if [ "$TRAVIS_RUBY_VERSION" = "2.1" ]; then + gem install bundler -v 1.16.1 +elif [ "$TRAVIS_RUBY_VERSION" = "2.2" ]; then + gem install bundler -v 1.17.3 +else + gem install bundler +fi diff --git a/.ci/deps.sh b/.ci/deps.sh deleted file mode 100755 index 7722a69ea5..0000000000 --- a/.ci/deps.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash - -set -e -set -x - -# NPM commands -ALEX=$(which alex || true) -# Delete 'alex' if it is not in a node_modules directory, -# which means it is ghc-alex. -if [[ -n "$ALEX" && "${ALEX/node_modules/}" == "${ALEX}" ]]; then - echo "Removing $ALEX" - sudo rm -rf $ALEX -fi -npm install -npm list --depth=0 - -# Ruby commands -bundle install --path=vendor/bundle --binstubs=vendor/bin --jobs=8 --retry=3 - -# Dart Lint commands -if ! dartanalyzer -v &> /dev/null; then - wget -nc -O ~/dart-sdk.zip https://storage.googleapis.com/dart-archive/channels/stable/release/1.14.2/sdk/dartsdk-linux-x64-release.zip - unzip -n ~/dart-sdk.zip -d ~/ - cp -rp ~/dart-sdk/* ~/.local/ -fi - -# VHDL Bakalint Installation -if [ ! -e ~/.local/bin/bakalint.pl ]; then - BAKALINT_VERSION=0.4.0 - wget "http://downloads.sourceforge.net/project/fpgalibre/bakalint/0.4.0/bakalint-0.4.0.tar.gz?r=https%3A%2F%2Fsourceforge.net%2Fprojects%2Ffpgalibre%2Ffiles%2Fbakalint%2F0.4.0%2F&ts=1461844926&use_mirror=netcologne" -O ~/bl.tar.gz - tar xf ~/bl.tar.gz -C ~/ - mv ~/bakalint-$BAKALINT_VERSION/bakalint.pl ~/.local/bin/ -fi - -# elm-format Installation -if [ ! -e ~/.local/bin/elm-format ]; then - curl -fsSL -o elm-format.tgz https://github.com/avh4/elm-format/releases/download/0.5.2-alpha/elm-format-0.17-0.5.2-alpha-linux-x64.tgz - tar -xvzf elm-format.tgz -C ~/.local/bin/ -fi - -# Julia commands -julia -e "Pkg.add(\"Lint\")" - -# Lua commands -luarocks install --local --deps-mode=none luacheck - -# PHPMD installation -if [ ! -e ~/.local/bin/phpmd ]; then - PHPMD='http://static.phpmd.org/php/latest/phpmd.phar' - mkdir -p ~/.local/bin - curl -fsSL -o ~/.local/bin/phpmd "$PHPMD" - chmod +x ~/.local/bin/phpmd -fi - -# Change environment for flawfinder from python to python2 -if [ ! -e ~/.local/bin/flawfinder ]; then - cp /usr/bin/flawfinder ~/.local/bin/flawfinder - sed -i '1s/.*/#!\/usr\/bin\/env python2/' ~/.local/bin/flawfinder - chmod +x ~/.local/bin/flawfinder -fi diff --git a/.ci/deps.tailor.sh b/.ci/deps.tailor.sh new file mode 100755 index 0000000000..2c681ab887 --- /dev/null +++ b/.ci/deps.tailor.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +set -e +set -x + +# Tailor (Swift) commands +# Comment out the hardcoded PREFIX, so we can put it into ~/.local +if [ ! -e ~/.local/tailor/tailor-latest ]; then + curl -fsSL -o install.sh https://tailor.sh/install.sh + sed -i 's/read -r CONTINUE < \/dev\/tty/CONTINUE=y/;;s/^PREFIX.*/# PREFIX=""/;' install.sh + PREFIX=$HOME/.local bash ./install.sh + # Provide a constant path for the executable + ln -s ~/.local/tailor/tailor-* ~/.local/tailor/tailor-latest +fi diff --git a/.ci/generate_coverage_thresholds.py b/.ci/generate_coverage_thresholds.py new file mode 100755 index 0000000000..0d5bb8d814 --- /dev/null +++ b/.ci/generate_coverage_thresholds.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 + +import json +import os +import sys + +IS_WIN = os.name == 'nt' + + +def main(): + args = sys.argv[1:] + thresholds = {} + if args == ['none']: + if os.path.exists(".threshold.json"): + os.remove(".threshold.json") + return + + for test in args: + bear = test.replace('tests/', 'bears/') + bear = bear.replace('Test.py', '.py').replace('*', '.*') + + threshold = 100 + if IS_WIN: + bear = bear.replace('/', '\\\\') + if 'CheckstyleBear' in bear or 'CMakeLintBear' in bear: + threshold = 90 + + thresholds[bear] = threshold + + with open('.threshold.json', 'w') as f: + json.dump(thresholds, f) + + +if __name__ == '__main__': + main() diff --git a/.ci/get_bears.py b/.ci/get_bears.py new file mode 100755 index 0000000000..7bdb67093e --- /dev/null +++ b/.ci/get_bears.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 + +import glob +import os +import os.path +import sys + +THIS_DIR = os.path.dirname(os.path.abspath(__file__)) + +PROJECT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) + +PROJECT_BEAR_DIR = os.path.abspath(os.path.join(PROJECT_DIR, 'bears')) + + +def main(): + args = sys.argv[1:] + do_missing = do_all = False + if args[0] == '--all': + do_all = True + args = args[1:] + elif args[0] == '--missing': + do_missing = True + args = args[1:] + + if do_all or do_missing: + all_bears = glob.glob('{}/**/*.py'.format(PROJECT_BEAR_DIR)) + all_bears = [ + bear[len(PROJECT_DIR) + 1:].replace(os.path.sep, '/') + for bear in all_bears + if not bear.endswith('__init__.py') + ] + if do_all: + print(' '.join(sorted(all_bears))) + return + + all_bears = set(all_bears) + + bears = set() + + for arg in args: + if arg.startswith('tests/'): + bear = arg.replace('tests/', 'bears/') + bear = bear[:bear.find('Test')] + '.py' + else: + bear = arg + bears.add(bear) + + if do_missing: + bears = all_bears - bears + + print(' '.join(sorted(bears))) + + +if __name__ == '__main__': + main() diff --git a/.ci/get_codecov_tags.py b/.ci/get_codecov_tags.py new file mode 100755 index 0000000000..d3b5edc768 --- /dev/null +++ b/.ci/get_codecov_tags.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 + +import sys + +# Tags like list, check and collectonly shouldnt appear on codecov +# but they also shouldnt be submitted to codecov, so they are not +# removed here as that would hide a bug in tox.ini +REJECT_TAGS = set(['codecov', 'skip', 'noskip']) + + +def main(): + env_factors = set(sys.argv[1].split('-')) + + print(','.join(sorted(env_factors - REJECT_TAGS))) + + +if __name__ == '__main__': + main() diff --git a/.ci/get_cov_args.py b/.ci/get_cov_args.py new file mode 100755 index 0000000000..2331a3f7d7 --- /dev/null +++ b/.ci/get_cov_args.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 + +import sys + + +def main(): + args = sys.argv[1:] + + bears = set() + + for test in args: + bear = test.replace('tests/', 'bears/') + bear = bear[:bear.find('Test')] + '.py' + bears.add(bear) + + print('--cov=' + ' --cov='.join(sorted(bears))) + + +if __name__ == '__main__': + main() diff --git a/.ci/get_tests.py b/.ci/get_tests.py new file mode 100755 index 0000000000..1b46f9fae0 --- /dev/null +++ b/.ci/get_tests.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 + +import glob +import os +import os.path +import sys + +from ruamel.yaml import YAML + +yaml = YAML() +THIS_DIR = os.path.dirname(os.path.abspath(__file__)) + +PROJECT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) + +IS_WIN = os.name == 'nt' + +# Clang DLLs x64 were nowadays installed, but the x64 version hangs, so we +# exclude according tests. See https://github.com/appveyor/ci/issues/495 and +# https://github.com/appveyor/ci/issues/688 + +WINDOWS_BROKEN = set(( + 'bakalint', # not installed + 'phpcs', # https://github.com/coala/coala-bears/issues/2916 + 'pmd', 'cpd', # https://github.com/coala/coala-bears/issues/2908 + 'mcs', # choco mono isnt providing this in the PATH + 'tailor', # installer fails + 'shellcheck', # https://github.com/coala/coala-bears/issues/2920 + # pip + 'apertium_lint', # https://gitlab.com/jpsinghgoud/apertium-lint/issues/5 + 'bandit', # RuntimeError: Unable to output report using 'json' formatter + 'clang', # see note above + 'cppclean', # https://github.com/myint/cppclean/issues/120 + 'scspell', # https://github.com/coala/coala-bears/issues/2926 + 'vint', # https://github.com/Kuniwak/vint/issues/290 + # gem + 'csvlint', # https://github.com/coala/coala-bears/issues/2909 + 'sqlint', # https://github.com/coala/coala-bears/issues/2923 + # npm ; try different version + 'alex', # https://github.com/coala/coala-bears/issues/2922 + 'coffeelint', # Extra windows results + 'csscomb', # Linter errors + 'dockerfile_lint', # test case bug + 'elm-format', # https://github.com/coala/coala-bears/issues/2925 + 'gherkin', # result json decode exception + 'jshint', # test case bug + 'remark', # remark result text difference due to unicode + 'postcss', # https://github.com/coala/coala-bears/issues/2921 + 'sass-lint', # rule `!important not allowed` not trigger + 'textlint', # Unexpected extra result in test + # Also textlint plugin for asciidoc requires a compiler. + # and should be replaced with plugin asciidoctor which does + # not need a compiler + + # No information from linter bear + 'eslint', # Two of tests fail + 'tslint', # Half of tests fail +)) + + +DISABLE_BEARS = set(os.environ.get('DISABLE_BEARS', '').split(' ')) + + +def get_metadata(): + with open('bear-metadata.yaml') as f: + metadata = yaml.load(f) + + return metadata + + +def get_bears(metadata, args, include_disabled=False): + bears = [] + + for arg in args: + for bear in metadata.values(): + tags = set(bear['tags']) + + if tags.intersection(DISABLE_BEARS): + tags.add('disabled') + + if IS_WIN and tags.intersection(WINDOWS_BROKEN): + tags.add('disabled') + + if arg in tags and (include_disabled or 'disabled' not in tags): + bears.append(bear) + + return bears + + +CLANG_EXTRA_TESTS = [ + 'tests/c_languages/codeclone_detection/ClangCountingConditionsTest.py', + 'tests/c_languages/codeclone_detection/ClangCountVectorCreatorTest.py', + 'tests/c_languages/codeclone_detection/CountVectorTest.py', + 'tests/c_languages/codeclone_detection/CloneDetectionRoutinesTest.py', +] + + +def get_tests(bears): + # Add 1 for the path separator after bears + project_dir_prefix_len = len(PROJECT_DIR) + 1 + + tests = set() + for bear in bears: + name = bear['name'] + if name.startswith('_'): + continue + subdir = bear['subdir'] + # A few test modules are FoobearSomethingTest.py, like + # PySafetyBearWithoutMockTest.py + testpath = os.path.join('tests', subdir, '{}*Test.py'.format(name)) + files = glob.glob(testpath) + for filename in files: + filename = filename.replace(os.path.sep, '/') + if filename.startswith('/'): + filename = filename[project_dir_prefix_len:] + tests.add(filename) + + if subdir == 'c_languages/codeclone_detection': + tests.update(CLANG_EXTRA_TESTS) + + elif subdir.startswith('vcs'): + tests.add('tests/vcs/CommitBearTest.py') + + return tests + + +def main(): + args_orig = sys.argv[1:] + metadata = get_metadata() + + include_disabled = False + if args_orig[0] == '--disabled': + include_disabled = True + args_orig = args_orig[1:] + + args = [] + for arg in args_orig: + if arg in ['ghc-mod', 'default-jre']: + args += [arg] + continue + args += arg.split('-') + + if 'java7' in args or 'java8' in args: + args.append('java') + + if 'pip' in args: + args.append('noreqs') + + # TODO: pass through any args which are literal test filenames + + bears = get_bears(metadata, args, include_disabled) + tests = get_tests(bears) + print(' '.join(sorted(tests))) + + +if __name__ == '__main__': + main() diff --git a/.ci/refreshenv.sh b/.ci/refreshenv.sh new file mode 100644 index 0000000000..49701d4348 --- /dev/null +++ b/.ci/refreshenv.sh @@ -0,0 +1,29 @@ +# shellcheck disable=SC2059,SC2154 +# as shellcheck believes the $ in the heredoc are shell variables + +function refreshenv +{ + powershell -NonInteractive - <<\EOF +Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" + +Update-SessionEnvironment + +# Round brackets in variable names cause problems with bash +Get-ChildItem env:* | %{ + if (!($_.Name.Contains('('))) { + $value = $_.Value + if ($_.Name -eq 'PATH') { + $value = $value -replace ';',':' + } + Write-Output ("export " + $_.Name + "='" + $value + "'") + } +} | Out-File -Encoding ascii $env:TEMP\refreshenv.sh + +EOF + + # shellcheck disable=SC1090 + # as shellcheck can not follow this `source` + source "$TEMP/refreshenv.sh" +} + +alias RefreshEnv=refreshenv diff --git a/.ci/run_with_env.cmd b/.ci/run_with_env.cmd deleted file mode 100644 index da8e020a54..0000000000 --- a/.ci/run_with_env.cmd +++ /dev/null @@ -1,88 +0,0 @@ -:: To build extensions for 64 bit Python 3, we need to configure environment -:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of: -:: MS Windows SDK for Windows 7 and .NET Framework 4 (SDK v7.1) -:: -:: To build extensions for 64 bit Python 2, we need to configure environment -:: variables to use the MSVC 2008 C++ compilers from GRMSDKX_EN_DVD.iso of: -:: MS Windows SDK for Windows 7 and .NET Framework 3.5 (SDK v7.0) -:: -:: 32 bit builds, and 64-bit builds for 3.5 and beyond, do not require specific -:: environment configurations. -:: -:: Note: this script needs to be run with the /E:ON and /V:ON flags for the -:: cmd interpreter, at least for (SDK v7.0) -:: -:: More details at: -:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows -:: http://stackoverflow.com/a/13751649/163740 -:: -:: Author: Olivier Grisel -:: License: CC0 1.0 Universal: https://creativecommons.org/publicdomain/zero/1.0/ -:: -:: Notes about batch files for Python people: -:: -:: Quotes in values are literally part of the values: -:: SET FOO="bar" -:: FOO is now five characters long: " b a r " -:: If you don't want quotes, don't include them on the right-hand side. -:: -:: The CALL lines at the end of this file look redundant, but if you move them -:: outside of the IF clauses, they do not run properly in the SET_SDK_64==Y -:: case, I don't know why. -@ECHO OFF - -SET COMMAND_TO_RUN=%* -SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows -SET WIN_WDK=c:\Program Files (x86)\Windows Kits\10\Include\wdf - -:: Extract the major and minor versions, and allow for the minor version to be -:: more than 9. This requires the version number to have two dots in it. -SET MAJOR_PYTHON_VERSION=%PYTHON_VERSION:~0,1% -IF "%PYTHON_VERSION:~3,1%" == "." ( - SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,1% -) ELSE ( - SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,2% -) - -:: Based on the Python version, determine what SDK version to use, and whether -:: to set the SDK for 64-bit. -IF %MAJOR_PYTHON_VERSION% == 2 ( - SET WINDOWS_SDK_VERSION="v7.0" - SET SET_SDK_64=Y -) ELSE ( - IF %MAJOR_PYTHON_VERSION% == 3 ( - SET WINDOWS_SDK_VERSION="v7.1" - IF %MINOR_PYTHON_VERSION% LEQ 4 ( - SET SET_SDK_64=Y - ) ELSE ( - SET SET_SDK_64=N - IF EXIST "%WIN_WDK%" ( - :: See: https://connect.microsoft.com/VisualStudio/feedback/details/1610302/ - REN "%WIN_WDK%" 0wdf - ) - ) - ) ELSE ( - ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%" - EXIT 1 - ) -) - -IF %PYTHON_ARCH% == 64 ( - IF %SET_SDK_64% == Y ( - ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture - SET DISTUTILS_USE_SDK=1 - SET MSSdk=1 - "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION% - "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release - ECHO Executing: %COMMAND_TO_RUN% - call %COMMAND_TO_RUN% || EXIT 1 - ) ELSE ( - ECHO Using default MSVC build environment for 64 bit architecture - ECHO Executing: %COMMAND_TO_RUN% - call %COMMAND_TO_RUN% || EXIT 1 - ) -) ELSE ( - ECHO Using default MSVC build environment for 32 bit architecture - ECHO Executing: %COMMAND_TO_RUN% - call %COMMAND_TO_RUN% || EXIT 1 -) diff --git a/.ci/store_env_in_registry.py b/.ci/store_env_in_registry.py new file mode 100755 index 0000000000..6f766d5c03 --- /dev/null +++ b/.ci/store_env_in_registry.py @@ -0,0 +1,85 @@ +import os + +KEY = 'System\\CurrentControlSet\\Control\\Session Manager\\Environment' + +DISCARD_KEYWORDS = tuple([ + 'awscli', + 'azure', + 'coverity', + 'dnvm', + 'mspec', + 'nunit', + 'odbc', + 'privateassemblies', + 'python27', + 'ruby193', + 'service fabric', + 'sql', + 'subversion', + 'testwindow', + 'xunit', +]) + +# TODOs: +# - Also get raw unexpanded values from registry to reduce length, and +# merge them with the current PATH which AppVeyor has populated +# - also fetch and filter user env vars, also de-duplicate wrt system vars +# (see https://github.com/reider-roque/pathvar) +# - Convert to Windows path names, not /c/foo/bar +# - Replace \\ and \.\ with \ + + +def get_tidy_path(original): + parts = [] + dups = set() + discard_matches = set() + + for part in original.split(';'): + # This will break directories with a trailing space + part = part.strip().rstrip('\\') + if part in parts: + dups.add(part) + continue + + part_lower = part.lower() + for word in DISCARD_KEYWORDS: + if word in part_lower: + discard_matches.add(word) + break + else: + parts.append(part) + + if dups: + print('Discarded dups:\n {}'.format('\n '.join(sorted(dups)))) + + if discard_matches: + print('Discarded keyword matches: ' + '{}'.format(', '.join(sorted(discard_matches)))) + + return ';'.join(parts) + + +def set_envvar_in_registry(envvar, value): + try: + import winreg + except ImportError: + import _winreg as winreg + + reg = winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) + with winreg.OpenKey(reg, KEY, 0, winreg.KEY_ALL_ACCESS) as regkey: + winreg.SetValueEx(regkey, envvar, 0, winreg.REG_EXPAND_SZ, value) + + +def do_all_environ(): + for key, value in os.environ.items(): + if key.upper() in ['PWD', 'OLDPWD', 'CWD']: + continue + + if key.upper() in ['PATH', 'PSMODULEPATH']: + value = get_tidy_path(value) + print('%s (len %d) set to:\n%s' % (key, len(value), value)) + set_envvar_in_registry(key, value) + + +if __name__ == '__main__': + do_all_environ() diff --git a/.ci/travis_extra_globals.sh b/.ci/travis_extra_globals.sh new file mode 100644 index 0000000000..219fb6c730 --- /dev/null +++ b/.ci/travis_extra_globals.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# This must be `source`d, and kept very basic to avoid breaking travis shell + +# Uses tox.ini selectors so that multiple languages can be handled in one job +if [ -z "$TOX_TEST_SELECTORS" ] && [ "$TRAVIS_LANGUAGE" ]; then + if [ "$TRAVIS_LANGUAGE" = "ruby" ]; then + TOX_TEST_SELECTORS=gem + elif [ "$TRAVIS_LANGUAGE" = "node_js" ]; then + TOX_TEST_SELECTORS=npm + elif [ "${BEARS/lua/}" != "$BEARS" ]; then + TOX_TEST_SELECTORS="$BEARS" + else + TOX_TEST_SELECTORS="$TRAVIS_LANGUAGE" + fi + export TOX_TEST_SELECTORS +fi + +if [ "${TOX_TEST_SELECTORS/gem/}" != "$TOX_TEST_SELECTORS" ]; then + # https://travis-ci.community/t/bundle-path-disappears/4260 + export BUNDLE_PATH="$TRAVIS_BUILD_DIR/vendor/bundle" + export BUNDLE_BIN="$BUNDLE_PATH/bin" + EXTRA_PATH="$EXTRA_PATH:$BUNDLE_BIN" +fi + +if [ "${TOX_TEST_SELECTORS/java/}" != "$TOX_TEST_SELECTORS" ]; then + EXTRA_PATH="$EXTRA_PATH:$HOME/.local/tailor/tailor-latest/bin" +fi + +if [ "${TOX_TEST_SELECTORS/php/}" != "$TOX_TEST_SELECTORS" ]; then + EXTRA_PATH="$EXTRA_PATH:$TRAVIS_BUILD_DIR/vendor/bin" +fi + +if [ "${TOX_TEST_SELECTORS/npm/}" != "$TOX_TEST_SELECTORS" ]; then + # Travis adds relative ./node_modules/.bin , but some tests change directory + EXTRA_PATH="$EXTRA_PATH:$TRAVIS_BUILD_DIR/node_modules/.bin" +fi + +if [ "${TOX_TEST_SELECTORS/lua/}" != "$TOX_TEST_SELECTORS" ]; then + EXTRA_PATH="$EXTRA_PATH:$HOME/.luarocks/bin" +fi + +# Remove leading colons +EXTRA_PATH="${EXTRA_PATH##:}" + +if [ "$EXTRA_PATH" != "" ]; then + echo "EXTRA_PATH=$EXTRA_PATH" + export PATH="$PATH:$EXTRA_PATH" +fi diff --git a/.ci/travis_init.ps1 b/.ci/travis_init.ps1 new file mode 100644 index 0000000000..40b36d6682 --- /dev/null +++ b/.ci/travis_init.ps1 @@ -0,0 +1,5 @@ +$PSVersionTable + +Set-MpPreference -DisableRealtimeMonitoring $true +Set-MpPreference -DisableArchiveScanning $true +Set-MpPreference -DisableBehaviorMonitoring $true diff --git a/.coafile b/.coafile index f78c659676..ca7e004baa 100644 --- a/.coafile +++ b/.coafile @@ -1,7 +1,8 @@ [all] files = *.py, bears/**/*.py, tests/**/*.py, .moban.dt/*.py.in ignore = tests/python/test_files/pylint_test.py, tests/python/bandit_test_files/*, - tests/python/vulture_test_files/* + tests/python/vulture_test_files/*, + .ci/Fudge*.ps* max_line_length = 80 use_spaces = True @@ -70,6 +71,7 @@ ignore = *.py [all.yml] bears = YAMLLintBear files = *.yml, *.yaml, .ci/*.yml, tests/**/*.yml +ignore = MYMETA.yml [bash] bears = ShellCheckBear diff --git a/.moban.dt/bears-appveyor.yml.jj2 b/.moban.dt/bears-appveyor.yml.jj2 new file mode 100644 index 0000000000..bc8a95c9d8 --- /dev/null +++ b/.moban.dt/bears-appveyor.yml.jj2 @@ -0,0 +1,10 @@ +{% extends 'ci/appveyor.yml.jj2' %} + +{% set _ = appveyor_global_environment.__setitem__( + 'TOX_FEATURES', '-'.join(tox.features)) %} +{% set _ = appveyor_global_environment.__setitem__( + 'BEAR_LIST', ' '.join(tox.win.extra_bears)) %} +{% set _ = appveyor_global_environment.__setitem__( + 'TOX_TEST_SELECTORS', '-'.join(tox.win.selectors)) %} +{% set _ = appveyor_global_environment.__setitem__( + 'TOXENV', 'py$(PYTHON_MINOR_NODOTS)-$(TOX_TEST_SELECTORS)-$(TOX_FEATURES)-win') %} diff --git a/.moban.dt/bears-test-requirements.txt.jj2 b/.moban.dt/bears-test-requirements.txt.jj2 index f35988528c..4bbc504435 100644 --- a/.moban.dt/bears-test-requirements.txt.jj2 +++ b/.moban.dt/bears-test-requirements.txt.jj2 @@ -1,2 +1,9 @@ {% include 'test-requirements.txt.jj2' %} +pbr!=2.1.0,>=2.0.0 +pytest-error-for-skips +git+https://github.com/krkd/pytest-cov-threshold#egg=pytest-cov-threshold ; python_version > '3.4' twine~=1.7.4 +tox~=3.12.0 +tox-travis +tox-backticks +tox-pyenv diff --git a/.moban.dt/bears-travis.yml.jj2 b/.moban.dt/bears-travis.yml.jj2 new file mode 100644 index 0000000000..86da97e13b --- /dev/null +++ b/.moban.dt/bears-travis.yml.jj2 @@ -0,0 +1,251 @@ +{% extends 'ci/travis.yml.jj2' %} + +{% block custom_python_versions %} +python: 3.6.3 +{% endblock %} + +{% block stages %} +stages: + - name: sentinel + if: branch != master OR type = pull_request + - test-languages + - test-other-versions + - moban + +{% endblock %} + +{% block jobs %} +{% macro job(language, version, seen) -%} +{% if language == 'python' and version.startswith('3.5') %} +{% set seen = False %} +{% endif %} +{% if language == 'python' and version.startswith('3.6') %} + - stage: sentinel +{% elif not seen %} + - stage: test-languages +{% else %} + - stage: test-other-versions +{% endif %} +{% set env = {} %} +{% set version_key = language %} +{% set dist = None %} +{% set cacher = False %} +{% if language == 'lua' %} +{% set env = {'BEARS': 'lua'} %} +{% set version_key = None %} +{% elif language == 'infer' %} +{% set version_key = None %} +{% set env = { + 'BEARS': 'opam', + 'INFER_VERSION': version, + 'PATH': '$PATH:$HOME/infer-linux64-v$INFER_VERSION/infer/bin', + } %} +{% set dist = 'trusty' %} +{% elif language == 'ruby' %} +{% set dist = 'trusty' %} +{% set version_key = 'rvm' %} +{% set cacher = 'bundler' %} +{% elif language == 'python' %} +{# clang-3.4 is not available on xenial #} +{% set dist = 'trusty' %} +{% set cacher = 'pip' %} +{% elif language == 'perl' %} +{% set dist = 'trusty' %} +{% elif language == 'php' %} +{% set dist = 'trusty' %} +{% elif language == 'r' %} +{% set cacher = 'packages' %} +{% elif language == 'node_js' %} +{% set cacher = 'npm' %} +{% elif language == 'java' %} +{% set version_key = 'jdk' %} +{% elif language == 'mono' %} +{% set language = 'csharp' %} +{% endif %} +{% if dist %} + dist: {{ dist }} +{% endif %} +{% if language in ['lua', 'infer'] %} + language: minimal +{% else %} + language: {{ language }} +{% endif %} +{% if version_key %} +{% if version.endswith('0') and not version.endswith('.0') %} + {{ version_key }}: '{{ version }}' +{% else %} + {{ version_key }}: {{ version }} +{% endif %} +{% endif %} +{% if env %} + env: +{% for name, value in env.items() %} + {{ name }}="{{ value }}" +{% endfor %} +{% endif %} +{% if cacher %} + cache: {{ cacher }} +{% endif %} +{% if language == 'python' %} + addons: + apt: + packages: clang-3.4 +{% elif language == 'go' %} + install: skip + cache: + directories: + - $HOME/.cache/go-build + - $HOME/gopath/pkg/mod +{% elif language == 'lua' %} + cache: + directories: + - $HOME/.luarocks +{% elif language == 'perl' %} + cache: + directories: + - $HOME/perl5 +{% elif language == 'scala' and version.startswith('2.12') %} + jdk: openjdk8 +{% elif language == 'julia' %} +{# default language is 0.6.4, and has problems with pre-installed Lint.jl #} +{# install not provided https://github.com/travis-ci/travis-build/pull/1571 #} + env: JL_PKG={{ appveyor_global_environment.JL_PKG }} + install: + - julia --color=yes .ci/deps.julia.jl + # Verify compilation works + - julia -e 'import Lint.lintfile' + cache: + directories: + - $HOME/.julia +{% elif language == 'infer' %} + addons: + apt: + sources: + - avsm + packages: + - camlp4-extra + - ocaml + - opam +{% endif %} +{% endmacro %} +{# reorder so python is first on Python 3.6 #} +{% set _supported_versions = {'python': python_versions} %} +{% set _ = _supported_versions.update(supported_versions) %} +{% set supported_versions = _supported_versions %} +jobs: + include: + # Manually added "language" entries should complete test coverage +{% macro manualmatrix() %}{% include 'travis-manual-matrix.yml' %}{% endmacro %} + {{ manualmatrix() | indent(4) }} + # Entries generates from `supported_versions` +{% set seen_languages = {} %} +{% for language, versions in supported_versions.items() %} +{% for version in versions %} +{% set version = ''.__class__(version) %} +{{ job(language, version, language in seen_languages) }} +{% set _ = seen_languages.__setitem__(language, 1) %} +{% endfor %} +{% endfor %} + - *moban + allow_failures: + - *moban + +cache: + pip: true + directories: + - docs/_build + # Installed language package caches + - ~/.cabal + - ~/.ghc + - ~/.ghc-mod + - $TRAVIS_BUILD_DIR/vendor + # coala managed data + - ~/nltk_data + # Installed linters + - ~/infer-linux64-v$INFER_VERSION + - ~/.local/ + +env: + global: + - TERM=dumb + - PATH="$HOME/.local/bin:$PATH" + # These are only needed by Windows + - NUGET_EXE_NO_PROMPT=true + - VIRTUALENV_NO_DOWNLOAD=1 + # Enable to debug tox + # VIRTUALENV_VERBOSE=1 + # This exceeds the travis maximum log length + # PIP_VERBOSE=1 + - PIP_DISABLE_PIP_VERSION_CHECK=1 + - PIP_YES=1 + - FudgeCI=${TRAVIS_BUILD_DIR}/.ci/ + - TOX_FEATURES={{ '-'.join(tox.features) }} +{% endblock %} + +{% block before_install %} +before_install: + - printenv + - mkdir -p ~/bin ~/.local/bin + - source .ci/travis_extra_globals.sh + + - if [ -z "$TRAVIS_PYTHON_VERSION" ]; then + .ci/deps.python36.sh; + fi + - if [ -d "$HOME/.pyenv/bin" ]; then + export PATH="$HOME/.pyenv/bin:$PATH"; + fi + - hash -r && pyenv versions --bare && python --version + - if [ "${TRAVIS_PYTHON_VERSION/3.4/}" != "$TRAVIS_PYTHON_VERSION" ]; then + pip install pip==9.0.3 setuptools==21.2.2; + fi + + - if [ -f ".ci/deps.$TRAVIS_LANGUAGE.sh" ]; then + bash -e -x ".ci/deps.$TRAVIS_LANGUAGE.sh"; + fi + + # https://github.com/coala/coala/issues/3183 + - cp requirements.txt requirements.orig + - printf '%s\n%s\n%s\n' + "$(cat test-requirements.txt)" + "$(grep -v '^-r' docs-requirements.txt)" + "$(cat bear-requirements.txt requirements.txt)" + > requirements.txt + +{% endblock %} + +{% block before_script %} +before_script: + - mv requirements.orig requirements.txt + - if [ -n "$TRAVIS_PYTHON_VERSION" ]; then + python setup.py bdist_wheel && + pip install $(ls ./dist/*.whl)"[alldeps]"; + fi + +{% endblock %} + +{% block script %} + - if [ -z "$TRAVIS_PYTHON_VERSION" ]; then + python -m pip install --upgrade --user -r test-requirements.txt; + fi + # Ensure metadata files are in sync with the bear metadata in the source + - if [ -n "$TRAVIS_PYTHON_VERSION" ]; then + PYTHONPATH=. .ci/generate_bear_metadata.py --debug --update; + fi + - if [ "${TRAVIS_PYTHON_VERSION/3.4/}" != "$TRAVIS_PYTHON_VERSION" ]; then + pip install --upgrade setuptools; + fi + - python -m tox + - if [ -n "$TRAVIS_PYTHON_VERSION" ]; then + python setup.py docs; + fi + - if [ -n "$TRAVIS_PYTHON_VERSION" ]; then + coala --non-interactive; + fi +{% endblock %} + +{% block end %} + +branches: + exclude: + - /^sils\// +{% endblock %} diff --git a/.moban.dt/travis-manual-matrix.yml b/.moban.dt/travis-manual-matrix.yml new file mode 100644 index 0000000000..a7f691d69c --- /dev/null +++ b/.moban.dt/travis-manual-matrix.yml @@ -0,0 +1,210 @@ +- stage: test-languages + dist: xenial + # This is in generic image, as language: haskell isnt working yet + # https://github.com/coala/coala-bears/issues/1384 + language: generic + env: DIST=xenial BEARS=adhoc BEAR_LIST="ghc-mod" + # ghc-mod needs parts of ghc, specifically at least /usr/lib/ghc/settings + # ghc-mod needs cabal-install + addons: + apt: + packages: + - cabal-install + - ghc + - ghc-mod + +- stage: test-languages + dist: xenial + language: generic + env: DIST=xenial BEARS=adhoc BEAR_LIST="apt_get bakalint default-jre" + DISABLE_BEARS="astyle flawfinder ghc-mod r_script shellcheck" + # astyle and shellcheck are failing in xenial + # xenial doesnt have Python 2.7 needed for flawfinder + # R bears and ghc_mod use separate jobs + addons: + apt: + packages: + - chktex + - cppcheck + - devscripts + - hlint + - indent + - libperl-critic-perl + - libxml2-utils + - mono-mcs + - php7.0-cli + - phpmd + - php-codesniffer + - verilator + +- stage: test-languages + dist: trusty + language: generic + env: DIST=trusty BEARS=adhoc BEAR_LIST="astyle flawfinder shellcheck" + addons: + apt: + sources: + - sourceline: # astyle + deb http://ppa.launchpad.net/cs50/ppa/ubuntu trusty main + key_url: + https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x5BDA2E974A0E822C + packages: + - astyle + - flawfinder + - shellcheck + +# Additonal manual entries +- stage: test-other-versions + os: windows + language: bash + python: '{{ choco_requirements.python.version }}' + env: + BEAR_LIST="{{ ' '.join(tox.win.extra_bears) }}" + TOX_TEST_SELECTORS="{{ '-'.join(tox.win.selectors) }}" + cache: + directories: + - C:/nuget_http_cache/ + - C:/choco_cache/ + - C:/pip_cache/ + before_install: + - bash --version + - powershell -c "Set-ExecutionPolicy -ExecutionPolicy Unrestricted + -Scope LocalMachine" + - powershell .ci/travis_init.ps1 + + - export NUGET_HTTP_CACHE_PATH=/C/nuget_http_cache + - export CHOCO_CACHE_DIR=/C/choco_cache + - export PIP_CACHE_DIR=/C/pip_cache + + - source .ci/travis_extra_globals.sh + + - export TOXENV=py36-${TOX_TEST_SELECTORS}-${TOX_FEATURES}-win + + - printenv + + - cp .ci/choco.config $ChocolateyInstall/config/chocolatey.config + + - python .ci/store_env_in_registry.py + - source .ci/refreshenv.sh + + # TODO: Add support for disabling pre-installed vctools which is + # disabled in choco_requirements. Uninstalling vctools fails + install: + - powershell -c ". .ci/Fudge.ps1 install" + - refreshenv + - taskkill -IM "gpg-agent.exe" || true + before_script: + - python --version + - cp tox.ini tox.orig + - "sed -i 's/^envlist.*$/envlist: '$TOXENV/ tox.ini" + - export TOX_SITEPACKAGES=true + - export VIRTUALENV_NO_PIP=1 + - export VIRTUALENV_NO_SETUPTOOLS=1 + - pip uninstall tox-venv || true + # Avoid verify_gemfile_dependencies_are_found errors due to + # Gemfile modifications in .ci/deps.ruby-packages.ps1 + - cp Gemfile.win Gemfile + script: + - python -m tox --sitepackages + after_script: + - cp tox.orig tox.ini + - cp Gemfile.orig Gemfile + +- stage: test-other-versions + language: ruby + rvm: 2.5 + cache: bundler + env: DISABLE_BEARS=csvlint + +- stage: test-other-versions + language: java + jdk: openjdk11 + env: DISABLE_BEARS="languagetool tailor" +# openjdk10 and openjdk9 are broken on Travis +# https://github.com/travis-ci/travis-cookbooks/issues/976 +# They need the same configuration as openjdk11: +# env: DISABLE_BEARS="languagetool tailor" +- stage: test-other-versions + language: java + jdk: openjdk7 + dist: trusty + env: DISABLE_BEARS=tailor + +# BEARS=apt_get allows matching bears to be skipped +- stage: test-other-versions + dist: bionic + language: generic + # https://travis-ci.community/t/apt-addon-broken-on-bionic/4061 + env: DIST=bionic BEARS=apt_get DISABLE_BEARS=shellcheck + +- stage: test-other-versions + dist: trusty + language: generic + env: DIST=trusty BEARS=apt_get + addons: + apt: + packages: + - chktex + - cppcheck + - devscripts + - indent + - libperl-critic-perl + - libxml2-utils + - mono-mcs + - php5-cli + - php-codesniffer + - verilator + +- stage: test-other-versions + dist: precise + language: generic + env: DIST=precise BEARS=apt_get + cache: + directories: + - $HOME/.pyenv + addons: + apt: + packages: + - chktex + - cppcheck + - devscripts + - indent + - libperl-critic-perl + - libxml2-utils + - mono-mcs + - php5-cli + - python3 + - verilator + +- python: 2.7 + language: python + stage: test-other-versions + env: PIP_NO_COMPILE=1 + addons: false + before_install: true + install: pip install 3to2 + before_script: true + script: .ci/check_unsupported.sh +- python: 3.3 + language: python + stage: test-other-versions + dist: trusty + env: PIP_NO_COMPILE=1 + addons: false + before_install: skip + install: skip + before_script: skip + script: .ci/check_unsupported.sh + +- python: 3.6 + language: python + stage: sentinel + addons: false + cache: pip + before_install: false + install: pip install moban + before_script: false + script: .ci/check_moban.sh + after_success: false + after_failure: false + if: branch = master AND type = push diff --git a/.moban.yaml b/.moban.yaml index 6ac3d2a1b3..ecd4649ac1 100644 --- a/.moban.yaml +++ b/.moban.yaml @@ -12,22 +12,143 @@ package_module: bears docs_source_dir: API docs_dir: docs test_prevent_skips: false +pip_constraints_file: bear-requirements.txt +lint_command: false +appveyor_global_environment: + # Needed for Julia + JL_PKG: CoalaBears + JULIA_PROJECT: "'@.'" + +python_versions: + - 3.6 + - 3.5 + - 3.4.4 + +tox: + features: + - check + - noskip + - codecov + win: + selectors: + - pip + - noreqs + - npm + - gem + - go + - perl + - php + - java8 + - adhoc + extra_bears: + - astyle + - cppcheck + - xmllint entry_points: coalabears: - coala_official_bears = bears +choco_requirements: + # overrides for package managers not yet used + visualstudio2017-workload-vctools: false + MinGW: false + miniconda3: false + R.Project: false # https://github.com/coala/coala-bears/issues/2919 + luarocks: false # https://github.com/coala/coala-bears/issues/2918 + julia: false # A bit tricky to get Lint.jl to build + ghc: false # Compiling from source broken on Travis + haskell-stack: false + rust: false # https://github.com/coala/coala-bears/issues/50 + bower: false + # extra deps for bears + elm-platform: false # https://github.com/coala/coala-bears/issues/2925 + astyle: true + cppcheck: true + xsltproc: true + ShellCheck: true # https://github.com/coala/coala-bears/issues/2920 + +supported_versions: + # python included from `python_version` + mono: + - 5.20.1 + r: + - release + - devel + - oldrel + node_js: + - 10 + - 9 + - 8 + - 7 + - 6 + julia: + - 1.1 + - 1.0 + - 0.7.0 + dart: + - 1.15.0 + - 1.14.2 + perl: + - '5.30' + - 5.22 + - 5.18 + - 5.14 + go: + - 1.11 + - '1.10' + ruby: + - 2.4 + - 2.3 + - 2.2 + - 2.1 + php: + - 7.2 + - hhvm-3.18 + - 5.5 + scala: + - 2.11 + - 2.12.2 + java: + - openjdk8 + lua: + - default + infer: + - 0.7.0 + +requires: + - https://gitlab.com/coala/package_manager/ + - type: git + url: https://gitlab.com/jayvdb/mobans + branch: bears-pr-final + configuration: template_dir: - .moban.dt/ - - .ci/ - - ../coala-mobans/templates/ - - ../coala-mobans/assets/ + - 'mobans:templates/' + - 'mobans:assets/' + - 'package_manager:' configuration: .moban.yaml - configuration_dir: ../coala-mobans/ + configuration_dir: 'mobans:' targets: - setup.py: bears-setup.py.jj2 - setup.cfg: bears-setup.cfg.jj2 + - Fudgefile: Fudgefile.jj2 + # Powershell Linting + - .ci/PSLint.ps1: ci/PSLint.ps1 + - .ci/Export-NUnitXml.psm1: ci/Export-NUnitXml.psm1 + - .ci/PSScriptAnalyzerSettings.psd1: ci/PSScriptAnalyzerSettings.psd1 + # Fudge + - .ci/Fudge.ps1: fudge/Fudge.ps1 + - .ci/Modules/FudgeTools.psm1: fudge/Modules/FudgeTools.psm1 + - .ci/FudgeCI.ps1: fudge/FudgeCI.ps1 + - .ci/PrepareAVVM.ps1: fudge/PrepareAVVM.ps1 + - .ci/FudgeGenerateFake.ps1: fudge/FudgeGenerateFake.ps1 + - .ci/FudgePostInstall.ps1: fudge/FudgePostInstall.ps1 + - .ci/store_env_in_registry.py: ci/store_env_in_registry.py + - .ci/refreshenv.sh: ci/refreshenv.sh + - .ci/constants.ps1: constants.ps1.jj2 + - .ci/deps.python-packages.ps1: fudge/deps.python-packages.ps1 - requirements.txt: requirements.txt.jj2 - test-requirements.txt: bears-test-requirements.txt.jj2 - bears/VERSION: VERSION.jj2 @@ -35,8 +156,13 @@ targets: - DESCRIPTION: DESCRIPTION.jj2 - package.json: package.json.jj2 - bear-requirements.txt: bear-requirements.txt.jj2 - - .ci/appveyor.yml: ci/appveyor.yml.jj2 - - .ci/run_with_env.cmd: run_with_env.cmd + - .travis.yml: bears-travis.yml.jj2 + - .ci/appveyor.yml: bears-appveyor.yml.jj2 + # from package manager + - .ci/deps.golang.ps1: .ci/deps.golang.ps1 + - .ci/deps.ActivePerl.ps1: .ci/deps.ActivePerl.ps1 + - .ci/deps.php.ps1: .ci/deps.php.ps1 + # Other - .ci/check_unsupported.sh: ci/check_unsupported.sh.jj2 - runtime.txt: runtime.txt - netlify.toml: docs/netlify.toml diff --git a/.travis.yml b/.travis.yml index 9544d7517b..995cc89015 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,22 @@ sudo: false - +dist: xenial language: python -python: - - 3.4 - - 3.5 - - 3.6 +notifications: + email: false +python: 3.6.3 + +stages: + - name: sentinel + if: branch != master OR type = pull_request + - test-languages + - test-other-versions + - moban .disable_global: &disable_global addons: false - cache: pip + cache: false + env: {} + python: false before_install: false install: false before_script: false @@ -18,33 +26,199 @@ python: before_deploy: false deploy: false -stages: - - name: sentinel - if: branch != master OR type = pull_request - - test - - moban - - name: unsupported - if: branch = master AND type = push - -.check_moban: &check_moban +.moban: &moban <<: *disable_global python: 3.6 stage: moban - install: pip install moban - script: .ci/check_moban.sh - if: branch != master OR type = pull_request + install: pip install moban>=0.0.4 + script: + - moban + - git diff --exit-code jobs: include: - - stage: sentinel - # All other jobs will be cancelled if the sentinel job fails - <<: *disable_global - python: 3.6 - install: pip install -r requirements.txt ".[alldeps]" - script: coala --non-interactive -V + # Manually added "language" entries should complete test coverage + - stage: test-languages + dist: xenial + # This is in generic image, as language: haskell isnt working yet + # https://github.com/coala/coala-bears/issues/1384 + language: generic + env: DIST=xenial BEARS=adhoc BEAR_LIST="ghc-mod" + # ghc-mod needs parts of ghc, specifically at least /usr/lib/ghc/settings + # ghc-mod needs cabal-install + addons: + apt: + packages: + - cabal-install + - ghc + - ghc-mod + + - stage: test-languages + dist: xenial + language: generic + env: DIST=xenial BEARS=adhoc BEAR_LIST="apt_get bakalint default-jre" + DISABLE_BEARS="astyle flawfinder ghc-mod r_script shellcheck" + # astyle and shellcheck are failing in xenial + # xenial doesnt have Python 2.7 needed for flawfinder + # R bears and ghc_mod use separate jobs + addons: + apt: + packages: + - chktex + - cppcheck + - devscripts + - hlint + - indent + - libperl-critic-perl + - libxml2-utils + - mono-mcs + - php7.0-cli + - phpmd + - php-codesniffer + - verilator + + - stage: test-languages + dist: trusty + language: generic + env: DIST=trusty BEARS=adhoc BEAR_LIST="astyle flawfinder shellcheck" + addons: + apt: + sources: + - sourceline: # astyle + deb http://ppa.launchpad.net/cs50/ppa/ubuntu trusty main + key_url: + https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x5BDA2E974A0E822C + packages: + - astyle + - flawfinder + - shellcheck + + # Additonal manual entries + - stage: test-other-versions + os: windows + language: bash + python: '3.6.8' + env: + BEAR_LIST="astyle cppcheck xmllint" + TOX_TEST_SELECTORS="pip-noreqs-npm-gem-go-perl-php-java8-adhoc" + cache: + directories: + - C:/nuget_http_cache/ + - C:/choco_cache/ + - C:/pip_cache/ + before_install: + - bash --version + - powershell -c "Set-ExecutionPolicy -ExecutionPolicy Unrestricted + -Scope LocalMachine" + - powershell .ci/travis_init.ps1 + + - export NUGET_HTTP_CACHE_PATH=/C/nuget_http_cache + - export CHOCO_CACHE_DIR=/C/choco_cache + - export PIP_CACHE_DIR=/C/pip_cache + + - source .ci/travis_extra_globals.sh + + - export TOXENV=py36-${TOX_TEST_SELECTORS}-${TOX_FEATURES}-win + + - printenv + + - cp .ci/choco.config $ChocolateyInstall/config/chocolatey.config + + - python .ci/store_env_in_registry.py + - source .ci/refreshenv.sh + + # TODO: Add support for disabling pre-installed vctools which is + # disabled in choco_requirements. Uninstalling vctools fails + install: + - powershell -c ". .ci/Fudge.ps1 install" + - refreshenv + - taskkill -IM "gpg-agent.exe" || true + before_script: + - python --version + - cp tox.ini tox.orig + - "sed -i 's/^envlist.*$/envlist: '$TOXENV/ tox.ini" + - export TOX_SITEPACKAGES=true + - export VIRTUALENV_NO_PIP=1 + - export VIRTUALENV_NO_SETUPTOOLS=1 + - pip uninstall tox-venv || true + # Avoid verify_gemfile_dependencies_are_found errors due to + # Gemfile modifications in .ci/deps.ruby-packages.ps1 + - cp Gemfile.win Gemfile + script: + - python -m tox --sitepackages + after_script: + - cp tox.orig tox.ini + - cp Gemfile.orig Gemfile + + - stage: test-other-versions + language: ruby + rvm: 2.5 + cache: bundler + env: DISABLE_BEARS=csvlint + + - stage: test-other-versions + language: java + jdk: openjdk11 + env: DISABLE_BEARS="languagetool tailor" + # openjdk10 and openjdk9 are broken on Travis + # https://github.com/travis-ci/travis-cookbooks/issues/976 + # They need the same configuration as openjdk11: + # env: DISABLE_BEARS="languagetool tailor" + - stage: test-other-versions + language: java + jdk: openjdk7 + dist: trusty + env: DISABLE_BEARS=tailor + + # BEARS=apt_get allows matching bears to be skipped + - stage: test-other-versions + dist: bionic + language: generic + # https://travis-ci.community/t/apt-addon-broken-on-bionic/4061 + env: DIST=bionic BEARS=apt_get DISABLE_BEARS=shellcheck + + - stage: test-other-versions + dist: trusty + language: generic + env: DIST=trusty BEARS=apt_get + addons: + apt: + packages: + - chktex + - cppcheck + - devscripts + - indent + - libperl-critic-perl + - libxml2-utils + - mono-mcs + - php5-cli + - php-codesniffer + - verilator + + - stage: test-other-versions + dist: precise + language: generic + env: DIST=precise BEARS=apt_get + cache: + directories: + - $HOME/.pyenv + addons: + apt: + packages: + - chktex + - cppcheck + - devscripts + - indent + - libperl-critic-perl + - libxml2-utils + - mono-mcs + - php5-cli + - python3 + - verilator - python: 2.7 - stage: unsupported + language: python + stage: test-other-versions env: PIP_NO_COMPILE=1 addons: false before_install: true @@ -52,15 +226,19 @@ jobs: before_script: true script: .ci/check_unsupported.sh - python: 3.3 - stage: unsupported + language: python + stage: test-other-versions + dist: trusty env: PIP_NO_COMPILE=1 addons: false - before_install: true - install: true - before_script: true + before_install: skip + install: skip + before_script: skip script: .ci/check_unsupported.sh + - python: 3.6 - stage: moban + language: python + stage: sentinel addons: false cache: pip before_install: false @@ -70,65 +248,252 @@ jobs: after_success: false after_failure: false if: branch = master AND type = push - - *check_moban + + # Entries generates from `supported_versions` + - stage: sentinel + dist: trusty + language: python + python: 3.6 + cache: pip + addons: + apt: + packages: clang-3.4 + + - stage: test-languages + dist: trusty + language: python + python: 3.5 + cache: pip + addons: + apt: + packages: clang-3.4 + + - stage: test-other-versions + dist: trusty + language: python + python: 3.4.4 + cache: pip + addons: + apt: + packages: clang-3.4 + + - stage: test-languages + language: csharp + mono: 5.20.1 + + - stage: test-languages + language: r + r: release + cache: packages + + - stage: test-other-versions + language: r + r: devel + cache: packages + + - stage: test-other-versions + language: r + r: oldrel + cache: packages + + - stage: test-languages + language: node_js + node_js: '10' + cache: npm + + - stage: test-other-versions + language: node_js + node_js: 9 + cache: npm + + - stage: test-other-versions + language: node_js + node_js: 8 + cache: npm + + - stage: test-other-versions + language: node_js + node_js: 7 + cache: npm + + - stage: test-other-versions + language: node_js + node_js: 6 + cache: npm + + - stage: test-languages + language: julia + julia: 1.1 + env: JL_PKG=CoalaBears + install: + - julia --color=yes .ci/deps.julia.jl + # Verify compilation works + - julia -e 'import Lint.lintfile' + cache: + directories: + - $HOME/.julia + + - stage: test-other-versions + language: julia + julia: 1.0 + env: JL_PKG=CoalaBears + install: + - julia --color=yes .ci/deps.julia.jl + # Verify compilation works + - julia -e 'import Lint.lintfile' + cache: + directories: + - $HOME/.julia + + - stage: test-other-versions + language: julia + julia: 0.7.0 + env: JL_PKG=CoalaBears + install: + - julia --color=yes .ci/deps.julia.jl + # Verify compilation works + - julia -e 'import Lint.lintfile' + cache: + directories: + - $HOME/.julia + + - stage: test-languages + language: dart + dart: 1.15.0 + + - stage: test-other-versions + language: dart + dart: 1.14.2 + + - stage: test-languages + dist: trusty + language: perl + perl: '5.30' + cache: + directories: + - $HOME/perl5 + + - stage: test-other-versions + dist: trusty + language: perl + perl: 5.22 + cache: + directories: + - $HOME/perl5 + + - stage: test-other-versions + dist: trusty + language: perl + perl: 5.18 + cache: + directories: + - $HOME/perl5 + + - stage: test-other-versions + dist: trusty + language: perl + perl: 5.14 + cache: + directories: + - $HOME/perl5 + + - stage: test-languages + language: go + go: 1.11 + install: skip + cache: + directories: + - $HOME/.cache/go-build + - $HOME/gopath/pkg/mod + + - stage: test-other-versions + language: go + go: '1.10' + install: skip + cache: + directories: + - $HOME/.cache/go-build + - $HOME/gopath/pkg/mod + + - stage: test-languages + dist: trusty + language: ruby + rvm: 2.4 + cache: bundler + + - stage: test-other-versions + dist: trusty + language: ruby + rvm: 2.3 + cache: bundler + + - stage: test-other-versions + dist: trusty + language: ruby + rvm: 2.2 + cache: bundler + + - stage: test-other-versions + dist: trusty + language: ruby + rvm: 2.1 + cache: bundler + + - stage: test-languages + dist: trusty + language: php + php: 7.2 + + - stage: test-other-versions + dist: trusty + language: php + php: hhvm-3.18 + + - stage: test-other-versions + dist: trusty + language: php + php: 5.5 + + - stage: test-languages + language: scala + scala: 2.11 + + - stage: test-other-versions + language: scala + scala: 2.12.2 + jdk: openjdk8 + + - stage: test-languages + language: java + jdk: openjdk8 + + - stage: test-languages + language: minimal + env: + BEARS="lua" + cache: + directories: + - $HOME/.luarocks + + - stage: test-languages + dist: trusty + language: minimal + env: + BEARS="opam" + INFER_VERSION="0.7.0" + PATH="$PATH:$HOME/infer-linux64-v$INFER_VERSION/infer/bin" + addons: + apt: + sources: + - avsm + packages: + - camlp4-extra + - ocaml + - opam + + - *moban allow_failures: - - *check_moban - -dist: trusty - -.apt_sources: &apt_sources - - ubuntu-toolchain-r-test - # avsm # OPAM stable - - hvr-ghc # Haskell - - sourceline: # R - deb https://cloud.r-project.org/bin/linux/ubuntu trusty-cran35/ - key_url: - https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x51716619E084DAB9 - - sourceline: # Julia - deb http://ppa.launchpad.net/staticfloat/juliareleases/ubuntu trusty main - key_url: - https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xCF979FFA3D3D3ACC - - sourceline: # astyle - deb http://ppa.launchpad.net/cs50/ppa/ubuntu trusty main - key_url: - https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x5BDA2E974A0E822C - -addons: - apt: - sources: *apt_sources - packages: - - aspcud - - astyle - - cabal-install-1.24 - - chktex - - clang-3.4 - - cppcheck - - devscripts - - flawfinder - - gfortran - - ghc - - happy - - indent - - julia - - libarpack2 - - libblas-dev - - libcolamd2.8.0 - - libfftw3-3 - - liblapack-dev - - libopenblas-base - - libpaper-utils - - libperl-critic-perl - - libumfpack5.6.2 - - libxml2-utils - - luarocks - - mercurial - # menhir - - mono-mcs - # ocaml - # opam - - php-codesniffer - - r-base - - verilator + - *moban cache: pip: true @@ -138,11 +503,6 @@ cache: - ~/.cabal - ~/.ghc - ~/.ghc-mod - - ~/R/Library - - ~/.julia - - ~/.luarocks - - $TRAVIS_BUILD_DIR/node_modules - - $TRAVIS_BUILD_DIR/.bundle - $TRAVIS_BUILD_DIR/vendor # coala managed data - ~/nltk_data @@ -153,25 +513,41 @@ cache: env: global: - TERM=dumb - - R_LIB_USER=~/R/Library - - LINTR_COMMENT_BOT=false - - CABAL_VERSION=1.24 - - INFER_VERSION=0.7.0 - - PATH="$HOME/.local/bin:/opt/cabal/$CABAL_VERSION/bin:$PATH:$TRAVIS_BUILD_DIR/node_modules/.bin:$TRAVIS_BUILD_DIR/vendor/bin:$HOME/.cabal/bin:$HOME/infer-linux64-v$INFER_VERSION/infer/bin:$HOME/.local/tailor/tailor-latest/bin:$HOME/.luarocks/bin" + - PATH="$HOME/.local/bin:$PATH" + # These are only needed by Windows + - NUGET_EXE_NO_PROMPT=true + - VIRTUALENV_NO_DOWNLOAD=1 + # Enable to debug tox + # VIRTUALENV_VERBOSE=1 + # This exceeds the travis maximum log length + # PIP_VERBOSE=1 + - PIP_DISABLE_PIP_VERSION_CHECK=1 + - PIP_YES=1 + - FudgeCI=${TRAVIS_BUILD_DIR}/.ci/ + - TOX_FEATURES=check-noskip-codecov + +stage: test before_install: - # Install latest stable version of Go using gimme - - gimme 1.11.5 > setup_go_root.sh - - source setup_go_root.sh - - nvm install 6.10.2 - # Remove Ruby directive from Gemfile as this image has 2.2.5 - - sed -i '/^ruby/d' Gemfile - - .ci/deps.sh - - .ci/deps.go.sh - - .ci/deps.cabal.sh - - .ci/deps.r.sh - # .ci/deps.opam.sh - - .ci/deps.java.sh + - printenv + - mkdir -p ~/bin ~/.local/bin + - source .ci/travis_extra_globals.sh + + - if [ -z "$TRAVIS_PYTHON_VERSION" ]; then + .ci/deps.python36.sh; + fi + - if [ -d "$HOME/.pyenv/bin" ]; then + export PATH="$HOME/.pyenv/bin:$PATH"; + fi + - hash -r && pyenv versions --bare && python --version + - if [ "${TRAVIS_PYTHON_VERSION/3.4/}" != "$TRAVIS_PYTHON_VERSION" ]; then + pip install pip==9.0.3 setuptools==21.2.2; + fi + + - if [ -f ".ci/deps.$TRAVIS_LANGUAGE.sh" ]; then + bash -e -x ".ci/deps.$TRAVIS_LANGUAGE.sh"; + fi + # https://github.com/coala/coala/issues/3183 - cp requirements.txt requirements.orig - printf '%s\n%s\n%s\n' @@ -182,22 +558,29 @@ before_install: before_script: - mv requirements.orig requirements.txt - - .ci/deps.coala-bears.sh + - if [ -n "$TRAVIS_PYTHON_VERSION" ]; then + python setup.py bdist_wheel && + pip install $(ls ./dist/*.whl)"[alldeps]"; + fi script: - - python setup.py bdist_wheel - - pip install $(ls ./dist/*.whl)"[alldeps]" - # Ensure bear requirements are in sync with the bear PipRequirement - - .ci/generate_bear_requirements.py --check --update - - coala --non-interactive - - rm bears/java/InferBear.py tests/java/InferBearTest.py - - rm bears/go/GoReturnsBear.py tests/go/GoReturnsBearTest.py - - pytest - - codecov - - python setup.py docs - -notifications: - email: false + - if [ -z "$TRAVIS_PYTHON_VERSION" ]; then + python -m pip install --upgrade --user -r test-requirements.txt; + fi + # Ensure metadata files are in sync with the bear metadata in the source + - if [ -n "$TRAVIS_PYTHON_VERSION" ]; then + PYTHONPATH=. .ci/generate_bear_metadata.py --debug --update; + fi + - if [ "${TRAVIS_PYTHON_VERSION/3.4/}" != "$TRAVIS_PYTHON_VERSION" ]; then + pip install --upgrade setuptools; + fi + - python -m tox + - if [ -n "$TRAVIS_PYTHON_VERSION" ]; then + python setup.py docs; + fi + - if [ -n "$TRAVIS_PYTHON_VERSION" ]; then + coala --non-interactive; + fi branches: exclude: diff --git a/Fudgefile b/Fudgefile new file mode 100644 index 0000000000..ba5a989ea2 --- /dev/null +++ b/Fudgefile @@ -0,0 +1,111 @@ +{ + "pack": { + "ActivePerl": ".ci/nuspecs/ActivePerl.nuspec", + "adoptopenjdk": ".ci/nuspecs/adoptopenjdk.nuspec", + "golang": ".ci/nuspecs/golang.nuspec", + "hg": ".ci/nuspecs/hg.nuspec", + "maven": ".ci/nuspecs/maven.nuspec", + "nodejs": ".ci/nuspecs/nodejs.nuspec", + "python": ".ci/nuspecs/python.nuspec", + "ruby": ".ci/nuspecs/ruby.nuspec", + "ruby2.devkit": ".ci/nuspecs/ruby2.devkit.nuspec" + }, + "packages": [ + { + "name": "msys2", + "params": "/InstallDir:C:\\msys64 /NoUpdate", + "source": "" + }, + { + "appveyor_id": true, + "name": "hg", + "source": "", + "version": "5.0" + }, + { + "appveyor_id": "python", + "name": "python", + "source": "", + "version": "3.6.8" + }, + { + "appveyor_id": "node", + "name": "nodejs", + "source": "", + "version": "11.13.0" + }, + { + "appveyor_id": "ruby", + "name": "ruby", + "source": "", + "version": "2.5.3.1" + }, + { + "appveyor_id": true, + "name": "ruby2.devkit", + "source": "", + "version": "4.7.2.2013022403" + }, + { + "appveyor_id": "go", + "name": "golang", + "source": "", + "version": "1.9.7" + }, + { + "appveyor_id": "jdk", + "name": "adoptopenjdk", + "source": "", + "version": "8.192" + }, + { + "appveyor_id": true, + "name": "ActivePerl", + "source": "", + "version": "5.24.3.2404" + }, + { + "appveyor_id": true, + "name": "maven", + "source": "", + "version": "3.5.4" + }, + { + "name": "php", + "source": "" + }, + { + "name": "composer", + "source": "" + }, + { + "name": "PSScriptAnalyzer", + "source": "" + }, + { + "name": "astyle", + "source": "" + }, + { + "name": "cppcheck", + "source": "" + }, + { + "name": "xsltproc", + "source": "" + }, + { + "name": "ShellCheck", + "source": "" + } + ], + "scripts": { + "post": { + "install": ". $env:FudgeCI/FudgePostInstall.ps1; Invoke-PostInstall" + }, + "pre": { + "install": ". $env:FudgeCI/FudgeCI.ps1; Invoke-FudgeCI" + } + }, + "source": "https://chocolatey.org/api/v2/" +} diff --git a/Makefile.PL b/Makefile.PL new file mode 100644 index 0000000000..e889d73de6 --- /dev/null +++ b/Makefile.PL @@ -0,0 +1,6 @@ +use ExtUtils::MakeMaker; +WriteMakefile( + NAME => 'Coala::Bears', + VERSION => '0.10', + PREREQ_PM => {Perl::Critic => 1.126}, +); diff --git a/bear-requirements.yaml b/bear-requirements.yaml index 1e25b04766..9f860344ec 100644 --- a/bear-requirements.yaml +++ b/bear-requirements.yaml @@ -1,32 +1,91 @@ # This is an automatically generated file. # And should not be edited by hand. -overrides: coala-build.yaml -gem_requirements: - brakeman: - version: ~>4.1.1 - csvlint: - version: ~>0.4.0 - fasterer: - version: ~>0.4.1 - haml_lint: - version: ~>0.27.0 - puppet-lint: - version: ~>2.1.1 - reek: - version: ~>4.6 - rubocop: - version: ~>0.51.0 - scss_lint: - version: ~>0.56.0 - sqlint: - version: ~>0.1.5 - travis: - version: ~>1.8.8 -r_script_requirements: - formatR: - version: '>1.5' - lintr: - version: '>=1.0.2' +overrides: pm-requirements.yaml +pip_requirements: + HTTPolice: + version: ~=0.5.2 + aenum: + version: ~=2.0.8 + apertium-lint: + version: ~=0.29 + autoflake: + version: ~=0.7 + autopep8: + version: ~=1.2 + bandit: + version: ~=1.2 + bashate: + version: ~=0.5.1 + cmakelint: + version: ~=1.3 + cppclean: + version: ~=0.12.0 + cpplint: + version: ~=1.3 + dennis: + version: ~=0.9 + docutils-ast-writer: + version: ~=0.1.2 + eradicate: + version: ~=0.1.6 + git-url-parse: + version: ~=1.1 + guess-language-spirit: + version: ~=0.5.2 + html-linter: + version: ~=0.4.0 + isort: + version: ~=4.2 + language-check: + version: ~=1.0 + libclang-py3: + version: ~=3.4.0 + lxml: + version: '>=1.0' + memento-client: + version: ~=0.6.1 + munkres3: + version: ~=1.0 + mypy: + version: ==0.590 + nbformat: + version: ~=4.1 + nltk: + version: ~=3.2 + proselint: + version: ~=0.7.0 + pycodestyle: + version: ~=2.2 + pydocstyle: + version: ~=2.0 + pyflakes: + version: ~=2.0.0 + pylint: + version: ~=1.7.2 + pyroma: + version: ~=2.2.0 + pyyaml: + version: ~=3.12 + radon: + version: ==1.4.0 + restructuredtext-lint: + version: ~=1.0 + rstcheck: + version: ~=3.1 + safety: + version: ~=1.8.2 + scspell3k: + version: ~=2.0 + sqlparse: + version: ~=0.2.4 + vim-vint: + version: ~=0.3.12,!=0.3.19 + vulture: + version: ~=0.25.0 + yamllint: + version: ~=1.12.0 + yapf: + version: ~=0.21.0 npm_requirements: alex: version: ~3 @@ -136,91 +195,27 @@ npm_requirements: version: ~12.0.0 write-good: version: ~0.9.1 -pip_requirements: - HTTPolice: - version: ~=0.5.2 - aenum: - version: ~=2.0.8 - apertium-lint: - version: ~=0.29 - autoflake: - version: ~=0.7 - autopep8: - version: ~=1.2 - bandit: - version: ~=1.2 - bashate: - version: ~=0.5.1 - cmakelint: - version: ~=1.3 - cppclean: - version: ~=0.12.0 - cpplint: - version: ~=1.3 - dennis: - version: ~=0.9 - docutils-ast-writer: - version: ~=0.1.2 - eradicate: - version: ~=0.1.6 - git-url-parse: - version: ~=1.1 - guess-language-spirit: - version: ~=0.5.2 - html-linter: - version: ~=0.4.0 - isort: - version: ~=4.2 - language-check: - version: ~=1.0 - libclang-py3: - version: ~=3.4.0 - lxml: - version: '>=1.0' - memento-client: - version: ~=0.6.1 - munkres3: - version: ~=1.0 - mypy: - version: ==0.590 - nbformat: - version: ~=4.1 - nltk: - version: ~=3.2 - proselint: - version: ~=0.7.0 - pycodestyle: - version: ~=2.2 - pydocstyle: - version: ~=2.0 - pyflakes: - version: ~=2.0.0 - pylint: - version: ~=1.7.2 - pyroma: - version: ~=2.2.0 - pyyaml: - version: ~=3.12 - radon: - version: ==1.4.0 - restructuredtext-lint: - version: ~=1.0 - rstcheck: - version: ~=3.1 - safety: - version: ~=1.8.2 - scspell3k: - version: ~=2.0 - sqlparse: - version: ~=0.2.4 - vim-vint: - version: ~=0.3.12,!=0.3.19 - vulture: - version: ~=0.25.0 - yamllint: - version: ~=1.12.0 - yapf: - version: ~=0.21.0 +gem_requirements: + brakeman: + version: ~>4.1.1 + csvlint: + version: ~>0.4.0 + fasterer: + version: ~>0.4.1 + haml_lint: + version: ~>0.27.0 + puppet-lint: + version: ~>2.1.1 + reek: + version: ~>4.6 + rubocop: + version: ~>0.51.0 + scss_lint: + version: ~>0.56.0 + sqlint: + version: ~>0.1.5 + travis: + version: ~>1.8.8 composer_requirements: phpmd/phpmd: version: ~2.6.0 @@ -233,3 +228,189 @@ cabal_requirements: version: ==5.6.0.0 hlint: version: ==1.9.27 +r_script_requirements: + formatR: + lintr: +distro_requirements: + astyle: + packages: + apt_get: astyle + dnf: astyle + chktex: + packages: + apt_get: chktex + brew: chktex + dnf: chktex + pacman: chktex + pkg: chktex + portage: chktex + xbps: chktex + yum: chktex + zypper: texlive-chktex + cppcheck: + packages: + apt_get: cppcheck + brew: cppcheck + dnf: cppcheck + pacman: cppcheck + pkg: cppcheck + portage: cppcheck + xbps: cppcheck + yum: cppcheck + zypper: cppcheck + dart: + packages: + brew: dart + default-jre: + packages: + apt_get: default-jre + devscripts: + packages: + apt_get: devscripts + dnf: licensecheck + portage: + zypper: devscripts + flawfinder: + packages: + apt_get: flawfinder + brew: flawfinder + dnf: flawfinder + pacman: flawfinder + pkg: flawfinder + portage: flawfinder + xbps: flawfinder + yum: flawfinder + zypper: flawfinder + ghc-mod: + packages: + apt_get: ghc-mod + brew: ghc-mod + dnf: ghc-mod + pacman: ghc-mod + pkg: ghc-mod + portage: ghc-mod + xbps: ghc-mod + yum: ghc-mod + zypper: ghc-mod + hlint: + packages: + apt_get: hlint + indent: + packages: + apt_get: indent + brew: indent + dnf: indent + pacman: indent + pkg: indent + portage: indent + xbps: indent + yum: indent + zypper: indent + libperl-critic-perl: + packages: + apt_get: libperl-critic-perl + brew: + dnf: perl-Perl-Critic + portage: dev-perl/Perl-Critic + xbps: + yum: perl-Perl-Critic + zypper: perl-Perl-Critic + libxml2: + packages: + apt_get: libxml2-utils + brew: libxml2 + dnf: libxml2 + pacman: libxml2 + pkg: libxml2 + portage: dev-libs/libxml2 + xbps: libxml2 + yum: libxml2 + zypper: libxml2 + mono: + packages: + apt_get: mono-mcs + brew: mono + dnf: mono + pacman: mono + pkg: mono + portage: dev-lang/mono + xbps: mono + yum: mono + zypper: mono + perl: + packages: + apt_get: perl + brew: perl + dnf: perl + pacman: perl + pkg: perl + portage: perl + xbps: perl + yum: perl + zypper: perl + php-cli: + packages: + apt_get: php-cli + php-codesniffer: + packages: + apt_get: php-codesniffer + zypper: php-pear-php_codesniffer + phpmd: + packages: + apt_get: phpmd + dnf: php-phpmd-PHP-PMD + r-base: + packages: + apt_get: r-base + version: '>=3.1.1' + r-cran-formatr: + packages: + apt_get: r-cran-formatr + zypper: R-formatR + ruby: + packages: + apt_get: ruby + brew: ruby + dnf: ruby + pacman: ruby + pkg: ruby + portage: ruby + xbps: ruby + yum: ruby + zypper: ruby + shellcheck: + packages: + apt_get: shellcheck + brew: shellcheck + dnf: shellcheck + pacman: shellcheck + pkg: shellcheck + portage: shellcheck + xbps: shellcheck + yum: shellcheck + zypper: shellcheck + verilator: + packages: + apt_get: verilator + brew: + dnf: verilator + portage: + yum: verilator + zypper: verilator +julia_requirements: + Lint: +go_requirements: + github.com/BurntSushi/toml/cmd/tomlv: + github.com/golang/lint/golint: + github.com/kisielk/errcheck: + golang.org/cmd/gofmt: + golang.org/cmd/vet: + golang.org/x/tools/cmd/goimports: + golang.org/x/tools/cmd/gotype: + sourcegraph.com/sqs/goreturns: +luarocks_requirements: + luacheck: +conda_requirements: + ruby: + version: '>=2.2.3' +exe_requirements: {} diff --git a/composer.json b/composer.json new file mode 100644 index 0000000000..d14f37262a --- /dev/null +++ b/composer.json @@ -0,0 +1,6 @@ +{ + "require-dev": { + "phpmd/phpmd": "^2.6", + "squizlabs/php_codesniffer": "^3.4" + } +} diff --git a/package.json b/package.json index 00f1517754..81ffb7fccd 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "stylelint": "~7", "stylint": "~1.5.9", "textlint": "~7.3.0", - "textlint-plugin-asciidoc-loose": "~1.0.1", + "textlint-plugin-asciidoctor": "~1.0.3", "textlint-plugin-html": "~0.1.5", "textlint-plugin-review": "~0.3.3", "textlint-plugin-rst": "~0.1.1", diff --git a/setup.cfg b/setup.cfg index 1c7ee45201..18c0be3a5c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -60,6 +60,7 @@ source = omit = tests/* setup.py + .ci/store_env_in_registry.py .ci/* [coverage:report] diff --git a/test-requirements.txt b/test-requirements.txt index 1643a4eb9a..fad513360a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -28,4 +28,11 @@ ipdb~=0.11 pip<10 six>=1.11.0 wheel~=0.29 +pbr!=2.1.0,>=2.0.0 +pytest-error-for-skips +git+https://github.com/krkd/pytest-cov-threshold#egg=pytest-cov-threshold ; python_version > '3.4' twine~=1.7.4 +tox~=3.12.0 +tox-travis +tox-backticks +tox-pyenv diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000000..9cb9521a39 --- /dev/null +++ b/tox.ini @@ -0,0 +1,114 @@ +[tox] +envlist = py{34,35,36,37}-{all,pip,clang,npm,gem,go,perl,php,cabal,java,java7,java8,scala,elm,r,dart,julia,lua,infer,opam,apt_get,adhoc,disabled,win,swift,mono}-{list,check,collectonly,skip,noskip}-codecov +minversion = 3.4 + +[travis:env] +TRAVIS = + true: codecov +TRAVIS_LANGUAGE = + python: pip-noskip + node_js: py36-npm-noskip + ruby: py36-gem-noskip + haskell: py36-cabal-noskip + go: py36-go-noskip + perl: py36-perl-noskip + php: py36-php-noskip + scala: py36-scala-noskip + elm: py36-elm-noskip + r: py36-r-noskip + dart: py36-dart-noskip + julia: py36-julia-noskip + objective_c: py36-swift-noskip + swift: py36-swift-noskip + objectivec: py36-swift-noskip + objective-c: py36-swift-noskip + csharp: py36-mono-noskip + minimal,generic: py36-adhoc-noskip +TRAVIS_JDK_VERSION = + oraclejdk11: py36-java-skip + oraclejdk9: py36-java-skip + oraclejdk8: py36-java8-noskip + openjdk11: py36-java-skip + openjdk10: py36-java-skip + openjdk9: py36-java-skip + openjdk8: py36-java8-noskip + openjdk7: py36-java7-noskip +# apt_get is the only group allowed to skip +BEARS = + apt_get: py36-apt_get-skip + lua: py36-lua-noskip + infer: py36-infer-noskip + opam: py36-opam-noskip + adhoc: py36-adhoc-noskip + clang: py36-clang-noskip + disabled: py36-disabled-noskip + +# pyyaml literally here to support get_tests.py +[testenv] +passenv = + HOME + PATH + CI CI_* + TRAVIS TRAVIS_* + APPVEYOR APPVEYOR_* + TOX_* + PIP_* + VIRTUALENV_* + LOCALAPPDATA + GEM_HOME + GEM_PATH + BUNDLE_PATH + BUNDLE_BIN + JULIA_PROJECT + GOROOT + GOPATH + BEARS + BEAR_LIST + DISABLE_BEARS + R_PROFILE + R_LIBS_USER + R_LIBS_SITE + _R_CHECK_CRAN_INCOMING_ + NOT_CRAN + R_PROFILE +pip_version = 9.0.1 +alwayscopy = true +skipsdist = true +skip_install = true +list_dependencies_command = python -m pip freeze --local +whitelist_externals = + pytest +deps = + git+https://github.com/coala/coala#egg=coala + pip: -rbear-requirements.txt + # aenum is needed during test collection + !pip: aenum + # PyYAML is needed for two gem bears, but it is already included for pip + gem-!pip: PyYAML + npm-!pip: docutils-ast-writer~=0.1.2 + {apt_get,clang,mono,adhoc}-!pip: libclang-py3~=3.4.0 + clang-!pip: munkres3~=1.0 + java{7,8}-!pip: language-check~=1.0 + java{7,8}-!pip: guess-language-spirit~=0.5.2 + -rtest-requirements.txt + # pytest-cov-threshold is incompatible with py34 + !py34: git+https://github.com/krkd/pytest-cov-threshold + noskip: pytest-error-for-skips +setenv = + LINTR_COMMENT_BOT=false + ENVNAMEBEARS=`python .ci/get_tests.py {envname}` + adhoc: ADHOCBEARS=`python .ci/get_tests.py {env:BEAR_LIST}` + disabled: DISABLEDBEARS=`python .ci/get_tests.py --disabled {env:BEAR_LIST}` + SELECTED={env:ENVNAMEBEARS:} {env:ADHOCBEARS:} {env:DISABLEDBEARS:} + noskip: PYTEST_ARGS=--error-for-skips + py34-noskip: PYTEST_ARGS=--error-for-skips -k 'not test_valid_async' + win-noskip: PYTEST_ARGS=--error-for-skips -k 'not test_language_french and not test_valid_async' + collectonly,list: PYTEST_ARGS=--collect-only + codecov: CODECOV_FLAGS=`python .ci/get_codecov_tags.py {envname}` +commands = + check,list,all: python .ci/get_bears.py --missing {env:SELECTED} + !py34,!apt_get: python .ci/generate_coverage_thresholds.py {posargs:{env:SELECTED}} + py34,apt_get: python .ci/generate_coverage_thresholds.py none + !list: pytest {env:PYTEST_ARGS:} --cov --cov-fail-under=0 --continue-on-collection-errors --cov-report term-missing:skip-covered --deselect=requirements.txt {posargs:{env:SELECTED}} +commands_post = + codecov: codecov --name={envname} --flags={env:CODECOV_FLAGS}