From 4ba60bed89a621561053a0ff4e3cef812a0e9002 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Mon, 6 Sep 2021 10:23:32 +0200 Subject: [PATCH 1/9] format file --- .drone.star | 2524 +++++++++++++++++++++++++-------------------------- 1 file changed, 1259 insertions(+), 1265 deletions(-) diff --git a/.drone.star b/.drone.star index 22c31e7..9fd96b1 100644 --- a/.drone.star +++ b/.drone.star @@ -49,1353 +49,1347 @@ config = { } def main(ctx): - before = beforePipelines() + before = beforePipelines() - stages = stagePipelines() - if (stages == False): - print('Errors detected. Review messages above.') - return [] + stages = stagePipelines() + if (stages == False): + print("Errors detected. Review messages above.") + return [] - dependsOn(before, stages) + dependsOn(before, stages) - after = afterPipelines() - dependsOn(stages, after) + after = afterPipelines() + dependsOn(stages, after) - return before + stages + after + return before + stages + after def beforePipelines(): - return codestyle() + jscodestyle() + phpstan() + phan() + return codestyle() + jscodestyle() + phpstan() + phan() def stagePipelines(): - buildPipelines = build() - jsPipelines = javascript() - phpunitPipelines = phptests('phpunit') - phpintegrationPipelines = phptests('phpintegration') - acceptancePipelines = acceptance() - if (buildPipelines == False) or (jsPipelines == False) or (phpunitPipelines == False) or (phpintegrationPipelines == False) or (acceptancePipelines == False): - return False + buildPipelines = build() + jsPipelines = javascript() + phpunitPipelines = phptests("phpunit") + phpintegrationPipelines = phptests("phpintegration") + acceptancePipelines = acceptance() + if (buildPipelines == False) or (jsPipelines == False) or (phpunitPipelines == False) or (phpintegrationPipelines == False) or (acceptancePipelines == False): + return False - return buildPipelines + jsPipelines + phpunitPipelines + phpintegrationPipelines + acceptancePipelines + return buildPipelines + jsPipelines + phpunitPipelines + phpintegrationPipelines + acceptancePipelines def afterPipelines(): - return [ - notify() - ] + return [ + notify(), + ] def codestyle(): - pipelines = [] - - if 'codestyle' not in config: - return pipelines - - default = { - 'phpVersions': ['7.2'], - } - - if 'defaults' in config: - if 'codestyle' in config['defaults']: - for item in config['defaults']['codestyle']: - default[item] = config['defaults']['codestyle'][item] - - codestyleConfig = config['codestyle'] - - if type(codestyleConfig) == "bool": - if codestyleConfig: - # the config has 'codestyle' true, so specify an empty dict that will get the defaults - codestyleConfig = {} - else: - return pipelines - - if len(codestyleConfig) == 0: - # 'codestyle' is an empty dict, so specify a single section that will get the defaults - codestyleConfig = {'doDefault': {}} - - for category, matrix in codestyleConfig.items(): - params = {} - for item in default: - params[item] = matrix[item] if item in matrix else default[item] - - for phpVersion in params['phpVersions']: - name = 'coding-standard-php%s' % phpVersion - - result = { - 'kind': 'pipeline', - 'type': 'docker', - 'name': name, - 'workspace' : { - 'base': '/var/www/owncloud', - 'path': 'server/apps/%s' % config['app'] - }, - 'steps': [ - { - 'name': 'coding-standard', - 'image': 'owncloudci/php:%s' % phpVersion, - 'pull': 'always', - 'commands': [ - 'make test-php-style' - ] - } - ], - 'depends_on': [], - 'trigger': { - 'ref': [ - 'refs/pull/**', - 'refs/tags/**' - ] - } - } - - for branch in config['branches']: - result['trigger']['ref'].append('refs/heads/%s' % branch) - - pipelines.append(result) - - return pipelines + pipelines = [] + + if "codestyle" not in config: + return pipelines + + default = { + "phpVersions": ["7.2"], + } + + if "defaults" in config: + if "codestyle" in config["defaults"]: + for item in config["defaults"]["codestyle"]: + default[item] = config["defaults"]["codestyle"][item] + + codestyleConfig = config["codestyle"] + + if type(codestyleConfig) == "bool": + if codestyleConfig: + # the config has 'codestyle' true, so specify an empty dict that will get the defaults + codestyleConfig = {} + else: + return pipelines + + if len(codestyleConfig) == 0: + # 'codestyle' is an empty dict, so specify a single section that will get the defaults + codestyleConfig = {"doDefault": {}} + + for category, matrix in codestyleConfig.items(): + params = {} + for item in default: + params[item] = matrix[item] if item in matrix else default[item] + + for phpVersion in params["phpVersions"]: + name = "coding-standard-php%s" % phpVersion + + result = { + "kind": "pipeline", + "type": "docker", + "name": name, + "workspace": { + "base": "/var/www/owncloud", + "path": "server/apps/%s" % config["app"], + }, + "steps": [ + { + "name": "coding-standard", + "image": "owncloudci/php:%s" % phpVersion, + "pull": "always", + "commands": [ + "make test-php-style", + ], + }, + ], + "depends_on": [], + "trigger": { + "ref": [ + "refs/pull/**", + "refs/tags/**", + ], + }, + } + + for branch in config["branches"]: + result["trigger"]["ref"].append("refs/heads/%s" % branch) + + pipelines.append(result) + + return pipelines def jscodestyle(): - pipelines = [] - - if 'jscodestyle' not in config: - return pipelines - - if type(config['jscodestyle']) == "bool": - if not config['jscodestyle']: - return pipelines - - result = { - 'kind': 'pipeline', - 'type': 'docker', - 'name': 'coding-standard-js', - 'workspace' : { - 'base': '/var/www/owncloud', - 'path': 'server/apps/%s' % config['app'] - }, - 'steps': [ - { - 'name': 'coding-standard-js', - 'image': 'owncloudci/php:7.2', - 'pull': 'always', - 'commands': [ - 'make test-js-style' - ] - } - ], - 'depends_on': [], - 'trigger': { - 'ref': [ - 'refs/pull/**', - 'refs/tags/**' - ] - } - } + pipelines = [] + + if "jscodestyle" not in config: + return pipelines + + if type(config["jscodestyle"]) == "bool": + if not config["jscodestyle"]: + return pipelines + + result = { + "kind": "pipeline", + "type": "docker", + "name": "coding-standard-js", + "workspace": { + "base": "/var/www/owncloud", + "path": "server/apps/%s" % config["app"], + }, + "steps": [ + { + "name": "coding-standard-js", + "image": "owncloudci/php:7.2", + "pull": "always", + "commands": [ + "make test-js-style", + ], + }, + ], + "depends_on": [], + "trigger": { + "ref": [ + "refs/pull/**", + "refs/tags/**", + ], + }, + } - for branch in config['branches']: - result['trigger']['ref'].append('refs/heads/%s' % branch) + for branch in config["branches"]: + result["trigger"]["ref"].append("refs/heads/%s" % branch) - pipelines.append(result) + pipelines.append(result) - return pipelines + return pipelines def phpstan(): - pipelines = [] - - if 'phpstan' not in config: - return pipelines - - default = { - 'phpVersions': ['7.2'], - 'logLevel': '2', - } - - if 'defaults' in config: - if 'phpstan' in config['defaults']: - for item in config['defaults']['phpstan']: - default[item] = config['defaults']['phpstan'][item] - - phpstanConfig = config['phpstan'] - - if type(phpstanConfig) == "bool": - if phpstanConfig: - # the config has 'phpstan' true, so specify an empty dict that will get the defaults - phpstanConfig = {} - else: - return pipelines - - if len(phpstanConfig) == 0: - # 'phpstan' is an empty dict, so specify a single section that will get the defaults - phpstanConfig = {'doDefault': {}} - - for category, matrix in phpstanConfig.items(): - params = {} - for item in default: - params[item] = matrix[item] if item in matrix else default[item] - - for phpVersion in params['phpVersions']: - name = 'phpstan-php%s' % phpVersion - - result = { - 'kind': 'pipeline', - 'type': 'docker', - 'name': name, - 'workspace' : { - 'base': '/var/www/owncloud', - 'path': 'server/apps/%s' % config['app'] - }, - 'steps': - installCore('daily-master-qa', 'sqlite', False) + - installApp(phpVersion) + - setupServerAndApp(phpVersion, params['logLevel']) + - [ - { - 'name': 'phpstan', - 'image': 'owncloudci/php:%s' % phpVersion, - 'pull': 'always', - 'commands': [ - 'make test-php-phpstan' - ] - } - ], - 'depends_on': [], - 'trigger': { - 'ref': [ - 'refs/pull/**', - 'refs/tags/**' - ] - } - } - - for branch in config['branches']: - result['trigger']['ref'].append('refs/heads/%s' % branch) - - pipelines.append(result) - - return pipelines + pipelines = [] + + if "phpstan" not in config: + return pipelines + + default = { + "phpVersions": ["7.2"], + "logLevel": "2", + } + + if "defaults" in config: + if "phpstan" in config["defaults"]: + for item in config["defaults"]["phpstan"]: + default[item] = config["defaults"]["phpstan"][item] + + phpstanConfig = config["phpstan"] + + if type(phpstanConfig) == "bool": + if phpstanConfig: + # the config has 'phpstan' true, so specify an empty dict that will get the defaults + phpstanConfig = {} + else: + return pipelines + + if len(phpstanConfig) == 0: + # 'phpstan' is an empty dict, so specify a single section that will get the defaults + phpstanConfig = {"doDefault": {}} + + for category, matrix in phpstanConfig.items(): + params = {} + for item in default: + params[item] = matrix[item] if item in matrix else default[item] + + for phpVersion in params["phpVersions"]: + name = "phpstan-php%s" % phpVersion + + result = { + "kind": "pipeline", + "type": "docker", + "name": name, + "workspace": { + "base": "/var/www/owncloud", + "path": "server/apps/%s" % config["app"], + }, + "steps": installCore("daily-master-qa", "sqlite", False) + + installApp(phpVersion) + + setupServerAndApp(phpVersion, params["logLevel"]) + + [ + { + "name": "phpstan", + "image": "owncloudci/php:%s" % phpVersion, + "pull": "always", + "commands": [ + "make test-php-phpstan", + ], + }, + ], + "depends_on": [], + "trigger": { + "ref": [ + "refs/pull/**", + "refs/tags/**", + ], + }, + } + + for branch in config["branches"]: + result["trigger"]["ref"].append("refs/heads/%s" % branch) + + pipelines.append(result) + + return pipelines def phan(): - pipelines = [] - - if 'phan' not in config: - return pipelines - - default = { - 'phpVersions': ['7.2', '7.3'], - } - - if 'defaults' in config: - if 'phan' in config['defaults']: - for item in config['defaults']['phan']: - default[item] = config['defaults']['phan'][item] - - phanConfig = config['phan'] - - if type(phanConfig) == "bool": - if phanConfig: - # the config has 'phan' true, so specify an empty dict that will get the defaults - phanConfig = {} - else: - return pipelines - - if len(phanConfig) == 0: - # 'phan' is an empty dict, so specify a single section that will get the defaults - phanConfig = {'doDefault': {}} - - for category, matrix in phanConfig.items(): - params = {} - for item in default: - params[item] = matrix[item] if item in matrix else default[item] - - for phpVersion in params['phpVersions']: - name = 'phan-php%s' % phpVersion - - result = { - 'kind': 'pipeline', - 'type': 'docker', - 'name': name, - 'workspace' : { - 'base': '/var/www/owncloud', - 'path': 'server/apps/%s' % config['app'] - }, - 'steps': - installCore('daily-master-qa', 'sqlite', False) + - [ - { - 'name': 'phan', - 'image': 'owncloudci/php:%s' % phpVersion, - 'pull': 'always', - 'commands': [ - 'make test-php-phan' - ] - } - ], - 'depends_on': [], - 'trigger': { - 'ref': [ - 'refs/pull/**', - 'refs/tags/**' - ] - } - } - - for branch in config['branches']: - result['trigger']['ref'].append('refs/heads/%s' % branch) - - pipelines.append(result) - - return pipelines + pipelines = [] + + if "phan" not in config: + return pipelines + + default = { + "phpVersions": ["7.2", "7.3"], + } + + if "defaults" in config: + if "phan" in config["defaults"]: + for item in config["defaults"]["phan"]: + default[item] = config["defaults"]["phan"][item] + + phanConfig = config["phan"] + + if type(phanConfig) == "bool": + if phanConfig: + # the config has 'phan' true, so specify an empty dict that will get the defaults + phanConfig = {} + else: + return pipelines + + if len(phanConfig) == 0: + # 'phan' is an empty dict, so specify a single section that will get the defaults + phanConfig = {"doDefault": {}} + + for category, matrix in phanConfig.items(): + params = {} + for item in default: + params[item] = matrix[item] if item in matrix else default[item] + + for phpVersion in params["phpVersions"]: + name = "phan-php%s" % phpVersion + + result = { + "kind": "pipeline", + "type": "docker", + "name": name, + "workspace": { + "base": "/var/www/owncloud", + "path": "server/apps/%s" % config["app"], + }, + "steps": installCore("daily-master-qa", "sqlite", False) + + [ + { + "name": "phan", + "image": "owncloudci/php:%s" % phpVersion, + "pull": "always", + "commands": [ + "make test-php-phan", + ], + }, + ], + "depends_on": [], + "trigger": { + "ref": [ + "refs/pull/**", + "refs/tags/**", + ], + }, + } + + for branch in config["branches"]: + result["trigger"]["ref"].append("refs/heads/%s" % branch) + + pipelines.append(result) + + return pipelines def build(): - pipelines = [] - - if 'build' not in config: - return pipelines - - default = { - 'phpVersions': ['7.2'], - 'commands': [ - 'make dist' - ], - 'extraEnvironment': {}, - 'configureTarOnTag': False, - } - - if 'defaults' in config: - if 'build' in config['defaults']: - for item in config['defaults']['build']: - default[item] = config['defaults']['build'][item] - - matrix = config['build'] - - if type(matrix) == "bool": - if matrix: - # the config has 'build' true, so specify an empty dict that will get the defaults - matrix = {} - else: - return pipelines - - params = {} - for item in default: - params[item] = matrix[item] if item in matrix else default[item] - - for phpVersion in params['phpVersions']: - result = { - 'kind': 'pipeline', - 'type': 'docker', - 'name': 'build', - 'workspace' : { - 'base': '/var/www/owncloud', - 'path': 'server/apps/%s' % config['app'] - }, - 'steps': [ - { - 'name': 'build', - 'image': 'owncloudci/php:%s' % phpVersion, - 'pull': 'always', - 'environment': params['extraEnvironment'], - 'commands': params['commands'] - } - ] + ([ - { - 'name': 'github_release', - 'image': 'plugins/github-release', - 'pull': 'always', - 'settings': { - 'checksum': 'sha256', - 'file_exists': 'overwrite', - 'files': 'build/dist/%s.tar.gz' % config['app'], - 'prerelease': True, - }, - 'environment': { - 'GITHUB_TOKEN': { - 'from_secret': 'github_token' - }, - }, - 'when': { - 'event': [ - 'tag' - ] - }, - } - ] if params['configureTarOnTag'] else []), - 'depends_on': [], - 'trigger': { - 'ref': [ - 'refs/pull/**', - 'refs/tags/**' - ] - } - } - - for branch in config['branches']: - result['trigger']['ref'].append('refs/heads/%s' % branch) - - pipelines.append(result) - - return pipelines + pipelines = [] + + if "build" not in config: + return pipelines + + default = { + "phpVersions": ["7.2"], + "commands": [ + "make dist", + ], + "extraEnvironment": {}, + "configureTarOnTag": False, + } + + if "defaults" in config: + if "build" in config["defaults"]: + for item in config["defaults"]["build"]: + default[item] = config["defaults"]["build"][item] + + matrix = config["build"] + + if type(matrix) == "bool": + if matrix: + # the config has 'build' true, so specify an empty dict that will get the defaults + matrix = {} + else: + return pipelines + + params = {} + for item in default: + params[item] = matrix[item] if item in matrix else default[item] + + for phpVersion in params["phpVersions"]: + result = { + "kind": "pipeline", + "type": "docker", + "name": "build", + "workspace": { + "base": "/var/www/owncloud", + "path": "server/apps/%s" % config["app"], + }, + "steps": [ + { + "name": "build", + "image": "owncloudci/php:%s" % phpVersion, + "pull": "always", + "environment": params["extraEnvironment"], + "commands": params["commands"], + }, + ] + ([ + { + "name": "github_release", + "image": "plugins/github-release", + "pull": "always", + "settings": { + "checksum": "sha256", + "file_exists": "overwrite", + "files": "build/dist/%s.tar.gz" % config["app"], + "prerelease": True, + }, + "environment": { + "GITHUB_TOKEN": { + "from_secret": "github_token", + }, + }, + "when": { + "event": [ + "tag", + ], + }, + }, + ] if params["configureTarOnTag"] else []), + "depends_on": [], + "trigger": { + "ref": [ + "refs/pull/**", + "refs/tags/**", + ], + }, + } + + for branch in config["branches"]: + result["trigger"]["ref"].append("refs/heads/%s" % branch) + + pipelines.append(result) + + return pipelines def javascript(): - pipelines = [] - - if 'javascript' not in config: - return pipelines - - default = { - 'coverage': False, - 'logLevel': '2', - 'extraSetup': [], - 'extraServices': [], - 'extraEnvironment': {}, - 'extraCommandsBeforeTestRun': [], - 'extraTeardown': [], - } - - if 'defaults' in config: - if 'javascript' in config['defaults']: - for item in config['defaults']['javascript']: - default[item] = config['defaults']['javascript'][item] - - matrix = config['javascript'] - - if type(matrix) == "bool": - if matrix: - # the config has 'javascript' true, so specify an empty dict that will get the defaults - matrix = {} - else: - return pipelines - - params = {} - for item in default: - params[item] = matrix[item] if item in matrix else default[item] - - result = { - 'kind': 'pipeline', - 'type': 'docker', - 'name': 'javascript-tests', - 'workspace' : { - 'base': '/var/www/owncloud', - 'path': 'server/apps/%s' % config['app'] - }, - 'steps': - installCore('daily-master-qa', 'sqlite', False) + - installApp('7.2') + - setupServerAndApp('7.2', params['logLevel']) + - params['extraSetup'] + - [ - { - 'name': 'js-tests', - 'image': 'owncloudci/php:7.2', - 'pull': 'always', - 'environment': params['extraEnvironment'], - 'commands': params['extraCommandsBeforeTestRun'] + [ - 'make test-js' - ] - } - ] + params['extraTeardown'], - 'services': params['extraServices'], - 'depends_on': [], - 'trigger': { - 'ref': [ - 'refs/pull/**', - 'refs/tags/**' - ] - } - } - - if params['coverage']: - result['steps'].append({ - 'name': 'codecov-js', - 'image': 'plugins/codecov:2', - 'pull': 'always', - 'settings': { - 'paths': [ - 'coverage/*.info', - ], - 'token': { - 'from_secret': 'codecov_token' - } - } - }) - - for branch in config['branches']: - result['trigger']['ref'].append('refs/heads/%s' % branch) - - return [result] + pipelines = [] + + if "javascript" not in config: + return pipelines + + default = { + "coverage": False, + "logLevel": "2", + "extraSetup": [], + "extraServices": [], + "extraEnvironment": {}, + "extraCommandsBeforeTestRun": [], + "extraTeardown": [], + } + + if "defaults" in config: + if "javascript" in config["defaults"]: + for item in config["defaults"]["javascript"]: + default[item] = config["defaults"]["javascript"][item] + + matrix = config["javascript"] + + if type(matrix) == "bool": + if matrix: + # the config has 'javascript' true, so specify an empty dict that will get the defaults + matrix = {} + else: + return pipelines + + params = {} + for item in default: + params[item] = matrix[item] if item in matrix else default[item] + + result = { + "kind": "pipeline", + "type": "docker", + "name": "javascript-tests", + "workspace": { + "base": "/var/www/owncloud", + "path": "server/apps/%s" % config["app"], + }, + "steps": installCore("daily-master-qa", "sqlite", False) + + installApp("7.2") + + setupServerAndApp("7.2", params["logLevel"]) + + params["extraSetup"] + + [ + { + "name": "js-tests", + "image": "owncloudci/php:7.2", + "pull": "always", + "environment": params["extraEnvironment"], + "commands": params["extraCommandsBeforeTestRun"] + [ + "make test-js", + ], + }, + ] + params["extraTeardown"], + "services": params["extraServices"], + "depends_on": [], + "trigger": { + "ref": [ + "refs/pull/**", + "refs/tags/**", + ], + }, + } + + if params["coverage"]: + result["steps"].append({ + "name": "codecov-js", + "image": "plugins/codecov:2", + "pull": "always", + "settings": { + "paths": [ + "coverage/*.info", + ], + "token": { + "from_secret": "codecov_token", + }, + }, + }) + + for branch in config["branches"]: + result["trigger"]["ref"].append("refs/heads/%s" % branch) + + return [result] def phptests(testType): - pipelines = [] - - if testType not in config: - return pipelines - - errorFound = False - - default = { - 'phpVersions': ['7.2', '7.3'], - 'databases': [ - 'sqlite', 'mariadb:10.2', 'mysql:5.5', 'mysql:5.7', 'postgres:9.4', 'oracle' - ], - 'coverage': True, - 'includeKeyInMatrixName': False, - 'logLevel': '2', - 'cephS3': False, - 'scalityS3': False, - 'extraSetup': [], - 'extraServices': [], - 'extraEnvironment': {}, - 'extraCommandsBeforeTestRun': [], - 'extraApps': {}, - 'extraTeardown': [], - } - - if 'defaults' in config: - if testType in config['defaults']: - for item in config['defaults'][testType]: - default[item] = config['defaults'][testType][item] - - phptestConfig = config[testType] - - if type(phptestConfig) == "bool": - if phptestConfig: - # the config has just True, so specify an empty dict that will get the defaults - phptestConfig = {} - else: - return pipelines - - if len(phptestConfig) == 0: - # the PHP test config is an empty dict, so specify a single section that will get the defaults - phptestConfig = {'doDefault': {}} - - for category, matrix in phptestConfig.items(): - params = {} - for item in default: - params[item] = matrix[item] if item in matrix else default[item] - - cephS3Params = params['cephS3'] - if type(cephS3Params) == "bool": - cephS3Needed = cephS3Params - filesPrimaryS3NeededForCeph = cephS3Params - else: - cephS3Needed = True - filesPrimaryS3NeededForCeph = cephS3Params['filesPrimaryS3Needed'] if 'filesPrimaryS3Needed' in cephS3Params else True - - scalityS3Params = params['scalityS3'] - if type(scalityS3Params) == "bool": - scalityS3Needed = scalityS3Params - filesPrimaryS3NeededForScality = scalityS3Params - else: - scalityS3Needed = True - filesPrimaryS3NeededForScality = scalityS3Params['filesPrimaryS3Needed'] if 'filesPrimaryS3Needed' in scalityS3Params else True - - if ((config['app'] != 'files_primary_s3') and (filesPrimaryS3NeededForCeph or filesPrimaryS3NeededForScality)): - # If we are not already 'files_primary_s3' and we need S3 storage, then install the 'files_primary_s3' app - extraAppsDict = { - 'files_primary_s3': 'composer install' - } - for app, command in params['extraApps'].items(): - extraAppsDict[app] = command - params['extraApps'] = extraAppsDict - - for phpVersion in params['phpVersions']: - - if testType == 'phpunit': - if params['coverage']: - command = 'make test-php-unit-dbg' - else: - command = 'make test-php-unit' - else: - if params['coverage']: - command = 'make test-php-integration-dbg' - else: - command = 'make test-php-integration' - - for db in params['databases']: - keyString = '-' + category if params['includeKeyInMatrixName'] else '' - name = '%s%s-php%s-%s' % (testType, keyString, phpVersion, db.replace(":", "")) - maxLength = 50 - nameLength = len(name) - if nameLength > maxLength: - print("Error: generated phpunit stage name of length", nameLength, "is not supported. The maximum length is " + str(maxLength) + ".", name) - errorFound = True - - result = { - 'kind': 'pipeline', - 'type': 'docker', - 'name': name, - 'workspace' : { - 'base': '/var/www/owncloud', - 'path': 'server/apps/%s' % config['app'] - }, - 'steps': - installCore('daily-master-qa', db, False) + - installApp(phpVersion) + - installExtraApps(phpVersion, params['extraApps']) + - setupServerAndApp(phpVersion, params['logLevel']) + - setupCeph(params['cephS3']) + - setupScality(params['scalityS3']) + - params['extraSetup'] + - [ - { - 'name': '%s-tests' % testType, - 'image': 'owncloudci/php:%s' % phpVersion, - 'pull': 'always', - 'environment': params['extraEnvironment'], - 'commands': params['extraCommandsBeforeTestRun'] + [ - command - ] - } - ] + params['extraTeardown'], - 'services': - databaseService(db) + - cephService(params['cephS3']) + - scalityService(params['scalityS3']) + - params['extraServices'], - 'depends_on': [], - 'trigger': { - 'ref': [ - 'refs/pull/**', - 'refs/tags/**' - ] - } - } - - if params['coverage']: - result['steps'].append({ - 'name': 'codecov-upload', - 'image': 'plugins/codecov:2', - 'pull': 'always', - 'settings': { - 'paths': [ - 'tests/output/clover.xml', - ], - 'token': { - 'from_secret': 'codecov_token' - } - } - }) - - for branch in config['branches']: - result['trigger']['ref'].append('refs/heads/%s' % branch) - - pipelines.append(result) - - if errorFound: - return False - - return pipelines + pipelines = [] + + if testType not in config: + return pipelines + + errorFound = False + + default = { + "phpVersions": ["7.2", "7.3"], + "databases": [ + "sqlite", + "mariadb:10.2", + "mysql:5.5", + "mysql:5.7", + "postgres:9.4", + "oracle", + ], + "coverage": True, + "includeKeyInMatrixName": False, + "logLevel": "2", + "cephS3": False, + "scalityS3": False, + "extraSetup": [], + "extraServices": [], + "extraEnvironment": {}, + "extraCommandsBeforeTestRun": [], + "extraApps": {}, + "extraTeardown": [], + } + + if "defaults" in config: + if testType in config["defaults"]: + for item in config["defaults"][testType]: + default[item] = config["defaults"][testType][item] + + phptestConfig = config[testType] + + if type(phptestConfig) == "bool": + if phptestConfig: + # the config has just True, so specify an empty dict that will get the defaults + phptestConfig = {} + else: + return pipelines + + if len(phptestConfig) == 0: + # the PHP test config is an empty dict, so specify a single section that will get the defaults + phptestConfig = {"doDefault": {}} + + for category, matrix in phptestConfig.items(): + params = {} + for item in default: + params[item] = matrix[item] if item in matrix else default[item] + + cephS3Params = params["cephS3"] + if type(cephS3Params) == "bool": + cephS3Needed = cephS3Params + filesPrimaryS3NeededForCeph = cephS3Params + else: + cephS3Needed = True + filesPrimaryS3NeededForCeph = cephS3Params["filesPrimaryS3Needed"] if "filesPrimaryS3Needed" in cephS3Params else True + + scalityS3Params = params["scalityS3"] + if type(scalityS3Params) == "bool": + scalityS3Needed = scalityS3Params + filesPrimaryS3NeededForScality = scalityS3Params + else: + scalityS3Needed = True + filesPrimaryS3NeededForScality = scalityS3Params["filesPrimaryS3Needed"] if "filesPrimaryS3Needed" in scalityS3Params else True + + if ((config["app"] != "files_primary_s3") and (filesPrimaryS3NeededForCeph or filesPrimaryS3NeededForScality)): + # If we are not already 'files_primary_s3' and we need S3 storage, then install the 'files_primary_s3' app + extraAppsDict = { + "files_primary_s3": "composer install", + } + for app, command in params["extraApps"].items(): + extraAppsDict[app] = command + params["extraApps"] = extraAppsDict + + for phpVersion in params["phpVersions"]: + if testType == "phpunit": + if params["coverage"]: + command = "make test-php-unit-dbg" + else: + command = "make test-php-unit" + elif params["coverage"]: + command = "make test-php-integration-dbg" + else: + command = "make test-php-integration" + + for db in params["databases"]: + keyString = "-" + category if params["includeKeyInMatrixName"] else "" + name = "%s%s-php%s-%s" % (testType, keyString, phpVersion, db.replace(":", "")) + maxLength = 50 + nameLength = len(name) + if nameLength > maxLength: + print("Error: generated phpunit stage name of length", nameLength, "is not supported. The maximum length is " + str(maxLength) + ".", name) + errorFound = True + + result = { + "kind": "pipeline", + "type": "docker", + "name": name, + "workspace": { + "base": "/var/www/owncloud", + "path": "server/apps/%s" % config["app"], + }, + "steps": installCore("daily-master-qa", db, False) + + installApp(phpVersion) + + installExtraApps(phpVersion, params["extraApps"]) + + setupServerAndApp(phpVersion, params["logLevel"]) + + setupCeph(params["cephS3"]) + + setupScality(params["scalityS3"]) + + params["extraSetup"] + + [ + { + "name": "%s-tests" % testType, + "image": "owncloudci/php:%s" % phpVersion, + "pull": "always", + "environment": params["extraEnvironment"], + "commands": params["extraCommandsBeforeTestRun"] + [ + command, + ], + }, + ] + params["extraTeardown"], + "services": databaseService(db) + + cephService(params["cephS3"]) + + scalityService(params["scalityS3"]) + + params["extraServices"], + "depends_on": [], + "trigger": { + "ref": [ + "refs/pull/**", + "refs/tags/**", + ], + }, + } + + if params["coverage"]: + result["steps"].append({ + "name": "codecov-upload", + "image": "plugins/codecov:2", + "pull": "always", + "settings": { + "paths": [ + "tests/output/clover.xml", + ], + "token": { + "from_secret": "codecov_token", + }, + }, + }) + + for branch in config["branches"]: + result["trigger"]["ref"].append("refs/heads/%s" % branch) + + pipelines.append(result) + + if errorFound: + return False + + return pipelines def acceptance(): - pipelines = [] - - if 'acceptance' not in config: - return pipelines - - if type(config['acceptance']) == "bool": - if not config['acceptance']: - return pipelines - - errorFound = False - - default = { - 'servers': ['daily-master-qa', 'latest'], - 'browsers': ['chrome'], - 'phpVersions': ['7.2'], - 'databases': ['mariadb:10.2'], - 'federatedServerNeeded': False, - 'filterTags': '', - 'logLevel': '2', - 'emailNeeded': False, - 'ldapNeeded': False, - 'cephS3': False, - 'scalityS3': False, - 'ssl': False, - 'xForwardedFor': False, - 'extraSetup': [], - 'extraServices': [], - 'extraTeardown': [], - 'extraEnvironment': {}, - 'extraCommandsBeforeTestRun': [], - 'extraApps': {}, - 'useBundledApp': False, - 'includeKeyInMatrixName': False, - 'runAllSuites': False, - 'runCoreTests': False, - 'numberOfParts': 1, - 'cron': '', - } - - if 'defaults' in config: - if 'acceptance' in config['defaults']: - for item in config['defaults']['acceptance']: - default[item] = config['defaults']['acceptance'][item] - - for category, matrix in config['acceptance'].items(): - if type(matrix['suites']) == "list": - suites = {} - for suite in matrix['suites']: - suites[suite] = suite - else: - suites = matrix['suites'] - - for suite, alternateSuiteName in suites.items(): - isWebUI = suite.startswith('webUI') - isAPI = suite.startswith('api') - isCLI = suite.startswith('cli') - - params = {} - for item in default: - params[item] = matrix[item] if item in matrix else default[item] - - if isAPI or isCLI: - params['browsers'] = [''] - - cephS3Params = params['cephS3'] - if type(cephS3Params) == "bool": - cephS3Needed = cephS3Params - filesPrimaryS3NeededForCeph = cephS3Params - else: - cephS3Needed = True - filesPrimaryS3NeededForCeph = cephS3Params['filesPrimaryS3Needed'] if 'filesPrimaryS3Needed' in cephS3Params else True - - scalityS3Params = params['scalityS3'] - if type(scalityS3Params) == "bool": - scalityS3Needed = scalityS3Params - filesPrimaryS3NeededForScality = scalityS3Params - else: - scalityS3Needed = True - filesPrimaryS3NeededForScality = scalityS3Params['filesPrimaryS3Needed'] if 'filesPrimaryS3Needed' in scalityS3Params else True - - if ((config['app'] != 'files_primary_s3') and (filesPrimaryS3NeededForCeph or filesPrimaryS3NeededForScality)): - # If we are not already 'files_primary_s3' and we need S3 object storage, then install the 'files_primary_s3' app - extraAppsDict = { - 'files_primary_s3': 'composer install' - } - for app, command in params['extraApps'].items(): - extraAppsDict[app] = command - params['extraApps'] = extraAppsDict - - for server in params['servers']: - for browser in params['browsers']: - for phpVersion in params['phpVersions']: - for db in params['databases']: - for runPart in range(1, params['numberOfParts'] + 1): - name = 'unknown' - - if isWebUI or isAPI or isCLI: - browserString = '' if browser == '' else '-' + browser - keyString = '-' + category if params['includeKeyInMatrixName'] else '' - partString = '' if params['numberOfParts'] == 1 else '-%d-%d' % (params['numberOfParts'], runPart) - name = '%s%s%s-%s%s-%s-php%s' % (alternateSuiteName, keyString, partString, server.replace('daily-', '').replace('-qa', ''), browserString, db.replace(':', ''), phpVersion) - maxLength = 50 - nameLength = len(name) - if nameLength > maxLength: - print("Error: generated stage name of length", nameLength, "is not supported. The maximum length is " + str(maxLength) + ".", name) - errorFound = True - - environment = {} - for env in params['extraEnvironment']: - environment[env] = params['extraEnvironment'][env] - - environment['TEST_SERVER_URL'] = 'http://server' - environment['BEHAT_FILTER_TAGS'] = params['filterTags'] - - if (params['runAllSuites'] == False): - environment['BEHAT_SUITE'] = suite - else: - environment['DIVIDE_INTO_NUM_PARTS'] = params['numberOfParts'] - environment['RUN_PART'] = runPart - - if isWebUI: - environment['SELENIUM_HOST'] = 'selenium' - environment['SELENIUM_PORT'] = '4444' - environment['BROWSER'] = browser - environment['PLATFORM'] = 'Linux' - if (params['runCoreTests']): - makeParameter = 'test-acceptance-core-webui' - else: - makeParameter = 'test-acceptance-webui' - - if isAPI: - if (params['runCoreTests']): - makeParameter = 'test-acceptance-core-api' - else: - makeParameter = 'test-acceptance-api' - - if isCLI: - if (params['runCoreTests']): - makeParameter = 'test-acceptance-core-cli' - else: - makeParameter = 'test-acceptance-cli' - - if params['emailNeeded']: - environment['MAILHOG_HOST'] = 'email' - - if params['ldapNeeded']: - environment['TEST_EXTERNAL_USER_BACKENDS'] = True - - if (cephS3Needed or scalityS3Needed): - environment['OC_TEST_ON_OBJECTSTORE'] = '1' - if (params['cephS3'] != False): - environment['S3_TYPE'] = 'ceph' - if (params['scalityS3'] != False): - environment['S3_TYPE'] = 'scality' - federationDbSuffix = '-federated' - - result = { - 'kind': 'pipeline', - 'type': 'docker', - 'name': name, - 'workspace' : { - 'base': '/var/www/owncloud', - 'path': 'testrunner/apps/%s' % config['app'] - }, - 'steps': - installCore(server, db, params['useBundledApp']) + - installTestrunner(phpVersion, params['useBundledApp']) + - (installFederated(server, phpVersion, params['logLevel'], db, federationDbSuffix) + owncloudLog('federated') if params['federatedServerNeeded'] else []) + - installApp(phpVersion) + - installExtraApps(phpVersion, params['extraApps']) + - setupServerAndApp(phpVersion, params['logLevel']) + - owncloudLog('server') + - setupCeph(params['cephS3']) + - setupScality(params['scalityS3']) + - params['extraSetup'] + - fixPermissions(phpVersion, params['federatedServerNeeded']) + - [ - ({ - 'name': 'acceptance-tests', - 'image': 'owncloudci/php:%s' % phpVersion, - 'pull': 'always', - 'environment': environment, - 'commands': params['extraCommandsBeforeTestRun'] + [ - 'touch /var/www/owncloud/saved-settings.sh', - '. /var/www/owncloud/saved-settings.sh', - 'make %s' % makeParameter - ] - }), - ] + params['extraTeardown'], - 'services': - databaseService(db) + - browserService(browser) + - emailService(params['emailNeeded']) + - ldapService(params['ldapNeeded']) + - cephService(params['cephS3']) + - scalityService(params['scalityS3']) + - params['extraServices'] + - owncloudService(server, phpVersion, 'server', '/var/www/owncloud/server', params['ssl'], params['xForwardedFor']) + - (( - owncloudService(server, phpVersion, 'federated', '/var/www/owncloud/federated', params['ssl'], params['xForwardedFor']) + - databaseServiceForFederation(db, federationDbSuffix) - ) if params['federatedServerNeeded'] else [] ), - 'depends_on': [], - 'trigger': {} - } - - if (params['cron'] == ''): - result['trigger']['ref'] = [ - 'refs/pull/**', - 'refs/tags/**' - ] - for branch in config['branches']: - result['trigger']['ref'].append('refs/heads/%s' % branch) - else: - result['trigger']['cron'] = params['cron'] - - pipelines.append(result) - - if errorFound: - return False - - return pipelines + pipelines = [] + + if "acceptance" not in config: + return pipelines + + if type(config["acceptance"]) == "bool": + if not config["acceptance"]: + return pipelines + + errorFound = False + + default = { + "servers": ["daily-master-qa", "latest"], + "browsers": ["chrome"], + "phpVersions": ["7.2"], + "databases": ["mariadb:10.2"], + "federatedServerNeeded": False, + "filterTags": "", + "logLevel": "2", + "emailNeeded": False, + "ldapNeeded": False, + "cephS3": False, + "scalityS3": False, + "ssl": False, + "xForwardedFor": False, + "extraSetup": [], + "extraServices": [], + "extraTeardown": [], + "extraEnvironment": {}, + "extraCommandsBeforeTestRun": [], + "extraApps": {}, + "useBundledApp": False, + "includeKeyInMatrixName": False, + "runAllSuites": False, + "runCoreTests": False, + "numberOfParts": 1, + "cron": "", + } + + if "defaults" in config: + if "acceptance" in config["defaults"]: + for item in config["defaults"]["acceptance"]: + default[item] = config["defaults"]["acceptance"][item] + + for category, matrix in config["acceptance"].items(): + if type(matrix["suites"]) == "list": + suites = {} + for suite in matrix["suites"]: + suites[suite] = suite + else: + suites = matrix["suites"] + + for suite, alternateSuiteName in suites.items(): + isWebUI = suite.startswith("webUI") + isAPI = suite.startswith("api") + isCLI = suite.startswith("cli") + + params = {} + for item in default: + params[item] = matrix[item] if item in matrix else default[item] + + if isAPI or isCLI: + params["browsers"] = [""] + + cephS3Params = params["cephS3"] + if type(cephS3Params) == "bool": + cephS3Needed = cephS3Params + filesPrimaryS3NeededForCeph = cephS3Params + else: + cephS3Needed = True + filesPrimaryS3NeededForCeph = cephS3Params["filesPrimaryS3Needed"] if "filesPrimaryS3Needed" in cephS3Params else True + + scalityS3Params = params["scalityS3"] + if type(scalityS3Params) == "bool": + scalityS3Needed = scalityS3Params + filesPrimaryS3NeededForScality = scalityS3Params + else: + scalityS3Needed = True + filesPrimaryS3NeededForScality = scalityS3Params["filesPrimaryS3Needed"] if "filesPrimaryS3Needed" in scalityS3Params else True + + if ((config["app"] != "files_primary_s3") and (filesPrimaryS3NeededForCeph or filesPrimaryS3NeededForScality)): + # If we are not already 'files_primary_s3' and we need S3 object storage, then install the 'files_primary_s3' app + extraAppsDict = { + "files_primary_s3": "composer install", + } + for app, command in params["extraApps"].items(): + extraAppsDict[app] = command + params["extraApps"] = extraAppsDict + + for server in params["servers"]: + for browser in params["browsers"]: + for phpVersion in params["phpVersions"]: + for db in params["databases"]: + for runPart in range(1, params["numberOfParts"] + 1): + name = "unknown" + + if isWebUI or isAPI or isCLI: + browserString = "" if browser == "" else "-" + browser + keyString = "-" + category if params["includeKeyInMatrixName"] else "" + partString = "" if params["numberOfParts"] == 1 else "-%d-%d" % (params["numberOfParts"], runPart) + name = "%s%s%s-%s%s-%s-php%s" % (alternateSuiteName, keyString, partString, server.replace("daily-", "").replace("-qa", ""), browserString, db.replace(":", ""), phpVersion) + maxLength = 50 + nameLength = len(name) + if nameLength > maxLength: + print("Error: generated stage name of length", nameLength, "is not supported. The maximum length is " + str(maxLength) + ".", name) + errorFound = True + + environment = {} + for env in params["extraEnvironment"]: + environment[env] = params["extraEnvironment"][env] + + environment["TEST_SERVER_URL"] = "http://server" + environment["BEHAT_FILTER_TAGS"] = params["filterTags"] + + if (params["runAllSuites"] == False): + environment["BEHAT_SUITE"] = suite + else: + environment["DIVIDE_INTO_NUM_PARTS"] = params["numberOfParts"] + environment["RUN_PART"] = runPart + + if isWebUI: + environment["SELENIUM_HOST"] = "selenium" + environment["SELENIUM_PORT"] = "4444" + environment["BROWSER"] = browser + environment["PLATFORM"] = "Linux" + if (params["runCoreTests"]): + makeParameter = "test-acceptance-core-webui" + else: + makeParameter = "test-acceptance-webui" + + if isAPI: + if (params["runCoreTests"]): + makeParameter = "test-acceptance-core-api" + else: + makeParameter = "test-acceptance-api" + + if isCLI: + if (params["runCoreTests"]): + makeParameter = "test-acceptance-core-cli" + else: + makeParameter = "test-acceptance-cli" + + if params["emailNeeded"]: + environment["MAILHOG_HOST"] = "email" + + if params["ldapNeeded"]: + environment["TEST_EXTERNAL_USER_BACKENDS"] = True + + if (cephS3Needed or scalityS3Needed): + environment["OC_TEST_ON_OBJECTSTORE"] = "1" + if (params["cephS3"] != False): + environment["S3_TYPE"] = "ceph" + if (params["scalityS3"] != False): + environment["S3_TYPE"] = "scality" + federationDbSuffix = "-federated" + + result = { + "kind": "pipeline", + "type": "docker", + "name": name, + "workspace": { + "base": "/var/www/owncloud", + "path": "testrunner/apps/%s" % config["app"], + }, + "steps": installCore(server, db, params["useBundledApp"]) + + installTestrunner(phpVersion, params["useBundledApp"]) + + (installFederated(server, phpVersion, params["logLevel"], db, federationDbSuffix) + owncloudLog("federated") if params["federatedServerNeeded"] else []) + + installApp(phpVersion) + + installExtraApps(phpVersion, params["extraApps"]) + + setupServerAndApp(phpVersion, params["logLevel"]) + + owncloudLog("server") + + setupCeph(params["cephS3"]) + + setupScality(params["scalityS3"]) + + params["extraSetup"] + + fixPermissions(phpVersion, params["federatedServerNeeded"]) + + [ + ({ + "name": "acceptance-tests", + "image": "owncloudci/php:%s" % phpVersion, + "pull": "always", + "environment": environment, + "commands": params["extraCommandsBeforeTestRun"] + [ + "touch /var/www/owncloud/saved-settings.sh", + ". /var/www/owncloud/saved-settings.sh", + "make %s" % makeParameter, + ], + }), + ] + params["extraTeardown"], + "services": databaseService(db) + + browserService(browser) + + emailService(params["emailNeeded"]) + + ldapService(params["ldapNeeded"]) + + cephService(params["cephS3"]) + + scalityService(params["scalityS3"]) + + params["extraServices"] + + owncloudService(server, phpVersion, "server", "/var/www/owncloud/server", params["ssl"], params["xForwardedFor"]) + + (( + owncloudService(server, phpVersion, "federated", "/var/www/owncloud/federated", params["ssl"], params["xForwardedFor"]) + + databaseServiceForFederation(db, federationDbSuffix) + ) if params["federatedServerNeeded"] else []), + "depends_on": [], + "trigger": {}, + } + + if (params["cron"] == ""): + result["trigger"]["ref"] = [ + "refs/pull/**", + "refs/tags/**", + ] + for branch in config["branches"]: + result["trigger"]["ref"].append("refs/heads/%s" % branch) + else: + result["trigger"]["cron"] = params["cron"] + + pipelines.append(result) + + if errorFound: + return False + + return pipelines def notify(): - result = { - 'kind': 'pipeline', - 'type': 'docker', - 'name': 'chat-notifications', - 'clone': { - 'disable': True - }, - 'steps': [ - { - 'name': 'notify-rocketchat', - 'image': 'plugins/slack:1', - 'pull': 'always', - 'settings': { - 'webhook': { - 'from_secret': config['rocketchat']['from_secret'] - }, - 'channel': config['rocketchat']['channel'] - } - } - ], - 'depends_on': [], - 'trigger': { - 'ref': [ - 'refs/tags/**' - ], - 'status': [ - 'success', - 'failure' - ] - } - } + result = { + "kind": "pipeline", + "type": "docker", + "name": "chat-notifications", + "clone": { + "disable": True, + }, + "steps": [ + { + "name": "notify-rocketchat", + "image": "plugins/slack:1", + "pull": "always", + "settings": { + "webhook": { + "from_secret": config["rocketchat"]["from_secret"], + }, + "channel": config["rocketchat"]["channel"], + }, + }, + ], + "depends_on": [], + "trigger": { + "ref": [ + "refs/tags/**", + ], + "status": [ + "success", + "failure", + ], + }, + } - for branch in config['branches']: - result['trigger']['ref'].append('refs/heads/%s' % branch) + for branch in config["branches"]: + result["trigger"]["ref"].append("refs/heads/%s" % branch) - return result + return result def databaseService(db): - dbName = getDbName(db) - if (dbName == 'mariadb') or (dbName == 'mysql'): - return [{ - 'name': dbName, - 'image': db, - 'pull': 'always', - 'environment': { - 'MYSQL_USER': getDbUsername(db), - 'MYSQL_PASSWORD': getDbPassword(db), - 'MYSQL_DATABASE': getDbDatabase(db), - 'MYSQL_ROOT_PASSWORD': getDbRootPassword() - } - }] - - if dbName == 'postgres': - return [{ - 'name': dbName, - 'image': db, - 'pull': 'always', - 'environment': { - 'POSTGRES_USER': getDbUsername(db), - 'POSTGRES_PASSWORD': getDbPassword(db), - 'POSTGRES_DB': getDbDatabase(db) - } - }] - - if dbName == 'oracle': - return [{ - 'name': dbName, - 'image': 'owncloudci/oracle-xe:latest', - 'pull': 'always', - 'environment': { - 'ORACLE_USER': getDbUsername(db), - 'ORACLE_PASSWORD': getDbPassword(db), - 'ORACLE_DB': getDbDatabase(db), - 'ORACLE_DISABLE_ASYNCH_IO': 'true', - } - }] - - return [] + dbName = getDbName(db) + if (dbName == "mariadb") or (dbName == "mysql"): + return [{ + "name": dbName, + "image": db, + "pull": "always", + "environment": { + "MYSQL_USER": getDbUsername(db), + "MYSQL_PASSWORD": getDbPassword(db), + "MYSQL_DATABASE": getDbDatabase(db), + "MYSQL_ROOT_PASSWORD": getDbRootPassword(), + }, + }] + + if dbName == "postgres": + return [{ + "name": dbName, + "image": db, + "pull": "always", + "environment": { + "POSTGRES_USER": getDbUsername(db), + "POSTGRES_PASSWORD": getDbPassword(db), + "POSTGRES_DB": getDbDatabase(db), + }, + }] + + if dbName == "oracle": + return [{ + "name": dbName, + "image": "owncloudci/oracle-xe:latest", + "pull": "always", + "environment": { + "ORACLE_USER": getDbUsername(db), + "ORACLE_PASSWORD": getDbPassword(db), + "ORACLE_DB": getDbDatabase(db), + "ORACLE_DISABLE_ASYNCH_IO": "true", + }, + }] + + return [] def browserService(browser): - if browser == 'chrome': - return [{ - 'name': 'selenium', - 'image': 'selenium/standalone-chrome-debug:3.141.59-oxygen', - 'pull': 'always', - 'environment': { - 'JAVA_OPTS': '-Dselenium.LOGGER.level=WARNING' - } - }] - - if browser == 'firefox': - return [{ - 'name': 'selenium', - 'image': 'selenium/standalone-firefox-debug:3.8.1', - 'pull': 'always', - 'environment': { - 'JAVA_OPTS': '-Dselenium.LOGGER.level=WARNING', - 'SE_OPTS': '-enablePassThrough false' - } - }] - - return [] + if browser == "chrome": + return [{ + "name": "selenium", + "image": "selenium/standalone-chrome-debug:3.141.59-oxygen", + "pull": "always", + "environment": { + "JAVA_OPTS": "-Dselenium.LOGGER.level=WARNING", + }, + }] + + if browser == "firefox": + return [{ + "name": "selenium", + "image": "selenium/standalone-firefox-debug:3.8.1", + "pull": "always", + "environment": { + "JAVA_OPTS": "-Dselenium.LOGGER.level=WARNING", + "SE_OPTS": "-enablePassThrough false", + }, + }] + + return [] def emailService(emailNeeded): - if emailNeeded: - return [{ - 'name': 'email', - 'image': 'mailhog/mailhog', - 'pull': 'always', - }] + if emailNeeded: + return [{ + "name": "email", + "image": "mailhog/mailhog", + "pull": "always", + }] - return [] + return [] def ldapService(ldapNeeded): - if ldapNeeded: - return [{ - 'name': 'ldap', - 'image': 'osixia/openldap', - 'pull': 'always', - 'environment': { - 'LDAP_DOMAIN': 'owncloud.com', - 'LDAP_ORGANISATION': 'owncloud', - 'LDAP_ADMIN_PASSWORD': 'admin', - 'LDAP_TLS_VERIFY_CLIENT': 'never', - 'HOSTNAME': 'ldap', - } - }] - - return [] + if ldapNeeded: + return [{ + "name": "ldap", + "image": "osixia/openldap", + "pull": "always", + "environment": { + "LDAP_DOMAIN": "owncloud.com", + "LDAP_ORGANISATION": "owncloud", + "LDAP_ADMIN_PASSWORD": "admin", + "LDAP_TLS_VERIFY_CLIENT": "never", + "HOSTNAME": "ldap", + }, + }] + + return [] def scalityService(serviceParams): - serviceEnvironment = { - 'HOST_NAME': 'scality' - } - - if type(serviceParams) == "bool": - if not serviceParams: - return [] - else: - if 'extraEnvironment' in serviceParams: - for env in serviceParams['extraEnvironment']: - serviceEnvironment[env] = serviceParams['extraEnvironment'][env] - - return [{ - 'name': 'scality', - 'image': 'owncloudci/scality-s3server', - 'pull': 'always', - 'environment': serviceEnvironment - }] + serviceEnvironment = { + "HOST_NAME": "scality", + } + + if type(serviceParams) == "bool": + if not serviceParams: + return [] + elif "extraEnvironment" in serviceParams: + for env in serviceParams["extraEnvironment"]: + serviceEnvironment[env] = serviceParams["extraEnvironment"][env] + + return [{ + "name": "scality", + "image": "owncloudci/scality-s3server", + "pull": "always", + "environment": serviceEnvironment, + }] def cephService(serviceParams): - serviceEnvironment = { - 'NETWORK_AUTO_DETECT': '4', - 'RGW_NAME': 'ceph', - 'CEPH_DEMO_UID': 'owncloud', - 'CEPH_DEMO_ACCESS_KEY': 'owncloud123456', - 'CEPH_DEMO_SECRET_KEY': 'secret123456', - } - - if type(serviceParams) == "bool": - if not serviceParams: - return [] - else: - if 'extraEnvironment' in serviceParams: - for env in serviceParams['extraEnvironment']: - serviceEnvironment[env] = serviceParams['extraEnvironment'][env] - - return [{ - 'name': 'ceph', - 'image': 'owncloudci/ceph:tag-build-master-jewel-ubuntu-16.04', - 'pull': 'always', - 'environment': serviceEnvironment - }] - -def owncloudService(version, phpVersion, name = 'server', path = '/var/www/owncloud/server', ssl = True, xForwardedFor = False): - if ssl: - environment = { - 'APACHE_WEBROOT': path, - 'APACHE_CONFIG_TEMPLATE': 'ssl', - 'APACHE_SSL_CERT_CN': 'server', - 'APACHE_SSL_CERT': '/var/www/owncloud/%s.crt' % name, - 'APACHE_SSL_KEY': '/var/www/owncloud/%s.key' % name - } - else: - environment = { - 'APACHE_WEBROOT': path - } - - return [{ - 'name': name, - 'image': 'owncloudci/php:%s' % phpVersion, - 'pull': 'always', - 'environment': environment, - 'commands': ([ - 'a2enmod remoteip', - 'cd /etc/apache2', - 'echo "RemoteIPHeader X-Forwarded-For" >> apache2.conf', - # This replaces the first occurrence of "%h with "%a in apache2.conf file telling Apache to log the client - # IP as recorded by mod_remoteip (%a) rather than hostname (%h). For more info check this out: - # https://www.digitalocean.com/community/questions/get-client-public-ip-on-apache-server-used-behind-load-balancer - 'sed -i \'0,/"%h/s//"%a/\' apache2.conf', - ] if xForwardedFor else []) + [ - '/usr/local/bin/apachectl -e debug -D FOREGROUND', - ] - }] + serviceEnvironment = { + "NETWORK_AUTO_DETECT": "4", + "RGW_NAME": "ceph", + "CEPH_DEMO_UID": "owncloud", + "CEPH_DEMO_ACCESS_KEY": "owncloud123456", + "CEPH_DEMO_SECRET_KEY": "secret123456", + } + + if type(serviceParams) == "bool": + if not serviceParams: + return [] + elif "extraEnvironment" in serviceParams: + for env in serviceParams["extraEnvironment"]: + serviceEnvironment[env] = serviceParams["extraEnvironment"][env] + + return [{ + "name": "ceph", + "image": "owncloudci/ceph:tag-build-master-jewel-ubuntu-16.04", + "pull": "always", + "environment": serviceEnvironment, + }] + +def owncloudService(version, phpVersion, name = "server", path = "/var/www/owncloud/server", ssl = True, xForwardedFor = False): + if ssl: + environment = { + "APACHE_WEBROOT": path, + "APACHE_CONFIG_TEMPLATE": "ssl", + "APACHE_SSL_CERT_CN": "server", + "APACHE_SSL_CERT": "/var/www/owncloud/%s.crt" % name, + "APACHE_SSL_KEY": "/var/www/owncloud/%s.key" % name, + } + else: + environment = { + "APACHE_WEBROOT": path, + } + + return [{ + "name": name, + "image": "owncloudci/php:%s" % phpVersion, + "pull": "always", + "environment": environment, + "commands": ([ + "a2enmod remoteip", + "cd /etc/apache2", + 'echo "RemoteIPHeader X-Forwarded-For" >> apache2.conf', + # This replaces the first occurrence of "%h with "%a in apache2.conf file telling Apache to log the client + # IP as recorded by mod_remoteip (%a) rather than hostname (%h). For more info check this out: + # https://www.digitalocean.com/community/questions/get-client-public-ip-on-apache-server-used-behind-load-balancer + 'sed -i \'0,/"%h/s//"%a/\' apache2.conf', + ] if xForwardedFor else []) + [ + "/usr/local/bin/apachectl -e debug -D FOREGROUND", + ], + }] def getDbName(db): - return db.split(':')[0] + return db.split(":")[0] def getDbUsername(db): - name = getDbName(db) + name = getDbName(db) - # The Oracle image has the Db Username hardcoded - if name == 'oracle': - return 'autotest' + # The Oracle image has the Db Username hardcoded + if name == "oracle": + return "autotest" - return 'owncloud' + return "owncloud" def getDbPassword(db): - name = getDbName(db) + name = getDbName(db) - # The Oracle image has the Db Password hardcoded - if name == 'oracle': - return 'owncloud' + # The Oracle image has the Db Password hardcoded + if name == "oracle": + return "owncloud" - return 'owncloud' + return "owncloud" def getDbRootPassword(): - return 'owncloud' + return "owncloud" def getDbDatabase(db): - name = getDbName(db) + name = getDbName(db) - # The Oracle image has the Db Name hardcoded - if name == 'oracle': - return 'XE' + # The Oracle image has the Db Name hardcoded + if name == "oracle": + return "XE" - return 'owncloud' + return "owncloud" def installCore(version, db, useBundledApp): - host = getDbName(db) - dbType = host - - username = getDbUsername(db) - password = getDbPassword(db) - database = getDbDatabase(db) - - if host == 'mariadb': - dbType = 'mysql' - - if host == 'postgres': - dbType = 'pgsql' - - if host == 'oracle': - dbType = 'oci' - - stepDefinition = { - 'name': 'install-core', - 'image': 'owncloudci/core', - 'pull': 'always', - 'settings': { - 'version': version, - 'core_path': '/var/www/owncloud/server', - 'db_type': dbType, - 'db_name': database, - 'db_host': host, - 'db_username': username, - 'db_password': password - } - } - - if not useBundledApp: - stepDefinition['settings']['exclude'] = 'apps/%s' % config['app'] - - return [stepDefinition] + host = getDbName(db) + dbType = host + + username = getDbUsername(db) + password = getDbPassword(db) + database = getDbDatabase(db) + + if host == "mariadb": + dbType = "mysql" + + if host == "postgres": + dbType = "pgsql" + + if host == "oracle": + dbType = "oci" + + stepDefinition = { + "name": "install-core", + "image": "owncloudci/core", + "pull": "always", + "settings": { + "version": version, + "core_path": "/var/www/owncloud/server", + "db_type": dbType, + "db_name": database, + "db_host": host, + "db_username": username, + "db_password": password, + }, + } + + if not useBundledApp: + stepDefinition["settings"]["exclude"] = "apps/%s" % config["app"] + + return [stepDefinition] def installTestrunner(phpVersion, useBundledApp): - return [{ - 'name': 'install-testrunner', - 'image': 'owncloudci/php:%s' % phpVersion, - 'pull': 'always', - 'commands': [ - 'mkdir /tmp/testrunner', - 'git clone -b master --depth=1 https://github.com/owncloud/core.git /tmp/testrunner', - 'rsync -aIX /tmp/testrunner /var/www/owncloud', - ] + ([ - 'cp -r /var/www/owncloud/testrunner/apps/%s /var/www/owncloud/server/apps/' % config['app'] - ] if not useBundledApp else []) - }] + return [{ + "name": "install-testrunner", + "image": "owncloudci/php:%s" % phpVersion, + "pull": "always", + "commands": [ + "mkdir /tmp/testrunner", + "git clone -b master --depth=1 https://github.com/owncloud/core.git /tmp/testrunner", + "rsync -aIX /tmp/testrunner /var/www/owncloud", + ] + ([ + "cp -r /var/www/owncloud/testrunner/apps/%s /var/www/owncloud/server/apps/" % config["app"], + ] if not useBundledApp else []), + }] def installExtraApps(phpVersion, extraApps): - commandArray = [] - for app, command in extraApps.items(): - commandArray.append('git clone https://github.com/owncloud/%s.git /var/www/owncloud/testrunner/apps/%s' % (app, app)) - commandArray.append('cp -r /var/www/owncloud/testrunner/apps/%s /var/www/owncloud/server/apps/' % app) - if (command != ''): - commandArray.append('cd /var/www/owncloud/server/apps/%s' % app) - commandArray.append(command) - commandArray.append('cd /var/www/owncloud/server') - commandArray.append('php occ a:l') - commandArray.append('php occ a:e %s' % app) - commandArray.append('php occ a:l') - - if (commandArray == []): - return [] - - return [{ - 'name': 'install-extra-apps', - 'image': 'owncloudci/php:%s' % phpVersion, - 'pull': 'always', - 'commands': commandArray - }] + commandArray = [] + for app, command in extraApps.items(): + commandArray.append("git clone https://github.com/owncloud/%s.git /var/www/owncloud/testrunner/apps/%s" % (app, app)) + commandArray.append("cp -r /var/www/owncloud/testrunner/apps/%s /var/www/owncloud/server/apps/" % app) + if (command != ""): + commandArray.append("cd /var/www/owncloud/server/apps/%s" % app) + commandArray.append(command) + commandArray.append("cd /var/www/owncloud/server") + commandArray.append("php occ a:l") + commandArray.append("php occ a:e %s" % app) + commandArray.append("php occ a:l") + + if (commandArray == []): + return [] + + return [{ + "name": "install-extra-apps", + "image": "owncloudci/php:%s" % phpVersion, + "pull": "always", + "commands": commandArray, + }] def installApp(phpVersion): - if 'appInstallCommand' not in config: - return [] - - return [{ - 'name': 'install-app-%s' % config['app'], - 'image': 'owncloudci/php:%s' % phpVersion, - 'pull': 'always', - 'commands': [ - 'cd /var/www/owncloud/server/apps/%s' % config['app'], - config['appInstallCommand'] - ] - }] + if "appInstallCommand" not in config: + return [] + + return [{ + "name": "install-app-%s" % config["app"], + "image": "owncloudci/php:%s" % phpVersion, + "pull": "always", + "commands": [ + "cd /var/www/owncloud/server/apps/%s" % config["app"], + config["appInstallCommand"], + ], + }] def setupServerAndApp(phpVersion, logLevel): - return [{ - 'name': 'setup-server-%s' % config['app'], - 'image': 'owncloudci/php:%s' % phpVersion, - 'pull': 'always', - 'commands': [ - 'cd /var/www/owncloud/server', - 'php occ a:l', - 'php occ a:e %s' % config['app'], - 'php occ a:e testing', - 'php occ a:l', - 'php occ config:system:set trusted_domains 1 --value=server', - 'php occ log:manage --level %s' % logLevel, - ] - }] + return [{ + "name": "setup-server-%s" % config["app"], + "image": "owncloudci/php:%s" % phpVersion, + "pull": "always", + "commands": [ + "cd /var/www/owncloud/server", + "php occ a:l", + "php occ a:e %s" % config["app"], + "php occ a:e testing", + "php occ a:l", + "php occ config:system:set trusted_domains 1 --value=server", + "php occ log:manage --level %s" % logLevel, + ], + }] def setupCeph(serviceParams): - if type(serviceParams) == "bool": - if serviceParams: - # specify an empty dict that will get the defaults - serviceParams = {} - else: - return [] - - createFirstBucket = serviceParams['createFirstBucket'] if 'createFirstBucket' in serviceParams else True - setupCommands = serviceParams['setupCommands'] if 'setupCommands' in serviceParams else [ - 'wait-for-it -t 600 ceph:80', - 'cd /var/www/owncloud/server/apps/files_primary_s3', - 'cp tests/drone/ceph.config.php /var/www/owncloud/server/config', - 'cd /var/www/owncloud/server', - ] - - return [{ - 'name': 'setup-ceph', - 'image': 'owncloudci/php:7.2', - 'pull': 'always', - 'commands': setupCommands + ([ - './apps/files_primary_s3/tests/drone/create-bucket.sh', - ] if createFirstBucket else []) - }] + if type(serviceParams) == "bool": + if serviceParams: + # specify an empty dict that will get the defaults + serviceParams = {} + else: + return [] + + createFirstBucket = serviceParams["createFirstBucket"] if "createFirstBucket" in serviceParams else True + setupCommands = serviceParams["setupCommands"] if "setupCommands" in serviceParams else [ + "wait-for-it -t 600 ceph:80", + "cd /var/www/owncloud/server/apps/files_primary_s3", + "cp tests/drone/ceph.config.php /var/www/owncloud/server/config", + "cd /var/www/owncloud/server", + ] + + return [{ + "name": "setup-ceph", + "image": "owncloudci/php:7.2", + "pull": "always", + "commands": setupCommands + ([ + "./apps/files_primary_s3/tests/drone/create-bucket.sh", + ] if createFirstBucket else []), + }] def setupScality(serviceParams): - if type(serviceParams) == "bool": - if serviceParams: - # specify an empty dict that will get the defaults - serviceParams = {} - else: - return [] - - specialConfig = '.' + serviceParams['config'] if 'config' in serviceParams else '' - configFile = 'scality%s.config.php' % specialConfig - createFirstBucket = serviceParams['createFirstBucket'] if 'createFirstBucket' in serviceParams else True - createExtraBuckets = serviceParams['createExtraBuckets'] if 'createExtraBuckets' in serviceParams else False - setupCommands = serviceParams['setupCommands'] if 'setupCommands' in serviceParams else [ - 'wait-for-it -t 600 scality:8000', - 'cd /var/www/owncloud/server/apps/files_primary_s3', - 'cp tests/drone/%s /var/www/owncloud/server/config' % configFile, - 'cd /var/www/owncloud/server' - ] - - return [{ - 'name': 'setup-scality', - 'image': 'owncloudci/php:7.2', - 'pull': 'always', - 'commands': setupCommands + ([ - 'php occ s3:create-bucket owncloud --accept-warning' - ] if createFirstBucket else []) + ([ - 'for I in $(seq 1 9); do php ./occ s3:create-bucket owncloud$I --accept-warning; done', - ] if createExtraBuckets else []) - }] + if type(serviceParams) == "bool": + if serviceParams: + # specify an empty dict that will get the defaults + serviceParams = {} + else: + return [] + + specialConfig = "." + serviceParams["config"] if "config" in serviceParams else "" + configFile = "scality%s.config.php" % specialConfig + createFirstBucket = serviceParams["createFirstBucket"] if "createFirstBucket" in serviceParams else True + createExtraBuckets = serviceParams["createExtraBuckets"] if "createExtraBuckets" in serviceParams else False + setupCommands = serviceParams["setupCommands"] if "setupCommands" in serviceParams else [ + "wait-for-it -t 600 scality:8000", + "cd /var/www/owncloud/server/apps/files_primary_s3", + "cp tests/drone/%s /var/www/owncloud/server/config" % configFile, + "cd /var/www/owncloud/server", + ] + + return [{ + "name": "setup-scality", + "image": "owncloudci/php:7.2", + "pull": "always", + "commands": setupCommands + ([ + "php occ s3:create-bucket owncloud --accept-warning", + ] if createFirstBucket else []) + ([ + "for I in $(seq 1 9); do php ./occ s3:create-bucket owncloud$I --accept-warning; done", + ] if createExtraBuckets else []), + }] def fixPermissions(phpVersion, federatedServerNeeded): - return [{ - 'name': 'fix-permissions', - 'image': 'owncloudci/php:%s' % phpVersion, - 'pull': 'always', - 'commands': [ - 'chown -R www-data /var/www/owncloud/server', - 'wait-for-it -t 600 server:80' - ] + ([ - 'chown -R www-data /var/www/owncloud/federated', - 'wait-for-it -t 600 federated:80' - ] if federatedServerNeeded else []) - }] + return [{ + "name": "fix-permissions", + "image": "owncloudci/php:%s" % phpVersion, + "pull": "always", + "commands": [ + "chown -R www-data /var/www/owncloud/server", + "wait-for-it -t 600 server:80", + ] + ([ + "chown -R www-data /var/www/owncloud/federated", + "wait-for-it -t 600 federated:80", + ] if federatedServerNeeded else []), + }] def owncloudLog(server): - return [{ - 'name': 'owncloud-log-%s' % server, - 'image': 'owncloud/ubuntu:20.04', - 'pull': 'always', - 'detach': True, - 'commands': [ - 'tail -f /var/www/owncloud/%s/data/owncloud.log' % server - ] - }] + return [{ + "name": "owncloud-log-%s" % server, + "image": "owncloud/ubuntu:20.04", + "pull": "always", + "detach": True, + "commands": [ + "tail -f /var/www/owncloud/%s/data/owncloud.log" % server, + ], + }] def dependsOn(earlierStages, nextStages): - for earlierStage in earlierStages: - for nextStage in nextStages: - nextStage['depends_on'].append(earlierStage['name']) - -def installFederated(federatedServerVersion, phpVersion, logLevel, db, dbSuffix = '-federated'): - host = getDbName(db) - dbType = host - - username = getDbUsername(db) - password = getDbPassword(db) - database = getDbDatabase(db) + dbSuffix - - if host == 'mariadb': - dbType = 'mysql' - elif host == 'postgres': - dbType = 'pgsql' - elif host == 'oracle': - dbType = 'oci' - return [ - { - 'name': 'install-federated', - 'image': 'owncloudci/core', - 'pull': 'always', - 'settings': { - 'version': federatedServerVersion, - 'core_path': '/var/www/owncloud/federated', - 'db_type': 'mysql', - 'db_name': database, - 'db_host': host + dbSuffix, - 'db_username': username, - 'db_password': password - }, - }, - { - 'name': 'configure-federation', - 'image': 'owncloudci/php:%s' % phpVersion, - 'pull': 'always', - 'commands': [ - 'echo "export TEST_SERVER_FED_URL=http://federated" > /var/www/owncloud/saved-settings.sh', - 'cd /var/www/owncloud/federated', - 'php occ a:l', - 'php occ a:e testing', - 'php occ a:l', - 'php occ config:system:set trusted_domains 1 --value=federated', - 'php occ log:manage --level %s' % logLevel, - 'php occ config:list' - ] - } - ] + for earlierStage in earlierStages: + for nextStage in nextStages: + nextStage["depends_on"].append(earlierStage["name"]) + +def installFederated(federatedServerVersion, phpVersion, logLevel, db, dbSuffix = "-federated"): + host = getDbName(db) + dbType = host + + username = getDbUsername(db) + password = getDbPassword(db) + database = getDbDatabase(db) + dbSuffix + + if host == "mariadb": + dbType = "mysql" + elif host == "postgres": + dbType = "pgsql" + elif host == "oracle": + dbType = "oci" + return [ + { + "name": "install-federated", + "image": "owncloudci/core", + "pull": "always", + "settings": { + "version": federatedServerVersion, + "core_path": "/var/www/owncloud/federated", + "db_type": "mysql", + "db_name": database, + "db_host": host + dbSuffix, + "db_username": username, + "db_password": password, + }, + }, + { + "name": "configure-federation", + "image": "owncloudci/php:%s" % phpVersion, + "pull": "always", + "commands": [ + 'echo "export TEST_SERVER_FED_URL=http://federated" > /var/www/owncloud/saved-settings.sh', + "cd /var/www/owncloud/federated", + "php occ a:l", + "php occ a:e testing", + "php occ a:l", + "php occ config:system:set trusted_domains 1 --value=federated", + "php occ log:manage --level %s" % logLevel, + "php occ config:list", + ], + }, + ] def databaseServiceForFederation(db, suffix): - dbName = getDbName(db) - - if dbName not in ['mariadb', 'mysql']: - print('Not implemented federated database for ', dbName) - return [] - - return [{ - 'name': dbName + suffix, - 'image': db, - 'pull': 'always', - 'environment': { - 'MYSQL_USER': getDbUsername(db), - 'MYSQL_PASSWORD': getDbPassword(db), - 'MYSQL_DATABASE': getDbDatabase(db) + suffix, - 'MYSQL_ROOT_PASSWORD': getDbRootPassword() - } - }] + dbName = getDbName(db) + + if dbName not in ["mariadb", "mysql"]: + print("Not implemented federated database for ", dbName) + return [] + + return [{ + "name": dbName + suffix, + "image": db, + "pull": "always", + "environment": { + "MYSQL_USER": getDbUsername(db), + "MYSQL_PASSWORD": getDbPassword(db), + "MYSQL_DATABASE": getDbDatabase(db) + suffix, + "MYSQL_ROOT_PASSWORD": getDbRootPassword(), + }, + }] From b17ed42bacef6e4ec40119411dd59afd3b65c236 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Mon, 6 Sep 2021 10:31:40 +0200 Subject: [PATCH 2/9] use streamlined CI from announcementcenter --- .drone.star | 1242 ++++++++++++++----- .drone/cancelBuilds.sh | 62 + composer.json | 23 +- composer.lock | 129 +- vendor-bin/owncloud-codestyle/composer.json | 4 +- vendor-bin/phan/composer.json | 4 +- vendor-bin/phpstan/composer.json | 2 +- 7 files changed, 1084 insertions(+), 382 deletions(-) create mode 100644 .drone/cancelBuilds.sh diff --git a/.drone.star b/.drone.star index 9fd96b1..26d5734 100644 --- a/.drone.star +++ b/.drone.star @@ -1,95 +1,136 @@ -config = { - 'app': 'qnap', - 'rocketchat': { - 'channel': 'builds', - 'from_secret': 'private_rocketchat' - }, - - 'branches': [ - 'master' - ], - - 'appInstallCommand': 'make vendor', - - 'codestyle': True, +dir = { + "base": "/var/www/owncloud", + "federated": "/var/www/owncloud/federated", + "server": "/var/www/owncloud/server", + "testrunner": "/var/www/owncloud/testrunner", +} - 'javascript': False, +config = { + "app": "qnap", + "rocketchat": { + "channel": "builds", + "from_secret": "private_rocketchat", + }, + "branches": [ + "master", + ], + "appInstallCommand": "make vendor", + "codestyle": True, + "javascript": False, + "phpstan": True, + "phan": True, + "build": { + "commands": [ + "make -f Makefile.release dist", + ], + }, + "phpunit": { + "codecov": { + "phpVersions": [ + "7.2", + ], + "databases": [ + "sqlite", + ], + "coverage": True, + }, + "reducedDatabases": { + "phpVersions": [ + "7.3", + ], + "databases": [ + "mysql:5.5", + "postgres:9.4", + ], + "coverage": False, + }, + }, +} - 'phpstan': True, +def main(ctx): + before = beforePipelines(ctx) - 'phan': True, + coverageTests = coveragePipelines(ctx) + if (coverageTests == False): + print("Errors detected in coveragePipelines. Review messages above.") + return [] - 'build': { - 'commands': [ - 'make -f Makefile.release dist', - ], - }, + dependsOn(before, coverageTests) - 'phpunit': { - 'codecov' : { - 'phpVersions': [ - '7.2', - ], - 'databases': [ - 'sqlite' - ], - 'coverage': True - }, - 'reducedDatabases' : { - 'phpVersions': [ - '7.3', - ], - 'databases': [ - 'mysql:5.5', - 'postgres:9.4', - ], - 'coverage': False - }, - }, -} + nonCoverageTests = nonCoveragePipelines(ctx) + if (nonCoverageTests == False): + print("Errors detected in nonCoveragePipelines. Review messages above.") + return [] -def main(ctx): - before = beforePipelines() + dependsOn(before, nonCoverageTests) - stages = stagePipelines() + stages = stagePipelines(ctx) if (stages == False): print("Errors detected. Review messages above.") return [] dependsOn(before, stages) - after = afterPipelines() - dependsOn(stages, after) + if (coverageTests == []): + afterCoverageTests = [] + else: + afterCoverageTests = afterCoveragePipelines(ctx) + dependsOn(coverageTests, afterCoverageTests) + + after = afterPipelines(ctx) + dependsOn(afterCoverageTests + nonCoverageTests + stages, after) + + return before + coverageTests + afterCoverageTests + nonCoverageTests + stages + after - return before + stages + after +def beforePipelines(ctx): + return codestyle(ctx) + jscodestyle(ctx) + checkForRecentBuilds(ctx) + phpstan(ctx) + phan(ctx) + phplint(ctx) + checkStarlark() -def beforePipelines(): - return codestyle() + jscodestyle() + phpstan() + phan() +def coveragePipelines(ctx): + # All unit test pipelines that have coverage or other test analysis reported + jsPipelines = javascript(ctx, True) + phpUnitPipelines = phpTests(ctx, "phpunit", True) + phpIntegrationPipelines = phpTests(ctx, "phpintegration", True) + if (jsPipelines == False) or (phpUnitPipelines == False) or (phpIntegrationPipelines == False): + return False + + return jsPipelines + phpUnitPipelines + phpIntegrationPipelines -def stagePipelines(): - buildPipelines = build() - jsPipelines = javascript() - phpunitPipelines = phptests("phpunit") - phpintegrationPipelines = phptests("phpintegration") - acceptancePipelines = acceptance() - if (buildPipelines == False) or (jsPipelines == False) or (phpunitPipelines == False) or (phpintegrationPipelines == False) or (acceptancePipelines == False): +def nonCoveragePipelines(ctx): + # All unit test pipelines that do not have coverage or other test analysis reported + jsPipelines = javascript(ctx, False) + phpUnitPipelines = phpTests(ctx, "phpunit", False) + phpIntegrationPipelines = phpTests(ctx, "phpintegration", False) + if (jsPipelines == False) or (phpUnitPipelines == False) or (phpIntegrationPipelines == False): return False - return buildPipelines + jsPipelines + phpunitPipelines + phpintegrationPipelines + acceptancePipelines + return jsPipelines + phpUnitPipelines + phpIntegrationPipelines + +def stagePipelines(ctx): + buildPipelines = build(ctx) + acceptancePipelines = acceptance(ctx) + if (buildPipelines == False) or (acceptancePipelines == False): + return False + + return buildPipelines + acceptancePipelines + +def afterCoveragePipelines(ctx): + return [ + sonarAnalysis(ctx), + ] -def afterPipelines(): +def afterPipelines(ctx): return [ notify(), ] -def codestyle(): +def codestyle(ctx): pipelines = [] if "codestyle" not in config: return pipelines default = { - "phpVersions": ["7.2"], + "phpVersions": ["7.3"], } if "defaults" in config: @@ -101,13 +142,13 @@ def codestyle(): if type(codestyleConfig) == "bool": if codestyleConfig: - # the config has 'codestyle' true, so specify an empty dict that will get the defaults + # the config has "codestyle" true, so specify an empty dict that will get the defaults codestyleConfig = {} else: return pipelines if len(codestyleConfig) == 0: - # 'codestyle' is an empty dict, so specify a single section that will get the defaults + # "codestyle" is an empty dict, so specify a single section that will get the defaults codestyleConfig = {"doDefault": {}} for category, matrix in codestyleConfig.items(): @@ -123,8 +164,8 @@ def codestyle(): "type": "docker", "name": name, "workspace": { - "base": "/var/www/owncloud", - "path": "server/apps/%s" % config["app"], + "base": dir["base"], + "path": "server/apps/%s" % ctx.repo.name, }, "steps": [ { @@ -152,7 +193,7 @@ def codestyle(): return pipelines -def jscodestyle(): +def jscodestyle(ctx): pipelines = [] if "jscodestyle" not in config: @@ -167,13 +208,13 @@ def jscodestyle(): "type": "docker", "name": "coding-standard-js", "workspace": { - "base": "/var/www/owncloud", - "path": "server/apps/%s" % config["app"], + "base": dir["base"], + "path": "server/apps/%s" % ctx.repo.name, }, "steps": [ { "name": "coding-standard-js", - "image": "owncloudci/php:7.2", + "image": "owncloudci/php:8.0", "pull": "always", "commands": [ "make test-js-style", @@ -196,15 +237,68 @@ def jscodestyle(): return pipelines -def phpstan(): +def checkForRecentBuilds(ctx): + pipelines = [] + + result = { + "kind": "pipeline", + "type": "docker", + "name": "stop-recent-builds", + "workspace": { + "base": dir["base"], + "path": "server/apps/%s" % ctx.repo.name, + }, + "steps": stopRecentBuilds(ctx), + "depends_on": [], + "trigger": { + "ref": [ + "refs/heads/master", + "refs/tags/**", + "refs/pull/**", + ], + }, + } + + pipelines.append(result) + + return pipelines + +def stopRecentBuilds(ctx): + app = "%s/apps/%s" % (dir["server"], ctx.repo.name) + return [{ + "name": "stop-recent-builds", + "image": "drone/cli:alpine", + "pull": "always", + "environment": { + "DRONE_SERVER": "https://drone.owncloud.com", + "DRONE_TOKEN": { + "from_secret": "drone_token", + }, + "OWNCLOUD_APP": ctx.repo.slug, + }, + "commands": [ + "drone build ls %s --status running > %s/recentBuilds.txt" % (ctx.repo.slug, app), + "drone build info %s ${DRONE_BUILD_NUMBER} > %s/thisBuildInfo.txt" % (ctx.repo.slug, app), + "cd %s && ./.drone/cancelBuilds.sh" % app, + ], + "when": { + "event": [ + "pull_request", + ], + }, + }] + +def phpstan(ctx): pipelines = [] if "phpstan" not in config: return pipelines default = { - "phpVersions": ["7.2"], + "phpVersions": ["7.3"], "logLevel": "2", + "extraApps": {}, + "enableApp": True, } if "defaults" in config: @@ -216,13 +310,13 @@ def phpstan(): if type(phpstanConfig) == "bool": if phpstanConfig: - # the config has 'phpstan' true, so specify an empty dict that will get the defaults + # the config has "phpstan" true, so specify an empty dict that will get the defaults phpstanConfig = {} else: return pipelines if len(phpstanConfig) == 0: - # 'phpstan' is an empty dict, so specify a single section that will get the defaults + # "phpstan" is an empty dict, so specify a single section that will get the defaults phpstanConfig = {"doDefault": {}} for category, matrix in phpstanConfig.items(): @@ -238,12 +332,13 @@ def phpstan(): "type": "docker", "name": name, "workspace": { - "base": "/var/www/owncloud", - "path": "server/apps/%s" % config["app"], + "base": dir["base"], + "path": "server/apps/%s" % ctx.repo.name, }, - "steps": installCore("daily-master-qa", "sqlite", False) + - installApp(phpVersion) + - setupServerAndApp(phpVersion, params["logLevel"]) + + "steps": installCore(ctx, "daily-master-qa", "sqlite", False) + + installApp(ctx, phpVersion) + + installExtraApps(phpVersion, params["extraApps"]) + + setupServerAndApp(ctx, phpVersion, params["logLevel"], False, params["enableApp"]) + [ { "name": "phpstan", @@ -270,14 +365,14 @@ def phpstan(): return pipelines -def phan(): +def phan(ctx): pipelines = [] if "phan" not in config: return pipelines default = { - "phpVersions": ["7.2", "7.3"], + "phpVersions": ["7.3", "7.4"], } if "defaults" in config: @@ -289,13 +384,13 @@ def phan(): if type(phanConfig) == "bool": if phanConfig: - # the config has 'phan' true, so specify an empty dict that will get the defaults + # the config has "phan" true, so specify an empty dict that will get the defaults phanConfig = {} else: return pipelines if len(phanConfig) == 0: - # 'phan' is an empty dict, so specify a single section that will get the defaults + # "phan" is an empty dict, so specify a single section that will get the defaults phanConfig = {"doDefault": {}} for category, matrix in phanConfig.items(): @@ -311,10 +406,10 @@ def phan(): "type": "docker", "name": name, "workspace": { - "base": "/var/www/owncloud", - "path": "server/apps/%s" % config["app"], + "base": dir["base"], + "path": "server/apps/%s" % ctx.repo.name, }, - "steps": installCore("daily-master-qa", "sqlite", False) + + "steps": installCore(ctx, "daily-master-qa", "sqlite", False) + [ { "name": "phan", @@ -341,14 +436,14 @@ def phan(): return pipelines -def build(): +def build(ctx): pipelines = [] if "build" not in config: return pipelines default = { - "phpVersions": ["7.2"], + "phpVersions": ["7.3"], "commands": [ "make dist", ], @@ -365,7 +460,7 @@ def build(): if type(matrix) == "bool": if matrix: - # the config has 'build' true, so specify an empty dict that will get the defaults + # the config has "build" true, so specify an empty dict that will get the defaults matrix = {} else: return pipelines @@ -380,8 +475,8 @@ def build(): "type": "docker", "name": "build", "workspace": { - "base": "/var/www/owncloud", - "path": "server/apps/%s" % config["app"], + "base": dir["base"], + "path": "server/apps/%s" % ctx.repo.name, }, "steps": [ { @@ -399,7 +494,7 @@ def build(): "settings": { "checksum": "sha256", "file_exists": "overwrite", - "files": "build/dist/%s.tar.gz" % config["app"], + "files": "build/dist/%s.tar.gz" % ctx.repo.name, "prerelease": True, }, "environment": { @@ -430,7 +525,7 @@ def build(): return pipelines -def javascript(): +def javascript(ctx, withCoverage): pipelines = [] if "javascript" not in config: @@ -444,6 +539,8 @@ def javascript(): "extraEnvironment": {}, "extraCommandsBeforeTestRun": [], "extraTeardown": [], + "skip": False, + "enableApp": True, } if "defaults" in config: @@ -455,7 +552,7 @@ def javascript(): if type(matrix) == "bool": if matrix: - # the config has 'javascript' true, so specify an empty dict that will get the defaults + # the config has "javascript" true, so specify an empty dict that will get the defaults matrix = {} else: return pipelines @@ -464,22 +561,33 @@ def javascript(): for item in default: params[item] = matrix[item] if item in matrix else default[item] + if params["skip"]: + return pipelines + + # if we only want pipelines with coverage, and this pipeline does not do coverage, then do not include it + if withCoverage and not params["coverage"]: + return pipelines + + # if we only want pipelines without coverage, and this pipeline does coverage, then do not include it + if not withCoverage and params["coverage"]: + return pipelines + result = { "kind": "pipeline", "type": "docker", "name": "javascript-tests", "workspace": { - "base": "/var/www/owncloud", - "path": "server/apps/%s" % config["app"], + "base": dir["base"], + "path": "server/apps/%s" % ctx.repo.name, }, - "steps": installCore("daily-master-qa", "sqlite", False) + - installApp("7.2") + - setupServerAndApp("7.2", params["logLevel"]) + + "steps": installCore(ctx, "daily-master-qa", "sqlite", False) + + installApp(ctx, "7.4") + + setupServerAndApp(ctx, "7.4", params["logLevel"], False, params["enableApp"]) + params["extraSetup"] + [ { "name": "js-tests", - "image": "owncloudci/php:7.2", + "image": "owncloudci/nodejs:%s" % getNodeJsVersion(), "pull": "always", "environment": params["extraEnvironment"], "commands": params["extraCommandsBeforeTestRun"] + [ @@ -499,15 +607,23 @@ def javascript(): if params["coverage"]: result["steps"].append({ - "name": "codecov-js", - "image": "plugins/codecov:2", + "name": "coverage-cache", + "image": "plugins/s3", "pull": "always", "settings": { - "paths": [ - "coverage/*.info", - ], - "token": { - "from_secret": "codecov_token", + "endpoint": { + "from_secret": "cache_s3_endpoint", + }, + "bucket": "cache", + "source": "./coverage/lcov.info", + "target": "%s/%s" % (ctx.repo.slug, ctx.build.commit + "-${DRONE_BUILD_NUMBER}"), + "path_style": True, + "strip_prefix": "./coverage", + "access_key": { + "from_secret": "cache_s3_access_key", + }, + "secret_key": { + "from_secret": "cache_s3_secret_key", }, }, }) @@ -517,7 +633,7 @@ def javascript(): return [result] -def phptests(testType): +def phpTests(ctx, testType, withCoverage): pipelines = [] if testType not in config: @@ -526,12 +642,11 @@ def phptests(testType): errorFound = False default = { - "phpVersions": ["7.2", "7.3"], + "phpVersions": ["7.3", "7.4"], "databases": [ "sqlite", "mariadb:10.2", - "mysql:5.5", - "mysql:5.7", + "mysql:8.0", "postgres:9.4", "oracle", ], @@ -546,6 +661,8 @@ def phptests(testType): "extraCommandsBeforeTestRun": [], "extraApps": {}, "extraTeardown": [], + "skip": False, + "enableApp": True, } if "defaults" in config: @@ -553,24 +670,35 @@ def phptests(testType): for item in config["defaults"][testType]: default[item] = config["defaults"][testType][item] - phptestConfig = config[testType] + phpTestConfig = config[testType] - if type(phptestConfig) == "bool": - if phptestConfig: + if type(phpTestConfig) == "bool": + if phpTestConfig: # the config has just True, so specify an empty dict that will get the defaults - phptestConfig = {} + phpTestConfig = {} else: return pipelines - if len(phptestConfig) == 0: + if len(phpTestConfig) == 0: # the PHP test config is an empty dict, so specify a single section that will get the defaults - phptestConfig = {"doDefault": {}} + phpTestConfig = {"doDefault": {}} - for category, matrix in phptestConfig.items(): + for category, matrix in phpTestConfig.items(): params = {} for item in default: params[item] = matrix[item] if item in matrix else default[item] + if params["skip"]: + continue + + # if we only want pipelines with coverage, and this pipeline does not do coverage, then do not include it + if withCoverage and not params["coverage"]: + continue + + # if we only want pipelines without coverage, and this pipeline does coverage, then do not include it + if not withCoverage and params["coverage"]: + continue + cephS3Params = params["cephS3"] if type(cephS3Params) == "bool": cephS3Needed = cephS3Params @@ -587,8 +715,8 @@ def phptests(testType): scalityS3Needed = True filesPrimaryS3NeededForScality = scalityS3Params["filesPrimaryS3Needed"] if "filesPrimaryS3Needed" in scalityS3Params else True - if ((config["app"] != "files_primary_s3") and (filesPrimaryS3NeededForCeph or filesPrimaryS3NeededForScality)): - # If we are not already 'files_primary_s3' and we need S3 storage, then install the 'files_primary_s3' app + if ((ctx.repo.name != "files_primary_s3") and (filesPrimaryS3NeededForCeph or filesPrimaryS3NeededForScality)): + # If we are not already "files_primary_s3" and we need S3 storage, then install the "files_primary_s3" app extraAppsDict = { "files_primary_s3": "composer install", } @@ -621,13 +749,13 @@ def phptests(testType): "type": "docker", "name": name, "workspace": { - "base": "/var/www/owncloud", - "path": "server/apps/%s" % config["app"], + "base": dir["base"], + "path": "server/apps/%s" % ctx.repo.name, }, - "steps": installCore("daily-master-qa", db, False) + - installApp(phpVersion) + + "steps": installCore(ctx, "daily-master-qa", db, False) + + installApp(ctx, phpVersion) + installExtraApps(phpVersion, params["extraApps"]) + - setupServerAndApp(phpVersion, params["logLevel"]) + + setupServerAndApp(ctx, phpVersion, params["logLevel"], False, params["enableApp"]) + setupCeph(params["cephS3"]) + setupScality(params["scalityS3"]) + params["extraSetup"] + @@ -657,15 +785,31 @@ def phptests(testType): if params["coverage"]: result["steps"].append({ - "name": "codecov-upload", - "image": "plugins/codecov:2", + "name": "coverage-rename", + "image": "owncloudci/php:%s" % phpVersion, + "pull": "always", + "commands": [ + "mv tests/output/clover.xml tests/output/clover-%s.xml" % (name), + ], + }) + result["steps"].append({ + "name": "coverage-cache-1", + "image": "plugins/s3", "pull": "always", "settings": { - "paths": [ - "tests/output/clover.xml", - ], - "token": { - "from_secret": "codecov_token", + "endpoint": { + "from_secret": "cache_s3_endpoint", + }, + "bucket": "cache", + "source": "tests/output/clover-%s.xml" % (name), + "target": "%s/%s" % (ctx.repo.slug, ctx.build.commit + "-${DRONE_BUILD_NUMBER}"), + "path_style": True, + "strip_prefix": "tests/output", + "access_key": { + "from_secret": "cache_s3_access_key", + }, + "secret_key": { + "from_secret": "cache_s3_secret_key", }, }, }) @@ -680,7 +824,7 @@ def phptests(testType): return pipelines -def acceptance(): +def acceptance(ctx): pipelines = [] if "acceptance" not in config: @@ -695,8 +839,9 @@ def acceptance(): default = { "servers": ["daily-master-qa", "latest"], "browsers": ["chrome"], - "phpVersions": ["7.2"], + "phpVersions": ["7.4"], "databases": ["mariadb:10.2"], + "esVersions": ["none"], "federatedServerNeeded": False, "filterTags": "", "logLevel": "2", @@ -712,12 +857,19 @@ def acceptance(): "extraEnvironment": {}, "extraCommandsBeforeTestRun": [], "extraApps": {}, + "externalScality": [], "useBundledApp": False, "includeKeyInMatrixName": False, "runAllSuites": False, "runCoreTests": False, "numberOfParts": 1, "cron": "", + "pullRequestAndCron": "nightly", + "skip": False, + "debugSuites": [], + "skipExceptParts": [], + "earlyFail": True, + "enableApp": True, } if "defaults" in config: @@ -733,6 +885,14 @@ def acceptance(): else: suites = matrix["suites"] + if "debugSuites" in matrix and len(matrix["debugSuites"]) != 0: + if type(matrix["debugSuites"]) == "list": + suites = {} + for suite in matrix["debugSuites"]: + suites[suite] = suite + else: + suites = matrix["debugSuites"] + for suite, alternateSuiteName in suites.items(): isWebUI = suite.startswith("webUI") isAPI = suite.startswith("api") @@ -742,6 +902,82 @@ def acceptance(): for item in default: params[item] = matrix[item] if item in matrix else default[item] + if params["skip"]: + continue + + # switch off earlyFail if the PR title contains full-ci + if ("full-ci" in ctx.build.title.lower()): + params["earlyFail"] = False + + # switch off earlyFail when running cron builds (for example, nightly CI) + if (ctx.build.event == "cron"): + params["earlyFail"] = False + + if "externalScality" in params and len(params["externalScality"]) != 0: + # We want to use an external scality server for this pipeline. + # That uses some "standard" extraSetup and extraTeardown. + # Put the needed setup and teardown in place. + params["extraSetup"] = [ + { + "name": "configure-app", + "image": "owncloudci/php:7.4", + "pull": "always", + "commands": [ + "cd /var/www/owncloud/server/apps/files_primary_s3", + "cp tests/drone/scality.config.php /var/www/owncloud/server/config", + "sed -i -e \"s/owncloud/owncloud-acceptance-tests-$DRONE_BUILD_NUMBER-$DRONE_STAGE_NUMBER/\" /var/www/owncloud/server/config/scality.config.php", + "sed -i -e \"s/accessKey1/$SCALITY_KEY/\" /var/www/owncloud/server/config/scality.config.php", + "sed -i -e \"s/verySecretKey1/$SCALITY_SECRET_ESCAPED/\" /var/www/owncloud/server/config/scality.config.php", + "sed -i -e \"s/http/https/\" /var/www/owncloud/server/config/scality.config.php", + "sed -i -e \"s/scality:8000/%s/\" /var/www/owncloud/server/config/scality.config.php" % params["externalScality"]["externalServerUrl"], + "cd /var/www/owncloud/server/", + "php occ s3:create-bucket owncloud-acceptance-tests-$DRONE_BUILD_NUMBER-$DRONE_STAGE_NUMBER --accept-warning", + "cd /var/www/owncloud/testrunner/apps/files_primary_s3", + ], + "environment": { + "SCALITY_KEY": { + "from_secret": params["externalScality"]["secrets"]["scality_key"], + }, + "SCALITY_SECRET": { + "from_secret": params["externalScality"]["secrets"]["scality_secret"], + }, + "SCALITY_SECRET_ESCAPED": { + "from_secret": params["externalScality"]["secrets"]["scality_secret_escaped"], + }, + }, + }, + ] + params["extraTeardown"] = [ + { + "name": "cleanup-scality-bucket", + "image": "banst/awscli", + "pull": "always", + "failure": "ignore", + "commands": [ + "aws configure set aws_access_key_id $SCALITY_KEY", + "aws configure set aws_secret_access_key $SCALITY_SECRET", + "aws --endpoint-url $SCALITY_ENDPOINT s3 rm --recursive s3://owncloud-acceptance-tests-$DRONE_BUILD_NUMBER-$DRONE_STAGE_NUMBER", + "/var/www/owncloud/testrunner/apps/files_primary_s3/tests/delete_all_object_versions.sh $SCALITY_ENDPOINT owncloud-acceptance-tests-$DRONE_BUILD_NUMBER-$DRONE_STAGE_NUMBER", + "aws --endpoint-url $SCALITY_ENDPOINT s3 rb --force s3://owncloud-acceptance-tests-$DRONE_BUILD_NUMBER-$DRONE_STAGE_NUMBER", + ], + "environment": { + "SCALITY_KEY": { + "from_secret": params["externalScality"]["secrets"]["scality_key"], + }, + "SCALITY_SECRET": { + "from_secret": params["externalScality"]["secrets"]["scality_secret"], + }, + "SCALITY_ENDPOINT": "https://%s" % params["externalScality"]["externalServerUrl"], + }, + "when": { + "status": [ + "failure", + "success", + ], + }, + }, + ] + if isAPI or isCLI: params["browsers"] = [""] @@ -761,8 +997,8 @@ def acceptance(): scalityS3Needed = True filesPrimaryS3NeededForScality = scalityS3Params["filesPrimaryS3Needed"] if "filesPrimaryS3Needed" in scalityS3Params else True - if ((config["app"] != "files_primary_s3") and (filesPrimaryS3NeededForCeph or filesPrimaryS3NeededForScality)): - # If we are not already 'files_primary_s3' and we need S3 object storage, then install the 'files_primary_s3' app + if ((ctx.repo.name != "files_primary_s3") and (filesPrimaryS3NeededForCeph or filesPrimaryS3NeededForScality)): + # If we are not already "files_primary_s3" and we need S3 object storage, then install the "files_primary_s3" app extraAppsDict = { "files_primary_s3": "composer install", } @@ -770,138 +1006,244 @@ def acceptance(): extraAppsDict[app] = command params["extraApps"] = extraAppsDict - for server in params["servers"]: - for browser in params["browsers"]: - for phpVersion in params["phpVersions"]: - for db in params["databases"]: - for runPart in range(1, params["numberOfParts"] + 1): - name = "unknown" - - if isWebUI or isAPI or isCLI: - browserString = "" if browser == "" else "-" + browser - keyString = "-" + category if params["includeKeyInMatrixName"] else "" - partString = "" if params["numberOfParts"] == 1 else "-%d-%d" % (params["numberOfParts"], runPart) - name = "%s%s%s-%s%s-%s-php%s" % (alternateSuiteName, keyString, partString, server.replace("daily-", "").replace("-qa", ""), browserString, db.replace(":", ""), phpVersion) - maxLength = 50 - nameLength = len(name) - if nameLength > maxLength: - print("Error: generated stage name of length", nameLength, "is not supported. The maximum length is " + str(maxLength) + ".", name) - errorFound = True - - environment = {} - for env in params["extraEnvironment"]: - environment[env] = params["extraEnvironment"][env] - - environment["TEST_SERVER_URL"] = "http://server" - environment["BEHAT_FILTER_TAGS"] = params["filterTags"] - - if (params["runAllSuites"] == False): - environment["BEHAT_SUITE"] = suite - else: - environment["DIVIDE_INTO_NUM_PARTS"] = params["numberOfParts"] - environment["RUN_PART"] = runPart - - if isWebUI: - environment["SELENIUM_HOST"] = "selenium" - environment["SELENIUM_PORT"] = "4444" - environment["BROWSER"] = browser - environment["PLATFORM"] = "Linux" - if (params["runCoreTests"]): - makeParameter = "test-acceptance-core-webui" - else: - makeParameter = "test-acceptance-webui" - - if isAPI: - if (params["runCoreTests"]): - makeParameter = "test-acceptance-core-api" - else: - makeParameter = "test-acceptance-api" - - if isCLI: - if (params["runCoreTests"]): - makeParameter = "test-acceptance-core-cli" - else: - makeParameter = "test-acceptance-cli" - - if params["emailNeeded"]: - environment["MAILHOG_HOST"] = "email" - - if params["ldapNeeded"]: - environment["TEST_EXTERNAL_USER_BACKENDS"] = True - - if (cephS3Needed or scalityS3Needed): - environment["OC_TEST_ON_OBJECTSTORE"] = "1" - if (params["cephS3"] != False): - environment["S3_TYPE"] = "ceph" - if (params["scalityS3"] != False): - environment["S3_TYPE"] = "scality" - federationDbSuffix = "-federated" - - result = { - "kind": "pipeline", - "type": "docker", - "name": name, - "workspace": { - "base": "/var/www/owncloud", - "path": "testrunner/apps/%s" % config["app"], - }, - "steps": installCore(server, db, params["useBundledApp"]) + - installTestrunner(phpVersion, params["useBundledApp"]) + - (installFederated(server, phpVersion, params["logLevel"], db, federationDbSuffix) + owncloudLog("federated") if params["federatedServerNeeded"] else []) + - installApp(phpVersion) + - installExtraApps(phpVersion, params["extraApps"]) + - setupServerAndApp(phpVersion, params["logLevel"]) + - owncloudLog("server") + - setupCeph(params["cephS3"]) + - setupScality(params["scalityS3"]) + - params["extraSetup"] + - fixPermissions(phpVersion, params["federatedServerNeeded"]) + - [ - ({ - "name": "acceptance-tests", - "image": "owncloudci/php:%s" % phpVersion, - "pull": "always", - "environment": environment, - "commands": params["extraCommandsBeforeTestRun"] + [ - "touch /var/www/owncloud/saved-settings.sh", - ". /var/www/owncloud/saved-settings.sh", - "make %s" % makeParameter, - ], - }), - ] + params["extraTeardown"], - "services": databaseService(db) + - browserService(browser) + - emailService(params["emailNeeded"]) + - ldapService(params["ldapNeeded"]) + - cephService(params["cephS3"]) + - scalityService(params["scalityS3"]) + - params["extraServices"] + - owncloudService(server, phpVersion, "server", "/var/www/owncloud/server", params["ssl"], params["xForwardedFor"]) + - (( - owncloudService(server, phpVersion, "federated", "/var/www/owncloud/federated", params["ssl"], params["xForwardedFor"]) + - databaseServiceForFederation(db, federationDbSuffix) - ) if params["federatedServerNeeded"] else []), - "depends_on": [], - "trigger": {}, - } - - if (params["cron"] == ""): - result["trigger"]["ref"] = [ - "refs/pull/**", - "refs/tags/**", - ] - for branch in config["branches"]: - result["trigger"]["ref"].append("refs/heads/%s" % branch) - else: - result["trigger"]["cron"] = params["cron"] - - pipelines.append(result) + for testConfig in buildTestConfig(params): + debugPartsEnabled = (len(testConfig["skipExceptParts"]) != 0) + if debugPartsEnabled and testConfig["runPart"] not in testConfig["skipExceptParts"]: + continue + + name = "unknown" + if isWebUI or isAPI or isCLI: + esString = "-es" + testConfig["esVersion"] if testConfig["esVersion"] != "none" else "" + browserString = "" if testConfig["browser"] == "" else "-" + testConfig["browser"] + keyString = "-" + category if testConfig["includeKeyInMatrixName"] else "" + partString = "" if testConfig["numberOfParts"] == 1 else "-%d-%d" % (testConfig["numberOfParts"], testConfig["runPart"]) + name = "%s%s%s-%s%s-%s-php%s%s" % (alternateSuiteName, keyString, partString, testConfig["server"].replace("daily-", "").replace("-qa", ""), browserString, testConfig["database"].replace(":", ""), testConfig["phpVersion"], esString) + maxLength = 50 + nameLength = len(name) + if nameLength > maxLength: + print("Error: generated stage name of length", nameLength, "is not supported. The maximum length is " + str(maxLength) + ".", name) + errorFound = True + + environment = {} + for env in testConfig["extraEnvironment"]: + environment[env] = testConfig["extraEnvironment"][env] + + environment["TEST_SERVER_URL"] = "http://server" + environment["BEHAT_FILTER_TAGS"] = testConfig["filterTags"] + + if (testConfig["runAllSuites"] == False): + environment["BEHAT_SUITE"] = suite + else: + environment["DIVIDE_INTO_NUM_PARTS"] = testConfig["numberOfParts"] + environment["RUN_PART"] = testConfig["runPart"] + + if isWebUI: + environment["SELENIUM_HOST"] = "selenium" + environment["SELENIUM_PORT"] = "4444" + environment["BROWSER"] = testConfig["browser"] + environment["PLATFORM"] = "Linux" + if (testConfig["runCoreTests"]): + makeParameter = "test-acceptance-core-webui" + else: + makeParameter = "test-acceptance-webui" + + if isAPI: + if (testConfig["runCoreTests"]): + makeParameter = "test-acceptance-core-api" + else: + makeParameter = "test-acceptance-api" + + if isCLI: + if (testConfig["runCoreTests"]): + makeParameter = "test-acceptance-core-cli" + else: + makeParameter = "test-acceptance-cli" + + if testConfig["emailNeeded"]: + environment["MAILHOG_HOST"] = "email" + + if testConfig["ldapNeeded"]: + environment["TEST_WITH_LDAP"] = True + + if (cephS3Needed or scalityS3Needed): + environment["OC_TEST_ON_OBJECTSTORE"] = "1" + if (testConfig["cephS3"] != False): + environment["S3_TYPE"] = "ceph" + if (testConfig["scalityS3"] != False): + environment["S3_TYPE"] = "scality" + federationDbSuffix = "-federated" + + result = { + "kind": "pipeline", + "type": "docker", + "name": name, + "workspace": { + "base": dir["base"], + "path": "testrunner/apps/%s" % ctx.repo.name, + }, + "steps": installCore(ctx, testConfig["server"], testConfig["database"], testConfig["useBundledApp"]) + + installTestrunner(ctx, "7.4", testConfig["useBundledApp"]) + + (installFederated(testConfig["server"], testConfig["phpVersion"], testConfig["logLevel"], testConfig["database"], federationDbSuffix) + owncloudLog("federated") if testConfig["federatedServerNeeded"] else []) + + installApp(ctx, testConfig["phpVersion"]) + + installExtraApps(testConfig["phpVersion"], testConfig["extraApps"]) + + setupServerAndApp(ctx, testConfig["phpVersion"], testConfig["logLevel"], testConfig["federatedServerNeeded"], params["enableApp"]) + + owncloudLog("server") + + setupCeph(testConfig["cephS3"]) + + setupScality(testConfig["scalityS3"]) + + setupElasticSearch(testConfig["esVersion"]) + + testConfig["extraSetup"] + + fixPermissions(testConfig["phpVersion"], testConfig["federatedServerNeeded"]) + + [ + ({ + "name": "acceptance-tests", + "image": "owncloudci/php:7.4", + "pull": "always", + "environment": environment, + "commands": testConfig["extraCommandsBeforeTestRun"] + [ + "touch %s/saved-settings.sh" % dir["base"], + ". %s/saved-settings.sh" % dir["base"], + "make %s" % makeParameter, + ], + }), + ] + testConfig["extraTeardown"] + buildGithubCommentForBuildStopped(name, params["earlyFail"]) + githubComment(params["earlyFail"]) + stopBuild(ctx, params["earlyFail"]), + "services": databaseService(testConfig["database"]) + + browserService(testConfig["browser"]) + + emailService(testConfig["emailNeeded"]) + + ldapService(testConfig["ldapNeeded"]) + + cephService(testConfig["cephS3"]) + + scalityService(testConfig["scalityS3"]) + + elasticSearchService(testConfig["esVersion"]) + + testConfig["extraServices"] + + owncloudService(testConfig["server"], testConfig["phpVersion"], "server", dir["server"], testConfig["ssl"], testConfig["xForwardedFor"]) + + (( + owncloudService(testConfig["server"], testConfig["phpVersion"], "federated", dir["federated"], testConfig["ssl"], testConfig["xForwardedFor"]) + + databaseServiceForFederation(testConfig["database"], federationDbSuffix) + ) if testConfig["federatedServerNeeded"] else []), + "depends_on": [], + "trigger": {}, + } + + if (testConfig["cron"] != ""): + result["trigger"]["cron"] = testConfig["cron"] + elif ((testConfig["pullRequestAndCron"] != "") and (ctx.build.event != "pull_request")): + result["trigger"]["cron"] = testConfig["pullRequestAndCron"] + else: + result["trigger"]["ref"] = [ + "refs/pull/**", + "refs/tags/**", + ] + + pipelines.append(result) if errorFound: return False return pipelines +def sonarAnalysis(ctx, phpVersion = "7.4"): + sonar_env = { + "SONAR_TOKEN": { + "from_secret": "sonar_token", + }, + "SONAR_SCANNER_OPTS": "-Xdebug", + } + + if ctx.build.event == "pull_request": + sonar_env.update({ + "SONAR_PULL_REQUEST_BASE": "%s" % (ctx.build.target), + "SONAR_PULL_REQUEST_BRANCH": "%s" % (ctx.build.source), + "SONAR_PULL_REQUEST_KEY": "%s" % (ctx.build.ref.replace("refs/pull/", "").split("/")[0]), + }) + + repo_slug = ctx.build.source_repo if ctx.build.source_repo else ctx.repo.slug + + result = { + "kind": "pipeline", + "type": "docker", + "name": "sonar-analysis", + "workspace": { + "base": dir["base"], + "path": "server/apps/%s" % ctx.repo.name, + }, + "clone": { + "disable": True, # Sonarcloud does not apply issues on already merged branch + }, + "steps": [ + { + "name": "clone", + "image": "owncloudci/alpine:latest", + "commands": [ + "git clone https://github.com/%s.git ." % repo_slug, + "git checkout $DRONE_COMMIT", + ], + }, + ] + + cacheRestore() + + composerInstall(phpVersion) + + installCore(ctx, "daily-master-qa", "sqlite", False) + + [ + { + "name": "sync-from-cache", + "image": "minio/mc:RELEASE.2020-12-18T10-53-53Z", + "pull": "always", + "environment": { + "MC_HOST_cache": { + "from_secret": "cache_s3_connection_url", + }, + }, + "commands": [ + "mkdir -p results", + "mc mirror cache/cache/%s/%s results/" % (ctx.repo.slug, ctx.build.commit + "-${DRONE_BUILD_NUMBER}"), + ], + }, + { + "name": "list-coverage-results", + "image": "owncloudci/php:%s" % phpVersion, + "pull": "always", + "commands": [ + "ls -l results", + ], + }, + { + "name": "sonarcloud", + "image": "sonarsource/sonar-scanner-cli", + "pull": "always", + "environment": sonar_env, + "when": { + "instance": [ + "drone.owncloud.services", + "drone.owncloud.com", + ], + }, + }, + { + "name": "purge-cache", + "image": "minio/mc:RELEASE.2020-12-18T10-53-53Z", + "environment": { + "MC_HOST_cache": { + "from_secret": "cache_s3_connection_url", + }, + }, + "commands": [ + "mc rm --recursive --force cache/cache/%s/%s" % (ctx.repo.slug, ctx.build.commit + "-${DRONE_BUILD_NUMBER}"), + ], + }, + ], + "depends_on": [], + "trigger": { + "ref": [ + "refs/heads/master", + "refs/pull/**", + "refs/tags/**", + ], + }, + } + + for branch in config["branches"]: + result["trigger"]["ref"].append("refs/heads/%s" % branch) + + return result + def notify(): result = { "kind": "pipeline", @@ -943,7 +1285,7 @@ def notify(): def databaseService(db): dbName = getDbName(db) if (dbName == "mariadb") or (dbName == "mysql"): - return [{ + service = { "name": dbName, "image": db, "pull": "always", @@ -953,7 +1295,10 @@ def databaseService(db): "MYSQL_DATABASE": getDbDatabase(db), "MYSQL_ROOT_PASSWORD": getDbRootPassword(), }, - }] + } + if (db == "mysql:8.0"): + service["command"] = ["--default-authentication-plugin=mysql_native_password"] + return [service] if dbName == "postgres": return [{ @@ -1027,12 +1372,25 @@ def ldapService(ldapNeeded): "LDAP_ORGANISATION": "owncloud", "LDAP_ADMIN_PASSWORD": "admin", "LDAP_TLS_VERIFY_CLIENT": "never", - "HOSTNAME": "ldap", }, }] return [] +def elasticSearchService(esVersion): + if esVersion == "none": + return [] + + return [{ + "name": "elasticsearch", + "image": "owncloudops/elasticsearch:%s" % esVersion, + "pull": "always", + "environment": { + "ELASTICSEARCH_ROOT_LOG_LEVEL": "warn", + "ELASTICSEARCH_BOOTSTRAP_MEMORY_LOCK": "false", + }, + }] + def scalityService(serviceParams): serviceEnvironment = { "HOST_NAME": "scality", @@ -1075,18 +1433,20 @@ def cephService(serviceParams): "environment": serviceEnvironment, }] -def owncloudService(version, phpVersion, name = "server", path = "/var/www/owncloud/server", ssl = True, xForwardedFor = False): +def owncloudService(version, phpVersion, name, path, ssl, xForwardedFor): if ssl: environment = { "APACHE_WEBROOT": path, "APACHE_CONFIG_TEMPLATE": "ssl", "APACHE_SSL_CERT_CN": "server", - "APACHE_SSL_CERT": "/var/www/owncloud/%s.crt" % name, - "APACHE_SSL_KEY": "/var/www/owncloud/%s.key" % name, + "APACHE_SSL_CERT": "%s/%s.crt" % (dir["base"], name), + "APACHE_SSL_KEY": "%s/%s.key" % (dir["base"], name), + "APACHE_LOGGING_PATH": "/dev/null", } else: environment = { "APACHE_WEBROOT": path, + "APACHE_LOGGING_PATH": "/dev/null", } return [{ @@ -1097,11 +1457,11 @@ def owncloudService(version, phpVersion, name = "server", path = "/var/www/owncl "commands": ([ "a2enmod remoteip", "cd /etc/apache2", - 'echo "RemoteIPHeader X-Forwarded-For" >> apache2.conf', + "echo 'RemoteIPHeader X-Forwarded-For' >> apache2.conf", # This replaces the first occurrence of "%h with "%a in apache2.conf file telling Apache to log the client # IP as recorded by mod_remoteip (%a) rather than hostname (%h). For more info check this out: # https://www.digitalocean.com/community/questions/get-client-public-ip-on-apache-server-used-behind-load-balancer - 'sed -i \'0,/"%h/s//"%a/\' apache2.conf', + "sed -i '0,/\"%h/s//\"%a/' apache2.conf", ] if xForwardedFor else []) + [ "/usr/local/bin/apachectl -e debug -D FOREGROUND", ], @@ -1140,7 +1500,52 @@ def getDbDatabase(db): return "owncloud" -def installCore(version, db, useBundledApp): +def getNodeJsVersion(): + if "nodeJsVersion" not in config: + # We use nodejs 14 as the default + return "14" + else: + return config["nodeJsVersion"] + +def cacheRestore(): + return [{ + "name": "cache-restore", + "image": "plugins/s3-cache:1", + "pull": "always", + "settings": { + "access_key": { + "from_secret": "cache_s3_access_key", + }, + "endpoint": { + "from_secret": "cache_s3_endpoint", + }, + "restore": True, + "secret_key": { + "from_secret": "cache_s3_secret_key", + }, + }, + "when": { + "instance": [ + "drone.owncloud.services", + "drone.owncloud.com", + ], + }, + }] + +def composerInstall(phpVersion): + return [{ + "name": "composer-install", + "image": "owncloudci/php:%s" % phpVersion, + "pull": "always", + "environment": { + "COMPOSER_HOME": "/drone/src/.cache/composer", + }, + "commands": [ + "make vendor", + ], + }] + +def installCore(ctx, version, db, useBundledApp): host = getDbName(db) dbType = host @@ -1163,7 +1568,7 @@ def installCore(version, db, useBundledApp): "pull": "always", "settings": { "version": version, - "core_path": "/var/www/owncloud/server", + "core_path": dir["server"], "db_type": dbType, "db_name": database, "db_host": host, @@ -1173,11 +1578,11 @@ def installCore(version, db, useBundledApp): } if not useBundledApp: - stepDefinition["settings"]["exclude"] = "apps/%s" % config["app"] + stepDefinition["settings"]["exclude"] = "apps/%s" % ctx.repo.name return [stepDefinition] -def installTestrunner(phpVersion, useBundledApp): +def installTestrunner(ctx, phpVersion, useBundledApp): return [{ "name": "install-testrunner", "image": "owncloudci/php:%s" % phpVersion, @@ -1185,21 +1590,21 @@ def installTestrunner(phpVersion, useBundledApp): "commands": [ "mkdir /tmp/testrunner", "git clone -b master --depth=1 https://github.com/owncloud/core.git /tmp/testrunner", - "rsync -aIX /tmp/testrunner /var/www/owncloud", + "rsync -aIX /tmp/testrunner %s" % dir["base"], ] + ([ - "cp -r /var/www/owncloud/testrunner/apps/%s /var/www/owncloud/server/apps/" % config["app"], + "cp -r %s/apps/%s %s/apps/" % (dir["testrunner"], ctx.repo.name, dir["server"]), ] if not useBundledApp else []), }] def installExtraApps(phpVersion, extraApps): commandArray = [] for app, command in extraApps.items(): - commandArray.append("git clone https://github.com/owncloud/%s.git /var/www/owncloud/testrunner/apps/%s" % (app, app)) - commandArray.append("cp -r /var/www/owncloud/testrunner/apps/%s /var/www/owncloud/server/apps/" % app) + commandArray.append("git clone https://github.com/owncloud/%s.git %s/apps/%s" % (app, dir["testrunner"], app)) + commandArray.append("cp -r %s/apps/%s %s/apps/" % (dir["testrunner"], app, dir["server"])) if (command != ""): - commandArray.append("cd /var/www/owncloud/server/apps/%s" % app) + commandArray.append("cd %s/apps/%s" % (dir["server"], app)) commandArray.append(command) - commandArray.append("cd /var/www/owncloud/server") + commandArray.append("cd %s" % dir["server"]) commandArray.append("php occ a:l") commandArray.append("php occ a:e %s" % app) commandArray.append("php occ a:l") @@ -1214,33 +1619,50 @@ def installExtraApps(phpVersion, extraApps): "commands": commandArray, }] -def installApp(phpVersion): +def installApp(ctx, phpVersion): if "appInstallCommand" not in config: return [] - return [{ - "name": "install-app-%s" % config["app"], + if "buildJsDeps" not in config: + installJsDeps = False + else: + installJsDeps = config["buildJsDeps"] + + return [ + { + "name": "install-app-js-%s" % config["app"], + "image": "owncloudci/nodejs:%s" % getNodeJsVersion(), + "pull": "always", + "commands": [ + "cd /var/www/owncloud/server/apps/%s" % config["app"], + "make install-js-deps", + "make build-dev", + ], + }, + ] if installJsDeps else [] + [{ + "name": "install-app-%s" % ctx.repo.name, "image": "owncloudci/php:%s" % phpVersion, "pull": "always", "commands": [ - "cd /var/www/owncloud/server/apps/%s" % config["app"], + "cd %s/apps/%s" % (dir["server"], ctx.repo.name), config["appInstallCommand"], ], }] -def setupServerAndApp(phpVersion, logLevel): +def setupServerAndApp(ctx, phpVersion, logLevel, federatedServerNeeded = False, enableApp = True): return [{ - "name": "setup-server-%s" % config["app"], + "name": "setup-server-%s" % ctx.repo.name, "image": "owncloudci/php:%s" % phpVersion, "pull": "always", "commands": [ - "cd /var/www/owncloud/server", + "cd %s" % dir["server"], "php occ a:l", - "php occ a:e %s" % config["app"], + "php occ a:e %s" % ctx.repo.name if enableApp else "", "php occ a:e testing", "php occ a:l", "php occ config:system:set trusted_domains 1 --value=server", "php occ log:manage --level %s" % logLevel, + "php occ config:system:set csrf.disabled --value=true" if federatedServerNeeded else "", ], }] @@ -1255,14 +1677,14 @@ def setupCeph(serviceParams): createFirstBucket = serviceParams["createFirstBucket"] if "createFirstBucket" in serviceParams else True setupCommands = serviceParams["setupCommands"] if "setupCommands" in serviceParams else [ "wait-for-it -t 600 ceph:80", - "cd /var/www/owncloud/server/apps/files_primary_s3", - "cp tests/drone/ceph.config.php /var/www/owncloud/server/config", - "cd /var/www/owncloud/server", + "cd %s/apps/files_primary_s3" % dir["server"], + "cp tests/drone/ceph.config.php %s/config" % dir["server"], + "cd %s" % dir["server"], ] return [{ "name": "setup-ceph", - "image": "owncloudci/php:7.2", + "image": "owncloudci/php:7.4", "pull": "always", "commands": setupCommands + ([ "./apps/files_primary_s3/tests/drone/create-bucket.sh", @@ -1283,32 +1705,48 @@ def setupScality(serviceParams): createExtraBuckets = serviceParams["createExtraBuckets"] if "createExtraBuckets" in serviceParams else False setupCommands = serviceParams["setupCommands"] if "setupCommands" in serviceParams else [ "wait-for-it -t 600 scality:8000", - "cd /var/www/owncloud/server/apps/files_primary_s3", - "cp tests/drone/%s /var/www/owncloud/server/config" % configFile, - "cd /var/www/owncloud/server", + "cd %s/apps/files_primary_s3" % dir["server"], + "cp tests/drone/%s %s/config" % (configFile, dir["server"]), + "cd %s" % dir["server"], ] return [{ "name": "setup-scality", - "image": "owncloudci/php:7.2", + "image": "owncloudci/php:7.4", "pull": "always", "commands": setupCommands + ([ "php occ s3:create-bucket owncloud --accept-warning", ] if createFirstBucket else []) + ([ - "for I in $(seq 1 9); do php ./occ s3:create-bucket owncloud$I --accept-warning; done", + "for I in $(seq 1 9); do php ./occ s3:create-bucket owncloud$I --accept-warning; done", ] if createExtraBuckets else []), }] +def setupElasticSearch(esVersion): + if esVersion == "none": + return [] + + return [{ + "name": "setup-es", + "image": "owncloudci/php:7.4", + "pull": "always", + "commands": [ + "cd %s" % dir["server"], + "php occ config:app:set search_elastic servers --value elasticsearch", + "wait-for-it -t 600 elasticsearch:9200", + "php occ search:index:reset --force", + ], + }] + def fixPermissions(phpVersion, federatedServerNeeded): return [{ "name": "fix-permissions", "image": "owncloudci/php:%s" % phpVersion, "pull": "always", "commands": [ - "chown -R www-data /var/www/owncloud/server", + "chown -R www-data %s" % dir["server"], "wait-for-it -t 600 server:80", ] + ([ - "chown -R www-data /var/www/owncloud/federated", + "chown -R www-data %s" % dir["federated"], "wait-for-it -t 600 federated:80", ] if federatedServerNeeded else []), }] @@ -1316,11 +1754,11 @@ def fixPermissions(phpVersion, federatedServerNeeded): def owncloudLog(server): return [{ "name": "owncloud-log-%s" % server, - "image": "owncloud/ubuntu:20.04", + "image": "owncloud/ubuntu:18.04", "pull": "always", "detach": True, "commands": [ - "tail -f /var/www/owncloud/%s/data/owncloud.log" % server, + "tail -f %s/%s/data/owncloud.log" % (dir["base"], server), ], }] @@ -1350,7 +1788,7 @@ def installFederated(federatedServerVersion, phpVersion, logLevel, db, dbSuffix "pull": "always", "settings": { "version": federatedServerVersion, - "core_path": "/var/www/owncloud/federated", + "core_path": dir["federated"], "db_type": "mysql", "db_name": database, "db_host": host + dbSuffix, @@ -1363,8 +1801,8 @@ def installFederated(federatedServerVersion, phpVersion, logLevel, db, dbSuffix "image": "owncloudci/php:%s" % phpVersion, "pull": "always", "commands": [ - 'echo "export TEST_SERVER_FED_URL=http://federated" > /var/www/owncloud/saved-settings.sh', - "cd /var/www/owncloud/federated", + "echo 'export TEST_SERVER_FED_URL=http://federated' > %s/saved-settings.sh" % dir["base"], + "cd %s" % dir["federated"], "php occ a:l", "php occ a:e testing", "php occ a:l", @@ -1382,7 +1820,7 @@ def databaseServiceForFederation(db, suffix): print("Not implemented federated database for ", dbName) return [] - return [{ + service = { "name": dbName + suffix, "image": db, "pull": "always", @@ -1392,4 +1830,196 @@ def databaseServiceForFederation(db, suffix): "MYSQL_DATABASE": getDbDatabase(db) + suffix, "MYSQL_ROOT_PASSWORD": getDbRootPassword(), }, + } + if (db == "mysql:8.0"): + service["command"] = ["--default-authentication-plugin=mysql_native_password"] + return [service] + +def buildTestConfig(params): + configs = [] + for server in params["servers"]: + for browser in params["browsers"]: + for phpVersion in params["phpVersions"]: + for db in params["databases"]: + for esVersion in params["esVersions"]: + for runPart in range(1, params["numberOfParts"] + 1): + config = dict(params) + config["server"] = server + config["browser"] = browser + config["phpVersion"] = phpVersion + config["database"] = db + config["esVersion"] = esVersion + config["runPart"] = runPart + configs.append(config) + return configs + +def stopBuild(ctx, earlyFail): + if (earlyFail): + return [{ + "name": "stop-build", + "image": "drone/cli:alpine", + "pull": "always", + "environment": { + "DRONE_SERVER": "https://drone.owncloud.com", + "DRONE_TOKEN": { + "from_secret": "drone_token", + }, + }, + "commands": [ + "drone build stop owncloud/%s ${DRONE_BUILD_NUMBER}" % ctx.repo.name, + ], + "when": { + "status": [ + "failure", + ], + "event": [ + "pull_request", + ], + }, + }] + + else: + return [] + +def buildGithubCommentForBuildStopped(alternateSuiteName, earlyFail): + if (earlyFail): + return [{ + "name": "build-github-comment-buildStop", + "image": "owncloud/ubuntu:16.04", + "pull": "always", + "commands": [ + "echo ':boom: Acceptance tests pipeline %s failed. The build has been cancelled.\\n\\n${DRONE_BUILD_LINK}/${DRONE_JOB_NUMBER}${DRONE_STAGE_NUMBER}/1\\n' >> %s/comments.file" % (alternateSuiteName, dir["base"]), + ], + "when": { + "status": [ + "failure", + ], + "event": [ + "pull_request", + ], + }, + }] + + else: + return [] + +def githubComment(earlyFail): + if (earlyFail): + return [{ + "name": "github-comment", + "image": "jmccann/drone-github-comment:1", + "pull": "if-not-exists", + "settings": { + "message_file": "%s/comments.file" % dir["base"], + }, + "environment": { + "GITHUB_TOKEN": { + "from_secret": "github_token", + }, + }, + "when": { + "status": [ + "failure", + ], + "event": [ + "pull_request", + ], + }, + }] + + else: + return [] + +def checkStarlark(): + return [{ + "kind": "pipeline", + "type": "docker", + "name": "check-starlark", + "steps": [ + { + "name": "format-check-starlark", + "image": "owncloudci/bazel-buildifier", + "pull": "always", + "commands": [ + "buildifier --mode=check .drone.star", + ], + }, + { + "name": "show-diff", + "image": "owncloudci/bazel-buildifier", + "pull": "always", + "commands": [ + "buildifier --mode=fix .drone.star", + "git diff", + ], + "when": { + "status": [ + "failure", + ], + }, + }, + ], + "depends_on": [], + "trigger": { + "ref": [ + "refs/pull/**", + ], + }, + }] + +def phplint(ctx): + pipelines = [] + + if "phplint" not in config: + return pipelines + + if type(config["phplint"]) == "bool": + if not config["phplint"]: + return pipelines + + result = { + "kind": "pipeline", + "type": "docker", + "name": "lint-test", + "workspace": { + "base": "/var/www/owncloud", + "path": "server/apps/%s" % ctx.repo.name, + }, + "steps": installNPM() + + lintTest(), + "depends_on": [], + "trigger": { + "ref": [ + "refs/heads/master", + "refs/tags/**", + "refs/pull/**", + ], + }, + } + + for branch in config["branches"]: + result["trigger"]["ref"].append("refs/heads/%s" % branch) + + pipelines.append(result) + + return pipelines + +def installNPM(): + return [{ + "name": "npm-install", + "image": "owncloudci/nodejs:%s" % getNodeJsVersion(), + "pull": "always", + "commands": [ + "yarn install --frozen-lockfile", + ], + }] + +def lintTest(): + return [{ + "name": "lint-test", + "image": "owncloudci/php:7.4", + "pull": "always", + "commands": [ + "make test-lint", + ], }] diff --git a/.drone/cancelBuilds.sh b/.drone/cancelBuilds.sh new file mode 100644 index 0000000..2d06b8e --- /dev/null +++ b/.drone/cancelBuilds.sh @@ -0,0 +1,62 @@ +SCRIPT_PATH=$(dirname "$0") +SCRIPT_PATH=$( cd "${SCRIPT_PATH}" && pwd ) # normalized and made absolute + +if [ -z "${OWNCLOUD_APP}" ] +then + echo "OWNCLOUD_APP environment variable is required to be defined as the organisation and repo name in the format: owncloud/app_name" + exit 1 +fi +# The recentBuilds.txt file contains all the information of the PR whose builds are recently running. +# Therefore, to filter out the BuildID and the Reference from all the recent builds which are running, +# the following awk command is used and the output containing the BuildId and the Reference of each +# builds/prs are stored in the filteredDescriptions.txt file. +awk "/Build #|Ref: refs\/pull/" ${SCRIPT_PATH}/../recentBuilds.txt > ${SCRIPT_PATH}/filteredDescriptions.txt + +# The thisBuildInfo.txt file contains the information of the current Build, including the BuildID, Reference, and +# other information of that particular build. To get the reference number/ pr number of the recent build, +# the following awk command and grep commands are used, where the Reference part ("Ref: refs/pull/5496/head") +# is stored in the "thisBuildFiltered.txt" file. From the reference, only the pr/reference number is extracted into +# the "NUMBER" variable. +awk "/Ref: refs\/pull/" ${SCRIPT_PATH}/../thisBuildInfo.txt > ${SCRIPT_PATH}/thisBuildFiltered.txt +NUMBER=$(grep -o -E "[0-9]+" ${SCRIPT_PATH}/thisBuildFiltered.txt) +referenceNum="Ref: refs/pull/"$NUMBER"/head" + +# From all the recent builds, the information about the BuildID and the reference which was stored in the +# "filteredDescriptions.txt" file, only the BuildID part from each of the buildInformation whose Reference is +# exactly same as that of the current build reference is filtered and stored in the "buildsToStop.txt" file. +# The buildIDs and the reference information of each build is stored sequentially in the "filteredDescriptions.txt" file as: + +# "Build #" +# "" +# "Build #" +# "" +# "Build #" +# "" + +# Therefore the following awk command extracts each lines just above the expected reference information. +awk -v ref="$referenceNum" 'index($0,ref){print p} {p=$0}' ${SCRIPT_PATH}/filteredDescriptions.txt > ${SCRIPT_PATH}/buildsToStop.txt + + +# The "buildsToStop.txt" file now contains the buildIDs of the recent builds whose reference id was equal to the reference of the +# current pr/build. + +# "Build #" +# "Build #" +# "Build #" +# "Build #" +# "Build #" + +# For each build in the "buildsToStop.txt" file, if the build number is older than the current drone build number, the build is cancelled. + +while IFS="" read -r p || [ -n "$p" ] +do + printf '%s\n' "$p" + buildNumber=$(echo "$p" | awk -F'#' '{print $(2)}') + + if [ $DRONE_BUILD_NUMBER \> "$buildNumber" ] + then + echo "CANCELLING BUILD: " $buildNumber + drone build stop $OWNCLOUD_APP $buildNumber + fi + +done <${SCRIPT_PATH}/buildsToStop.txt diff --git a/composer.json b/composer.json index 7b752b6..676a797 100644 --- a/composer.json +++ b/composer.json @@ -1,12 +1,17 @@ { - "name": "owncloud/qnap", - "require": {}, - "require-dev": { - "bamarni/composer-bin-plugin": "^1.4" - }, - "extra": { - "bamarni-bin": { - "bin-links": false - } + "name": "owncloud/qnap", + "config": { + "platform": { + "php": "7.3" } + }, + "require": {}, + "require-dev": { + "bamarni/composer-bin-plugin": "^1.4" + }, + "extra": { + "bamarni-bin": { + "bin-links": false + } + } } diff --git a/composer.lock b/composer.lock index ba47e85..ba2d443 100644 --- a/composer.lock +++ b/composer.lock @@ -1,65 +1,70 @@ { - "_readme": [ - "This file locks the dependencies of your project to a known state", - "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", - "This file is @generated automatically" - ], - "content-hash": "0a65c627e4da68add3b483a76ae1b6e9", - "packages": [], - "packages-dev": [ - { - "name": "bamarni/composer-bin-plugin", - "version": "1.4.1", - "source": { - "type": "git", - "url": "https://github.com/bamarni/composer-bin-plugin.git", - "reference": "9329fb0fbe29e0e1b2db8f4639a193e4f5406225" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/bamarni/composer-bin-plugin/zipball/9329fb0fbe29e0e1b2db8f4639a193e4f5406225", - "reference": "9329fb0fbe29e0e1b2db8f4639a193e4f5406225", - "shasum": "" - }, - "require": { - "composer-plugin-api": "^1.0 || ^2.0", - "php": "^5.5.9 || ^7.0 || ^8.0" - }, - "require-dev": { - "composer/composer": "^1.0 || ^2.0", - "symfony/console": "^2.5 || ^3.0 || ^4.0" - }, - "type": "composer-plugin", - "extra": { - "class": "Bamarni\\Composer\\Bin\\Plugin" - }, - "autoload": { - "psr-4": { - "Bamarni\\Composer\\Bin\\": "src" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "description": "No conflicts for your bin dependencies", - "keywords": [ - "composer", - "conflict", - "dependency", - "executable", - "isolation", - "tool" - ], - "time": "2020-05-03T08:27:20+00:00" + "_readme": [ + "This file locks the dependencies of your project to a known state", + "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", + "This file is @generated automatically" + ], + "content-hash": "08bce8888d90a8cac412ebd890217822", + "packages": [], + "packages-dev": [ + { + "name": "bamarni/composer-bin-plugin", + "version": "1.4.1", + "source": { + "type": "git", + "url": "https://github.com/bamarni/composer-bin-plugin.git", + "reference": "9329fb0fbe29e0e1b2db8f4639a193e4f5406225" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/bamarni/composer-bin-plugin/zipball/9329fb0fbe29e0e1b2db8f4639a193e4f5406225", + "reference": "9329fb0fbe29e0e1b2db8f4639a193e4f5406225", + "shasum": "" + }, + "require": { + "composer-plugin-api": "^1.0 || ^2.0", + "php": "^5.5.9 || ^7.0 || ^8.0" + }, + "require-dev": { + "composer/composer": "^1.0 || ^2.0", + "symfony/console": "^2.5 || ^3.0 || ^4.0" + }, + "type": "composer-plugin", + "extra": { + "class": "Bamarni\\Composer\\Bin\\Plugin" + }, + "autoload": { + "psr-4": { + "Bamarni\\Composer\\Bin\\": "src" } - ], - "aliases": [], - "minimum-stability": "stable", - "stability-flags": [], - "prefer-stable": false, - "prefer-lowest": false, - "platform": [], - "platform-dev": [], - "plugin-api-version": "1.1.0" + }, + "notification-url": "https://packagist.org/downloads/", + "license": ["MIT"], + "description": "No conflicts for your bin dependencies", + "keywords": [ + "composer", + "conflict", + "dependency", + "executable", + "isolation", + "tool" + ], + "support": { + "issues": "https://github.com/bamarni/composer-bin-plugin/issues", + "source": "https://github.com/bamarni/composer-bin-plugin/tree/master" + }, + "time": "2020-05-03T08:27:20+00:00" + } + ], + "aliases": [], + "minimum-stability": "stable", + "stability-flags": [], + "prefer-stable": false, + "prefer-lowest": false, + "platform": [], + "platform-dev": [], + "platform-overrides": { + "php": "7.3" + }, + "plugin-api-version": "2.1.0" } diff --git a/vendor-bin/owncloud-codestyle/composer.json b/vendor-bin/owncloud-codestyle/composer.json index bb481bd..2d5dfea 100644 --- a/vendor-bin/owncloud-codestyle/composer.json +++ b/vendor-bin/owncloud-codestyle/composer.json @@ -1,5 +1,5 @@ { "require": { - "owncloud/coding-standard": "^2.0" + "owncloud/coding-standard": "^3.0" } -} \ No newline at end of file +} diff --git a/vendor-bin/phan/composer.json b/vendor-bin/phan/composer.json index 2a38112..4a5d0b5 100644 --- a/vendor-bin/phan/composer.json +++ b/vendor-bin/phan/composer.json @@ -1,5 +1,5 @@ { "require": { - "phan/phan": "^1.0.0" + "phan/phan": "^4.0" } -} \ No newline at end of file +} diff --git a/vendor-bin/phpstan/composer.json b/vendor-bin/phpstan/composer.json index 1f6c28b..47c0c31 100644 --- a/vendor-bin/phpstan/composer.json +++ b/vendor-bin/phpstan/composer.json @@ -1,5 +1,5 @@ { "require": { - "phpstan/phpstan": "^0.12.4" + "phpstan/phpstan": "^0.12" } } From 69c75f17e013bbf23c3723ba20a694088a80baba Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Mon, 6 Sep 2021 11:00:14 +0200 Subject: [PATCH 3/9] get streamlined makefile from files_external_s3 --- .drone/cancelBuilds.sh | 0 .php-cs-fixer.dist.php | 13 +++ .php_cs.dist | 13 --- Makefile | 200 +++++++++++++++++++++-------------------- Makefile.release | 74 --------------- 5 files changed, 114 insertions(+), 186 deletions(-) mode change 100644 => 100755 .drone/cancelBuilds.sh create mode 100644 .php-cs-fixer.dist.php delete mode 100644 .php_cs.dist delete mode 100644 Makefile.release diff --git a/.drone/cancelBuilds.sh b/.drone/cancelBuilds.sh old mode 100644 new mode 100755 diff --git a/.php-cs-fixer.dist.php b/.php-cs-fixer.dist.php new file mode 100644 index 0000000..4db0f4f --- /dev/null +++ b/.php-cs-fixer.dist.php @@ -0,0 +1,13 @@ +setUsingCache(true) + ->getFinder() + ->exclude('l10n') + ->exclude('vendor') + ->exclude('vendor-bin') + ->in(__DIR__); + +return $config; diff --git a/.php_cs.dist b/.php_cs.dist deleted file mode 100644 index c12c70f..0000000 --- a/.php_cs.dist +++ /dev/null @@ -1,13 +0,0 @@ -setUsingCache(true) - ->getFinder() - ->in(__DIR__) - ->exclude('build') - ->exclude('vendor-bin') - ->exclude('vendor'); - -return $config; \ No newline at end of file diff --git a/Makefile b/Makefile index 0c83f01..5f9a6a7 100644 --- a/Makefile +++ b/Makefile @@ -1,89 +1,108 @@ SHELL := /bin/bash -YARN := $(shell command -v yarn 2> /dev/null) -NODE_PREFIX=$(shell pwd) COMPOSER_BIN := $(shell command -v composer 2> /dev/null) -NPM := $(shell command -v npm 2> /dev/null) -ifndef NPM - $(error npm is not available on your system, please install npm) -endif +# directories +app_name=$(notdir $(CURDIR)) +build_dir=$(CURDIR)/build +dist_dir=$(build_dir)/dist +doc_files=README.md LICENSE +src_dirs=appinfo lib vendor +all_src=$(src_dirs) $(doc_files) -app_name=qnap +acceptance_test_deps=vendor-bin/behat/vendor # bin file definitions -PHPUNIT=php -d zend.enable_gc=0 "$(PWD)/../../lib/composer/bin/phpunit" -PHPUNITDBG=phpdbg -qrr -d memory_limit=4096M -d zend.enable_gc=0 "$(PWD)/../../lib/composer/bin/phpunit" +PHPUNIT=php -d zend.enable_gc=0 ../../lib/composer/bin/phpunit +PHPUNITDBG=phpdbg -qrr -d memory_limit=4096M -d zend.enable_gc=0 "../../lib/composer/bin/phpunit" PHP_CS_FIXER=php -d zend.enable_gc=0 vendor-bin/owncloud-codestyle/vendor/bin/php-cs-fixer PHAN=php -d zend.enable_gc=0 vendor-bin/phan/vendor/bin/phan PHPSTAN=php -d zend.enable_gc=0 vendor-bin/phpstan/vendor/bin/phpstan +BEHAT_BIN=vendor-bin/behat/vendor/bin/behat + +occ?=$(CURDIR)/../../occ +private_key?=$(HOME)/.owncloud/certificates/$(app_name).key +certificate?=$(HOME)/.owncloud/certificates/$(app_name).crt +sign?=$(occ) integrity:sign-app --privateKey="$(private_key)" --certificate="$(certificate)" +sign_skip_msg="Skipping signing, either no key and certificate found in $(private_key) and $(certificate) or occ can not be found at $(occ)" +ifneq (,$(wildcard $(private_key))) +ifneq (,$(wildcard $(certificate))) +ifneq (,$(wildcard $(occ))) + CAN_SIGN=true +endif +endif +endif -KARMA=$(NODE_PREFIX)/node_modules/.bin/karma - -.DEFAULT_GOAL := all +# start with displaying help +.DEFAULT_GOAL := help help: @fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//' | sed -e 's/##//' | sed -e 's/ */ /' | column -t -s : -## -## Entrypoints -##---------------------- - -.PHONY: all -all: install-deps - -# Remove the appstore build .PHONY: clean -clean: clean-nodejs-deps clean-composer-deps - rm -rf ./build - -.PHONY: clean-nodejs-deps -clean-nodejs-deps: - rm -Rf $(nodejs_deps) +clean: clean-composer-deps clean-build-dir clean-vendor .PHONY: clean-composer-deps clean-composer-deps: - rm -rf ./vendor rm -Rf vendor-bin/**/vendor vendor-bin/**/composer.lock -.PHONY: dev -dev: ## Initialize dev environment -dev: install-deps +.PHONY: clean-vendor +clean-vendor: + rm -Rf vendor + +.PHONY: clean-build-dir +clean-build-dir: + rm -Rf $(build_dir) + +##--------------------- +## Build targets +##--------------------- -# -# Release -# make this app compatible with the ownCloud -# default build tools -# .PHONY: dist -dist: - make -f Makefile.release dist +dist: ## Build distribution +dist: vendor distdir sign package + +.PHONY: distdir +distdir: + rm -rf $(build_dir) + mkdir -p $(dist_dir)/$(app_name) + cp -R $(all_src) $(dist_dir)/$(app_name) + +.PHONY: sign +sign: +ifdef CAN_SIGN + $(sign) --path="$(dist_dir)/$(app_name)" +else + @echo $(sign_skip_msg) +endif -$(KARMA): $(nodejs_deps) +.PHONY: package +package: + tar -czf $(dist_dir)/$(app_name).tar.gz -C $(dist_dir) $(app_name) -## +##--------------------- ## Tests -##---------------------- +##--------------------- .PHONY: test-php-unit test-php-unit: ## Run php unit tests -test-php-unit: vendor/bin/phpunit - $(PHPUNIT) --configuration ./phpunit.xml --testsuite qnap-unit +test-php-unit: ../../lib/composer/bin/phpunit + $(PHPUNIT) --configuration ./phpunit.xml --testsuite unit .PHONY: test-php-unit-dbg test-php-unit-dbg: ## Run php unit tests using phpdbg -test-php-unit-dbg: vendor/bin/phpunit - $(PHPUNITDBG) --configuration ./phpunit.xml --testsuite qnap-unit +test-php-unit-dbg: ../../lib/composer/bin/phpunit + $(PHPUNITDBG) --configuration ./phpunit.xml --testsuite unit .PHONY: test-php-style test-php-style: ## Run php-cs-fixer and check owncloud code-style test-php-style: vendor-bin/owncloud-codestyle/vendor - $(PHP_CS_FIXER) fix -v --diff --diff-format udiff --allow-risky yes --dry-run + $(PHP_CS_FIXER) fix -v --diff --allow-risky yes --dry-run .PHONY: test-php-style-fix test-php-style-fix: ## Run php-cs-fixer and fix code style issues test-php-style-fix: vendor-bin/owncloud-codestyle/vendor - $(PHP_CS_FIXER) fix -v --diff --diff-format udiff --allow-risky yes + $(PHP_CS_FIXER) fix -v --diff --allow-risky yes .PHONY: test-php-phan test-php-phan: ## Run phan @@ -95,81 +114,64 @@ test-php-phpstan: ## Run phpstan test-php-phpstan: vendor-bin/phpstan/vendor $(PHPSTAN) analyse --memory-limit=4G --configuration=./phpstan.neon --no-progress --level=5 appinfo lib -.PHONY: test-js -test-js: $(nodejs_deps) - $(KARMA) start tests/js/karma.config.js --single-run - -# -# Translation -#-------------------------------------- - -.PHONY: l10n-push -l10n-push: - cd l10n && tx -d push -s --skip --no-interactive - -.PHONY: l10n-pull -l10n-pull: - cd l10n && tx -d pull -a --skip --minimum-perc=75 - -.PHONY: l10n-clean -l10n-clean: - rm -rf l10n/l10n.pl - find l10n -type f -name \*.po -or -name \*.pot | xargs rm -f - find l10n -type f -name uz.\* -or -name yo.\* -or -name ne.\* -or -name or_IN.\* | xargs git rm -f || true +.PHONY: test-acceptance-api +test-acceptance-api: ## Run API acceptance tests +test-acceptance-api: $(acceptance_test_deps) + BEHAT_BIN=$(BEHAT_BIN) ../../tests/acceptance/run.sh --remote --type api -.PHONY: l10n-read -l10n-read: l10n/l10n.pl - cd l10n && perl l10n.pl $(app_name) read +.PHONY: test-acceptance-cli +test-acceptance-cli: ## Run CLI acceptance tests +test-acceptance-cli: $(acceptance_test_deps) + BEHAT_BIN=$(BEHAT_BIN) ../../tests/acceptance/run.sh --remote --type cli -.PHONY: l10n-write -l10n-write: l10n/l10n.pl - cd l10n && perl l10n.pl $(app_name) write +.PHONY: test-acceptance-webui +test-acceptance-webui: ## Run webUI acceptance tests +test-acceptance-webui: $(acceptance_test_deps) + BEHAT_BIN=$(BEHAT_BIN) ../../tests/acceptance/run.sh --remote --type webUI -l10n/l10n.pl: - wget -qO l10n/l10n.pl https://raw.githubusercontent.com/owncloud-ci/transifex/d1c63674d791fe8812216b29da9d8f2f26e7e138/rootfs/usr/bin/l10n +.PHONY: test-acceptance-core-api +test-acceptance-core-api: ## Run core API acceptance tests +test-acceptance-core-api: $(acceptance_test_deps) + BEHAT_BIN=$(BEHAT_BIN) ../../tests/acceptance/run.sh --remote --type api -c ../../tests/acceptance/config/behat.yml --tags '~@skipOnEncryption&&~@skipOnEncryptionType:${ENCRYPTION_TYPE}&&~@skip' -## -## Dependency management -##---------------------- +.PHONY: test-acceptance-core-webui +test-acceptance-core-webui: ## Run core webUI acceptance tests +test-acceptance-core-webui: $(acceptance_test_deps) + BEHAT_BIN=$(BEHAT_BIN) ../../tests/acceptance/run.sh --remote --type webui -c ../../tests/acceptance/config/behat.yml --tags '~@skipOnEncryption&&~@skipOnEncryptionType:${ENCRYPTION_TYPE}&&~@skip' -.PHONY: install-deps -install-deps: ## Install dependencies -install-deps: install-php-deps install-js-deps +# +# Dependency management +#---------------------- composer.lock: composer.json @echo composer.lock is not up to date. -.PHONY: install-php-deps -install-php-deps: ## Install PHP dependencies -install-php-deps: vendor vendor-bin composer.json composer.lock - -.PHONY: install-js-deps -install-js-deps: ## Install PHP dependencies -install-js-deps: $(nodejs_deps) - -vendor: composer.lock - $(COMPOSER_BIN) install --no-dev +vendor: + composer install --no-dev -vendor/bin/phpunit: composer.lock - $(COMPOSER_BIN) install - -vendor/bamarni/composer-bin-plugin: composer.lock - $(COMPOSER_BIN) install +vendor/bamarni/composer-bin-plugin: + composer install vendor-bin/owncloud-codestyle/vendor: vendor/bamarni/composer-bin-plugin vendor-bin/owncloud-codestyle/composer.lock - $(COMPOSER_BIN) bin owncloud-codestyle install --no-progress + composer bin owncloud-codestyle install --no-progress vendor-bin/owncloud-codestyle/composer.lock: vendor-bin/owncloud-codestyle/composer.json @echo owncloud-codestyle composer.lock is not up to date. vendor-bin/phan/vendor: vendor/bamarni/composer-bin-plugin vendor-bin/phan/composer.lock - $(COMPOSER_BIN) bin phan install --no-progress + composer bin phan install --no-progress vendor-bin/phan/composer.lock: vendor-bin/phan/composer.json @echo phan composer.lock is not up to date. vendor-bin/phpstan/vendor: vendor/bamarni/composer-bin-plugin vendor-bin/phpstan/composer.lock - $(COMPOSER_BIN) bin phpstan install --no-progress + composer bin phpstan install --no-progress vendor-bin/phpstan/composer.lock: vendor-bin/phpstan/composer.json @echo phpstan composer.lock is not up to date. + +vendor-bin/behat/vendor: vendor/bamarni/composer-bin-plugin vendor-bin/behat/composer.lock + composer bin behat install --no-progress + +vendor-bin/behat/composer.lock: vendor-bin/behat/composer.json + @echo behat composer.lock is not up to date. \ No newline at end of file diff --git a/Makefile.release b/Makefile.release deleted file mode 100644 index 0ba0c5d..0000000 --- a/Makefile.release +++ /dev/null @@ -1,74 +0,0 @@ -SHELL := /bin/bash - -COMPOSER_BIN := $(shell command -v composer 2> /dev/null) -ifndef COMPOSER_BIN - $(error composer is not available on your system, please install composer) -endif - -NPM := $(shell command -v npm 2> /dev/null) -ifndef NPM - $(error npm is not available on your system, please install npm) -endif - -GIT := $(shell command -v git 2> /dev/null) -ifndef GIT - $(error git is not available on your system, please install git) -endif - - -app_name=qnap -build_dir=$(CURDIR)/build -dist_dir=$(build_dir)/dist -src_files=README.md LICENSE -src_dirs=appinfo img lib vendor templates -all_src=$(src_dirs) $(src_files) - -occ=$(CURDIR)/../../occ -private_key=$(HOME)/.owncloud/certificates/$(app_name).key -certificate=$(HOME)/.owncloud/certificates/$(app_name).crt -sign=$(occ) integrity:sign-app --privateKey="$(private_key)" --certificate="$(certificate)" -sign_skip_msg="Skipping signing, either no key and certificate found in $(private_key) and $(certificate) or occ can not be found at $(occ)" -ifneq (,$(wildcard $(private_key))) -ifneq (,$(wildcard $(certificate))) -ifneq (,$(wildcard $(occ))) - CAN_SIGN=true -endif -endif -endif - -.DEFAULT_GOAL := help - -help: - @fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//' | sed -e 's/##//' - -## -## Build targets -##-------------------------------------- - -.PHONY: dist -dist: ## Build distribution -dist: composer distdir sign package - - -.PHONY: composer -composer: - $(COMPOSER_BIN) install --no-dev - - -.PHONY: distdir -distdir: - rm -rf $(dist_dir) - mkdir -p $(dist_dir)/$(app_name) - cp -R $(all_src) $(dist_dir)/$(app_name) - -.PHONY: sign -sign: -ifdef CAN_SIGN - $(sign) --path="$(dist_dir)/$(app_name)" -else - @echo $(sign_skip_msg) -endif - -.PHONY: package -package: - tar -czf $(dist_dir)/$(app_name).tar.gz -C $(dist_dir) $(app_name) From 8d2e76b73e24d27ff0a55d16f8824686281d5eb9 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Mon, 6 Sep 2021 11:13:43 +0200 Subject: [PATCH 4/9] fix format, supress nullable --- lib/LicenseParser.php | 4 ++-- tests/unit/CheckActiveUsersTest.php | 20 ++++++++++++-------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/lib/LicenseParser.php b/lib/LicenseParser.php index 29a91b0..465bd25 100644 --- a/lib/LicenseParser.php +++ b/lib/LicenseParser.php @@ -34,8 +34,8 @@ public function loadLicensesText(string $licenseText): void { $now = $this->timeFactory->getTime(); foreach ($licenses as $r) { $license = \json_decode($r['license_info_json_str'] ?? '{}', true); - $validFrom = \DateTime::createFromFormat(self::DATETIME_FORMAT, $license['valid_from']); - $validUntil = \DateTime::createFromFormat(self::DATETIME_FORMAT, $license['valid_until']); + $validFrom = \DateTime::createFromFormat(self::DATETIME_FORMAT, $license['valid_from']); // @phan-suppress-current-line PhanTypeArraySuspiciousNullable + $validUntil = \DateTime::createFromFormat(self::DATETIME_FORMAT, $license['valid_until']); // @phan-suppress-current-line PhanTypeArraySuspiciousNullable # invalid license information # TODO: log if ($validFrom === false || $validUntil === false) { diff --git a/tests/unit/CheckActiveUsersTest.php b/tests/unit/CheckActiveUsersTest.php index 69276e2..017e822 100644 --- a/tests/unit/CheckActiveUsersTest.php +++ b/tests/unit/CheckActiveUsersTest.php @@ -56,11 +56,11 @@ class CheckActiveusersTest extends TestCase { private $userAllowance = 0; /** @var array */ - const ENABLED_ADMIN_USER = ['enabled' => true, 'admin' => true, 'guest' => false]; - const ENABLED_GUEST_USER = ['enabled' => true, 'admin' => false, 'guest' => true]; - const DISABLED_GUEST_USER = ['enabled' => false, 'admin' => false, 'guest' => true]; - const ENABLED_NORMAL_USER = ['enabled' => true, 'admin' => false, 'guest' => false]; - const DISABLED_NORMAL_USER = ['enabled' => false, 'admin' => false, 'guest' => false]; + public const ENABLED_ADMIN_USER = ['enabled' => true, 'admin' => true, 'guest' => false]; + public const ENABLED_GUEST_USER = ['enabled' => true, 'admin' => false, 'guest' => true]; + public const DISABLED_GUEST_USER = ['enabled' => false, 'admin' => false, 'guest' => true]; + public const ENABLED_NORMAL_USER = ['enabled' => true, 'admin' => false, 'guest' => false]; + public const DISABLED_NORMAL_USER = ['enabled' => false, 'admin' => false, 'guest' => false]; /** * @dataProvider providesUserAllowance @@ -253,9 +253,13 @@ protected function setUp(): void { $this->urlGenerator = $this->createMock(IURLGenerator::class); $command = new CheckActiveUsers( - $this->userManager, $this->mailer, $this->l10n, - $this->groupManager, $this->notificationManager, - $this->urlGenerator, $this->licenseManager, + $this->userManager, + $this->mailer, + $this->l10n, + $this->groupManager, + $this->notificationManager, + $this->urlGenerator, + $this->licenseManager, $this->userTypeHelper ); $this->commandTester = new CommandTester($command); From 2c38975b6b9f516eba4035b8d83486f20e02547b Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Mon, 6 Sep 2021 11:30:23 +0200 Subject: [PATCH 5/9] test against recent postgres --- .drone.star | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.drone.star b/.drone.star index 26d5734..bf10927 100644 --- a/.drone.star +++ b/.drone.star @@ -31,16 +31,18 @@ config = { ], "databases": [ "sqlite", + "mariadb:10.2", + "postgres:13.3", ], - "coverage": True, }, "reducedDatabases": { "phpVersions": [ "7.3", ], "databases": [ - "mysql:5.5", - "postgres:9.4", + "sqlite", + "mariadb:10.2", + "postgres:13.3", ], "coverage": False, }, From 61194bad0b54d9776263ca342f283d67c8c61320 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Mon, 6 Sep 2021 12:59:30 +0200 Subject: [PATCH 6/9] add sonarcloud config --- sonar-project.properties | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 sonar-project.properties diff --git a/sonar-project.properties b/sonar-project.properties new file mode 100644 index 0000000..4fd25f8 --- /dev/null +++ b/sonar-project.properties @@ -0,0 +1,33 @@ +# Organization and project keys are displayed in the right sidebar of the project homepage +sonar.organization=owncloud-1 +sonar.projectKey=qnap +sonar.projectVersion=1.0 +sonar.host.url=https://sonarcloud.io + +# ===================================================== +# Meta-data for the project +# ===================================================== + +sonar.links.homepage=https://github.com/owncloud/qnap +sonar.links.ci=https://drone.owncloud.com/owncloud/qnap/ +sonar.links.scm=https://github.com/owncloud/qnap +sonar.links.issue=https://github.com/owncloud/qnap/issues + +# ===================================================== +# Properties that will be shared amongst all modules +# ===================================================== + +# Just look in these directories for code +sonar.sources=. +sonar.inclusions=appinfo/**,lib/** + +# Pull Requests +sonar.pullrequest.provider=GitHub +sonar.pullrequest.github.repository=owncloud/qnap +sonar.pullrequest.base=${env.SONAR_PULL_REQUEST_BASE} +sonar.pullrequest.branch=${env.SONAR_PULL_REQUEST_BRANCH} +sonar.pullrequest.key=${env.SONAR_PULL_REQUEST_KEY} + +# Properties specific to language plugins: +sonar.php.coverage.reportPaths=results/clover-phpunit-php7.3-sqlite.xml,results/clover-phpunit-php7.3-mariadb10.2.xml,results/clover-phpunit-php7.3-postgres13.3-sqlite.xml,results/clover-phpunit-php7.4-sqlite.xml,results/clover-phpunit-php7.4-mariadb10.2.xml +sonar.javascript.lcov.reportPaths=results/lcov.info From ffa193bf158060b86b1e2ccc12f9f67622c653b2 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Mon, 6 Sep 2021 13:15:34 +0200 Subject: [PATCH 7/9] readd l10n --- Makefile | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5f9a6a7..4ee0e56 100644 --- a/Makefile +++ b/Makefile @@ -139,6 +139,35 @@ test-acceptance-core-webui: ## Run core webUI acceptance tests test-acceptance-core-webui: $(acceptance_test_deps) BEHAT_BIN=$(BEHAT_BIN) ../../tests/acceptance/run.sh --remote --type webui -c ../../tests/acceptance/config/behat.yml --tags '~@skipOnEncryption&&~@skipOnEncryptionType:${ENCRYPTION_TYPE}&&~@skip' +# +# Translation +#-------------------------------------- + +.PHONY: l10n-push +l10n-push: + cd l10n && tx -d push -s --skip --no-interactive + +.PHONY: l10n-pull +l10n-pull: + cd l10n && tx -d pull -a --skip --minimum-perc=75 + +.PHONY: l10n-clean +l10n-clean: + rm -rf l10n/l10n.pl + find l10n -type f -name \*.po -or -name \*.pot | xargs rm -f + find l10n -type f -name uz.\* -or -name yo.\* -or -name ne.\* -or -name or_IN.\* | xargs git rm -f || true + +.PHONY: l10n-read +l10n-read: l10n/l10n.pl + cd l10n && perl l10n.pl $(app_name) read + +.PHONY: l10n-write +l10n-write: l10n/l10n.pl + cd l10n && perl l10n.pl $(app_name) write + +l10n/l10n.pl: + wget -qO l10n/l10n.pl https://raw.githubusercontent.com/owncloud-ci/transifex/d1c63674d791fe8812216b29da9d8f2f26e7e138/rootfs/usr/bin/l10n + # # Dependency management #---------------------- @@ -174,4 +203,4 @@ vendor-bin/behat/vendor: vendor/bamarni/composer-bin-plugin vendor-bin/behat/com composer bin behat install --no-progress vendor-bin/behat/composer.lock: vendor-bin/behat/composer.json - @echo behat composer.lock is not up to date. \ No newline at end of file + @echo behat composer.lock is not up to date. From a3589ad00a197a1579b26a9b7a34fbfc44d3d38e Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Mon, 6 Sep 2021 13:53:04 +0200 Subject: [PATCH 8/9] fix config --- .drone.star | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/.drone.star b/.drone.star index bf10927..d40871a 100644 --- a/.drone.star +++ b/.drone.star @@ -14,20 +14,13 @@ config = { "branches": [ "master", ], - "appInstallCommand": "make vendor", "codestyle": True, - "javascript": False, "phpstan": True, "phan": True, - "build": { - "commands": [ - "make -f Makefile.release dist", - ], - }, "phpunit": { - "codecov": { + "allDatabases": { "phpVersions": [ - "7.2", + "7.3", ], "databases": [ "sqlite", @@ -37,7 +30,7 @@ config = { }, "reducedDatabases": { "phpVersions": [ - "7.3", + "7.4", ], "databases": [ "sqlite", From 02c80800c0c95b58a904e5b532cab15db6a12590 Mon Sep 17 00:00:00 2001 From: Willy Kloucek <34452982+wkloucek@users.noreply.github.com> Date: Tue, 7 Sep 2021 15:10:02 +0200 Subject: [PATCH 9/9] Update sonar-project.properties Co-authored-by: Phil Davis --- sonar-project.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sonar-project.properties b/sonar-project.properties index 4fd25f8..094bcbf 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -29,5 +29,5 @@ sonar.pullrequest.branch=${env.SONAR_PULL_REQUEST_BRANCH} sonar.pullrequest.key=${env.SONAR_PULL_REQUEST_KEY} # Properties specific to language plugins: -sonar.php.coverage.reportPaths=results/clover-phpunit-php7.3-sqlite.xml,results/clover-phpunit-php7.3-mariadb10.2.xml,results/clover-phpunit-php7.3-postgres13.3-sqlite.xml,results/clover-phpunit-php7.4-sqlite.xml,results/clover-phpunit-php7.4-mariadb10.2.xml +sonar.php.coverage.reportPaths=results/clover-phpunit-php7.3-sqlite.xml,results/clover-phpunit-php7.3-mariadb10.2.xml,results/clover-phpunit-php7.3-postgres13.3-sqlite.xml sonar.javascript.lcov.reportPaths=results/lcov.info