From 69e0da0096dcdd3a63e1e9c44b7408d3aba0ec58 Mon Sep 17 00:00:00 2001 From: Michael Barz Date: Wed, 30 Sep 2020 11:42:26 +0200 Subject: [PATCH 1/4] Send coverage to S3 Cache --- .drone.star | 79 ++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 72 insertions(+), 7 deletions(-) diff --git a/.drone.star b/.drone.star index 3c64628d0..e8178d241 100644 --- a/.drone.star +++ b/.drone.star @@ -117,6 +117,7 @@ def testPipelines(ctx): pipelines.append(testing(ctx, module)) pipelines += [ + uploadCoverage(ctx), localApiTests(ctx, config['apiTests']['coreBranch'], config['apiTests']['coreCommit'], 'owncloud'), localApiTests(ctx, config['apiTests']['coreBranch'], config['apiTests']['coreCommit'], 'ocis') ] @@ -182,6 +183,7 @@ def testing(ctx, module): 'commands': [ 'cd %s' % (module), 'make test', + 'mv coverage.out %s_coverage.out' % (module), ], 'volumes': [ { @@ -191,15 +193,25 @@ def testing(ctx, module): ], }, { - 'name': 'codacy', - 'image': 'plugins/codacy:1', - 'pull': 'always', + 'name': 'coverage-cache', + 'image': 'plugins/s3', 'settings': { - 'token': { - 'from_secret': 'codacy_token', + 'endpoint': { + 'from_secret': 'cache_s3_endpoint' }, - }, - }, + 'bucket': 'cache', + 'source': '%s/%s_coverage.out' % (module, module), + 'target': '%s/%s/coverage' % (ctx.repo.slug, ctx.build.commit + '-${DRONE_BUILD_NUMBER}'), + 'path_style': True, + 'strip_prefix': module, + 'access_key': { + 'from_secret': 'cache_s3_access_key' + }, + 'secret_key': { + 'from_secret': 'cache_s3_secret_key' + } + } + } ] if config['modules'][module] == 'frontend': @@ -224,6 +236,57 @@ def testing(ctx, module): }, } +def uploadCoverage(ctx): + return { + 'kind': 'pipeline', + 'type': 'docker', + 'name': 'upload-coverage', + 'platform': { + 'os': 'linux', + 'arch': 'amd64', + }, + 'steps': [ + { + 'name': 'sync-from-cache', + 'image': 'minio/mc', + 'environment': { + 'MC_HOST_cache': { + 'from_secret': 'cache_s3_connection_url' + } + }, + 'commands': [ + 'mc mirror cache/cache/%s/%s/coverage coverage/' % (ctx.repo.slug, ctx.build.commit + '-${DRONE_BUILD_NUMBER}'), + ] + }, + { + 'name': 'check', + 'image': 'alpine', + 'commands': [ + 'ls -la coverage/', + 'cat coverage/*' + ] + }, + { + 'name': 'codacy', + 'image': 'plugins/codacy:1', + 'pull': 'always', + 'settings': { + 'token': { + 'from_secret': 'codacy_token', + }, + }, + } + ], + 'trigger': { + 'ref': [ + 'refs/heads/master', + 'refs/tags/v*', + 'refs/pull/**', + ], + }, + 'depends_on': getTestSuiteNames(), + } + def localApiTests(ctx, coreBranch = 'master', coreCommit = '', storage = 'owncloud'): return { 'kind': 'pipeline', @@ -480,6 +543,7 @@ def docker(ctx, arch): ], 'depends_on': getTestSuiteNames() + [ + 'upload-coverage', 'localApiTests-owncloud-storage', 'localApiTests-ocis-storage', ] + getCoreApiTestPipelineNames() + getUITestSuiteNames(), @@ -635,6 +699,7 @@ def binary(ctx, name): ], 'depends_on': getTestSuiteNames() + [ + 'upload-coverage', 'localApiTests-owncloud-storage', 'localApiTests-ocis-storage', ] + getCoreApiTestPipelineNames() + getUITestSuiteNames(), From ab5631a8e21f3586fee5676e9c7ed8db8b3c4860 Mon Sep 17 00:00:00 2001 From: Michael Barz Date: Wed, 30 Sep 2020 15:09:30 +0200 Subject: [PATCH 2/4] purge coverage cache after upload --- .drone.star | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/.drone.star b/.drone.star index e8178d241..19c19940b 100644 --- a/.drone.star +++ b/.drone.star @@ -258,14 +258,6 @@ def uploadCoverage(ctx): 'mc mirror cache/cache/%s/%s/coverage coverage/' % (ctx.repo.slug, ctx.build.commit + '-${DRONE_BUILD_NUMBER}'), ] }, - { - 'name': 'check', - 'image': 'alpine', - 'commands': [ - 'ls -la coverage/', - 'cat coverage/*' - ] - }, { 'name': 'codacy', 'image': 'plugins/codacy:1', @@ -275,7 +267,19 @@ def uploadCoverage(ctx): 'from_secret': 'codacy_token', }, }, - } + }, + { + 'name': 'purge-cache', + 'image': 'minio/mc', + 'environment': { + 'MC_HOST_cache': { + 'from_secret': 'cache_s3_connection_url' + } + }, + 'commands': [ + 'mc rm --recursive --force cache/cache/%s/%s/coverage' % (ctx.repo.slug, ctx.build.commit + '-${DRONE_BUILD_NUMBER}'), + ] + }, ], 'trigger': { 'ref': [ From 57a1eec5f62c372f8db9459334c50e434ecf6e58 Mon Sep 17 00:00:00 2001 From: Michael Barz Date: Wed, 30 Sep 2020 19:28:14 +0200 Subject: [PATCH 3/4] bring back source_repo var in changelog pipeline --- .drone.star | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.star b/.drone.star index 19c19940b..e32bf5517 100644 --- a/.drone.star +++ b/.drone.star @@ -811,7 +811,7 @@ def manifest(ctx): } def changelog(ctx): - repo_slug = ctx.build.source if ctx.build.source else ctx.repo.slug + repo_slug = ctx.build.source_repo if ctx.build.source_repo else ctx.repo.slug return { 'kind': 'pipeline', 'type': 'docker', From 3f86d062c7b4a16859a85db0da9cb9de9e9e3839 Mon Sep 17 00:00:00 2001 From: Michael Barz Date: Thu, 1 Oct 2020 09:38:18 +0200 Subject: [PATCH 4/4] Create coverage folder first Co-authored-by: Phil Davis --- .drone.star | 1 + 1 file changed, 1 insertion(+) diff --git a/.drone.star b/.drone.star index e32bf5517..f022140dd 100644 --- a/.drone.star +++ b/.drone.star @@ -255,6 +255,7 @@ def uploadCoverage(ctx): } }, 'commands': [ + 'mkdir -p coverage', 'mc mirror cache/cache/%s/%s/coverage coverage/' % (ctx.repo.slug, ctx.build.commit + '-${DRONE_BUILD_NUMBER}'), ] },