From 4b3516ec927918ae4820cf2c7b04c718bfcc8f48 Mon Sep 17 00:00:00 2001 From: Ralf Haferkamp Date: Thu, 19 Oct 2023 13:54:04 +0200 Subject: [PATCH] Try to run linter and unit test from top-level directory Instead of starting a separate pipeline for each service we now just run the linter and unit tests from the top-level directory. This allows us to run the linter it with '--modules-download-mode vendor' and should also reduce the resource usage. --- .drone.star | 74 ++++++++-------------------------------- Makefile | 8 +++-- sonar-project.properties | 5 +-- 3 files changed, 20 insertions(+), 67 deletions(-) diff --git a/.drone.star b/.drone.star index 2c9b0ca58..508b7aea7 100644 --- a/.drone.star +++ b/.drone.star @@ -53,49 +53,6 @@ dirs = { # configuration config = { - "modules": [ - # if you add a module here please also add it to the root level Makefile - "services/antivirus", - "services/app-provider", - "services/app-registry", - "services/audit", - "services/auth-basic", - "services/auth-bearer", - "services/auth-machine", - "services/auth-service", - "services/clientlog", - "services/eventhistory", - "services/frontend", - "services/gateway", - "services/graph", - "services/groups", - "services/idm", - "services/idp", - "services/invitations", - "services/nats", - "services/notifications", - "services/ocdav", - "services/ocs", - "services/policies", - "services/proxy", - "services/search", - "services/settings", - "services/sharing", - "services/sse", - "services/storage-system", - "services/storage-publiclink", - "services/storage-shares", - "services/storage-users", - "services/store", - "services/thumbnails", - "services/userlog", - "services/users", - "services/web", - "services/webdav", - "services/webfinger", - "ocis-pkg", - "ocis", - ], "cs3ApiTests": { "skip": False, }, @@ -268,7 +225,7 @@ def main(ctx): buildWebCache(ctx) + \ getGoBinForTesting(ctx) + \ [buildOcisBinaryForTesting(ctx)] + \ - testOcisModules(ctx) + \ + testOcisAndUploadResults(ctx) + \ testPipelines(ctx) build_release_pipelines = \ @@ -339,15 +296,13 @@ def buildWebCache(ctx): cachePipeline("web-pnpm", generateWebPnpmCache(ctx)), ] -def testOcisModules(ctx): - pipelines = [] - for module in config["modules"]: - pipelines.append(testOcisModule(ctx, module)) +def testOcisAndUploadResults(ctx): + pipeline = testOcis(ctx) scan_result_upload = uploadScanResults(ctx) - scan_result_upload["depends_on"] = getPipelineNames(pipelines) + scan_result_upload["depends_on"] = getPipelineNames([pipeline]) - return pipelines + [scan_result_upload] + return [pipeline, scan_result_upload] def testPipelines(ctx): pipelines = [] @@ -477,15 +432,15 @@ def restoreGoBinCache(): }, ] -def testOcisModule(ctx, module): - steps = skipIfUnchanged(ctx, "unit-tests") + restoreGoBinCache() + makeGoGenerate(module) + [ +def testOcis(ctx): + steps = skipIfUnchanged(ctx, "unit-tests") + restoreGoBinCache() + makeGoGenerate("") + [ { "name": "golangci-lint", "image": OC_CI_GOLANG, "commands": [ "mkdir -p cache/checkstyle", - "make -C %s ci-golangci-lint" % (module), - "mv %s/checkstyle.xml cache/checkstyle/$(basename %s)_checkstyle.xml" % (module, module), + "make ci-golangci-lint", + "mv checkstyle.xml cache/checkstyle/checkstyle.xml", ], "environment": { "HTTP_PROXY": { @@ -502,8 +457,8 @@ def testOcisModule(ctx, module): "image": OC_CI_GOLANG, "commands": [ "mkdir -p cache/coverage", - "make -C %s test" % (module), - "mv %s/coverage.out cache/coverage/$(basename %s)_coverage.out" % (module, module), + "make test", + "mv coverage.out cache/coverage/", ], "volumes": [stepVolumeGo], }, @@ -531,7 +486,7 @@ def testOcisModule(ctx, module): return { "kind": "pipeline", "type": "docker", - "name": "linting&unitTests-%s" % (module), + "name": "linting_and_unitTests", "platform": { "os": "linux", "arch": "amd64", @@ -540,7 +495,6 @@ def testOcisModule(ctx, module): "trigger": { "ref": [ "refs/heads/master", - "refs/tags/%s/v*" % (module), "refs/pull/**", ], }, @@ -1291,7 +1245,7 @@ def dockerRelease(ctx, arch): "REVISION=%s" % (ctx.build.commit), "VERSION=%s" % (ctx.build.ref.replace("refs/tags/", "") if ctx.build.event == "tag" else "latest"), ] - depends_on = getPipelineNames(testOcisModules(ctx) + testPipelines(ctx)) + depends_on = getPipelineNames(testOcisAndUploadResults(ctx) + testPipelines(ctx)) if ctx.build.event == "tag": depends_on = [] @@ -1380,7 +1334,7 @@ def binaryReleases(ctx): def binaryRelease(ctx, name): # uploads binary to https://download.owncloud.com/ocis/ocis/daily/ target = "/ocis/%s/daily" % (ctx.repo.name.replace("ocis-", "")) - depends_on = getPipelineNames(testOcisModules(ctx) + testPipelines(ctx)) + depends_on = getPipelineNames(testOcisAndUploadResults(ctx) + testPipelines(ctx)) if ctx.build.event == "tag": # uploads binary to eg. https://download.owncloud.com/ocis/ocis/1.0.0-beta9/ folder = "stable" diff --git a/Makefile b/Makefile index b83a0c323..4eb2a4c91 100644 --- a/Makefile +++ b/Makefile @@ -193,9 +193,7 @@ go-mod-tidy: .PHONY: test test: - @for mod in $(OCIS_MODULES); do \ - $(MAKE) --no-print-directory -C $$mod test || exit 1; \ - done + @go test -v -tags '$(TAGS)' -coverprofile coverage.out ./... .PHONY: go-coverage go-coverage: @@ -216,6 +214,10 @@ golangci-lint: $(MAKE) --no-print-directory -C $$mod golangci-lint; \ done +.PHONY: ci-golangci-lint +ci-golangci-lint: $(GOLANGCI_LINT) + $(GOLANGCI_LINT) run --modules-download-mode vendor --timeout 15m0s --issues-exit-code 0 --out-format checkstyle > checkstyle.xml + .PHONY: golangci-lint-fix golangci-lint-fix: @for mod in $(OCIS_MODULES); do \ diff --git a/sonar-project.properties b/sonar-project.properties index 469c4cf04..74398c2cc 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -30,10 +30,7 @@ sonar.pullrequest.key=${env.SONAR_PULL_REQUEST_KEY} # Properties specific to language plugins: sonar.go.coverage.reportPaths=cache/coverage/* -# golangci-lint does not support wildcard -# https://github.com/SonarSource/slang/blob/85b05160bc1b31c6072a35f8818da4376b107afa/sonar-go-plugin/src/main/java/org/sonar/go/externalreport/GolangCILintReportSensor.java#L37 -# https://github.com/SonarSource/slang/blob/85b05160bc1b31c6072a35f8818da4376b107afa/sonar-go-plugin/src/main/java/org/sonar/go/externalreport/AbstractReportSensor.java#L76-L90 -sonar.go.golangci-lint.reportPaths=cache/checkstyle/app-provider_checkstyle.xml,cache/checkstyle/app-registry_checkstyle.xml,cache/checkstyle/audit_checkstyle.xml,cache/checkstyle/auth-basic_checkstyle.xml,cache/checkstyle/auth-bearer_checkstyle.xml,cache/checkstyle/auth-machine_checkstyle.xml,cache/checkstyle/frontend_checkstyle.xml,cache/checkstyle/gateway_checkstyle.xml,cache/checkstyle/graph_checkstyle.xml,cache/checkstyle/groups_checkstyle.xml,cache/checkstyle/idm_checkstyle.xml,cache/checkstyle/idp_checkstyle.xml,cache/checkstyle/nats_checkstyle.xml,cache/checkstyle/notifications_checkstyle.xml,cache/checkstyle/ocdav_checkstyle.xml,cache/checkstyle/ocs_checkstyle.xml,cache/checkstyle/proxy_checkstyle.xml,cache/checkstyle/search_checkstyle.xml,cache/checkstyle/settings_checkstyle.xml,cache/checkstyle/sharing_checkstyle.xml,cache/checkstyle/storage-publiclink_checkstyle.xml,cache/checkstyle/storage-shares_checkstyle.xml,cache/checkstyle/storage-system_checkstyle.xml,cache/checkstyle/storage-users_checkstyle.xml,cache/checkstyle/store_checkstyle.xml,cache/checkstyle/thumbnails_checkstyle.xml,cache/checkstyle/users_checkstyle.xml,cache/checkstyle/web_checkstyle.xml,cache/checkstyle/webdav_checkstyle.xml +sonar.go.golangci-lint.reportPaths=cache/checkstyle/checkstyle.xml # Exclude files sonar.exclusions=**/third_party,docs/**,changelog/**,**/package.json,**/rollup.config.js,CHANGELOG.md,deployments/**,tests/**,vendor/**,vendor-bin/**,README.md,**/mocks/**,/protogen/**,services/search/pkg/query/kql/dictionary_gen.go