[full-ci] run linter and unit test from top-level (#7542)

* Try to run linter and unit test from top-level directory

Instead of starting a separate pipeline for each service we now just run
the linter and unit tests from the top-level directory. This allows us
to run the linter it with '--modules-download-mode vendor' and should also
reduce the resource usage.

* cleanup: remove no longer valid submodule release step

---------

Co-authored-by: Ralf Haferkamp <rhaferkamp@owncloud.com>

.drone.star

@@ -52,46 +52,6 @@ dirs = {
 
 # configuration
 config = {
-    "modules": [
-        # if you add a module here please also add it to the root level Makefile
-        "services/antivirus",
-        "services/app-provider",
-        "services/app-registry",
-        "services/audit",
-        "services/auth-basic",
-        "services/auth-bearer",
-        "services/auth-machine",
-        "services/eventhistory",
-        "services/frontend",
-        "services/gateway",
-        "services/graph",
-        "services/groups",
-        "services/idm",
-        "services/idp",
-        "services/invitations",
-        "services/nats",
-        "services/notifications",
-        "services/ocdav",
-        "services/ocs",
-        "services/policies",
-        "services/proxy",
-        "services/search",
-        "services/settings",
-        "services/sharing",
-        "services/storage-system",
-        "services/storage-publiclink",
-        "services/storage-shares",
-        "services/storage-users",
-        "services/store",
-        "services/thumbnails",
-        "services/userlog",
-        "services/users",
-        "services/web",
-        "services/webdav",
-        "services/webfinger",
-        "ocis-pkg",
-        "ocis",
-    ],
     "cs3ApiTests": {
         "skip": False,
     },
@@ -263,14 +223,13 @@ def main(ctx):
         buildWebCache(ctx) + \
         getGoBinForTesting(ctx) + \
         [buildOcisBinaryForTesting(ctx)] + \
-        testOcisModules(ctx) + \
+        testOcisAndUploadResults(ctx) + \
         testPipelines(ctx)
 
     build_release_pipelines = \
         [licenseCheck(ctx)] + \
         dockerReleases(ctx) + \
-        binaryReleases(ctx) + \
+        binaryReleases(ctx)
-        [releaseSubmodule(ctx)]
 
     build_release_helpers = [
         changelog(),
@@ -334,15 +293,13 @@ def buildWebCache(ctx):
         cachePipeline("web-pnpm", generateWebPnpmCache(ctx)),
     ]
 
-def testOcisModules(ctx):
+def testOcisAndUploadResults(ctx):
-    pipelines = []
+    pipeline = testOcis(ctx)
-    for module in config["modules"]:
-        pipelines.append(testOcisModule(ctx, module))
 
     scan_result_upload = uploadScanResults(ctx)
-    scan_result_upload["depends_on"] = getPipelineNames(pipelines)
+    scan_result_upload["depends_on"] = getPipelineNames([pipeline])
 
-    return pipelines + [scan_result_upload]
+    return [pipeline, scan_result_upload]
 
 def testPipelines(ctx):
     pipelines = []
@@ -471,15 +428,15 @@ def restoreGoBinCache():
         },
     ]
 
-def testOcisModule(ctx, module):
+def testOcis(ctx):
-    steps = skipIfUnchanged(ctx, "unit-tests") + restoreGoBinCache() + makeGoGenerate(module) + [
+    steps = skipIfUnchanged(ctx, "unit-tests") + restoreGoBinCache() + makeGoGenerate("") + [
         {
             "name": "golangci-lint",
             "image": OC_CI_GOLANG,
             "commands": [
                 "mkdir -p cache/checkstyle",
-                "make -C %s ci-golangci-lint" % (module),
+                "make ci-golangci-lint",
-                "mv %s/checkstyle.xml cache/checkstyle/$(basename %s)_checkstyle.xml" % (module, module),
+                "mv checkstyle.xml cache/checkstyle/checkstyle.xml",
             ],
             "environment": {
                 "HTTP_PROXY": {
@@ -496,8 +453,8 @@ def testOcisModule(ctx, module):
             "image": OC_CI_GOLANG,
             "commands": [
                 "mkdir -p cache/coverage",
-                "make -C %s test" % (module),
+                "make test",
-                "mv %s/coverage.out cache/coverage/$(basename %s)_coverage.out" % (module, module),
+                "mv coverage.out cache/coverage/",
             ],
             "volumes": [stepVolumeGo],
         },
@@ -525,7 +482,7 @@ def testOcisModule(ctx, module):
     return {
         "kind": "pipeline",
         "type": "docker",
-        "name": "linting&unitTests-%s" % (module),
+        "name": "linting_and_unitTests",
         "platform": {
             "os": "linux",
             "arch": "amd64",
@@ -534,7 +491,6 @@ def testOcisModule(ctx, module):
         "trigger": {
             "ref": [
                 "refs/heads/stable-*",
-                "refs/tags/%s/v*" % (module),
                 "refs/pull/**",
             ],
         },
@@ -1286,7 +1242,7 @@ def dockerRelease(ctx, arch):
         "REVISION=%s" % (ctx.build.commit),
         "VERSION=%s" % (ctx.build.ref.replace("refs/tags/", "") if ctx.build.event == "tag" else "latest"),
     ]
-    depends_on = getPipelineNames(testOcisModules(ctx) + testPipelines(ctx))
+    depends_on = getPipelineNames(testOcisAndUploadResults(ctx) + testPipelines(ctx))
 
     if ctx.build.event == "tag":
         depends_on = []
@@ -1375,7 +1331,7 @@ def binaryReleases(ctx):
 def binaryRelease(ctx, name):
     # uploads binary to https://download.owncloud.com/ocis/ocis/daily/
     target = "/ocis/%s/daily" % (ctx.repo.name.replace("ocis-", ""))
-    depends_on = getPipelineNames(testOcisModules(ctx) + testPipelines(ctx))
+    depends_on = getPipelineNames(testOcisAndUploadResults(ctx) + testPipelines(ctx))
     if ctx.build.event == "tag":
         # uploads binary to eg. https://download.owncloud.com/ocis/ocis/1.0.0-beta9/
         folder = "stable"
@@ -1624,49 +1580,6 @@ def licenseCheck(ctx):
         "volumes": [pipelineVolumeGo],
     }
 
-def releaseSubmodule(ctx):
-    depends = []
-    if len(ctx.build.ref.replace("refs/tags/", "").split("/")) == 2:
-        depends = ["linting&unitTests-%s" % (ctx.build.ref.replace("refs/tags/", "").split("/")[0])]
-
-    return {
-        "kind": "pipeline",
-        "type": "docker",
-        "name": "release-%s" % (ctx.build.ref.replace("refs/tags/", "")),
-        "platform": {
-            "os": "linux",
-            "arch": "amd64",
-        },
-        "steps": [
-            {
-                "name": "release-submodule",
-                "image": PLUGINS_GITHUB_RELEASE,
-                "settings": {
-                    "api_key": {
-                        "from_secret": "github_token",
-                    },
-                    "files": [
-                    ],
-                    "title": ctx.build.ref.replace("refs/tags/", "").replace("/v", " "),
-                    "note": "Release %s submodule" % (ctx.build.ref.replace("refs/tags/", "").replace("/v", " ")),
-                    "overwrite": True,
-                    "prerelease": len(ctx.build.ref.split("-")) > 1,
-                },
-                "when": {
-                    "ref": [
-                        "refs/tags/*/v*",
-                    ],
-                },
-            },
-        ],
-        "depends_on": depends,
-        "trigger": {
-            "ref": [
-                "refs/tags/*/v*",
-            ],
-        },
-    }
-
 def releaseDockerManifest():
     return {
         "kind": "pipeline",

Makefile

@@ -188,9 +188,7 @@ go-mod-tidy:
 
 .PHONY: test
 test:
-	@for mod in $(OCIS_MODULES); do \
+	@go test -v -tags '$(TAGS)' -coverprofile coverage.out ./...
-        $(MAKE) --no-print-directory -C $$mod test || exit 1; \
-    done
 
 .PHONY: go-coverage
 go-coverage:
@@ -211,6 +209,10 @@ golangci-lint:
         $(MAKE) --no-print-directory -C $$mod golangci-lint; \
     done
 
+.PHONY: ci-golangci-lint
+ci-golangci-lint: $(GOLANGCI_LINT)
+	$(GOLANGCI_LINT) run --modules-download-mode vendor --timeout 15m0s --issues-exit-code 0 --out-format checkstyle > checkstyle.xml
+
 .PHONY: golangci-lint-fix
 golangci-lint-fix:
 	@for mod in $(OCIS_MODULES); do \

sonar-project.properties

@@ -30,10 +30,7 @@ sonar.pullrequest.key=${env.SONAR_PULL_REQUEST_KEY}
 
 # Properties specific to language plugins:
 sonar.go.coverage.reportPaths=cache/coverage/*
-# golangci-lint does not support wildcard
+sonar.go.golangci-lint.reportPaths=cache/checkstyle/checkstyle.xml
-# https://github.com/SonarSource/slang/blob/85b05160bc1b31c6072a35f8818da4376b107afa/sonar-go-plugin/src/main/java/org/sonar/go/externalreport/GolangCILintReportSensor.java#L37
-# https://github.com/SonarSource/slang/blob/85b05160bc1b31c6072a35f8818da4376b107afa/sonar-go-plugin/src/main/java/org/sonar/go/externalreport/AbstractReportSensor.java#L76-L90
-sonar.go.golangci-lint.reportPaths=cache/checkstyle/app-provider_checkstyle.xml,cache/checkstyle/app-registry_checkstyle.xml,cache/checkstyle/audit_checkstyle.xml,cache/checkstyle/auth-basic_checkstyle.xml,cache/checkstyle/auth-bearer_checkstyle.xml,cache/checkstyle/auth-machine_checkstyle.xml,cache/checkstyle/frontend_checkstyle.xml,cache/checkstyle/gateway_checkstyle.xml,cache/checkstyle/graph_checkstyle.xml,cache/checkstyle/groups_checkstyle.xml,cache/checkstyle/idm_checkstyle.xml,cache/checkstyle/idp_checkstyle.xml,cache/checkstyle/nats_checkstyle.xml,cache/checkstyle/notifications_checkstyle.xml,cache/checkstyle/ocdav_checkstyle.xml,cache/checkstyle/ocs_checkstyle.xml,cache/checkstyle/proxy_checkstyle.xml,cache/checkstyle/search_checkstyle.xml,cache/checkstyle/settings_checkstyle.xml,cache/checkstyle/sharing_checkstyle.xml,cache/checkstyle/storage-publiclink_checkstyle.xml,cache/checkstyle/storage-shares_checkstyle.xml,cache/checkstyle/storage-system_checkstyle.xml,cache/checkstyle/storage-users_checkstyle.xml,cache/checkstyle/store_checkstyle.xml,cache/checkstyle/thumbnails_checkstyle.xml,cache/checkstyle/users_checkstyle.xml,cache/checkstyle/web_checkstyle.xml,cache/checkstyle/webdav_checkstyle.xml
 
 # Exclude files
 sonar.exclusions=**/third_party,docs/**,changelog/**,**/package.json,**/rollup.config.js,CHANGELOG.md,deployments/**,tests/**,vendor/**,vendor-bin/**,README.md,**/mocks/**,/protogen/**