From 4b78e06e043b89336847ac50c2fdd8c62895f1e4 Mon Sep 17 00:00:00 2001 From: Swikriti Tripathi <41103328+SwikritiT@users.noreply.github.com> Date: Fri, 2 Jun 2023 17:11:17 +0545 Subject: [PATCH] Add acceptance tests for antivirus (#6405) test on ci test on ci run on detached mode add wait for clam fix syntax fix syntax try without wait add clamservice to steps use different wait for command put sleep test wait for clamav service Clean up tests and enable other tests in the ci address reviews add documentation add docs for docker address reviews refactor docs remove user premission as it's by default root capitalize the letters --- .drone.star | 58 +++++++++- docs/ocis/development/testing.md | 79 +++++++++++++- tests/TestHelpers/HttpRequestHelper.php | 1 - tests/acceptance/config/behat.yml | 18 ++++ .../features/apiAntivirus/antivirus.feature | 102 ++++++++++++++++++ .../delayPostprocessing.feature | 2 +- .../bootstrap/NotificationContext.php | 62 ++++++++--- .../filesForUpload/filesWithVirus/eicar.com | 1 + .../filesWithVirus/eicar_com.zip | Bin 0 -> 184 bytes .../filesWithVirus/eicarcom2.zip | Bin 0 -> 308 bytes 10 files changed, 304 insertions(+), 19 deletions(-) create mode 100644 tests/acceptance/features/apiAntivirus/antivirus.feature create mode 100644 tests/acceptance/filesForUpload/filesWithVirus/eicar.com create mode 100644 tests/acceptance/filesForUpload/filesWithVirus/eicar_com.zip create mode 100644 tests/acceptance/filesForUpload/filesWithVirus/eicarcom2.zip diff --git a/.drone.star b/.drone.star index b7ea613ff..c9b608b4a 100644 --- a/.drone.star +++ b/.drone.star @@ -9,6 +9,7 @@ INBUCKET_INBUCKET = "inbucket/inbucket" MINIO_MC = "minio/mc:RELEASE.2021-10-07T04-19-58Z" OC_CI_ALPINE = "owncloudci/alpine:latest" OC_CI_BAZEL_BUILDIFIER = "owncloudci/bazel-buildifier:latest" +OC_CI_CLAMAVD = "owncloudci/clamavd" OC_CI_DRONE_ANSIBLE = "owncloudci/drone-ansible:latest" OC_CI_DRONE_CANCEL_PREVIOUS_BUILDS = "owncloudci/drone-cancel-previous-builds" OC_CI_DRONE_SKIP_PIPELINE = "owncloudci/drone-skip-pipeline" @@ -134,6 +135,21 @@ config = { "NOTIFICATIONS_SMTP_INSECURE": "true", }, }, + "apiAntivirus": { + "suites": [ + "apiAntivirus", + ], + "skip": False, + "earlyFail": True, + "antivirusNeeded": True, + "extraServerEnvironment": { + "ANTIVIRUS_SCANNER_TYPE": "clamav", + "ANTIVIRUS_CLAMAV_SOCKET": "/var/run/clamav/clamd.sock", + "POSTPROCESSING_STEPS": "virusscan", + "OCIS_ASYNC_UPLOADS": True, + "OCIS_ADD_RUN_SERVICES": "antivirus", + }, + }, }, "apiTests": { "numberOfParts": 10, @@ -181,6 +197,22 @@ pipelineVolumeGo = \ "temp": {}, } +# volume for pipeline to share clamav socket between steps of a pipeline +# to be used in combination with stepVolumeClamav +pipelineVolumeClamav = \ + { + "name": "sockets", + "temp": {}, + } + +# volume for steps to share clamav socket between steps of a pipeline +# socket path must be set to /var/run/clamav/ inside the image, which is the case +stepVolumeClamav = \ + { + "name": "sockets", + "path": "/var/run/clamav/", + } + # minio mc environment variables MINIO_MC_ENV = { "CACHE_BUCKET": { @@ -761,6 +793,7 @@ def localApiTestPipeline(ctx): "storages": ["ocis"], "accounts_hash_difficulty": 4, "emailNeeded": False, + "antivirusNeeded": False, } if "localApiTests" in config: @@ -782,7 +815,9 @@ def localApiTestPipeline(ctx): }, "steps": skipIfUnchanged(ctx, "acceptance-tests") + restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin") + - ocisServer(storage, params["accounts_hash_difficulty"], extra_server_environment = params["extraServerEnvironment"], with_wrapper = True) + + (clamavService() if params["antivirusNeeded"] else []) + + (waitForClamavService() if params["antivirusNeeded"] else []) + + ocisServer(storage, params["accounts_hash_difficulty"], extra_server_environment = params["extraServerEnvironment"], with_wrapper = True, volumes = [stepVolumeClamav] if params["antivirusNeeded"] else []) + (waitForEmailService() if params["emailNeeded"] else []) + localApiTests(suite, storage, params["extraEnvironment"]) + failEarly(ctx, early_fail), @@ -794,6 +829,7 @@ def localApiTestPipeline(ctx): "refs/pull/**", ], }, + "volumes": [pipelineVolumeClamav] if params["antivirusNeeded"] else [], } pipelines.append(pipeline) return pipelines @@ -2919,3 +2955,23 @@ def waitForEmailService(): "wait-for -it email:9000 -t 600", ], }] + +def clamavService(): + return [{ + "name": "clamav", + "image": OC_CI_CLAMAVD, + "volumes": [{ + "name": "sockets", + "path": "/var/run/clamav/", + }], + "detach": True, + }] + +def waitForClamavService(): + return [{ + "name": "wait-for-clamav", + "image": OC_CI_WAIT_FOR, + "commands": [ + "wait-for -it clamav:3310 -t 600", + ], + }] diff --git a/docs/ocis/development/testing.md b/docs/ocis/development/testing.md index 3e9660095..1d3295915 100644 --- a/docs/ocis/development/testing.md +++ b/docs/ocis/development/testing.md @@ -313,7 +313,7 @@ While writing tests for a new oCIS ENV configuration, please make sure to follow 2. Use `OcisConfigHelper.php` for helper functions - provides functions to reconfigure the running oCIS instance. 3. Recommended: add the new step implementations in `OcisConfigContext.php` -## Running Test Suite With Email Service (@Email) +## Running Test Suite With Email Service (@email) Test suites that are tagged with `@email` require an email service. We use inbucket as the email service in our tests. @@ -406,3 +406,80 @@ make test-paralleldeployment-api \ ... \ BEHAT_FEATURE="tests/parallelDeployAcceptance/features/apiShareManagement/acceptShares.feature" ``` + +## Running Test Suite With Antivirus Service (@antivirus) +Test suites that are tagged with `@antivirus` require antivirus service. The available antivirus and the configuration related to them can be found [here](https://doc.owncloud.com/ocis/next/deployment/services/s-list/antivirus.html). This documentation is only going to use `clamAv` as antivirus. + +### Setup clamAV +#### 1. Setup Locally +Run the following command to set up calmAV and clamAV daemon +```bash +sudo apt install clamav clamav-daemon -y +``` + +Make sure that the clamAV daemon is up and running + +```bash +sudo service clamav-daemon status +``` +{{< hint info >}} +The commands are ubuntu specific and may differ according to your system. You can find information related to installation of clamAV in their official documentation [here](https://docs.clamav.net/manual/Installing/Packages.html). +{{< /hint>}} + +#### 2. Setup clamAV With Docker +##### a. Create a Volume +For `clamAV` only local sockets can currently be configured we need to create a volume in order to share the socket with `oCIS server`. Run the following command to do so: +```bash + docker volume create -d local -o device=/your/local/filesystem/path/ -o o=bind -o type=none clamav_vol +``` +##### b. Run the Container +Run `clamAV` through docker and bind the path to the socket of clamAV from the image to the pre-created volume +```bash +docker run -v clamav_vol:/var/run/clamav/ owncloudci/clamavd +``` +{{< hint info >}} +The path to the socket i.e. `/var/run/clamav/` may differ as per the image you are using. Make sure that you're providing the correct path to the socket if you're using image other than `owncloudci/clamavd`. +{{< /hint>}} + +##### b. Change Ownership +Change the ownership of the path of your local filesystem that the volume `clamav_vol` is mounted on. After running `clamav` through docker the ownership of the bound path gets changed. As we need to provide this path to ocis server the ownership should be changed back to $USER or whatever ownership that your server requires. +```bash + sudo chown -R $USER:$USER /your/local/filesystem/path/ +``` +{{< hint info >}} +Make sure that `clamAV` is fully up before running this command. The command is ubuntu specific and may differ according to your system. +{{< /hint>}} + +{{< hint info >}} +If you want to use the same volume after the container is down. Before running the container once again you need to either remove all the data inside `/your/local/filesystem/path/` or give the ownership back. For instance, it ubuntu it might be `sudo chown -R systemd-network:systemd-journal /your/local/filesystem/path/` and repeat step 2 and 3` +{{< /hint>}} + +### Run oCIS + +As `antivirus` service is not enabled by default we need to enable the service while running oCIS server. We also need to enable `async upload` and as virus scan is performed in post-processing step, we need to set it as well. Documentation for environment variables related to antivirus is available [here](https://owncloud.dev/services/antivirus/#environment-variables) + +```bash +# run oCIS +PROXY_ENABLE_BASIC_AUTH=true \ +ANTIVIRUS_SCANNER_TYPE="clamav" \ +ANTIVIRUS_CLAMAV_SOCKET="/var/run/clamav/clamd.ctl" \ +POSTPROCESSING_STEPS="virusscan" \ +OCIS_ASYNC_UPLOADS=true \ +OCIS_ADD_RUN_SERVICES="antivirus" +ocis/bin/ocis server +``` +{{< hint info >}} +The value for `ANTIVIRUS_CLAMAV_SOCKET` is an example which needs adaption according your OS. If you are running `clamAv` with docker as per this documentation check the path that you mounted the volume i.e. `/your/local/filesystem/path/` to make sure the socket exists and give the full path to socket i.e. `/your/local/filesystem/path/clamd.sock` to `ANTIVIRUS_CLAMAV_SOCKET`. +{{< /hint>}} + +#### Run the Acceptance Test + +Run the acceptance test with the following command: + +```bash +TEST_WITH_GRAPH_API=true \ +TEST_OCIS=true \ +TEST_SERVER_URL="https://localhost:9200" \ +BEHAT_FEATURE="tests/acceptance/features/apiAntivirus/antivirus.feature" \ +make test-acceptance-api +``` diff --git a/tests/TestHelpers/HttpRequestHelper.php b/tests/TestHelpers/HttpRequestHelper.php index 9d327e12c..6ea218786 100644 --- a/tests/TestHelpers/HttpRequestHelper.php +++ b/tests/TestHelpers/HttpRequestHelper.php @@ -200,7 +200,6 @@ class HttpRequestHelper { $sendRetryLimit = self::numRetriesOnHttpTooEarly(); $sendCount = 0; $sendExceptionHappened = false; - do { $response = self::sendRequestOnce( $url, diff --git a/tests/acceptance/config/behat.yml b/tests/acceptance/config/behat.yml index 74d2e8fda..3149eefad 100644 --- a/tests/acceptance/config/behat.yml +++ b/tests/acceptance/config/behat.yml @@ -187,6 +187,24 @@ default: - FilesVersionsContext: - SettingsContext: + apiAntivirus: + paths: + - '%paths.base%/../features/apiAntivirus' + context: *common_ldap_suite_context + contexts: + - NotificationContext: + - SpacesContext: + - FeatureContext: *common_feature_context_params + - WebDavPropertiesContext: + - OCSContext: + - GraphContext: + - TrashbinContext: + - FavoritesContext: + - ChecksumContext: + - FilesVersionsContext: + - SettingsContext: + - OcisConfigContext: + extensions: rdx\behatvars\BehatVariablesExtension: ~ diff --git a/tests/acceptance/features/apiAntivirus/antivirus.feature b/tests/acceptance/features/apiAntivirus/antivirus.feature new file mode 100644 index 000000000..0bb5e2b1a --- /dev/null +++ b/tests/acceptance/features/apiAntivirus/antivirus.feature @@ -0,0 +1,102 @@ +@api @antivirus +Feature: antivirus + As a system administrator and user + I want to protect myself and others from known viruses + So that I can prevent files with viruses from being uploaded + + + Background: + Given user "Alice" has been created with default attributes and without skeleton files + + + Scenario Outline: upload a normal file without virus + Given using DAV path + When user "Alice" uploads file "filesForUpload/textfile.txt" to "/normalfile.txt" using the WebDAV API + Then the HTTP status code should be "201" + And as "Alice" file "/normalfile.txt" should exist + And the content of file "/normalfile.txt" for user "Alice" should be: + """ + This is a testfile. + + Cheers. + """ + Examples: + | dav-path-version | + | old | + | new | + + @skipOnRevaMaster + Examples: + | dav-path-version | + | spaces | + + Scenario Outline: upload a file with virus + Given using DAV path + When user "Alice" uploads file "filesForUpload/filesWithVirus/eicar.com" to "/aFileWithVirus.txt" using the WebDAV API + # antivirus service can scan files during post-processing. on demand scanning is currently not available + Then the HTTP status code should be "201" + And user "Alice" should get a notification with subject "Virus found" and message: + | message | + | Virus found in aFileWithVirus.txt. Upload not possible. Virus: Win.Test.EICAR_HDB-1 | + And as "Alice" file "/aFileWithVirus.txt" should not exist + Examples: + | dav-path-version | + | old | + | new | + + @skipOnRevaMaster + Examples: + | dav-path-version | + | spaces | + + + Scenario Outline: upload a file with virus and a file without virus + Given using DAV path + When user "Alice" uploads file "filesForUpload/filesWithVirus/eicar.com" to "/aFileWithVirus.txt" using the WebDAV API + # antivirus service can scan files during post-processing. on demand scanning is currently not available + Then the HTTP status code should be "201" + And user "Alice" uploads file "filesForUpload/textfile.txt" to "/normalfile.txt" using the WebDAV API + And the HTTP status code should be "201" + And user "Alice" should get a notification with subject "Virus found" and message: + | message | + | Virus found in aFileWithVirus.txt. Upload not possible. Virus: Win.Test.EICAR_HDB-1 | + And as "Alice" file "/aFileWithVirus.txt" should not exist + But as "Alice" file "/normalfile.txt" should exist + And the content of file "/normalfile.txt" for user "Alice" should be: + """ + This is a testfile. + + Cheers. + """ + Examples: + | dav-path-version | + | old | + | new | + + @skipOnRevaMaster + Examples: + | dav-path-version | + | spaces | + + + Scenario Outline: upload a file with virus in chunks + Given using DAV path + When user "Alice" uploads the following chunks to "/myChunkedFile.txt" with old chunking and using the WebDAV API + | number | content | + | 1 | X5O!P%@AP[4\PZX54(P^)7C | + | 2 | C)7}$EICAR-STANDARD-ANT | + | 3 | IVIRUS-TEST-FILE!$H+H* | + # antivirus service can scan files during post-processing. on demand scanning is currently not available + Then the HTTP status code should be "201" + And user "Alice" should get a notification with subject "Virus found" and message: + | message | + | Virus found in myChunkedFile.txt. Upload not possible. Virus: Win.Test.EICAR_HDB-1 | + And as "Alice" file "/myChunkedFile.txt" should not exist + Examples: + | dav-path-version | + | old | + + @skipOnRevaMaster + Examples: + | dav-path-version | + | spaces | diff --git a/tests/acceptance/features/apiAsyncUpload/delayPostprocessing.feature b/tests/acceptance/features/apiAsyncUpload/delayPostprocessing.feature index b09a2590c..fb4942336 100644 --- a/tests/acceptance/features/apiAsyncUpload/delayPostprocessing.feature +++ b/tests/acceptance/features/apiAsyncUpload/delayPostprocessing.feature @@ -43,4 +43,4 @@ Feature: delay post-processing of uploaded files | dav_path | | /remote.php/webdav/my_data | | /remote.php/dav/files/%username%/my_data | - | /dav/spaces/%spaceid%/my_data | \ No newline at end of file + | /dav/spaces/%spaceid%/my_data | diff --git a/tests/acceptance/features/bootstrap/NotificationContext.php b/tests/acceptance/features/bootstrap/NotificationContext.php index 4fdaf1a76..251ca52d3 100644 --- a/tests/acceptance/features/bootstrap/NotificationContext.php +++ b/tests/acceptance/features/bootstrap/NotificationContext.php @@ -13,6 +13,7 @@ use Behat\Gherkin\Node\PyStringNode; use TestHelpers\EmailHelper; use PHPUnit\Framework\Assert; use TestHelpers\GraphHelper; +use Behat\Gherkin\Node\TableNode; require_once 'bootstrap.php'; @@ -80,7 +81,7 @@ class NotificationContext implements Context { } /** - * @Then /^user "([^"]*)" lists all notifications$/ + * @When /^user "([^"]*)" lists all notifications$/ * * @param string $user * @@ -127,20 +128,7 @@ class NotificationContext implements Context { string $subject, PyStringNode $schemaString ): void { - if (isset($this->featureContext->getJsonDecodedResponseBodyContent()->ocs->data)) { - $responseBody = $this->featureContext->getJsonDecodedResponseBodyContent()->ocs->data; - foreach ($responseBody as $value) { - if (isset($value->subject) && $value->subject === $subject) { - $responseBody = $value; - // set notificationId - $this->notificationIds[] = $value->notification_id; - break; - } - } - } else { - $responseBody = $this->featureContext->getJsonDecodedResponseBodyContent(); - } - + $responseBody = $this->filterResponseAccordingToNotificationSubject($subject); // substitute the value here $schemaString = $schemaString->getRaw(); $schemaString = $this->featureContext->substituteInLineCodes( @@ -157,6 +145,50 @@ class NotificationContext implements Context { ); } + /** + * @param string $subject + * + * @return object + */ + public function filterResponseAccordingToNotificationSubject(string $subject): object { + $responseBody = null; + if (isset($this->featureContext->getJsonDecodedResponseBodyContent()->ocs->data)) { + $responseBody = $this->featureContext->getJsonDecodedResponseBodyContent()->ocs->data; + foreach ($responseBody as $value) { + if (isset($value->subject) && $value->subject === $subject) { + $responseBody = $value; + // set notificationId + $this->notificationIds[] = $value->notification_id; + break; + } + } + } else { + $responseBody = $this->featureContext->getJsonDecodedResponseBodyContent(); + } + return $responseBody; + } + + /** + * @Then user :user should get a notification with subject :subject and message: + * + * @param string $user + * @param string $subject + * @param TableNode $table + * + * @return void + */ + public function userShouldGetANotificationWithMessage(string $user, string $subject, TableNode $table):void { + $this->userListAllNotifications($user); + $this->featureContext->theHTTPStatusCodeShouldBe(200); + $actualMessage = $this->filterResponseAccordingToNotificationSubject($subject)->message; + $expectedMessage = $table->getColumnsHash()[0]['message']; + Assert::assertSame( + $expectedMessage, + $actualMessage, + __METHOD__ . "expected message to be '$expectedMessage' but found'$actualMessage'" + ); + } + /** * @Then user :user should have received the following email from user :sender about the share of project space :spaceName * diff --git a/tests/acceptance/filesForUpload/filesWithVirus/eicar.com b/tests/acceptance/filesForUpload/filesWithVirus/eicar.com new file mode 100644 index 000000000..a2463df6d --- /dev/null +++ b/tests/acceptance/filesForUpload/filesWithVirus/eicar.com @@ -0,0 +1 @@ +X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H* \ No newline at end of file diff --git a/tests/acceptance/filesForUpload/filesWithVirus/eicar_com.zip b/tests/acceptance/filesForUpload/filesWithVirus/eicar_com.zip new file mode 100644 index 0000000000000000000000000000000000000000..e7e8cefb257c8d4905538feabae7fe9309d792d1 GIT binary patch literal 184 zcmWIWW@h1H0D%WHc4*j~56o}@vO$;=h*L9@6N~hc^K&Ci{S^aL9UKFqO=1F~B1}y* z0^&5yot-t!YgJr5ogIU8gF_tsTpWX3bRGRdJi|PLLW6ZfT!TY&-8_9<6;(X6J+uP6 m8JR>Fa9aknkAaavf#H86hy}Mgz?+o~BE`sH4rFA5I1B)Pq$ZvK literal 0 HcmV?d00001 diff --git a/tests/acceptance/filesForUpload/filesWithVirus/eicarcom2.zip b/tests/acceptance/filesForUpload/filesWithVirus/eicarcom2.zip new file mode 100644 index 0000000000000000000000000000000000000000..4bda6119ad1e57f1087d87804a0dfab63e1e4f46 GIT binary patch literal 308 zcmWIWW@h1H00E;luQe_z+&0_+WP>m-5T|A)ClxQ@nhv>R_`nW2pcxZcQ1$Z+ui7-Iih0QXceGH5Y3Jm`nK`gK?7#YlgjBF5x0ogEYo&+lb8nz8#7{bd41FeDT9)r~a0HhZ{KL7v# literal 0 HcmV?d00001