collaborative posix test (#672)

* collaborative posix test

* fix after review

.woodpecker.star

@@ -132,6 +132,7 @@ config = {
             "suites": [
                 "apiGraph",
                 "apiServiceAvailability",
+                "collaborativePosix",
             ],
             "skip": False,
             "withRemotePhp": [True],
@@ -939,12 +940,13 @@ def localApiTests(name, suites, storage = "decomposed", extra_environment = {},
         "SEND_SCENARIO_LINE_REFERENCES": True,
         "STORAGE_DRIVER": storage,
         "BEHAT_SUITES": ",".join(suites),
-        "BEHAT_FILTER_TAGS": "~@skip&&~@skipOnGraph&&~@skipOnOpencloud-%s-Storage" % storage,
+        "BEHAT_FILTER_TAGS": "~@skip&&~@skipOnOpencloud-%s-Storage" % storage,
         "EXPECTED_FAILURES_FILE": expected_failures_file,
         "UPLOAD_DELETE_WAIT_TIME": "1" if storage == "owncloud" else 0,
         "OC_WRAPPER_URL": "http://%s:5200" % OC_SERVER_NAME,
         "WITH_REMOTE_PHP": with_remote_php,
         "COLLABORATION_SERVICE_URL": "http://wopi-fakeoffice:9300",
+        "OC_STORAGE_PATH": "$HOME/.opencloud/storage/users/users",
     }
 
     for item in extra_environment:
@@ -1107,7 +1109,7 @@ def coreApiTests(ctx, part_number = 1, number_of_parts = 1, with_remote_php = Fa
     storage = "posix"
     if "[decomposed]" in ctx.build.title.lower():
         storage = "decomposed"
-    filterTags = "~@skipOnGraph&&~@skipOnOpencloud-%s-Storage" % storage
+    filterTags = "~@skipOnOpencloud-%s-Storage" % storage
     test_dir = "%s/tests/acceptance" % dirs["base"]
     expected_failures_file = "%s/expected-failures-API-on-%s-storage.md" % (test_dir, storage)
 
@@ -2001,6 +2003,8 @@ def opencloudServer(storage = "decomposed", accounts_hash_difficulty = 4, depend
             },
         },
         "commands": [
+            "apt-get update",
+            "apt-get install -y inotify-tools",
             "%s init --insecure true" % dirs["opencloudBin"],
             "cat $OC_CONFIG_DIR/opencloud.yaml",
             "cp tests/config/woodpecker/app-registry.yaml $OC_CONFIG_DIR/app-registry.yaml",

tests/acceptance/TestHelpers/CliHelper.php

@@ -20,9 +20,7 @@
 
 namespace TestHelpers;
 
-use GuzzleHttp\Exception\ConnectException;
 use GuzzleHttp\Exception\GuzzleException;
-use GuzzleHttp\Psr7\Request;
 use Psr\Http\Message\ResponseInterface;
 use TestHelpers\OcConfigHelper;
 

tests/acceptance/bootstrap/CliContext.php

@@ -34,6 +34,15 @@ class CliContext implements Context {
 	private FeatureContext $featureContext;
 	private SpacesContext $spacesContext;
 
+	/**
+	 * opencloud user storage path
+	 *
+	 * @return string
+	 */
+	public static function getStoragePath(): string {
+		return getenv('OC_STORAGE_PATH') ?: '/var/lib/opencloud/storage/users/users';
+	}
+
 	/**
 	 * @BeforeScenario
 	 *
@@ -85,8 +94,8 @@ class CliContext implements Context {
 	): void {
 		$command = "idm resetpassword -u $user";
 		$body = [
-			"command" => $command,
+		  "command" => $command,
-			"inputs" => [$password, $password]
+		  "inputs" => [$password, $password]
 		];
 
 		$this->featureContext->setResponse(CliHelper::runCommand($body));
@@ -105,7 +114,7 @@ class CliContext implements Context {
 		$path = $this->featureContext->getStorageUsersRoot();
 		$command = "trash purge-empty-dirs -p $path --dry-run=false";
 		$body = [
-			"command" => $command
+		  "command" => $command
 		];
 		$this->featureContext->setResponse(CliHelper::runCommand($body));
 	}
@@ -119,7 +128,7 @@ class CliContext implements Context {
 		$path = $this->featureContext->getStorageUsersRoot();
 		$command = "backup consistency -p $path";
 		$body = [
-			"command" => $command
+		  "command" => $command
 		];
 		$this->featureContext->setResponse(CliHelper::runCommand($body));
 	}
@@ -139,7 +148,7 @@ class CliContext implements Context {
 		$user = $this->featureContext->getActualUserName($user);
 		$command = "auth-app create --user-name=$user --expiration=$expirationTime";
 		$body = [
-			"command" => $command
+		  "command" => $command
 		];
 		$this->featureContext->setResponse(CliHelper::runCommand($body));
 	}
@@ -159,7 +168,7 @@ class CliContext implements Context {
 		$user = $this->featureContext->getActualUserName($user);
 		$command = "auth-app create --user-name=$user --expiration=$expirationTime";
 		$body = [
-			"command" => $command
+		  "command" => $command
 		];
 
 		$response = CliHelper::runCommand($body);
@@ -182,7 +191,7 @@ class CliContext implements Context {
 		$path = $this->featureContext->getStorageUsersRoot();
 		$command = "revisions purge -p $path --dry-run=false";
 		$body = [
-			"command" => $command
+		  "command" => $command
 		];
 		$this->featureContext->setResponse(CliHelper::runCommand($body));
 	}
@@ -201,7 +210,7 @@ class CliContext implements Context {
 		$fileId = $this->spacesContext->getFileId($user, $space, $file);
 		$command = "revisions purge -p $path -r $fileId --dry-run=false";
 		$body = [
-			"command" => $command
+		  "command" => $command
 		];
 		$this->featureContext->setResponse(CliHelper::runCommand($body));
 	}
@@ -214,7 +223,7 @@ class CliContext implements Context {
 	public function theAdministratorReindexesAllSpacesUsingTheCli(): void {
 		$command = "search index --all-spaces";
 		$body = [
-			"command" => $command
+		  "command" => $command
 		];
 		$this->featureContext->setResponse(CliHelper::runCommand($body));
 	}
@@ -230,7 +239,7 @@ class CliContext implements Context {
 		$spaceId = $this->spacesContext->getSpaceIdByName($this->featureContext->getAdminUsername(), $spaceName);
 		$command = "search index --space $spaceId";
 		$body = [
-			"command" => $command
+		  "command" => $command
 		];
 		$this->featureContext->setResponse(CliHelper::runCommand($body));
 	}
@@ -248,7 +257,7 @@ class CliContext implements Context {
 		$spaceId = $this->spacesContext->getSpaceIdByName($adminUsername, $space);
 		$command = "revisions purge -p $path -r $spaceId --dry-run=false";
 		$body = [
-			"command" => $command
+		  "command" => $command
 		];
 		$this->featureContext->setResponse(CliHelper::runCommand($body));
 	}
@@ -306,7 +315,7 @@ class CliContext implements Context {
 		}
 		$command = "storage-users uploads sessions --json $flag";
 		$body = [
-			"command" => $command
+		  "command" => $command
 		];
 		$this->featureContext->setResponse(CliHelper::runCommand($body));
 	}
@@ -326,7 +335,7 @@ class CliContext implements Context {
 		$flagString = trim($flag);
 		$command = "storage-users uploads sessions $flagString --clean --json";
 		$body = [
-			"command" => $command
+		  "command" => $command
 		];
 		$this->featureContext->setResponse(CliHelper::runCommand($body));
 	}
@@ -339,7 +348,7 @@ class CliContext implements Context {
 	public function theAdministratorRestartsTheUploadSessionsThatAreInPostprocessing(): void {
 		$command = "storage-users uploads sessions --processing --restart --json";
 		$body = [
-			"command" => $command
+		  "command" => $command
 		];
 		$this->featureContext->setResponse(CliHelper::runCommand($body));
 	}
@@ -365,7 +374,7 @@ class CliContext implements Context {
 
 		$command = "storage-users uploads sessions --id=$uploadId --restart --json";
 		$body = [
-			"command" => $command
+		  "command" => $command
 		];
 		$this->featureContext->setResponse(CliHelper::runCommand($body));
 	}
@@ -431,9 +440,45 @@ class CliContext implements Context {
 	public function cleanUploadsSessions(): void {
 		$command = "storage-users uploads sessions --clean";
 		$body = [
-			"command" => $command
+		  "command" => $command
 		];
 		$response = CliHelper::runCommand($body);
 		Assert::assertEquals("200", $response->getStatusCode(), "Failed to clean upload sessions");
 	}
+
+	/**
+	 * @When the administrator creates folder :folder for user :user on the POSIX filesystem
+	 *
+	 * @param string $folder
+	 * @param string $user
+	 *
+	 * @return void
+	 */
+	public function theAdministratorCreatesFolder(string $folder, string $user): void {
+		$userUuid = $this->featureContext->getUserIdByUserName($user);
+		$storagePath = $this->getStoragePath();
+		$body = [
+		  "command" => "mkdir -p $storagePath/$userUuid/$folder",
+		  "raw" => true
+		];
+		$this->featureContext->setResponse(CliHelper::runCommand($body));
+		sleep(1);
+	}
+
+	/**
+	 * @When the administrator lists the content of the POSIX storage folder of user :user
+	 *
+	 * @param string $user
+	 *
+	 * @return void
+	 */
+	public function theAdministratorCheckUsersFolder(string $user): void {
+		$userUuid = $this->featureContext->getUserIdByUserName($user);
+		$storagePath = $this->getStoragePath();
+		$body = [
+		  "command" => "ls -la $storagePath/$userUuid",
+		  "raw" => true
+		];
+		$this->featureContext->setResponse(CliHelper::runCommand($body));
+	}
 }

tests/acceptance/config/behat.yml

@@ -458,6 +458,15 @@ default:
         - TrashbinContext:
         - SpacesTUSContext:
 
+    collaborativePosix:
+      paths:
+        - "%paths.base%/../features/collaborativePosix"
+      context: *common_ldap_suite_context
+      contexts:
+        - FeatureContext: *common_feature_context_params
+        - CliContext:
+        - OcConfigContext:
+
     coreApiMain:
       paths:
         - "%paths.base%/../features/coreApiMain"

tests/acceptance/docker/Makefile

@@ -48,8 +48,8 @@ else
 	SEARCH_EXTRACTOR_TYPE := basic
 endif
 
-# default to decomposedfs
+# default to posix
-STORAGE_DRIVER ?= decomposed
+STORAGE_DRIVER ?= posix
 ifeq ($(STORAGE_DRIVER),posix)
 	# posix requires a additional driver config
 	COMPOSE_FILE := $(COMPOSE_FILE):src/posix.yml

tests/acceptance/docker/src/opencloud-base.yml

@@ -10,6 +10,7 @@ services:
       WITH_WRAPPER: $WITH_WRAPPER
       OC_URL: "https://opencloud-server:9200"
       STORAGE_USERS_DRIVER: $STORAGE_DRIVER
+      STORAGE_USERS_POSIX_WATCH_FS: "true"
       STORAGE_USERS_DRIVER_LOCAL_ROOT: /srv/app/tmp/opencloud/local/root
       STORAGE_USERS_DRIVER_OC_ROOT: /srv/app/tmp/opencloud/storage/users
       STORAGE_SYSTEM_DRIVER_OC_ROOT: /srv/app/tmp/opencloud/storage/metadata

tests/acceptance/docker/src/opencloud.Dockerfile

@@ -6,6 +6,7 @@ FROM opencloudeu/opencloud:${OC_IMAGE_TAG} AS opencloud
 
 FROM ubuntu:22.04
 COPY --from=opencloud /usr/bin/opencloud /usr/bin/opencloud
+RUN apt-get update && apt-get install -y inotify-tools
 
 COPY ["./serve-opencloud.sh", "/usr/bin/serve-opencloud"]
 RUN chmod +x /usr/bin/serve-opencloud

tests/acceptance/docker/src/posix.yml

@@ -6,3 +6,4 @@ services:
       STORAGE_USERS_DRIVER: posix
       # posix requires a shared cache store
       STORAGE_USERS_ID_CACHE_STORE: "nats-js-kv"
+      STORAGE_USERS_POSIX_WATCH_FS: "true"

tests/acceptance/features/collaborativePosix/collaborativePosixFS.feature

@@ -0,0 +1,15 @@
+@env-config @skipOnOpencloud-decomposed-Storage @skipOnOpencloud-decomposeds3-Storage
+Feature: create a resources using collaborative posixfs
+
+  Background:
+    Given the config "STORAGE_USERS_POSIX_WATCH_FS" has been set to "true"
+    And user "Alice" has been created with default attributes
+
+
+  Scenario: create folder
+    Given user "Alice" has uploaded file with content "content" to "textfile.txt"
+    When the administrator creates folder "myFolder" for user "Alice" on the POSIX filesystem
+    Then the command should be successful
+    When the administrator lists the content of the POSIX storage folder of user "Alice"
+    Then the command output should contain "myFolder"
+    And as "Alice" folder "/myFolder" should exist

tests/ocwrapper/opencloud/opencloud.go

@@ -271,3 +271,36 @@ func RunCommand(command string, inputs []string) (int, string) {
 
 	return c.ProcessState.ExitCode(), cmdOutput
 }
+
+func RunRawCommand(command string, inputs []string) (int, string) {
+	logs := new(strings.Builder)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	fmt.Print("Running command: ", command)
+
+	c := exec.CommandContext(ctx, "bash", "-c", command)
+
+	ptyF, err := pty.Start(c)
+	if err != nil {
+		log.Panic(err)
+	}
+	defer ptyF.Close()
+
+	for _, input := range inputs {
+		fmt.Fprintf(ptyF, "%s\n", input)
+	}
+
+	var cmdOutput string
+	if err := c.Wait(); err != nil {
+		if ctx.Err() == context.DeadlineExceeded {
+			cmdOutput = "Command timed out:\n"
+		}
+	}
+
+	io.Copy(logs, ptyF)
+	cmdOutput += logs.String()
+	cmdOutput = strings.TrimLeft(cmdOutput, strings.Join(inputs, "\r\n"))
+
+	return c.ProcessState.ExitCode(), cmdOutput
+}

tests/ocwrapper/wrapper/handlers/handler.go

@@ -198,7 +198,19 @@ func CommandHandler(res http.ResponseWriter, req *http.Request) {
 			}
 		}
 	}
+	raw := false
+	if r, ok := body["raw"].(bool); ok {
+		raw = r
+	}
 
-	exitCode, output := opencloud.RunCommand(command, stdIn)
+	var exitCode int
+	var output string
+
+
+	if raw {
+		exitCode, output = opencloud.RunRawCommand(command, stdIn)
+	} else {
+		exitCode, output = opencloud.RunCommand(command, stdIn)
+	}
 	sendCmdResponse(res, exitCode, output)
 }