[docs-only]Update testing docs (#5444)

* Update testing docs

* make ocis run with new config

* fix wording

* fix sording

* fix command and run.sh

* Edit makefile to make it dynamically run suites

* adress reviews

* address reviews

* address reviews

tests/acceptance/docker/Makefile

@@ -29,6 +29,7 @@ OCIS_IMAGE_TAG ?= dev
 # static
 DIVIDE_INTO_NUM_PARTS := 10
 PARTS = 1 2 3 4 5 6 7 8 9 10
+LOCAL_API_SUITES = $(shell ls ../features | grep ^api*)
 
 COMPOSE_PROJECT_NAME := ocis-acceptance-tests
 
@@ -42,8 +43,8 @@ help:
 	@echo -e "oCIS feature tests and test suites can be found here:"
 	@echo -e "\thttps://github.com/owncloud/ocis/tree/master/tests/acceptance/features"
 	@echo
-	@echo -e "ownCloud feature tests and test suites can be found here:"
-	@echo -e "\thttps://github.com/owncloud/core/tree/master/tests/acceptance/features"
+	@echo -e "test suites that test ownCloud 10 API core compatibility are found here and they start with prefix coreApi-:"
+	@echo -e "\thttps://github.com/owncloud/ocis/tree/master/tests/acceptance/features"
 	@echo
 	@echo -e "The oCIS to be tested will be build from your current working state."
 	@echo -e "You also can select the oCIS Docker image for all tests by setting"
@@ -52,13 +53,10 @@ help:
 	@echo -e "https://hub.docker.com/r/owncloud/ocis."
 	@echo
 	@echo -e "${GREEN}Run full oCIS test suites against oCIS with oCIS storage:${RESET}\n"
-	@echo -e "\tmake localApiTests-apiAccountsHashDifficulty-ocis\t\t${BLUE}run apiAccountsHashDifficulty test suite${RESET}"
+	@echo -e "\tmake localApiTests-apiAccountsHashDifficulty-ocis\t\t${BLUE}run apiAccountsHashDifficulty test suite,  where available test suite are apiAccountsHashDifficulty apiArchiver apiContract apiGraph apiSpaces apiSpacesShares apiAsyncUpload apiCors${RESET}"
 	@echo
 	@echo -e "${GREEN}Run full oCIS test suites against oCIS with s3ng storage:${RESET}\n"
-	@echo -e "\tmake localApiTests-apiAccountsHashDifficulty-s3ng\t\t${BLUE}run apiAccountsHashDifficulty test suite${RESET}"
-	@echo
-	@echo -e "${GREEN}Run full oCIS test suites against oCIS with ownCloud storage:${RESET}\n"
-	@echo -e "\tmake localApiTests-apiAccountsHashDifficulty-owncloud\t\t${BLUE}run apiAccountsHashDifficulty test suite${RESET}"
+	@echo -e "\tmake localApiTests-apiAccountsHashDifficulty-s3ng\t\t${BLUE}run apiAccountsHashDifficulty test suite, where available test suite are apiAccountsHashDifficulty apiArchiver apiContract apiGraph apiSpaces apiSpacesShares apiAsyncUpload apiCors${RESET}"
 	@echo
 	@echo -e "${GREEN}Run full ownCloud test suites against oCIS with oCIS storage:${RESET}\n"
 	@echo -e "\tmake Core-API-Tests-ocis-storage-${RED}X${RESET}\t\t${BLUE}run test suite number X, where ${RED}X = 1 .. 10${RESET}"
@@ -82,19 +80,19 @@ help:
 	@echo -e "\texample: ${RED}tests/acceptance/features/apiAccountsHashDifficulty/addUser.feature${RESET}"
 	@echo
 	@echo -e "${GREEN}Run an ownCloud feature test against oCIS with oCIS storage:${RESET}\n"
-	@echo -e "\tmake test-oc10-feature-ocis-storage ${YELLOW}BEHAT_FEATURE='...'${RESET}\t${BLUE}run single feature test${RESET}"
+	@echo -e "\tmake test-core-feature-ocis-storage ${YELLOW}BEHAT_FEATURE='...'${RESET}\t${BLUE}run single feature test${RESET}"
 	@echo
 	@echo -e "\twhere ${YELLOW}BEHAT_FEATURE='...'${RESET} contains a relative path to the feature definition."
-	@echo -e "\texample: ${RED}tests/acceptance/features/apiAuth/cors.feature${RESET}"
+	@echo -e "\texample: ${RED}tests/acceptance/features/coreApiAuth/webDavAuth.feature${RESET}"
 	@echo
 	@echo -e "${GREEN}Run an ownCloud feature test against oCIS with s3ng storage:${RESET}\n"
-	@echo -e "\tmake test-oc10-feature-s3ng-storage ${YELLOW}BEHAT_FEATURE='...'${RESET}\t${BLUE}run single feature test${RESET}"
+	@echo -e "\tmake test-core-feature-s3ng-storage ${YELLOW}BEHAT_FEATURE='...'${RESET}\t${BLUE}run single feature test${RESET}"
 	@echo
 	@echo -e "\twhere ${YELLOW}BEHAT_FEATURE='...'${RESET} contains a relative path to the feature definition."
-	@echo -e "\texample: ${RED}tests/acceptance/features/apiAuth/cors.feature${RESET}"
+	@echo -e "\texample: ${RED}tests/acceptance/features/coreApiAuth/webDavAuth.feature${RESET}"
 	@echo
 	@echo -e "\twhere ${YELLOW}BEHAT_FEATURE='...'${RESET} contains a relative path to the feature definition."
-	@echo -e "\texample: ${RED}tests/acceptance/features/apiAuth/cors.feature${RESET}"
+	@echo -e "\texample: ${RED}tests/acceptance/features/coreApiAuth/webDavAuth.feature${RESET}"
 	@echo
 	@echo
 	@echo -e "${GREEN}Show output of tests:${RESET}\n"
@@ -121,44 +119,57 @@ test-ocis-feature-s3ng-storage: ## test a ocis feature with s3ng storage, usage:
 	START_CEPH=1 \
 	$(MAKE) --no-print-directory testSuite
 
-.PHONY: test-oc10-feature-ocis-storage
-test-oc10-feature-ocis-storage: ## test a oC10 feature with oCIS storage, usage: make ... BEHAT_FEATURE='tests/acceptance/features/apiAuth/cors.feature'
+.PHONY: test-core-feature-ocis-storage
+test-core-feature-ocis-storage: ## test a core feature with oCIS storage, usage: make ... BEHAT_FEATURE='tests/acceptance/features/coreApiAuth/webDavAuth.feature'
 	@TEST_SOURCE=oc10 \
 	STORAGE_DRIVER=ocis \
 	BEHAT_FEATURE=$(BEHAT_FEATURE) \
 	$(MAKE) --no-print-directory testSuite
 
-.PHONY: test-oc10-feature-s3ng-storage
-test-oc10-feature-s3ng-storage: ## test a oC10 feature with s3ng storage, usage: make ... BEHAT_FEATURE='tests/acceptance/features/apiAuth/cors.feature'
+.PHONY: test-core-feature-s3ng-storage
+test-core-feature-s3ng-storage: ## test a core feature with s3ng storage, usage: make ... BEHAT_FEATURE='tests/acceptance/features/coreApiAuth/webDavAuth.feature'
 	@TEST_SOURCE=oc10 \
 	STORAGE_DRIVER=s3ng \
 	BEHAT_FEATURE=$(BEHAT_FEATURE) \
 	START_CEPH=1 \
 	$(MAKE) --no-print-directory testSuite
 
-.PHONY: localApiTests-apiAccountsHashDifficulty-owncloud
-localApiTests-apiAccountsHashDifficulty-owncloud: ## run apiAccountsHashDifficulty test suite with owncloud storage
-	@TEST_SOURCE=ocis \
-	STORAGE_DRIVER=owncloud \
-	BEHAT_SUITE=apiAccountsHashDifficulty \
-	$(MAKE) --no-print-directory testSuite
-
-.PHONY: localApiTests-apiAccountsHashDifficulty-ocis
-localApiTests-apiAccountsHashDifficulty-ocis: ## run apiAccountsHashDifficulty test suite with ocis storage
+localSuiteOcis = $(addprefix localApiTests-, $(addsuffix -ocis,${LOCAL_API_SUITES}))
+.PHONY: $(localSuiteOcis)
+$(localSuiteOcis): ## run local api test suite with ocis storage
+	@$(eval BEHAT_SUITE=$(shell echo "$@" | cut -d'-' -f2))
 	@TEST_SOURCE=ocis \
 	STORAGE_DRIVER=ocis \
-	BEHAT_SUITE=apiAccountsHashDifficulty \
+	BEHAT_SUITE=$(BEHAT_SUITE) \
 	$(MAKE) --no-print-directory testSuite
 
-targets = $(addprefix Core-API-Tests-ocis-storage-,$(PARTS))
-.PHONY: $(targets)
-$(targets):
+localSuites3ng = $(addprefix localApiTests-, $(addsuffix -s3ng,${LOCAL_API_SUITES}))
+.PHONY: $(localSuites3ng)
+$(localSuites3ng): ## run local api test suite with s3 storage
+	@$(eval BEHAT_SUITE=$(shell echo "$@" | cut -d'-' -f2))
+	@TEST_SOURCE=ocis \
+	STORAGE_DRIVER=s3ng \
+	BEHAT_SUITE=$(BEHAT_SUITE) \
+	$(MAKE) --no-print-directory testSuite
+
+targetsOCIS = $(addprefix Core-API-Tests-ocis-storage-,$(PARTS))
+.PHONY: $(targetsOCIS)
+$(targetsOCIS):
 	@$(eval RUN_PART=$(shell echo "$@" | tr -dc '0-9'))
 	@TEST_SOURCE=oc10 \
 	STORAGE_DRIVER=ocis \
 	RUN_PART=$(RUN_PART) \
 	$(MAKE) --no-print-directory testSuite
 
+targetsS3ng = $(addprefix Core-API-Tests-s3ng-storage-,$(PARTS))
+.PHONY: $(targetsS3ng)
+$(targets):
+	@$(eval RUN_PART=$(shell echo "$@" | tr -dc '0-9'))
+	@TEST_SOURCE=oc10 \
+	STORAGE_DRIVER=s3ng \
+	RUN_PART=$(RUN_PART) \
+	$(MAKE) --no-print-directory testSuite
+
 .PHONY: testSuite
 testSuite: build-dev-image clean-docker-container
 	@if [ -n "${START_CEPH}" ]; then \
@@ -220,7 +231,7 @@ clean-docker-volumes: ## clean docker volumes created during acceptance tests
 
 .PHONY: clean-files
 clean-files:
-	@$(MAKE) --no-print-directory -C ../. clean-tests
+	@$(MAKE) --no-print-directory -C ../../../. clean-tests
 
 .PHONY: clean
 clean: clean-docker-container clean-docker-volumes clean-dev-docker-image clean-files ## clean all

tests/acceptance/docker/src/acceptance.yml

@@ -4,12 +4,10 @@ services:
     command: /bin/bash /test/run-tests.sh
     environment:
       OCIS_ROOT: /drone/src
-      PATH_TO_CORE: /srv/app/testrunner
-      SKELETON_DIR: /srv/app/tmp/testing/data/apiSkeleton
       TEST_OCIS: "true"
       RUN_ON_OCIS: "true"
       TEST_SERVER_URL: https://ocis-server:9200
-      TESTING_DIR: /srv/app/tmp/testing
+      TEST_WITH_GRAPH_API: "true"
 
       STORAGE_DRIVER: $STORAGE_DRIVER
       TEST_SOURCE: $TEST_SOURCE
@@ -21,11 +19,5 @@ services:
       - ../../../../.drone.env
     volumes:
       - ./run-tests.sh:/test/run-tests.sh
-      - oCISownCloud10testsuite:/srv
       - ../../../../:/drone/src
       - ../../../../vendor-bin/behat/composer.json:/tmp/vendor-bin/behat/composer.json
-      - oCISownCloud10testsuite-vendor:/drone/src/vendor
-
-volumes:
-  oCISownCloud10testsuite:
-  oCISownCloud10testsuite-vendor:

tests/acceptance/docker/src/ocis-base.yml

@@ -8,12 +8,20 @@ services:
       STORAGE_USERS_DRIVER_LOCAL_ROOT: /srv/app/tmp/ocis/local/root
       STORAGE_USERS_DRIVER_OCIS_ROOT: /srv/app/tmp/ocis/storage/users
       STORAGE_SYSTEM_DRIVER_OCIS_ROOT: /srv/app/tmp/ocis/storage/metadata
-      STORAGE_SHARING_USER_JSON_FILE: /srv/app/tmp/ocis/shares.json
+      SHARING_USER_JSON_FILE: /srv/app/tmp/ocis/shares.json
       SETTINGS_DATA_PATH: "/srv/app/tmp/ocis/settings"
       PROXY_ENABLE_BASIC_AUTH: "true"
       WEB_UI_CONFIG: /drone/src/tests/config/drone/ocis-config.json
       ACCOUNTS_HASH_DIFFICULTY: 4
       OCIS_INSECURE: "true"
+      "IDM_CREATE_DEMO_USERS": "true"
+      "IDM_ADMIN_PASSWORD": "admin"
+      "FRONTEND_SEARCH_MIN_LENGTH": "2"
+
+      "OCIS_CORS_ALLOW_ORIGINS": "https://aphno.badal"
+
+      "POSTPROCESSING_DELAY": ${POSTPROCESSING_DELAY:-0s}
+
       # s3ng specific settings
       STORAGE_USERS_S3NG_ENDPOINT: http://ceph:8080
       STORAGE_USERS_S3NG_REGION: default
@@ -22,7 +30,8 @@ services:
       STORAGE_USERS_S3NG_BUCKET: test
     volumes:
       - ../../../config:/drone/src/tests/config
-      - oCISownCloud10testsuite:/srv
-
-volumes:
-  oCISownCloud10testsuite:
+    entrypoint: /bin/sh
+    # run ocis init to initialize a configuration file with random secrets
+    # it will fail on subsequent runs, because the config file already exists
+    # therefore we ignore the error and then start the ocis server
+    command: ["-c", "ocis init || true; ocis server"]

tests/acceptance/docker/src/run-tests.sh

@@ -5,42 +5,21 @@
 
 git config --global advice.detachedHead false
 
-## GET DEPENDENCIES
-
-if cd $TESTING_DIR > /dev/null 2>&1
-then
-    git pull
-else
-    git clone -b master --depth=1 https://github.com/owncloud/testing.git $TESTING_DIR
-fi
-
-if cd $PATH_TO_CORE > /dev/null 2>&1
-then
-    git checkout $CORE_BRANCH
-    git pull
-    git checkout $CORE_COMMITID
-else
-    git clone -b $CORE_BRANCH --single-branch --no-tags https://github.com/owncloud/core.git $PATH_TO_CORE
-    cd $PATH_TO_CORE
-    git checkout $CORE_COMMITID
-fi
-
 ## CONFIGURE TEST
 
 if [ "$TEST_SOURCE" = "oc10" ]
 then
-    if [ "$STORAGE_DRIVER" = "owncloud" ]
-    then
-        export OCIS_REVA_DATA_ROOT='/srv/app/tmp/ocis/owncloud/data/'
-        export BEHAT_FILTER_TAGS='~@notToImplementOnOCIS&&~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@local_storage&&~@skipOnOcis-OC-Storage'
-        export OCIS_SKELETON_STRATEGY='copy'
-        export EXPECTED_FAILURES_FILE='/drone/src/tests/acceptance/expected-failures-API-on-OWNCLOUD-storage.md'
-    elif [ "$STORAGE_DRIVER" = "ocis" ]
+    if [ "$STORAGE_DRIVER" = "ocis" ]
     then
         export OCIS_REVA_DATA_ROOT=''
         export BEHAT_FILTER_TAGS='~@notToImplementOnOCIS&&~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage'
         export OCIS_SKELETON_STRATEGY='upload'
         export EXPECTED_FAILURES_FILE='/drone/src/tests/acceptance/expected-failures-API-on-OCIS-storage.md'
+    elif [ "$STORAGE_DRIVER" = "s3ng" ]
+    then
+        export BEHAT_FILTER_TAGS='~@skip&&~@skipOnOcis-S3NG-Storage'
+        export OCIS_REVA_DATA_ROOT=''
+        export OCIS_SKELETON_STRATEGY='upload'
     else
         echo "non existing STORAGE selected"
         exit 1
@@ -51,12 +30,7 @@ then
 elif [ "$TEST_SOURCE" = "ocis" ]
 then
 
-    if [ "$STORAGE_DRIVER" = "owncloud" ]
-    then
-        export BEHAT_FILTER_TAGS='~@skip&&~@skipOnOcis-OC-Storage'
-        export OCIS_REVA_DATA_ROOT='/srv/app/tmp/ocis/owncloud/data/'
-        export OCIS_SKELETON_STRATEGY='copy'
-    elif [ "$STORAGE_DRIVER" = "ocis" ]
+    if [ "$STORAGE_DRIVER" = "ocis" ]
     then
         export BEHAT_FILTER_TAGS='~@skip&&~@skipOnOcis-OCIS-Storage'
         export OCIS_REVA_DATA_ROOT=''
@@ -95,12 +69,12 @@ fi
 
 if [ "$TEST_SOURCE" = "oc10" ]
 then
-    make -C /srv/app/testrunner test-acceptance-api
+   sleep 10
+   make -C $OCIS_ROOT test-acceptance-from-core-api
 elif [ "$TEST_SOURCE" = "ocis" ]
 then
-    cd $OCIS_ROOT
     sleep 10
-    make test-acceptance-api
+    make -C $OCIS_ROOT test-acceptance-api
 else
     echo "non existing TEST_SOURCE selected"
     exit 1