Merge pull request #435 from aduffeck/fix-ci

Always run CLI tests with the decomposed storage driver
This commit is contained in:
Andre Duffeck
2025-03-20 15:01:24 +01:00
committed by GitHub
38 changed files with 18102 additions and 60 deletions
+1 -1
View File
@@ -1,3 +1,3 @@
# The test runner source for UI tests # The test runner source for UI tests
WEB_COMMITID=4a2f3a1d14009676a3a9dfef536ed4fd3e7f4c21 WEB_COMMITID=a85b8b2f0b22d2e8fa133fee2dae8cc866c0c8c2
WEB_BRANCH=main WEB_BRANCH=main
+2 -1
View File
@@ -296,6 +296,7 @@ config = {
"ANTIVIRUS_CLAMAV_SOCKET": "tcp://clamav:3310", "ANTIVIRUS_CLAMAV_SOCKET": "tcp://clamav:3310",
"OC_ASYNC_UPLOADS": True, "OC_ASYNC_UPLOADS": True,
"OC_ADD_RUN_SERVICES": "antivirus", "OC_ADD_RUN_SERVICES": "antivirus",
"STORAGE_USERS_DRIVER": "decomposed",
}, },
}, },
}, },
@@ -1143,7 +1144,7 @@ def coreApiTests(ctx, part_number = 1, number_of_parts = 1, with_remote_php = Fa
storage = "decomposed" storage = "decomposed"
filterTags = "~@skipOnGraph&&~@skipOnOpencloud-%s-Storage" % storage filterTags = "~@skipOnGraph&&~@skipOnOpencloud-%s-Storage" % storage
test_dir = "%s/tests/acceptance" % dirs["base"] test_dir = "%s/tests/acceptance" % dirs["base"]
expected_failures_file = "%s/expected-failures-API-on-decomposed-storage.md" % (test_dir) expected_failures_file = "%s/expected-failures-API-on-%s-storage.md" % (test_dir, storage)
return { return {
"name": "Core-API-Tests-%s%s-%s" % (part_number, "-withoutRemotePhp" if not with_remote_php else "", storage), "name": "Core-API-Tests-%s%s-%s" % (part_number, "-withoutRemotePhp" if not with_remote_php else "", storage),
+4 -3
View File
@@ -60,10 +60,10 @@ require (
github.com/oklog/run v1.1.0 github.com/oklog/run v1.1.0
github.com/olekukonko/tablewriter v0.0.5 github.com/olekukonko/tablewriter v0.0.5
github.com/onsi/ginkgo v1.16.5 github.com/onsi/ginkgo v1.16.5
github.com/onsi/ginkgo/v2 v2.23.0 github.com/onsi/ginkgo/v2 v2.23.1
github.com/onsi/gomega v1.36.2 github.com/onsi/gomega v1.36.2
github.com/open-policy-agent/opa v1.2.0 github.com/open-policy-agent/opa v1.2.0
github.com/opencloud-eu/reva/v2 v2.28.1-0.20250318145617-dd5b9b6fb606 github.com/opencloud-eu/reva/v2 v2.28.1-0.20250320105919-be91238e6b11
github.com/orcaman/concurrent-map v1.0.0 github.com/orcaman/concurrent-map v1.0.0
github.com/owncloud/libre-graph-api-go v1.0.5-0.20240829135935-80dc00d6f5ea github.com/owncloud/libre-graph-api-go v1.0.5-0.20240829135935-80dc00d6f5ea
github.com/pkg/errors v0.9.1 github.com/pkg/errors v0.9.1
@@ -253,7 +253,7 @@ require (
github.com/minio/crc64nvme v1.0.1 // indirect github.com/minio/crc64nvme v1.0.1 // indirect
github.com/minio/highwayhash v1.0.3 // indirect github.com/minio/highwayhash v1.0.3 // indirect
github.com/minio/md5-simd v1.1.2 // indirect github.com/minio/md5-simd v1.1.2 // indirect
github.com/minio/minio-go/v7 v7.0.87 // indirect github.com/minio/minio-go/v7 v7.0.88 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@@ -287,6 +287,7 @@ require (
github.com/segmentio/ksuid v1.0.4 // indirect github.com/segmentio/ksuid v1.0.4 // indirect
github.com/sercand/kuberesolver/v5 v5.1.1 // indirect github.com/sercand/kuberesolver/v5 v5.1.1 // indirect
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
github.com/sethvargo/go-diceware v0.5.0 // indirect
github.com/sethvargo/go-password v0.3.1 // indirect github.com/sethvargo/go-password v0.3.1 // indirect
github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c // indirect github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c // indirect
github.com/shurcooL/vfsgen v0.0.0-20230704071429-0000e147ea92 // indirect github.com/shurcooL/vfsgen v0.0.0-20230704071429-0000e147ea92 // indirect
+8 -6
View File
@@ -785,8 +785,8 @@ github.com/minio/highwayhash v1.0.3 h1:kbnuUMoHYyVl7szWjSxJnxw11k2U709jqFPPmIUyD
github.com/minio/highwayhash v1.0.3/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= github.com/minio/highwayhash v1.0.3/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.87 h1:nkr9x0u53PespfxfUqxP3UYWiE2a41gaofgNnC4Y8WQ= github.com/minio/minio-go/v7 v7.0.88 h1:v8MoIJjwYxOkehp+eiLIuvXk87P2raUtoU5klrAAshs=
github.com/minio/minio-go/v7 v7.0.87/go.mod h1:33+O8h0tO7pCeCWwBVa07RhVVfB/3vS4kEX7rwYKmIg= github.com/minio/minio-go/v7 v7.0.88/go.mod h1:33+O8h0tO7pCeCWwBVa07RhVVfB/3vS4kEX7rwYKmIg=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
@@ -852,8 +852,8 @@ github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
github.com/onsi/ginkgo/v2 v2.23.0 h1:FA1xjp8ieYDzlgS5ABTpdUDB7wtngggONc8a7ku2NqQ= github.com/onsi/ginkgo/v2 v2.23.1 h1:Ox0cOPv/t8RzKJUfDo9ZKtRvBOJY369sFJnl00CjqwY=
github.com/onsi/ginkgo/v2 v2.23.0/go.mod h1:zXTP6xIp3U8aVuXN8ENK9IXRaTjFnpVB9mGmaSRvxnM= github.com/onsi/ginkgo/v2 v2.23.1/go.mod h1:zXTP6xIp3U8aVuXN8ENK9IXRaTjFnpVB9mGmaSRvxnM=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
@@ -861,8 +861,8 @@ github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8=
github.com/onsi/gomega v1.36.2/go.mod h1:DdwyADRjrc825LhMEkD76cHR5+pUnjhUN8GlHlRPHzY= github.com/onsi/gomega v1.36.2/go.mod h1:DdwyADRjrc825LhMEkD76cHR5+pUnjhUN8GlHlRPHzY=
github.com/open-policy-agent/opa v1.2.0 h1:88NDVCM0of1eO6Z4AFeL3utTEtMuwloFmWWU7dRV1z0= github.com/open-policy-agent/opa v1.2.0 h1:88NDVCM0of1eO6Z4AFeL3utTEtMuwloFmWWU7dRV1z0=
github.com/open-policy-agent/opa v1.2.0/go.mod h1:30euUmOvuBoebRCcJ7DMF42bRBOPznvt0ACUMYDUGVY= github.com/open-policy-agent/opa v1.2.0/go.mod h1:30euUmOvuBoebRCcJ7DMF42bRBOPznvt0ACUMYDUGVY=
github.com/opencloud-eu/reva/v2 v2.28.1-0.20250318145617-dd5b9b6fb606 h1:ASUV6F7hHgar1RrnPfTQhtd+/KMeTCn7LhLzda0+HKY= github.com/opencloud-eu/reva/v2 v2.28.1-0.20250320105919-be91238e6b11 h1:MjfgrhEs73BezOXQZUgEtNTZsmXDVixFpGzZljR5lrk=
github.com/opencloud-eu/reva/v2 v2.28.1-0.20250318145617-dd5b9b6fb606/go.mod h1:XWp81Uok1opSID0HeITjvxJqdorltHVx+iJv4IlWzPo= github.com/opencloud-eu/reva/v2 v2.28.1-0.20250320105919-be91238e6b11/go.mod h1:iK0tNdLgqK0zBi0l7Q4uWSn9GPUbYtNxz3YAMfYvYNg=
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
@@ -1000,6 +1000,8 @@ github.com/sercand/kuberesolver/v5 v5.1.1 h1:CYH+d67G0sGBj7q5wLK61yzqJJ8gLLC8aep
github.com/sercand/kuberesolver/v5 v5.1.1/go.mod h1:Fs1KbKhVRnB2aDWN12NjKCB+RgYMWZJ294T3BtmVCpQ= github.com/sercand/kuberesolver/v5 v5.1.1/go.mod h1:Fs1KbKhVRnB2aDWN12NjKCB+RgYMWZJ294T3BtmVCpQ=
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8=
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
github.com/sethvargo/go-diceware v0.5.0 h1:exrQ7GpaBo00GqRVM1N8ChXSsi3oS7tjQiIehsD+yR0=
github.com/sethvargo/go-diceware v0.5.0/go.mod h1:Lg1SyPS7yQO6BBgTN5r4f2MUDkqGfLWsOjHPY0kA8iw=
github.com/sethvargo/go-password v0.3.1 h1:WqrLTjo7X6AcVYfC6R7GtSyuUQR9hGyAj/f1PYQZCJU= github.com/sethvargo/go-password v0.3.1 h1:WqrLTjo7X6AcVYfC6R7GtSyuUQR9hGyAj/f1PYQZCJU=
github.com/sethvargo/go-password v0.3.1/go.mod h1:rXofC1zT54N7R8K/h1WDUdkf9BOx5OptoxrMBcrXzvs= github.com/sethvargo/go-password v0.3.1/go.mod h1:rXofC1zT54N7R8K/h1WDUdkf9BOx5OptoxrMBcrXzvs=
github.com/shamaton/msgpack/v2 v2.2.3 h1:uDOHmxQySlvlUYfQwdjxyybAOzjlQsD1Vjy+4jmO9NM= github.com/shamaton/msgpack/v2 v2.2.3 h1:uDOHmxQySlvlUYfQwdjxyybAOzjlQsD1Vjy+4jmO9NM=
@@ -0,0 +1,172 @@
## Scenarios from core API tests that are expected to fail with decomposed storage while running with the Graph API
### File
Basic file management like up and download, move, copy, properties, trash, versions and chunking.
#### [Custom dav properties with namespaces are rendered incorrectly](https://github.com/owncloud/ocis/issues/2140)
_ocdav: double-check the webdav property parsing when custom namespaces are used_
- [coreApiWebdavProperties/setFileProperties.feature:128](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavProperties/setFileProperties.feature#L128)
- [coreApiWebdavProperties/setFileProperties.feature:129](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavProperties/setFileProperties.feature#L129)
- [coreApiWebdavProperties/setFileProperties.feature:130](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavProperties/setFileProperties.feature#L130)
### Sync
Synchronization features like etag propagation, setting mtime and locking files
#### [Uploading an old method chunked file with checksum should fail using new DAV path](https://github.com/owncloud/ocis/issues/2323)
- [coreApiMain/checksums.feature:233](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiMain/checksums.feature#L233)
- [coreApiMain/checksums.feature:234](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiMain/checksums.feature#L234)
- [coreApiMain/checksums.feature:235](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiMain/checksums.feature#L235)
### Share
#### [d:quota-available-bytes in dprop of PROPFIND give wrong response value](https://github.com/owncloud/ocis/issues/8197)
- [coreApiWebdavProperties/getQuota.feature:57](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavProperties/getQuota.feature#L57)
- [coreApiWebdavProperties/getQuota.feature:58](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavProperties/getQuota.feature#L58)
- [coreApiWebdavProperties/getQuota.feature:59](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavProperties/getQuota.feature#L59)
- [coreApiWebdavProperties/getQuota.feature:73](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavProperties/getQuota.feature#L73)
- [coreApiWebdavProperties/getQuota.feature:74](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavProperties/getQuota.feature#L74)
- [coreApiWebdavProperties/getQuota.feature:75](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavProperties/getQuota.feature#L75)
#### [deleting a file inside a received shared folder is moved to the trash-bin of the sharer not the receiver](https://github.com/owncloud/ocis/issues/1124)
- [coreApiTrashbin/trashbinSharingToShares.feature:54](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L54)
- [coreApiTrashbin/trashbinSharingToShares.feature:55](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L55)
- [coreApiTrashbin/trashbinSharingToShares.feature:56](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L56)
- [coreApiTrashbin/trashbinSharingToShares.feature:83](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L83)
- [coreApiTrashbin/trashbinSharingToShares.feature:84](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L84)
- [coreApiTrashbin/trashbinSharingToShares.feature:85](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L85)
- [coreApiTrashbin/trashbinSharingToShares.feature:142](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L142)
- [coreApiTrashbin/trashbinSharingToShares.feature:143](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L143)
- [coreApiTrashbin/trashbinSharingToShares.feature:144](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L144)
- [coreApiTrashbin/trashbinSharingToShares.feature:202](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L202)
- [coreApiTrashbin/trashbinSharingToShares.feature:203](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L203)
### Other
API, search, favorites, config, capabilities, not existing endpoints, CORS and others
#### [sending MKCOL requests to another or non-existing user's webDav endpoints as normal user should return 404](https://github.com/owncloud/ocis/issues/5049)
_ocdav: api compatibility, return correct status code_
- [coreApiAuth/webDavMKCOLAuth.feature:42](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiAuth/webDavMKCOLAuth.feature#L42)
- [coreApiAuth/webDavMKCOLAuth.feature:53](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiAuth/webDavMKCOLAuth.feature#L53)
#### [trying to lock file of another user gives http 500](https://github.com/owncloud/ocis/issues/2176)
- [coreApiAuth/webDavLOCKAuth.feature:46](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiAuth/webDavLOCKAuth.feature#L46)
- [coreApiAuth/webDavLOCKAuth.feature:58](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiAuth/webDavLOCKAuth.feature#L58)
#### [Support for favorites](https://github.com/owncloud/ocis/issues/1228)
- [coreApiFavorites/favorites.feature:101](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiFavorites/favorites.feature#L101)
- [coreApiFavorites/favorites.feature:102](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiFavorites/favorites.feature#L102)
- [coreApiFavorites/favorites.feature:103](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiFavorites/favorites.feature#L103)
- [coreApiFavorites/favorites.feature:124](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiFavorites/favorites.feature#L124)
- [coreApiFavorites/favorites.feature:125](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiFavorites/favorites.feature#L125)
- [coreApiFavorites/favorites.feature:126](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiFavorites/favorites.feature#L126)
- [coreApiFavorites/favorites.feature:189](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiFavorites/favorites.feature#L189)
- [coreApiFavorites/favorites.feature:190](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiFavorites/favorites.feature#L190)
- [coreApiFavorites/favorites.feature:191](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiFavorites/favorites.feature#L191)
- [coreApiFavorites/favorites.feature:145](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiFavorites/favorites.feature#L145)
- [coreApiFavorites/favorites.feature:146](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiFavorites/favorites.feature#L146)
- [coreApiFavorites/favorites.feature:147](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiFavorites/favorites.feature#L147)
- [coreApiFavorites/favorites.feature:174](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiFavorites/favorites.feature#L174)
- [coreApiFavorites/favorites.feature:175](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiFavorites/favorites.feature#L175)
- [coreApiFavorites/favorites.feature:176](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiFavorites/favorites.feature#L176)
- [coreApiFavorites/favoritesSharingToShares.feature:91](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiFavorites/favoritesSharingToShares.feature#L91)
- [coreApiFavorites/favoritesSharingToShares.feature:92](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiFavorites/favoritesSharingToShares.feature#L92)
- [coreApiFavorites/favoritesSharingToShares.feature:93](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiFavorites/favoritesSharingToShares.feature#L93)
#### [WWW-Authenticate header for unauthenticated requests is not clear](https://github.com/owncloud/ocis/issues/2285)
- [coreApiWebdavOperations/refuseAccess.feature:21](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavOperations/refuseAccess.feature#L21)
- [coreApiWebdavOperations/refuseAccess.feature:22](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavOperations/refuseAccess.feature#L22)
#### [PATCH request for TUS upload with wrong checksum gives incorrect response](https://github.com/owncloud/ocis/issues/1755)
- [coreApiWebdavUploadTUS/checksums.feature:74](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L74)
- [coreApiWebdavUploadTUS/checksums.feature:75](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L75)
- [coreApiWebdavUploadTUS/checksums.feature:76](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L76)
- [coreApiWebdavUploadTUS/checksums.feature:77](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L77)
- [coreApiWebdavUploadTUS/checksums.feature:79](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L79)
- [coreApiWebdavUploadTUS/checksums.feature:78](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L78)
- [coreApiWebdavUploadTUS/checksums.feature:147](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L147)
- [coreApiWebdavUploadTUS/checksums.feature:148](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L148)
- [coreApiWebdavUploadTUS/checksums.feature:149](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L149)
- [coreApiWebdavUploadTUS/checksums.feature:192](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L192)
- [coreApiWebdavUploadTUS/checksums.feature:193](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L193)
- [coreApiWebdavUploadTUS/checksums.feature:194](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L194)
- [coreApiWebdavUploadTUS/checksums.feature:195](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L195)
- [coreApiWebdavUploadTUS/checksums.feature:196](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L196)
- [coreApiWebdavUploadTUS/checksums.feature:197](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L197)
- [coreApiWebdavUploadTUS/checksums.feature:240](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L240)
- [coreApiWebdavUploadTUS/checksums.feature:241](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L241)
- [coreApiWebdavUploadTUS/checksums.feature:242](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L242)
- [coreApiWebdavUploadTUS/checksums.feature:243](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L243)
- [coreApiWebdavUploadTUS/checksums.feature:244](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L244)
- [coreApiWebdavUploadTUS/checksums.feature:245](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L245)
- [coreApiWebdavUploadTUS/uploadToShare.feature:255](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L255)
- [coreApiWebdavUploadTUS/uploadToShare.feature:256](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L256)
- [coreApiWebdavUploadTUS/uploadToShare.feature:279](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L279)
- [coreApiWebdavUploadTUS/uploadToShare.feature:280](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L280)
- [coreApiWebdavUploadTUS/uploadToShare.feature:375](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L375)
- [coreApiWebdavUploadTUS/uploadToShare.feature:376](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L376)
#### [Renaming resource to banned name is allowed in spaces webdav](https://github.com/owncloud/ocis/issues/3099)
- [coreApiWebdavMove2/moveFile.feature:143](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavMove2/moveFile.feature#L143)
- [coreApiWebdavMove1/moveFolder.feature:36](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavMove1/moveFolder.feature#L36)
- [coreApiWebdavMove1/moveFolder.feature:50](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavMove1/moveFolder.feature#L50)
- [coreApiWebdavMove1/moveFolder.feature:64](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavMove1/moveFolder.feature#L64)
#### [Trying to delete other user's trashbin item returns 409 for spaces path instead of 404](https://github.com/owncloud/ocis/issues/9791)
- [coreApiTrashbin/trashbinDelete.feature:92](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiTrashbin/trashbinDelete.feature#L92)
#### [MOVE a file into same folder with same name returns 404 instead of 403](https://github.com/owncloud/ocis/issues/1976)
- [coreApiWebdavMove2/moveFile.feature:100](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavMove2/moveFile.feature#L100)
- [coreApiWebdavMove2/moveFile.feature:101](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavMove2/moveFile.feature#L101)
- [coreApiWebdavMove2/moveFile.feature:102](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavMove2/moveFile.feature#L102)
- [coreApiWebdavMove1/moveFolder.feature:217](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavMove1/moveFolder.feature#L217)
- [coreApiWebdavMove1/moveFolder.feature:218](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavMove1/moveFolder.feature#L218)
- [coreApiWebdavMove1/moveFolder.feature:219](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavMove1/moveFolder.feature#L219)
- [coreApiWebdavMove2/moveShareOnOpencloud.feature:334](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavMove2/moveShareOnOpencloud.feature#L334)
- [coreApiWebdavMove2/moveShareOnOpencloud.feature:337](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavMove2/moveShareOnOpencloud.feature#L337)
- [coreApiWebdavMove2/moveShareOnOpencloud.feature:340](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavMove2/moveShareOnOpencloud.feature#L340)
#### [COPY file/folder to same name is possible (but 500 code error for folder with spaces path)](https://github.com/owncloud/ocis/issues/8711)
- [coreApiSharePublicLink2/copyFromPublicLink.feature:198](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiSharePublicLink2/copyFromPublicLink.feature#L198)
- [coreApiWebdavProperties/copyFile.feature:1094](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavProperties/copyFile.feature#L1094)
- [coreApiWebdavProperties/copyFile.feature:1095](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavProperties/copyFile.feature#L1095)
- [coreApiWebdavProperties/copyFile.feature:1096](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavProperties/copyFile.feature#L1096)
#### [Trying to restore personal file to file of share received folder returns 403 but the share file is deleted (new dav path)](https://github.com/owncloud/ocis/issues/10356)
- [coreApiTrashbin/trashbinSharingToShares.feature:277](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L277)
#### [Uploading file with mtime gives 500 error](https://github.com/opencloud-eu/opencloud/issues/391)
- [coreApiWebdavUpload/uploadFile.feature:400](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavUpload/uploadFile.feature#L400)
- [coreApiWebdavUpload/uploadFile.feature:401](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavUpload/uploadFile.feature#L401)
- [coreApiWebdavUpload/uploadFile.feature:402](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavUpload/uploadFile.feature#L402)
- [coreApiWebdavUploadTUS/uploadFileMtime.feature:79](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavUploadTUS/uploadFileMtime.feature#L79)
- [coreApiWebdavUploadTUS/uploadFileMtime.feature:80](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavUploadTUS/uploadFileMtime.feature#L80)
- [coreApiWebdavUploadTUS/uploadFileMtime.feature:81](https://github.com/opencloud-eu/opencloud/blob/main/tests/acceptance/features/coreApiWebdavUploadTUS/uploadFileMtime.feature#L81)
### Won't fix
Not everything needs to be implemented for opencloud.
- _Blacklisted ignored files are no longer required because opencloud can handle `.htaccess` files without security implications introduced by serving user provided files with apache._
Note: always have an empty line at the end of this file.
The bash script that processes this file requires that the last line has a newline on the end.
@@ -27,7 +27,7 @@ Feature: backup consistency
Then the command should be successful Then the command should be successful
And the command output should contain "💚 No inconsistency found. The backup in '%storage_path%' seems to be valid." And the command output should contain "💚 No inconsistency found. The backup in '%storage_path%' seems to be valid."
@issue-9498 @issue-391 @skipOnOpencloud-decomposed-Storage @issue-9498 @issue-391
Scenario: check backup consistency after uploading file multiple times via TUS Scenario: check backup consistency after uploading file multiple times via TUS
Given user "Alice" uploads a file "filesForUpload/textfile.txt" to "/today.txt" with mtime "today" via TUS inside of the space "Personal" using the WebDAV API Given user "Alice" uploads a file "filesForUpload/textfile.txt" to "/today.txt" with mtime "today" via TUS inside of the space "Personal" using the WebDAV API
And user "Alice" uploads a file "filesForUpload/textfile.txt" to "/today.txt" with mtime "today" via TUS inside of the space "Personal" using the WebDAV API And user "Alice" uploads a file "filesForUpload/textfile.txt" to "/today.txt" with mtime "today" via TUS inside of the space "Personal" using the WebDAV API
@@ -41,7 +41,7 @@ Feature: backup consistency
Then the HTTP status code should be "207" Then the HTTP status code should be "207"
And the number of versions should be "1" And the number of versions should be "1"
@issue-9498 @issue-428 @skipOnOpencloud-decomposed-Storage @issue-9498 @issue-428
Scenario: check backup consistency after uploading a file multiple times Scenario: check backup consistency after uploading a file multiple times
Given user "Alice" has uploaded file with content "hello world" to "/textfile0.txt" Given user "Alice" has uploaded file with content "hello world" to "/textfile0.txt"
And user "Alice" has uploaded file with content "hello world" to "/textfile0.txt" And user "Alice" has uploaded file with content "hello world" to "/textfile0.txt"
+2
View File
@@ -212,6 +212,8 @@ type ObjectInfo struct {
// not to be confused with `Expires` HTTP header. // not to be confused with `Expires` HTTP header.
Expiration time.Time Expiration time.Time
ExpirationRuleID string ExpirationRuleID string
// NumVersions is the number of versions of the object.
NumVersions int
Restore *RestoreInfo Restore *RestoreInfo
+57 -14
View File
@@ -22,6 +22,7 @@ import (
"fmt" "fmt"
"net/http" "net/http"
"net/url" "net/url"
"slices"
"time" "time"
"github.com/minio/minio-go/v7/pkg/s3utils" "github.com/minio/minio-go/v7/pkg/s3utils"
@@ -421,20 +422,17 @@ func (c *Client) listObjectVersions(ctx context.Context, bucketName string, opts
var ( var (
keyMarker = "" keyMarker = ""
versionIDMarker = "" versionIDMarker = ""
preName = ""
preKey = ""
perVersions []Version
numVersions int
) )
send := func(vers []Version) {
for { if opts.WithVersions && opts.ReverseVersions {
// Get list of objects a maximum of 1000 per request. slices.Reverse(vers)
result, err := c.listObjectVersionsQuery(ctx, bucketName, opts, keyMarker, versionIDMarker, delimiter) numVersions = len(vers)
if err != nil {
sendObjectInfo(ObjectInfo{
Err: err,
})
return
} }
for _, version := range vers {
// If contents are available loop through and send over channel.
for _, version := range result.Versions {
info := ObjectInfo{ info := ObjectInfo{
ETag: trimEtag(version.ETag), ETag: trimEtag(version.ETag),
Key: version.Key, Key: version.Key,
@@ -448,6 +446,7 @@ func (c *Client) listObjectVersions(ctx context.Context, bucketName string, opts
UserTags: version.UserTags, UserTags: version.UserTags,
UserMetadata: version.UserMetadata, UserMetadata: version.UserMetadata,
Internal: version.Internal, Internal: version.Internal,
NumVersions: numVersions,
} }
select { select {
// Send object version info. // Send object version info.
@@ -457,6 +456,38 @@ func (c *Client) listObjectVersions(ctx context.Context, bucketName string, opts
return return
} }
} }
}
for {
// Get list of objects a maximum of 1000 per request.
result, err := c.listObjectVersionsQuery(ctx, bucketName, opts, keyMarker, versionIDMarker, delimiter)
if err != nil {
sendObjectInfo(ObjectInfo{
Err: err,
})
return
}
if opts.WithVersions && opts.ReverseVersions {
for _, version := range result.Versions {
if preName == "" {
preName = result.Name
preKey = version.Key
}
if result.Name == preName && preKey == version.Key {
// If the current name is same as previous name,
// we need to append the version to the previous version.
perVersions = append(perVersions, version)
continue
}
// Send the file versions.
send(perVersions)
perVersions = perVersions[:0]
perVersions = append(perVersions, version)
preName = result.Name
preKey = version.Key
}
} else {
send(result.Versions)
}
// Send all common prefixes if any. // Send all common prefixes if any.
// NOTE: prefixes are only present if the request is delimited. // NOTE: prefixes are only present if the request is delimited.
@@ -480,10 +511,20 @@ func (c *Client) listObjectVersions(ctx context.Context, bucketName string, opts
versionIDMarker = result.NextVersionIDMarker versionIDMarker = result.NextVersionIDMarker
} }
// Listing ends result is not truncated, return right here. // If context is canceled, return here.
if !result.IsTruncated { if contextCanceled(ctx) {
return return
} }
// Listing ends result is not truncated, return right here.
if !result.IsTruncated {
// sent the lasted file with versions
if opts.ReverseVersions && len(perVersions) > 0 {
send(perVersions)
}
return
}
} }
}(resultCh) }(resultCh)
return resultCh return resultCh
@@ -683,6 +724,8 @@ func (c *Client) listObjectsQuery(ctx context.Context, bucketName, objectPrefix,
// ListObjectsOptions holds all options of a list object request // ListObjectsOptions holds all options of a list object request
type ListObjectsOptions struct { type ListObjectsOptions struct {
// ReverseVersions - reverse the order of the object versions
ReverseVersions bool
// Include objects versions in the listing // Include objects versions in the listing
WithVersions bool WithVersions bool
// Include objects metadata in the listing // Include objects metadata in the listing
+1 -1
View File
@@ -155,7 +155,7 @@ type Options struct {
// Global constants. // Global constants.
const ( const (
libraryName = "minio-go" libraryName = "minio-go"
libraryVersion = "v7.0.87" libraryVersion = "v7.0.88"
) )
// User Agent should always following the below style. // User Agent should always following the below style.
+38 -4
View File
@@ -868,8 +868,20 @@ type ReplQNodeStats struct {
XferStats map[MetricName]XferStats `json:"transferSummary"` XferStats map[MetricName]XferStats `json:"transferSummary"`
TgtXferStats map[string]map[MetricName]XferStats `json:"tgtTransferStats"` TgtXferStats map[string]map[MetricName]XferStats `json:"tgtTransferStats"`
QStats InQueueMetric `json:"queueStats"` QStats InQueueMetric `json:"queueStats"`
MRFStats ReplMRFStats `json:"mrfStats"` MRFStats ReplMRFStats `json:"mrfStats"`
Retries CounterSummary `json:"retries"`
Errors CounterSummary `json:"errors"`
}
// CounterSummary denotes the stats counter summary
type CounterSummary struct {
// Counted last 1hr
Last1hr uint64 `json:"last1hr"`
// Counted last 1m
Last1m uint64 `json:"last1m"`
// Total counted since uptime
Total uint64 `json:"total"`
} }
// ReplQueueStats holds stats for replication queue across nodes // ReplQueueStats holds stats for replication queue across nodes
@@ -914,8 +926,10 @@ type ReplQStats struct {
XferStats map[MetricName]XferStats `json:"xferStats"` XferStats map[MetricName]XferStats `json:"xferStats"`
TgtXferStats map[string]map[MetricName]XferStats `json:"tgtXferStats"` TgtXferStats map[string]map[MetricName]XferStats `json:"tgtXferStats"`
QStats InQueueMetric `json:"qStats"` QStats InQueueMetric `json:"qStats"`
MRFStats ReplMRFStats `json:"mrfStats"` MRFStats ReplMRFStats `json:"mrfStats"`
Retries CounterSummary `json:"retries"`
Errors CounterSummary `json:"errors"`
} }
// QStats returns cluster level stats for objects in replication queue // QStats returns cluster level stats for objects in replication queue
@@ -958,6 +972,12 @@ func (q ReplQueueStats) QStats() (r ReplQStats) {
r.MRFStats.LastFailedCount += node.MRFStats.LastFailedCount r.MRFStats.LastFailedCount += node.MRFStats.LastFailedCount
r.MRFStats.TotalDroppedCount += node.MRFStats.TotalDroppedCount r.MRFStats.TotalDroppedCount += node.MRFStats.TotalDroppedCount
r.MRFStats.TotalDroppedBytes += node.MRFStats.TotalDroppedBytes r.MRFStats.TotalDroppedBytes += node.MRFStats.TotalDroppedBytes
r.Retries.Last1hr += node.Retries.Last1hr
r.Retries.Last1m += node.Retries.Last1m
r.Retries.Total += node.Retries.Total
r.Errors.Last1hr += node.Errors.Last1hr
r.Errors.Last1m += node.Errors.Last1m
r.Errors.Total += node.Errors.Total
r.Uptime += node.Uptime r.Uptime += node.Uptime
} }
if len(q.Nodes) > 0 { if len(q.Nodes) > 0 {
@@ -971,4 +991,18 @@ type MetricsV2 struct {
Uptime int64 `json:"uptime"` Uptime int64 `json:"uptime"`
CurrentStats Metrics `json:"currStats"` CurrentStats Metrics `json:"currStats"`
QueueStats ReplQueueStats `json:"queueStats"` QueueStats ReplQueueStats `json:"queueStats"`
DowntimeInfo DowntimeInfo `json:"downtimeInfo"`
}
// DowntimeInfo represents the downtime info
type DowntimeInfo struct {
Duration Stat `json:"duration"`
Count Stat `json:"count"`
}
// Stat represents the aggregates
type Stat struct {
Total int64 `json:"total"`
Avg int64 `json:"avg"`
Max int64 `json:"max"`
} }
+15
View File
@@ -1,3 +1,18 @@
## 2.23.1
## 🚨 For users on MacOS 🚨
A long-standing Ginkgo performance issue on MacOS seems to be due to mac's antimalware XProtect. You can follow the instructions [here](https://onsi.github.io/ginkgo/#if-you-are-running-on-macos) to disable it in your terminal. Doing so sped up Ginkgo's own test suite from 1m8s to 47s.
### Fixes
Ginkgo's CLI is now a bit clearer if you pass flags in incorrectly:
- make it clearer that you need to pass a filename to the various profile flags, not an absolute directory [a0e52ff]
- emit an error and exit if the ginkgo invocation includes flags after positional arguments [b799d8d]
This might cause existing CI builds to fail. If so then it's likely that your CI build was misconfigured and should be corrected. Open an issue if you need help.
## 2.23.0 ## 2.23.0
Ginkgo 2.23.0 adds a handful of methods to `GinkgoT()` to make it compatible with the `testing.TB` interface in Go 1.24. `GinkgoT().Context()`, in particular, is a useful shorthand for generating a new context that will clean itself up in a `DeferCleanup()`. This has subtle behavior differences from the golang implementation but should make sense in a Ginkgo... um... context. Ginkgo 2.23.0 adds a handful of methods to `GinkgoT()` to make it compatible with the `testing.TB` interface in Go 1.24. `GinkgoT().Context()`, in particular, is a useful shorthand for generating a new context that will clean itself up in a `DeferCleanup()`. This has subtle behavior differences from the golang implementation but should make sense in a Ginkgo... um... context.
+5 -1
View File
@@ -24,7 +24,11 @@ func (c Command) Run(args []string, additionalArgs []string) {
if err != nil { if err != nil {
AbortWithUsage(err.Error()) AbortWithUsage(err.Error())
} }
for _, arg := range args {
if strings.HasPrefix(arg, "-") {
AbortWith("Malformed arguments - make sure all flags appear {{bold}}after{{/}} the Ginkgo subcommand and {{bold}}before{{/}} your list of packages.\n{{gray}}e.g. 'ginkgo run -p my_package' is valid `ginkgo -p run my_package` is not.{{/}}")
}
}
c.Command(args, additionalArgs) c.Command(args, additionalArgs)
} }
+23 -3
View File
@@ -257,8 +257,12 @@ var FlagSections = GinkgoFlagSections{
{Key: "filter", Style: "{{cyan}}", Heading: "Filtering Tests"}, {Key: "filter", Style: "{{cyan}}", Heading: "Filtering Tests"},
{Key: "failure", Style: "{{red}}", Heading: "Failure Handling"}, {Key: "failure", Style: "{{red}}", Heading: "Failure Handling"},
{Key: "output", Style: "{{magenta}}", Heading: "Controlling Output Formatting"}, {Key: "output", Style: "{{magenta}}", Heading: "Controlling Output Formatting"},
{Key: "code-and-coverage-analysis", Style: "{{orange}}", Heading: "Code and Coverage Analysis"}, {Key: "code-and-coverage-analysis", Style: "{{orange}}", Heading: "Code and Coverage Analysis",
{Key: "performance-analysis", Style: "{{coral}}", Heading: "Performance Analysis"}, Description: "When generating a cover files, please pass a filename {{bold}}not{{/}} a path. To specify a different directory use {{magenta}}--output-dir{{/}}.",
},
{Key: "performance-analysis", Style: "{{coral}}", Heading: "Performance Analysis",
Description: "When generating profile files, please pass filenames {{bold}}not{{/}} a path. Ginkgo will generate a profile file with the given name in the package's directory. To specify a different directory use {{magenta}}--output-dir{{/}}.",
},
{Key: "debug", Style: "{{blue}}", Heading: "Debugging Tests", {Key: "debug", Style: "{{blue}}", Heading: "Debugging Tests",
Description: "In addition to these flags, Ginkgo supports a few debugging environment variables. To change the parallel server protocol set {{blue}}GINKGO_PARALLEL_PROTOCOL{{/}} to {{bold}}HTTP{{/}}. To avoid pruning callstacks set {{blue}}GINKGO_PRUNE_STACK{{/}} to {{bold}}FALSE{{/}}."}, Description: "In addition to these flags, Ginkgo supports a few debugging environment variables. To change the parallel server protocol set {{blue}}GINKGO_PARALLEL_PROTOCOL{{/}} to {{bold}}HTTP{{/}}. To avoid pruning callstacks set {{blue}}GINKGO_PRUNE_STACK{{/}} to {{bold}}FALSE{{/}}."},
{Key: "watch", Style: "{{light-yellow}}", Heading: "Controlling Ginkgo Watch"}, {Key: "watch", Style: "{{light-yellow}}", Heading: "Controlling Ginkgo Watch"},
@@ -572,7 +576,7 @@ var GoBuildFlags = GinkgoFlags{
// GoRunFlags provides flags for the Ginkgo CLI run, and watch commands that capture go's run-time flags. These are passed to the compiled test binary by the ginkgo CLI // GoRunFlags provides flags for the Ginkgo CLI run, and watch commands that capture go's run-time flags. These are passed to the compiled test binary by the ginkgo CLI
var GoRunFlags = GinkgoFlags{ var GoRunFlags = GinkgoFlags{
{KeyPath: "Go.CoverProfile", Name: "coverprofile", UsageArgument: "file", SectionKey: "code-and-coverage-analysis", {KeyPath: "Go.CoverProfile", Name: "coverprofile", UsageArgument: "file", SectionKey: "code-and-coverage-analysis",
Usage: `Write a coverage profile to the file after all tests have passed. Sets -cover.`}, Usage: `Write a coverage profile to the file after all tests have passed. Sets -cover. Must be passed a filename, not a path. Use output-dir to control the location of the output.`},
{KeyPath: "Go.BlockProfile", Name: "blockprofile", UsageArgument: "file", SectionKey: "performance-analysis", {KeyPath: "Go.BlockProfile", Name: "blockprofile", UsageArgument: "file", SectionKey: "performance-analysis",
Usage: `Write a goroutine blocking profile to the specified file when all tests are complete. Preserves test binary.`}, Usage: `Write a goroutine blocking profile to the specified file when all tests are complete. Preserves test binary.`},
{KeyPath: "Go.BlockProfileRate", Name: "blockprofilerate", UsageArgument: "rate", SectionKey: "performance-analysis", {KeyPath: "Go.BlockProfileRate", Name: "blockprofilerate", UsageArgument: "rate", SectionKey: "performance-analysis",
@@ -600,6 +604,22 @@ func VetAndInitializeCLIAndGoConfig(cliConfig CLIConfig, goFlagsConfig GoFlagsCo
errors = append(errors, GinkgoErrors.BothRepeatAndUntilItFails()) errors = append(errors, GinkgoErrors.BothRepeatAndUntilItFails())
} }
if strings.ContainsRune(goFlagsConfig.CoverProfile, os.PathSeparator) {
errors = append(errors, GinkgoErrors.ExpectFilenameNotPath("--coverprofile", goFlagsConfig.CoverProfile))
}
if strings.ContainsRune(goFlagsConfig.CPUProfile, os.PathSeparator) {
errors = append(errors, GinkgoErrors.ExpectFilenameNotPath("--cpuprofile", goFlagsConfig.CPUProfile))
}
if strings.ContainsRune(goFlagsConfig.MemProfile, os.PathSeparator) {
errors = append(errors, GinkgoErrors.ExpectFilenameNotPath("--memprofile", goFlagsConfig.MemProfile))
}
if strings.ContainsRune(goFlagsConfig.BlockProfile, os.PathSeparator) {
errors = append(errors, GinkgoErrors.ExpectFilenameNotPath("--blockprofile", goFlagsConfig.BlockProfile))
}
if strings.ContainsRune(goFlagsConfig.MutexProfile, os.PathSeparator) {
errors = append(errors, GinkgoErrors.ExpectFilenameNotPath("--mutexprofile", goFlagsConfig.MutexProfile))
}
//initialize the output directory //initialize the output directory
if cliConfig.OutputDir != "" { if cliConfig.OutputDir != "" {
err := os.MkdirAll(cliConfig.OutputDir, 0777) err := os.MkdirAll(cliConfig.OutputDir, 0777)
+7
View File
@@ -629,6 +629,13 @@ func (g ginkgoErrors) BothRepeatAndUntilItFails() error {
} }
} }
func (g ginkgoErrors) ExpectFilenameNotPath(flag string, path string) error {
return GinkgoError{
Heading: fmt.Sprintf("%s expects a filename but was given a path: %s", flag, path),
Message: fmt.Sprintf("%s takes a filename, not a path. Use --output-dir to specify a directory to collect all test outputs.", flag),
}
}
/* Stack-Trace parsing errors */ /* Stack-Trace parsing errors */
func (g ginkgoErrors) FailedToParseStackTrace(message string) error { func (g ginkgoErrors) FailedToParseStackTrace(message string) error {
+1 -1
View File
@@ -1,3 +1,3 @@
package types package types
const VERSION = "2.23.0" const VERSION = "2.23.1"
@@ -27,7 +27,6 @@ type Config struct {
FavoriteStorageDrivers map[string]map[string]interface{} `mapstructure:"favorite_storage_drivers"` FavoriteStorageDrivers map[string]map[string]interface{} `mapstructure:"favorite_storage_drivers"`
Version string `mapstructure:"version"` Version string `mapstructure:"version"`
VersionString string `mapstructure:"version_string"` VersionString string `mapstructure:"version_string"`
Edition string `mapstructure:"edition"`
Product string `mapstructure:"product"` Product string `mapstructure:"product"`
ProductName string `mapstructure:"product_name"` ProductName string `mapstructure:"product_name"`
ProductVersion string `mapstructure:"product_version"` ProductVersion string `mapstructure:"product_version"`
@@ -34,7 +34,6 @@ func (s *svc) doStatus(w http.ResponseWriter, r *http.Request) {
NeedsDBUpgrade: false, NeedsDBUpgrade: false,
Version: s.c.Version, Version: s.c.Version,
VersionString: s.c.VersionString, VersionString: s.c.VersionString,
Edition: s.c.Edition,
ProductName: s.c.ProductName, ProductName: s.c.ProductName,
ProductVersion: s.c.ProductVersion, ProductVersion: s.c.ProductVersion,
Product: s.c.Product, Product: s.c.Product,
@@ -69,9 +69,6 @@ func (h *Handler) Init(c *config.Config) {
if h.c.Capabilities.Core.Status.VersionString == "" { if h.c.Capabilities.Core.Status.VersionString == "" {
h.c.Capabilities.Core.Status.VersionString = "10.0.11" // TODO make build determined h.c.Capabilities.Core.Status.VersionString = "10.0.11" // TODO make build determined
} }
if h.c.Capabilities.Core.Status.Edition == "" {
h.c.Capabilities.Core.Status.Edition = "" // TODO make build determined
}
if h.c.Capabilities.Core.Status.ProductName == "" { if h.c.Capabilities.Core.Status.ProductName == "" {
h.c.Capabilities.Core.Status.ProductName = "reva" // TODO make build determined h.c.Capabilities.Core.Status.ProductName = "reva" // TODO make build determined
} }
@@ -220,7 +217,6 @@ func (h *Handler) Init(c *config.Config) {
Minor: 0, Minor: 0,
Micro: 11, Micro: 11,
String: "10.0.11", String: "10.0.11",
Edition: "",
Product: "reva", Product: "reva",
ProductVersion: "", ProductVersion: "",
} }
@@ -0,0 +1,494 @@
package jsoncs3
import (
"context"
"encoding/json"
"fmt"
"strings"
"sync"
"time"
"github.com/alexedwards/argon2id"
apppb "github.com/cs3org/go-cs3apis/cs3/auth/applications/v1beta1"
authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1"
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
"github.com/google/uuid"
"github.com/mitchellh/mapstructure"
"github.com/opencloud-eu/reva/v2/pkg/appauth"
"github.com/opencloud-eu/reva/v2/pkg/appauth/manager/registry"
"github.com/opencloud-eu/reva/v2/pkg/appctx"
ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx"
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/metadata"
"github.com/opencloud-eu/reva/v2/pkg/utils"
"github.com/pkg/errors"
"github.com/sethvargo/go-diceware/diceware"
"github.com/sethvargo/go-password/password"
"go.opentelemetry.io/otel/codes"
)
type PasswordGenerator interface {
GeneratePassword() (string, error)
}
func init() {
registry.Register("jsoncs3", New)
}
type manager struct {
sync.RWMutex // for lazy initialization
mds metadata.Storage
generator PasswordGenerator
initialized bool
}
type config struct {
ProviderAddr string `mapstructure:"provider_addr"`
ServiceUserID string `mapstructure:"service_user_id"`
ServiceUserIdp string `mapstructure:"service_user_idp"`
MachineAuthAPIKey string `mapstructure:"machine_auth_apikey"`
Generator string `mapstructure:"password_generator"`
GeneratorConfig map[string]any `mapstructure:"generator_config"`
}
type updaterFunc func(map[string]*apppb.AppPassword) (map[string]*apppb.AppPassword, error)
const tracerName = "jsoncs3"
func New(m map[string]any) (appauth.Manager, error) {
c := &config{}
if err := mapstructure.Decode(m, c); err != nil {
err = errors.Wrap(err, "error creating a new manager")
return nil, err
}
if c.ProviderAddr == "" {
return nil, fmt.Errorf("appauth jsoncs3 manager: provider_addr not set")
}
if c.ServiceUserID == "" {
return nil, fmt.Errorf("appauth jsoncs3 manager: service_user_id not set")
}
if c.ServiceUserIdp == "" {
return nil, fmt.Errorf("appauth jsoncs3 manager: service_user_idp not set")
}
if c.MachineAuthAPIKey == "" {
return nil, fmt.Errorf("appauth jsoncs3 manager: machine_auth_apikey not set")
}
if c.Generator == "" {
c.Generator = "diceware"
}
var pwgen PasswordGenerator
var err error
switch c.Generator {
case "diceware":
pwgen, err = NewDicewareGenerator(c.GeneratorConfig)
case "random":
pwgen, err = NewRandGenerator(c.GeneratorConfig)
default:
return nil, fmt.Errorf("appauth jsoncs3 manager: unknown generator: %s", c.Generator)
}
if err != nil {
return nil, fmt.Errorf("appauth jsoncs3 manager: failed initialize password generator: %w", err)
}
cs3, err := metadata.NewCS3Storage(c.ProviderAddr, c.ProviderAddr, c.ServiceUserID, c.ServiceUserIdp, c.MachineAuthAPIKey)
if err != nil {
return nil, err
}
return NewWithOptions(cs3, pwgen)
}
func NewWithOptions(mds metadata.Storage, generator PasswordGenerator) (*manager, error) {
return &manager{
mds: mds,
generator: generator,
}, nil
}
// GenerateAppPassword creates a password with specified scope to be used by
// third-party applications.
func (m *manager) GenerateAppPassword(ctx context.Context, scope map[string]*authpb.Scope, label string, expiration *typespb.Timestamp) (*apppb.AppPassword, error) {
ctx, span := appctx.GetTracerProvider(ctx).Tracer(tracerName).Start(ctx, "GenerateAppPassword")
defer span.End()
if err := m.initialize(ctx); err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return nil, err
}
token, err := m.generator.GeneratePassword()
if err != nil {
return nil, errors.Wrap(err, "error creating new token")
}
tokenHashed, err := argon2id.CreateHash(token, argon2id.DefaultParams)
if err != nil {
return nil, errors.Wrap(err, "error creating new token")
}
var userID *userpb.UserId
if user, ok := ctxpkg.ContextGetUser(ctx); ok {
userID = user.GetId()
} else {
return nil, errtypes.BadRequest("no user in context")
}
cTime := &typespb.Timestamp{Seconds: uint64(time.Now().Unix())}
// For persisting we use the hashed password, since we don't
// want to store it in cleartext
appPass := &apppb.AppPassword{
Password: tokenHashed,
TokenScope: scope,
Label: label,
Expiration: expiration,
Ctime: cTime,
Utime: cTime,
User: userID,
}
id := uuid.New().String()
err = m.updateWithRetry(ctx, 5, true, userID, func(a map[string]*apppb.AppPassword) (map[string]*apppb.AppPassword, error) {
a[id] = appPass
return a, nil
})
if err != nil {
return nil, err
}
// Here we need to resplace the hash with the cleartext password again since
// the requestor needs to know the cleartext value.
appPass.Password = token
return appPass, nil
}
// ListAppPasswords lists the application passwords created by a user.
func (m *manager) ListAppPasswords(ctx context.Context) ([]*apppb.AppPassword, error) {
log := appctx.GetLogger(ctx)
ctx, span := appctx.GetTracerProvider(ctx).Tracer(tracerName).Start(ctx, "ListAppPasswords")
defer span.End()
if err := m.initialize(ctx); err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return nil, err
}
var userID *userpb.UserId
if user, ok := ctxpkg.ContextGetUser(ctx); ok {
userID = user.GetId()
} else {
return nil, errtypes.BadRequest("no user in context")
}
_, userAppPasswords, err := m.getUserAppPasswords(ctx, userID)
if err != nil {
if _, ok := err.(errtypes.NotFound); ok {
return []*apppb.AppPassword{}, nil
}
log.Error().Err(err).Msg("getUserAppPasswords failed")
return nil, err
}
userAppPasswordSlice := make([]*apppb.AppPassword, 0, len(userAppPasswords))
for _, p := range userAppPasswords {
userAppPasswordSlice = append(userAppPasswordSlice, p)
}
return userAppPasswordSlice, nil
}
// InvalidateAppPassword invalidates a generated password.
func (m *manager) InvalidateAppPassword(ctx context.Context, secret string) error {
log := appctx.GetLogger(ctx)
ctx, span := appctx.GetTracerProvider(ctx).Tracer(tracerName).Start(ctx, "InvalidateAppPassword")
defer span.End()
if err := m.initialize(ctx); err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return err
}
var userID *userpb.UserId
if user, ok := ctxpkg.ContextGetUser(ctx); ok {
userID = user.GetId()
} else {
return errtypes.BadRequest("no user in context")
}
updater := func(a map[string]*apppb.AppPassword) (map[string]*apppb.AppPassword, error) {
for key, pw := range a {
ok, err := argon2id.ComparePasswordAndHash(secret, pw.Password)
switch {
case err != nil:
log.Debug().Err(err).Msg("Error comparing password and hash")
case ok:
delete(a, key)
return a, nil
}
}
return a, errtypes.NotFound("password not found")
}
err := m.updateWithRetry(ctx, 5, false, userID, updater)
if err != nil {
log.Error().Err(err).Msg("getUserAppPasswords failed")
return errtypes.NotFound("password not found")
}
return nil
}
// GetAppPassword retrieves the password information by the combination of username and password.
func (m *manager) GetAppPassword(ctx context.Context, user *userpb.UserId, secret string) (*apppb.AppPassword, error) {
log := appctx.GetLogger(ctx)
ctx, span := appctx.GetTracerProvider(ctx).Tracer(tracerName).Start(ctx, "GetAppPassword")
defer span.End()
if err := m.initialize(ctx); err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return nil, err
}
errUpdateSkipped := errors.New("update skipped")
var matchedPw *apppb.AppPassword
updater := func(a map[string]*apppb.AppPassword) (map[string]*apppb.AppPassword, error) {
matchedPw = nil
for id, pw := range a {
ok, err := argon2id.ComparePasswordAndHash(secret, pw.Password)
switch {
case err != nil:
log.Debug().Err(err).Msg("Error comparing password and hash")
case ok:
// password found
if pw.Expiration != nil && pw.Expiration.Seconds != 0 && uint64(time.Now().Unix()) > pw.Expiration.Seconds {
log.Debug().Str("AppPasswordId", id).Msg("password expired")
return nil, errtypes.NotFound("password not found")
}
matchedPw = pw
// password not expired
// Updating the Utime will cause an Upload for every single GetAppPassword request. We are limiting this to one
// update per 5 minutes otherwise this backend will become unusable.
if time.Since(utils.TSToTime(pw.Utime)) > 5*time.Minute {
a[id].Utime = utils.TSNow()
return a, nil
}
return a, errUpdateSkipped
}
}
return nil, errtypes.NotFound("password not found")
}
err := m.updateWithRetry(ctx, 5, false, user, updater)
switch {
case err == nil:
fallthrough
case errors.Is(err, errUpdateSkipped):
return matchedPw, nil
}
return nil, errtypes.NotFound("password not found")
}
func (m *manager) initialize(ctx context.Context) error {
_, span := appctx.GetTracerProvider(ctx).Tracer(tracerName).Start(ctx, "initialize")
defer span.End()
if m.initialized {
span.SetStatus(codes.Ok, "already initialized")
return nil
}
m.Lock()
defer m.Unlock()
if m.initialized { // check if initialization happened while grabbing the lock
span.SetStatus(codes.Ok, "initialized while grabbing lock")
return nil
}
ctx = context.Background()
err := m.mds.Init(ctx, "jsoncs3-appauth-data")
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return err
}
m.initialized = true
return nil
}
func (m *manager) updateWithRetry(ctx context.Context, retries int, createIfNotFound bool, userid *userpb.UserId, updater updaterFunc) error {
_, span := appctx.GetTracerProvider(ctx).Tracer(tracerName).Start(ctx, "initialize")
defer span.End()
retry := true
var (
etag string
userAppPasswords map[string]*apppb.AppPassword
err error
)
// retry for the specified number of times, then error out
for i := 0; i < retries && retry; i++ {
etag, userAppPasswords, err = m.getUserAppPasswords(ctx, userid)
switch err.(type) {
case nil:
// empty
case errtypes.NotFound:
if createIfNotFound {
userAppPasswords = map[string]*apppb.AppPassword{}
} else {
span.RecordError(err)
span.SetStatus(codes.Error, "downloading app tokens failed")
return err
}
default:
span.RecordError(err)
span.SetStatus(codes.Error, "downloading app tokens failed")
return err
}
userAppPasswords, err = updater(userAppPasswords)
if err != nil {
return err
}
err = m.updateUserAppPassword(ctx, userid, userAppPasswords, etag)
switch err.(type) {
case nil:
retry = false
case errtypes.PreconditionFailed:
retry = true
default:
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return err
}
}
if retry {
span.RecordError(err)
span.SetStatus(codes.Error, "updating app tokens failed")
return err
}
return nil
}
func (m *manager) updateUserAppPassword(ctx context.Context, userid *userpb.UserId, appPasswords map[string]*apppb.AppPassword, ifMatchEtag string) error {
log := appctx.GetLogger(ctx)
ctx, span := appctx.GetTracerProvider(ctx).Tracer(tracerName).Start(ctx, "getUserAppPasswords")
jsonPath := userAppTokenJSONPath(userid)
pwBytes, err := json.Marshal(appPasswords)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return err
}
ur := metadata.UploadRequest{
Path: jsonPath,
Content: pwBytes,
IfMatchEtag: ifMatchEtag,
}
// If there is no etag, make sure to only upload if the file wasn't craeted yet
if ifMatchEtag == "" {
ur.IfNoneMatch = []string{"*"}
}
_, err = m.mds.Upload(ctx, ur)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
log.Debug().Err(err).Msg("persisting provider cache failed")
return err
}
return nil
}
func (m *manager) getUserAppPasswords(ctx context.Context, userid *userpb.UserId) (string, map[string]*apppb.AppPassword, error) {
log := appctx.GetLogger(ctx)
ctx, span := appctx.GetTracerProvider(ctx).Tracer(tracerName).Start(ctx, "getUserAppPasswords")
jsonPath := userAppTokenJSONPath(userid)
dlreq := metadata.DownloadRequest{
Path: jsonPath,
}
var userAppPasswords = map[string]*apppb.AppPassword{}
dlres, err := m.mds.Download(ctx, dlreq)
switch err.(type) {
case nil:
err = json.Unmarshal(dlres.Content, &userAppPasswords)
if err != nil {
log.Error().Err(err).Msg("unmarshaling app tokens failed")
return "", nil, err
}
case errtypes.NotFound:
return "", nil, errtypes.NotFound("password not found")
default:
span.RecordError(err)
span.SetStatus(codes.Error, "downloading app tokens failed")
return "", nil, err
}
return dlres.Etag, userAppPasswords, nil
}
func userAppTokenJSONPath(userID *userpb.UserId) string {
return userID.GetOpaqueId() + ".json"
}
type randomPassword struct {
Strength int `mapstructure:"token_strength"`
}
func NewRandGenerator(config map[string]any) (*randomPassword, error) {
r := &randomPassword{}
if err := mapstructure.Decode(config, r); err != nil {
err = errors.Wrap(err, "error configuring password generator")
return nil, err
}
if r.Strength <= 0 {
r.Strength = 11
}
return r, nil
}
func (r randomPassword) GeneratePassword() (string, error) {
token, err := password.Generate(r.Strength, r.Strength/2, 0, false, false)
if err != nil {
return "", errors.Wrap(err, "error creating new token")
}
return token, nil
}
type dicewarePassword struct {
NumWords int `mapstructure:"number_of_words"`
}
func NewDicewareGenerator(config map[string]any) (*dicewarePassword, error) {
d := &dicewarePassword{}
if err := mapstructure.Decode(config, d); err != nil {
err = errors.Wrap(err, "error creating a new manager")
return nil, err
}
if d.NumWords <= 0 {
d.NumWords = 6
}
return d, nil
}
func (d dicewarePassword) GeneratePassword() (string, error) {
token, err := diceware.Generate(d.NumWords)
if err != nil {
return "", errors.Wrap(err, "error creating new token")
}
return strings.Join(token, " "), nil
}
@@ -21,5 +21,6 @@ package loader
import ( import (
// Load core application auth manager drivers. // Load core application auth manager drivers.
_ "github.com/opencloud-eu/reva/v2/pkg/appauth/manager/json" _ "github.com/opencloud-eu/reva/v2/pkg/appauth/manager/json"
_ "github.com/opencloud-eu/reva/v2/pkg/appauth/manager/jsoncs3"
// Add your own here // Add your own here
) )
-7
View File
@@ -297,13 +297,6 @@ func VersionString(val string) Option {
} }
} }
// Edition provides a function to set the Edition config option.
func Edition(val string) Option {
return func(o *Options) {
o.config.Edition = val
}
}
// Product provides a function to set the Product config option. // Product provides a function to set the Product config option.
func Product(val string) Option { func Product(val string) Option {
return func(o *Options) { return func(o *Options) {
@@ -142,7 +142,6 @@ type Status struct {
NeedsDBUpgrade ocsBool `json:"needsDbUpgrade" xml:"needsDbUpgrade"` NeedsDBUpgrade ocsBool `json:"needsDbUpgrade" xml:"needsDbUpgrade"`
Version string `json:"version" xml:"version"` Version string `json:"version" xml:"version"`
VersionString string `json:"versionstring" xml:"versionstring"` VersionString string `json:"versionstring" xml:"versionstring"`
Edition string `json:"edition" xml:"edition"`
ProductName string `json:"productname" xml:"productname"` ProductName string `json:"productname" xml:"productname"`
Product string `json:"product" xml:"product"` Product string `json:"product" xml:"product"`
ProductVersion string `json:"productversion" xml:"productversion"` ProductVersion string `json:"productversion" xml:"productversion"`
@@ -309,7 +308,6 @@ type Version struct {
Minor int `json:"minor" xml:"minor"` Minor int `json:"minor" xml:"minor"`
Micro int `json:"micro" xml:"micro"` // = patch level Micro int `json:"micro" xml:"micro"` // = patch level
String string `json:"string" xml:"string"` String string `json:"string" xml:"string"`
Edition string `json:"edition" xml:"edition"`
Product string `json:"product" xml:"product"` Product string `json:"product" xml:"product"`
ProductVersion string `json:"productversion" xml:"productversion"` ProductVersion string `json:"productversion" xml:"productversion"`
} }
@@ -276,6 +276,11 @@ func (lu *Lookup) InternalRoot() string {
return lu.Options.Root return lu.Options.Root
} }
// InternalSpaceRoot returns the internal path for a space
func (lu *Lookup) InternalSpaceRoot(spaceID string) string {
return lu.InternalPath(spaceID, spaceID)
}
// InternalPath returns the internal path for a given ID // InternalPath returns the internal path for a given ID
func (lu *Lookup) InternalPath(spaceID, nodeID string) string { func (lu *Lookup) InternalPath(spaceID, nodeID string) string {
if strings.Contains(nodeID, node.RevisionIDDelimiter) || strings.HasSuffix(nodeID, node.CurrentIDDelimiter) { if strings.Contains(nodeID, node.RevisionIDDelimiter) || strings.HasSuffix(nodeID, node.CurrentIDDelimiter) {
+1 -1
View File
@@ -139,7 +139,7 @@ func New(m map[string]interface{}, stream events.Stream, log *zerolog.Logger) (s
Permissions: p, Permissions: p,
EventStream: stream, EventStream: stream,
UserMapper: um, UserMapper: um,
DisableVersioning: false, DisableVersioning: o.DisableVersioning,
Trashbin: trashbin, Trashbin: trashbin,
} }
@@ -62,7 +62,18 @@ func (tp *Tree) CreateRevision(ctx context.Context, n *node.Node, version string
defer sf.Close() defer sf.Close()
vf, err := os.OpenFile(versionPath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0600) vf, err := os.OpenFile(versionPath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0600)
if err != nil { if err != nil {
return "", err if os.IsExist(err) {
err := os.Remove(versionPath)
if err != nil {
return "", err
}
vf, err = os.OpenFile(versionPath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0600)
if err != nil {
return "", err
}
} else {
return "", err
}
} }
defer vf.Close() defer vf.Close()
if _, err := io.Copy(vf, sf); err != nil { if _, err := io.Copy(vf, sf); err != nil {
@@ -263,6 +263,11 @@ func (lu *Lookup) InternalRoot() string {
return lu.Options.Root return lu.Options.Root
} }
// InternalSpaceRoot returns the internal path for a space
func (lu *Lookup) InternalSpaceRoot(spaceID string) string {
return filepath.Join(lu.Options.Root, "spaces", Pathify(spaceID, 1, 2))
}
// InternalPath returns the internal path for a given ID // InternalPath returns the internal path for a given ID
func (lu *Lookup) InternalPath(spaceID, nodeID string) string { func (lu *Lookup) InternalPath(spaceID, nodeID string) string {
return filepath.Join(lu.Options.Root, "spaces", Pathify(spaceID, 1, 2), "nodes", Pathify(nodeID, 4, 2)) return filepath.Join(lu.Options.Root, "spaces", Pathify(spaceID, 1, 2), "nodes", Pathify(nodeID, 4, 2))
@@ -156,6 +156,7 @@ type PathLookup interface {
GenerateSpaceID(spaceType string, owner *userpb.User) (string, error) GenerateSpaceID(spaceType string, owner *userpb.User) (string, error)
InternalRoot() string InternalRoot() string
InternalSpaceRoot(spaceID string) string
InternalPath(spaceID, nodeID string) string InternalPath(spaceID, nodeID string) string
VersionPath(spaceID, nodeID, version string) string VersionPath(spaceID, nodeID, version string) string
Path(ctx context.Context, n *Node, hasPermission PermissionFunc) (path string, err error) Path(ctx context.Context, n *Node, hasPermission PermissionFunc) (path string, err error)
@@ -786,8 +786,11 @@ func (fs *Decomposedfs) DeleteStorageSpace(ctx context.Context, req *provider.De
} }
// remove space metadata // remove space metadata
if err := os.RemoveAll(root); err != nil { spaceRoot := fs.lu.InternalSpaceRoot(spaceID)
return err if spaceRoot != "" {
if err := os.RemoveAll(fs.lu.InternalSpaceRoot(spaceID)); err != nil {
return err
}
} }
// invalidate id in cache // invalidate id in cache
@@ -30,6 +30,7 @@ import (
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/rogpeppe/go-internal/lockedfile" "github.com/rogpeppe/go-internal/lockedfile"
"github.com/shamaton/msgpack/v2"
"github.com/opencloud-eu/reva/v2/pkg/appctx" "github.com/opencloud-eu/reva/v2/pkg/appctx"
"github.com/opencloud-eu/reva/v2/pkg/errtypes" "github.com/opencloud-eu/reva/v2/pkg/errtypes"
@@ -61,7 +62,40 @@ func (tp *Tree) CreateRevision(ctx context.Context, n *node.Node, version string
// create version node // create version node
vf, err := os.OpenFile(versionPath, os.O_CREATE|os.O_EXCL, 0600) vf, err := os.OpenFile(versionPath, os.O_CREATE|os.O_EXCL, 0600)
if err != nil { if err != nil {
return "", err if os.IsExist(err) {
revisionNode := node.NewBaseNode(n.SpaceID, n.ID+node.RevisionIDDelimiter+version, tp.lookup)
revisionPath := tp.lookup.MetadataBackend().MetadataPath(revisionNode)
b, err := os.ReadFile(revisionPath)
if err != nil {
return "", err
}
m := map[string][]byte{}
if err := msgpack.Unmarshal(b, &m); err != nil {
return "", err
}
bid := m["user.oc.blobid"]
if string(bid) != "" {
if err := tp.DeleteBlob(&node.Node{
BaseNode: *revisionNode,
BlobID: string(bid),
}); err != nil {
return "", err
}
}
err = os.Remove(versionPath)
if err != nil {
return "", err
}
vf, err = os.OpenFile(versionPath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0600)
if err != nil {
return "", err
}
} else {
return "", err
}
} }
defer vf.Close() defer vf.Close()
+20
View File
@@ -0,0 +1,20 @@
Copyright 2017 Seth Vargo <seth@sethvargo.com>
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+195
View File
@@ -0,0 +1,195 @@
package diceware
import (
"crypto/rand"
"fmt"
"io"
"math"
"math/big"
)
// sides is the number of sides on a die.
var sides = big.NewInt(6)
var _ DicewareGenerator = (*Generator)(nil)
// Generator is the stateful generator which can be used to customize the word
// list and other generation options.
type Generator struct {
wordList WordList
randReader io.Reader
}
// GeneratorInput is used as input to the NewGenerator function.
type GeneratorInput struct {
// WordList is the word list to use. There are built-in word lists like
// WordListEffBig (default), WordListEffSmall, and WordListOriginal. You can
// also bring your own word list by implementing the WordList interface.
WordList WordList
// RandReader is an optional reader to use in place of the default
// (crypto/rand.Reader), which can be used to generate repeatable sets of
// words
RandReader io.Reader
}
// NewGenerator creates a new Generator from the specified configuration. If no
// input is given, all the default values are used. This function is safe for
// concurrent use.
func NewGenerator(i *GeneratorInput) (*Generator, error) {
if i == nil {
i = new(GeneratorInput)
}
if i.WordList == nil {
i.WordList = WordListEffLarge()
}
gen := &Generator{
wordList: i.WordList,
randReader: i.RandReader,
}
if gen.randReader == nil {
gen.randReader = rand.Reader
}
return gen, nil
}
// Generate generates a collection of diceware words, specified by the numWords
// parameter.
//
// The algorithm is fast, but it's not designed to be performant, favoring
// entropy over speed.
//
// This function is safe for concurrent use, but there is a possibility of
// concurrent invocations generating overlapping words. To generate multiple
// non-overlapping words, use a single invocation of the function and split the
// resulting string list.
func (g *Generator) Generate(numWords int) ([]string, error) {
if typ, ok := g.wordList.(WordListNumWordser); ok {
if l := typ.NumWords(); numWords > l {
return nil, fmt.Errorf("number of requested words (%d) cannot exceed the size of the wordlist (%d)",
numWords, l)
}
}
list := make([]string, 0, numWords)
seen := make(map[string]struct{}, numWords)
for i := 0; i < numWords; i++ {
n, err := g.RollWord(g.wordList.Digits())
if err != nil {
return nil, err
}
word := g.wordList.WordAt(n)
if _, ok := seen[word]; ok {
i--
continue
}
list = append(list, word)
seen[word] = struct{}{}
}
return list, nil
}
// MustGenerate is the same as Generate, but panics on error.
func (g *Generator) MustGenerate(numWords int) []string {
list, err := g.Generate(numWords)
if err != nil {
panic(err)
}
return list
}
// Generate - see Generator.Generate for usage.
func Generate(numWords int) ([]string, error) {
gen, err := NewGenerator(nil)
if err != nil {
return nil, err
}
return gen.Generate(numWords)
}
// MustGenerate - see Generator.MustGenerate for usage.
func MustGenerate(numWords int) []string {
gen, err := NewGenerator(nil)
if err != nil {
panic(err)
}
return gen.MustGenerate(numWords)
}
// GenerateWithWordList generates a list of the given number of words from the
// given word list.
func GenerateWithWordList(numWords int, wordList WordList) ([]string, error) {
gen, err := NewGenerator(&GeneratorInput{
WordList: wordList,
})
if err != nil {
return nil, err
}
return gen.Generate(numWords)
}
// WordAt retrieves the word at the given index from EFF's large wordlist.
//
// Deprecated: Use WordList.WordAt instead.
func WordAt(i int) string {
return WordListEffLarge().WordAt(i)
}
// RollDie rolls a single 6-sided die and returns a value between [1,6].
//
// Internally this creates a new Generator with a nil configuration and calls
// Generator.RollDie.
func RollDie() (int, error) {
gen, err := NewGenerator(nil)
if err != nil {
return 0, err
}
return gen.RollDie()
}
// RollWord rolls and aggregates dice to represent one word in the list. The
// result is the index of the word in the list.
//
// Internally this creates a new Generator with a nil configuration and calls
// Generator.RollWord.
func RollWord(d int) (int, error) {
gen, err := NewGenerator(nil)
if err != nil {
return 0, err
}
return gen.RollWord(d)
}
// RollDie rolls a single 6-sided die and returns a value between [1,6].
func (g *Generator) RollDie() (int, error) {
r, err := rand.Int(g.randReader, sides)
if err != nil {
return 0, fmt.Errorf("failed to generate a random number: %w", err)
}
return int(r.Int64()) + 1, nil
}
// RollWord rolls and aggregates dice to represent one word in the list. The
// result is the index of the word in the list.
func (g *Generator) RollWord(d int) (int, error) {
var final int
for i := d; i > 0; i-- {
res, err := g.RollDie()
if err != nil {
return 0, err
}
final += res * int(math.Pow(10, float64(i-1)))
}
return final, nil
}
+6
View File
@@ -0,0 +1,6 @@
package diceware
type DicewareGenerator interface {
Generate(int) ([]string, error)
MustGenerate(int) []string
}
+38
View File
@@ -0,0 +1,38 @@
package diceware
var _ DicewareGenerator = (*mockGenerator)(nil)
type mockGenerator struct {
result []string
err error
}
// NewMockGenerator creates a new generator that satisfies the DicewareGenerator
// interface. If an error is provided, the error is returned. If a result if
// provided, the result is always returned, regardless of what parameters are
// passed into the Generate or MustGenerate methods.
//
// This function is most useful for tests where you want to have predicable
// results for a transitive resource that depends on go-diceware.
func NewMockGenerator(result []string, err error) *mockGenerator {
return &mockGenerator{
result: result,
err: err,
}
}
// Generate returns the mocked result or error.
func (g *mockGenerator) Generate(int) ([]string, error) {
if g.err != nil {
return nil, g.err
}
return g.result, nil
}
// MustGenerate returns the mocked result or panics if an error was given.
func (g *mockGenerator) MustGenerate(int) []string {
if g.err != nil {
panic(g.err)
}
return g.result
}
+42
View File
@@ -0,0 +1,42 @@
package diceware
// WordList is an interface that must be implemented to be considered a word
// list for use in the diceware algorithm. This interface can be implemented by
// other libraries.
type WordList interface {
// Digits is the number of digits for indexes in the word list. This
// determines the number of dice rolls.
Digits() int
// WordAt returns the word at the given integer in the word list.
WordAt(int) string
}
// WordListNumWordser is an auxiliary interface that returns the number of words
// in the list. This is a separate interface for backwards compatibility.
type WordListNumWordser interface {
// NumWords returns the total number of words in the list.
NumWords() int
}
var (
_ WordList = (*wordListInternal)(nil)
_ WordListNumWordser = (*wordListInternal)(nil)
)
type wordListInternal struct {
digits int
words map[int]string
}
func (w *wordListInternal) Digits() int {
return w.digits
}
func (w *wordListInternal) WordAt(i int) string {
return w.words[i]
}
func (w *wordListInternal) NumWords() int {
return len(w.words)
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+7 -3
View File
@@ -942,7 +942,7 @@ github.com/minio/highwayhash
# github.com/minio/md5-simd v1.1.2 # github.com/minio/md5-simd v1.1.2
## explicit; go 1.14 ## explicit; go 1.14
github.com/minio/md5-simd github.com/minio/md5-simd
# github.com/minio/minio-go/v7 v7.0.87 # github.com/minio/minio-go/v7 v7.0.88
## explicit; go 1.22 ## explicit; go 1.22
github.com/minio/minio-go/v7 github.com/minio/minio-go/v7
github.com/minio/minio-go/v7/pkg/cors github.com/minio/minio-go/v7/pkg/cors
@@ -1057,7 +1057,7 @@ github.com/onsi/ginkgo/reporters/stenographer
github.com/onsi/ginkgo/reporters/stenographer/support/go-colorable github.com/onsi/ginkgo/reporters/stenographer/support/go-colorable
github.com/onsi/ginkgo/reporters/stenographer/support/go-isatty github.com/onsi/ginkgo/reporters/stenographer/support/go-isatty
github.com/onsi/ginkgo/types github.com/onsi/ginkgo/types
# github.com/onsi/ginkgo/v2 v2.23.0 # github.com/onsi/ginkgo/v2 v2.23.1
## explicit; go 1.23.0 ## explicit; go 1.23.0
github.com/onsi/ginkgo/v2 github.com/onsi/ginkgo/v2
github.com/onsi/ginkgo/v2/config github.com/onsi/ginkgo/v2/config
@@ -1191,7 +1191,7 @@ github.com/open-policy-agent/opa/v1/types
github.com/open-policy-agent/opa/v1/util github.com/open-policy-agent/opa/v1/util
github.com/open-policy-agent/opa/v1/util/decoding github.com/open-policy-agent/opa/v1/util/decoding
github.com/open-policy-agent/opa/v1/version github.com/open-policy-agent/opa/v1/version
# github.com/opencloud-eu/reva/v2 v2.28.1-0.20250318145617-dd5b9b6fb606 # github.com/opencloud-eu/reva/v2 v2.28.1-0.20250320105919-be91238e6b11
## explicit; go 1.24.1 ## explicit; go 1.24.1
github.com/opencloud-eu/reva/v2/cmd/revad/internal/grace github.com/opencloud-eu/reva/v2/cmd/revad/internal/grace
github.com/opencloud-eu/reva/v2/cmd/revad/runtime github.com/opencloud-eu/reva/v2/cmd/revad/runtime
@@ -1295,6 +1295,7 @@ github.com/opencloud-eu/reva/v2/pkg/app/registry/registry
github.com/opencloud-eu/reva/v2/pkg/app/registry/static github.com/opencloud-eu/reva/v2/pkg/app/registry/static
github.com/opencloud-eu/reva/v2/pkg/appauth github.com/opencloud-eu/reva/v2/pkg/appauth
github.com/opencloud-eu/reva/v2/pkg/appauth/manager/json github.com/opencloud-eu/reva/v2/pkg/appauth/manager/json
github.com/opencloud-eu/reva/v2/pkg/appauth/manager/jsoncs3
github.com/opencloud-eu/reva/v2/pkg/appauth/manager/loader github.com/opencloud-eu/reva/v2/pkg/appauth/manager/loader
github.com/opencloud-eu/reva/v2/pkg/appauth/manager/registry github.com/opencloud-eu/reva/v2/pkg/appauth/manager/registry
github.com/opencloud-eu/reva/v2/pkg/appctx github.com/opencloud-eu/reva/v2/pkg/appctx
@@ -1763,6 +1764,9 @@ github.com/sercand/kuberesolver/v5
# github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 # github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3
## explicit; go 1.13 ## explicit; go 1.13
github.com/sergi/go-diff/diffmatchpatch github.com/sergi/go-diff/diffmatchpatch
# github.com/sethvargo/go-diceware v0.5.0
## explicit; go 1.22
github.com/sethvargo/go-diceware/diceware
# github.com/sethvargo/go-password v0.3.1 # github.com/sethvargo/go-password v0.3.1
## explicit; go 1.21 ## explicit; go 1.21
github.com/sethvargo/go-password/password github.com/sethvargo/go-password/password