Initial QSfera import

This commit is contained in:
Курнат Андрей
2026-06-07 10:20:04 +03:00
commit 2315f25754
16485 changed files with 4826827 additions and 0 deletions
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,150 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package eosclient
import (
"context"
"io"
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/acl"
)
// EOSClient is the interface which enables access to EOS instances through various interfaces.
type EOSClient interface {
AddACL(ctx context.Context, auth, rootAuth Authorization, path string, position uint, a *acl.Entry) error
RemoveACL(ctx context.Context, auth, rootAuth Authorization, path string, a *acl.Entry) error
UpdateACL(ctx context.Context, auth, rootAuth Authorization, path string, position uint, a *acl.Entry) error
GetACL(ctx context.Context, auth Authorization, path, aclType, target string) (*acl.Entry, error)
ListACLs(ctx context.Context, auth Authorization, path string) ([]*acl.Entry, error)
GetFileInfoByInode(ctx context.Context, auth Authorization, inode uint64) (*FileInfo, error)
GetFileInfoByFXID(ctx context.Context, auth Authorization, fxid string) (*FileInfo, error)
GetFileInfoByPath(ctx context.Context, auth Authorization, path string) (*FileInfo, error)
SetAttr(ctx context.Context, auth Authorization, attr *Attribute, errorIfExists, recursive bool, path string) error
UnsetAttr(ctx context.Context, auth Authorization, attr *Attribute, recursive bool, path string) error
GetAttr(ctx context.Context, auth Authorization, key, path string) (*Attribute, error)
GetQuota(ctx context.Context, username string, rootAuth Authorization, path string) (*QuotaInfo, error)
SetQuota(ctx context.Context, rooAuth Authorization, info *SetQuotaInfo) error
Touch(ctx context.Context, auth Authorization, path string) error
Chown(ctx context.Context, auth, chownauth Authorization, path string) error
Chmod(ctx context.Context, auth Authorization, mode, path string) error
CreateDir(ctx context.Context, auth Authorization, path string) error
Remove(ctx context.Context, auth Authorization, path string, noRecycle bool) error
Rename(ctx context.Context, auth Authorization, oldPath, newPath string) error
List(ctx context.Context, auth Authorization, path string) ([]*FileInfo, error)
Read(ctx context.Context, auth Authorization, path string) (io.ReadCloser, error)
Write(ctx context.Context, auth Authorization, path string, stream io.ReadCloser) error
WriteFile(ctx context.Context, auth Authorization, path, source string) error
ListDeletedEntries(ctx context.Context, auth Authorization) ([]*DeletedEntry, error)
RestoreDeletedEntry(ctx context.Context, auth Authorization, key string) error
PurgeDeletedEntries(ctx context.Context, auth Authorization) error
ListVersions(ctx context.Context, auth Authorization, p string) ([]*FileInfo, error)
RollbackToVersion(ctx context.Context, auth Authorization, path, version string) error
ReadVersion(ctx context.Context, auth Authorization, p, version string) (io.ReadCloser, error)
GenerateToken(ctx context.Context, auth Authorization, path string, a *acl.Entry) (string, error)
}
// AttrType is the type of extended attribute,
// either system (sys) or user (user).
type AttrType uint32
// Attribute represents an EOS extended attribute.
type Attribute struct {
Type AttrType
Key, Val string
}
// FileInfo represents the metadata information returned by querying the EOS namespace.
type FileInfo struct {
IsDir bool
MTimeNanos uint32
Inode uint64 `json:"inode"`
FID uint64 `json:"fid"`
UID uint64 `json:"uid"`
GID uint64 `json:"gid"`
TreeSize uint64 `json:"tree_size"`
MTimeSec uint64 `json:"mtime_sec"`
Size uint64 `json:"size"`
TreeCount uint64 `json:"tree_count"`
File string `json:"eos_file"`
ETag string `json:"etag"`
Instance string `json:"instance"`
XS *Checksum `json:"xs"`
SysACL *acl.ACLs `json:"sys_acl"`
Attrs map[string]string `json:"attrs"`
}
// DeletedEntry represents an entry from the trashbin.
type DeletedEntry struct {
RestorePath string
RestoreKey string
Size uint64
DeletionMTime uint64
IsDir bool
}
// Checksum represents a cheksum entry for a file returned by EOS.
type Checksum struct {
XSSum string
XSType string
}
// QuotaInfo reports the available bytes and inodes for a particular user.
// eos reports all quota values are unsigned long, see https://github.com/cern-eos/eos/blob/93515df8c0d5a858982853d960bec98f983c1285/mgm/Quota.hh#L135
type QuotaInfo struct {
AvailableBytes, UsedBytes uint64
AvailableInodes, UsedInodes uint64
}
// SetQuotaInfo encapsulates the information needed to
// create a quota space in EOS for a user
type SetQuotaInfo struct {
Username string
UID string
GID string
QuotaNode string
MaxBytes uint64
MaxFiles uint64
}
// Constants for ACL position
const (
EndPosition uint = 0
StartPosition uint = 1
)
// Role holds the attributes required to authenticate to EOS via role-based access.
type Role struct {
UID, GID string
}
// Authorization specifies the mechanisms through which EOS can be accessed.
// One of the data members must be set.
type Authorization struct {
Role Role
Token string
}
// AttrAlreadyExistsError is the error raised when setting
// an already existing attr on a resource
const AttrAlreadyExistsError = errtypes.BadRequest("attr already exists")
// AttrNotExistsError is the error raised when removing
// an attribute that does not exist
const AttrNotExistsError = errtypes.BadRequest("attr not exists")
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,604 @@
// @project The CERN Tape Archive (CTA)
// @brief CTA-EOS gRPC API for CASTOR-EOS migration
// @copyright Copyright 2019 CERN
// @license This program is free software: you can redistribute it and/or
// modify
// it under the terms of the GNU General Public License as
// published by the Free Software Foundation, either version 3
// of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be
// useful, but WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
// PURPOSE. See the GNU General Public License for more
// details.
//
// You should have received a copy of the GNU General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
// NOTE: Compile for Go with:
// protoc ./eos_grpc.proto --go_out=plugins=grpc:.
syntax = "proto3";
package eos.rpc;
option java_multiple_files = true;
option java_package = "io.grpc.eos.rpc";
option java_outer_classname = "EosProto";
option objc_class_prefix = "EOS";
option go_package = "github.com/cern-eos/grpc-proto/protobuf;eos_grpc";
service Eos {
// Replies to a ping
rpc Ping(PingRequest) returns (PingReply) {}
// ---------------------------------------------------------------------
// NAMESPACE
// ---------------------------------------------------------------------
// Replies to MD requests with a stream
rpc MD(MDRequest) returns (stream MDResponse) {}
// Replies to Find requests with a stream
rpc Find(FindRequest) returns (stream MDResponse) {}
// Replies to a NsStat operation
rpc NsStat(NsStatRequest) returns (NsStatResponse) {}
// Replies to an insert
rpc ContainerInsert(ContainerInsertRequest) returns (InsertReply) {}
rpc FileInsert(FileInsertRequest) returns (InsertReply) {}
// Replies to a NsRequest operation
rpc Exec(NSRequest) returns (NSResponse) {}
// ---------------------------------------------------------------------
// OPENSTACK
// ---------------------------------------------------------------------
// Manila Driver
rpc ManilaServerRequest(ManilaRequest) returns (ManilaResponse) {}
}
message PingRequest {
string authkey = 1;
bytes message = 2;
}
message PingReply { bytes message = 1; }
// ---------------------------------------------------------------------
// NAMESPACE
// ---------------------------------------------------------------------
message ContainerInsertRequest {
repeated ContainerMdProto container = 1;
string authkey = 2;
bool inherit_md = 3;
}
message FileInsertRequest {
repeated FileMdProto files = 1;
string authkey = 2;
}
message InsertReply {
repeated string message = 1;
repeated uint32 retc = 2;
}
message Time {
uint64 sec = 1;
uint64 n_sec = 2;
}
message Checksum {
bytes value = 1;
string type = 2;
}
message FileMdProto {
uint64 id = 1;
uint64 cont_id = 2;
uint64 uid = 3;
uint64 gid = 4;
uint64 size = 5;
uint32 layout_id = 6;
uint32 flags = 7;
bytes name = 8;
bytes link_name = 9;
Time ctime = 10; // change time
Time mtime = 11; // modification time
Checksum checksum = 12;
repeated uint32 locations = 13;
repeated uint32 unlink_locations = 14;
map<string, bytes> xattrs = 15;
bytes path = 16;
string etag = 17;
uint64 inode = 18;
}
message ContainerMdProto {
uint64 id = 1;
uint64 parent_id = 2;
uint64 uid = 3;
uint64 gid = 4;
int64 tree_size = 6;
uint32 mode = 5;
uint32 flags = 7;
bytes name = 8;
Time ctime = 9; // change time
Time mtime = 10; // modification time
Time stime = 11; // sync time
map<string, bytes> xattrs = 12;
bytes path = 13;
string etag = 14;
uint64 inode = 15;
}
enum TYPE {
FILE = 0;
CONTAINER = 1;
LISTING = 2;
STAT = 3;
}
enum QUOTATYPE {
USER = 0;
GROUP = 2;
PROJECT = 3;
}
enum QUOTAOP {
GET = 0;
SET = 1;
RM = 2;
RMNODE = 3;
}
enum QUOTAENTRY {
NONE = 0;
VOLUME = 1;
INODE = 2;
}
message QuotaProto {
bytes path = 1; // quota node path
string name = 2; // associated name for the given type
QUOTATYPE type = 3; // user,group,project or all quota
uint64 usedbytes = 4; // bytes used physical
uint64 usedlogicalbytes = 5; // bytes used logical
uint64 usedfiles = 6; // number of files used
uint64 maxbytes = 7; // maximum number of bytes (volume quota)
uint64 maxlogicalbytes =
8; // maximum number of logical bytes (logical volume quota)
uint64 maxfiles = 9; // maximum number of files (inode quota)
float percentageusedbytes =
10; // percentage of volume quota used from 0 to 100
float percentageusedfiles = 11; // percentag of inode quota used from 0 to 100
string statusbytes = 12; // status string for volume quota ok,warning,exceeded
string statusfiles = 13; // status string for inode quota ok,warning,exceeded
}
message RoleId {
uint64 uid = 1;
uint64 gid = 2;
string username = 3;
string groupname = 4;
}
message MDId {
bytes path = 1;
fixed64 id = 2;
fixed64 ino = 3;
TYPE type = 4;
}
message Limit {
bool zero = 1;
uint64 min = 2;
uint64 max = 3;
}
message MDSelection {
bool select = 1;
Limit ctime = 2;
Limit mtime = 3;
Limit stime = 4;
Limit size = 5;
Limit treesize = 6;
Limit children = 7;
Limit locations = 8;
Limit unlinked_locations = 9;
uint64 layoutid = 10;
uint64 flags = 11;
bool symlink = 12;
Checksum checksum = 13;
uint32 owner = 14;
uint32 group = 15;
bool owner_root = 16;
bool group_root = 17;
bytes regexp_filename = 18;
bytes regexp_dirname = 19;
map<string, bytes> xattr = 20;
}
message MDRequest {
TYPE type = 1;
MDId id = 2;
string authkey = 3;
RoleId role = 4;
MDSelection selection = 5;
}
message MDResponse {
TYPE type = 1;
FileMdProto fmd = 2;
ContainerMdProto cmd = 3;
}
message FindRequest {
TYPE type = 1;
MDId id = 2;
RoleId role = 3;
string authkey = 4;
uint64 maxdepth = 5;
MDSelection selection = 6;
}
message ShareAuth {
string prot = 1;
string name = 2;
string host = 3;
}
message ShareProto {
string permission = 1;
uint64 expires = 2;
string owner = 3;
string group = 4;
uint64 generation = 5;
string path = 6;
bool allowtree = 7;
string vtoken = 8;
repeated ShareAuth origins = 9;
}
message ShareToken {
ShareProto token = 1;
bytes signature = 2;
bytes serialized = 3;
int32 seed = 4;
}
message NSRequest {
message MkdirRequest {
MDId id = 1;
bool recursive = 2;
int64 mode = 3;
}
message RmdirRequest { MDId id = 1; }
message TouchRequest { MDId id = 1; }
message UnlinkRequest {
MDId id = 1;
bool norecycle = 3;
}
message RmRequest {
MDId id = 1;
bool recursive = 2;
bool norecycle = 3;
}
message RenameRequest {
MDId id = 1;
bytes target = 2;
}
message SymlinkRequest {
MDId id = 1;
bytes target = 2;
}
message VersionRequest {
enum VERSION_CMD {
CREATE = 0;
PURGE = 1;
LIST = 2;
GRAB = 3;
}
MDId id = 1;
VERSION_CMD cmd = 2;
int32 maxversion = 3;
string grabversion = 4;
}
message RecycleRequest {
string key = 1;
enum RECYCLE_CMD {
RESTORE = 0;
PURGE = 1;
LIST = 2;
}
RECYCLE_CMD cmd = 2;
message RestoreFlags {
bool force = 1;
bool mkpath = 2;
bool versions = 3;
}
message PurgeDate {
int32 year = 1;
int32 month = 2;
int32 day = 3;
}
RestoreFlags restoreflag = 3;
PurgeDate purgedate = 4;
}
message SetXAttrRequest {
MDId id = 1;
map<string, bytes> xattrs = 2;
bool recursive = 3;
repeated string keystodelete = 4;
bool create = 5;
}
message ChownRequest {
MDId id = 1;
RoleId owner = 2;
}
message ChmodRequest {
MDId id = 1;
int64 mode = 2;
}
message AclRequest {
enum ACL_COMMAND {
NONE = 0;
MODIFY = 1;
LIST = 2;
}
enum ACL_TYPE {
USER_ACL = 0;
SYS_ACL = 1;
}
MDId id = 1;
ACL_COMMAND cmd = 2;
bool recursive = 3;
ACL_TYPE type = 4;
string rule = 5;
uint32 position = 6;
}
message TokenRequest { ShareToken token = 1; }
message QuotaRequest {
bytes path = 1;
RoleId id = 2;
QUOTAOP op = 3; // get or set, rm or rmnode
uint64 maxfiles = 4; // maximum number of bytes (volume quota) for setting
uint64 maxbytes = 5; // maximum number of bytes (volume quota) for setting
QUOTAENTRY entry = 6; // select volume or inode entry for deletion
}
message ShareRequest {
message LsShare {
enum OutFormat {
NONE = 0; //
MONITORING = 1; // [-m]
LISTING = 2; // [-l]
JSON = 3; // [grpc]
}
OutFormat outformat = 1; //
string selection = 2; //
}
message OperateShare {
enum Op {
CREATE = 0;
REMOVE = 1;
SHARE = 2;
UNSHARE = 3;
ACCESS = 4;
MODIFY = 5;
}
Op op = 1;
string share = 2;
string acl = 3;
string path = 4;
string user = 5;
string group = 6;
}
oneof subcmd {
LsShare ls = 1;
OperateShare op = 2;
}
}
string authkey = 1;
RoleId role = 2;
// Actual request data object
oneof command {
MkdirRequest mkdir = 21;
RmdirRequest rmdir = 22;
TouchRequest touch = 23;
UnlinkRequest unlink = 24;
RmRequest rm = 25;
RenameRequest rename = 26;
SymlinkRequest symlink = 27;
VersionRequest version = 28;
RecycleRequest recycle = 29;
SetXAttrRequest xattr = 30;
ChownRequest chown = 31;
ChmodRequest chmod = 32;
AclRequest acl = 33;
TokenRequest token = 34;
QuotaRequest quota = 35;
ShareRequest share = 36;
}
}
message NSResponse {
message ErrorResponse {
int64 code = 1;
string msg = 2;
}
message VersionResponse {
message VersionInfo {
MDId id = 1;
Time mtime = 2;
}
int64 code = 1;
string msg = 2;
repeated VersionInfo versions = 3;
}
message RecycleResponse {
int64 code = 1;
string msg = 2;
message RecycleInfo {
enum DELETIONTYPE {
FILE = 0;
TREE = 1;
}
MDId id = 1;
RoleId owner = 2;
Time dtime = 3;
uint64 size = 4;
DELETIONTYPE type = 5;
string key = 6;
}
repeated RecycleInfo recycles = 3;
}
message AclResponse {
int64 code = 1;
string msg = 2;
string rule = 3;
}
message QuotaResponse {
int64 code = 1;
string msg = 2;
repeated QuotaProto quotanode = 3;
}
message ShareInfo {
string name = 1;
string root = 2;
string rule = 3;
uint64 uid = 4;
uint64 nshared = 5;
}
message ShareAccess {
string name = 1;
bool granted = 2;
}
message ShareResponse {
int64 code = 1;
string msg = 2;
repeated ShareInfo shares = 3;
repeated ShareAccess access = 4;
}
ErrorResponse error = 1;
VersionResponse version = 2;
RecycleResponse recycle = 3;
AclResponse acl = 4;
QuotaResponse quota = 5;
ShareResponse share = 6;
}
message NsStatRequest { string authkey = 1; }
message NsStatResponse {
int64 code = 1;
string emsg = 2;
string state = 3;
uint64 nfiles = 4;
uint64 ncontainers = 5;
uint64 boot_time = 6;
uint64 current_fid = 7;
uint64 current_cid = 8;
uint64 mem_virtual = 9;
uint64 mem_resident = 10;
uint64 mem_share = 11;
uint64 mem_growth = 12;
uint64 threads = 13;
uint64 fds = 14;
uint64 uptime = 15;
}
// ---------------------------------------------------------------------
// OPENSTACK
// ---------------------------------------------------------------------
enum MANILA_REQUEST_TYPE {
CREATE_SHARE = 0;
DELETE_SHARE = 1;
EXTEND_SHARE = 2;
SHRINK_SHARE = 3;
MANAGE_EXISTING = 4;
UNMANAGE = 5;
GET_CAPACITIES = 6;
/* EXTRA FUNCTIONS NOT IMPLEMENTED */
/*
CREATE_SNAPSHOT = 7;
DELETE_SNAPSHOT = 8;
CREATE_SHARE_FROM_SNAPSHOT = 9;
ENSURE_SHARE = 10;
ALLOW_ACCESS = 11;
DENY_ACCESS = 12;
GET_SHARE_STATS = 13;
DO_SETUP = 14;
SETUP_SERVER = 15;
TEARDOWN_SERVER = 16;
GET_NETWORK_ALLOCATIONS_NUMBER = 17;
VERIFY_SHARE_SERVER_HANDLING = 18;
CREATE_SHARE_GROUP = 19;
DELETE_SHARE_GROUP = 20;
*/
}
message ManilaRequest {
MANILA_REQUEST_TYPE request_type = 1;
string auth_key = 2;
string protocol = 3;
string share_name = 4;
string description = 5;
string share_id = 6;
string share_group_id = 7;
int32 quota = 8;
string creator = 9;
string egroup = 10;
string admin_egroup = 11;
string share_host = 12;
string share_location = 13;
}
message ManilaResponse {
string msg = 1; // for generic messages
int32 code = 2; // < 1 is an error -- > 1 is OK
int64 total_used = 3;
int64 total_capacity = 4;
int64 new_share_quota = 5;
string new_share_path = 6;
}
@@ -0,0 +1,445 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.2.0
// - protoc v3.19.1
// source: Rpc.proto
package eos_grpc
import (
context "context"
grpc "google.golang.org/grpc"
codes "google.golang.org/grpc/codes"
status "google.golang.org/grpc/status"
)
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
// Requires gRPC-Go v1.32.0 or later.
const _ = grpc.SupportPackageIsVersion7
// EosClient is the client API for Eos service.
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
type EosClient interface {
// Replies to a ping
Ping(ctx context.Context, in *PingRequest, opts ...grpc.CallOption) (*PingReply, error)
// Replies to MD requests with a stream
MD(ctx context.Context, in *MDRequest, opts ...grpc.CallOption) (Eos_MDClient, error)
// Replies to Find requests with a stream
Find(ctx context.Context, in *FindRequest, opts ...grpc.CallOption) (Eos_FindClient, error)
// Replies to a NsStat operation
NsStat(ctx context.Context, in *NsStatRequest, opts ...grpc.CallOption) (*NsStatResponse, error)
// Replies to an insert
ContainerInsert(ctx context.Context, in *ContainerInsertRequest, opts ...grpc.CallOption) (*InsertReply, error)
FileInsert(ctx context.Context, in *FileInsertRequest, opts ...grpc.CallOption) (*InsertReply, error)
// Replies to a NsRequest operation
Exec(ctx context.Context, in *NSRequest, opts ...grpc.CallOption) (*NSResponse, error)
// Manila Driver
ManilaServerRequest(ctx context.Context, in *ManilaRequest, opts ...grpc.CallOption) (*ManilaResponse, error)
}
type eosClient struct {
cc grpc.ClientConnInterface
}
func NewEosClient(cc grpc.ClientConnInterface) EosClient {
return &eosClient{cc}
}
func (c *eosClient) Ping(ctx context.Context, in *PingRequest, opts ...grpc.CallOption) (*PingReply, error) {
out := new(PingReply)
err := c.cc.Invoke(ctx, "/eos.rpc.Eos/Ping", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *eosClient) MD(ctx context.Context, in *MDRequest, opts ...grpc.CallOption) (Eos_MDClient, error) {
stream, err := c.cc.NewStream(ctx, &Eos_ServiceDesc.Streams[0], "/eos.rpc.Eos/MD", opts...)
if err != nil {
return nil, err
}
x := &eosMDClient{stream}
if err := x.ClientStream.SendMsg(in); err != nil {
return nil, err
}
if err := x.ClientStream.CloseSend(); err != nil {
return nil, err
}
return x, nil
}
type Eos_MDClient interface {
Recv() (*MDResponse, error)
grpc.ClientStream
}
type eosMDClient struct {
grpc.ClientStream
}
func (x *eosMDClient) Recv() (*MDResponse, error) {
m := new(MDResponse)
if err := x.ClientStream.RecvMsg(m); err != nil {
return nil, err
}
return m, nil
}
func (c *eosClient) Find(ctx context.Context, in *FindRequest, opts ...grpc.CallOption) (Eos_FindClient, error) {
stream, err := c.cc.NewStream(ctx, &Eos_ServiceDesc.Streams[1], "/eos.rpc.Eos/Find", opts...)
if err != nil {
return nil, err
}
x := &eosFindClient{stream}
if err := x.ClientStream.SendMsg(in); err != nil {
return nil, err
}
if err := x.ClientStream.CloseSend(); err != nil {
return nil, err
}
return x, nil
}
type Eos_FindClient interface {
Recv() (*MDResponse, error)
grpc.ClientStream
}
type eosFindClient struct {
grpc.ClientStream
}
func (x *eosFindClient) Recv() (*MDResponse, error) {
m := new(MDResponse)
if err := x.ClientStream.RecvMsg(m); err != nil {
return nil, err
}
return m, nil
}
func (c *eosClient) NsStat(ctx context.Context, in *NsStatRequest, opts ...grpc.CallOption) (*NsStatResponse, error) {
out := new(NsStatResponse)
err := c.cc.Invoke(ctx, "/eos.rpc.Eos/NsStat", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *eosClient) ContainerInsert(ctx context.Context, in *ContainerInsertRequest, opts ...grpc.CallOption) (*InsertReply, error) {
out := new(InsertReply)
err := c.cc.Invoke(ctx, "/eos.rpc.Eos/ContainerInsert", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *eosClient) FileInsert(ctx context.Context, in *FileInsertRequest, opts ...grpc.CallOption) (*InsertReply, error) {
out := new(InsertReply)
err := c.cc.Invoke(ctx, "/eos.rpc.Eos/FileInsert", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *eosClient) Exec(ctx context.Context, in *NSRequest, opts ...grpc.CallOption) (*NSResponse, error) {
out := new(NSResponse)
err := c.cc.Invoke(ctx, "/eos.rpc.Eos/Exec", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *eosClient) ManilaServerRequest(ctx context.Context, in *ManilaRequest, opts ...grpc.CallOption) (*ManilaResponse, error) {
out := new(ManilaResponse)
err := c.cc.Invoke(ctx, "/eos.rpc.Eos/ManilaServerRequest", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
// EosServer is the server API for Eos service.
// All implementations must embed UnimplementedEosServer
// for forward compatibility
type EosServer interface {
// Replies to a ping
Ping(context.Context, *PingRequest) (*PingReply, error)
// Replies to MD requests with a stream
MD(*MDRequest, Eos_MDServer) error
// Replies to Find requests with a stream
Find(*FindRequest, Eos_FindServer) error
// Replies to a NsStat operation
NsStat(context.Context, *NsStatRequest) (*NsStatResponse, error)
// Replies to an insert
ContainerInsert(context.Context, *ContainerInsertRequest) (*InsertReply, error)
FileInsert(context.Context, *FileInsertRequest) (*InsertReply, error)
// Replies to a NsRequest operation
Exec(context.Context, *NSRequest) (*NSResponse, error)
// Manila Driver
ManilaServerRequest(context.Context, *ManilaRequest) (*ManilaResponse, error)
mustEmbedUnimplementedEosServer()
}
// UnimplementedEosServer must be embedded to have forward compatible implementations.
type UnimplementedEosServer struct {
}
func (UnimplementedEosServer) Ping(context.Context, *PingRequest) (*PingReply, error) {
return nil, status.Errorf(codes.Unimplemented, "method Ping not implemented")
}
func (UnimplementedEosServer) MD(*MDRequest, Eos_MDServer) error {
return status.Errorf(codes.Unimplemented, "method MD not implemented")
}
func (UnimplementedEosServer) Find(*FindRequest, Eos_FindServer) error {
return status.Errorf(codes.Unimplemented, "method Find not implemented")
}
func (UnimplementedEosServer) NsStat(context.Context, *NsStatRequest) (*NsStatResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method NsStat not implemented")
}
func (UnimplementedEosServer) ContainerInsert(context.Context, *ContainerInsertRequest) (*InsertReply, error) {
return nil, status.Errorf(codes.Unimplemented, "method ContainerInsert not implemented")
}
func (UnimplementedEosServer) FileInsert(context.Context, *FileInsertRequest) (*InsertReply, error) {
return nil, status.Errorf(codes.Unimplemented, "method FileInsert not implemented")
}
func (UnimplementedEosServer) Exec(context.Context, *NSRequest) (*NSResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method Exec not implemented")
}
func (UnimplementedEosServer) ManilaServerRequest(context.Context, *ManilaRequest) (*ManilaResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method ManilaServerRequest not implemented")
}
func (UnimplementedEosServer) mustEmbedUnimplementedEosServer() {}
// UnsafeEosServer may be embedded to opt out of forward compatibility for this service.
// Use of this interface is not recommended, as added methods to EosServer will
// result in compilation errors.
type UnsafeEosServer interface {
mustEmbedUnimplementedEosServer()
}
func RegisterEosServer(s grpc.ServiceRegistrar, srv EosServer) {
s.RegisterService(&Eos_ServiceDesc, srv)
}
func _Eos_Ping_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(PingRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(EosServer).Ping(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/eos.rpc.Eos/Ping",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(EosServer).Ping(ctx, req.(*PingRequest))
}
return interceptor(ctx, in, info, handler)
}
func _Eos_MD_Handler(srv interface{}, stream grpc.ServerStream) error {
m := new(MDRequest)
if err := stream.RecvMsg(m); err != nil {
return err
}
return srv.(EosServer).MD(m, &eosMDServer{stream})
}
type Eos_MDServer interface {
Send(*MDResponse) error
grpc.ServerStream
}
type eosMDServer struct {
grpc.ServerStream
}
func (x *eosMDServer) Send(m *MDResponse) error {
return x.ServerStream.SendMsg(m)
}
func _Eos_Find_Handler(srv interface{}, stream grpc.ServerStream) error {
m := new(FindRequest)
if err := stream.RecvMsg(m); err != nil {
return err
}
return srv.(EosServer).Find(m, &eosFindServer{stream})
}
type Eos_FindServer interface {
Send(*MDResponse) error
grpc.ServerStream
}
type eosFindServer struct {
grpc.ServerStream
}
func (x *eosFindServer) Send(m *MDResponse) error {
return x.ServerStream.SendMsg(m)
}
func _Eos_NsStat_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(NsStatRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(EosServer).NsStat(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/eos.rpc.Eos/NsStat",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(EosServer).NsStat(ctx, req.(*NsStatRequest))
}
return interceptor(ctx, in, info, handler)
}
func _Eos_ContainerInsert_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ContainerInsertRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(EosServer).ContainerInsert(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/eos.rpc.Eos/ContainerInsert",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(EosServer).ContainerInsert(ctx, req.(*ContainerInsertRequest))
}
return interceptor(ctx, in, info, handler)
}
func _Eos_FileInsert_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(FileInsertRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(EosServer).FileInsert(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/eos.rpc.Eos/FileInsert",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(EosServer).FileInsert(ctx, req.(*FileInsertRequest))
}
return interceptor(ctx, in, info, handler)
}
func _Eos_Exec_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(NSRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(EosServer).Exec(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/eos.rpc.Eos/Exec",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(EosServer).Exec(ctx, req.(*NSRequest))
}
return interceptor(ctx, in, info, handler)
}
func _Eos_ManilaServerRequest_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ManilaRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(EosServer).ManilaServerRequest(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/eos.rpc.Eos/ManilaServerRequest",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(EosServer).ManilaServerRequest(ctx, req.(*ManilaRequest))
}
return interceptor(ctx, in, info, handler)
}
// Eos_ServiceDesc is the grpc.ServiceDesc for Eos service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
var Eos_ServiceDesc = grpc.ServiceDesc{
ServiceName: "eos.rpc.Eos",
HandlerType: (*EosServer)(nil),
Methods: []grpc.MethodDesc{
{
MethodName: "Ping",
Handler: _Eos_Ping_Handler,
},
{
MethodName: "NsStat",
Handler: _Eos_NsStat_Handler,
},
{
MethodName: "ContainerInsert",
Handler: _Eos_ContainerInsert_Handler,
},
{
MethodName: "FileInsert",
Handler: _Eos_FileInsert_Handler,
},
{
MethodName: "Exec",
Handler: _Eos_Exec_Handler,
},
{
MethodName: "ManilaServerRequest",
Handler: _Eos_ManilaServerRequest_Handler,
},
},
Streams: []grpc.StreamDesc{
{
StreamName: "MD",
Handler: _Eos_MD_Handler,
ServerStreams: true,
},
{
StreamName: "Find",
Handler: _Eos_Find_Handler,
ServerStreams: true,
},
},
Metadata: "Rpc.proto",
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,500 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package eosgrpc
import (
"bytes"
"context"
"crypto/tls"
"fmt"
"io"
"net/http"
"net/url"
"os"
"strconv"
"time"
"github.com/opencloud-eu/reva/v2/pkg/appctx"
"github.com/opencloud-eu/reva/v2/pkg/eosclient"
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
"github.com/opencloud-eu/reva/v2/pkg/logger"
)
// HTTPOptions to configure the Client.
type HTTPOptions struct {
// HTTP URL of the EOS MGM.
// Default is https://eos-example.org
BaseURL string
// Timeout in seconds for connecting to the service
ConnectTimeout int
// Timeout in seconds for sending a request to the service and getting a response
// Does not include redirections
RWTimeout int
// Timeout in seconds for performing an operation. Includes every redirection, retry, etc
OpTimeout int
// Max idle conns per Transport
MaxIdleConns int
// Max conns per transport per destination host
MaxConnsPerHost int
// Max idle conns per transport per destination host
MaxIdleConnsPerHost int
// TTL for an idle conn per transport
IdleConnTimeout int
// If the URL is https, then we need to configure this client
// with the usual TLS stuff
// Defaults are /etc/grid-security/hostcert.pem and /etc/grid-security/hostkey.pem
ClientCertFile string
ClientKeyFile string
// These will override the defaults, which are common system paths hardcoded
// in the go x509 implementation (why did they do that?!?!?)
// of course /etc/grid-security/certificates is NOT in those defaults!
ClientCADirs string
ClientCAFiles string
}
// Init fills the basic fields
func (opt *HTTPOptions) init() {
if opt.BaseURL == "" {
opt.BaseURL = "https://eos-example.org"
}
if opt.ConnectTimeout == 0 {
opt.ConnectTimeout = 30
}
if opt.RWTimeout == 0 {
opt.RWTimeout = 180
}
if opt.OpTimeout == 0 {
opt.OpTimeout = 360
}
if opt.MaxIdleConns == 0 {
opt.MaxIdleConns = 100
}
if opt.MaxConnsPerHost == 0 {
opt.MaxConnsPerHost = 64
}
if opt.MaxIdleConnsPerHost == 0 {
opt.MaxIdleConnsPerHost = 8
}
if opt.IdleConnTimeout == 0 {
opt.IdleConnTimeout = 30
}
if opt.ClientCertFile == "" {
opt.ClientCertFile = "/etc/grid-security/hostcert.pem"
}
if opt.ClientKeyFile == "" {
opt.ClientKeyFile = "/etc/grid-security/hostkey.pem"
}
if opt.ClientCAFiles != "" {
os.Setenv("SSL_CERT_FILE", opt.ClientCAFiles)
}
if opt.ClientCADirs != "" {
os.Setenv("SSL_CERT_DIR", opt.ClientCADirs)
} else {
os.Setenv("SSL_CERT_DIR", "/etc/grid-security/certificates")
}
}
// EOSHTTPClient performs HTTP-based tasks (e.g. upload, download)
// against a EOS management node (MGM)
// using the EOS XrdHTTP interface.
// In this module we wrap eos-related behaviour, e.g. headers or r/w retries
type EOSHTTPClient struct {
opt *HTTPOptions
cl *http.Client
}
// NewEOSHTTPClient creates a new client with the given options.
func NewEOSHTTPClient(opt *HTTPOptions) (*EOSHTTPClient, error) {
log := logger.New().With().Int("pid", os.Getpid()).Logger()
log.Debug().Str("func", "New").Str("Creating new eoshttp client. opt: ", "'"+fmt.Sprintf("%#v", opt)+"' ").Msg("")
if opt == nil {
log.Debug().Str("opt is nil, error creating http client ", "").Msg("")
return nil, errtypes.InternalError("HTTPOptions is nil")
}
opt.init()
cert, err := tls.LoadX509KeyPair(opt.ClientCertFile, opt.ClientKeyFile)
if err != nil {
return nil, err
}
// TODO: the error reporting of http.transport is insufficient
// we may want to check manually at least the existence of the certfiles
// The point is that also the error reporting of the context that calls this function
// is weak
t := &http.Transport{
TLSClientConfig: &tls.Config{
Certificates: []tls.Certificate{cert},
},
MaxIdleConns: opt.MaxIdleConns,
MaxConnsPerHost: opt.MaxConnsPerHost,
MaxIdleConnsPerHost: opt.MaxIdleConnsPerHost,
IdleConnTimeout: time.Duration(opt.IdleConnTimeout) * time.Second,
DisableCompression: true,
}
cl := &http.Client{
Transport: t,
CheckRedirect: func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
},
}
return &EOSHTTPClient{
opt: opt,
cl: cl,
}, nil
}
// Format a human readable line that describes a response
func rspdesc(rsp *http.Response) string {
desc := "'" + fmt.Sprintf("%d", rsp.StatusCode) + "'" + ": '" + rsp.Status + "'"
buf := new(bytes.Buffer)
r := "<none>"
n, e := buf.ReadFrom(rsp.Body)
if e != nil {
r = "Error reading body: '" + e.Error() + "'"
} else if n > 0 {
r = buf.String()
}
desc += " - '" + r + "'"
return desc
}
// If the error is not nil, take that
// If there is an error coming from EOS, erturn a descriptive error
func (c *EOSHTTPClient) getRespError(rsp *http.Response, err error) error {
if err != nil {
return err
}
if rsp.StatusCode == 0 {
return nil
}
switch rsp.StatusCode {
case 0, 200, 201:
return nil
case 403:
return errtypes.PermissionDenied(rspdesc(rsp))
case 404:
return errtypes.NotFound(rspdesc(rsp))
}
err2 := errtypes.InternalError("Err from EOS: " + rspdesc(rsp))
return err2
}
// From the basepath and the file path... build an url
func (c *EOSHTTPClient) buildFullURL(urlpath string, auth eosclient.Authorization) (string, error) {
u, err := url.Parse(c.opt.BaseURL)
if err != nil {
return "", err
}
u, err = u.Parse(url.PathEscape(urlpath))
if err != nil {
return "", err
}
// Prohibit malicious users from injecting a false uid/gid into the url
v := u.Query()
if v.Get("eos.ruid") != "" || v.Get("eos.rgid") != "" {
return "", errtypes.PermissionDenied("Illegal malicious url " + urlpath)
}
if len(auth.Role.UID) > 0 {
v.Set("eos.ruid", auth.Role.UID)
}
if len(auth.Role.GID) > 0 {
v.Set("eos.rgid", auth.Role.GID)
}
u.RawQuery = v.Encode()
return u.String(), nil
}
// GETFile does an entire GET to download a full file. Returns a stream to read the content from
func (c *EOSHTTPClient) GETFile(ctx context.Context, remoteuser string, auth eosclient.Authorization, urlpath string, stream io.WriteCloser) (io.ReadCloser, error) {
log := appctx.GetLogger(ctx)
log.Info().Str("func", "GETFile").Str("remoteuser", remoteuser).Str("uid,gid", auth.Role.UID+","+auth.Role.GID).Str("path", urlpath).Msg("")
// Now send the req and see what happens
finalurl, err := c.buildFullURL(urlpath, auth)
if err != nil {
log.Error().Str("func", "GETFile").Str("url", finalurl).Str("err", err.Error()).Msg("can't create request")
return nil, err
}
req, err := http.NewRequestWithContext(ctx, "GET", finalurl, nil)
if err != nil {
log.Error().Str("func", "GETFile").Str("url", finalurl).Str("err", err.Error()).Msg("can't create request")
return nil, err
}
ntries := 0
nredirs := 0
timebegin := time.Now().Unix()
for {
// Check for a max count of redirections or retries
// Check for a global timeout in any case
tdiff := time.Now().Unix() - timebegin
if tdiff > int64(c.opt.OpTimeout) {
log.Error().Str("func", "GETFile").Str("url", finalurl).Int64("timeout", tdiff).Int("ntries", ntries).Msg("")
return nil, errtypes.InternalError("Timeout with url" + finalurl)
}
// Execute the request. I don't like that there is no explicit timeout or buffer control on the input stream
log.Debug().Str("func", "GETFile").Msg("sending req")
resp, err := c.cl.Do(req)
// Let's support redirections... and if we retry we have to retry at the same FST, avoid going back to the MGM
if resp != nil && (resp.StatusCode == http.StatusFound || resp.StatusCode == http.StatusTemporaryRedirect) {
// io.Copy(io.Discard, resp.Body)
// resp.Body.Close()
loc, err := resp.Location()
if err != nil {
log.Error().Str("func", "GETFile").Str("url", finalurl).Str("err", err.Error()).Msg("can't get a new location for a redirection")
return nil, err
}
req, err = http.NewRequestWithContext(ctx, "GET", loc.String(), nil)
if err != nil {
log.Error().Str("func", "GETFile").Str("url", loc.String()).Str("err", err.Error()).Msg("can't create redirected request")
return nil, err
}
req.Close = true
log.Debug().Str("func", "GETFile").Str("location", loc.String()).Msg("redirection")
nredirs++
resp = nil
err = nil
continue
}
// And get an error code (if error) that is worth propagating
e := c.getRespError(resp, err)
if e != nil {
if os.IsTimeout(e) {
ntries++
log.Warn().Str("func", "GETFile").Str("url", finalurl).Str("err", e.Error()).Int("try", ntries).Msg("recoverable network timeout")
continue
}
log.Error().Str("func", "GETFile").Str("url", finalurl).Str("err", e.Error()).Msg("")
return nil, e
}
log.Debug().Str("func", "GETFile").Str("url", finalurl).Str("resp:", fmt.Sprintf("%#v", resp)).Msg("")
if resp == nil {
return nil, errtypes.NotFound(fmt.Sprintf("url: %s", finalurl))
}
if stream != nil {
// Streaming versus localfile. If we have bene given a dest stream then copy the body into it
_, err = io.Copy(stream, resp.Body)
return nil, err
}
// If we have not been given a stream to write into then return our stream to read from
return resp.Body, nil
}
}
// PUTFile does an entire PUT to upload a full file, taking the data from a stream
func (c *EOSHTTPClient) PUTFile(ctx context.Context, remoteuser string, auth eosclient.Authorization, urlpath string, stream io.ReadCloser, length int64) error {
log := appctx.GetLogger(ctx)
log.Info().Str("func", "PUTFile").Str("remoteuser", remoteuser).Str("uid,gid", auth.Role.UID+","+auth.Role.GID).Str("path", urlpath).Int64("length", length).Msg("")
// Now send the req and see what happens
finalurl, err := c.buildFullURL(urlpath, auth)
if err != nil {
log.Error().Str("func", "PUTFile").Str("url", finalurl).Str("err", err.Error()).Msg("can't create request")
return err
}
req, err := http.NewRequestWithContext(ctx, "PUT", finalurl, nil)
if err != nil {
log.Error().Str("func", "PUTFile").Str("url", finalurl).Str("err", err.Error()).Msg("can't create request")
return err
}
req.Close = true
ntries := 0
nredirs := 0
timebegin := time.Now().Unix()
for {
// Check for a max count of redirections or retries
// Check for a global timeout in any case
tdiff := time.Now().Unix() - timebegin
if tdiff > int64(c.opt.OpTimeout) {
log.Error().Str("func", "PUTFile").Str("url", finalurl).Int64("timeout", tdiff).Int("ntries", ntries).Msg("")
return errtypes.InternalError("Timeout with url" + finalurl)
}
// Execute the request. I don't like that there is no explicit timeout or buffer control on the input stream
log.Debug().Str("func", "PUTFile").Msg("sending req")
resp, err := c.cl.Do(req)
if resp != nil {
resp.Body.Close()
}
// Let's support redirections... and if we retry we retry at the same FST
if resp != nil && resp.StatusCode == 307 {
// io.Copy(io.Discard, resp.Body)
// resp.Body.Close()
loc, err := resp.Location()
if err != nil {
log.Error().Str("func", "PUTFile").Str("url", finalurl).Str("err", err.Error()).Msg("can't get a new location for a redirection")
return err
}
req, err = http.NewRequestWithContext(ctx, "PUT", loc.String(), stream)
if err != nil {
log.Error().Str("func", "PUTFile").Str("url", loc.String()).Str("err", err.Error()).Msg("can't create redirected request")
return err
}
if length >= 0 {
log.Debug().Str("func", "PUTFile").Int64("Content-Length", length).Msg("setting header")
req.Header.Set("Content-Length", strconv.FormatInt(length, 10))
}
if err != nil {
log.Error().Str("func", "PUTFile").Str("url", loc.String()).Str("err", err.Error()).Msg("can't create redirected request")
return err
}
if length >= 0 {
log.Debug().Str("func", "PUTFile").Int64("Content-Length", length).Msg("setting header")
req.Header.Set("Content-Length", strconv.FormatInt(length, 10))
}
log.Debug().Str("func", "PUTFile").Str("location", loc.String()).Msg("redirection")
nredirs++
resp = nil
err = nil
continue
}
// And get an error code (if error) that is worth propagating
e := c.getRespError(resp, err)
if e != nil {
if os.IsTimeout(e) {
ntries++
log.Warn().Str("func", "PUTFile").Str("url", finalurl).Str("err", e.Error()).Int("try", ntries).Msg("recoverable network timeout")
continue
}
log.Error().Str("func", "PUTFile").Str("url", finalurl).Str("err", e.Error()).Msg("")
return e
}
log.Debug().Str("func", "PUTFile").Str("url", finalurl).Str("resp:", fmt.Sprintf("%#v", resp)).Msg("")
if resp == nil {
return errtypes.NotFound(fmt.Sprintf("url: %s", finalurl))
}
return nil
}
}
// Head performs a HEAD req. Useful to check the server
func (c *EOSHTTPClient) Head(ctx context.Context, remoteuser string, auth eosclient.Authorization, urlpath string) error {
log := appctx.GetLogger(ctx)
log.Info().Str("func", "Head").Str("remoteuser", remoteuser).Str("uid,gid", auth.Role.UID+","+auth.Role.GID).Str("path", urlpath).Msg("")
// Now send the req and see what happens
finalurl, err := c.buildFullURL(urlpath, auth)
if err != nil {
log.Error().Str("func", "Head").Str("url", finalurl).Str("err", err.Error()).Msg("can't create request")
return err
}
req, err := http.NewRequestWithContext(ctx, "HEAD", finalurl, nil)
if err != nil {
log.Error().Str("func", "Head").Str("remoteuser", remoteuser).Str("uid,gid", auth.Role.UID+","+auth.Role.GID).Str("url", finalurl).Str("err", err.Error()).Msg("can't create request")
return err
}
ntries := 0
timebegin := time.Now().Unix()
for {
tdiff := time.Now().Unix() - timebegin
if tdiff > int64(c.opt.OpTimeout) {
log.Error().Str("func", "Head").Str("url", finalurl).Int64("timeout", tdiff).Int("ntries", ntries).Msg("")
return errtypes.InternalError("Timeout with url" + finalurl)
}
// Execute the request. I don't like that there is no explicit timeout or buffer control on the input stream
resp, err := c.cl.Do(req)
if resp != nil {
resp.Body.Close()
}
// And get an error code (if error) that is worth propagating
e := c.getRespError(resp, err)
if e != nil {
if os.IsTimeout(e) {
ntries++
log.Warn().Str("func", "Head").Str("url", finalurl).Str("err", e.Error()).Int("try", ntries).Msg("recoverable network timeout")
continue
}
log.Error().Str("func", "Head").Str("url", finalurl).Str("err", e.Error()).Msg("")
return e
}
log.Debug().Str("func", "Head").Str("url", finalurl).Str("resp:", fmt.Sprintf("%#v", resp)).Msg("")
if resp == nil {
return errtypes.NotFound(fmt.Sprintf("url: %s", finalurl))
}
}
// return nil
}
+61
View File
@@ -0,0 +1,61 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package eosclient
import (
"fmt"
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
)
const (
// SystemAttr is the system extended attribute.
SystemAttr AttrType = iota
// UserAttr is the user extended attribute.
UserAttr
)
// AttrStringToType converts a string to an AttrType
func AttrStringToType(t string) (AttrType, error) {
switch t {
case "sys":
return SystemAttr, nil
case "user":
return UserAttr, nil
default:
return 0, errtypes.InternalError("attr type not existing")
}
}
// AttrTypeToString converts a type to a string representation.
func AttrTypeToString(at AttrType) string {
switch at {
case SystemAttr:
return "sys"
case UserAttr:
return "user"
default:
return "invalid"
}
}
// GetKey returns the key considering the type of attribute.
func (a *Attribute) GetKey() string {
return fmt.Sprintf("%s.%s", AttrTypeToString(a.Type), a.Key)
}