Merge pull request #662 from owncloud/eos-ocis

add eos-ocis docker images

.codacy.yml

@@ -10,5 +10,7 @@ exclude_paths:
   - 'konnectd/assets/identifier/**'
   - 'settings/rollup.config.js'
   - 'accounts/rollup.config.js'
+  - 'ocis/docker/eos-ocis/Dockerfile'
+  - 'ocis/docker/eos-ocis-dev/Dockerfile'
 
-...
+...

.drone.star

@@ -102,6 +102,7 @@ def main(ctx):
     docker(ctx, 'amd64'),
     docker(ctx, 'arm64'),
     docker(ctx, 'arm'),
+    dockerEos(ctx),
     binary(ctx, 'linux'),
     binary(ctx, 'darwin'),
     binary(ctx, 'windows'),
@@ -117,7 +118,7 @@ def main(ctx):
     updateDeployment(ctx)
   ]
 
-  if '[docs-only]' in ctx.build.title:
+  if '[docs-only]' in (ctx.build.title + ctx.build.message):
     pipelines = docs(ctx)
     pipelines['depends_on'] = []
   else:
@@ -742,6 +743,79 @@ def docker(ctx, arch):
     },
   }
 
+def dockerEos(ctx):
+  return {
+    'kind': 'pipeline',
+    'type': 'docker',
+    'name': 'docker-eos-ocis',
+    'platform': {
+      'os': 'linux',
+      'arch': 'amd64',
+    },
+    'steps':
+      generate('ocis') +
+      build() + [
+        {
+          'name': 'dryrun-eos-ocis',
+          'image': 'plugins/docker:18.09',
+          'pull': 'always',
+          'settings': {
+            'dry_run': True,
+            'context': 'ocis/docker/eos-ocis',
+            'tags': 'linux-eos-ocis',
+            'dockerfile': 'ocis/docker/eos-ocis/Dockerfile',
+            'repo': 'owncloud/eos-ocis',
+          },
+          'when': {
+            'ref': {
+              'include': [
+                'refs/pull/**',
+              ],
+            },
+          },
+        },
+        {
+          'name': 'docker-eos-ocis',
+          'image': 'plugins/docker:18.09',
+          'pull': 'always',
+          'settings': {
+            'username': {
+              'from_secret': 'docker_username',
+            },
+            'password': {
+              'from_secret': 'docker_password',
+            },
+            'auto_tag': True,
+            'context': 'ocis/docker/eos-ocis',
+            'auto_tag_suffix': 'linux-amd64',
+            'dockerfile': 'ocis/docker/eos-ocis/Dockerfile',
+            'repo': 'owncloud/eos-ocis',
+          },
+          'when': {
+            'ref': {
+              'exclude': [
+                'refs/pull/**',
+              ],
+            },
+          },
+        },
+      ],
+    'volumes': [
+      {
+        'name': 'gopath',
+        'temp': {},
+      },
+    ],
+    'depends_on': getDependsOnAllTestPipelines(ctx),
+    'trigger': {
+      'ref': [
+        'refs/heads/master',
+        'refs/tags/v*',
+        'refs/pull/**',
+      ],
+    },
+  }
+
 def binary(ctx, name):
   if ctx.build.event == "tag":
     settings = {
@@ -975,6 +1049,7 @@ def manifest(ctx):
       'docker-amd64',
       'docker-arm64',
       'docker-arm',
+      'docker-eos-ocis',
       'binaries-linux',
       'binaries-darwin',
       'binaries-windows',
@@ -1129,6 +1204,7 @@ def badges(ctx):
       'docker-amd64',
       'docker-arm64',
       'docker-arm',
+      'docker-eos-ocis',
     ],
     'trigger': {
       'ref': [

accounts/Makefile

@@ -156,8 +156,8 @@ $(GOPATH)/bin/protoc-gen-micro:
 $(GOPATH)/bin/protoc-gen-microweb:
 	GO111MODULE=off go get -v github.com/owncloud/protoc-gen-microweb
 
-$(GOPATH)/bin/protoc-gen-swagger:
-	GO111MODULE=off go get -v github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger
+$(GOPATH)/bin/protoc-gen-openapiv2:
+	GO111MODULE=off go get -v github.com/grpc-ecosystem/grpc-gateway/protoc-gen-openapiv2
 
 $(PROTO_SRC)/accounts.pb.go: $(PROTO_SRC)/accounts.proto
 	protoc \
@@ -186,5 +186,5 @@ $(PROTO_SRC)/accounts.swagger.json: $(PROTO_SRC)/accounts.proto
 		--swagger_out=$(PROTO_SRC) accounts.proto
 
 .PHONY: protobuf
-protobuf: $(GOPATH)/bin/protoc-gen-go $(GOPATH)/bin/protoc-gen-micro $(GOPATH)/bin/protoc-gen-microweb $(GOPATH)/bin/protoc-gen-swagger \
+protobuf: $(GOPATH)/bin/protoc-gen-go $(GOPATH)/bin/protoc-gen-micro $(GOPATH)/bin/protoc-gen-microweb $(GOPATH)/bin/protoc-gen-openapiv2 \
 		  $(PROTO_SRC)/accounts.pb.go $(PROTO_SRC)/accounts.pb.micro.go $(PROTO_SRC)/accounts.pb.web.go $(PROTO_SRC)/accounts.swagger.json

ocis/docker/eos-ocis-dev/Dockerfile

@@ -0,0 +1,24 @@
+FROM owncloud/eos-base:4.6.5
+
+ENV CGO_ENABLED=1
+ENV GOOS=linux
+
+RUN rpm --rebuilddb && yum -y install \
+    wget \
+    time \
+    make \
+    gcc \
+    git
+
+RUN wget -q https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz
+RUN mkdir -p /usr/local/bin
+RUN tar xf go1.14.2.linux-amd64.tar.gz -C /usr/local
+RUN ln -s /usr/local/go/bin/* /usr/local/bin
+
+COPY entrypoint /entrypoint
+COPY start-ldap /start-ldap
+
+VOLUME [ "/ocis" ]
+WORKDIR /ocis
+
+

ocis/docker/eos-ocis-dev/README.md

@@ -0,0 +1,84 @@
+# Docker image for ocis development with eos
+
+Image is based on [owncloud/eos-base](https://hub.docker.com/r/owncloud/eos-base) from [eos-stack](https://github.com/owncloud-docker/eos-stack)
+
+## Build
+```shell
+docker build -t owncloud/eos-ocis-dev:latest .
+```
+
+## Publish
+```shell
+docker push owncloud/eos-ocis-dev:latest
+```
+
+## Maintainer
+
+* [Felix Böhm](https://github.com/felixboehm)
+
+## Disclaimer
+Use only for development or testing. Setup is not secured nor tested.
+
+## oCIS development on eos
+
+### Setup oCIS
+
+To build and run your local ocis code with default storage driver
+
+```shell
+docker run --rm -ti --name ocis -v $PWD:/ocis -p 9200:9200 owncloud/eos-ocis-dev
+```
+
+ocis will use the owncloud storage driver and store files in the container at /var/tmp/reva/data/<username>/files
+
+Data is here: `docker exec -it ocis ll /var/tmp/reva/`
+
+Alternative: With the [docker-compose.yml file in ocis repo](https://github.com/owncloud/ocis/blob/master/docker-compose.yml) you can also start ocis via compose:
+
+```shell
+docker-compose up -d ocis
+```
+
+Now try to list the running services
+
+```shell
+docker-compose exec ocis ./bin/ocis list
+```
+
+## Setup eos storage
+
+1. Start the eos cluster and ocis via the compose stack
+
+```shell
+docker-compose up -d
+```
+
+2. Configure to use eos storage driver instead of default storage driver
+
+* kill the home storage and data providers. we need to switch them to the eoshome driver:
+
+```shell
+docker-compose exec ocis ./bin/ocis kill reva-storage-home
+docker-compose exec ocis ./bin/ocis kill reva-storage-home-data
+```
+
+* restart them with the eoshome driver and a new layout:
+
+```shell
+docker-compose exec -e REVA_STORAGE_EOS_LAYOUT="{{substr 0 1 .Username}}/{{.Username}}" -e REVA_STORAGE_HOME_DRIVER=eoshome -d ocis ./bin/ocis run reva-storage-home
+docker-compose exec -e REVA_STORAGE_EOS_LAYOUT="{{substr 0 1 .Username}}/{{.Username}}" -e REVA_STORAGE_HOME_DATA_DRIVER=eoshome -d ocis ./bin/ocis run reva-storage-home-data
+```
+
+* restart the reva frontend with a new namespace (pointing to the eos storage provider) for the dav files endpoint
+
+```shell
+docker-compose exec ocis ./bin/ocis kill reva-frontend
+docker-compose exec -e DAV_FILES_NAMESPACE="/eos/" -d ocis ./bin/ocis run reva-frontend
+```
+
+* login with `einstein / relativity`, upload a file to einsteins home and verify the file is there using
+
+```shell
+docker-compose exec ocis eos ls -l /eos/dockertest/reva/users/e/einstein/
+-rw-r--r--   1 einstein users              10 Jul  1 15:24 newfile.txt
+```

ocis/docker/eos-ocis-dev/entrypoint

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+make build
+/start-ldap &
+
+exec bin/ocis server

ocis/docker/eos-ocis-dev/start-ldap

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+LDAP_BINDDN=${LDAP_BINDDN:-cn=reva,ou=sysusers,dc=example,dc=org}
+LDAP_BINDPW=${LDAP_BINDPW:-reva}
+
+echo "Waiting for EOS MGM"
+echo "until nc -z -w 3 $EOS_MGM_ALIAS 1094; do sleep 2; done;" > /wait-for-mgm
+chmod +x /wait-for-mgm
+
+/wait-for-mgm;
+echo "----- [ocis] LDAP setup -----";
+authconfig --enableldap --enableldapauth --ldapserver=${EOS_LDAP_HOST} --ldapbasedn="dc=example,dc=org" --update;
+sed -i "s/#binddn cn=.*/binddn ${LDAP_BINDDN}/" /etc/nslcd.conf
+sed -i "s/#bindpw .*/bindpw ${LDAP_BINDPW}/" /etc/nslcd.conf
+# start in debug mode;
+
+nslcd -d

ocis/docker/eos-ocis/Dockerfile

@@ -0,0 +1,37 @@
+FROM centos:7 as build
+ARG BRANCH=master
+
+ENV CGO_ENABLED=1
+ENV GOOS=linux
+RUN rpm --rebuilddb && yum -y install \
+    wget \
+    time \
+    make \
+    gcc \
+    git
+RUN wget -q https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz
+RUN mkdir -p /usr/local/bin
+RUN tar xf go1.14.2.linux-amd64.tar.gz -C /usr/local
+RUN ln -s /usr/local/go/bin/* /usr/local/bin
+RUN git clone https://github.com/owncloud/ocis.git
+RUN cd ocis && git checkout ${BRANCH}
+RUN cd ocis && git log -n 1
+RUN cd ocis/ocis && go build -v -o ../bin/ocis -toolexec '/usr/bin/time -v' ./cmd/ocis
+# RUN cd ocis && make generate bin/ocis
+
+# build user lookup test program
+RUN mkdir -p /build
+COPY user.go /build/user.go
+RUN cd /build && go build -o user user.go
+
+
+FROM owncloud/eos-base:4.6.5
+RUN mkdir -p /usr/local/bin
+COPY --from=build ocis/bin/ocis /usr/local/bin/
+RUN chmod +x /usr/local/bin/ocis
+
+COPY --from=build /build/user /usr/local/bin/user
+RUN chmod +x /usr/local/bin/user
+
+COPY setup /setup
+COPY entrypoint /entrypoint

ocis/docker/eos-ocis/README.md

@@ -0,0 +1,30 @@
+# Docker image for ocis running on eos
+
+Image is based on [owncloud/eos-base](https://hub.docker.com/r/owncloud/eos-base) from [eos-stack](https://github.com/owncloud-docker/eos-stack)
+
+## Build
+Build owncloud/ocis master branch
+```shell
+docker build -t owncloud/eos-ocis:latest .
+```
+
+Or build a certain branch / tag
+```shell
+docker build -t owncloud/eos-ocis:1.0.0 --build-arg BRANCH=v1.0.0./eos-ocis
+```
+
+## Publish
+```shell
+docker push owncloud/eos-ocis:latest
+```
+
+## Maintainer
+
+  * [Felix Böhm](https://github.com/felixboehm)
+
+## Disclaimer
+Use only for development or testing. Setup is not secured nor tested.
+
+## Example Usage
+
+See <https://github.com/owncloud-docker/compose-playground/tree/master/examples/eos-compose>

ocis/docker/eos-ocis/entrypoint

@@ -0,0 +1,41 @@
+#!/bin/sh
+
+# todo: log all ocis services to stdout
+
+echo "Waiting for EOS MGM"
+echo "until nc -z -w 3 $EOS_MGM_ALIAS 1094; do sleep 2; done;" > /wait-for-mgm
+chmod +x /wait-for-mgm
+mkdir -p /var/log/ocis
+
+(/wait-for-mgm; /setup > /var/log/ocis/setup.log; cat /var/log/ocis/setup.log ) &
+
+echo "----- [ocis] DOMAIN setup -----"
+echo "Domain / IP: $OCIS_DOMAIN"
+mkdir -p /etc/ocis
+sed -e "s|@IPADDR@|${OCIS_DOMAIN}|g" /config/identifier-registration.yml.tmpl > /etc/ocis/identifier-registration.yml
+
+# TODO do we still need to precreate this folder?
+mkdir -p /var/tmp/reva
+
+echo "----- [ocis] Starting oCIS -----"
+# todo start ocis as daemon not as root
+ocis reva-storage-home &
+ocis reva-storage-home-data &
+ocis reva-storage-eos & 
+ocis reva-storage-eos-data &
+ocis reva-storage-public-link &
+ocis micro &
+ocis glauth &
+ocis graph-explorer &
+ocis graph &
+ocis konnectd &
+ocis phoenix &
+ocis thumbnails &
+ocis webdav &
+ocis reva-auth-basic &
+ocis reva-auth-bearer &
+ocis reva-frontend &
+ocis reva-gateway &
+ocis reva-sharing &
+ocis reva-users &
+exec ocis proxy

ocis/docker/eos-ocis/setup

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -x
+
+echo "----- [ocis] LDAP setup -----"
+authconfig --enableldap --enableldapauth --ldapserver=${EOS_LDAP_HOST} --ldapbasedn="dc=example,dc=org" --update
+sed -i "s/#binddn cn=.*/binddn ${LDAP_BINDDN}/" /etc/nslcd.conf
+sed -i "s/#bindpw .*/bindpw ${LDAP_BINDPW}/" /etc/nslcd.conf
+# start in debug mode
+nslcd -d &
+
+# echo "----- [ocis] eos setup -----"
+# eos -r 0 0 -b vid set membership daemon -uids adm
+# eos -r 0 0 -b vid set membership daemon -gids adm
+# eos -r 0 0 -b vid set membership daemon +sudo
+# eos -r 0 0 -b vid set map -unix "<pwd>" vuid:0 vgid:0
+# # eos -r 0 0 -b vid add gateway ocis.testnet
+# eos -r 0 0 -b vid add gateway ocis
+
+# todo start ocis as daemon not as root
+# eos -r 0 0 -b vid set membership root -uids adm
+# eos -r 0 0 -b vid set membership root -gids adm
+# eos -r 0 0 -b vid set membership root +sudo

ocis/docker/eos-ocis/user.go

@@ -0,0 +1,13 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	gouser "os/user"
+)
+
+func main() {
+	username := os.Args[1]
+	user, err := gouser.Lookup(username)
+	fmt.Printf("User %#v %#v %#v\n", username, user, err)
+}