Initial QSfera import

This commit is contained in:
Курнат Андрей
2026-06-07 10:20:04 +03:00
commit 2315f25754
16485 changed files with 4826827 additions and 0 deletions
+661
View File
@@ -0,0 +1,661 @@
package runtime
import (
"archive/tar"
"compress/gzip"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/nxadm/tail"
log "go-micro.dev/v4/logger"
"go-micro.dev/v4/runtime/local/git"
)
// defaultNamespace to use if not provided as an option.
const defaultNamespace = "default"
type runtime struct {
// options configure runtime
options *Options
// used to stop the runtime
closed chan bool
// used to start new services
start chan *service
// namespaces stores services grouped by namespace, e.g. namespaces["foo"]["go.micro.auth:latest"]
// would return the latest version of go.micro.auth from the foo namespace
namespaces map[string]map[string]*service
sync.RWMutex
// indicates if we're running
running bool
}
// NewRuntime creates new local runtime and returns it.
func NewRuntime(opts ...Option) Runtime {
// get default options
options := NewOptions(opts...)
// make the logs directory
path := filepath.Join(os.TempDir(), "micro", "logs")
_ = os.MkdirAll(path, 0755)
return &runtime{
options: options,
closed: make(chan bool),
start: make(chan *service, 128),
namespaces: make(map[string]map[string]*service),
}
}
// @todo move this to runtime default.
func (r *runtime) checkoutSourceIfNeeded(s *Service) error {
// Runtime service like config have no source.
// Skip checkout in that case
if len(s.Source) == 0 {
return nil
}
// @todo make this come from config
cpath := filepath.Join(os.TempDir(), "micro", "uploads", s.Source)
path := strings.ReplaceAll(cpath, ".tar.gz", "")
if ex, _ := exists(cpath); ex {
err := os.RemoveAll(path)
if err != nil {
return err
}
err = os.MkdirAll(path, 0777)
if err != nil {
return err
}
err = uncompress(cpath, path)
if err != nil {
return err
}
s.Source = path
return nil
}
source, err := git.ParseSourceLocal("", s.Source)
if err != nil {
return err
}
source.Ref = s.Version
err = git.CheckoutSource(os.TempDir(), source)
if err != nil {
return err
}
s.Source = source.FullPath
return nil
}
// modified version of: https://gist.github.com/mimoo/25fc9716e0f1353791f5908f94d6e726
func uncompress(src string, dst string) error {
file, err := os.OpenFile(src, os.O_RDWR|os.O_CREATE, 0666)
defer file.Close()
if err != nil {
return err
}
// ungzip
zr, err := gzip.NewReader(file)
if err != nil {
return err
}
// untar
tr := tar.NewReader(zr)
// uncompress each element
for {
header, err := tr.Next()
if err == io.EOF {
break // End of archive
}
if err != nil {
return err
}
target := header.Name
// validate name against path traversal
if !validRelPath(header.Name) {
return fmt.Errorf("tar contained invalid name error %q\n", target)
}
// add dst + re-format slashes according to system
target = filepath.Join(dst, header.Name)
// if no join is needed, replace with ToSlash:
// target = filepath.ToSlash(header.Name)
// check the type
switch header.Typeflag {
// if its a dir and it doesn't exist create it (with 0755 permission)
case tar.TypeDir:
if _, err := os.Stat(target); err != nil {
// @todo think about this:
// if we don't nuke the folder, we might end up with files from
// the previous decompress.
if err := os.MkdirAll(target, 0755); err != nil {
return err
}
}
// if it's a file create it (with same permission)
case tar.TypeReg:
// the truncating is probably unnecessary due to the `RemoveAll` of folders
// above
fileToWrite, err := os.OpenFile(target, os.O_TRUNC|os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
if err != nil {
return err
}
// copy over contents
if _, err := io.Copy(fileToWrite, tr); err != nil {
return err
}
// manually close here after each file operation; defering would cause each file close
// to wait until all operations have completed.
fileToWrite.Close()
}
}
return nil
}
// check for path traversal and correct forward slashes.
func validRelPath(p string) bool {
if p == "" || strings.Contains(p, `\`) || strings.HasPrefix(p, "/") || strings.Contains(p, "../") {
return false
}
return true
}
// Init initializes runtime options.
func (r *runtime) Init(opts ...Option) error {
r.Lock()
defer r.Unlock()
for _, o := range opts {
o(r.options)
}
return nil
}
// run runs the runtime management loop.
func (r *runtime) run(events <-chan Event) {
t := time.NewTicker(time.Second * 5)
defer t.Stop()
logger := r.options.Logger
// process event processes an incoming event
processEvent := func(event Event, service *service, ns string) error {
// get current vals
r.RLock()
name := service.Name
updated := service.updated
r.RUnlock()
// only process if the timestamp is newer
if !event.Timestamp.After(updated) {
return nil
}
logger.Logf(log.DebugLevel, "Runtime updating service %s in %v namespace", name, ns)
// this will cause a delete followed by created
if err := r.Update(service.Service, UpdateNamespace(ns)); err != nil {
return err
}
// update the local timestamp
r.Lock()
service.updated = updated
r.Unlock()
return nil
}
for {
select {
case <-t.C:
// check running services
r.RLock()
for _, sevices := range r.namespaces {
for _, service := range sevices {
if !service.ShouldStart() {
continue
}
// TODO: check service error
logger.Logf(log.DebugLevel, "Runtime starting %s", service.Name)
if err := service.Start(); err != nil {
logger.Logf(log.DebugLevel, "Runtime error starting %s: %v", service.Name, err)
}
}
}
r.RUnlock()
case service := <-r.start:
if !service.ShouldStart() {
continue
}
// TODO: check service error
logger.Logf(log.DebugLevel, "Runtime starting service %s", service.Name)
if err := service.Start(); err != nil {
logger.Logf(log.DebugLevel, "Runtime error starting service %s: %v", service.Name, err)
}
case event := <-events:
logger.Logf(log.DebugLevel, "Runtime received notification event: %v", event)
// NOTE: we only handle Update events for now
switch event.Type {
case Update:
if event.Service != nil {
ns := defaultNamespace
if event.Options != nil && len(event.Options.Namespace) > 0 {
ns = event.Options.Namespace
}
r.RLock()
if _, ok := r.namespaces[ns]; !ok {
logger.Logf(log.DebugLevel, "Runtime unknown namespace: %s", ns)
r.RUnlock()
continue
}
service, ok := r.namespaces[ns][fmt.Sprintf("%v:%v", event.Service.Name, event.Service.Version)]
r.RUnlock()
if !ok {
logger.Logf(log.DebugLevel, "Runtime unknown service: %s", event.Service)
}
if err := processEvent(event, service, ns); err != nil {
logger.Logf(log.DebugLevel, "Runtime error updating service %s: %v", event.Service, err)
}
continue
}
r.RLock()
namespaces := r.namespaces
r.RUnlock()
// if blank service was received we update all services
for ns, services := range namespaces {
for _, service := range services {
if err := processEvent(event, service, ns); err != nil {
logger.Logf(log.DebugLevel, "Runtime error updating service %s: %v", service.Name, err)
}
}
}
}
case <-r.closed:
logger.Logf(log.DebugLevel, "Runtime stopped")
return
}
}
}
func logFile(serviceName string) string {
// make the directory
name := strings.Replace(serviceName, "/", "-", -1)
path := filepath.Join(os.TempDir(), "micro", "logs")
return filepath.Join(path, fmt.Sprintf("%v.log", name))
}
func serviceKey(s *Service) string {
return fmt.Sprintf("%v:%v", s.Name, s.Version)
}
// Create creates a new service which is then started by runtime.
func (r *runtime) Create(s *Service, opts ...CreateOption) error {
err := r.checkoutSourceIfNeeded(s)
if err != nil {
return err
}
r.Lock()
defer r.Unlock()
var options CreateOptions
for _, o := range opts {
o(&options)
}
if len(options.Namespace) == 0 {
options.Namespace = defaultNamespace
}
if len(options.Command) == 0 {
options.Command = []string{"go"}
options.Args = []string{"run", "."}
}
if _, ok := r.namespaces[options.Namespace]; !ok {
r.namespaces[options.Namespace] = make(map[string]*service)
}
if _, ok := r.namespaces[options.Namespace][serviceKey(s)]; ok {
return errors.New("service already running")
}
// create new service
service := newService(s, options, r.options.Logger)
f, err := os.OpenFile(logFile(service.Name), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
r.options.Logger.Log(log.FatalLevel, err)
}
if service.output != nil {
service.output = io.MultiWriter(service.output, f)
} else {
service.output = f
}
// start the service
if err := service.Start(); err != nil {
return err
}
// save service
r.namespaces[options.Namespace][serviceKey(s)] = service
return nil
}
// exists returns whether the given file or directory exists.
func exists(path string) (bool, error) {
_, err := os.Stat(path)
if err == nil {
return true, nil
}
if os.IsNotExist(err) {
return false, nil
}
return true, err
}
// @todo: Getting existing lines is not supported yet.
// The reason for this is because it's hard to calculate line offset
// as opposed to character offset.
// This logger streams by default and only supports the `StreamCount` option.
func (r *runtime) Logs(s *Service, options ...LogsOption) (LogStream, error) {
lopts := LogsOptions{}
for _, o := range options {
o(&lopts)
}
ret := &logStream{
service: s.Name,
stream: make(chan LogRecord),
stop: make(chan bool),
logger: r.options.Logger,
}
fpath := logFile(s.Name)
if ex, err := exists(fpath); err != nil {
return nil, err
} else if !ex {
return nil, fmt.Errorf("Log file %v does not exists", fpath)
}
// have to check file size to avoid too big of a seek
fi, err := os.Stat(fpath)
if err != nil {
return nil, err
}
size := fi.Size()
whence := 2
// Multiply by length of an average line of log in bytes
offset := lopts.Count * 200
if offset > size {
offset = size
}
offset *= -1
t, err := tail.TailFile(fpath, tail.Config{Follow: lopts.Stream, Location: &tail.SeekInfo{
Whence: whence,
Offset: int64(offset),
}, Logger: tail.DiscardingLogger})
if err != nil {
return nil, err
}
ret.tail = t
go func() {
for {
select {
case line, ok := <-t.Lines:
if !ok {
ret.Stop()
return
}
ret.stream <- LogRecord{Message: line.Text}
case <-ret.stop:
return
}
}
}()
return ret, nil
}
type logStream struct {
err error
logger log.Logger
tail *tail.Tail
stream chan LogRecord
stop chan bool
service string
sync.Mutex
}
func (l *logStream) Chan() chan LogRecord {
return l.stream
}
func (l *logStream) Error() error {
return l.err
}
func (l *logStream) Stop() error {
l.Lock()
defer l.Unlock()
select {
case <-l.stop:
return nil
default:
close(l.stop)
close(l.stream)
err := l.tail.Stop()
if err != nil {
l.logger.Logf(log.ErrorLevel, "Error stopping tail: %v", err)
return err
}
}
return nil
}
// Read returns all instances of requested service
// If no service name is provided we return all the track services.
func (r *runtime) Read(opts ...ReadOption) ([]*Service, error) {
r.Lock()
defer r.Unlock()
gopts := ReadOptions{}
for _, o := range opts {
o(&gopts)
}
if len(gopts.Namespace) == 0 {
gopts.Namespace = defaultNamespace
}
save := func(k, v string) bool {
if len(k) == 0 {
return true
}
return k == v
}
//nolint:prealloc
var services []*Service
if _, ok := r.namespaces[gopts.Namespace]; !ok {
return make([]*Service, 0), nil
}
for _, service := range r.namespaces[gopts.Namespace] {
if !save(gopts.Service, service.Name) {
continue
}
if !save(gopts.Version, service.Version) {
continue
}
// TODO deal with service type
// no version has sbeen requested, just append the service
services = append(services, service.Service)
}
return services, nil
}
// Update attempts to update the service.
func (r *runtime) Update(s *Service, opts ...UpdateOption) error {
var options UpdateOptions
for _, o := range opts {
o(&options)
}
if len(options.Namespace) == 0 {
options.Namespace = defaultNamespace
}
err := r.checkoutSourceIfNeeded(s)
if err != nil {
return err
}
r.Lock()
srvs, ok := r.namespaces[options.Namespace]
r.Unlock()
if !ok {
return errors.New("Service not found")
}
r.Lock()
service, ok := srvs[serviceKey(s)]
r.Unlock()
if !ok {
return errors.New("Service not found")
}
if err := service.Stop(); err != nil && err.Error() != "no such process" {
r.options.Logger.Logf(log.ErrorLevel, "Error stopping service %s: %s", service.Name, err)
return err
}
return service.Start()
}
// Delete removes the service from the runtime and stops it.
func (r *runtime) Delete(s *Service, opts ...DeleteOption) error {
r.Lock()
defer r.Unlock()
var options DeleteOptions
for _, o := range opts {
o(&options)
}
if len(options.Namespace) == 0 {
options.Namespace = defaultNamespace
}
srvs, ok := r.namespaces[options.Namespace]
if !ok {
return nil
}
r.options.Logger.Logf(log.DebugLevel, "Runtime deleting service %s", s.Name)
service, ok := srvs[serviceKey(s)]
if !ok {
return nil
}
// check if running
if !service.Running() {
delete(srvs, service.key())
r.namespaces[options.Namespace] = srvs
return nil
}
// otherwise stop it
if err := service.Stop(); err != nil {
return err
}
// delete it
delete(srvs, service.key())
r.namespaces[options.Namespace] = srvs
return nil
}
// Start starts the runtime.
func (r *runtime) Start() error {
r.Lock()
defer r.Unlock()
// already running
if r.running {
return nil
}
// set running
r.running = true
r.closed = make(chan bool)
var events <-chan Event
if r.options.Scheduler != nil {
var err error
events, err = r.options.Scheduler.Notify()
if err != nil {
// TODO: should we bail here?
r.options.Logger.Logf(log.DebugLevel, "Runtime failed to start update notifier")
}
}
go r.run(events)
return nil
}
// Stop stops the runtime.
func (r *runtime) Stop() error {
r.Lock()
defer r.Unlock()
if !r.running {
return nil
}
select {
case <-r.closed:
return nil
default:
close(r.closed)
// set not running
r.running = false
// stop all the services
for _, services := range r.namespaces {
for _, service := range services {
r.options.Logger.Logf(log.DebugLevel, "Runtime stopping %s", service.Name)
service.Stop()
}
}
// stop the scheduler
if r.options.Scheduler != nil {
return r.options.Scheduler.Close()
}
}
return nil
}
// String implements stringer interface.
func (r *runtime) String() string {
return "local"
}
@@ -0,0 +1,34 @@
// Package build builds a micro runtime package
package build
import (
"go-micro.dev/v4/runtime/local/source"
)
// Builder builds binaries.
type Builder interface {
// Build builds a package
Build(*Source) (*Package, error)
// Clean deletes the package
Clean(*Package) error
}
// Source is the source of a build.
type Source struct {
// Location of the source
Repository *source.Repository
// Language is the language of code
Language string
}
// Package is micro service package.
type Package struct {
// Source of the binary
Source *Source
// Name of the binary
Name string
// Location of the binary
Path string
// Type of binary
Type string
}
@@ -0,0 +1,15 @@
package build
type Options struct {
// local path to download source
Path string
}
type Option func(o *Options)
// Local path for repository.
func Path(p string) Option {
return func(o *Options) {
o.Path = p
}
}
+341
View File
@@ -0,0 +1,341 @@
package git
import (
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"regexp"
"strings"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/plumbing"
)
type Gitter interface {
Clone(repo string) error
FetchAll(repo string) error
Checkout(repo, branchOrCommit string) error
RepoDir(repo string) string
}
type libGitter struct {
folder string
}
func (g libGitter) Clone(repo string) error {
fold := filepath.Join(g.folder, dirifyRepo(repo))
exists, err := pathExists(fold)
if err != nil {
return err
}
if exists {
return nil
}
_, err = git.PlainClone(fold, false, &git.CloneOptions{
URL: repo,
Progress: os.Stdout,
})
return err
}
func (g libGitter) FetchAll(repo string) error {
repos, err := git.PlainOpen(filepath.Join(g.folder, dirifyRepo(repo)))
if err != nil {
return err
}
remotes, err := repos.Remotes()
if err != nil {
return err
}
err = remotes[0].Fetch(&git.FetchOptions{
RefSpecs: []config.RefSpec{"refs/*:refs/*", "HEAD:refs/heads/HEAD"},
Progress: os.Stdout,
Depth: 1,
})
if err != nil && err != git.NoErrAlreadyUpToDate {
return err
}
return nil
}
func (g libGitter) Checkout(repo, branchOrCommit string) error {
if branchOrCommit == "latest" {
branchOrCommit = "master"
}
repos, err := git.PlainOpen(filepath.Join(g.folder, dirifyRepo(repo)))
if err != nil {
return err
}
worktree, err := repos.Worktree()
if err != nil {
return err
}
if plumbing.IsHash(branchOrCommit) {
return worktree.Checkout(&git.CheckoutOptions{
Hash: plumbing.NewHash(branchOrCommit),
Force: true,
})
}
return worktree.Checkout(&git.CheckoutOptions{
Branch: plumbing.NewBranchReferenceName(branchOrCommit),
Force: true,
})
}
func (g libGitter) RepoDir(repo string) string {
return filepath.Join(g.folder, dirifyRepo(repo))
}
type binaryGitter struct {
folder string
}
func (g binaryGitter) Clone(repo string) error {
fold := filepath.Join(g.folder, dirifyRepo(repo), ".git")
exists, err := pathExists(fold)
if err != nil {
return err
}
if exists {
return nil
}
fold = filepath.Join(g.folder, dirifyRepo(repo))
cmd := exec.Command("git", "clone", repo, ".")
err = os.MkdirAll(fold, 0777)
if err != nil {
return err
}
cmd.Dir = fold
_, err = cmd.Output()
if err != nil {
return err
}
return err
}
func (g binaryGitter) FetchAll(repo string) error {
cmd := exec.Command("git", "fetch", "--all")
cmd.Dir = filepath.Join(g.folder, dirifyRepo(repo))
outp, err := cmd.CombinedOutput()
if err != nil {
return errors.New(string(outp))
}
return err
}
func (g binaryGitter) Checkout(repo, branchOrCommit string) error {
if branchOrCommit == "latest" {
branchOrCommit = "master"
}
cmd := exec.Command("git", "checkout", "-f", branchOrCommit)
cmd.Dir = filepath.Join(g.folder, dirifyRepo(repo))
outp, err := cmd.CombinedOutput()
if err != nil {
return errors.New(string(outp))
}
return nil
}
func (g binaryGitter) RepoDir(repo string) string {
return filepath.Join(g.folder, dirifyRepo(repo))
}
func NewGitter(folder string) Gitter {
if commandExists("git") {
return binaryGitter{folder}
}
return libGitter{folder}
}
func commandExists(cmd string) bool {
_, err := exec.LookPath(cmd)
return err == nil
}
func dirifyRepo(s string) string {
s = strings.ReplaceAll(s, "https://", "")
s = strings.ReplaceAll(s, "/", "-")
return s
}
// exists returns whether the given file or directory exists.
func pathExists(path string) (bool, error) {
_, err := os.Stat(path)
if err == nil {
return true, nil
}
if os.IsNotExist(err) {
return false, nil
}
return true, err
}
// GetRepoRoot determines the repo root from a full path.
// Returns empty string and no error if not found.
func GetRepoRoot(fullPath string) (string, error) {
// traverse parent directories
prev := fullPath
for {
current := prev
exists, err := pathExists(filepath.Join(current, ".git"))
if err != nil {
return "", err
}
if exists {
return current, nil
}
prev = filepath.Dir(current)
// reached top level, see:
// https://play.golang.org/p/rDgVdk3suzb
if current == prev {
break
}
}
return "", nil
}
const defaultRepo = "github.com/micro/services"
// Source is not just git related @todo move.
type Source struct {
// absolute path to service folder in local mode
FullPath string
// path of folder to repo root
// be it local or github repo
Folder string
// github ref
Ref string
// for cloning purposes
// blank for local
Repo string
// dir to repo root
// blank for non local
LocalRepoRoot string
// is it a local folder intended for a local runtime?
Local bool
}
// Name to be passed to RPC call runtime.Create Update Delete
// eg: `helloworld/api`, `crufter/myrepo/helloworld/api`, `localfolder`
func (s *Source) RuntimeName() string {
if s.Repo == "github.com/micro/services" || s.Repo == "" {
return s.Folder
}
return fmt.Sprintf("%v/%v", strings.ReplaceAll(s.Repo, "github.com/", ""), s.Folder)
}
// Source to be passed to RPC call runtime.Create Update Delete
// eg: `helloworld`, `github.com/crufter/myrepo/helloworld`, `/path/to/localrepo/localfolder`
func (s *Source) RuntimeSource() string {
if s.Local {
return s.FullPath
}
if s.Repo == "github.com/micro/services" || s.Repo == "" {
return s.Folder
}
return fmt.Sprintf("%v/%v", s.Repo, s.Folder)
}
// ParseSource parses a `micro run/update/kill` source.
func ParseSource(source string) (*Source, error) {
// If github is not present, we got a shorthand for `micro/services`
if !strings.Contains(source, "github.com") {
source = "github.com/micro/services/" + source
}
if !strings.Contains(source, "@") {
source += "@latest"
}
ret := &Source{}
refs := strings.Split(source, "@")
ret.Ref = refs[1]
parts := strings.Split(refs[0], "/")
ret.Repo = strings.Join(parts[0:3], "/")
if len(parts) > 1 {
ret.Folder = strings.Join(parts[3:], "/")
}
return ret, nil
}
// ParseSourceLocal detects and handles local pathes too
// workdir should be used only from the CLI @todo better interface for this function.
// PathExistsFunc exists only for testing purposes, to make the function side effect free.
func ParseSourceLocal(workDir, source string, pathExistsFunc ...func(path string) (bool, error)) (*Source, error) {
var pexists func(string) (bool, error)
if len(pathExistsFunc) == 0 {
pexists = pathExists
} else {
pexists = pathExistsFunc[0]
}
var localFullPath string
if len(workDir) > 0 {
localFullPath = filepath.Join(workDir, source)
} else {
localFullPath = source
}
if exists, err := pexists(localFullPath); err == nil && exists {
localRepoRoot, err := GetRepoRoot(localFullPath)
if err != nil {
return nil, err
}
var folder string
// If the local repo root is a top level folder, we are not in a git repo.
// In this case, we should take the last folder as folder name.
if localRepoRoot == "" {
folder = filepath.Base(localFullPath)
} else {
folder = strings.ReplaceAll(localFullPath, localRepoRoot+string(filepath.Separator), "")
}
return &Source{
Local: true,
Folder: folder,
FullPath: localFullPath,
LocalRepoRoot: localRepoRoot,
Ref: "latest", // @todo consider extracting branch from git here
}, nil
}
return ParseSource(source)
}
// CheckoutSource for the local runtime server
// folder is the folder to check out the source code to
// Modifies source path to set it to checked out repo absolute path locally.
func CheckoutSource(folder string, source *Source) error {
// if it's a local folder, do nothing
if exists, err := pathExists(source.FullPath); err == nil && exists {
return nil
}
gitter := NewGitter(folder)
repo := source.Repo
if !strings.Contains(repo, "https://") {
repo = "https://" + repo
}
// Always clone, it's idempotent and only clones if needed
err := gitter.Clone(repo)
if err != nil {
return err
}
source.FullPath = filepath.Join(gitter.RepoDir(source.Repo), source.Folder)
return gitter.Checkout(repo, source.Ref)
}
// code below is not used yet
var nameExtractRegexp = regexp.MustCompile(`((micro|web)\.Name\(")(.*)("\))`)
func extractServiceName(fileContent []byte) string {
hits := nameExtractRegexp.FindAll(fileContent, 1)
if len(hits) == 0 {
return ""
}
hit := string(hits[0])
return strings.Split(hit, "\"")[1]
}
@@ -0,0 +1,5 @@
package process
type Options struct{}
type Option func(o *Options)
@@ -0,0 +1,95 @@
//go:build !windows
// +build !windows
// Package os runs processes locally
package os
import (
"fmt"
"os"
"os/exec"
"strconv"
"syscall"
"go-micro.dev/v4/runtime/local/process"
)
func (p *Process) Exec(exe *process.Executable) error {
cmd := exec.Command(exe.Package.Path)
cmd.Dir = exe.Dir
return cmd.Run()
}
func (p *Process) Fork(exe *process.Executable) (*process.PID, error) {
// create command
cmd := exec.Command(exe.Package.Path, exe.Args...)
cmd.Dir = exe.Dir
// set env vars
cmd.Env = append(cmd.Env, os.Environ()...)
cmd.Env = append(cmd.Env, exe.Env...)
// create process group
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
in, err := cmd.StdinPipe()
if err != nil {
return nil, err
}
out, err := cmd.StdoutPipe()
if err != nil {
return nil, err
}
er, err := cmd.StderrPipe()
if err != nil {
return nil, err
}
// start the process
if err := cmd.Start(); err != nil {
return nil, err
}
return &process.PID{
ID: fmt.Sprintf("%d", cmd.Process.Pid),
Input: in,
Output: out,
Error: er,
}, nil
}
func (p *Process) Kill(pid *process.PID) error {
id, err := strconv.Atoi(pid.ID)
if err != nil {
return err
}
if _, err := os.FindProcess(id); err != nil {
return err
}
// now kill it
// using -ve PID kills the process group which we created in Fork()
return syscall.Kill(-id, syscall.SIGTERM)
}
func (p *Process) Wait(pid *process.PID) error {
id, err := strconv.Atoi(pid.ID)
if err != nil {
return err
}
pr, err := os.FindProcess(id)
if err != nil {
return err
}
ps, err := pr.Wait()
if err != nil {
return err
}
if ps.Success() {
return nil
}
return fmt.Errorf(ps.String())
}
@@ -0,0 +1,89 @@
// Package os runs processes locally
package os
import (
"fmt"
"os"
"os/exec"
"strconv"
"go-micro.dev/v4/runtime/local/process"
)
func (p *Process) Exec(exe *process.Executable) error {
cmd := exec.Command(exe.Package.Path)
return cmd.Run()
}
func (p *Process) Fork(exe *process.Executable) (*process.PID, error) {
// create command
cmd := exec.Command(exe.Package.Path, exe.Args...)
// set env vars
cmd.Env = append(cmd.Env, exe.Env...)
in, err := cmd.StdinPipe()
if err != nil {
return nil, err
}
out, err := cmd.StdoutPipe()
if err != nil {
return nil, err
}
er, err := cmd.StderrPipe()
if err != nil {
return nil, err
}
// start the process
if err := cmd.Start(); err != nil {
return nil, err
}
return &process.PID{
ID: fmt.Sprintf("%d", cmd.Process.Pid),
Input: in,
Output: out,
Error: er,
}, nil
}
func (p *Process) Kill(pid *process.PID) error {
id, err := strconv.Atoi(pid.ID)
if err != nil {
return err
}
pr, err := os.FindProcess(id)
if err != nil {
return err
}
// now kill it
err = pr.Kill()
// return the kill error
return err
}
func (p *Process) Wait(pid *process.PID) error {
id, err := strconv.Atoi(pid.ID)
if err != nil {
return err
}
pr, err := os.FindProcess(id)
if err != nil {
return err
}
ps, err := pr.Wait()
if err != nil {
return err
}
if ps.Success() {
return nil
}
return fmt.Errorf(ps.String())
}
@@ -0,0 +1,12 @@
// Package os runs processes locally
package os
import (
"go-micro.dev/v4/runtime/local/process"
)
type Process struct{}
func NewProcess(opts ...process.Option) process.Process {
return &Process{}
}
@@ -0,0 +1,43 @@
// Package process executes a binary
package process
import (
"io"
"go-micro.dev/v4/runtime/local/build"
)
// Process manages a running process.
type Process interface {
// Executes a process to completion
Exec(*Executable) error
// Creates a new process
Fork(*Executable) (*PID, error)
// Kills the process
Kill(*PID) error
// Waits for a process to exit
Wait(*PID) error
}
type Executable struct {
// Package containing executable
Package *build.Package
// Initial working directory
Dir string
// The env variables
Env []string
// Args to pass
Args []string
}
// PID is the running process.
type PID struct {
// Stdin
Input io.Writer
// Stdout
Output io.Reader
// Stderr
Error io.Reader
// ID of the process
ID string
}
@@ -0,0 +1,15 @@
package source
type Options struct {
// local path to download source
Path string
}
type Option func(o *Options)
// Local path for repository.
func Path(p string) Option {
return func(o *Options) {
o.Path = p
}
}
@@ -0,0 +1,22 @@
// Package source retrieves source code
package source
// Source retrieves source code.
type Source interface {
// Fetch repo from a url
Fetch(url string) (*Repository, error)
// Commit and upload repo
Commit(*Repository) error
// The sourcerer
String() string
}
// Repository is the source repository.
type Repository struct {
// Name or repo
Name string
// Local path where repo is stored
Path string
// URL from which repo was retrieved
URL string
}
+309
View File
@@ -0,0 +1,309 @@
package runtime
import (
"context"
"io"
"go-micro.dev/v4/client"
"go-micro.dev/v4/logger"
)
type Option func(o *Options)
// Options configure runtime.
type Options struct {
// Scheduler for updates
Scheduler Scheduler
// Client to use when making requests
Client client.Client
// Logger underline logger
Logger logger.Logger
// Service type to manage
Type string
// Source of the services repository
Source string
// Base image to use
Image string
}
func NewOptions(opts ...Option) *Options {
options := &Options{
Logger: logger.DefaultLogger,
}
for _, o := range opts {
o(options)
}
return options
}
// WithSource sets the base image / repository.
func WithSource(src string) Option {
return func(o *Options) {
o.Source = src
}
}
// WithScheduler specifies a scheduler for updates.
func WithScheduler(n Scheduler) Option {
return func(o *Options) {
o.Scheduler = n
}
}
// WithType sets the service type to manage.
func WithType(t string) Option {
return func(o *Options) {
o.Type = t
}
}
// WithImage sets the image to use.
func WithImage(t string) Option {
return func(o *Options) {
o.Image = t
}
}
// WithClient sets the client to use.
func WithClient(c client.Client) Option {
return func(o *Options) {
o.Client = c
}
}
// WithLogger sets the underline logger.
func WithLogger(l logger.Logger) Option {
return func(o *Options) {
o.Logger = l
}
}
type CreateOption func(o *CreateOptions)
type ReadOption func(o *ReadOptions)
// CreateOptions configure runtime services.
type CreateOptions struct {
// Log output
Output io.Writer
// Specify the context to use
Context context.Context
// Type of service to create
Type string
// Specify the image to use
Image string
// Namespace to create the service in
Namespace string
// Command to execut
Command []string
// Args to pass into command
Args []string
// Environment to configure
Env []string
// Retries before failing deploy
Retries int
}
// ReadOptions queries runtime services.
type ReadOptions struct {
// Specify the context to use
Context context.Context
// Service name
Service string
// Version queries services with given version
Version string
// Type of service
Type string
// Namespace the service is running in
Namespace string
}
// CreateType sets the type of service to create.
func CreateType(t string) CreateOption {
return func(o *CreateOptions) {
o.Type = t
}
}
// CreateImage sets the image to use.
func CreateImage(img string) CreateOption {
return func(o *CreateOptions) {
o.Image = img
}
}
// CreateNamespace sets the namespace.
func CreateNamespace(ns string) CreateOption {
return func(o *CreateOptions) {
o.Namespace = ns
}
}
// CreateContext sets the context.
func CreateContext(ctx context.Context) CreateOption {
return func(o *CreateOptions) {
o.Context = ctx
}
}
// WithCommand specifies the command to execute.
func WithCommand(cmd ...string) CreateOption {
return func(o *CreateOptions) {
// set command
o.Command = cmd
}
}
// WithArgs specifies the command to execute.
func WithArgs(args ...string) CreateOption {
return func(o *CreateOptions) {
// set command
o.Args = args
}
}
func WithRetries(retries int) CreateOption {
return func(o *CreateOptions) {
o.Retries = retries
}
}
// WithEnv sets the created service environment.
func WithEnv(env []string) CreateOption {
return func(o *CreateOptions) {
o.Env = env
}
}
// WithOutput sets the arg output.
func WithOutput(out io.Writer) CreateOption {
return func(o *CreateOptions) {
o.Output = out
}
}
// ReadService returns services with the given name.
func ReadService(service string) ReadOption {
return func(o *ReadOptions) {
o.Service = service
}
}
// ReadVersion confifgures service version.
func ReadVersion(version string) ReadOption {
return func(o *ReadOptions) {
o.Version = version
}
}
// ReadType returns services of the given type.
func ReadType(t string) ReadOption {
return func(o *ReadOptions) {
o.Type = t
}
}
// ReadNamespace sets the namespace.
func ReadNamespace(ns string) ReadOption {
return func(o *ReadOptions) {
o.Namespace = ns
}
}
// ReadContext sets the context.
func ReadContext(ctx context.Context) ReadOption {
return func(o *ReadOptions) {
o.Context = ctx
}
}
type UpdateOption func(o *UpdateOptions)
type UpdateOptions struct {
// Specify the context to use
Context context.Context
// Namespace the service is running in
Namespace string
}
// UpdateNamespace sets the namespace.
func UpdateNamespace(ns string) UpdateOption {
return func(o *UpdateOptions) {
o.Namespace = ns
}
}
// UpdateContext sets the context.
func UpdateContext(ctx context.Context) UpdateOption {
return func(o *UpdateOptions) {
o.Context = ctx
}
}
type DeleteOption func(o *DeleteOptions)
type DeleteOptions struct {
// Specify the context to use
Context context.Context
// Namespace the service is running in
Namespace string
}
// DeleteNamespace sets the namespace.
func DeleteNamespace(ns string) DeleteOption {
return func(o *DeleteOptions) {
o.Namespace = ns
}
}
// DeleteContext sets the context.
func DeleteContext(ctx context.Context) DeleteOption {
return func(o *DeleteOptions) {
o.Context = ctx
}
}
// LogsOption configures runtime logging.
type LogsOption func(o *LogsOptions)
// LogsOptions configure runtime logging.
type LogsOptions struct {
// Specify the context to use
Context context.Context
// Namespace the service is running in
Namespace string
// How many existing lines to show
Count int64
// Stream new lines?
Stream bool
}
// LogsExistingCount confiures how many existing lines to show.
func LogsCount(count int64) LogsOption {
return func(l *LogsOptions) {
l.Count = count
}
}
// LogsStream configures whether to stream new lines.
func LogsStream(stream bool) LogsOption {
return func(l *LogsOptions) {
l.Stream = stream
}
}
// LogsNamespace sets the namespace.
func LogsNamespace(ns string) LogsOption {
return func(o *LogsOptions) {
o.Namespace = ns
}
}
// LogsContext sets the context.
func LogsContext(ctx context.Context) LogsOption {
return func(o *LogsOptions) {
o.Context = ctx
}
}
+110
View File
@@ -0,0 +1,110 @@
// Package runtime is a service runtime manager
package runtime
import (
"errors"
"time"
)
var (
// DefaultRuntime is default micro runtime.
DefaultRuntime Runtime = NewRuntime()
// DefaultName is default runtime service name.
DefaultName = "go.micro.runtime"
ErrAlreadyExists = errors.New("already exists")
)
// Runtime is a service runtime manager.
type Runtime interface {
// Init initializes runtime
Init(...Option) error
// Create registers a service
Create(*Service, ...CreateOption) error
// Read returns the service
Read(...ReadOption) ([]*Service, error)
// Update the service in place
Update(*Service, ...UpdateOption) error
// Remove a service
Delete(*Service, ...DeleteOption) error
// Logs returns the logs for a service
Logs(*Service, ...LogsOption) (LogStream, error)
// Start starts the runtime
Start() error
// Stop shuts down the runtime
Stop() error
// String describes runtime
String() string
}
// Stream returns a log stream.
type LogStream interface {
Error() error
Chan() chan LogRecord
Stop() error
}
type LogRecord struct {
Metadata map[string]string
Message string
}
// Scheduler is a runtime service scheduler.
type Scheduler interface {
// Notify publishes schedule events
Notify() (<-chan Event, error)
// Close stops the scheduler
Close() error
}
// EventType defines schedule event.
type EventType int
const (
// Create is emitted when a new build has been craeted.
Create EventType = iota
// Update is emitted when a new update become available.
Update
// Delete is emitted when a build has been deleted.
Delete
)
// String returns human readable event type.
func (t EventType) String() string {
switch t {
case Create:
return "create"
case Delete:
return "delete"
case Update:
return "update"
default:
return "unknown"
}
}
// Event is notification event.
type Event struct {
// Timestamp is event timestamp
Timestamp time.Time
// Service the event relates to
Service *Service
// Options to use when processing the event
Options *CreateOptions
// ID of the event
ID string
// Type is event type
Type EventType
}
// Service is runtime service.
type Service struct {
// Metadata stores metadata
Metadata map[string]string
// Name of the service
Name string
// Version of the service
Version string
// url location of source
Source string
}
+230
View File
@@ -0,0 +1,230 @@
package runtime
import (
"fmt"
"io"
"strconv"
"strings"
"sync"
"time"
log "go-micro.dev/v4/logger"
"go-micro.dev/v4/runtime/local/build"
"go-micro.dev/v4/runtime/local/process"
proc "go-micro.dev/v4/runtime/local/process/os"
)
type service struct {
updated time.Time
// to be used logger
Logger log.Logger
// output for logs
output io.Writer
err error
// process creator
Process *proc.Process
closed chan bool
// service to manage
*Service
// Exec
Exec *process.Executable
// process pid
PID *process.PID
retries int
maxRetries int
sync.RWMutex
running bool
}
func newService(s *Service, c CreateOptions, l log.Logger) *service {
var exec string
var args []string
// set command
exec = strings.Join(c.Command, " ")
args = c.Args
return &service{
Service: s,
Process: new(proc.Process),
Exec: &process.Executable{
Package: &build.Package{
Name: s.Name,
Path: exec,
},
Env: c.Env,
Args: args,
Dir: s.Source,
},
Logger: log.LoggerOrDefault(l),
closed: make(chan bool),
output: c.Output,
updated: time.Now(),
maxRetries: c.Retries,
}
}
func (s *service) streamOutput() {
go io.Copy(s.output, s.PID.Output)
go io.Copy(s.output, s.PID.Error)
}
func (s *service) shouldStart() bool {
if s.running {
return false
}
return s.retries <= s.maxRetries
}
func (s *service) key() string {
return fmt.Sprintf("%v:%v", s.Name, s.Version)
}
func (s *service) ShouldStart() bool {
s.RLock()
defer s.RUnlock()
return s.shouldStart()
}
func (s *service) Running() bool {
s.RLock()
defer s.RUnlock()
return s.running
}
// Start starts the service.
func (s *service) Start() error {
s.Lock()
defer s.Unlock()
if !s.shouldStart() {
return nil
}
// reset
s.err = nil
s.closed = make(chan bool)
s.retries = 0
if s.Metadata == nil {
s.Metadata = make(map[string]string)
}
s.Status("starting", nil)
// TODO: pull source & build binary
s.Logger.Log(log.DebugLevel, "Runtime service %s forking new process", s.Service.Name)
p, err := s.Process.Fork(s.Exec)
if err != nil {
s.Status("error", err)
return err
}
// set the pid
s.PID = p
// set to running
s.running = true
// set status
s.Status("running", nil)
// set started
s.Metadata["started"] = time.Now().Format(time.RFC3339)
if s.output != nil {
s.streamOutput()
}
// wait and watch
go s.Wait()
return nil
}
// Status updates the status of the service. Assumes it's called under a lock as it mutates state.
func (s *service) Status(status string, err error) {
s.Metadata["lastStatusUpdate"] = time.Now().Format(time.RFC3339)
s.Metadata["status"] = status
if err == nil {
delete(s.Metadata, "error")
return
}
s.Metadata["error"] = err.Error()
}
// Stop stops the service.
func (s *service) Stop() error {
s.Lock()
defer s.Unlock()
select {
case <-s.closed:
return nil
default:
close(s.closed)
s.running = false
s.retries = 0
if s.PID == nil {
return nil
}
// set status
s.Status("stopping", nil)
// kill the process
err := s.Process.Kill(s.PID)
if err == nil {
// wait for it to exit
s.Process.Wait(s.PID)
}
// set status
s.Status("stopped", err)
// return the kill error
return err
}
}
// Error returns the last error service has returned.
func (s *service) Error() error {
s.RLock()
defer s.RUnlock()
return s.err
}
// Wait waits for the service to finish running.
func (s *service) Wait() {
// wait for process to exit
s.RLock()
thisPID := s.PID
s.RUnlock()
err := s.Process.Wait(thisPID)
s.Lock()
defer s.Unlock()
if s.PID.ID != thisPID.ID {
// trying to update when it's already been switched out, ignore
s.Logger.Logf(log.WarnLevel, "Trying to update a process status but PID doesn't match. Old %s, New %s. Skipping update.", thisPID.ID, s.PID.ID)
return
}
// save the error
if err != nil {
s.Logger.Logf(log.ErrorLevel, "Service %s terminated with error %s", s.Name, err)
s.retries++
s.Status("error", err)
s.Metadata["retries"] = strconv.Itoa(s.retries)
s.err = err
} else {
s.Status("done", nil)
}
// no longer running
s.running = false
}