Skip to content

tingweiwang / pipeline-test

Go to a project
Commit cdbcc85e by tingweiwang
Options

fix

0 parents
Inline Side-by-side
Showing with 6426 additions and 0 deletions
Dockerfile 0 → 100644
FROM hb.seetatech.com/k8s/ubuntu-basic:16.04
COPY core--* /adl/bin/
COPY conf/ /adl/bin/conf
COPY conf/config.json /root/.docker/config.json
ENV PATH /adl/bin:$PATH
WORKDIR /adl/bin/
\ No newline at end of file
Jenkinsfile 0 → 100644
def label = "mypod-${UUID.randomUUID().toString()}"
podTemplate(label: label, cloud: 'kubernetes') {
node('private') {
stage('Clone') {
echo "开始下载autodl项目代码"
git url: "https://tingweiwang@gitlab.seetatech.com/tingweiwang/jenkins-slave-test.git"
git credentialsId: 'd2625477-5e8f-4661-84cf-aefabf33bc2f' }
stage('go test') {
sh "go version"
}
stage('docker test') {
sh "docker version"
}
stage('k8s test') {
sh "kubectl get node"
sh "sleep 1000000"
sh "make redeploy_all mode=dev"
}
}
}
Makefile 0 → 100644
KUBENETES=kubectl
REGISTRY=harbor_host/core
VERSION=1.11
GOPROXY=export GO111MODULE=on && export GOPROXY=http://goproxy.seetatech.com
GOCMD=$(GOPROXY) && go
GORUN=$(GOCMD) run
DOCKER=docker
now:=$(shell date +%Y%m%d%H%M%S)
.PHONY: clean build redeploy
build: build_server build_worker build_collector build_monitor
redeploy_all: build docker_build_worker docker_push_worker docker_build_nginx docker_push_nginx docker_build_collector docker_push_collector rewrite_yaml delete_server create_server delete_worker create_worker delete_collecor create_collecor delete_monitor create_monitor delete_nginx create_nginx recover_yaml
update: build docker_build_worker docker_push_worker docker_build_nginx docker_push_nginx docker_build_collector docker_push_collector rewrite_yaml apply_server apply_worker apply_collecor apply_monitor apply_nginx recover_yaml
private_deploy: docker_build_worker docker_push_worker docker_build_nginx docker_push_nginx docker_build_collector docker_push_collector rewrite_yaml delete_server create_server delete_worker create_worker delete_collecor create_collecor delete_monitor create_monitor delete_nginx create_nginx recover_yaml
redeploy_monitor: build docker_build_worker docker_push_worker rewrite_yaml delete_monitor create_monitor recover_yaml
redeploy_collector: build docker_build_worker docker_push_worker rewrite_yaml delete_collecor create_collecor recover_yaml
redeploy_nginx: docker_build_nginx docker_push_nginx delete_nginx rewrite_yaml create_nginx recover_yaml
redeploy_log_stash: build_log_stash docker_build_log_stash docker_push_log_stash delete_log_stash rewrite_yaml create_log_stash recover_yaml
build_server:
$(GOCMD) build -o core--server ./cmd/server/main.go
build_worker:
$(GOCMD) build -o core--worker ./cmd/worker/main.go
build_collector:
$(GOCMD) build -o core--collector ./cmd/collector/main.go
build_monitor:
$(GOCMD) build -o core--monitor ./cmd/monitor/main.go
build_log_stash:
$(GOCMD) build -o log-stash-v1 ./cmd/log_stash/main.go
create_server:
@$(KUBENETES) create -f $(DeployYAMLPath)/server-deployment.yaml || $(call panic_and_recover_yaml)
@$(KUBENETES) create -f $(DeployYAMLPath)/server-svc.yaml || $(call panic_and_recover_yaml)
create_worker:
@$(KUBENETES) create -f $(DeployYAMLPath)/worker-deployment.yaml || $(call panic_and_recover_yaml)
create_collecor:
@$(KUBENETES) create -f $(DeployYAMLPath)/collector-deployment.yaml || $(call panic_and_recover_yaml)
create_monitor:
@$(KUBENETES) create -f $(DeployYAMLPath)/monitor-deployment.yaml || $(call panic_and_recover_yaml)
create_nginx:
@$(KUBENETES) create -f $(DeployYAMLPath)/nginx.yaml || $(call panic_and_recover_yaml)
@$(KUBENETES) create -f $(DeployYAMLPath)/nginx-svc.yaml || $(call panic_and_recover_yaml)
create_log_stash:
@$(KUBENETES) create -f $(DeployYAMLPath)/log-stash-deployment*.yaml || $(call panic_and_recover_yaml)
apply_server:
@$(KUBENETES) apply -f $(DeployYAMLPath)/server-deployment.yaml --record || $(call panic_and_recover_yaml)
apply_worker:
@$(KUBENETES) apply -f $(DeployYAMLPath)/worker-deployment.yaml --record || $(call panic_and_recover_yaml)
apply_collecor:
@$(KUBENETES) apply -f $(DeployYAMLPath)/collector-deployment.yaml --record || $(call panic_and_recover_yaml)
apply_nginx:
@$(KUBENETES) apply -f $(DeployYAMLPath)/nginx.yaml --record || $(call panic_and_recover_yaml)
apply_monitor:
@$(KUBENETES) apply -f $(DeployYAMLPath)/monitor-deployment.yaml --record || $(call panic_and_recover_yaml)
delete_server:
-@$(KUBENETES) delete -f $(DeployYAMLPath)/server-deployment.yaml
-@$(KUBENETES) delete -f $(DeployYAMLPath)/server-svc.yaml
delete_worker:
-@$(KUBENETES) delete -f $(DeployYAMLPath)/worker-deployment.yaml
delete_collecor:
-@$(KUBENETES) delete -f $(DeployYAMLPath)/collector-deployment.yaml
delete_monitor:
-@$(KUBENETES) delete -f $(DeployYAMLPath)/monitor-deployment.yaml
delete_nginx:
-@$(KUBENETES) delete -f $(DeployYAMLPath)/nginx.yaml
-@$(KUBENETES) delete -f $(DeployYAMLPath)/nginx-svc.yaml
delete_log_stash:
-@$(KUBENETES) delete -f $(DeployYAMLPath)/log-stash-deployment*.yaml
docker_build_worker:
@echo "build tag is: "$(now)
$(DOCKER) build -f Dockerfile -t $(REGISTRY)/adl-core-v1:$(now) .
docker_push_worker:
$(DOCKER) push $(REGISTRY)/adl-core-v1:$(now)
docker_build_nginx:
$(DOCKER) build -f cmd/nginx/Dockerfile -t $(REGISTRY)/core--nginx:$(now) cmd/nginx/
docker_push_nginx:
$(DOCKER) push $(REGISTRY)/core--nginx:$(now)
docker_build_collector:
$(DOCKER) build -f cmd/collector/Dockerfile -t $(REGISTRY)/core--collector:$(now) .
docker_push_collector:
$(DOCKER) push $(REGISTRY)/core--collector:$(now)
docker_build_log_stash:
$(DOCKER) build -f cmd/log_stash/Dockerfile -t $(REGISTRY)/log-stash-v1:$(now) .
docker_push_log_stash:
$(DOCKER) push $(REGISTRY)/log-stash-v1:$(now)
rewrite_yaml:
@sed -i "s#DOCKER_REGISTRY#$(REGISTRY)#g" `find $(DeployYAMLPath) -type f -name "*.yaml"`
@sed -i "s#latest#$(now)#g" `find $(DeployYAMLPath) -type f -name "*.yaml"`
recover_yaml:
@sed -i "s#$(now)#latest#g" `find $(DeployYAMLPath) -type f -name "*.yaml"`
@sed -i "s#$(REGISTRY)#DOCKER_REGISTRY#g" `find $(DeployYAMLPath) -type f -name "*.yaml"`
define panic_and_recover_yaml
(sed -i "s#$(now)#latest#g" `find $(DeployYAMLPath) -type f -name "*.yaml"` && sed -i "s#$(REGISTRY)#DOCKER_REGISTRY#g" `find $(DeployYAMLPath) -type f -name "*.yaml"` && exit 1)
endef
help:
$(info You should set the mode, e.g. `make {command} mode=dev` || `make {command} mode=release`)
$(info )
$(info )
clean:
-ls | grep core-- | xargs rm
-ls | grep "\."log | xargs rm
ifeq ($(mode),dev)
DeployYAMLPath=deploy/develop
else
ifeq ($(mode),release)
DeployYAMLPath=deploy/release
else
$(error You should set the mode, e.g. `make {command} mode=dev` || `make {command} mode=release`)
endif
endif
README.md 0 → 100644
## 部署方式
### 服务部署
1. 修改Makefile中的 REGISTRY
2. make redeploy_all
### 注意事项
1. k8s node的name需要和/etc/hostname保持一致
\ No newline at end of file
cmd/collector/Dockerfile 0 → 100644
FROM hb.seetatech.com/k8s/cuda:10.0-base-ubuntu16.04
COPY conf/ /adl/bin/conf
COPY conf/config.json /root/.docker/config.json
COPY core--collector /adl/bin/
WORKDIR /adl/bin/
ENV PATH /adl/bin:$PATH
cmd/collector/main.go 0 → 100644
package main
import (
_ "autodl-core/conf"
"autodl-core/service"
"context"
)
func main() {
collectorFactory := service.InitializeCollectorFactory()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go collectorFactory.Collector.CollectNodeResourceInfo(ctx)
collectorFactory.Collector.CollectTaskLog()
}
cmd/log_stash/Dockerfile 0 → 100644
FROM ubuntu:16.04
RUN apt-get update && \
apt-get install -y git && \
export DEBIAN_FRONTEND=noninteractive && \
apt-get install -y tzdata && \
ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \
dpkg-reconfigure -f noninteractive tzdata
COPY conf/ /adl/bin/conf
COPY conf/config.json /root/.docker/config.json
COPY log-stash-v1 /adl/bin/
WORKDIR /adl/bin/
ENV PATH /adl/bin:$PATH
cmd/log_stash/main.go 0 → 100644
package main
import (
_ "autodl-core/conf"
"autodl-core/service"
"github.com/spf13/viper"
"os"
)
func main() {
logStash := service.InitializeLogStashFactory()
if len(os.Args) > 1 {
logStash.LogStash.CollectTaskLog(os.Args[1])
} else {
logStash.LogStash.CollectTaskLog(viper.GetString("k8s.namespace"))
}
}
cmd/monitor/main.go 0 → 100644
package main
import (
_ "autodl-core/conf"
"autodl-core/service"
"context"
)
func main() {
monitor := service.InitializeMonitorFactory()
stop := make(chan error)
ctx, cancelFunc := context.WithCancel(context.Background())
go monitor.Monitor.WatchTaskStatus(ctx, stop)
go monitor.Monitor.HandleTaskStatus(ctx, stop)
select {
case err := <-stop:
cancelFunc()
if err != nil {
panic(err.Error())
}
}
}
cmd/nginx/502.html 0 → 100644
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>AutoDL</title>
<style>
.content {
position: absolute;
top: 40%;
left: 50%;
transform: translate(-50%);
text-align: center;
display: flex;
flex-direction: column;
align-items: center;
font-size: 40px;
color: #1F95FF;
}
</style>
</head>
<body>
<div class="content">
<span>服务无法访问,请稍后再试</span>
</div>
</body>
</html>
\ No newline at end of file
cmd/nginx/Dockerfile 0 → 100644
FROM nginx
COPY nginx.conf /etc/nginx/nginx.conf
COPY apiserver.conf /etc/nginx/conf.d/default.conf
COPY *.html /etc/nginx/
cmd/nginx/apiserver.conf 0 → 100644
server {
listen 80;
location ~ /tensorboard/([-_.:\w]+)/(.*) {
resolver kube-dns.kube-system.svc.cluster.local valid=5s;
rewrite_log on;
rewrite ^/tensorboard/([-_.:\w]+)/(.*) /$2 break;
proxy_pass http://$1;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_set_header Origin "";
}
location ~ /jupyter/([-_.:\w]+)/(.*) {
resolver kube-dns.kube-system.svc.cluster.local valid=1s;
rewrite_log on;
rewrite ^/jupyter/([-_.:\w]+)/(.*) /jupyter/$1/$2 break;
proxy_pass http://$1;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_set_header Origin "";
}
location ~ /standard/([-_.:\w]+)/(.*) {
resolver kube-dns.kube-system.svc.cluster.local valid=5s;
rewrite_log on;
rewrite ^/standard/([-_.:\w]+)/(.*) /$2 break;
proxy_pass http://$1;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_set_header Origin "";
}
error_page 404 502 /502.html;
location = /502.html {
root /etc/nginx;
}
}
cmd/nginx/nginx.conf 0 → 100644
user nginx;
worker_processes 2;
worker_rlimit_nofile 65535;
error_log /var/log/nginx/error.log warn;
pid /var/run/nginx.pid;
events {
worker_connections 4096;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
sendfile on;
#tcp_nopush on;
keepalive_timeout 65;
#gzip on;
include /etc/nginx/conf.d/*.conf;
}
cmd/server/main.go 0 → 100644
package main
import (
_ "autodl-core/conf"
"autodl-core/service"
log "github.com/cihub/seelog"
"github.com/spf13/viper"
"gitlab.seetatech.com/autodl.com/autodl-base.git/protoV1"
"google.golang.org/grpc"
"net"
)
func main() {
serverFactory := service.InitializeServerFactory()
lis, err := net.Listen("tcp", viper.GetString("app.rpc_port"))
if err != nil {
log.Errorf("failed to listen: %v", err)
}
s := grpc.NewServer()
proto_v1.RegisterCoreServer(s, serverFactory.RpcServer)
if err := s.Serve(lis); err != nil {
log.Errorf("failed to serve: %v", err)
}
}
cmd/worker/main.go 0 → 100644
package main
import (
_ "autodl-core/conf"
"autodl-core/service"
)
func main() {
workerFactory := service.InitializeWorkerFactory()
if err := workerFactory.Worker.Run(); err != nil {
panic("panic: " + err.Error())
}
}
conf/app.yaml 0 → 100644
app:
rpc_port: ":30100"
mount_path: "/mnt/ceph"
database:
type: "mysql"
host: mysql_host
name: autodl-core
user: mysql_user
password: mysql_password
url: "%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local"
redis:
addr: "redis_host"
password: "redis_password"
mongodb:
addr: "mongo_host"
k8s:
namespace: autodl
pvc: adl-pvc
docker:
registry_host: harbor_host
registry_group_name: core
host:
nginx_host: core_nginx
conf/config.json 0 → 100644
{
"auths": {
"172.24.14.24:5000": {
"auth": "YWRtaW46YXV0b2Nubl9zZWV0YXRlY2g="
}
},
"HttpHeaders": {
"User-Agent": "Docker-Client/18.03.1-ce (linux)"
}
}
conf/init_config.go 0 → 100644
package conf
import (
log "github.com/cihub/seelog"
"github.com/spf13/viper"
)
const (
viperConfigFile = "conf/app.yaml"
logConfigFile = "conf/log.xml"
)
func init() {
viper.Reset()
viper.SetConfigType("yaml")
viper.SetConfigFile(viperConfigFile)
if e := viper.ReadInConfig(); e != nil {
panic(e)
}
logger, err := log.LoggerFromConfigAsFile(logConfigFile)
if err != nil {
panic(err)
}
_ = log.ReplaceLogger(logger)
}
conf/log.xml 0 → 100644
<seelog type="sync">
<outputs formatid="main">
<console/>
</outputs>
<formats>
<format id="main" format="%Date/%Time [%LEV] [%FullPath:%Line %Func] %Msg%n"/>
</formats>
</seelog>
\ No newline at end of file
deploy/develop/collector-deployment.yaml 0 → 100644
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
name: core--collector
namespace: autodl
spec:
template:
metadata:
labels:
app: core--collector
logCollect: "true"
spec:
serviceAccountName: autodl-serviceaccount
containers:
- name: core--collector
image: DOCKER_REGISTRY/core--collector:latest
imagePullPolicy: Always
volumeMounts:
- mountPath: /mnt/ceph
name: adl-volume
- name: docker-sock
mountPath: /var/run/docker.sock
subPath: docker.sock
- name: hostname
mountPath: /etc/autocnn_hostname
command: ["core--collector"]
resources:
requests:
cpu: "500m"
memory: "512Mi"
limits:
cpu: "1000m"
memory: "1024Mi"
volumes:
- name: adl-volume
persistentVolumeClaim:
claimName: adl-pvc
readOnly: false
- name: docker-sock
hostPath:
path: /var/run/
- name: hostname
hostPath:
path: /etc/hostname
deploy/develop/log-stash-deployment-seetaas.yaml 0 → 100644
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: log-stash-v1
namespace: seetaas
spec:
template:
metadata:
labels:
app: log-stash-v1
logCollect: "true"
spec:
serviceAccountName: seetaas-serviceaccount
containers:
- name: log-stash-v1
image: DOCKER_REGISTRY/log-stash-v1:latest
imagePullPolicy: Always
volumeMounts:
- mountPath: /mnt/ceph
name: adl-volume
- name: docker-sock
mountPath: /var/run/docker.sock
subPath: docker.sock
- name: hostname
mountPath: /etc/autocnn_hostname
command: ["log-stash-v1", "seetaas"]
resources:
requests:
cpu: "500m"
memory: "512Mi"
limits:
cpu: "1000m"
memory: "1024Mi"
volumes:
- name: adl-volume
persistentVolumeClaim:
claimName: adl-pvc
readOnly: false
- name: docker-sock
hostPath:
path: /var/run/
- name: hostname
hostPath:
path: /etc/hostname
deploy/develop/log-stash-deployment.yaml 0 → 100644
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: log-stash-v1
namespace: autodl
spec:
template:
metadata:
labels:
app: log-stash-v1
logCollect: "true"
spec:
serviceAccountName: autodl-serviceaccount
containers:
- name: log-stash-v1
image: DOCKER_REGISTRY/log-stash-v1:latest
imagePullPolicy: Always
volumeMounts:
- mountPath: /mnt/ceph
name: adl-volume
- name: docker-sock
mountPath: /var/run/docker.sock
subPath: docker.sock
- name: hostname
mountPath: /etc/autocnn_hostname
command: ["log-stash-v1"]
resources:
requests:
cpu: "500m"
memory: "512Mi"
limits:
cpu: "1000m"
memory: "1024Mi"
volumes:
- name: adl-volume
persistentVolumeClaim:
claimName: adl-pvc
readOnly: false
- name: docker-sock
hostPath:
path: /var/run/
- name: hostname
hostPath:
path: /etc/hostname
deploy/develop/monitor-deployment.yaml 0 → 100644
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: core--monitor
namespace: autodl
spec:
replicas: 1
template:
metadata:
labels:
app: core--monitor
logCollect: "true"
spec:
serviceAccountName: autodl-serviceaccount
nodeSelector:
internal_service_node: "true"
containers:
- name: core--monitor
image: DOCKER_REGISTRY/adl-core-v1:latest
imagePullPolicy: Always
volumeMounts:
- mountPath: /mnt/ceph
name: adl-volume
- name: docker-sock
mountPath: /var/run/docker.sock
subPath: docker.sock
command: ["core--monitor"]
resources:
requests:
cpu: "500m"
memory: "512Mi"
limits:
cpu: "1000m"
memory: "1024Mi"
volumes:
- name: adl-volume
persistentVolumeClaim:
claimName: adl-pvc
readOnly: false
- name: docker-sock
hostPath:
path: /var/run/
deploy/develop/nginx-svc.yaml 0 → 100644
apiVersion: v1
kind: Service
metadata:
name: core--nginx-svc
namespace: autodl
spec:
type: NodePort
selector:
app: core--nginx
ports:
- port: 80
name: port-80
targetPort: 80
nodePort: 30099
\ No newline at end of file
deploy/develop/nginx.yaml 0 → 100644
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: core--nginx
namespace: autodl
spec:
replicas: 1
template:
metadata:
labels:
app: core--nginx
spec:
nodeSelector:
internal_service_node: "true"
containers:
- name: core--nginx
image: DOCKER_REGISTRY/core--nginx:latest
imagePullPolicy: Always
ports:
- containerPort: 80
name: port-80
resources:
requests:
cpu: "500m"
memory: "512Mi"
limits:
cpu: "500m"
memory: "1024Mi"
\ No newline at end of file
deploy/develop/server-deployment.yaml 0 → 100644
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: core--server
namespace: autodl
spec:
replicas: 1
template:
metadata:
labels:
app: core--server
logCollect: "true"
spec:
serviceAccountName: autodl-serviceaccount
nodeSelector:
internal_service_node: "true"
containers:
- name: core--server
image: DOCKER_REGISTRY/adl-core-v1:latest
imagePullPolicy: Always
volumeMounts:
- mountPath: /mnt/ceph
name: adl-volume
- name: docker-sock
mountPath: /var/run/docker.sock
subPath: docker.sock
command: ["core--server"]
resources:
requests:
cpu: "500m"
memory: "512Mi"
limits:
cpu: "1000m"
memory: "1024Mi"
volumes:
- name: adl-volume
persistentVolumeClaim:
claimName: adl-pvc
readOnly: false
- name: docker-sock
hostPath:
path: /var/run/
deploy/develop/server-svc.yaml 0 → 100644
apiVersion: v1
kind: Service
metadata:
labels:
app: core--server
name: core--server
namespace: autodl
spec:
type: NodePort
selector:
app: core--server
ports:
- port: 30100
name: port-30100
targetPort: 30100
nodePort: 30100
\ No newline at end of file
deploy/develop/worker-deployment.yaml 0 → 100644
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: core--worker
namespace: autodl
spec:
replicas: 1
template:
metadata:
labels:
app: core--worker
logCollect: "true"
spec:
serviceAccountName: autodl-serviceaccount
nodeSelector:
internal_service_node: "true"
containers:
- name: core--worker
image: DOCKER_REGISTRY/adl-core-v1:latest
imagePullPolicy: Always
volumeMounts:
- mountPath: /mnt/ceph
name: adl-volume
- name: docker-sock
mountPath: /var/run/docker.sock
subPath: docker.sock
command: ["core--worker"]
resources:
requests:
cpu: "500m"
memory: "512Mi"
limits:
cpu: "1000m"
memory: "1024Mi"
volumes:
- name: adl-volume
persistentVolumeClaim:
claimName: adl-pvc
readOnly: false
- name: docker-sock
hostPath:
path: /var/run/
git_repo/git.go 0 → 100644
package git_repo
import (
"bytes"
"errors"
"fmt"
log "github.com/cihub/seelog"
"os"
"os/exec"
"path/filepath"
"regexp"
"strings"
)
// ConfigMap maps config keys to their values.
type ConfigMap map[string]string
// Ref is the basic way to point at an individual commit in Git.
type Ref struct {
SHA, Path string
r *Repo
}
// RefMap maps ref names into Ref structs.
type RefMap map[string]*Ref
// Repo is the main struct that we use to track Git repositories.
type Repo struct {
// GitDir is the directory that the Git metadata is in for this repo.
GitDir string
// WorkDir is the directory that holds the working tree for this repo.
WorkDir string
// refs holds the cached RefMap.
refs RefMap
// cfg holds the cached config data.
cfg ConfigMap
}
var gitCmd string
var statusRE *regexp.Regexp
var statMap = map[string]string{
" ": "unmodified",
"M": "modified",
"A": "added",
"D": "deleted",
"R": "renamed",
"C": "copied",
"U": "unmerged",
"?": "untracked",
"!": "ignored",
}
func init() {
var err error
if gitCmd, err = exec.LookPath("git"); err != nil {
panic("Cannot find git command!")
}
statusRE = regexp.MustCompile("^([ MADRCU!?])([ MADRCU?!]) (.*)$")
}
func findRepo(path string) (found bool, gitdir, workdir string) {
stat, err := os.Stat(path)
if err != nil {
panic("Could not stat " + path)
}
if !stat.IsDir() {
panic(path + " is not a directory!")
}
if strings.HasSuffix(path, ".git") {
if stat, err = os.Stat(filepath.Join(path, "config")); err == nil {
found = true
gitdir = path
workdir = ""
return
}
}
if stat, err = os.Stat(filepath.Join(path, ".git", "config")); err != nil {
found = false
return
}
found = true
gitdir = filepath.Join(path, ".git")
workdir = path
return
}
// Open the first git repository that "owns" path.
func Open(path string) (repo *Repo, err error) {
if path == "" {
path = "."
}
path, err = filepath.Abs(path)
basepath := path
if err != nil {
return
}
for {
found, gitdir, workdir := findRepo(path)
if found {
repo = new(Repo)
repo.GitDir = gitdir
repo.WorkDir = workdir
return
}
parent := filepath.Dir(path)
if parent == path {
break
}
path = parent
}
return nil, errors.New(fmt.Sprintf("Could not find a Git repository in %s or any of its parents!", basepath))
}
// Git is a helper for creating exec.Cmd types and arranging to capture
// the output and erro streams of the command into bytes.Buffers
func Git(cmd string, args ...string) (res *exec.Cmd, stdout, stderr *bytes.Buffer) {
cmdArgs := make([]string, 1)
cmdArgs[0] = cmd
cmdArgs = append(cmdArgs, args...)
res = exec.Command(gitCmd, cmdArgs...)
stdout, stderr = new(bytes.Buffer), new(bytes.Buffer)
res.Stdout, res.Stderr = stdout, stderr
return
}
// Git is a helper for making sure that the Git command runs in the proper repository.
func (r *Repo) Git(cmd string, args ...string) (res *exec.Cmd, out, err *bytes.Buffer) {
var path string
if r.WorkDir == "" {
path = r.GitDir
} else {
path = r.WorkDir
}
res, out, err = Git(cmd, args...)
res.Dir = path
return
}
func (r *Repo) Commit() error {
cmd, stdout, stderr := r.Git("add", "-A")
if err := cmd.Run(); err != nil {
return errors.New(stdout.String() + " " + stderr.String())
}
cmd, stdout, stderr = r.Git("-c", "user.email='autocnn@seetatech.com'", "-c", "user.name='autocnn", "commit", "-m", "'commit'")
if err := cmd.Run(); err != nil {
return errors.New(stdout.String() + " " + stderr.String())
}
return nil
}
func (r *Repo) GetLastCommit() (string, error) {
cmd, out, stderr := r.Git("--no-pager", "log", "--pretty=oneline", "-1", r.WorkDir)
if err := cmd.Run(); err != nil {
return "", errors.New(stderr.String())
}
stdout := out.String()
if len(stdout) > 0 && strings.Contains(stdout, " ") {
return strings.Split(stdout, " ")[0], nil
}
return "", errors.New("no commit found")
}
func (r *Repo) Checkout(commit string) error {
if len(commit) == 0 {
commit = "master"
}
cmd, stdout, stderr := r.Git("checkout", commit)
if err := cmd.Run(); err != nil {
return errors.New(stderr.String() + ";" + stdout.String())
}
return nil
}
func (r *Repo) Reset(commit string) error {
cmd, _, stderr := r.Git("reset", "--hard", commit)
if err := cmd.Run(); err != nil {
return errors.New(stderr.String())
}
return nil
}
// Init initializes new Get metadata at the passed path.
// The rest of the args are passed to the 'git init' command unchanged.
func Init(path string, args ...string) (res *Repo, err error) {
cmd, _, stderr := Git("init", append(args, path)...)
if err = cmd.Run(); err != nil {
return nil, errors.New(stderr.String())
}
res, err = Open(path)
return
}
func InitAndCommit(path string) (string, error) {
repo, err := Init(path)
if err != nil {
log.Info("repos maybe initialized before")
repo, err = Open(path)
}
err = repo.Commit()
if err != nil {
log.Warn("repos commit failed: ", err.Error())
}
commit, err := repo.GetLastCommit()
if err != nil {
return "", err
}
return commit, nil
}
// Clone a new git repository. The clone will be created in the current
// directory.
func Clone(source, target string, args ...string) (res *Repo, err error) {
cmd, _, stderr := Git("clone", append(args, source, target)...)
if err = cmd.Run(); err != nil {
return nil, errors.New(stderr.String())
}
res, err = Open(target)
return
}
// StatLine holds interesting bits of git status output.
type StatLine struct {
indexStat, workStat, oldPath, newPath string
}
// StatLines is a slice of statuses.
type StatLines []*StatLine
// Print prints a StatLine in human readable format.
func (s *StatLine) Print() string {
var res string
if s.indexStat == "R" {
res = fmt.Sprintf("%s was renamed to %s\n", s.oldPath, s.newPath)
}
res = res + fmt.Sprintf("%s is %s in the index and %s in the working tree.",
s.newPath,
statMap[s.indexStat],
statMap[s.workStat])
return res
}
func (r *Repo) mapStatus() (res StatLines) {
var thisStat *StatLine
cmd, out, err := r.Git("status", "--porcelain", "-z")
if cmd.Run() != nil {
panic(err.String())
}
for {
line, err := out.ReadString(0)
if err != nil {
break
}
parts := statusRE.FindStringSubmatch(line)
if parts != nil {
if thisStat != nil {
res = append(res, thisStat)
}
thisStat = new(StatLine)
thisStat.indexStat = parts[1]
thisStat.workStat = parts[2]
thisStat.oldPath = parts[3]
thisStat.newPath = parts[3]
} else if thisStat != nil {
thisStat.newPath = line
} else {
panic("Cannot happen!")
}
}
if thisStat != nil {
res = append(res, thisStat)
}
return
}
go.mod 0 → 100644
module autodl-core
require (
cloud.google.com/go v0.44.3 // indirect
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
github.com/Azure/go-autorest v13.0.0+incompatible // indirect
github.com/Microsoft/go-winio v0.0.0-20190322214808-dd3d7fa17846 // indirect
github.com/NYTimes/gziphandler v1.1.1 // indirect
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
github.com/Unknwon/com v0.0.0-20181010210213-41959bdd855f // indirect
github.com/adl-golang/quicktemplate v1.0.3
github.com/c9s/goprocinfo v0.0.0-20191125144613-4acdd056c72d
github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575
github.com/containerd/continuity v0.0.0-20181203112020-004b46473808 // indirect
github.com/coreos/bbolt v1.3.3 // indirect
github.com/coreos/etcd v3.3.15+incompatible // indirect
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect
github.com/docker/docker v0.0.0-20180616010903-de0abf4315fd // indirect
github.com/docker/go-connections v0.0.0-20180821093606-97c2040d34df // indirect
github.com/docker/go-units v0.3.3 // indirect
github.com/docker/libnetwork v0.0.0-20190314230531-ebcade70ad10 // indirect
github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c // indirect
github.com/elazarl/goproxy v0.0.0-20190711103511-473e67f1d7d2 // indirect
github.com/emicklei/go-restful v2.9.6+incompatible // indirect
github.com/evanphx/json-patch v4.5.0+incompatible // indirect
github.com/fsouza/go-dockerclient v1.2.1
github.com/gin-gonic/gin v0.0.0-20180531061340-caf3e350a548
github.com/go-kit/kit v0.9.0 // indirect
github.com/go-openapi/spec v0.19.2 // indirect
github.com/go-openapi/swag v0.19.5 // indirect
github.com/go-redis/redis v6.15.6+incompatible
github.com/go-sql-driver/mysql v1.4.1
github.com/gogo/protobuf v1.3.0 // indirect
github.com/gogs/git-module v0.0.0-20181023105832-dfc2c1e6d377
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect
github.com/golang/snappy v0.0.1 // indirect
github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 // indirect
github.com/google/pprof v0.0.0-20190723021845-34ac40c74b70 // indirect
github.com/google/wire v0.4.0
github.com/googleapis/gnostic v0.3.1 // indirect
github.com/gophercloud/gophercloud v0.3.0 // indirect
github.com/gorilla/websocket v1.4.1 // indirect
github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.11.0 // indirect
github.com/hashicorp/golang-lru v0.5.3 // indirect
github.com/imdario/mergo v0.3.7 // indirect
github.com/jinzhu/gorm v1.9.10
github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
github.com/kr/pty v1.1.8 // indirect
github.com/levigross/grequests v0.0.0-20171009010347-bf9788368aa0
github.com/mcuadros/go-version v0.0.0-20180611085657-6d5863ca60fa // indirect
github.com/mindprince/gonvml v0.0.0-20190828220739-9ebdce4bb989
github.com/munnerz/goautoneg v0.0.0-20190414153302-2ae31c8b6b30 // indirect
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
github.com/novln/docker-parser v0.0.0-20170503100553-6030251119d6
github.com/onsi/ginkgo v1.10.3
github.com/onsi/gomega v1.7.1
github.com/opencontainers/go-digest v0.0.0-20190228220655-ac19fd6e7483 // indirect
github.com/opencontainers/image-spec v0.0.0-20190321123305-da296dcb1e47 // indirect
github.com/opencontainers/runc v0.0.0-20190322180631-11fc498ffa5c // indirect
github.com/pkg/errors v0.8.1
github.com/prometheus/client_golang v1.1.0 // indirect
github.com/prometheus/procfs v0.0.4 // indirect
github.com/rogpeppe/fastuuid v1.2.0 // indirect
github.com/rogpeppe/go-internal v1.3.1 // indirect
github.com/satori/go.uuid v1.2.0
github.com/sirupsen/logrus v1.4.2 // indirect
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 // indirect
github.com/spf13/viper v1.4.0
github.com/tidwall/pretty v1.0.0 // indirect
github.com/ugorji/go v1.1.7 // indirect
github.com/vishvananda/netlink v1.0.0 // indirect
github.com/vishvananda/netns v0.0.0-20190625233234-7109fa855b0f // indirect
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c // indirect
github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc // indirect
gitlab.seetatech.com/autodl.com/autodl-base.git v0.0.0-20200102092617-103b8ce89985
gitlab.seetatech.com/autodl.com/autodl-k8s.git v0.0.0-20190902080041-53a58554394c
go.etcd.io/bbolt v1.3.3 // indirect
go.mongodb.org/mongo-driver v0.0.0-20190315150634-c72645a64800
golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 // indirect
golang.org/x/exp v0.0.0-20190829153037-c13cbed26979 // indirect
golang.org/x/image v0.0.0-20190902063713-cb417be4ba39 // indirect
golang.org/x/mobile v0.0.0-20190830201351-c6da95954960 // indirect
golang.org/x/net v0.0.0-20191116160921-f9c825593386
golang.org/x/sys v0.0.0-20190830142957-1e83adbbebd0 // indirect
golang.org/x/tools v0.0.0-20190830223141-573d9926052a // indirect
google.golang.org/api v0.9.0 // indirect
google.golang.org/appengine v1.6.2 // indirect
google.golang.org/grpc v1.25.1
gopkg.in/inf.v0 v0.9.1 // indirect
gotest.tools v2.3.0+incompatible // indirect
honnef.co/go/tools v0.0.1-2019.2.2 // indirect
k8s.io/api v0.0.0-20190831074750-7364b6bdad65
k8s.io/apimachinery v0.0.0-20190831074630-461753078381
k8s.io/gengo v0.0.0-20190826232639-a874a240740c // indirect
k8s.io/utils v0.0.0-20190829053155-3a4a5477acf8 // indirect
sigs.k8s.io/structured-merge-diff v0.0.0-20190820212518-960c3cc04183 // indirect
)
go 1.13
go.sum 0 → 100644
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7hw=
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
cloud.google.com/go v0.44.3 h1:0sMegbmn/8uTwpNkB0q9cLEpZ2W5a6kl+wtBQgPWBJQ=
cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8=
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
github.com/Azure/go-autorest v11.1.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
github.com/Azure/go-autorest v13.0.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/Microsoft/go-winio v0.0.0-20190322214808-dd3d7fa17846 h1:WzgKzHWRC8dJ0YMZk9/Tighx6yKsLCkqNSa4EIahIKU=
github.com/Microsoft/go-winio v0.0.0-20190322214808-dd3d7fa17846/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw=
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
github.com/Unknwon/com v0.0.0-20181010210213-41959bdd855f h1:m1tYqjD/N0vF/S8s/ZKz/eccUr8RAAcrOK2MhXeTegA=
github.com/Unknwon/com v0.0.0-20181010210213-41959bdd855f/go.mod h1:KYCjqMOeHpNuTOiFQU6WEcTG7poCJrUs0YgyHNtn1no=
github.com/adl-golang/quicktemplate v1.0.3 h1:Jk6nisekPAPLn0EWdjj83HW8gA/FvBc9IxA16Vqj1+o=
github.com/adl-golang/quicktemplate v1.0.3/go.mod h1:TAbPIaVNmjvWIJCperDHh4dl0Rbrn65U9K4gyUAUE+c=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/c9s/goprocinfo v0.0.0-20191125144613-4acdd056c72d h1:MQGrhPHSxg08x+LKgQTOnnjfXt+p+128WCECqAYXJsU=
github.com/c9s/goprocinfo v0.0.0-20191125144613-4acdd056c72d/go.mod h1:uEyr4WpAH4hio6LFriaPkL938XnrvLpNPmQHBdrmbIE=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 h1:kHaBemcxl8o/pQ5VM1c8PVE1PubbNx3mjUr09OqWGCs=
github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575/go.mod h1:9d6lWj8KzO/fd/NrVaLscBKmPigpZpn5YawRPw+e3Yo=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/containerd/continuity v0.0.0-20181203112020-004b46473808 h1:4BX8f882bXEDKfWIf0wa8HRvpnBoPszJJXL+TVbBw4M=
github.com/containerd/continuity v0.0.0-20181203112020-004b46473808/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/etcd v3.3.15+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/denisenkom/go-mssqldb v0.0.0-20190515213511-eb9f6a1743f3 h1:tkum0XDgfR0jcVVXuTsYv/erY2NnEDqwRojbxR1rBYA=
github.com/denisenkom/go-mssqldb v0.0.0-20190515213511-eb9f6a1743f3/go.mod h1:zAg7JM8CkOJ43xKXIj7eRO9kmWm/TW578qo+oDO6tuM=
github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/dgryski/go-sip13 v0.0.0-20190329191031-25c5027a8c7b/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/docker/docker v0.0.0-20180616010903-de0abf4315fd h1:PSbXzo3G5Uws4PczuR8Lsl76mhaOxyrfTzq/mvTzm4w=
github.com/docker/docker v0.0.0-20180616010903-de0abf4315fd/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.0.0-20180821093606-97c2040d34df h1:ADMjlaDGEn0OOQIieyxanhAt41jcngf8rf78X2eKNLw=
github.com/docker/go-connections v0.0.0-20180821093606-97c2040d34df/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk=
github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/docker/libnetwork v0.0.0-20190314230531-ebcade70ad10 h1:A8QkYACiT7KU+8xiXxu2zhDShhVzndUuFq0l5zzSkvA=
github.com/docker/libnetwork v0.0.0-20190314230531-ebcade70ad10/go.mod h1:93m0aTqz6z+g32wla4l4WxTrdtvBRmVzYRkYvasA5Z8=
github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
github.com/elazarl/goproxy v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
github.com/emicklei/go-restful v2.9.6+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y=
github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsouza/go-dockerclient v1.2.1 h1:ZcSDAjMR2wkfuAOOaoCOML8NZKuXRi8L0aib5ZtGPoc=
github.com/fsouza/go-dockerclient v1.2.1/go.mod h1:KpcjM623fQYE9MZiTGzKhjfxXAV9wbyX2C1cyRHfhl0=
github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/gin-gonic/gin v0.0.0-20180531061340-caf3e350a548 h1:YQ4MldjZCM+FNF1pwZVKhjPikjKLndpnrPxHTwtGFP4=
github.com/gin-gonic/gin v0.0.0-20180531061340-caf3e350a548/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=
github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=
github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY=
github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-redis/redis v0.0.0-20190725120116-ec312e4dd5bf h1:nAs8spv+TXm9/ygiWHYY4SiBGRB5RakUL7J6TQ9yYbY=
github.com/go-redis/redis v0.0.0-20190725120116-ec312e4dd5bf/go.mod h1:nuQKdm6S7SnV28NJEN2ZNbKpddAM1O76Z2LMJcIxJVM=
github.com/go-redis/redis v6.15.6+incompatible h1:H9evprGPLI8+ci7fxQx6WNZHJSb7be8FqJQRhdQZ5Sg=
github.com/go-redis/redis v6.15.6+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I=
github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
github.com/gogo/protobuf v1.3.0 h1:G8O7TerXerS4F6sx9OV7/nRfJdnXgHZu/S/7F2SN+UE=
github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
github.com/gogs/git-module v0.0.0-20181023105832-dfc2c1e6d377 h1:VdMM5u+FB5ap/oNyF/6nWIv5v9Lh4rQgf3dhFS1ZYOc=
github.com/gogs/git-module v0.0.0-20181023105832-dfc2c1e6d377/go.mod h1:OP3DbwwwhG0m2RJ/Jm2VgdlnovIipQuktM9uFTPLBgc=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/btree v0.0.0-20160524151835-7d79101e329e/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 h1:zLTLjkaOFEFIOxY5BWLFLwh+cL8vOBW4XJ2aqLE/Tf0=
github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190723021845-34ac40c74b70/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/wire v0.2.1 h1:TYj4Z2qjqxa2ufb34UJqVeO9aznL+i0fLO6TqThKZ7Y=
github.com/google/wire v0.2.1/go.mod h1:ptBl5bWD3nzmJHVNwYHV3v4wdtKzBMlU2YbtKQCG9GI=
github.com/google/wire v0.4.0 h1:kXcsA/rIGzJImVqPdhfnr6q0xsS9gU0515q1EPpJ9fE=
github.com/google/wire v0.4.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/googleapis/gnostic v0.0.0-20170426233943-68f4ded48ba9/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
github.com/googleapis/gnostic v0.3.1 h1:WeAefnSUHlBb0iJKwxFDZdbfGwkd7xRNuV+IpXMJhYk=
github.com/googleapis/gnostic v0.3.1/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU=
github.com/gophercloud/gophercloud v0.0.0-20190126172459-c818fa66e4c8/go.mod h1:3WdhXV3rUYy9p6AUW8d94kr+HS62Y4VL9mBnFxsD8q4=
github.com/gophercloud/gophercloud v0.3.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/grpc-ecosystem/grpc-gateway v1.9.6/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/grpc-ecosystem/grpc-gateway v1.11.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI=
github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/jinzhu/gorm v1.9.10 h1:HvrsqdhCW78xpJF67g1hMxS6eCToo9PZH4LDB8WKPac=
github.com/jinzhu/gorm v1.9.10/go.mod h1:Kh6hTsSGffh4ui079FHrR5Gg+5D0hgihqDcsDN2BBJY=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M=
github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo=
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0=
github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
github.com/levigross/grequests v0.0.0-20171009010347-bf9788368aa0 h1:QpdhtrR7SX3R7OlEv9dZarsXogM3PM/tl1ibRH/eHbQ=
github.com/levigross/grequests v0.0.0-20171009010347-bf9788368aa0/go.mod h1:uCZIhROSrVmuF/BPYFPwDeiiQ6juSLp0kikFoEcNcEs=
github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4=
github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM=
github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o=
github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mcuadros/go-version v0.0.0-20180611085657-6d5863ca60fa h1:XvNrttGMJfVrUqblGju4IkjYXwx6l5OAAyjaIsydzsk=
github.com/mcuadros/go-version v0.0.0-20180611085657-6d5863ca60fa/go.mod h1:76rfSfYPWj01Z85hUf/ituArm797mNKcvINh1OlsZKo=
github.com/mindprince/gonvml v0.0.0-20190828220739-9ebdce4bb989 h1:PS1dLCGtD8bb9RPKJrc8bS7qHL6JnW1CZvwzH9dPoUs=
github.com/mindprince/gonvml v0.0.0-20190828220739-9ebdce4bb989/go.mod h1:2eu9pRWp8mo84xCg6KswZ+USQHjwgRhNp06sozOdsTY=
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/munnerz/goautoneg v0.0.0-20190414153302-2ae31c8b6b30/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
github.com/novln/docker-parser v0.0.0-20170503100553-6030251119d6 h1:1iA6ohNspg/6AkPh6LhXSqxWNYBCPUvZ7EHh8FtXImU=
github.com/novln/docker-parser v0.0.0-20170503100553-6030251119d6/go.mod h1:iDeGFp41QxV9wavwD8gqv5/Bhyh2pWJQb4TI3vhG7VQ=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/oklog/ulid v2.0.0+incompatible/go.mod h1:9+sfj31oWp8ZMW/ijX7Tz254TfS+jktjmFqEK9jtXa8=
github.com/olivere/elastic v6.2.22+incompatible h1:kfNhtWbbRep8LXUpHo4siKjnbqfy21BE0GwlquWx2bc=
github.com/olivere/elastic v6.2.22+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8=
github.com/olivere/elastic v6.2.23+incompatible h1:oRGUA/8fKcnkDcqLuwGb5YCzgbgEBo+Y9gamsWqZ0qU=
github.com/olivere/elastic v6.2.23+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8=
github.com/olivere/elastic v6.2.26+incompatible h1:3PjUHKyt8xKwbFQpRC5cgtEY7Qz6ejopBkukhI7UWvE=
github.com/olivere/elastic v6.2.26+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8=
github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.9.0 h1:SZjF721BByVj8QH636/8S2DnX4n0Re3SteMmw3N+tzc=
github.com/onsi/ginkgo v1.9.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo=
github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v0.0.0-20190113212917-5533ce8a0da3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.6.0 h1:8XTW0fcJZEq9q+Upcyws4JSGua2MFysCL5xkaSgHc+M=
github.com/onsi/gomega v1.6.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/opencontainers/go-digest v0.0.0-20190228220655-ac19fd6e7483 h1:MeQlJsws42k27LSQ4S+zRPxMMhQrD4iSElkbD6Mhvck=
github.com/opencontainers/go-digest v0.0.0-20190228220655-ac19fd6e7483/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
github.com/opencontainers/image-spec v0.0.0-20190321123305-da296dcb1e47 h1:AQGmFjxTJ1tIUzcY2eB2geQbec7Yb0CBsgE78Ye32ic=
github.com/opencontainers/image-spec v0.0.0-20190321123305-da296dcb1e47/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
github.com/opencontainers/runc v0.0.0-20190322180631-11fc498ffa5c h1:jQhbb5xdguoK4hYbLuGCLcIe+yIiVaQwhOJEAiXPRe4=
github.com/opencontainers/runc v0.0.0-20190322180631-11fc498ffa5c/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
github.com/pborman/getopt v0.0.0-20190409184431-ee0cd42419d3/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pelletier/go-toml v1.4.0 h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfSeg=
github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
github.com/prometheus/procfs v0.0.4/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/prometheus/tsdb v0.10.0/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSgwXEyGCt4=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.3.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8=
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
github.com/streadway/amqp v0.0.0-20190815230801-eade30b20f1d h1:5HjJ7tDIRsO3UbqNihgIU34zn4EK41IK838h38KFx8o=
github.com/streadway/amqp v0.0.0-20190815230801-eade30b20f1d/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271 h1:WhxRHzgeVGETMlmVfqhRn8RIeeNoPr2Czh33I4Zdccw=
github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v0.0.0-20180905170723-c6fd90e432cc/go.mod h1:+g/po7GqyG5E+1CNgquiIxJnsXEi5vwFn5weFujbO78=
github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
github.com/vishvananda/netlink v1.0.0 h1:bqNY2lgheFIu1meHUFSH3d7vG93AFyqg3oGbJCOJgSM=
github.com/vishvananda/netlink v1.0.0/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
github.com/vishvananda/netns v0.0.0-20190625233234-7109fa855b0f h1:nBX3nTcmxEtHSERBJaIo1Qa26VwRaopnZmfDQUXsF4I=
github.com/vishvananda/netns v0.0.0-20190625233234-7109fa855b0f/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk=
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc h1:n+nNi93yXLkJvKwXNP9d55HC7lGK4H/SRcwB5IaUZLo=
github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
gitlab.seetatech.com/autodl.com/autodl-base.git v0.0.0-20190718062439-81aa51766927 h1:i4z+L5P/96fhFjfDkRZaeYjvFx2jBp/mr1W8PBEscWA=
gitlab.seetatech.com/autodl.com/autodl-base.git v0.0.0-20190718062439-81aa51766927/go.mod h1:KhnrSVPzIlTth95LKAwta/twb2rhY+Wqkrqn4TxR96g=
gitlab.seetatech.com/autodl.com/autodl-base.git v0.0.0-20200102084505-b32a3678327b h1:SeNAsPVjZXOPRvCeMcrjY9YnNnxOnF/rXRbCl076A9M=
gitlab.seetatech.com/autodl.com/autodl-base.git v0.0.0-20200102084505-b32a3678327b/go.mod h1:49rpvalx88RS7J3B7LHIte59x1Spgtzko7tK8+OsRbc=
gitlab.seetatech.com/autodl.com/autodl-base.git v0.0.0-20200102090022-cd40cead03c4 h1:36VAMwpeKYTLCHE6OFVsJvRcMygC1lOwb/dcgWHlDWw=
gitlab.seetatech.com/autodl.com/autodl-base.git v0.0.0-20200102090022-cd40cead03c4/go.mod h1:49rpvalx88RS7J3B7LHIte59x1Spgtzko7tK8+OsRbc=
gitlab.seetatech.com/autodl.com/autodl-base.git v0.0.0-20200102090452-0c83222fdd11 h1:m2GJwAjKQSlp3l9jpv/iy4jXRUFL83U2ZQ5wAUMUYeI=
gitlab.seetatech.com/autodl.com/autodl-base.git v0.0.0-20200102090452-0c83222fdd11/go.mod h1:49rpvalx88RS7J3B7LHIte59x1Spgtzko7tK8+OsRbc=
gitlab.seetatech.com/autodl.com/autodl-base.git v0.0.0-20200102092617-103b8ce89985 h1:MUniZn1XLKrrLV5SusTDjzaUqtTwcSrh8bqmy5/RkNc=
gitlab.seetatech.com/autodl.com/autodl-base.git v0.0.0-20200102092617-103b8ce89985/go.mod h1:49rpvalx88RS7J3B7LHIte59x1Spgtzko7tK8+OsRbc=
gitlab.seetatech.com/autodl.com/autodl-k8s.git v0.0.0-20190821094546-89ee0a889cc2 h1:KjcXk3eDTsPkRYgN7bFWU30ItHZFA3XuLRM/WQtnHcE=
gitlab.seetatech.com/autodl.com/autodl-k8s.git v0.0.0-20190821094546-89ee0a889cc2/go.mod h1:axgFeDajdXHDbl+X0NkUmjp7piWrIyf3Tgq/OSFy65k=
gitlab.seetatech.com/autodl.com/autodl-k8s.git v0.0.0-20190821115728-952b1bfdd4aa h1:zTuwZS46llmnwiCm2RdLv/vpTETCidO3jZ+dh1yqx2g=
gitlab.seetatech.com/autodl.com/autodl-k8s.git v0.0.0-20190821115728-952b1bfdd4aa/go.mod h1:axgFeDajdXHDbl+X0NkUmjp7piWrIyf3Tgq/OSFy65k=
gitlab.seetatech.com/autodl.com/autodl-k8s.git v0.0.0-20190821121547-2f4916720f89 h1:xTuI5mRZ9y4+NTz4bWzpFBaI5mDFV6E/Oo7sWYM2y3Q=
gitlab.seetatech.com/autodl.com/autodl-k8s.git v0.0.0-20190821121547-2f4916720f89/go.mod h1:axgFeDajdXHDbl+X0NkUmjp7piWrIyf3Tgq/OSFy65k=
gitlab.seetatech.com/autodl.com/autodl-k8s.git v0.0.0-20190902031150-e9a77ba285dd h1:lbnzqGfsg7A3do4Y1vnebNvd3gTp3mzehYKorMij+6o=
gitlab.seetatech.com/autodl.com/autodl-k8s.git v0.0.0-20190902031150-e9a77ba285dd/go.mod h1:axgFeDajdXHDbl+X0NkUmjp7piWrIyf3Tgq/OSFy65k=
gitlab.seetatech.com/autodl.com/autodl-k8s.git v0.0.0-20190902075250-c964761b6b36 h1:WpxCL8ZZn8rLLhVTN12gsAPBfOUzClUv4JUjEkq5RsY=
gitlab.seetatech.com/autodl.com/autodl-k8s.git v0.0.0-20190902075250-c964761b6b36/go.mod h1:axgFeDajdXHDbl+X0NkUmjp7piWrIyf3Tgq/OSFy65k=
gitlab.seetatech.com/autodl.com/autodl-k8s.git v0.0.0-20190902080041-53a58554394c h1:orG8/5n3jb5kCtNG4EBLkEk2PGjLZ6UOdXH9aU9+DCg=
gitlab.seetatech.com/autodl.com/autodl-k8s.git v0.0.0-20190902080041-53a58554394c/go.mod h1:axgFeDajdXHDbl+X0NkUmjp7piWrIyf3Tgq/OSFy65k=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.mongodb.org/mongo-driver v0.0.0-20190315150634-c72645a64800 h1:83WhW9Jkk1JTNPFsDG053As+rKto2Kx0Sqv/CHxrXxY=
go.mongodb.org/mongo-driver v0.0.0-20190315150634-c72645a64800/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181025213731-e84da0312774/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586 h1:7KByu05hhLed2MO29w7p1XfZvZ13m8mub3shuVftRs0=
golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 h1:Gv7RPwsi3eZ2Fgewe3CBsuOebPwO27PoXzRpJPsvSSM=
golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4=
golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.0.0-20190829233526-b3c06291d021/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.0.0-20190902063713-cb417be4ba39/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
golang.org/x/mobile v0.0.0-20190814143026-e8b3e6111d02/go.mod h1:z5wpDCy2wbnXyFdvEuY3LhY9gBUL86/IOILm+Hsjx+E=
golang.org/x/mobile v0.0.0-20190830201351-c6da95954960/go.mod h1:mJOp/i0LXPxJZ9weeIadcPqKVfS05Ai7m6/t9z1Hs/Y=
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190206173232-65e2d4e15006/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA=
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191116160921-f9c825593386 h1:ktbWvQrW08Txdxno1PiDpSxPXG6ndGsfnJjRRtkM0LQ=
golang.org/x/net v0.0.0-20191116160921-f9c825593386/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ=
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190830142957-1e83adbbebd0 h1:7z820YPX9pxWR59qM7BE5+fglp4D/mKqAwCvGt11b+8=
golang.org/x/sys v0.0.0-20190830142957-1e83adbbebd0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/time v0.0.0-20161028155119-f51c12702a4d/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180810170437-e96c4e24768d/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181017214349-06f26fdaaa28/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190808195139-e713427fea3f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190820205717-547ecf7b1ef1/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190830223141-573d9926052a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I=
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
google.golang.org/appengine v1.6.2 h1:j8RI1yW0SkI+paT6uGwMlrMI/6zwYA6/CFil8rxOzGI=
google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1 h1:wdKvqQk7IttEw92GoRyKG2IDrUIpgpj6H6m81yfeMW0=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gotest.tools v2.3.0+incompatible h1:oTCKjb7ZuVfn37AQodk7UysjQQL/S5Dep3pzi59u1NQ=
gotest.tools v2.3.0+incompatible/go.mod h1:R//lfYlUuTOTfblYI3lGoAAAebUdzjvbmQsuB7Ykd90=
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.1-2019.2.2/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
k8s.io/api v0.0.0-20190620084959-7cf5895f2711/go.mod h1:TBhBqb1AWbBQbW3XRusr7n7E4v2+5ZY8r8sAMnyFC5A=
k8s.io/api v0.0.0-20190820101039-d651a1528133 h1:6XLrpVprGENt40uO1F50pRfgP+lOyyz05NfwsbmWXMM=
k8s.io/api v0.0.0-20190820101039-d651a1528133/go.mod h1:AlhL1I0Xqh5Tyz0HsxjEhy+iKci9l1Qy3UMDFW7iG3A=
k8s.io/api v0.0.0-20190831074750-7364b6bdad65 h1:J9uZfyjvqkRa9p2Z22vCCraDjq1Dwf/x9wXzPf6fqSk=
k8s.io/api v0.0.0-20190831074750-7364b6bdad65/go.mod h1:u09ZxrpPFcoUNEQM2GsqT/KpglKAtXdEcK+tSMilQ3Q=
k8s.io/apimachinery v0.0.0-20190612205821-1799e75a0719/go.mod h1:I4A+glKBHiTgiEjQiCCQfCAIcIMFGt291SmsvcrFzJA=
k8s.io/apimachinery v0.0.0-20190820100750-21ddcbbef9e1/go.mod h1:EZoIMuAgG/4v58YL+bz0kqnivqupk28fKYxFCa5e6X8=
k8s.io/apimachinery v0.0.0-20190820100751-ac02f8882ef6 h1:lpB4MEqLgFxnb5zcDsI3MPBoXQ89pFcg8MQLwrwyL2s=
k8s.io/apimachinery v0.0.0-20190820100751-ac02f8882ef6/go.mod h1:EZoIMuAgG/4v58YL+bz0kqnivqupk28fKYxFCa5e6X8=
k8s.io/apimachinery v0.0.0-20190831074630-461753078381 h1:gySvpxrHatsZtG3qOkyPIHjWY7D5ogkrrWnD7+5/RGs=
k8s.io/apimachinery v0.0.0-20190831074630-461753078381/go.mod h1:nL6pwRT8NgfF8TT68DBI8uEePRt89cSvoXUVqbkWHq4=
k8s.io/client-go v12.0.0+incompatible h1:YlJxncpeVUC98/WMZKC3JZGk/OXQWCZjAB4Xr3B17RY=
k8s.io/client-go v12.0.0+incompatible/go.mod h1:E95RaSlHr79aHaX0aGSwcPNfygDiPKOVXdmivCIZT0k=
k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
k8s.io/gengo v0.0.0-20190813173942-955ffa8fcfc9/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
k8s.io/gengo v0.0.0-20190826232639-a874a240740c/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
k8s.io/klog v0.4.0 h1:lCJCxf/LIowc2IGS9TPjWDyXY4nOmdGdfcwwDQCOURQ=
k8s.io/klog v0.4.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc=
k8s.io/kube-openapi v0.0.0-20190709113604-33be087ad058/go.mod h1:nfDlWeOsu3pUf4yWGL+ERqohP4YsZcBJXWMK+gkzOA4=
k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
k8s.io/utils v0.0.0-20190221042446-c2654d5206da/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0=
k8s.io/utils v0.0.0-20190809000727-6c36bc71fc4a h1:uy5HAgt4Ha5rEMbhZA+aM1j2cq5LmR6LQ71EYC2sVH4=
k8s.io/utils v0.0.0-20190809000727-6c36bc71fc4a/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
k8s.io/utils v0.0.0-20190829053155-3a4a5477acf8 h1:khtxGxwSe3nyReEEggzTwQigMT3g40enrlivMlMeaGY=
k8s.io/utils v0.0.0-20190829053155-3a4a5477acf8/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
sigs.k8s.io/structured-merge-diff v0.0.0-20190820212518-960c3cc04183/go.mod h1:IIgPezJWb76P0hotTxzDbWsMYB8APh18qZnxkomBpxA=
sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
k8s_plugin/manager.go 0 → 100644
package k8s_plugin
import (
"github.com/spf13/viper"
"gitlab.seetatech.com/autodl.com/autodl-k8s.git/manager"
)
type K8sManager struct {
K8sManager *manager.K8SManager
}
func K8sManagerProvider() *K8sManager {
k8sManager, err := manager.NewK8SManager(viper.GetString("k8s.namespace"), false)
if err != nil {
panic("init k8s manager failed: " + err.Error())
}
return &K8sManager{
K8sManager: k8sManager,
}
}
libs/constants.go 0 → 100644
package libs
const (
PodTaskIDLabel = "task_id"
PodServiceIDLabel = "service_id"
TaskLogIndexName = "task_log"
)
const K8S_API_VERSION_V1 = "v1"
const K8S_POD_KIND = "Pod"
libs/delay.go 0 → 100644
package libs
import "time"
func RunnerFailedRetry(runner func() (err error), retryTimes int) (lastErr error) {
for i := 0; i < retryTimes; i++ {
lastErr = runner()
if lastErr != nil {
Delay(2 * i)
continue
}
return
}
return
}
func Delay(times int) {
if times > 30 {
time.Sleep(time.Second * 30)
return
}
time.Sleep(time.Second * time.Duration(times))
}
libs/docker/auth.go 0 → 100644
package docker
import (
"strings"
api "github.com/fsouza/go-dockerclient"
)
func getAuth() *api.AuthConfigurations {
c, err := api.NewAuthConfigurationsFromDockerCfg()
if err != nil {
return &api.AuthConfigurations{}
}
return c
}
func getAuthWithRegistry(registry string) api.AuthConfiguration {
auth := getAuth()
if auth != nil {
for k, v := range auth.Configs {
if indexName(k) == indexName(registry) {
return v
}
}
}
return api.AuthConfiguration{}
}
func getAuthWithImage(image string) api.AuthConfiguration {
empty := api.AuthConfiguration{}
auth := getAuthWithRegistry(image)
if auth == empty {
return getAuthWithRegistry("docker.io")
}
return auth
}
func indexName(val string) string {
val = toHostname(val)
// 'index.docker.io' => 'docker.io'
if val == "index.docker.io" {
return "docker.io"
}
return val
}
func toHostname(url string) string {
s := url
if strings.HasPrefix(url, "http://") {
s = strings.Replace(url, "http://", "", 1)
} else if strings.HasPrefix(url, "https://") {
s = strings.Replace(url, "https://", "", 1)
}
p := strings.SplitN(s, "/", 2)
return p[0]
}
libs/docker/build.go 0 → 100644
package docker
import (
api "github.com/fsouza/go-dockerclient"
)
// Build options
const (
buildQuiet = false
buildRm = true
buildForceRm = true
buildMemory = 0
buildMemswap = -1
)
// BuildOptions contains the build configuration for the docker daemon.
type BuildOptions struct {
Name string
Directory string
Pull bool
NoCache bool
}
// See Docker interface
func (d docker) Build(option BuildOptions, stream LogStream) error {
return d.client.BuildImage(buildImageOptions(option, stream))
}
func buildImageOptions(option BuildOptions, stream LogStream) api.BuildImageOptions {
json, output := stream.OutputStream()
opts := api.BuildImageOptions{
Name: option.Name,
NoCache: option.NoCache,
SuppressOutput: buildQuiet,
RmTmpContainer: buildRm,
ForceRmTmpContainer: buildForceRm,
Pull: option.Pull,
OutputStream: output,
ContextDir: option.Directory,
RawJSONStream: json,
Memory: buildMemory,
Memswap: buildMemswap,
}
if auth := getAuth(); auth != nil {
opts.AuthConfigs = *auth
}
return opts
}
libs/docker/build_test.go 0 → 100644
package docker
import (
"fmt"
"testing"
)
func TestDockerBuild(t *testing.T) {
endpoint := "unix:///var/run/docker.sock"
docker, err := New(endpoint)
//client, err := docker.NewClient(endpoint)
if err != nil {
panic(err)
}
buildOption := BuildOptions{
Name: "daiab_test",
Directory: "/tmp/docker1/",
Pull: true,
NoCache: true,
}
logStream := NewLogStream()
docker.Build(buildOption, logStream)
fmt.Println(logStream)
}
libs/docker/container.go 0 → 100644
package docker
import (
api "github.com/fsouza/go-dockerclient"
"time"
)
func (d docker) GetContainer(id string) (*api.Container, error) {
return d.client.InspectContainer(id)
}
func (d docker) GetStats(id string, statsChan chan *api.Stats, done <-chan bool, stream bool) error {
return d.client.Stats(api.StatsOptions{
ID: id,
Stats: statsChan,
Timeout: time.Second * 2,
Done: done,
Stream: stream})
}
libs/docker/decoder.go 0 → 100644
package docker
import (
"bufio"
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
)
type streamDecoderWrapper struct {
out *bufio.Writer
err *bufio.Writer
buffer *bytes.Buffer
tty bool
lines map[string]int
diff int
}
type message struct {
Stream string `json:"stream,omitempty"`
Status string `json:"status,omitempty"`
Progress string `json:"progress,omitempty"`
ID string `json:"id,omitempty"`
Error string `json:"error,omitempty"`
}
func newStreamDecoderWrapper(out io.Writer, err io.Writer, tty bool) io.Writer {
buffer := &bytes.Buffer{}
lines := make(map[string]int)
diff := 0
return &streamDecoderWrapper{
out: bufio.NewWriter(out),
err: bufio.NewWriter(err),
buffer: buffer,
tty: tty,
lines: lines,
diff: diff}
}
func (w *streamDecoderWrapper) Write(p []byte) (int, error) {
defer w.Flush()
n, err := w.buffer.Write(p)
if err != nil {
return n, err
}
return n, w.Decode()
}
func (w streamDecoderWrapper) Flush() {
_ = w.err.Flush()
_ = w.out.Flush()
}
func (w streamDecoderWrapper) getOutWriter() io.Writer {
return w.out
}
func (w streamDecoderWrapper) getErrWriter() io.Writer {
return w.err
}
func (w streamDecoderWrapper) encode(m message) {
var endl string
if w.tty && m.Stream == "" && (m.Progress != "" || m.Status != "") {
// <ESC>[2K = erase entire current line
fmt.Fprintf(w.getOutWriter(), "%c[2K\r", 27)
endl = "\r"
} else if m.Progress != "" { //disable progressbar in non-terminal
return
}
if m.ID != "" {
fmt.Fprintf(w.getOutWriter(), "%s: ", m.ID)
}
if m.Progress != "" && w.tty {
fmt.Fprintf(w.getOutWriter(), "%s %s%s", m.Status, m.Progress, endl)
} else if m.Stream != "" {
fmt.Fprintf(w.getOutWriter(), "%s%s", m.Stream, endl)
} else {
fmt.Fprintf(w.getOutWriter(), "%s%s\n", m.Status, endl)
}
}
func (w *streamDecoderWrapper) setCursor(id string) {
if id != "" {
line, ok := w.lines[id]
if !ok {
// NOTE: This approach of using len(lines) to
// figure out the number of lines of history
// only works as long as we clear the history
// when we output something that's not
// accounted for in the map, such as a line
// with no ID.
line = len(w.lines)
w.lines[id] = line
if w.tty {
fmt.Fprintf(w.getOutWriter(), "\n")
}
} else {
w.diff = len(w.lines) - line
}
if w.tty {
// NOTE: this appears to be necessary even if
// diff == 0.
// <ESC>[{diff}A = move cursor up diff rows
fmt.Fprintf(w.getOutWriter(), "%c[%dA", 27, w.diff)
}
} else {
// When outputting something that isn't progress
// output, clear the history of previous lines. We
// don't want progress entries from some previous
// operation to be updated (for example, pull -a
// with multiple tags).
w.lines = make(map[string]int)
}
}
func (w *streamDecoderWrapper) Decode() error {
decoder := json.NewDecoder(w.buffer)
for {
m := message{}
if err := decoder.Decode(&m); err != nil {
if err != io.EOF {
return err
}
// Recopy remaining bytes into buffer to be available again for json decoder.
b, err := ioutil.ReadAll(decoder.Buffered())
if err != nil {
return err
}
w.buffer = bytes.NewBuffer(b)
return nil
}
if m.Error != "" {
return errors.New(m.Error)
}
w.setCursor(m.ID)
w.encode(m)
if m.ID != "" && w.tty {
// NOTE: this appears to be necessary even if
// diff == 0.
// <ESC>[{diff}B = move cursor down diff rows
fmt.Fprintf(w.getOutWriter(), "%c[%dB", 27, w.diff)
}
}
}
libs/docker/docker.go 0 → 100644
package docker
import (
api "github.com/fsouza/go-dockerclient"
)
// Docker wrap underlaying docker client to expose only required functions.
type Docker interface {
// Ping pings the docker server
Ping() error
// Logs attach a stream on a running container to read stdout and stderr
// output from docker logs.
Logs(id string, stream LogStream) error
// Run create and start a container to execute a runnable.
// Return the exit code of the container status, an error otherwise.
Run(option RunOptions, stream LogStream) (int, error)
// Build create a new image from a dockerfile.
Build(option BuildOptions, stream LogStream) error
// tag adds a tag to the image for a repository.
Tag(option TagOptions) error
// Push pushes an image to a remote registry.
Push(option PushOptions, stream LogStream) error
// ImageID returns an image ID by its name.
ImageID(name string) string
// RemoveImage removes an image by its name or ID.
RemoveImage(name string) error
GetContainer(id string) (*api.Container, error)
GetStats(id string, statsChan chan *api.Stats, done <-chan bool, stream bool) error
}
// The default implementation of the Docker interface.
type docker struct {
client *api.Client
}
// See Docker interface
func (d docker) Ping() error {
return d.client.Ping()
}
// New return a Docker client
func New(endpoint string) (Docker, error) {
c, err := api.NewClient(endpoint)
if err != nil {
return nil, err
}
d := &docker{}
d.client = c
if err = d.Ping(); err != nil {
return nil, err
}
return d, nil
}
libs/docker/image.go 0 → 100644
package docker
import (
api "github.com/fsouza/go-dockerclient"
)
// Image options
const (
rmiForce = false
rmiNoPrune = false
)
func (d docker) ImageID(name string) string {
i, err := d.client.InspectImage(name)
if err != nil {
return ""
}
return i.ID
}
func (d docker) RemoveImage(name string) error {
if i, _ := d.client.InspectImage(name); i != nil {
if err := d.client.RemoveImageExtended(name, removeImageOptions()); err != nil {
return err
}
}
return nil
}
func removeImageOptions() api.RemoveImageOptions {
return api.RemoveImageOptions{
Force: rmiForce,
NoPrune: rmiNoPrune,
}
}
libs/docker/log.go 0 → 100644
package docker
import (
"io"
"os"
//"github.com/andrew-d/go-termutil"
api "github.com/fsouza/go-dockerclient"
)
// LogStream contains two io.Writer for respectively, stdout and stderr, and also a JSON Decoder for the Docker API.
type LogStream struct {
Out io.Writer
Err io.Writer
Decoder io.Writer
}
// OutputStream return a stream output handler with its encoding.
func (l LogStream) OutputStream() (json bool, stream io.Writer) {
return true, l.Decoder
}
// See Docker interface
func (d docker) Logs(id string, stream LogStream) error {
return d.client.Logs(logsOptions(id, stream))
}
// NewLogStream return a default LogStream using OS stdout and stderr.
func NewLogStream() LogStream {
out := os.Stdout
err := os.Stderr
//decoder := newStreamDecoderWrapper(out, err, termutil.Isatty(out.Fd()))
decoder := newStreamDecoderWrapper(out, err, true)
return LogStream{Out: out, Err: err, Decoder: decoder}
}
func logsOptions(container string, stream LogStream) api.LogsOptions {
return api.LogsOptions{
Container: container,
OutputStream: stream.Out,
ErrorStream: stream.Err,
Follow: true,
Stdout: true,
Stderr: true,
Timestamps: false,
}
}
libs/docker/mock.go 0 → 100644
package docker
// Mock a Docker interface.
type Mock struct {
// Handler
RunHandler func(option RunOptions, stream LogStream) (int, error)
PingHandler func() error
LogsHandler func(id string, stream LogStream) error
BuildHandler func(option BuildOptions, stream LogStream) error
TagHandler func(option TagOptions) error
PushHandler func(option PushOptions, stream LogStream) error
ImageIDHandler func(name string) string
RemoveImageHandler func(name string) error
// Values
RunCalled bool
PingCalled bool
LogsCalled bool
BuildCalled bool
TagCalled bool
PushCalled bool
ImageIDCalled bool
RemoveImageCalled bool
RunOptions RunOptions
LogsID string
BuildOptions BuildOptions
TagOptions TagOptions
PushOptions PushOptions
ImageIDName string
RemoveImageName string
}
// NewMock return a mock of Docker interface.
func NewMock() *Mock {
d := &Mock{}
d.RunHandler = func(option RunOptions, stream LogStream) (int, error) {
return 0, nil
}
d.PingHandler = func() error {
return nil
}
d.LogsHandler = func(id string, stream LogStream) error {
return nil
}
d.BuildHandler = func(option BuildOptions, stream LogStream) error {
return nil
}
d.TagHandler = func(option TagOptions) error {
return nil
}
d.PushHandler = func(option PushOptions, stream LogStream) error {
return nil
}
d.ImageIDHandler = func(name string) string {
return ""
}
d.RemoveImageHandler = func(name string) error {
return nil
}
return d
}
// Run is a mock function for Docker interface.
func (m *Mock) Run(option RunOptions, stream LogStream) (int, error) {
m.RunCalled = true
m.RunOptions = option
return m.RunHandler(option, stream)
}
// Ping is a mock function for Docker interface.
func (m *Mock) Ping() error {
m.PingCalled = true
return m.PingHandler()
}
// Logs is a mock function for Docker interface.
func (m *Mock) Logs(id string, stream LogStream) error {
m.LogsCalled = true
m.LogsID = id
return m.LogsHandler(id, stream)
}
// Build is a mock function for Docker interface.
func (m *Mock) Build(option BuildOptions, stream LogStream) error {
m.BuildCalled = true
m.BuildOptions = option
return m.BuildHandler(option, stream)
}
// tag is a mock function for Docker interface.
func (m *Mock) Tag(option TagOptions) error {
m.TagCalled = true
m.TagOptions = option
return m.TagHandler(option)
}
// Push is a mock function for Docker interface.
func (m *Mock) Push(option PushOptions, stream LogStream) error {
m.PushCalled = true
m.PushOptions = option
return m.PushHandler(option, stream)
}
// ImageID is a mock function for Docker interface.
func (m *Mock) ImageID(name string) string {
m.ImageIDCalled = true
m.ImageIDName = name
return m.ImageIDHandler(name)
}
// RemoveImage is a mock function for Docker interface.
func (m *Mock) RemoveImage(name string) error {
m.RemoveImageCalled = true
m.RemoveImageName = name
return m.RemoveImageHandler(name)
}
libs/docker/parser.go 0 → 100644
package docker
import (
"strings"
parser "github.com/novln/docker-parser/docker"
)
// Reference is an opaque object that include identifier such as a name, tag, repository, registry, etc...
type Reference struct {
named parser.Named
tag string
}
// Name returns the image's name. (ie: debian[:8.2])
func (r Reference) Name() string {
return r.named.RemoteName() + r.tag
}
// ShortName returns the image's name (ie: debian)
func (r Reference) ShortName() string {
return r.named.RemoteName()
}
// tag returns the image's tag (or digest).
func (r Reference) Tag() string {
if len(r.tag) > 1 {
return r.tag[1:]
}
return ""
}
// Registry returns the image's registry. (ie: host[:port])
func (r Reference) Registry() string {
return r.named.Hostname()
}
// Repository returns the image's repository. (ie: registry/name)
func (r Reference) Repository() string {
return r.named.FullName()
}
// Remote returns the image's remote identifier. (ie: registry/name[:tag])
func (r Reference) Remote() string {
return r.named.FullName() + r.tag
}
func clean(url string) string {
s := url
if strings.HasPrefix(url, "http://") {
s = strings.Replace(url, "http://", "", 1)
} else if strings.HasPrefix(url, "https://") {
s = strings.Replace(url, "https://", "", 1)
}
return s
}
// Parse returns a Reference from analyzing the given remote identifier.
func Parse(remote string) (*Reference, error) {
n, err := parser.ParseNamed(clean(remote))
if err != nil {
return nil, err
}
n = parser.WithDefaultTag(n)
var t string
switch x := n.(type) {
case parser.Canonical:
t = "@" + x.Digest().String()
case parser.NamedTagged:
t = ":" + x.Tag()
}
return &Reference{named: n, tag: t}, nil
}
libs/docker/push.go 0 → 100644
package docker
import (
api "github.com/fsouza/go-dockerclient"
)
// PushOptions contains the push configuration for the docker daemon.
type PushOptions struct {
Name string
Repository string
Registry string
Tag string
}
// See Docker interface
func (d docker) Push(option PushOptions, stream LogStream) error {
return d.client.PushImage(pushImageOptions(option, stream), pushAuthConfiguration(option))
}
func pushImageOptions(option PushOptions, stream LogStream) api.PushImageOptions {
json, output := stream.OutputStream()
return api.PushImageOptions{
Name: option.Repository,
Tag: option.Tag,
OutputStream: output,
RawJSONStream: json,
}
}
func pushAuthConfiguration(option PushOptions) api.AuthConfiguration {
return getAuthWithRegistry(option.Registry)
}
libs/docker/render/docker.qtpl 0 → 100644
{% func RenderDockerfile(fromImage, srcCode, workdir string, copyCode bool, steps []string, envs map[string]string ) %}
FROM {%s fromImage %}
ENV SHELL /bin/bash
{% if copyCode %}
COPY {%s srcCode %} {%s workdir %}
{% endif %}
WORKDIR {%s workdir %}
{%for _, step := range steps%}
RUN {%s step %}
{% endfor %}
{%for key, value := range envs%}
ENV {%s key + " " + value %}
{% endfor %}
{% endfunc %}
libs/docker/render/docker.qtpl.go 0 → 100644
// This file is automatically generated by qtc from "docker.qtpl".
// See https://github.com/valyala/quicktemplate for details.
//line docker.qtpl:1
package render
//line docker.qtpl:1
import (
qt422016 "github.com/adl-golang/quicktemplate"
qtio422016 "io"
)
//line docker.qtpl:1
var (
_ = qtio422016.Copy
_ = qt422016.AcquireByteBuffer
)
//line docker.qtpl:1
func StreamRenderDockerfile(qw422016 *qt422016.Writer, fromImage, srcCode, workdir string, copyCode bool, steps []string, envs map[string]string) {
//line docker.qtpl:1
qw422016.N().S(`
FROM `)
//line docker.qtpl:2
qw422016.E().S(fromImage)
//line docker.qtpl:2
qw422016.N().S(`
ENV SHELL /bin/bash
`)
//line docker.qtpl:5
if copyCode {
//line docker.qtpl:5
qw422016.N().S(`
COPY `)
//line docker.qtpl:6
qw422016.E().S(srcCode)
//line docker.qtpl:6
qw422016.N().S(` `)
//line docker.qtpl:6
qw422016.E().S(workdir)
//line docker.qtpl:6
qw422016.N().S(`
`)
//line docker.qtpl:7
}
//line docker.qtpl:7
qw422016.N().S(`
WORKDIR `)
//line docker.qtpl:9
qw422016.E().S(workdir)
//line docker.qtpl:9
qw422016.N().S(`
`)
//line docker.qtpl:11
for _, step := range steps {
//line docker.qtpl:11
qw422016.N().S(`
RUN `)
//line docker.qtpl:12
qw422016.E().S(step)
//line docker.qtpl:12
qw422016.N().S(`
`)
//line docker.qtpl:13
}
//line docker.qtpl:13
qw422016.N().S(`
`)
//line docker.qtpl:15
for key, value := range envs {
//line docker.qtpl:15
qw422016.N().S(`
ENV `)
//line docker.qtpl:16
qw422016.E().S(key + " " + value)
//line docker.qtpl:16
qw422016.N().S(`
`)
//line docker.qtpl:17
}
//line docker.qtpl:17
qw422016.N().S(`
`)
//line docker.qtpl:19
}
//line docker.qtpl:1
func StreamRenderDockerfileRunFirst(qw422016 *qt422016.Writer, fromImage, srcCode, workdir string, copyCode bool, steps []string, envs map[string]string) {
//line docker.qtpl:1
qw422016.N().S(`
FROM `)
//line docker.qtpl:2
qw422016.E().S(fromImage)
//line docker.qtpl:2
qw422016.N().S(`
ENV SHELL /bin/bash
`)
//line docker.qtpl:7
qw422016.N().S(`
WORKDIR `)
//line docker.qtpl:9
qw422016.E().S(workdir)
//line docker.qtpl:9
qw422016.N().S(`
`)
//line docker.qtpl:11
for _, step := range steps {
//line docker.qtpl:11
qw422016.N().S(`
RUN `)
//line docker.qtpl:12
qw422016.E().S(step)
//line docker.qtpl:12
qw422016.N().S(`
`)
//line docker.qtpl:13
}
//line docker.qtpl:13
qw422016.N().S(`
`)
//line docker.qtpl:5
if copyCode {
//line docker.qtpl:5
qw422016.N().S(`
COPY `)
//line docker.qtpl:6
qw422016.E().S(srcCode)
//line docker.qtpl:6
qw422016.N().S(` `)
//line docker.qtpl:6
qw422016.E().S(workdir)
//line docker.qtpl:6
qw422016.N().S(`
`)
//line docker.qtpl:7
}
//line docker.qtpl:15
for key, value := range envs {
//line docker.qtpl:15
qw422016.N().S(`
ENV `)
//line docker.qtpl:16
qw422016.E().S(key + " " + value)
//line docker.qtpl:16
qw422016.N().S(`
`)
//line docker.qtpl:17
}
//line docker.qtpl:17
qw422016.N().S(`
`)
//line docker.qtpl:19
}
//line docker.qtpl:19
func WriteRenderDockerfile(qq422016 qtio422016.Writer, fromImage, srcCode, workdir string, copyCode bool, steps []string, envs map[string]string, copyAfterRun bool) {
//line docker.qtpl:19
qw422016 := qt422016.AcquireWriter(qq422016)
//line docker.qtpl:19
if copyAfterRun {
StreamRenderDockerfileRunFirst(qw422016, fromImage, srcCode, workdir, copyCode, steps, envs)
} else {
StreamRenderDockerfile(qw422016, fromImage, srcCode, workdir, copyCode, steps, envs)
}
//line docker.qtpl:19
qt422016.ReleaseWriter(qw422016)
//line docker.qtpl:19
}
//line docker.qtpl:19
//func RenderDockerfile(fromImage, srcCode, workdir string, copyCode bool, steps []string, envs map[string]string) string {
// //line docker.qtpl:19
// qb422016 := qt422016.AcquireByteBuffer()
// //line docker.qtpl:19
// WriteRenderDockerfile(qb422016, fromImage, srcCode, workdir, copyCode, steps, envs)
// //line docker.qtpl:19
// qs422016 := string(qb422016.B)
// //line docker.qtpl:19
// qt422016.ReleaseByteBuffer(qb422016)
// //line docker.qtpl:19
// return qs422016
// //line docker.qtpl:19
//}
libs/docker/render/render_test.go 0 → 100644
package render
import (
"bytes"
"fmt"
"testing"
)
func TestRender(t *testing.T) {
var buf bytes.Buffer
WriteRenderDockerfile(&buf, "ubuntu:16.04", "/source_code",
"/code", true, []string{"run 1", "run 2"},
map[string]string{"env1": "value1", "env2": "value2"}, false)
fmt.Printf("buf=\n%s", buf.Bytes())
}
libs/docker/run.go 0 → 100644
package docker
import (
"fmt"
"strings"
api "github.com/fsouza/go-dockerclient"
)
// Run options
const (
containerName = ""
useTTy = false
attachStdout = true
attachStderr = true
hostNetworkMode = "bridge"
networkDisabled = false
removeVolumes = true
forceRemove = false
)
// RunOptions contains the run configuration of the docker container
type RunOptions struct {
Image string
Command string
Env []string
Volumes []string
Links []string
}
// See Docker interface
func (d docker) Run(option RunOptions, stream LogStream) (int, error) {
r, err := Parse(option.Image)
if err != nil {
return 0, err
}
err = d.client.PullImage(pullImageOptions(r.Remote(), stream), pullAuthConfiguration(option))
if err != nil {
return 0, err
}
e, err := d.client.CreateContainer(createContainerOptions(option))
if err != nil {
return 0, err
}
id := e.ID
if err = d.client.StartContainer(id, nil); err != nil {
return 0, err
}
err = d.Logs(id, stream)
if err != nil {
fmt.Fprint(stream.Err, err)
}
exit, err := d.client.WaitContainer(id)
if err != nil {
return 0, err
}
if err = d.client.RemoveContainer(removeContainerOptions(id)); err != nil {
return 0, err
}
return exit, nil
}
func pullAuthConfiguration(option RunOptions) api.AuthConfiguration {
return getAuthWithImage(option.Image)
}
func pullImageOptions(remote string, stream LogStream) api.PullImageOptions {
json, output := stream.OutputStream()
return api.PullImageOptions{
Repository: remote,
OutputStream: output,
RawJSONStream: json,
}
}
func createContainerOptions(option RunOptions) api.CreateContainerOptions {
return api.CreateContainerOptions{
Name: containerName,
Config: &api.Config{
AttachStdout: attachStdout,
AttachStderr: attachStderr,
Tty: useTTy,
Env: option.Env,
NetworkDisabled: networkDisabled,
Image: option.Image,
Cmd: strings.Fields(option.Command),
},
HostConfig: &api.HostConfig{
NetworkMode: hostNetworkMode,
Binds: option.Volumes,
Links: option.Links,
},
}
}
func removeContainerOptions(id string) api.RemoveContainerOptions {
return api.RemoveContainerOptions{
ID: id,
RemoveVolumes: removeVolumes,
Force: forceRemove,
}
}
libs/docker/tag.go 0 → 100644
package docker
import (
api "github.com/fsouza/go-dockerclient"
)
// tag options
const (
tagForce = true
)
// TagOptions contains the tag configuration for the docker daemon.
type TagOptions struct {
Name string
Repository string
Tag string
}
// See Docker interface
func (d docker) Tag(option TagOptions) error {
return d.client.TagImage(option.Name, tagImageOptions(option))
}
func tagImageOptions(option TagOptions) api.TagImageOptions {
return api.TagImageOptions{
Repo: option.Repository,
Tag: option.Tag,
Force: tagForce,
}
}
libs/errors.go 0 → 100644
package libs
import "errors"
var (
TaskIsDoneCanNotSetStatusError = errors.New("This task is done, so you can not set status on it, drop it! ")
TaskStatusExistCanNotSetStatusError = errors.New("This task status is exist, so you can not set status on it, drop it! ")
)
libs/function.go 0 → 100644
package libs
import (
"fmt"
)
func FailOnError(err error, msg string) {
if err != nil {
panic(fmt.Sprintf("%s: %s", msg, err))
}
}
libs/parser/spawner.go 0 → 100644
package parser
import (
log "github.com/cihub/seelog"
"github.com/gin-gonic/gin/json"
"gitlab.seetatech.com/autodl.com/autodl-base.git"
"gitlab.seetatech.com/autodl.com/autodl-base.git/protoV1"
"gitlab.seetatech.com/autodl.com/autodl-k8s.git/spawner"
)
func GetSpawnerParserConfig(task *proto_v1.Task, imageNameTag, serviceID string) (config *spawner.ParsedConfig) {
config = &spawner.ParsedConfig{
ID: task.TaskId,
Mount: []autodl_base.MountPath{},
Envs: make(map[string]string),
Cmd: task.BuildInfo.Cmd,
Image: imageNameTag,
Resource: autodl_base.ATCResource{},
MachineSelector: make(map[string]string),
HasService: task.PodInfo.ServiceType != proto_v1.PodInfo_NoService,
HasShareMemory: task.PodInfo.HaveShm,
ServiceID: serviceID,
}
for _, mount := range task.BuildInfo.Mounts {
if len(mount.CephPath) == 0 || len(mount.MountPath) == 0 {
continue
}
config.Mount = append(config.Mount, autodl_base.MountPath{
CephPath: mount.CephPath,
MountPath: mount.MountPath,
ReadOnly: mount.ReadOnly,
})
}
for _, env := range task.BuildInfo.Envs {
config.Envs[env.Key] = env.Value
}
config.Envs["PYTHONUNBUFFERED"] = "1"
config.Resource.Default = autodl_base.ATCPodResource{
CPU: autodl_base.ATCK8SResource{
Requests: task.PodInfo.Resource.Cpu.Requests,
Limits: task.PodInfo.Resource.Cpu.Limits,
},
GPU: autodl_base.ATCK8SResource{
Requests: task.PodInfo.Resource.Gpu.Requests,
Limits: task.PodInfo.Resource.Gpu.Limits,
},
Mem: autodl_base.ATCK8SResource{
Requests: task.PodInfo.Resource.Memory.Requests,
Limits: task.PodInfo.Resource.Memory.Limits,
},
}
if task.PodInfo.ServiceType != proto_v1.PodInfo_NoService {
for _, port := range task.PodInfo.Ports {
config.Ports = append(config.Ports, int(port))
}
}
for _, machineSelector := range task.PodInfo.MachineSelector {
config.MachineSelector[machineSelector.Key] = machineSelector.Value
}
t, e := json.Marshal(config)
log.Info("task config: ", string(t), e)
return
}
libs/path/base.go 0 → 100644
package path
import (
log "github.com/cihub/seelog"
"io"
"os"
)
func DeletePath(path string) error {
if _, err := os.Stat(path); err == nil {
return os.RemoveAll(path)
}
return nil
}
func CopyFolder(source string, dest string) (err error) {
sourceinfo, err := os.Stat(source)
if err != nil {
return err
}
err = os.MkdirAll(dest, sourceinfo.Mode())
if err != nil {
return err
}
directory, _ := os.Open(source)
objects, err := directory.Readdir(-1)
for _, obj := range objects {
sourcefilepointer := source + "/" + obj.Name()
destinationfilepointer := dest + "/" + obj.Name()
if obj.IsDir() {
err = CopyFolder(sourcefilepointer, destinationfilepointer)
if err != nil {
log.Error("copy folder error: ", err.Error())
}
} else {
err = CopyFile(sourcefilepointer, destinationfilepointer)
if err != nil {
log.Error("copy file error: ", err.Error())
}
}
}
return
}
func CopyFile(source string, dest string) (err error) {
sourcefile, err := os.Open(source)
if err != nil {
return err
}
defer sourcefile.Close()
destfile, err := os.Create(dest)
if err != nil {
return err
}
defer destfile.Close()
_, err = io.Copy(destfile, sourcefile)
if err == nil {
sourceinfo, err := os.Stat(source)
if err == nil {
err = os.Chmod(dest, sourceinfo.Mode())
}
}
return
}
libs/path/log.go 0 → 100644
package path
import (
"fmt"
"github.com/spf13/viper"
"os"
)
func InitServiceDir(serviceID string) error {
return os.MkdirAll(fmt.Sprintf("%s/adl_core/log/%s", viper.GetString("app.mount_path"), serviceID), 0777)
}
func GetTaskLogFilePath(serviceID, taskID string) string {
return fmt.Sprintf("%s/adl_core/log/%s/%s.log", viper.GetString("app.mount_path"), serviceID, taskID)
}
libs/resource.go 0 → 100644
package libs
import (
"encoding/json"
linuxproc "github.com/c9s/goprocinfo/linux"
"github.com/mindprince/gonvml"
"github.com/pkg/errors"
"runtime"
"time"
)
type ResourceSummary struct {
NodeName string `json:"node_name"`
CpuInfo *CpuInfo `json:"cpu_info"`
GpuInfo *GpuInfo `json:"gpu_info"`
MemInfo *MemInfo `json:"mem_info"`
UpdatedAt time.Time `json:"updated_at"`
}
func (rs *ResourceSummary) Parse(input string) error {
return json.Unmarshal([]byte(input), &rs)
}
func (rs *ResourceSummary) ToString() string {
tmp, _ := json.Marshal(rs)
return string(tmp)
}
type GpuInfo struct {
DriverVersion string `json:"driver_version"`
GPUS []GpuUnit `json:"gpus"`
}
type GpuUnit struct {
Index uint `json:"index"`
UUID string `json:"uuid"`
Name string `json:"name"`
MemoryUsed uint64 `json:"memory_used"`
MemoryTotal uint64 `json:"memory_total"`
GpuUtilization uint `json:"gpu_utilization"`
MemoryUtilization uint `json:"mem_utilization"`
PowerDraw uint `json:"power_draw"`
Temperature uint `json:"temperature"`
FanSpeed uint `json:"fan_speed"`
}
func GetGpuResourcesSummary() (gpuInfo *GpuInfo, err error) {
err = gonvml.Initialize()
if err != nil {
return
}
defer gonvml.Shutdown()
var driverVersion string
driverVersion, err = gonvml.SystemDriverVersion()
if err != nil {
err = errors.Wrap(err, "SystemDriverVersion() failed")
return
}
var numDevices uint
numDevices, err = gonvml.DeviceCount()
if err != nil {
err = errors.Wrap(err, "DeviceCount() failed")
return
}
gpuInfo = &GpuInfo{
DriverVersion: driverVersion,
GPUS: make([]GpuUnit, numDevices),
}
for i := 0; i < int(numDevices); i++ {
gpuRes := GpuUnit{}
var dev gonvml.Device
dev, err = gonvml.DeviceHandleByIndex(uint(i))
if err != nil {
err = errors.Wrap(err, "DeviceHandleByIndex() failed")
return
}
gpuRes.UUID, err = dev.UUID()
if err != nil {
err = errors.Wrap(err, "dev.UUID() failed")
return
}
gpuRes.Index, err = dev.MinorNumber()
if err != nil {
err = errors.Wrap(err, "dev.MinorNumber() failed")
return
}
gpuRes.Name, err = dev.Name()
if err != nil {
err = errors.Wrap(err, "dev.Name() failed")
return
}
gpuRes.MemoryTotal, gpuRes.MemoryUsed, err = dev.MemoryInfo()
if err != nil {
err = errors.Wrap(err, "dev.MemoryInfo() failed")
return
}
gpuRes.GpuUtilization, gpuRes.MemoryUtilization, err = dev.UtilizationRates()
if err != nil {
err = errors.Wrap(err, "dev.UtilizationRates() failed")
return
}
gpuRes.PowerDraw, err = dev.PowerUsage()
if err != nil {
err = errors.Wrap(err, "dev.PowerUsage() failed")
return
}
gpuRes.FanSpeed, err = dev.FanSpeed()
if err != nil {
err = errors.Wrap(err, "dev.FanSpeed() failed")
return
}
gpuRes.Temperature, err = dev.Temperature()
if err != nil {
err = errors.Wrap(err, "dev.Temperature() failed")
return
}
gpuInfo.GPUS[i] = gpuRes
}
return
}
type CpuInfo struct {
Count int `json:"count"`
linuxproc.LoadAvg
}
func GetCpuInfo() (cpuInfo *CpuInfo, err error) {
var loadInfo *linuxproc.LoadAvg
cpuInfo = &CpuInfo{Count: runtime.NumCPU()}
loadInfo, err = linuxproc.ReadLoadAvg("/proc/loadavg")
if err != nil {
return
}
cpuInfo.LoadAvg = *loadInfo
return
}
type MemInfo struct {
MemTotal uint64 `json:"mem_total"`
MemFree uint64 `json:"mem_free"`
MemAvailable uint64 `json:"mem_available"`
}
func GetMemInfo() (memInfo *MemInfo, err error) {
var readMem *linuxproc.MemInfo
memInfo = &MemInfo{}
readMem, err = linuxproc.ReadMemInfo("/proc/meminfo")
if err != nil {
return
}
memInfo.MemTotal = readMem.MemTotal
memInfo.MemFree = readMem.MemFree
memInfo.MemAvailable = readMem.MemAvailable
return
}
libs/task.go 0 → 100644
package libs
type TaskType string
const (
RunTask TaskType = "run"
StopTask TaskType = "stop"
)
models/db.go 0 → 100644
package models
import (
"fmt"
log "github.com/cihub/seelog"
_ "github.com/go-sql-driver/mysql"
"github.com/jinzhu/gorm"
"github.com/pkg/errors"
"github.com/spf13/viper"
"time"
)
var (
NoFilterSetError = errors.New("No filter set is forbidden in database")
)
func DBProvider() *gorm.DB {
var typ = viper.GetString("database.type")
var url = fmt.Sprintf(viper.GetString("database.url"), viper.GetString("database.user"), viper.GetString("database.password"), viper.GetString("database.host"), viper.GetString("database.name"))
var err error
log.Info("connect to database...", typ, url)
dbConn, err := gorm.Open(typ, url)
// defer dbConn.Close()
for err != nil {
log.Error("connect to db failed: ", err.Error())
time.Sleep(5 * time.Second)
dbConn, err = gorm.Open("mysql", url)
}
log.Info("connect to database success...")
dbConn.DB().SetMaxIdleConns(10)
dbConn.DB().SetConnMaxLifetime(time.Hour * 8)
dbConn.DB().SetMaxOpenConns(10)
dbConn.LogMode(true)
return dbConn
}
models/job.go 0 → 100644
package models
import (
"errors"
"github.com/jinzhu/gorm"
"time"
)
const jobTableName = "job"
type Job struct {
ID int `gorm:"column:id;NOT NULL;PRIMARY KEY;" json:"id"`
ServiceID string `gorm:"column:service_id;NOT NULL;" json:"service_id"`
TaskID string `gorm:"column:task_id;NOT NULL;" json:"task_id"`
Definition []byte `gorm:"column:definition;NOT NULL;" json:"definition"`
CreatedAt time.Time `gorm:"column:created_at;NOT NULL;" json:"created_at"`
UpdatedAt time.Time `gorm:"column:updated_at;NOT NULL;" json:"updated_at"`
HasService bool `gorm:"column:has_service" json:"has_service"`
ServiceIsClean bool `gorm:"column:service_is_clean" json:"service_is_clean"`
LogIsPersist bool `gorm:"column:log_is_persist" json:"log_is_persist"`
LogIsFlushOver bool `gorm:"column:log_is_flush_over" json:"log_is_flush_over"`
}
func (j *Job) TableName() string {
return jobTableName
}
type JobModel struct{}
func JobModelProvider() *JobModel {
return &JobModel{}
}
func (model *JobModel) Create(dbConn *gorm.DB, job *Job) (err error) {
return dbConn.Create(job).Error
}
func (model *JobModel) Get(dbConn *gorm.DB, serviceID, taskID string) (job *Job, err error) {
if len(taskID) == 0 || len(serviceID) == 0 {
err = gorm.ErrRecordNotFound
return
}
job = &Job{}
err = dbConn.Where(&Job{ServiceID: serviceID, TaskID: taskID}).First(job).Error
return
}
func (model *JobModel) GetInTaskIDs(dbConn *gorm.DB, serviceID string, taskIDs []string) (job []Job, err error) {
if len(taskIDs) == 0 || len(serviceID) == 0 {
err = gorm.ErrRecordNotFound
return
}
err = dbConn.Where(&Job{ServiceID: serviceID}).Where(" task_id in (?)", taskIDs).Find(&job).Error
return
}
func (model *JobModel) CleanServiceFinished(dbConn *gorm.DB, serviceID, taskID string) (err error) {
if len(taskID) == 0 {
err = errors.New("can not update when no filter")
return
}
err = dbConn.Table(jobTableName).Where(&Job{ServiceID: serviceID, TaskID: taskID}).Updates(&Job{
ServiceIsClean: true,
}).Error
return
}
func (model *JobModel) UpdateDefinition(dbConn *gorm.DB, serviceID, taskID string, definition []byte) (err error) {
if len(taskID) == 0 {
err = errors.New("can not update when no filter")
return
}
err = dbConn.Table(jobTableName).Where(&Job{ServiceID: serviceID, TaskID: taskID}).Updates(&Job{
Definition: definition,
}).Error
return
}
func (model *JobModel) LogPersistFinished(dbConn *gorm.DB, serviceID, taskID string) (err error) {
if len(taskID) == 0 {
err = errors.New("can not update when no filter")
return
}
err = dbConn.Table(jobTableName).Where(&Job{ServiceID: serviceID, TaskID: taskID}).Updates(&Job{
LogIsPersist: true,
}).Error
return
}
func (model *JobModel) LogFlushOver(dbConn *gorm.DB, serviceID, taskID string) (err error) {
if len(taskID) == 0 {
err = errors.New("can not update when no filter")
return
}
err = dbConn.Table(jobTableName).Where(&Job{ServiceID: serviceID, TaskID: taskID}).Updates(&Job{
LogIsFlushOver: true,
}).Error
return
}
\ No newline at end of file
models/service.go 0 → 100644
package models
import (
"github.com/jinzhu/gorm"
"time"
)
const serviceTableName = "service"
type Service struct {
ID int `gorm:"column:id;NOT NULL;PRIMARY KEY;" json:"id"`
ServiceID string `gorm:"column:service_id;NOT NULL;" json:"service_id"`
NotiAPI string `gorm:"column:noti_api;NOT NULL;" json:"noti_api"`
CreatedAt time.Time `gorm:"column:created_at;NOT NULL;" json:"created_at"`
}
func (j *Service) TableName() string {
return serviceTableName
}
type ServiceModel struct{}
func ServiceModelProvider() *ServiceModel {
return &ServiceModel{}
}
func (model *ServiceModel) Get(dbConn *gorm.DB, serviceID string) (service *Service, err error) {
if len(serviceID) == 0 {
err = gorm.ErrRecordNotFound
return
}
service = &Service{}
err = dbConn.Where(&Service{
ServiceID: serviceID,
}).First(service).Error
return
}
models/task.go 0 → 100644
package models
import (
"autodl-core/libs"
"github.com/jinzhu/gorm"
"time"
)
const taskTableName = "task"
type Task struct {
ID int `gorm:"column:id;NOT NULL;PRIMARY KEY;" json:"-"`
ServiceID string `gorm:"column:service_id;NOT NULL;" json:"service_id"`
TaskType libs.TaskType `gorm:"column:task_type;" json:"task_type"`
TaskList string `gorm:"column:task_list;" json:"task_list"`
CreatedAt time.Time `gorm:"column:created_at;NOT NULL;" json:"created_at"`
}
func (ts *Task) TableName() string {
return taskTableName
}
type TaskModel struct{}
func TaskModelProvider() *TaskModel {
return &TaskModel{}
}
func (model *TaskModel) Create(dbConn *gorm.DB, task *Task) (id int, err error) {
err = dbConn.Create(task).Error
id = task.ID
return
}
func (model *TaskModel) GetLast(dbConn *gorm.DB, id int) (task *Task, err error) {
if id == 0 {
err = gorm.ErrRecordNotFound
return
}
task = &Task{}
err = dbConn.Where(&Task{
ID: id,
}).Last(task).Error
return
}
models/task_status.go 0 → 100644
package models
import (
"autodl-core/libs"
log "github.com/cihub/seelog"
"github.com/jinzhu/gorm"
"gitlab.seetatech.com/autodl.com/autodl-base.git"
"strings"
"time"
)
const taskStatusTableName = "task_status"
type TaskStatus struct {
ID int `gorm:"column:id;NOT NULL;PRIMARY KEY;" json:"-"`
ServiceID string `gorm:"column:service_id;NOT NULL;" json:"service_id"`
TaskID string `gorm:"column:task_id;" json:"task_id"`
Status string `gorm:"column:status;" json:"status"`
Msg string `gorm:"column:msg;" json:"msg"`
IsNotifySuccess bool `gorm:"column:is_notify_success" json:"is_notify_success"`
CreatedAt time.Time `gorm:"column:created_at;NOT NULL;" json:"created_at"`
}
func (ts *TaskStatus) TableName() string {
return taskStatusTableName
}
type TaskStatusModel struct{}
func TaskStatusModelProvider() *TaskStatusModel {
return &TaskStatusModel{}
}
func (model *TaskStatusModel) create(dbConn *gorm.DB, taskStatus *TaskStatus) (err error) {
return dbConn.Create(taskStatus).Error
}
func (model *TaskStatusModel) GetLast(dbConn *gorm.DB, serviceID, taskID string) (taskStatus *TaskStatus, err error) {
if len(serviceID) == 0 || len(taskID) == 0 {
err = gorm.ErrRecordNotFound
return
}
taskStatus = &TaskStatus{}
err = dbConn.Where(&TaskStatus{
ServiceID: serviceID,
TaskID: taskID,
}).Last(taskStatus).Error
return
}
func (model *TaskStatusModel) getLast(dbConn *gorm.DB, filter *TaskStatus) (taskStatus *TaskStatus, err error) {
taskStatus = &TaskStatus{}
err = dbConn.Where(filter).Last(taskStatus).Error
return
}
func (model *TaskStatusModel) IsStatusNotifySuccess(dbConn *gorm.DB, serviceID, taskID, status string) bool {
ts, err := model.getLast(dbConn, &TaskStatus{ServiceID: serviceID, TaskID: taskID, Status: status})
if gorm.IsRecordNotFoundError(err) {
return false
}
return ts.IsNotifySuccess
}
func (model *TaskStatusModel) SetStatusNotifySuccess(dbConn *gorm.DB, serviceID, taskID, status string) error {
err := dbConn.Table(taskStatusTableName).Where(&TaskStatus{ServiceID: serviceID, TaskID: taskID, Status: status}).Updates(&TaskStatus{IsNotifySuccess: true}).Error
return err
}
func (model *TaskStatusModel) checkTaskStatusCanSet(dbConn *gorm.DB, serviceID, taskID, status string) error {
ts, err := model.getLast(dbConn, &TaskStatus{ServiceID: serviceID, TaskID: taskID})
if err == nil {
if autodl_base.TaskIsDone(ts.Status) {
return libs.TaskIsDoneCanNotSetStatusError
}
if ts.Status == status {
return libs.TaskStatusExistCanNotSetStatusError
}
}
_, err = model.getLast(dbConn, &TaskStatus{ServiceID: serviceID, TaskID: taskID, Status: status})
if !gorm.IsRecordNotFoundError(err) {
return libs.TaskStatusExistCanNotSetStatusError
}
return nil
}
func (model *TaskStatusModel) SetTaskStatus(dbConn *gorm.DB, serviceID, taskID, status, msg string) (err error) {
log.Info("set status: ", strings.Join([]string{serviceID, taskID, status, msg}, ", "))
err = model.checkTaskStatusCanSet(dbConn, serviceID, taskID, status)
if err != nil {
return
}
newTaskStatus := TaskStatus{
ServiceID: serviceID,
TaskID: taskID,
Status: status,
Msg: msg,
CreatedAt: time.Now(),
}
err = model.create(dbConn, &newTaskStatus)
return
}
mongodb_plugin/mongodb_plugin_suite_test.go 0 → 100644
package mongodb_plugin_test
import (
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestMongodbPlugin(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "MongodbPlugin Suite")
}
mongodb_plugin/options.go 0 → 100644
package mongodb_plugin
import (
"autodl-core/libs"
"context"
log "github.com/cihub/seelog"
"github.com/pkg/errors"
"github.com/spf13/viper"
"gitlab.seetatech.com/autodl.com/autodl-base.git"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"go.mongodb.org/mongo-driver/mongo/readpref"
"go.mongodb.org/mongo-driver/x/bsonx"
"io"
"time"
)
type LogMongoClient struct {
mongoClient *mongo.Client
}
func MongoClientProvider() (mongoClient *LogMongoClient) {
mongoClient = &LogMongoClient{}
err := libs.RunnerFailedRetry(mongoClient.initMongoClient, 10)
if err != nil {
panic("panic!!!! failed to init mongodb: " + err.Error())
}
return
}
func (mc *LogMongoClient) initMongoClient() (err error) {
log.Info("init mongo client: ", viper.GetString("mongodb.addr"))
ctx, _ := context.WithTimeout(context.Background(), 5*time.Second)
mc.mongoClient, err = mongo.Connect(ctx, options.Client().ApplyURI(viper.GetString("mongodb.addr")), options.Client().SetMaxPoolSize(30))
if err != nil {
err = errors.Wrap(err, "init mongo db failed: "+err.Error())
log.Error(err.Error())
return
}
err = mc.mongoClient.Ping(ctx, readpref.Primary())
if err != nil {
err = errors.Wrap(err, "init mongo db failed: "+err.Error())
log.Error(err.Error())
return
}
err = mc.addIndex()
if err != nil {
err = errors.Wrap(err, "error when add index"+err.Error())
log.Error(err.Error())
}
return
}
type mongoIndex struct {
Key map[string]int
NS string
Name string
}
func (mc *LogMongoClient) addIndex() (err error) {
col := mc.mongoClient.Database(autodl_base.MongoDBCoreDatabaseName).Collection(autodl_base.MongoDBCoreLogCollectionName)
cursor, err := col.Indexes().List(context.Background())
if err != nil {
err = errors.Wrap(err, "get mongo index failed")
return
}
for cursor.Next(context.Background()) {
var idx mongoIndex
err = cursor.Decode(&idx)
if err != nil {
err = errors.Wrap(err, "decode mongo index failed")
continue
}
if _, ok := idx.Key["service_id"]; ok {
if _, ok := idx.Key["task_id"]; ok {
return
}
}
}
_, err = col.Indexes().CreateOne(
context.Background(),
mongo.IndexModel{
Keys: bsonx.Doc{{"service_id", bsonx.Int32(1)}, {"task_id", bsonx.Int32(1)}},
Options: options.Index().
SetBackground(true).
SetName("service_id-task_id").
SetVersion(1),
},
)
return
}
func (mc *LogMongoClient) AddLog(taskLogs ...autodl_base.TaskLog) (err error) {
err = mc.addLog(taskLogs...)
if err != nil {
mc.mongoClient = MongoClientProvider().mongoClient
err = mc.addLog(taskLogs...)
}
return
}
func (mc *LogMongoClient) addLog(taskLogs ...autodl_base.TaskLog) (err error) {
if len(taskLogs) == 0 {
return
}
col := mc.mongoClient.Database(autodl_base.MongoDBCoreDatabaseName).Collection(autodl_base.MongoDBCoreLogCollectionName)
ctx, _ := context.WithTimeout(context.Background(), 5*time.Second)
var content autodl_base.TaskLog
for index, taskLog := range taskLogs {
if index == 0 {
content = taskLog
} else {
content.Message += taskLog.Message
}
}
_, err = col.InsertOne(ctx, content)
return
}
func (mc *LogMongoClient) DeleteLog(serviceID, taskID string) (deleteCount int64, err error) {
col := mc.mongoClient.Database(autodl_base.MongoDBCoreDatabaseName).Collection(autodl_base.MongoDBCoreLogCollectionName)
query := bson.M{
"service_id": serviceID,
"task_id": taskID,
}
var deleteResult *mongo.DeleteResult
deleteResult, err = col.DeleteMany(context.Background(), query, &options.DeleteOptions{})
if err != nil {
return
}
deleteCount = deleteResult.DeletedCount
return
}
func (mc *LogMongoClient) GetAllLog(serviceID, taskID string, logChan chan<- autodl_base.TaskLogChanStruct) {
var err error
tlStruct := autodl_base.TaskLogChanStruct{}
defer func() {
if err == nil {
err = io.EOF
}
tlStruct.Error = err
logChan <- tlStruct
}()
query := bson.M{
"service_id": serviceID,
"task_id": taskID,
}
col := mc.mongoClient.Database(autodl_base.MongoDBCoreDatabaseName).Collection(autodl_base.MongoDBCoreLogCollectionName)
ctx, _ := context.WithTimeout(context.Background(), 5*time.Second)
var cur *mongo.Cursor
cur, err = col.Find(ctx, query, &options.FindOptions{})
if err != nil {
return
}
var taskLog autodl_base.TaskLog
for cur.Next(ctx) {
err = cur.Decode(&taskLog)
if err != nil {
err = errors.Wrap(err, "decode log failed")
return
}
tlStruct.TaskLog = taskLog
logChan <- tlStruct
}
if cur.Err() != nil {
err = cur.Err()
}
}
func (mc *LogMongoClient) GetPagedLogOrderByIDASC(serviceID, taskID, offsetID string, limit int64, logChan chan<- autodl_base.TaskLogChanStruct) {
var err error
tlStruct := autodl_base.TaskLogChanStruct{}
defer func() {
if err == nil {
err = io.EOF
}
tlStruct.Error = err
logChan <- tlStruct
}()
if len(taskID) == 0 {
return
}
if offsetID != "" && len(offsetID) < 10 {
err = errors.Errorf("wrong offsetID, only log offsetID is permitted, %s provided", offsetID)
return
}
col := mc.mongoClient.Database(autodl_base.MongoDBCoreDatabaseName).Collection(autodl_base.MongoDBCoreLogCollectionName)
ctx, _ := context.WithTimeout(context.Background(), 5*time.Second)
query := bson.M{
"service_id": serviceID,
"task_id": taskID,
}
if offsetID != "" {
var objectID primitive.ObjectID
objectID, err = primitive.ObjectIDFromHex(offsetID)
query["_id"] = bson.D{{"$gt", objectID}}
}
findOptions := &options.FindOptions{}
findOptions.SetSort(bsonx.Doc{{"_id", bsonx.Int32(1)}})
if limit > 0 {
if limit > 200 {
limit = 200
}
} else {
limit = 10
}
findOptions.SetLimit(limit)
var cur *mongo.Cursor
cur, err = col.Find(ctx, query, findOptions)
if err != nil {
err = errors.Wrap(err, "mongodb col find failed")
return
}
var taskLog autodl_base.TaskLog
for cur.Next(ctx) {
err = cur.Decode(&taskLog)
if err != nil {
err = errors.Wrap(err, "decode log failed")
return
}
tlStruct.TaskLog = taskLog
logChan <- tlStruct
}
if cur.Err() != nil {
err = cur.Err()
}
}
// offsetID == -1, return last 10 log
func (mc *LogMongoClient) GetPagedLogOrderByIDDESC(serviceID, taskID, offsetID string, limit int64) (logList []autodl_base.TaskLog, err error) {
if len(taskID) == 0 {
err = errors.Errorf("task id can not be nil")
return
}
if len(offsetID) < 10 && offsetID != "-1" {
err = errors.Errorf("wrong offsetID, only -1 or log offsetID is permitted, %s provided", offsetID)
return
}
col := mc.mongoClient.Database(autodl_base.MongoDBCoreDatabaseName).Collection(autodl_base.MongoDBCoreLogCollectionName)
ctx, _ := context.WithTimeout(context.Background(), 5*time.Second)
query := bson.M{
"service_id": serviceID,
"task_id": taskID,
}
if offsetID != "-1" {
var objectID primitive.ObjectID
objectID, err = primitive.ObjectIDFromHex(offsetID)
query["_id"] = bson.D{{"$lt", objectID}}
}
findOptions := &options.FindOptions{}
findOptions.SetSort(bsonx.Doc{{"_id", bsonx.Int32(-1)}})
if limit > 0 {
if limit > 200 {
limit = 200
}
findOptions.SetLimit(limit)
}
if offsetID == "-1" && limit == 0 {
findOptions.SetLimit(10)
}
var cur *mongo.Cursor
cur, err = col.Find(ctx, query, findOptions)
if err != nil {
errors.Wrap(err, "mongodb col find failed")
return
}
var taskLog autodl_base.TaskLog
for cur.Next(ctx) {
err = cur.Decode(&taskLog)
if err != nil {
err = errors.Wrap(err, "decode log failed")
return
}
logList = append(logList, taskLog)
}
if cur.Err() != nil {
err = cur.Err()
}
reverseLogList(logList)
return
}
func reverseLogList(s []autodl_base.TaskLog) {
for from, to := 0, len(s)-1; from < to; from, to = from+1, to-1 {
s[from], s[to] = s[to], s[from]
}
return
}
mongodb_plugin/options_test.go 0 → 100644
package mongodb_plugin_test
import (
"fmt"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
uuid "github.com/satori/go.uuid"
"github.com/spf13/viper"
autodl_base "gitlab.seetatech.com/autodl.com/autodl-base.git"
"strconv"
"time"
. "autodl-core/mongodb_plugin"
)
var _ = Describe("Options", func() {
var lm *LogMongoClient
var err error
BeforeEach(func() {
viper.Set("mongodb.addr", "mongodb://127.0.0.1:30011,127.0.0.1:30012,127.0.0.1:30013/seetaas?replicaSet=vision-set&authSource=admin")
lm = MongoClientProvider()
})
It("should pub", func() {
for i:=0; i < 10000; i ++ {
fmt.Println("add log of ", i)
err = lm.AddLog(autodl_base.TaskLog{
ServiceID: "service_id_a",
TaskID: strconv.Itoa(i),
Message: uuid.NewV4().String(),
CreatedAt: time.Time{},
})
Expect(err).NotTo(HaveOccurred())
time.Sleep(time.Second)
}
})
})
nohup.out 0 → 100644
Sending build context to Docker daemon 625.6MB
Step 1/6 : FROM hb.seetatech.com/k8s/ubuntu-basic:16.04
16.04: Pulling from k8s/ubuntu-basic
Digest: sha256:85104b6a9da33239971554534c75793db6d5ed0a95feaab9ef2725ca6f0e3632
Status: Downloaded newer image for hb.seetatech.com/k8s/ubuntu-basic:16.04
---> ffd9714fee87
Step 2/6 : COPY core--* /adl/bin/
COPY failed: no source files were specified
redis_plugin/resource.go 0 → 100644
package redis_plugin
import (
"github.com/go-redis/redis"
)
type ResourcePlugin struct {
client *redis.Client
}
func ResourcePluginProvider(client *redis.Client) *ResourcePlugin {
return &ResourcePlugin{client: client}
}
func (hash *ResourcePlugin) tableName() string {
return "autodl_core_resource_hash_table"
}
func (hash *ResourcePlugin) Set(key, value string) (err error) {
err = hash.client.HSet(hash.tableName(), key, value).Err()
return
}
func (hash *ResourcePlugin) Get(key string) (content string, err error) {
content, err = hash.client.HGet(hash.tableName(), key).Result()
return
}
func (hash *ResourcePlugin) GetAll() (content map[string]string, err error) {
content, err = hash.client.HGetAll(hash.tableName()).Result()
return
}
redis_plugin/task.go 0 → 100644
package redis_plugin
import (
"github.com/go-redis/redis"
"gitlab.seetatech.com/autodl.com/autodl-base.git"
"strconv"
"time"
)
type TaskPlugin struct {
client *redis.Client
}
func TaskPluginProvider(client *redis.Client) *TaskPlugin {
return &TaskPlugin{client: client}
}
func (task *TaskPlugin) linkedListName() string {
return autodl_base.CoreWorkerLinkedList
}
func (task *TaskPlugin) AddBottom(taskUUID int) (err error) {
err = task.push(strconv.Itoa(taskUUID), false)
return
}
func (task *TaskPlugin) push(content string, leftPush bool) (err error) {
for i := 0; i < 5; i++ {
if leftPush {
err = task.client.LPush(task.linkedListName(), content).Err()
} else {
err = task.client.RPush(task.linkedListName(), content).Err()
}
if err != nil {
time.Sleep(time.Millisecond * 5)
continue
}
break
}
return
}
func (task *TaskPlugin) Pop() (taskUUID int, err error) {
var content string
content, err = task.client.LPop(task.linkedListName()).Result()
if err != nil {
return
}
taskUUID, err = strconv.Atoi(content)
return
}
func (task *TaskPlugin) AddTop(taskUUID int) (err error) {
err = task.push(strconv.Itoa(taskUUID), true)
return
}
redis_plugin/task_status.go 0 → 100644
package redis_plugin
import (
"encoding/json"
"fmt"
"github.com/go-redis/redis"
"gitlab.seetatech.com/autodl.com/autodl-base.git"
"strings"
"time"
)
type TaskStatusPlugin struct {
client *redis.Client
}
func TaskStatusPluginProvider(client *redis.Client) *TaskStatusPlugin {
return &TaskStatusPlugin{client: client}
}
func (task *TaskStatusPlugin) linkedListName(serviceID string) string {
return fmt.Sprintf("%s%s", autodl_base.CoreMonitorLinkedListPrefix, serviceID)
}
func (task *TaskStatusPlugin) GetServiceNameByTableName(tableName string) string {
return strings.TrimPrefix(tableName, autodl_base.CoreMonitorLinkedListPrefix)
}
func (task *TaskStatusPlugin) AddTaskStatusOfService(serviceID, taskID, taskStatus, msg string) (err error) {
var content []byte
val := autodl_base.TaskStatus{TaskIDString: taskID, TaskStatus: taskStatus, Msg: msg, UpdateAt: time.Now()}
content, err = json.Marshal(&val)
if err != nil {
return
}
for i := 0; i < 5; i++ {
err = task.client.RPush(task.linkedListName(serviceID), string(content)).Err()
if err != nil {
time.Sleep(time.Millisecond * 5)
continue
}
break
}
return
}
func (task *TaskStatusPlugin) GetTaskStatusLenOfService(serviceID string) (l int64, err error) {
l, err = task.client.LLen(task.linkedListName(serviceID)).Result()
return
}
func (task *TaskStatusPlugin) GetAllServiceTable() (tableList []string, err error) {
tableList, err = task.client.Keys(fmt.Sprintf("%s*", autodl_base.CoreMonitorLinkedListPrefix)).Result()
return
}
func (task *TaskStatusPlugin) PopTaskStatusOfService(serviceID string) (taskStatus *autodl_base.TaskStatus, err error) {
var content string
content, err = task.client.LPop(task.linkedListName(serviceID)).Result()
if err != nil {
return
}
taskStatus = &autodl_base.TaskStatus{}
err = json.Unmarshal([]byte(content), taskStatus)
return
}
func (task *TaskStatusPlugin) GetBackTaskStatusOfService(serviceID string, taskStatus *autodl_base.TaskStatus) (err error) {
if taskStatus == nil {
return
}
var content []byte
content, err = json.Marshal(&taskStatus)
if err != nil {
return
}
for i := 0; i < 5; i++ {
err = task.client.LPush(task.linkedListName(serviceID), string(content)).Err()
if err != nil {
time.Sleep(time.Millisecond * 5)
continue
}
break
}
return
}
redis_plugin/wire.go 0 → 100644
package redis_plugin
import (
"github.com/go-redis/redis"
"github.com/spf13/viper"
)
func RedisClientProvider() *redis.Client {
return redis.NewClient(&redis.Options{
Addr: viper.GetString("redis.addr"),
Password: viper.GetString("redis.password"), // no password set IRisiI8m8JElsvJ7aChgWPcv1lwRGdTu
DB: 0, // use default DB
})
}
service/collector.go 0 → 100644
package service
import (
"autodl-core/k8s_plugin"
"autodl-core/libs"
"autodl-core/models"
"autodl-core/mongodb_plugin"
"autodl-core/redis_plugin"
"bufio"
"context"
"fmt"
log "github.com/cihub/seelog"
"github.com/jinzhu/gorm"
"github.com/spf13/viper"
"gitlab.seetatech.com/autodl.com/autodl-base.git"
"gitlab.seetatech.com/autodl.com/autodl-k8s.git/manager"
"gitlab.seetatech.com/autodl.com/autodl-k8s.git/template"
"io/ioutil"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"strings"
"time"
)
type Collector struct {
k8sManager *k8s_plugin.K8sManager
logMongoClient *mongodb_plugin.LogMongoClient
dbConn *gorm.DB
jobModel *models.JobModel
resourcePlugin *redis_plugin.ResourcePlugin
}
func (cs *Collector) CollectNodeResourceInfo(ctx context.Context) {
h, err := ioutil.ReadFile("/etc/autocnn_hostname")
if err != nil {
log.Error("/etc/autocnn_hostname file is not found")
return
}
hostname := strings.ToLower(strings.TrimSpace(string(h)))
resourceTicker := time.NewTicker(time.Second)
for {
select {
case <-ctx.Done():
return
case <-resourceTicker.C:
gpuInfo, err := libs.GetGpuResourcesSummary()
if err != nil {
log.Info("get gpu info failed: ", err.Error())
gpuInfo = &libs.GpuInfo{}
}
cpuInfo, err := libs.GetCpuInfo()
if err != nil {
log.Info("get cpu info failed: ", err.Error())
cpuInfo = &libs.CpuInfo{}
}
memInfo, err := libs.GetMemInfo()
if err != nil {
log.Info("get mem info failed: ", err.Error())
memInfo = &libs.MemInfo{}
}
rs := libs.ResourceSummary{
NodeName: hostname,
CpuInfo: cpuInfo,
GpuInfo: gpuInfo,
MemInfo: memInfo,
UpdatedAt: time.Now(),
}
err = cs.resourcePlugin.Set(hostname, rs.ToString())
if err != nil {
log.Error("set resource info failed, ", err.Error())
}
}
}
}
func (cs *Collector) CollectTaskLog() {
clientset := cs.k8sManager.K8sManager.Clientset
startTime := metav1.NewTime(time.Now())
h, err := ioutil.ReadFile("/etc/autocnn_hostname")
if err != nil {
log.Error("/etc/autocnn_hostname file is not found")
return
}
hostname := strings.ToLower(strings.TrimSpace(string(h)))
fieldSelector, _ := fields.ParseSelector("spec.nodeName=" + hostname)
namespace := viper.GetString("k8s.namespace")
labelSelector := fmt.Sprintf("role = %s", template.GetWorkerRoleName(namespace))
listOption := metav1.ListOptions{TypeMeta: metav1.TypeMeta{Kind: manager.K8S_POD_KIND,
APIVersion: manager.K8S_API_VERSION_V1},
LabelSelector: labelSelector,
FieldSelector: fieldSelector.String()}
var podMap = make(map[string]bool)
podList, err := clientset.CoreV1().Pods(namespace).List(listOption)
if err != nil {
log.Error("list pod failed when initialize: ", err.Error())
return
}
for _, p := range podList.Items {
if p.Status.Phase == v1.PodSucceeded || p.Status.Phase == v1.PodFailed || p.Status.Phase == v1.PodUnknown {
lenCond := len(p.Status.Conditions)
if lenCond == 0 {
log.Error("conditions length is zero")
continue
}
lastCond := p.Status.Conditions[lenCond-1]
if time.Now().After(lastCond.LastTransitionTime.Time.Add(time.Second * 60)) {
log.Info("history pod. skip ", p.Name)
continue
}
}
if _, ok := podMap[p.Name]; !ok {
go cs.watchLog(p.Name, p.Namespace, startTime)
podMap[p.Name] = true
}
}
for {
watcher, err := clientset.CoreV1().Pods(namespace).Watch(listOption)
if err != nil {
log.Error("watch pod for collect log failed: ", err.Error())
return
}
ch := watcher.ResultChan()
for event := range ch {
pod, ok := event.Object.(*v1.Pod)
if !ok {
log.Error("unexpected type")
continue
}
podStatus := pod.Status
phase := podStatus.Phase
if phase == v1.PodPending || phase == v1.PodRunning {
if _, ok := podMap[pod.Name]; !ok {
log.Info("start to collect log. pod name: ", pod.Name)
go cs.watchLog(pod.Name, pod.Namespace, startTime)
podMap[pod.Name] = true
}
}
}
}
}
func (cs *Collector) watchLog(podName, namespace string, startTime metav1.Time) {
log.Info("start watch job: ", podName)
var labels map[string]string
var isRunning = true
for {
pod, err := cs.k8sManager.K8sManager.Clientset.CoreV1().Pods(namespace).Get(podName, metav1.GetOptions{})
if err != nil {
log.Error("Pod not found", err.Error())
return
}
status := pod.Status.Phase
if status == v1.PodRunning {
labels = pod.Labels
break
} else if status == v1.PodFailed || status == v1.PodSucceeded {
isRunning = false
labels = pod.Labels
break
} else {
log.Info("status in waiting: ", status)
}
time.Sleep(time.Second)
}
requests := cs.k8sManager.K8sManager.Clientset.CoreV1().
Pods(namespace).
GetLogs(podName, &v1.PodLogOptions{TypeMeta: metav1.TypeMeta{APIVersion: "v1"}, Follow: isRunning, SinceTime: &startTime})
readCloser, err := requests.Stream()
if err != nil {
log.Error("maybe job have been over", err.Error())
return
}
defer readCloser.Close()
scanner := bufio.NewScanner(readCloser)
logCh := make(chan string, 10)
over := make(chan bool)
defer close(over)
go func() {
lastLogSyncTime := time.Now()
var logList []autodl_base.TaskLog
for {
select {
case logLine, ok := <-logCh:
if !ok {
log.Infof("log channel closed: len of log list %d", len(logList))
if len(logList) != 0 {
err = cs.logMongoClient.AddLog(logList...)
if err != nil {
log.Error("add log to mongo db failed, ", err.Error())
}
}
over <- true
return
}
logList = append(logList, autodl_base.TaskLog{
ServiceID: labels[libs.PodServiceIDLabel],
TaskID: labels[libs.PodTaskIDLabel],
CreatedAt: time.Now(),
Type: autodl_base.TaskLogTypeRuntime,
Message: logLine,
})
if len(logList) > 30 || time.Now().After(lastLogSyncTime.Add(time.Second)) {
log.Info("force add log")
err = cs.logMongoClient.AddLog(logList...)
if err != nil {
log.Error("add log to mongo db failed, ", err.Error())
}
lastLogSyncTime = time.Now()
logList = make([]autodl_base.TaskLog, 0)
}
}
}
}()
for scanner.Scan() {
logLine := scanner.Text()
if strings.HasPrefix(logLine, "Environment information not found") {
continue
}
if !strings.HasSuffix(logLine, "\n") {
logLine += "\n"
}
logCh <- logLine
}
log.Info("scan finished, close log channel")
close(logCh)
log.Info("wait over")
<-over
err = cs.jobModel.LogFlushOver(cs.dbConn, labels[libs.PodServiceIDLabel], labels[libs.PodTaskIDLabel])
if err != nil {
log.Errorf("set task log flush over failed: %s-%s, %v", labels[libs.PodServiceIDLabel], labels[libs.PodTaskIDLabel], err)
}
log.Info("job watch over: ", podName)
}
service/log_stash.go 0 → 100644
package service
import (
"autodl-core/k8s_plugin"
"bufio"
"fmt"
log "github.com/cihub/seelog"
"github.com/spf13/viper"
"gitlab.seetatech.com/autodl.com/autodl-k8s.git/manager"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"os"
"strings"
"time"
)
type LogStash struct {
k8sManager *k8s_plugin.K8sManager
}
type logContent struct {
PodName string
LogContent string
}
func (svc *LogStash) CollectTaskLog(namespace string) {
startTime := metav1.NewTime(time.Now())
listOption := metav1.ListOptions{TypeMeta: metav1.TypeMeta{Kind: manager.K8S_POD_KIND,
APIVersion: manager.K8S_API_VERSION_V1},
LabelSelector: "app"}
var podMap = make(map[string]bool)
podList, err := svc.k8sManager.K8sManager.Clientset.CoreV1().Pods(namespace).List(listOption)
if err != nil {
log.Error("list pod failed when initialize: ", err.Error())
return
}
logCh := make(chan logContent)
go svc.handleLog(logCh)
defer func() {
close(logCh)
}()
for _, p := range podList.Items {
if p.Status.Phase != v1.PodRunning {
continue
}
fmt.Println("found: ", p.Name)
if _, ok := podMap[p.Name]; !ok {
go svc.watchLog(p.Name, p.Namespace, startTime, logCh)
podMap[p.Name] = true
}
}
for {
watcher, err := svc.k8sManager.K8sManager.Clientset.CoreV1().Pods(namespace).Watch(listOption)
if err != nil {
log.Error("watch pod for collect log failed: ", err.Error())
return
}
ch := watcher.ResultChan()
for event := range ch {
pod, ok := event.Object.(*v1.Pod)
if !ok {
log.Error("unexpected type")
continue
}
podStatus := pod.Status
phase := podStatus.Phase
log.Info("monitor pod: ", pod.Name, " pod status: ", phase)
if phase == v1.PodPending || phase == v1.PodRunning {
if _, ok := podMap[pod.Name]; !ok {
log.Info("start to collect log. pod name: ", pod.Name)
go svc.watchLog(pod.Name, pod.Namespace, startTime, logCh)
podMap[pod.Name] = true
}
}
}
}
}
func (svc *LogStash) watchLog(podName, namespace string, startTime metav1.Time, logCh chan logContent) {
log.Info("start watch pod: ", podName)
for {
pod, err := svc.k8sManager.K8sManager.Clientset.CoreV1().Pods(namespace).Get(podName, metav1.GetOptions{})
if err != nil {
log.Error("Pod not found", err.Error())
return
}
status := pod.Status.Phase
if status == v1.PodRunning {
break
} else if status == v1.PodFailed || status == v1.PodSucceeded {
return
} else {
log.Info("status in waiting: ", status)
}
time.Sleep(time.Second)
}
requests := svc.k8sManager.K8sManager.Clientset.CoreV1().
Pods(namespace).
GetLogs(podName, &v1.PodLogOptions{TypeMeta: metav1.TypeMeta{APIVersion: "v1"}, Follow: true, SinceTime: &startTime})
readCloser, err := requests.Stream()
if err != nil {
log.Error("maybe job have been over", err.Error())
return
}
defer readCloser.Close()
scanner := bufio.NewScanner(readCloser)
var lastLine string
for scanner.Scan() {
logLine := scanner.Text()
if containErrInfo(&logLine) {
logCh <- logContent{
PodName: podName,
LogContent: lastLine + "\n" + logLine + "\n",
}
}
lastLine = logLine
}
log.Info("pod watch over: ", podName)
}
func containErrInfo(line *string) bool {
return strings.Contains(*line, "ERR") || strings.Contains(*line, "err") || strings.Contains(*line, "panic") || strings.Contains(*line, "fail") || strings.Contains(*line, "goroutine")
}
func getLogFinePath(podName string) string {
today := time.Now()
return viper.GetString("app.mount_path") + "/log_collector/" + fmt.Sprintf("%d_%d_%d__%s", today.Year(), today.Month(), today.Day(), podName) + ".log"
}
func (svc *LogStash) handleLog(logCh chan logContent) {
var f *os.File
var err error
defer func() {
if f != nil {
f.Close()
}
}()
tick := time.Tick(time.Second * 5)
lastFlushTime := time.Now()
var record []logContent
for {
select {
case lcontent, ok := <-logCh:
if !ok {
log.Info("close log transfer channel")
return
}
record = append(record, lcontent)
case <-tick:
if len(record) == 0 {
continue
}
if len(record) < 100 && time.Since(lastFlushTime) < 20*time.Second {
continue
}
f, err = os.OpenFile(getLogFinePath(record[0].PodName), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644)
if err != nil {
log.Error("can not open/write log to log file! ", err.Error())
continue
}
for _, content := range record {
_, err = f.Write([]byte(content.LogContent))
if err != nil {
log.Error(err.Error())
log.Error("can not write log, print err: ", content)
}
}
f.Close()
record = record[:0]
}
}
}
service/monitor.go 0 → 100644
package service
import (
"autodl-core/k8s_plugin"
"autodl-core/libs"
"autodl-core/libs/path"
"autodl-core/models"
"autodl-core/mongodb_plugin"
"autodl-core/redis_plugin"
"context"
"fmt"
log "github.com/cihub/seelog"
"github.com/go-redis/redis"
"github.com/jinzhu/gorm"
"github.com/levigross/grequests"
"github.com/pkg/errors"
"github.com/spf13/viper"
"gitlab.seetatech.com/autodl.com/autodl-base.git"
"gitlab.seetatech.com/autodl.com/autodl-k8s.git/spawner"
"gitlab.seetatech.com/autodl.com/autodl-k8s.git/template"
"io"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"net/http"
"os"
"time"
)
type Monitor struct {
k8sManager *k8s_plugin.K8sManager
taskStatusModel *models.TaskStatusModel
dbConn *gorm.DB
jobModel *models.JobModel
taskStatusPlugin *redis_plugin.TaskStatusPlugin
serviceModel *models.ServiceModel
logMongoPlugin *mongodb_plugin.LogMongoClient
*Status
}
func (ms *Monitor) WatchTaskStatus(ctx context.Context, stop chan<- error) {
var err error
defer func() {
stop <- err
}()
namespace := viper.GetString("k8s.namespace")
labelSelector := fmt.Sprintf("role = %s", template.GetWorkerRoleName(namespace))
for {
select {
case <-ctx.Done():
return
default:
watcher, err := ms.k8sManager.K8sManager.Clientset.CoreV1().Pods(namespace).Watch(metav1.ListOptions{LabelSelector: labelSelector})
if err != nil {
log.Error("monitor status watcher failed: ", err.Error())
}
ch := watcher.ResultChan()
for event := range ch {
pod, ok := event.Object.(*v1.Pod)
if !ok {
log.Error("unexpected type")
continue
}
taskStatus, serviceID, err := ms.getTaskStatus(pod)
if err != nil {
continue
}
err = ms.UpdateTaskStatus(serviceID, taskStatus.TaskIDString, taskStatus.TaskStatus, taskStatus.Msg)
if err != nil && err != libs.TaskIsDoneCanNotSetStatusError && err != libs.TaskStatusExistCanNotSetStatusError {
err = errors.Wrap(err, "push task status to redis linked list failed")
return
}
}
}
}
}
func (ms *Monitor) getTaskStatus(pod *v1.Pod) (ts *autodl_base.TaskStatus, serviceID string, err error) {
ts = &autodl_base.TaskStatus{}
if len(pod.Labels[libs.PodTaskIDLabel]) > 0 && len(pod.Labels[libs.PodServiceIDLabel]) > 0 {
ts.TaskIDString = pod.Labels[libs.PodTaskIDLabel]
ts.TaskStatus = string(pod.Status.Phase)
ts.Msg = pod.Status.Message
if pod.Status.Phase == v1.PodFailed {
if len(pod.Status.ContainerStatuses) > 0 {
ts.Msg += pod.Status.ContainerStatuses[0].State.Terminated.Reason + " " + pod.Status.ContainerStatuses[0].State.Terminated.Message
}
}
ts.UpdateAt = time.Now()
serviceID = pod.Labels[libs.PodServiceIDLabel]
return
}
err = errors.New("Label selector does not work, we got a pod without podtaskIDLabel || PodServiceIDLabel, " + pod.String())
return
}
func (ms *Monitor) HandleTaskStatus(ctx context.Context, stop chan<- error) {
var err error
defer func() {
stop <- err
}()
sonCtx, cancelFunc := context.WithCancel(context.Background())
monitorServerHub := make(map[string]bool)
for {
select {
case <-ctx.Done():
cancelFunc()
return
default:
var tableList []string
tableList, err = ms.taskStatusPlugin.GetAllServiceTable()
if err != nil {
if err == redis.Nil {
continue
}
cancelFunc()
err = errors.Wrap(err, "get all service table failed")
return
}
for _, serviceTableName := range tableList {
if _, ok := monitorServerHub[serviceTableName]; !ok {
go ms.notifyService(sonCtx, ms.taskStatusPlugin.GetServiceNameByTableName(serviceTableName))
monitorServerHub[serviceTableName] = true
}
}
time.Sleep(time.Second * 10)
}
}
}
func (ms *Monitor) notifyService(ctx context.Context, serviceID string) {
log.Info("start send notify of ", serviceID)
var failedNotiTimes int
for {
select {
case <-ctx.Done():
return
default:
taskStatus, err := ms.taskStatusPlugin.PopTaskStatusOfService(serviceID)
if err != nil {
if err != redis.Nil {
log.Info("error when pop: ", err.Error())
}
time.Sleep(time.Second * 2)
continue
}
lastTaskStatus, _ := ms.taskStatusModel.GetLast(ms.dbConn, serviceID, taskStatus.TaskIDString)
if lastTaskStatus != nil {
if lastTaskStatus.Status == autodl_base.TaskStopped {
var spa *spawner.ExpSpawner
spa, err = spawner.NewExpStopSpawner(serviceID, taskStatus.TaskIDString, ms.k8sManager.K8sManager)
err = spa.StopExp()
if err != nil {
log.Error("delete exp master failed: ", err.Error(), " ", serviceID, " ", taskStatus.TaskIDString)
}
}
}
// handle log persist and svc clean
if autodl_base.TaskIsDone(taskStatus.TaskStatus) {
task, err := ms.jobModel.Get(ms.dbConn, serviceID, taskStatus.TaskIDString)
if err != nil {
log.Error("Find task by ID fail, ", err.Error(), task.TaskID)
continue
}
if !task.LogIsPersist {
go ms.persistLog(serviceID, task.TaskID)
}
if task.HasService && !task.ServiceIsClean {
spa, err := spawner.NewExpStopSpawner(serviceID, task.TaskID, ms.k8sManager.K8sManager)
if err != nil {
log.Error("delete svc fail, init Spawner fail: ", err.Error(), task.TaskID)
} else {
err = spa.Manager.DeleteService(template.GetServiceName(serviceID, task.TaskID))
if err != nil {
log.Error("delete svc fail, delete fail: ", err.Error(), task.TaskID)
}
err = ms.jobModel.CleanServiceFinished(ms.dbConn, serviceID, task.TaskID)
if err != nil {
log.Error("update service is clean failed: ", err.Error())
}
}
}
}
//notify to service api
if !ms.taskStatusModel.IsStatusNotifySuccess(ms.dbConn, serviceID, taskStatus.TaskIDString, taskStatus.TaskStatus) {
service, err := ms.serviceModel.Get(ms.dbConn, serviceID)
if err != nil {
log.Error("handle task status of service failed (get service): ", serviceID)
continue
}
res, err := grequests.Post(service.NotiAPI, &grequests.RequestOptions{
JSON: taskStatus,
RequestTimeout: time.Second * 10,
})
if err != nil || res.StatusCode != http.StatusOK {
log.Info("send failed:", err, ", status code: ", res.StatusCode)
err := ms.taskStatusPlugin.GetBackTaskStatusOfService(serviceID, taskStatus)
if err != nil {
log.Error("!! get back task status to list failed: ", err.Error())
}
failedNotiTimes++
libs.Delay(failedNotiTimes)
continue
} else {
failedNotiTimes = 0
}
log.Info("send status to notify api success")
err = ms.taskStatusModel.SetStatusNotifySuccess(ms.dbConn, serviceID, taskStatus.TaskIDString, taskStatus.TaskStatus)
if err != nil {
log.Error("set notify success failed: ", err)
}
}
}
}
}
func (ms *Monitor) persistLog(serviceID, taskID string) (err error) {
defer func() {
if err != nil {
log.Error("!! log persist failed: ", err.Error())
}
}()
for i := 1; i < 5; i++ {
var job *models.Job
job, err = ms.jobModel.Get(ms.dbConn, serviceID, taskID)
if err != nil {
log.Errorf("get job from db failed: %v", err)
break
}
if job.LogIsFlushOver {
break
} else {
time.Sleep(time.Second)
}
}
var f *os.File
err = path.InitServiceDir(serviceID)
if err != nil {
err = errors.Wrapf(err, "init service [%s] dir failed", serviceID)
return
}
// if monitor have multi deployment, use this to avoid persist at same time
//_, err = os.Stat(path.GetTaskLogFilePath(serviceID, taskID))
//if !os.IsNotExist(err) {
// return nil
//}
f, err = os.OpenFile(path.GetTaskLogFilePath(serviceID, taskID), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
err = errors.Wrapf(err, "open log file [%s] failed", path.GetTaskLogFilePath(serviceID, taskID))
return
}
defer f.Close()
logChan := make(chan autodl_base.TaskLogChanStruct)
defer close(logChan)
go ms.logMongoPlugin.GetAllLog(serviceID, taskID, logChan)
for {
select {
case taskLog, ok := <-logChan:
if !ok {
err = errors.Errorf("get log failed: %v", ok)
return
}
if taskLog.Error != nil {
if taskLog.Error == io.EOF {
err = ms.jobModel.LogPersistFinished(ms.dbConn, serviceID, taskID)
if err != nil {
log.Errorf("set log persist finished failed: %v", err)
}
var deleteCount int64
deleteCount, err = ms.logMongoPlugin.DeleteLog(serviceID, taskID)
log.Infof("try to delete log of %s-%s [Count: %d]", serviceID, taskID, deleteCount)
if err != nil {
log.Error("failed to delete log of ", serviceID, " ", taskID, err.Error())
}
} else {
err = errors.Errorf("get log failed 2: %v", taskLog.Error)
}
return
}
_, err = f.WriteString(taskLog.Message)
if err != nil {
return
}
}
}
}
service/rpc_server.go 0 → 100644
package service
import (
"autodl-core/k8s_plugin"
"autodl-core/libs"
"autodl-core/libs/path"
"autodl-core/models"
"autodl-core/mongodb_plugin"
"autodl-core/redis_plugin"
"autodl-core/service/utils"
"encoding/json"
"fmt"
log "github.com/cihub/seelog"
"github.com/jinzhu/gorm"
"github.com/pkg/errors"
"github.com/spf13/viper"
"gitlab.seetatech.com/autodl.com/autodl-base.git"
"gitlab.seetatech.com/autodl.com/autodl-base.git/protoV1"
"go.mongodb.org/mongo-driver/bson/primitive"
"golang.org/x/net/context"
"io"
"os"
)
type Rpc struct {
dbConn *gorm.DB
taskStatusPlugin *redis_plugin.TaskStatusPlugin
taskStatusModel *models.TaskStatusModel
jobModel *models.JobModel
logPlugin *mongodb_plugin.LogMongoClient
k8sManager *k8s_plugin.K8sManager
serviceModel *models.ServiceModel
taskPlugin *redis_plugin.TaskPlugin
taskModel *models.TaskModel
resourcePlugin *redis_plugin.ResourcePlugin
}
func (s *Rpc) RunTask(ctx context.Context, runTaskRequest *proto_v1.RunTaskRequest) (reply *proto_v1.RunTaskReply, rpcError error) {
log.Info("get task from: ", runTaskRequest.ServiceId)
err := s.validateRunTaskRequest(runTaskRequest)
reply = &proto_v1.RunTaskReply{
ErrCode: proto_v1.Success,
}
if err != nil {
reply.ErrMsg = err.Error()
reply.ErrCode = proto_v1.Failed
return
}
for _, t := range runTaskRequest.Tasks {
var svcName string
switch t.PodInfo.ServiceType {
case proto_v1.PodInfo_NoService:
case proto_v1.PodInfo_Jupyter:
//TODO: add port in backend
//TODO: service 访问时需要最后加一个/,需要处理一下
//runTaskRequest.Tasks[index].BuildInfo.Cmd += " --NotebookApp.base_url=" + template.GetJupyterUrlPath(viper.GetString("k8s.namespace"), runTaskRequest.ServiceId, t.TaskId, t.PodInfo.Ports[0])
svcName = utils.GetJupyterHost(viper.GetString("host.nginx_host"))
case proto_v1.PodInfo_TensorBoard:
svcName = utils.GetTensorboardHost(viper.GetString("host.nginx_host"))
case proto_v1.PodInfo_Standard:
svcName = utils.GetStandardHost(viper.GetString("host.nginx_host"))
}
reply.TaskSvcName = append(reply.TaskSvcName, &proto_v1.TaskSvcName{
TaskId: t.TaskId,
SvcHost: svcName,
SvcName: utils.GetServiceName(viper.GetString("k8s.namespace"), runTaskRequest.ServiceId, t.TaskId),
})
}
var taskList []byte
taskList, err = json.Marshal(runTaskRequest)
if err != nil {
reply.ErrCode = proto_v1.Failed
reply.ErrMsg = fmt.Sprintf("marshal task list failed: %s", err.Error())
return
}
var taskUUID int
taskUUID, err = s.taskModel.Create(s.dbConn, &models.Task{
ServiceID: runTaskRequest.ServiceId,
TaskType: libs.RunTask,
TaskList: string(taskList),
})
if err != nil {
reply.ErrCode = proto_v1.Failed
reply.ErrMsg = fmt.Sprintf("record task failed: %s", err.Error())
return
}
err = s.taskPlugin.AddBottom(taskUUID)
if err != nil {
reply.ErrCode = proto_v1.Failed
reply.ErrMsg = fmt.Sprintf("run task failed when init: %s", err.Error())
return
}
return
}
func (s *Rpc) validateRunTaskRequest(tasks *proto_v1.RunTaskRequest) (err error) {
if tasks == nil {
return errors.Errorf("nil request is invalid")
}
var errMsg string
if len(tasks.ServiceId) == 0 {
errMsg += " missing " + "[service_id]"
} else {
service, err := s.serviceModel.Get(s.dbConn, tasks.ServiceId)
if err != nil {
errMsg += " service_id [" + tasks.ServiceId + "] is forbidden "
} else if len(service.NotiAPI) == 0 {
errMsg += " service's notification api can not be null "
}
}
if len(tasks.Tasks) == 0 {
errMsg += " missing " + "[tasks]"
} else {
var taskIDs []string
for _, t := range tasks.Tasks {
if t.TaskId == "0" {
errMsg += " task_id shouldn't be 0"
}
taskIDs = append(taskIDs, t.TaskId)
}
var jobs []models.Job
jobs, err = s.jobModel.GetInTaskIDs(s.dbConn, tasks.ServiceId, taskIDs)
if err != nil {
errMsg += " core database error "
} else if len(jobs) > 0 {
errMsg += fmt.Sprintf(" serviceId %s and taskId", tasks.ServiceId)
for _, j := range jobs {
errMsg += "[" + j.TaskID + "]"
}
errMsg += " already exist"
}
}
if len(errMsg) > 0 {
err = errors.Errorf("Parameter error: %s", errMsg)
}
return
}
func (s *Rpc) StopTask(ctx context.Context, stopTaskRequest *proto_v1.StopTaskRequest) (reply *proto_v1.StopTaskReply, rpcError error) {
err := s.validateStopTaskRequest(stopTaskRequest)
reply = &proto_v1.StopTaskReply{
ErrCode: proto_v1.Success,
}
if err != nil {
reply.ErrMsg = err.Error()
reply.ErrCode = proto_v1.Failed
return
}
var taskList []byte
taskList, err = json.Marshal(stopTaskRequest)
if err != nil {
reply.ErrCode = proto_v1.Failed
reply.ErrMsg = fmt.Sprintf("marshal task list failed: %s", err.Error())
return
}
var taskUUID int
taskUUID, err = s.taskModel.Create(s.dbConn, &models.Task{
ServiceID: stopTaskRequest.ServiceId,
TaskType: libs.StopTask,
TaskList: string(taskList),
})
if err != nil {
reply.ErrCode = proto_v1.Failed
reply.ErrMsg = fmt.Sprintf("record task failed: %s", err.Error())
return
}
err = s.taskPlugin.AddBottom(taskUUID)
if err != nil {
reply.ErrCode = proto_v1.Failed
reply.ErrMsg = fmt.Sprintf("stop task failed when init: %s", err.Error())
}
return
}
func (s *Rpc) validateStopTaskRequest(stopTaskRequest *proto_v1.StopTaskRequest) (err error) {
if stopTaskRequest == nil {
return errors.Errorf("nil request is invalid")
}
var errMsg string
if len(stopTaskRequest.ServiceId) == 0 {
errMsg += " missing " + "[service_id]"
}
if len(stopTaskRequest.Tasks) == 0 {
errMsg += " missing " + "[tasks]"
}
if len(errMsg) > 0 {
err = errors.Errorf("Parameter missing: %s", errMsg)
}
return
}
func (s *Rpc) GetPagedLogs(request *proto_v1.GetPagedLogsRequest, stream proto_v1.Core_GetPagedLogsServer) (rpcError error) {
if request == nil {
return errors.Errorf("nil request is invalid")
}
if len(request.ServiceId) == 0 || len(request.TaskId) == 0 {
rpcError = errors.Errorf("can not find serviceId + taskId (%s-%s) task", request.ServiceId, request.TaskId)
return
}
defer func() {
if rpcError != nil {
log.Error("get paged logs failed: ", rpcError.Error())
}
}()
if request.Offset == "" {
return s.streamLogAsc(request, stream)
} else if request.Offset == "-1" {
return s.streamLogDesc(request, stream)
}
if request.PageOption == proto_v1.GetPagedLogsRequest_Backward {
return s.streamLogDesc(request, stream)
} else if request.PageOption == proto_v1.GetPagedLogsRequest_Forward {
return s.streamLogAsc(request, stream)
} else {
rpcError = errors.Errorf("wrong request params, check your request body. %v", request)
}
return
}
func (s *Rpc) streamLogAsc(request *proto_v1.GetPagedLogsRequest, stream proto_v1.Core_GetPagedLogsServer) (rpcError error) {
if request == nil {
return errors.Errorf("nil request is invalid")
}
logChan := make(chan autodl_base.TaskLogChanStruct)
go s.logPlugin.GetPagedLogOrderByIDASC(request.ServiceId, request.TaskId, request.Offset, request.Limit, logChan)
for {
select {
case taskLog, ok := <-logChan:
if !ok {
rpcError = errors.Errorf("get log failed: %v", ok)
return
}
if taskLog.Error != nil {
if taskLog.Error != io.EOF {
rpcError = errors.Errorf("failed to get record of serviceId: %s, taskId: %s, detail: %s", request.ServiceId, request.TaskId, taskLog.Error)
}
return
}
err := stream.Send(&proto_v1.GetPagedLogsReply{
Offset: taskLog.MongoID.(primitive.ObjectID).Hex(),
Content: []byte(taskLog.Message),
})
if err != nil {
rpcError = errors.Wrapf(err, "send log content failed")
return
}
}
}
}
func (s *Rpc) streamLogDesc(request *proto_v1.GetPagedLogsRequest, stream proto_v1.Core_GetPagedLogsServer) (rpcError error) {
if request == nil {
return errors.Errorf("nil request is invalid")
}
// 倒序查询 (包括 offset == -1 )
taskLogList, err := s.logPlugin.GetPagedLogOrderByIDDESC(request.ServiceId, request.TaskId, request.Offset, request.Limit)
if err != nil {
rpcError = errors.Wrap(err, "error when GetPagedLogOrderByIDDESC")
return
}
for _, v := range taskLogList {
err := stream.Send(&proto_v1.GetPagedLogsReply{
Offset: v.MongoID.(primitive.ObjectID).Hex(),
Content: []byte(v.Message),
})
if err != nil {
rpcError = errors.Wrapf(err, "send log content failed")
return
}
}
return
}
func (s *Rpc) GetAllLogs(request *proto_v1.GetAllLogsRequest, stream proto_v1.Core_GetAllLogsServer) (rpcError error) {
if request == nil {
return errors.Errorf("nil request is invalid")
}
if len(request.ServiceId) == 0 || len(request.TaskId) == 0 {
rpcError = errors.Errorf("can not find serviceId + taskId (%s-%s) task", request.ServiceId, request.TaskId)
return
}
defer func() {
if rpcError != nil {
log.Error("get all logs failed: ", rpcError.Error())
}
}()
jobDetail, err := s.jobModel.Get(s.dbConn, request.ServiceId, request.TaskId)
if err != nil {
rpcError = errors.Wrapf(err, "failed to get record of serviceId: %s, taskId: %s", request.ServiceId, request.TaskId)
log.Error(rpcError.Error())
return
}
if jobDetail.LogIsPersist {
f, err := os.OpenFile(path.GetTaskLogFilePath(request.ServiceId, request.TaskId), os.O_RDONLY, 0644)
if err != nil {
rpcError = errors.Wrapf(err, "open log file [%s] failed", path.GetTaskLogFilePath(request.ServiceId, request.TaskId))
return
}
defer f.Close()
buf := make([]byte, 1024*1024)
for {
l, err := f.Read(buf)
if err != nil {
if err == io.EOF {
err = stream.Send(&proto_v1.GetAllLogsReply{
Content: []byte{},
IsLastChunk: true,
})
if err != nil {
rpcError = errors.Wrapf(err, "send log content failed")
}
return
}
rpcError = errors.Wrapf(err, "read log file [%s] failed", path.GetTaskLogFilePath(request.ServiceId, request.TaskId))
return
}
err = stream.Send(&proto_v1.GetAllLogsReply{
Content: buf[:l],
IsLastChunk: false,
})
if err != nil {
rpcError = errors.Wrapf(err, "send log content failed")
return
}
}
} else {
logChan := make(chan autodl_base.TaskLogChanStruct)
go s.logPlugin.GetAllLog(request.ServiceId, request.TaskId, logChan)
for {
select {
case taskLog, ok := <-logChan:
if !ok {
rpcError = errors.Errorf("get log failed: %v", ok)
return
}
if taskLog.Error != nil {
if taskLog.Error == io.EOF {
err = stream.Send(&proto_v1.GetAllLogsReply{
Content: []byte{},
IsLastChunk: true,
})
if err != nil {
rpcError = errors.Wrapf(err, "send log content failed")
}
return
}
rpcError = errors.Errorf("failed to get record of serviceId: %s, taskId: %s, detail: %s", request.ServiceId, request.TaskId, taskLog.Error)
return
}
err := stream.Send(&proto_v1.GetAllLogsReply{
Content: []byte(taskLog.Message),
IsLastChunk: false,
})
if err != nil {
rpcError = errors.Wrapf(err, "send log content failed")
return
}
}
}
}
}
func (s *Rpc) GetTaskStatus(ctx context.Context, getRequest *proto_v1.GetTaskStatusRequest) (getTaskStatusReply *proto_v1.GetTaskStatusReply, rpcError error) {
if getRequest == nil {
return nil, errors.Errorf("nil request is invalid")
}
if len(getRequest.ServiceId) == 0 || len(getRequest.TaskId) == 0 {
rpcError = errors.Errorf("can not find serviceId + taskId (%s-%s) task", getRequest.ServiceId, getRequest.TaskId)
return
}
taskStatus, err := s.taskStatusModel.GetLast(s.dbConn, getRequest.ServiceId, getRequest.TaskId)
if err != nil {
rpcError = errors.Wrapf(err, "can not find serviceId + taskId (%s-%s) task", taskStatus.ServiceID, taskStatus.TaskID)
return
}
getTaskStatusReply = &proto_v1.GetTaskStatusReply{
Status: taskStatus.Status,
Msg: taskStatus.Msg,
UpdateAt: taskStatus.CreatedAt.Unix(),
}
return
}
func (s *Rpc) GetResourceInfo(ctx context.Context, request *proto_v1.GetResourceInfoRequest) (reply *proto_v1.GetResourceInfoReply, rpcError error) {
reply = &proto_v1.GetResourceInfoReply{}
nodeResources, err := s.resourcePlugin.GetAll()
if err != nil {
rpcError = errors.Wrapf(err, "can not find nodes resources"+err.Error())
return
}
for _, nodeInfo := range nodeResources {
var nodeResource libs.ResourceSummary
err = nodeResource.Parse(nodeInfo)
if err != nil {
log.Error("nodeResource.Parse failed: ", err.Error())
continue
}
gpuInfo := []*proto_v1.GpuUnitSummary{}
gpuDriverVersion := nodeResource.GpuInfo.DriverVersion
for _, g := range nodeResource.GpuInfo.GPUS {
gpuInfo = append(gpuInfo, &proto_v1.GpuUnitSummary{
Index: int64(g.Index),
Uuid: g.UUID,
Name: g.Name,
MemoryUsed: int64(g.MemoryUsed),
MemoryTotal: int64(g.MemoryTotal),
GpuUtilization: int64(g.GpuUtilization),
MemUtilization: int64(g.MemoryUtilization),
PowerDraw: int64(g.PowerDraw),
Temperature: int64(g.Temperature),
FanSpeed: int64(g.FanSpeed),
})
}
reply.NodeInfo = append(reply.NodeInfo, &proto_v1.NodeResourceInfo{
NodeName: nodeResource.NodeName,
CpuInfo: &proto_v1.CpuResourceSummary{
Count: int64(nodeResource.CpuInfo.Count),
Last1Min: float32(nodeResource.CpuInfo.Last1Min),
Last5Min: float32(nodeResource.CpuInfo.Last5Min),
Last15Min: float32(nodeResource.CpuInfo.Last15Min),
ProcessRunning: int64(nodeResource.CpuInfo.ProcessRunning),
ProcessTotal: int64(nodeResource.CpuInfo.ProcessTotal),
},
MemInfo: &proto_v1.MemResourceSummary{
MemTotal: int64(nodeResource.MemInfo.MemTotal),
MemFree: int64(nodeResource.MemInfo.MemFree),
MemAvailable: int64(nodeResource.MemInfo.MemAvailable),
},
GpuInfo: &proto_v1.GpuResourceSummary{
DriverVersion: gpuDriverVersion,
Gpus: gpuInfo,
},
UpdatedAt: nodeResource.UpdatedAt.Unix(),
})
}
return
}
service/status.go 0 → 100644
package service
import (
"autodl-core/libs"
"autodl-core/models"
"autodl-core/redis_plugin"
log "github.com/cihub/seelog"
"github.com/jinzhu/gorm"
)
type Status struct {
taskStatusModel *models.TaskStatusModel
dbConn *gorm.DB
taskStatusPlugin *redis_plugin.TaskStatusPlugin
}
func (svc *Status) UpdateTaskStatus(serviceID, taskID, status, msg string) (err error) {
err = svc.taskStatusModel.SetTaskStatus(svc.dbConn, serviceID, taskID, status, msg)
switch err {
case nil:
case libs.TaskStatusExistCanNotSetStatusError:
return
case libs.TaskIsDoneCanNotSetStatusError:
return
default:
log.Error("set task status in model failed: ", err)
return
}
err = svc.taskStatusPlugin.AddTaskStatusOfService(serviceID, taskID, status, msg)
return
}
service/utils/logstream.go 0 → 100644
package utils
import (
"autodl-core/mongodb_plugin"
"bytes"
"encoding/json"
"errors"
"fmt"
"gitlab.seetatech.com/autodl.com/autodl-base.git"
"io"
"io/ioutil"
"strings"
"time"
)
type decoderWrapper struct {
buffer *bytes.Buffer
lines map[string]int
diff int
taskID string
serviceID string
client *mongodb_plugin.LogMongoClient
}
type logStreamMessage struct {
Stream string `json:"stream,omitempty"`
Status string `json:"status,omitempty"`
Progress string `json:"progress,omitempty"`
ID string `json:"id,omitempty"`
Error string `json:"error,omitempty"`
}
func DockerLogDecoderWrapper(serviceID, taskID string, client *mongodb_plugin.LogMongoClient) io.WriteCloser {
buffer := &bytes.Buffer{}
lines := make(map[string]int)
diff := 0
return &decoderWrapper{
buffer: buffer,
lines: lines,
diff: diff,
serviceID: serviceID,
taskID: taskID,
client: client,
}
}
func (w *decoderWrapper) Close() error {
w.client.AddLog(autodl_base.TaskLog{
ServiceID: w.serviceID,
TaskID: w.taskID,
Message: "======== your experiment building over. wait to scheduler to machine and running ========\n",
CreatedAt: time.Now(),
Type: autodl_base.TaskLogTypeBuilding,
})
return nil
}
func (w *decoderWrapper) Write(p []byte) (int, error) {
//defer w.Flush()
n, err := w.buffer.Write(p)
if err != nil {
return n, err
}
return n, w.Decode()
}
//func (w decoderWrapper) Flush() {
//
//}
func (w *decoderWrapper) encode(m logStreamMessage) {
var endl string
if m.Stream == "" && (m.Progress != "" || m.Status != "") {
endl = "\r"
}
var line string
if m.ID != "" {
line += fmt.Sprintf("%s: ", m.ID)
}
if m.Progress != "" {
line += fmt.Sprintf("%s %s%s", m.Status, m.Progress, endl)
} else if m.Stream != "" {
line += fmt.Sprintf("%s%s", m.Stream, endl)
} else {
line += fmt.Sprintf("%s%s", m.Status, endl)
}
if !strings.HasSuffix(line, "\n") {
line += "\n"
}
w.client.AddLog(autodl_base.TaskLog{
ServiceID: w.serviceID,
TaskID: w.taskID,
Message: line,
CreatedAt: time.Now(),
Type: autodl_base.TaskLogTypeBuilding,
})
}
func (w *decoderWrapper) setCursor(id string) {
if id != "" {
line, ok := w.lines[id]
if !ok {
line = len(w.lines)
w.lines[id] = line
fmt.Print("\n")
} else {
w.diff = len(w.lines) - line
}
} else {
w.lines = make(map[string]int)
}
}
func (w *decoderWrapper) Decode() error {
decoder := json.NewDecoder(w.buffer)
for {
m := logStreamMessage{}
if err := decoder.Decode(&m); err != nil {
if err != io.EOF {
return err
}
// Recopy remaining bytes into buffer to be available again for json decoder.
b, err := ioutil.ReadAll(decoder.Buffered())
if err != nil {
return err
}
w.buffer = bytes.NewBuffer(b)
return nil
}
if m.Error != "" {
return errors.New(m.Error)
}
w.setCursor(m.ID)
w.encode(m)
}
}
service/utils/service.go 0 → 100644
package utils
import (
"fmt"
"gitlab.seetatech.com/autodl.com/autodl-k8s.git/template"
)
func GetJupyterHost(host string) (jupyterHost string) {
jupyterHost = fmt.Sprintf("%s/jupyter/", host)
return
}
func GetTensorboardHost(host string) (jupyterHost string) {
jupyterHost = fmt.Sprintf("%s/tensorboard/", host, )
return
}
func GetStandardHost(host string) (jupyterHost string) {
jupyterHost = fmt.Sprintf("%s/standard/", host, )
return
}
func GetServiceName(namespace, serviceID, taskID string) string {
return fmt.Sprintf("%s.%s.svc.cluster.local", template.GetServiceName(serviceID, taskID), namespace)
}
service/utils/task.go 0 → 100644
package utils
import (
"autodl-core/git_repo"
"autodl-core/libs/docker"
"autodl-core/libs/docker/render"
"autodl-core/libs/path"
"autodl-core/mongodb_plugin"
"bytes"
"encoding/json"
"fmt"
log "github.com/cihub/seelog"
"github.com/gogs/git-module"
"github.com/pkg/errors"
"github.com/spf13/viper"
"gitlab.seetatech.com/autodl.com/autodl-base.git/protoV1"
"io/ioutil"
"os"
"strings"
)
type Task struct {
proto_v1.Task
}
func NewTask(task proto_v1.Task) *Task {
return &Task{
task,
}
}
func (t *Task) tag() string {
return "latest"
}
func (t *Task) imageName() string {
return fmt.Sprintf("%s/%s/%s", viper.GetString("docker.registry_host"), viper.GetString("docker.registry_group_name"), t.Task.TaskId)
}
func (t *Task) ImageNameTag() string {
return t.imageName() + ":" + t.tag()
}
func (t *Task) CodeAbsPath() string {
return fmt.Sprintf("%s/%s", viper.GetString("app.mount_path"), strings.TrimLeft(t.Task.BuildInfo.CodePath, "/"))
}
/*
├── /tmp/t.ID(tempPath)
│ ├── Dockerfile
│ └── t.ID(codePath)
│ ├── code.*
│ ├── code.content
*/
func (t *Task) tempPath() string {
return fmt.Sprintf("/tmp/%s", t.Task.TaskId)
}
func (t *Task) codePath() string {
return fmt.Sprintf("/tmp/%s/%s", t.Task.TaskId, t.Task.TaskId)
}
func (t *Task) dockerfile(copyCode bool) *bytes.Buffer {
//env := make(map[string]string)
//for _, v := range t.Task.BuildInfo.Envs {
// env[v.Key] = v.Value
//}
var buf bytes.Buffer
render.WriteRenderDockerfile(&buf, t.Task.BuildInfo.FromImage,
t.Task.TaskId,
"/code",
copyCode,
t.Task.BuildInfo.Runs,
nil,
t.Task.BuildInfo.CopyAfterRun)
return &buf
}
func (t *Task) ToString() string {
b, err := json.Marshal(t)
if err != nil {
log.Error("Task marshal fail: ", err.Error())
}
return string(b)
}
func (t *Task) Validate() (err error) {
var errMsg string
if len(t.Task.TaskId) == 0 {
errMsg += " missing " + "[Task id]"
}
if t.Task.PodInfo == nil {
errMsg += " missing " + "[pod info]"
} else {
if t.Task.PodInfo.Resource == nil {
errMsg += " missing " + "[resource]"
} else {
if t.Task.PodInfo.Resource.Cpu == nil || t.Task.PodInfo.Resource.Gpu == nil || t.Task.PodInfo.Resource.Memory == nil {
errMsg += " missing " + "[resource.Cpu | resource.Gpu | resource.Memory]"
}
}
}
if t.Task.BuildInfo == nil {
errMsg += " missing " + "[build info]"
} else {
if len(t.Task.BuildInfo.FromImage) == 0 {
errMsg += " missing " + "[from image]"
}
if len(t.Task.BuildInfo.Cmd) == 0 {
errMsg += " missing " + "[cmd]"
}
if t.Task.BuildInfo.HaveCode {
switch t.Task.BuildInfo.CodeType {
case proto_v1.TaskBuildInfo_Normal, proto_v1.TaskBuildInfo_GitRepo, proto_v1.TaskBuildInfo_GitBareRepo:
if len(t.Task.BuildInfo.CodePath) == 0 {
errMsg += " missing " + "[code path]"
}
default:
errMsg += " missing " + "[code type]"
}
}
}
if len(errMsg) > 0 {
err = errors.Errorf("Parameter missing: %s", errMsg)
}
return
}
func (t *Task) BuildAndPush(serviceID string, logPlugin *mongodb_plugin.LogMongoClient) (err error) {
decoder := DockerLogDecoderWrapper(serviceID, t.Task.TaskId, logPlugin)
logStream := docker.LogStream{Decoder: decoder}
endpoint := "unix:///var/run/docker.sock"
var do docker.Docker
do, err = docker.New(endpoint)
if err != nil {
log.Error("docker new error")
return
}
path.DeletePath(t.codePath())
os.MkdirAll(t.tempPath(), 0644)
//defer path.DeletePath(t.tempPath())
var copyCode bool
if t.Task.BuildInfo.HaveCode {
switch t.Task.BuildInfo.CodeType {
case proto_v1.TaskBuildInfo_Normal:
err = path.CopyFolder(t.CodeAbsPath(), t.codePath())
if err != nil {
err = errors.Wrap(err, "copy code to tmp code path failed")
return
}
copyCode = true
case proto_v1.TaskBuildInfo_GitRepo:
err = path.CopyFolder(t.CodeAbsPath(), t.codePath())
if err != nil {
err = errors.Wrap(err, "copy code to tmp code path failed")
return
}
path.DeletePath(fmt.Sprintf("%s/.git", t.codePath()))
copyCode = true
case proto_v1.TaskBuildInfo_GitBareRepo:
err = git.Clone(t.CodeAbsPath(), t.codePath(), git.CloneRepoOptions{Branch: t.Task.BuildInfo.CodeBranch})
if err != nil {
err = fmt.Errorf("copy repository to tmp path error: %s", err.Error())
return
}
if len(t.Task.BuildInfo.CodeCommit) != 0 {
var repoHub *git_repo.Repo
repoHub, err = git_repo.Open(t.codePath())
if err != nil {
return
}
err = repoHub.Checkout(t.Task.BuildInfo.CodeCommit)
if err != nil {
return
}
path.DeletePath(fmt.Sprintf("%s/.git", t.codePath()))
}
copyCode = true
default:
}
}
err = ioutil.WriteFile(t.tempPath()+"/Dockerfile", t.dockerfile(copyCode).Bytes(), 0644)
if err != nil {
return
}
err = do.Build(docker.BuildOptions{
Name: t.imageName(),
Directory: t.tempPath(),
Pull: true,
NoCache: false,
}, logStream)
if err != nil {
log.Error("docker build error")
return
}
err = do.Push(docker.PushOptions{
Repository: t.imageName(),
Registry: viper.GetString("docker.registry_host"),
Tag: t.tag(),
}, logStream)
decoder.Close()
return
}
service/wire.go 0 → 100644
//+build wireinject
package service
import (
"autodl-core/k8s_plugin"
"autodl-core/models"
"autodl-core/mongodb_plugin"
"autodl-core/redis_plugin"
"github.com/google/wire"
"github.com/jinzhu/gorm"
)
func collectorProvider(k8sManager *k8s_plugin.K8sManager, logMongoClient *mongodb_plugin.LogMongoClient, dbConn *gorm.DB, jobModel *models.JobModel, plugin *redis_plugin.ResourcePlugin, ) *Collector {
return &Collector{k8sManager, logMongoClient, dbConn, jobModel, plugin}
}
func monitorProvider(k8sManager *k8s_plugin.K8sManager, taskStatusModel *models.TaskStatusModel, dbConn *gorm.DB, jobModel *models.JobModel, taskStatusPlugin *redis_plugin.TaskStatusPlugin, serviceModel *models.ServiceModel, logMongoPlugin *mongodb_plugin.LogMongoClient, cs *Status) *Monitor {
return &Monitor{k8sManager, taskStatusModel, dbConn, jobModel, taskStatusPlugin, serviceModel, logMongoPlugin, cs}
}
func rpcProvider(
dbConn *gorm.DB,
taskStatusPlugin *redis_plugin.TaskStatusPlugin,
taskStatusModel *models.TaskStatusModel,
jobModel *models.JobModel,
logPlugin *mongodb_plugin.LogMongoClient,
serviceModel *models.ServiceModel,
taskPlugin *redis_plugin.TaskPlugin,
taskModel *models.TaskModel,
plugin *redis_plugin.ResourcePlugin,
k8sManager *k8s_plugin.K8sManager) *Rpc {
return &Rpc{dbConn, taskStatusPlugin, taskStatusModel, jobModel, logPlugin, k8sManager, serviceModel, taskPlugin, taskModel, plugin}
}
func statusProvider(taskStatusModel *models.TaskStatusModel, dbConn *gorm.DB, taskStatusPlugin *redis_plugin.TaskStatusPlugin) *Status {
return &Status{taskStatusModel, dbConn, taskStatusPlugin}
}
func workerProvider(
dbConn *gorm.DB,
taskStatusPlugin *redis_plugin.TaskStatusPlugin,
jobModel *models.JobModel,
logPlugin *mongodb_plugin.LogMongoClient,
taskPlugin *redis_plugin.TaskPlugin,
cs *Status,
taskModel *models.TaskModel,
k8sManager *k8s_plugin.K8sManager) *Worker {
return &Worker{dbConn, taskStatusPlugin, jobModel, logPlugin, k8sManager, taskPlugin, taskModel, cs}
}
func logStashProvider(k8sManager *k8s_plugin.K8sManager) *LogStash {
return &LogStash{k8sManager}
}
type ServerFactory struct {
RpcServer *Rpc
}
var serverSet = wire.NewSet(
rpcProvider,
models.DBProvider,
models.TaskStatusModelProvider,
models.JobModelProvider,
models.ServiceModelProvider,
models.TaskModelProvider,
redis_plugin.TaskStatusPluginProvider,
redis_plugin.RedisClientProvider,
redis_plugin.TaskPluginProvider,
redis_plugin.ResourcePluginProvider,
mongodb_plugin.MongoClientProvider,
k8s_plugin.K8sManagerProvider,
wire.Struct(new(ServerFactory), "*"),
)
func InitializeServerFactory() ServerFactory {
wire.Build(serverSet)
return ServerFactory{}
}
type WorkerFactory struct {
Worker *Worker
}
var workerSet = wire.NewSet(
workerProvider,
models.DBProvider,
models.TaskStatusModelProvider,
models.JobModelProvider,
models.ServiceModelProvider,
models.TaskModelProvider,
redis_plugin.TaskStatusPluginProvider,
redis_plugin.RedisClientProvider,
redis_plugin.TaskPluginProvider,
mongodb_plugin.MongoClientProvider,
k8s_plugin.K8sManagerProvider,
statusProvider,
wire.Struct(new(WorkerFactory), "*"),
)
func InitializeWorkerFactory() WorkerFactory {
wire.Build(workerSet)
return WorkerFactory{}
}
type CollectorFactory struct {
Collector *Collector
}
var collectorSet = wire.NewSet(
collectorProvider,
mongodb_plugin.MongoClientProvider,
models.DBProvider,
models.JobModelProvider,
k8s_plugin.K8sManagerProvider,
redis_plugin.RedisClientProvider,
redis_plugin.ResourcePluginProvider,
wire.Struct(new(CollectorFactory), "*"),
)
func InitializeCollectorFactory() CollectorFactory {
wire.Build(collectorSet)
return CollectorFactory{}
}
type MonitorFactory struct {
Monitor *Monitor
}
var monitorSet = wire.NewSet(
monitorProvider,
models.DBProvider,
models.TaskStatusModelProvider,
models.JobModelProvider,
models.ServiceModelProvider,
redis_plugin.RedisClientProvider,
redis_plugin.TaskStatusPluginProvider,
mongodb_plugin.MongoClientProvider,
k8s_plugin.K8sManagerProvider,
statusProvider,
wire.Struct(new(MonitorFactory), "*"),
)
func InitializeMonitorFactory() MonitorFactory {
wire.Build(monitorSet)
return MonitorFactory{}
}
type LogStashFactory struct {
LogStash *LogStash
}
var logStashSet = wire.NewSet(
logStashProvider,
k8s_plugin.K8sManagerProvider,
wire.Struct(new(LogStashFactory), "*"),
)
func InitializeLogStashFactory() LogStashFactory {
wire.Build(logStashSet)
return LogStashFactory{}
}
service/wire_gen.go 0 → 100644
// Code generated by Wire. DO NOT EDIT.
//go:generate wire
//+build !wireinject
package service
import (
"autodl-core/k8s_plugin"
"autodl-core/models"
"autodl-core/mongodb_plugin"
"autodl-core/redis_plugin"
"github.com/google/wire"
"github.com/jinzhu/gorm"
)
// Injectors from wire.go:
func InitializeServerFactory() ServerFactory {
db := models.DBProvider()
client := redis_plugin.RedisClientProvider()
taskStatusPlugin := redis_plugin.TaskStatusPluginProvider(client)
taskStatusModel := models.TaskStatusModelProvider()
jobModel := models.JobModelProvider()
logMongoClient := mongodb_plugin.MongoClientProvider()
serviceModel := models.ServiceModelProvider()
taskPlugin := redis_plugin.TaskPluginProvider(client)
taskModel := models.TaskModelProvider()
resourcePlugin := redis_plugin.ResourcePluginProvider(client)
k8sManager := k8s_plugin.K8sManagerProvider()
rpc := rpcProvider(db, taskStatusPlugin, taskStatusModel, jobModel, logMongoClient, serviceModel, taskPlugin, taskModel, resourcePlugin, k8sManager)
serverFactory := ServerFactory{
RpcServer: rpc,
}
return serverFactory
}
func InitializeWorkerFactory() WorkerFactory {
db := models.DBProvider()
client := redis_plugin.RedisClientProvider()
taskStatusPlugin := redis_plugin.TaskStatusPluginProvider(client)
jobModel := models.JobModelProvider()
logMongoClient := mongodb_plugin.MongoClientProvider()
taskPlugin := redis_plugin.TaskPluginProvider(client)
taskStatusModel := models.TaskStatusModelProvider()
status := statusProvider(taskStatusModel, db, taskStatusPlugin)
taskModel := models.TaskModelProvider()
k8sManager := k8s_plugin.K8sManagerProvider()
worker := workerProvider(db, taskStatusPlugin, jobModel, logMongoClient, taskPlugin, status, taskModel, k8sManager)
workerFactory := WorkerFactory{
Worker: worker,
}
return workerFactory
}
func InitializeCollectorFactory() CollectorFactory {
k8sManager := k8s_plugin.K8sManagerProvider()
logMongoClient := mongodb_plugin.MongoClientProvider()
db := models.DBProvider()
jobModel := models.JobModelProvider()
client := redis_plugin.RedisClientProvider()
resourcePlugin := redis_plugin.ResourcePluginProvider(client)
collector := collectorProvider(k8sManager, logMongoClient, db, jobModel, resourcePlugin)
collectorFactory := CollectorFactory{
Collector: collector,
}
return collectorFactory
}
func InitializeMonitorFactory() MonitorFactory {
k8sManager := k8s_plugin.K8sManagerProvider()
taskStatusModel := models.TaskStatusModelProvider()
db := models.DBProvider()
jobModel := models.JobModelProvider()
client := redis_plugin.RedisClientProvider()
taskStatusPlugin := redis_plugin.TaskStatusPluginProvider(client)
serviceModel := models.ServiceModelProvider()
logMongoClient := mongodb_plugin.MongoClientProvider()
status := statusProvider(taskStatusModel, db, taskStatusPlugin)
monitor := monitorProvider(k8sManager, taskStatusModel, db, jobModel, taskStatusPlugin, serviceModel, logMongoClient, status)
monitorFactory := MonitorFactory{
Monitor: monitor,
}
return monitorFactory
}
func InitializeLogStashFactory() LogStashFactory {
k8sManager := k8s_plugin.K8sManagerProvider()
logStash := logStashProvider(k8sManager)
logStashFactory := LogStashFactory{
LogStash: logStash,
}
return logStashFactory
}
// wire.go:
func collectorProvider(k8sManager *k8s_plugin.K8sManager, logMongoClient *mongodb_plugin.LogMongoClient, dbConn *gorm.DB, jobModel *models.JobModel, plugin *redis_plugin.ResourcePlugin) *Collector {
return &Collector{k8sManager, logMongoClient, dbConn, jobModel, plugin}
}
func monitorProvider(k8sManager *k8s_plugin.K8sManager, taskStatusModel *models.TaskStatusModel, dbConn *gorm.DB, jobModel *models.JobModel, taskStatusPlugin *redis_plugin.TaskStatusPlugin, serviceModel *models.ServiceModel, logMongoPlugin *mongodb_plugin.LogMongoClient, cs *Status) *Monitor {
return &Monitor{k8sManager, taskStatusModel, dbConn, jobModel, taskStatusPlugin, serviceModel, logMongoPlugin, cs}
}
func rpcProvider(
dbConn *gorm.DB,
taskStatusPlugin *redis_plugin.TaskStatusPlugin,
taskStatusModel *models.TaskStatusModel,
jobModel *models.JobModel,
logPlugin *mongodb_plugin.LogMongoClient,
serviceModel *models.ServiceModel,
taskPlugin *redis_plugin.TaskPlugin,
taskModel *models.TaskModel,
plugin *redis_plugin.ResourcePlugin,
k8sManager *k8s_plugin.K8sManager) *Rpc {
return &Rpc{dbConn, taskStatusPlugin, taskStatusModel, jobModel, logPlugin, k8sManager, serviceModel, taskPlugin, taskModel, plugin}
}
func statusProvider(taskStatusModel *models.TaskStatusModel, dbConn *gorm.DB, taskStatusPlugin *redis_plugin.TaskStatusPlugin) *Status {
return &Status{taskStatusModel, dbConn, taskStatusPlugin}
}
func workerProvider(
dbConn *gorm.DB,
taskStatusPlugin *redis_plugin.TaskStatusPlugin,
jobModel *models.JobModel,
logPlugin *mongodb_plugin.LogMongoClient,
taskPlugin *redis_plugin.TaskPlugin,
cs *Status,
taskModel *models.TaskModel,
k8sManager *k8s_plugin.K8sManager) *Worker {
return &Worker{dbConn, taskStatusPlugin, jobModel, logPlugin, k8sManager, taskPlugin, taskModel, cs}
}
func logStashProvider(k8sManager *k8s_plugin.K8sManager) *LogStash {
return &LogStash{k8sManager}
}
type ServerFactory struct {
RpcServer *Rpc
}
var serverSet = wire.NewSet(
rpcProvider, models.DBProvider, models.TaskStatusModelProvider, models.JobModelProvider, models.ServiceModelProvider, models.TaskModelProvider, redis_plugin.TaskStatusPluginProvider, redis_plugin.RedisClientProvider, redis_plugin.TaskPluginProvider, redis_plugin.ResourcePluginProvider, mongodb_plugin.MongoClientProvider, k8s_plugin.K8sManagerProvider, wire.Struct(new(ServerFactory), "*"),
)
type WorkerFactory struct {
Worker *Worker
}
var workerSet = wire.NewSet(
workerProvider, models.DBProvider, models.TaskStatusModelProvider, models.JobModelProvider, models.ServiceModelProvider, models.TaskModelProvider, redis_plugin.TaskStatusPluginProvider, redis_plugin.RedisClientProvider, redis_plugin.TaskPluginProvider, mongodb_plugin.MongoClientProvider, k8s_plugin.K8sManagerProvider, statusProvider, wire.Struct(new(WorkerFactory), "*"),
)
type CollectorFactory struct {
Collector *Collector
}
var collectorSet = wire.NewSet(
collectorProvider, mongodb_plugin.MongoClientProvider, models.DBProvider, models.JobModelProvider, k8s_plugin.K8sManagerProvider, redis_plugin.RedisClientProvider, redis_plugin.ResourcePluginProvider, wire.Struct(new(CollectorFactory), "*"),
)
type MonitorFactory struct {
Monitor *Monitor
}
var monitorSet = wire.NewSet(
monitorProvider, models.DBProvider, models.TaskStatusModelProvider, models.JobModelProvider, models.ServiceModelProvider, redis_plugin.RedisClientProvider, redis_plugin.TaskStatusPluginProvider, mongodb_plugin.MongoClientProvider, k8s_plugin.K8sManagerProvider, statusProvider, wire.Struct(new(MonitorFactory), "*"),
)
type LogStashFactory struct {
LogStash *LogStash
}
var logStashSet = wire.NewSet(
logStashProvider, k8s_plugin.K8sManagerProvider, wire.Struct(new(LogStashFactory), "*"),
)
service/worker.go 0 → 100644
package service
import (
"autodl-core/k8s_plugin"
"autodl-core/libs"
"autodl-core/libs/parser"
"autodl-core/models"
"autodl-core/mongodb_plugin"
"autodl-core/redis_plugin"
"autodl-core/service/utils"
"encoding/json"
"fmt"
log "github.com/cihub/seelog"
"github.com/go-redis/redis"
"github.com/jinzhu/gorm"
"github.com/pkg/errors"
"github.com/spf13/viper"
"gitlab.seetatech.com/autodl.com/autodl-base.git"
"gitlab.seetatech.com/autodl.com/autodl-base.git/protoV1"
"gitlab.seetatech.com/autodl.com/autodl-k8s.git/manager"
"gitlab.seetatech.com/autodl.com/autodl-k8s.git/spawner"
"time"
)
type Worker struct {
dbConn *gorm.DB
taskStatusPlugin *redis_plugin.TaskStatusPlugin
jobModel *models.JobModel
logPlugin *mongodb_plugin.LogMongoClient
k8sManager *k8s_plugin.K8sManager
taskPlugin *redis_plugin.TaskPlugin
taskModel *models.TaskModel
*Status
}
type getBackTask struct {
taskUUID int
err error
}
func (s *Worker) Run() (err error) {
retryTime, maxRetryTime := 0, 2
getBackChan := make(chan getBackTask, 10)
defer close(getBackChan)
go func() {
for {
select {
case content, ok := <-getBackChan:
if !ok {
return
}
if content.err != nil {
if retryTime < maxRetryTime {
s.taskPlugin.AddTop(content.taskUUID)
retryTime++
} else {
log.Errorf("task [%d] failed over %d times, drop it. %v", content.taskUUID, maxRetryTime, content.err)
retryTime = 0
}
}
}
}
}()
for {
var taskUUID int
taskUUID, err = s.taskPlugin.Pop()
if err != nil {
if err != redis.Nil {
return
}
time.Sleep(time.Second * 2)
continue
}
var task *models.Task
task, err = s.taskModel.GetLast(s.dbConn, taskUUID)
if err != nil {
log.Errorf("get task [%d] from task table failed: %s", taskUUID, err.Error())
continue
}
switch task.TaskType {
case libs.RunTask:
var runTaskRequest proto_v1.RunTaskRequest
err = json.Unmarshal([]byte(task.TaskList), &runTaskRequest)
if err != nil {
log.Errorf("unmarshal [%d] task failed: %s (%s)", taskUUID, err.Error(), task.TaskList)
continue
}
go s.taskPublish(taskUUID, &runTaskRequest, getBackChan)
case libs.StopTask:
var stopTaskRequest proto_v1.StopTaskRequest
err = json.Unmarshal([]byte(task.TaskList), &stopTaskRequest)
if err != nil {
log.Errorf("unmarshal [%d] task failed: %s (%s)", taskUUID, err.Error(), task.TaskList)
continue
}
go s.taskStop(taskUUID, &stopTaskRequest, getBackChan)
}
}
}
func (s *Worker) taskPublish(taskUUID int, t *proto_v1.RunTaskRequest, getBackChan chan<- getBackTask) {
for _, task := range t.Tasks {
err := s.run(task, t.ServiceId)
if err != nil {
getBackChan <- getBackTask{
taskUUID: taskUUID,
err: errors.Wrap(err, "run task failed"),
}
return
}
}
}
func (s *Worker) run(task *proto_v1.Task, serviceID string) (err error) {
t := utils.NewTask(*task)
err = t.Validate()
if err != nil {
return
}
err = s.jobModel.Create(s.dbConn, &models.Job{
ServiceID: serviceID,
TaskID: t.TaskId,
HasService: task.PodInfo.ServiceType != proto_v1.PodInfo_NoService,
Definition: []byte("{}"),
})
if err != nil {
log.Error("create job record failed: %v", err)
}
err = s.Building(serviceID, t.TaskId)
if err == libs.TaskIsDoneCanNotSetStatusError {
log.Info("task was stopped, skip it before build")
return
}
//不要跳过所有build的过程了, 跳过会导致的问题 1: image pull失败 k8s不会更新status状态, 2: 所有节点都需要从公网下载image, 速度很慢
err = t.BuildAndPush(serviceID, s.logPlugin)
if err != nil {
log.Error(fmt.Sprintf("Build fail: %s. (%s)", err.Error(), t.ToString()))
log.Error("image info: ", t.ImageNameTag())
s.Failed(serviceID, t.TaskId, fmt.Sprintf("Build fail: %s. ", err.Error()))
return
}
err = s.Scheduled(serviceID, t.TaskId)
if err == libs.TaskIsDoneCanNotSetStatusError {
log.Info("task was stopped, skip it before schedule")
return
}
pc := parser.GetSpawnerParserConfig(&t.Task, t.ImageNameTag(), serviceID)
spa, err := spawner.NewExpSpawner(pc, viper.GetString("k8s.namespace"), viper.GetString("app.mount_path"), viper.GetString("k8s.pvc"), s.k8sManager.K8sManager)
if err != nil {
err = errors.Wrap(err, "new spawner failed")
s.Failed(serviceID, t.TaskId, fmt.Sprintf("Init Spawner fail: %s. (%s)", err.Error(), t.ToString()))
return
}
pod, err := spa.StartExp(pc.ID)
if err != nil {
err = errors.Wrap(err, "start task failed")
s.Failed(serviceID, t.TaskId, fmt.Sprintf("Run task fail: : %s. (%s)", err.Error(), t.ToString()))
return
}
s.Starting(serviceID, t.TaskId)
definition, _ := json.Marshal(pod[manager.MASTER_ROLE])
err = s.jobModel.UpdateDefinition(s.dbConn, serviceID, t.TaskId, definition)
if err != nil {
log.Error("update job record failed: %v", err)
}
return
}
func (s *Worker) taskStop(taskUUID int, t *proto_v1.StopTaskRequest, getBackChan chan<- getBackTask) {
for _, task := range t.Tasks {
err := s.stop(t.ServiceId, task.TaskId)
if err != nil {
getBackChan <- getBackTask{
taskUUID: taskUUID,
err: errors.Wrap(err, "stop task failed"),
}
return
}
}
}
func (s *Worker) stop(serviceID, taskID string) (err error) {
err = s.UpdateTaskStatus(serviceID, taskID, autodl_base.TaskStopped, "Stopped by yourself.")
if err != nil && err != libs.TaskIsDoneCanNotSetStatusError && err != libs.TaskStatusExistCanNotSetStatusError {
return
}
var spa *spawner.ExpSpawner
spa, err = spawner.NewExpStopSpawner(serviceID, taskID, s.k8sManager.K8sManager)
err = spa.StopExp()
if err != nil {
log.Error("delete exp master failed: ", err.Error(), " ", serviceID, " ", taskID)
}
return
}
func (s *Worker) Building(serviceID, taskID string) error {
return s.UpdateTaskStatus(serviceID, taskID, autodl_base.TaskBuilding, "")
}
func (s *Worker) Scheduled(serviceID, taskID string) error {
return s.UpdateTaskStatus(serviceID, taskID, autodl_base.TaskScheduled, "")
}
func (s *Worker) Starting(serviceID, taskID string) {
s.UpdateTaskStatus(serviceID, taskID, autodl_base.TaskStarting, "")
}
func (s *Worker) Failed(serviceID, taskID string, msg string) {
s.UpdateTaskStatus(serviceID, taskID, autodl_base.TaskFailed, msg)
}
func (s *Worker) Stopped(serviceID, taskID string, msg string) error {
err := s.UpdateTaskStatus(serviceID, taskID, autodl_base.TaskStopped, msg)
if err == libs.TaskIsDoneCanNotSetStatusError {
return nil
}
return err
}
sql/1106.sql 0 → 100644
create table `autodl-core`.task_status
(
id int auto_increment,
task_id varchar(64) not null,
status varchar(64) not null,
created_at datetime null,
msg text null,
constraint task_id_uindex unique (id));
alter table `autodl-core`.task_status
add primary key (id)
;
create table `autodl-core`.job
(
id int auto_increment,
task_id varchar(64) not null,
definition json not null,
created_at datetime null,
updated_at datetime null,
constraint job_id_uindex unique (id)
)
;
alter table `autodl-core`.job
add primary key (id)
;
sql/1121.sql 0 → 100644
ALTER TABLE job ADD mode varchar(63) NULL
\ No newline at end of file
sql/190321.sql 0 → 100644
create table `autodl-core`.service
(
id int auto_increment,
service_id varchar(255) not null,
noti_api text not null,
created_at datetime null,
constraint service_id_uindex unique (id),
constraint service_service_id_uindex unique (service_id)
)
;
alter table `autodl-core`.service
add primary key (id)
;
ALTER TABLE job ADD has_service tinyint DEFAULT 0 NULL;
ALTER TABLE job ADD service_is_clean tinyint DEFAULT 0 NULL;
ALTER TABLE job DROP mode;
ALTER TABLE job ADD log_is_persist tinyint DEFAULT 0 NULL;
ALTER TABLE job ADD service_id varchar(255) DEFAULT "" NOT NULL;
ALTER TABLE task_status ADD service_id varchar(255) DEFAULT "" NOT NULL;
ALTER TABLE task_status MODIFY COLUMN service_id varchar(255) NOT NULL DEFAULT '' AFTER id;
ALTER TABLE task_status ADD is_notify_success tinyint DEFAULT 0 NOT NULL;
create table `autodl-core`.task
(
id int auto_increment,
service_id varchar(255) not null,
task_type varchar(255) not null,
task_list text not null,
created_at datetime null,
constraint task_id_uindex unique (id)
)
;
alter table `autodl-core`.task
add primary key (id)
;
sql/190402.sql 0 → 100644
ALTER TABLE job ADD log_is_flush_over tinyint DEFAULT 0 NULL;
\ No newline at end of file
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!