Skip to content

shenyizhong / minsync

Go to a project
Commit c664042b by shenyizhong
Options

* upload files

1 parent 01a664cc
Inline Side-by-side
Showing with 281 additions and 43 deletions
.gitignore
src/config.json
src/minsync
src/minsync.log
src/Makefile
src/auth/auth.go
package auth
import (
"fmt"
"log"
"os"
"minsync/nodetree"
)
func init() {
logFile, err := os.OpenFile("./minsync.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
if err != nil {
fmt.Println("open log file failed, err:", err)
return
}
log.SetOutput(logFile)
log.SetFlags(log.Lshortfile | log.Ldate | log.Ltime)
log.Println("auth.init() called")
}
......@@ -25,7 +16,5 @@ func Login(user string, password string) (string, error) {
}
func GetToken() string {
var token_op string
token_op = "96eb895c9a07435ea6fb6ba61543c4b2"
return token_op
return nodetree.StrToken
}
src/download/download.go
package download
import (
"github.com/sirupsen/logrus"
"log"
//"os"
//"github.com/sirupsen/logrus"
"minsync/nodetree"
"os"
)
func init() {
log.Println("minsync.download.init() called")
var log01 = logrus.New()
log01.Formatter = new(logrus.JSONFormatter)
log01.Formatter = new(logrus.TextFormatter) //default
log01.Formatter.(*logrus.TextFormatter).DisableColors = true // remove colors
log01.Formatter.(*logrus.TextFormatter).DisableTimestamp = true // remove timestamp from test output
log01.Level = logrus.TraceLevel
log01.Out = os.Stdout
//var log01 = logrus.New()
//log01.Formatter = new(logrus.JSONFormatter)
//log01.Formatter = new(logrus.TextFormatter) //default
//log01.Formatter.(*logrus.TextFormatter).DisableColors = true // remove colors
//log01.Formatter.(*logrus.TextFormatter).DisableTimestamp = true // remove timestamp from test output
//log01.Level = logrus.TraceLevel
//log01.Out = os.Stdout
log01.Debug("minsync.download.init() called, from logrus")
//log01.Debug("minsync.download.init() called, from logrus")
}
func Download(src string, dst string) (nodetree.Task, error) {
......
src/httpcli/httpcli.go
package httpcli
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"minsync/auth"
"minsync/nodetree"
"net/http"
"os"
"os/exec"
"strconv"
"strings"
)
var host string
var port int
func init() {
log.Println("minsync.httpcli.init() called")
file, _ := os.Open("./config.json")
defer file.Close()
decoder := json.NewDecoder(file)
conf := nodetree.Configuration{}
err := decoder.Decode(&conf)
if err != nil {
log.Println("Error:", err)
}
host = conf.Host
port = conf.Port
}
func Get(url string) ([]byte, error) {
......@@ -30,3 +51,103 @@ func Delete(postdata string) ([]byte, error) {
func Update(postdata string) ([]byte, error) {
return nil, nil
}
func Mkdir(path string) {
var tmpStrs = strings.Split(path, nodetree.StrSrc)
if len(tmpStrs) < 2 {
return
}
// "/front/data/createfolder?name=data/folder3/"
url := "http://"
url += host
url += ":"
url += strconv.Itoa(port)
url += "/front/data/createfolder?name=data"
url += tmpStrs[1]
url += "/"
log.Println("httpcli.Mkdir() called, GET request:", url)
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("token", auth.GetToken())
var resp *http.Response
resp, err := http.DefaultClient.Do(req)
if err != nil {
log.Println(err)
}
body, _ := ioutil.ReadAll(resp.Body)
log.Println(string(body))
}
func Upload(node *nodetree.NodeLoc) {
var tmpStrs = strings.Split(node.Path, nodetree.StrSrc)
if len(tmpStrs) < 2 {
return
}
// "/front/data/upload?name=data/folder3/ddd.txt"
url := "http://"
url += host
url += ":"
url += strconv.Itoa(port)
url += "/front/data/upload?name=data"
url += tmpStrs[1]
log.Println("httpcli.Upload() called, GET request:", url)
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("token", auth.GetToken())
var resp *http.Response
resp, err := http.DefaultClient.Do(req)
if err != nil {
log.Println(err)
}
body, _ := ioutil.ReadAll(resp.Body)
var jsonRes nodetree.UploadFile
json.Unmarshal(body, &jsonRes)
strCmd01 := jsonRes.Data.Url
strCmd02 := "x-amz-signature="
strCmd02 += jsonRes.Data.X_amz_signature
strCmd03 := "bucket="
strCmd03 += jsonRes.Data.Bucket
strCmd04 := "x-amz-meta-user="
strCmd04 += jsonRes.Data.X_amz_meta_user
strCmd05 := "policy="
strCmd05 += jsonRes.Data.Policy
strCmd06 := "x-amz-algorithm="
strCmd06 += jsonRes.Data.X_amz_algorithm
strCmd07 := "x-amz-credential="
strCmd07 += jsonRes.Data.X_amz_credential
strCmd08 := "x-amz-date="
strCmd08 += jsonRes.Data.X_amz_date
strCmd09 := "key="
strCmd09 += jsonRes.Data.Key
strCmd10 := "file=@"
strCmd10 += node.Path
cmd := exec.Command("curl", strCmd01, "-F", strCmd02, "-F", strCmd03, "-F", strCmd04, "-F", strCmd05, "-F", strCmd06, "-F", strCmd07, "-F", strCmd08, "-F", strCmd09, "-F", strCmd10)
stdout, err := cmd.StdoutPipe()
if err != nil {
log.Fatal(err)
}
defer stdout.Close()
if err := cmd.Start(); err != nil {
log.Fatal(err)
}
if opBytes, err := ioutil.ReadAll(stdout); err != nil {
log.Fatal(err)
} else {
fmt.Printf("File uploaded successful : [%v]\n", node.Path)
log.Println(string(opBytes))
}
}
src/main.go
......@@ -3,7 +3,7 @@ package main
import (
"bytes"
"encoding/json"
_ "fmt"
"fmt"
_ "io/ioutil"
"log"
_ "minsync/auth"
......@@ -11,31 +11,86 @@ import (
_ "minsync/encrypt"
_ "minsync/httpcli"
"minsync/nodetree"
_ "minsync/travloc"
"minsync/travloc"
"minsync/travremot"
_ "minsync/upload"
_ "net/http"
"os"
"runtime"
_ "runtime"
)
func printUsage() {
fmt.Println("Usage: minsync --token ef0c26da821c [upload|download] src dst")
}
func main() {
var nodeloc nodetree.NodeLoc
nodeloc.Name = "look"
log.Printf("main() called, os %v, %v", runtime.GOOS, nodeloc.Name)
var option_remot nodetree.OptionRemot
option_remot.RootPath = []string{"data/"}
resp, _ := travremot.Explorer(option_remot)
// minsync --token ef0c26da821c upload /home/syz/tmp/abc /part1/box2
if len(os.Args) < 6 {
printUsage()
return
}
if !(os.Args[1] == "--token" || os.Args[4] == "--token") {
printUsage()
return
}
if os.Args[1] == "--token" {
nodetree.StrToken = os.Args[2]
nodetree.StrOp = os.Args[3]
nodetree.StrSrc = os.Args[4]
nodetree.StrDst = os.Args[5]
} else if os.Args[4] == "" {
nodetree.StrToken = os.Args[5]
nodetree.StrOp = os.Args[1]
nodetree.StrSrc = os.Args[2]
nodetree.StrDst = os.Args[3]
}
fmt.Printf("%v %v %v %v\n", nodetree.StrToken, nodetree.StrOp, nodetree.StrSrc, nodetree.StrDst)
b, err := json.Marshal(resp)
file, _ := os.Open("./config.json")
defer file.Close()
decoder := json.NewDecoder(file)
conf := nodetree.Configuration{}
err := decoder.Decode(&conf)
if err != nil {
log.Println(err)
log.Println("Error:", err)
}
var out bytes.Buffer
err = json.Indent(&out, b, "", " ")
if nodetree.StrOp == "upload" {
out.WriteTo(os.Stdout)
var option_loc nodetree.OptionLoc
option_loc.RootPath = []string{nodetree.StrSrc}
option_loc.SubFlag = true
option_loc.IgnorePath = []string{".git"}
resp_loc, _ := travloc.Explorer(option_loc, "mkdir")
b_loc, err := json.Marshal(resp_loc)
if err != nil {
log.Println(err)
}
var out_loc bytes.Buffer
err = json.Indent(&out_loc, b_loc, "", " ")
// TODO, debug in config
//out_loc.WriteTo(os.Stdout)
} else if nodetree.StrOp == "download" {
var option_remot nodetree.OptionRemot
option_remot.RootPath = []string{"data/"}
resp_remot, _ := travremot.Explorer(option_remot)
b_remot, err := json.Marshal(resp_remot)
if err != nil {
log.Println(err)
}
var out_remot bytes.Buffer
err = json.Indent(&out_remot, b_remot, "", " ")
// TODO, debug in config
//out_remot.WriteTo(os.Stdout)
}
}
src/nodetree/nodetree.go
package nodetree
import (
_ "fmt"
"fmt"
"log"
"os"
)
type Configuration struct {
Host string
Port int
}
type NodeLoc struct {
Name string `json:"name"`
Path string `json:"path"`
......@@ -48,12 +54,43 @@ type OptionRemot struct {
Token string `yaml:"token"`
}
type UploadFileData struct {
X_amz_credential string `json:"x-amz-credential"`
X_amz_date string `json:"x-amz-date"`
X_amz_signature string `json:"x-amz-signature"`
X_amz_algorithm string `json:"x-amz-algorithm"`
X_amz_meta_user string `json:"x-amz-meta-user"`
Bucket string `json:"bucket"`
Policy string `json:"policy"`
Key string `json:"key"`
Url string `json:"Url"`
}
type UploadFile struct {
Data UploadFileData `json:"data"`
Msg string `json:"msg"`
}
type Task struct {
Id string `json:"id"`
Progress int `json:"progress"`
State int `json:"state"`
}
var StrOp string
var StrSrc string
var StrDst string
var StrToken string
func init() {
logFile, err := os.OpenFile("./minsync.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
if err != nil {
fmt.Println("open log file failed, err:", err)
return
}
log.SetOutput(logFile)
log.SetFlags(log.Lshortfile | log.Ldate | log.Ltime)
log.Println("minsync.nodetree init() called")
}
src/travloc/travloc.go
......@@ -4,6 +4,7 @@ import (
_ "fmt"
"io/ioutil"
"log"
"minsync/httpcli"
"minsync/nodetree"
"os"
"path"
......@@ -14,7 +15,7 @@ func init() {
log.Println("minsync.travloc.init() called")
}
func Explorer(option nodetree.OptionLoc) (nodetree.NodeLoc, error) {
func Explorer(option nodetree.OptionLoc, op string) (nodetree.NodeLoc, error) {
var root nodetree.NodeLoc
for _, p := range option.RootPath {
......@@ -26,7 +27,7 @@ func Explorer(option nodetree.OptionLoc) (nodetree.NodeLoc, error) {
child.Path = p
exploreRecursive(&child, &option)
exploreRecursive(&child, &option, op)
root.Children = append(root.Children, &child)
}
......@@ -34,7 +35,7 @@ func Explorer(option nodetree.OptionLoc) (nodetree.NodeLoc, error) {
return root, nil
}
func exploreRecursive(node *nodetree.NodeLoc, option *nodetree.OptionLoc) {
func exploreRecursive(node *nodetree.NodeLoc, option *nodetree.OptionLoc, op string) {
p, err := os.Stat(node.Path)
if err != nil {
log.Println(err)
......@@ -62,10 +63,14 @@ func exploreRecursive(node *nodetree.NodeLoc, option *nodetree.OptionLoc) {
child.IsDir = f.IsDir()
if f.IsDir() {
// mkdir remot
httpcli.Mkdir(child.Path)
if option.SubFlag {
if !IsInSlice(option.IgnorePath, f.Name()) {
node.Children = append(node.Children, &child)
exploreRecursive(&child, option)
exploreRecursive(&child, option, op)
}
} else {
if !IsInSlice(option.IgnoreFile, f.Name()) {
......@@ -75,6 +80,10 @@ func exploreRecursive(node *nodetree.NodeLoc, option *nodetree.OptionLoc) {
}
} else {
if !IsInSlice(option.IgnoreFile, f.Name()) {
// upload file
httpcli.Upload(&child)
child.Name = f.Name()
node.Children = append(node.Children, &child)
}
......
src/travremot/travremot.go
......@@ -7,11 +7,27 @@ import (
"minsync/auth"
"minsync/nodetree"
"net/http"
"os"
"strconv"
"strings"
)
var host string
var port int
func init() {
log.Println("minsync.travremot.init() called")
file, _ := os.Open("./config.json")
defer file.Close()
decoder := json.NewDecoder(file)
conf := nodetree.Configuration{}
err := decoder.Decode(&conf)
if err != nil {
log.Println("Error:", err)
}
host = conf.Host
port = conf.Port
}
func Explorer(option nodetree.OptionRemot) (nodetree.NodeRemot, error) {
......@@ -39,8 +55,14 @@ func exploreRecursive(node *nodetree.NodeRemot, option *nodetree.OptionRemot) {
log.Printf("travremot.exploreRecursive() node.Name: %s", node.Name)
url := "http://122.112.207.59:8989/front/data/query_list?pages=1&pagesize=20&prefix="
// "http://122.112.207.59:8989/front/data/query_list?pages=1&pagesize=20&prefix="
url := "http://"
url += host
url += ":"
url += strconv.Itoa(port)
url += "/front/data/query_list?pages=1&pagesize=20&prefix="
url += node.Name
log.Println(url)
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("token", auth.GetToken())
......@@ -62,6 +84,9 @@ func exploreRecursive(node *nodetree.NodeRemot, option *nodetree.OptionRemot) {
var child nodetree.NodeRemot
child.Name = tmp
child.Type = f.Type
child.UpdtDate = f.UpdtDate
child.Creator = f.Creator
child.Size = f.Size
if f.Type == "folder" {
child.Name += "/"
......
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!