Skip to content

shenyizhong / minsync

Go to a project
Commit c664042b by shenyizhong
Options

* upload files

1 parent 01a664cc
Inline Side-by-side
Showing with 281 additions and 43 deletions
.gitignore
src/config.json
src/minsync src/minsync
src/minsync.log src/minsync.log
src/Makefile
src/auth/auth.go
package auth package auth
import ( import (
"fmt"
"log" "log"
"os" "minsync/nodetree"
) )
func init() { func init() {
logFile, err := os.OpenFile("./minsync.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
if err != nil {
fmt.Println("open log file failed, err:", err)
return
}
log.SetOutput(logFile)
log.SetFlags(log.Lshortfile | log.Ldate | log.Ltime)
log.Println("auth.init() called") log.Println("auth.init() called")
} }
...@@ -25,7 +16,5 @@ func Login(user string, password string) (string, error) { ...@@ -25,7 +16,5 @@ func Login(user string, password string) (string, error) {
} }
func GetToken() string { func GetToken() string {
var token_op string return nodetree.StrToken
token_op = "96eb895c9a07435ea6fb6ba61543c4b2"
return token_op
} }
src/download/download.go
package download package download
import ( import (
"github.com/sirupsen/logrus"
"log" "log"
//"os"
//"github.com/sirupsen/logrus"
"minsync/nodetree" "minsync/nodetree"
"os"
) )
func init() { func init() {
log.Println("minsync.download.init() called") log.Println("minsync.download.init() called")
var log01 = logrus.New() //var log01 = logrus.New()
log01.Formatter = new(logrus.JSONFormatter) //log01.Formatter = new(logrus.JSONFormatter)
log01.Formatter = new(logrus.TextFormatter) //default //log01.Formatter = new(logrus.TextFormatter) //default
log01.Formatter.(*logrus.TextFormatter).DisableColors = true // remove colors //log01.Formatter.(*logrus.TextFormatter).DisableColors = true // remove colors
log01.Formatter.(*logrus.TextFormatter).DisableTimestamp = true // remove timestamp from test output //log01.Formatter.(*logrus.TextFormatter).DisableTimestamp = true // remove timestamp from test output
log01.Level = logrus.TraceLevel //log01.Level = logrus.TraceLevel
log01.Out = os.Stdout //log01.Out = os.Stdout
log01.Debug("minsync.download.init() called, from logrus") //log01.Debug("minsync.download.init() called, from logrus")
} }
func Download(src string, dst string) (nodetree.Task, error) { func Download(src string, dst string) (nodetree.Task, error) {
......
src/httpcli/httpcli.go
package httpcli package httpcli
import ( import (
"encoding/json"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"log" "log"
"minsync/auth"
"minsync/nodetree"
"net/http" "net/http"
"os"
"os/exec"
"strconv"
"strings"
) )
var host string
var port int
func init() { func init() {
log.Println("minsync.httpcli.init() called") log.Println("minsync.httpcli.init() called")
file, _ := os.Open("./config.json")
defer file.Close()
decoder := json.NewDecoder(file)
conf := nodetree.Configuration{}
err := decoder.Decode(&conf)
if err != nil {
log.Println("Error:", err)
}
host = conf.Host
port = conf.Port
} }
func Get(url string) ([]byte, error) { func Get(url string) ([]byte, error) {
...@@ -30,3 +51,103 @@ func Delete(postdata string) ([]byte, error) { ...@@ -30,3 +51,103 @@ func Delete(postdata string) ([]byte, error) {
func Update(postdata string) ([]byte, error) { func Update(postdata string) ([]byte, error) {
return nil, nil return nil, nil
} }
func Mkdir(path string) {
var tmpStrs = strings.Split(path, nodetree.StrSrc)
if len(tmpStrs) < 2 {
return
}
// "/front/data/createfolder?name=data/folder3/"
url := "http://"
url += host
url += ":"
url += strconv.Itoa(port)
url += "/front/data/createfolder?name=data"
url += tmpStrs[1]
url += "/"
log.Println("httpcli.Mkdir() called, GET request:", url)
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("token", auth.GetToken())
var resp *http.Response
resp, err := http.DefaultClient.Do(req)
if err != nil {
log.Println(err)
}
body, _ := ioutil.ReadAll(resp.Body)
log.Println(string(body))
}
func Upload(node *nodetree.NodeLoc) {
var tmpStrs = strings.Split(node.Path, nodetree.StrSrc)
if len(tmpStrs) < 2 {
return
}
// "/front/data/upload?name=data/folder3/ddd.txt"
url := "http://"
url += host
url += ":"
url += strconv.Itoa(port)
url += "/front/data/upload?name=data"
url += tmpStrs[1]
log.Println("httpcli.Upload() called, GET request:", url)
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("token", auth.GetToken())
var resp *http.Response
resp, err := http.DefaultClient.Do(req)
if err != nil {
log.Println(err)
}
body, _ := ioutil.ReadAll(resp.Body)
var jsonRes nodetree.UploadFile
json.Unmarshal(body, &jsonRes)
strCmd01 := jsonRes.Data.Url
strCmd02 := "x-amz-signature="
strCmd02 += jsonRes.Data.X_amz_signature
strCmd03 := "bucket="
strCmd03 += jsonRes.Data.Bucket
strCmd04 := "x-amz-meta-user="
strCmd04 += jsonRes.Data.X_amz_meta_user
strCmd05 := "policy="
strCmd05 += jsonRes.Data.Policy
strCmd06 := "x-amz-algorithm="
strCmd06 += jsonRes.Data.X_amz_algorithm
strCmd07 := "x-amz-credential="
strCmd07 += jsonRes.Data.X_amz_credential
strCmd08 := "x-amz-date="
strCmd08 += jsonRes.Data.X_amz_date
strCmd09 := "key="
strCmd09 += jsonRes.Data.Key
strCmd10 := "file=@"
strCmd10 += node.Path
cmd := exec.Command("curl", strCmd01, "-F", strCmd02, "-F", strCmd03, "-F", strCmd04, "-F", strCmd05, "-F", strCmd06, "-F", strCmd07, "-F", strCmd08, "-F", strCmd09, "-F", strCmd10)
stdout, err := cmd.StdoutPipe()
if err != nil {
log.Fatal(err)
}
defer stdout.Close()
if err := cmd.Start(); err != nil {
log.Fatal(err)
}
if opBytes, err := ioutil.ReadAll(stdout); err != nil {
log.Fatal(err)
} else {
fmt.Printf("File uploaded successful : [%v]\n", node.Path)
log.Println(string(opBytes))
}
}
src/main.go
...@@ -3,7 +3,7 @@ package main ...@@ -3,7 +3,7 @@ package main
import ( import (
"bytes" "bytes"
"encoding/json" "encoding/json"
_ "fmt" "fmt"
_ "io/ioutil" _ "io/ioutil"
"log" "log"
_ "minsync/auth" _ "minsync/auth"
...@@ -11,31 +11,86 @@ import ( ...@@ -11,31 +11,86 @@ import (
_ "minsync/encrypt" _ "minsync/encrypt"
_ "minsync/httpcli" _ "minsync/httpcli"
"minsync/nodetree" "minsync/nodetree"
_ "minsync/travloc" "minsync/travloc"
"minsync/travremot" "minsync/travremot"
_ "minsync/upload" _ "minsync/upload"
_ "net/http" _ "net/http"
"os" "os"
"runtime" _ "runtime"
) )
func printUsage() {
fmt.Println("Usage: minsync --token ef0c26da821c [upload|download] src dst")
}
func main() { func main() {
var nodeloc nodetree.NodeLoc
nodeloc.Name = "look"
log.Printf("main() called, os %v, %v", runtime.GOOS, nodeloc.Name)
var option_remot nodetree.OptionRemot // minsync --token ef0c26da821c upload /home/syz/tmp/abc /part1/box2
option_remot.RootPath = []string{"data/"} if len(os.Args) < 6 {
resp, _ := travremot.Explorer(option_remot) printUsage()
return
}
if !(os.Args[1] == "--token" || os.Args[4] == "--token") {
printUsage()
return
}
if os.Args[1] == "--token" {
nodetree.StrToken = os.Args[2]
nodetree.StrOp = os.Args[3]
nodetree.StrSrc = os.Args[4]
nodetree.StrDst = os.Args[5]
} else if os.Args[4] == "" {
nodetree.StrToken = os.Args[5]
nodetree.StrOp = os.Args[1]
nodetree.StrSrc = os.Args[2]
nodetree.StrDst = os.Args[3]
}
fmt.Printf("%v %v %v %v\n", nodetree.StrToken, nodetree.StrOp, nodetree.StrSrc, nodetree.StrDst)
b, err := json.Marshal(resp) file, _ := os.Open("./config.json")
defer file.Close()
decoder := json.NewDecoder(file)
conf := nodetree.Configuration{}
err := decoder.Decode(&conf)
if err != nil { if err != nil {
log.Println(err) log.Println("Error:", err)
} }
var out bytes.Buffer if nodetree.StrOp == "upload" {
err = json.Indent(&out, b, "", " ")
out.WriteTo(os.Stdout) var option_loc nodetree.OptionLoc
option_loc.RootPath = []string{nodetree.StrSrc}
option_loc.SubFlag = true
option_loc.IgnorePath = []string{".git"}
resp_loc, _ := travloc.Explorer(option_loc, "mkdir")
b_loc, err := json.Marshal(resp_loc)
if err != nil {
log.Println(err)
}
var out_loc bytes.Buffer
err = json.Indent(&out_loc, b_loc, "", " ")
// TODO, debug in config
//out_loc.WriteTo(os.Stdout)
} else if nodetree.StrOp == "download" {
var option_remot nodetree.OptionRemot
option_remot.RootPath = []string{"data/"}
resp_remot, _ := travremot.Explorer(option_remot)
b_remot, err := json.Marshal(resp_remot)
if err != nil {
log.Println(err)
}
var out_remot bytes.Buffer
err = json.Indent(&out_remot, b_remot, "", " ")
// TODO, debug in config
//out_remot.WriteTo(os.Stdout)
}
} }
src/nodetree/nodetree.go
package nodetree package nodetree
import ( import (
_ "fmt" "fmt"
"log" "log"
"os"
) )
type Configuration struct {
Host string
Port int
}
type NodeLoc struct { type NodeLoc struct {
Name string `json:"name"` Name string `json:"name"`
Path string `json:"path"` Path string `json:"path"`
...@@ -48,12 +54,43 @@ type OptionRemot struct { ...@@ -48,12 +54,43 @@ type OptionRemot struct {
Token string `yaml:"token"` Token string `yaml:"token"`
} }
type UploadFileData struct {
X_amz_credential string `json:"x-amz-credential"`
X_amz_date string `json:"x-amz-date"`
X_amz_signature string `json:"x-amz-signature"`
X_amz_algorithm string `json:"x-amz-algorithm"`
X_amz_meta_user string `json:"x-amz-meta-user"`
Bucket string `json:"bucket"`
Policy string `json:"policy"`
Key string `json:"key"`
Url string `json:"Url"`
}
type UploadFile struct {
Data UploadFileData `json:"data"`
Msg string `json:"msg"`
}
type Task struct { type Task struct {
Id string `json:"id"` Id string `json:"id"`
Progress int `json:"progress"` Progress int `json:"progress"`
State int `json:"state"` State int `json:"state"`
} }
var StrOp string
var StrSrc string
var StrDst string
var StrToken string
func init() { func init() {
logFile, err := os.OpenFile("./minsync.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
if err != nil {
fmt.Println("open log file failed, err:", err)
return
}
log.SetOutput(logFile)
log.SetFlags(log.Lshortfile | log.Ldate | log.Ltime)
log.Println("minsync.nodetree init() called") log.Println("minsync.nodetree init() called")
} }
src/travloc/travloc.go
...@@ -4,6 +4,7 @@ import ( ...@@ -4,6 +4,7 @@ import (
_ "fmt" _ "fmt"
"io/ioutil" "io/ioutil"
"log" "log"
"minsync/httpcli"
"minsync/nodetree" "minsync/nodetree"
"os" "os"
"path" "path"
...@@ -14,7 +15,7 @@ func init() { ...@@ -14,7 +15,7 @@ func init() {
log.Println("minsync.travloc.init() called") log.Println("minsync.travloc.init() called")
} }
func Explorer(option nodetree.OptionLoc) (nodetree.NodeLoc, error) { func Explorer(option nodetree.OptionLoc, op string) (nodetree.NodeLoc, error) {
var root nodetree.NodeLoc var root nodetree.NodeLoc
for _, p := range option.RootPath { for _, p := range option.RootPath {
...@@ -26,7 +27,7 @@ func Explorer(option nodetree.OptionLoc) (nodetree.NodeLoc, error) { ...@@ -26,7 +27,7 @@ func Explorer(option nodetree.OptionLoc) (nodetree.NodeLoc, error) {
child.Path = p child.Path = p
exploreRecursive(&child, &option) exploreRecursive(&child, &option, op)
root.Children = append(root.Children, &child) root.Children = append(root.Children, &child)
} }
...@@ -34,7 +35,7 @@ func Explorer(option nodetree.OptionLoc) (nodetree.NodeLoc, error) { ...@@ -34,7 +35,7 @@ func Explorer(option nodetree.OptionLoc) (nodetree.NodeLoc, error) {
return root, nil return root, nil
} }
func exploreRecursive(node *nodetree.NodeLoc, option *nodetree.OptionLoc) { func exploreRecursive(node *nodetree.NodeLoc, option *nodetree.OptionLoc, op string) {
p, err := os.Stat(node.Path) p, err := os.Stat(node.Path)
if err != nil { if err != nil {
log.Println(err) log.Println(err)
...@@ -62,10 +63,14 @@ func exploreRecursive(node *nodetree.NodeLoc, option *nodetree.OptionLoc) { ...@@ -62,10 +63,14 @@ func exploreRecursive(node *nodetree.NodeLoc, option *nodetree.OptionLoc) {
child.IsDir = f.IsDir() child.IsDir = f.IsDir()
if f.IsDir() { if f.IsDir() {
// mkdir remot
httpcli.Mkdir(child.Path)
if option.SubFlag { if option.SubFlag {
if !IsInSlice(option.IgnorePath, f.Name()) { if !IsInSlice(option.IgnorePath, f.Name()) {
node.Children = append(node.Children, &child) node.Children = append(node.Children, &child)
exploreRecursive(&child, option) exploreRecursive(&child, option, op)
} }
} else { } else {
if !IsInSlice(option.IgnoreFile, f.Name()) { if !IsInSlice(option.IgnoreFile, f.Name()) {
...@@ -75,6 +80,10 @@ func exploreRecursive(node *nodetree.NodeLoc, option *nodetree.OptionLoc) { ...@@ -75,6 +80,10 @@ func exploreRecursive(node *nodetree.NodeLoc, option *nodetree.OptionLoc) {
} }
} else { } else {
if !IsInSlice(option.IgnoreFile, f.Name()) { if !IsInSlice(option.IgnoreFile, f.Name()) {
// upload file
httpcli.Upload(&child)
child.Name = f.Name() child.Name = f.Name()
node.Children = append(node.Children, &child) node.Children = append(node.Children, &child)
} }
......
src/travremot/travremot.go
...@@ -7,11 +7,27 @@ import ( ...@@ -7,11 +7,27 @@ import (
"minsync/auth" "minsync/auth"
"minsync/nodetree" "minsync/nodetree"
"net/http" "net/http"
"os"
"strconv"
"strings" "strings"
) )
var host string
var port int
func init() { func init() {
log.Println("minsync.travremot.init() called") log.Println("minsync.travremot.init() called")
file, _ := os.Open("./config.json")
defer file.Close()
decoder := json.NewDecoder(file)
conf := nodetree.Configuration{}
err := decoder.Decode(&conf)
if err != nil {
log.Println("Error:", err)
}
host = conf.Host
port = conf.Port
} }
func Explorer(option nodetree.OptionRemot) (nodetree.NodeRemot, error) { func Explorer(option nodetree.OptionRemot) (nodetree.NodeRemot, error) {
...@@ -39,8 +55,14 @@ func exploreRecursive(node *nodetree.NodeRemot, option *nodetree.OptionRemot) { ...@@ -39,8 +55,14 @@ func exploreRecursive(node *nodetree.NodeRemot, option *nodetree.OptionRemot) {
log.Printf("travremot.exploreRecursive() node.Name: %s", node.Name) log.Printf("travremot.exploreRecursive() node.Name: %s", node.Name)
url := "http://122.112.207.59:8989/front/data/query_list?pages=1&pagesize=20&prefix=" // "http://122.112.207.59:8989/front/data/query_list?pages=1&pagesize=20&prefix="
url := "http://"
url += host
url += ":"
url += strconv.Itoa(port)
url += "/front/data/query_list?pages=1&pagesize=20&prefix="
url += node.Name url += node.Name
log.Println(url)
req, _ := http.NewRequest("GET", url, nil) req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("token", auth.GetToken()) req.Header.Add("token", auth.GetToken())
...@@ -62,6 +84,9 @@ func exploreRecursive(node *nodetree.NodeRemot, option *nodetree.OptionRemot) { ...@@ -62,6 +84,9 @@ func exploreRecursive(node *nodetree.NodeRemot, option *nodetree.OptionRemot) {
var child nodetree.NodeRemot var child nodetree.NodeRemot
child.Name = tmp child.Name = tmp
child.Type = f.Type child.Type = f.Type
child.UpdtDate = f.UpdtDate
child.Creator = f.Creator
child.Size = f.Size
if f.Type == "folder" { if f.Type == "folder" {
child.Name += "/" child.Name += "/"
......
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!