From d9740653eb419f02f73d6e0220a0b0886beae0d6 Mon Sep 17 00:00:00 2001 From: Bigsk Date: Wed, 20 Nov 2024 22:12:35 +0800 Subject: [PATCH] first version: v0.0.1 Alpha --- .gitignore | 50 +++---- controller.go | 73 +++++++++++ controller_test.go | 71 ++++++++++ go.mod | 5 + go.sum | 2 + richkago.go | 316 +++++++++++++++++++++++++++++++++++++++++++++ richkago_test.go | 16 +++ 7 files changed, 510 insertions(+), 23 deletions(-) create mode 100644 controller.go create mode 100644 controller_test.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 richkago.go create mode 100644 richkago_test.go diff --git a/.gitignore b/.gitignore index 73d710f..93195bc 100644 --- a/.gitignore +++ b/.gitignore @@ -46,7 +46,8 @@ $RECYCLE.BIN/ .LSOverride # Icon must end with two \r -Icon +Icon + # Thumbnails ._* @@ -485,31 +486,34 @@ FodyWeavers.xsd # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 +# Whole .idea +.idea + # User-specific stuff -.idea/**/workspace.xml -.idea/**/tasks.xml -.idea/**/usage.statistics.xml -.idea/**/dictionaries -.idea/**/shelf +#.idea/**/workspace.xml +#.idea/**/tasks.xml +#.idea/**/usage.statistics.xml +#.idea/**/dictionaries +#.idea/**/shelf # AWS User-specific -.idea/**/aws.xml +#.idea/**/aws.xml # Generated files -.idea/**/contentModel.xml +#.idea/**/contentModel.xml # Sensitive or high-churn files -.idea/**/dataSources/ -.idea/**/dataSources.ids -.idea/**/dataSources.local.xml -.idea/**/sqlDataSources.xml -.idea/**/dynamic.xml -.idea/**/uiDesigner.xml -.idea/**/dbnavigator.xml +#.idea/**/dataSources/ +#.idea/**/dataSources.ids +#.idea/**/dataSources.local.xml +#.idea/**/sqlDataSources.xml +#.idea/**/dynamic.xml +#.idea/**/uiDesigner.xml +#.idea/**/dbnavigator.xml # Gradle -.idea/**/gradle.xml -.idea/**/libraries +#.idea/**/gradle.xml +#.idea/**/libraries # Gradle and Maven with auto-import # When using Gradle or Maven with auto-import, you should exclude module files, @@ -528,7 +532,7 @@ FodyWeavers.xsd cmake-build-*/ # Mongo Explorer plugin -.idea/**/mongoSettings.xml +#.idea/**/mongoSettings.xml # File-based project format *.iws @@ -537,16 +541,16 @@ cmake-build-*/ out/ # mpeltonen/sbt-idea plugin -.idea_modules/ +#.idea_modules/ # JIRA plugin atlassian-ide-plugin.xml # Cursive Clojure plugin -.idea/replstate.xml +#.idea/replstate.xml # SonarLint plugin -.idea/sonarlint/ +#.idea/sonarlint/ # Crashlytics plugin (for Android Studio and IntelliJ) com_crashlytics_export_strings.xml @@ -555,8 +559,8 @@ crashlytics-build.properties fabric.properties # Editor-based Rest Client -.idea/httpRequests +#.idea/httpRequests # Android studio 3.1+ serialized cache file -.idea/caches/build_file_checksums.ser +#.idea/caches/build_file_checksums.ser diff --git a/controller.go b/controller.go new file mode 100644 index 0000000..6f6c87f --- /dev/null +++ b/controller.go @@ -0,0 +1,73 @@ +package richkago + +import "sync" + +type Controller struct { + paused bool + excepted bool + totalSize int64 + downloadedSize int64 + downloadedSlices map[string]int64 + mu sync.Mutex +} + +// NewController build a new controller +func NewController() *Controller { + return &Controller{ + downloadedSlices: make(map[string]int64), + } +} + +// UpdateProgress update progress into controller +func (c *Controller) UpdateProgress(size int64, chunkID string) { + // Get lock + c.mu.Lock() + defer c.mu.Unlock() + + if chunkID == "" && len(c.downloadedSlices) == 0 { + // Init variable + c.downloadedSize = size + } else { + // Update progress + c.downloadedSlices[chunkID] = size + c.downloadedSize = 0 + // Sum up + for _, v := range c.downloadedSlices { + c.downloadedSize += v + } + } +} + +// Pause pause a progress +func (c *Controller) Pause() { + c.paused = true +} + +// Unpause unpause a progress +func (c *Controller) Unpause() { + c.paused = false +} + +// Status gets a status of a controller +func (c *Controller) Status() int { + if c.downloadedSize == 0 && !c.excepted { + return -1 // Not started + } else if c.paused { + return -2 // Paused + } else if c.excepted { + return -3 // Excepted + } else if c.downloadedSize == c.totalSize { + return 0 // Done + } else { + return 1 // Downloading + } +} + +// Progress gets progress of a controller +func (c *Controller) Progress() float64 { + if c.totalSize == 0 { + return -1 + } + + return float64(c.downloadedSize) / float64(c.totalSize) * 100 +} diff --git a/controller_test.go b/controller_test.go new file mode 100644 index 0000000..8bcae62 --- /dev/null +++ b/controller_test.go @@ -0,0 +1,71 @@ +package richkago + +import "testing" + +// TestNewController test example that builds a new controller +func TestNewController(t *testing.T) { + controller := NewController() + if controller == nil { + t.Error("controller is nil") + return + } +} + +// TestController_UpdateProgress test example that updates progress +func TestController_UpdateProgress(t *testing.T) { + controller := NewController() + controller.totalSize = 1000 + + controller.UpdateProgress(100, "1") + if controller.Progress() != 10 { + t.Error("progress is wrong", controller.Progress()) + return + } + + if controller.Status() != 1 { + t.Error("status is wrong", controller.Status()) + return + } +} + +// TestController_Pause test example that pause a progress +func TestController_Pause(t *testing.T) { + controller := NewController() + controller.totalSize = 1000 + + controller.UpdateProgress(100, "1") + if controller.Progress() != 10 { + t.Error("progress is wrong", controller.Progress()) + return + } + + controller.Pause() + if controller.Status() != -2 { + t.Error("status is wrong", controller.Status()) + return + } +} + +// TestController_Unpause test example that unpause a progress +func TestController_Unpause(t *testing.T) { + controller := NewController() + controller.totalSize = 1000 + + controller.UpdateProgress(100, "1") + if controller.Progress() != 10 { + t.Error("progress is wrong", controller.Progress()) + return + } + + controller.Pause() + if controller.Status() != -2 { + t.Error("status is wrong", controller.Status()) + return + } + + controller.Unpause() + if controller.Status() != 1 { + t.Error("status is wrong", controller.Status()) + return + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..47744e4 --- /dev/null +++ b/go.mod @@ -0,0 +1,5 @@ +module github.com/ghinknet/richkago + +go 1.23.2 + +require golang.org/x/sync v0.9.0 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..fc7734d --- /dev/null +++ b/go.sum @@ -0,0 +1,2 @@ +golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ= +golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= diff --git a/richkago.go b/richkago.go new file mode 100644 index 0000000..1c92b94 --- /dev/null +++ b/richkago.go @@ -0,0 +1,316 @@ +package richkago + +import ( + "errors" + "fmt" + "golang.org/x/sync/errgroup" + "io" + "net/http" + "os" + "strconv" + "time" +) + +const ( + version = "Alpha/0.0.1" + userAgent = "Richkago" + version + coroutineLimit = 10 + sliceThreshold = 10 * 1024 * 1024 // 10 MiB + timeout = 3 * time.Second + retryTimes = 5 + chunkSize = 102400 +) + +// readWithTimeout read data from source with timeout limit +func readWithTimeout(reader io.Reader, buffer []byte, controller *Controller, file *os.File, downloaded *int64, chunkID string) error { + // Make a error chan + ch := make(chan error, 1) + + go func() { + // Read data + n, err := reader.Read(buffer) + if n > 0 { + // Write to file + _, err = file.Write(buffer[:n]) + if err != nil { + ch <- err + return + } + // Calc amount of downloaded data + *downloaded += int64(n) + } + ch <- err + }() + + // Error and timeout handler + select { + case err := <-ch: + // Update amount of downloaded data + if controller != nil { + controller.UpdateProgress(*downloaded, chunkID) + } + + return err + case <-time.After(timeout): + return errors.New("timeout while reading data") + } +} + +// downloadRange download a file with many slices +func downloadRange(client *http.Client, url string, start, end int64, destination string, controller *Controller) error { + // Build request header + headers := map[string]string{ + "User-Agent": userAgent, + "Range": fmt.Sprintf("bytes=%d-%d", start, end), + } + + retries := retryTimes + var err error + + for retries > 0 { + // Get header info + var req *http.Request + req, err = http.NewRequest("GET", url, nil) + if err != nil { + retries-- + time.Sleep(1 * time.Second) + continue + } + for k, v := range headers { + req.Header.Add(k, v) + } + + // Read http header + var resp *http.Response + resp, err = client.Do(req) + if err != nil || resp.StatusCode != http.StatusPartialContent { + retries-- + time.Sleep(1 * time.Second) + continue + } + + defer func(Body io.ReadCloser) { + err = Body.Close() + if err != nil { + panic(err) + } + }(resp.Body) + + // Pre-create file + file, _ := os.OpenFile(destination, os.O_WRONLY, 0644) + defer func(file *os.File) { + err = file.Close() + if err != nil { + panic(err) + } + }(file) + _, err = file.Seek(start, io.SeekStart) + if err != nil { + retries-- + time.Sleep(1 * time.Second) + continue + } + + // Create goroutines to download + buffer := make([]byte, chunkSize) + var downloaded int64 + for { + // Controller pin + if controller.paused { + time.Sleep(1 * time.Second) + continue + } + + // Start read stream + err = readWithTimeout(resp.Body, buffer, controller, file, &downloaded, fmt.Sprintf("%d-%d", start, end)) + if err == io.EOF { + break + } else if err != nil { + break + } + } + + // Error handler + if err != nil && err != io.EOF { + retries-- + time.Sleep(1 * time.Second) + continue + } + + return nil + } + + return err +} + +// downloadSingle download a file directly +func downloadSingle(client *http.Client, url, destination string, controller *Controller) error { + // Build request header + headers := map[string]string{ + "User-Agent": userAgent, + } + retries := retryTimes + + for retries > 0 { + // Get header info + req, _ := http.NewRequest("GET", url, nil) + for k, v := range headers { + req.Header.Add(k, v) + } + + // Read http header + resp, err := client.Do(req) + if err != nil || resp.StatusCode != http.StatusOK { + retries-- + time.Sleep(1 * time.Second) + continue + } + + defer func(Body io.ReadCloser) { + err = Body.Close() + if err != nil { + panic(err) + } + }(resp.Body) + + // Pre-create file + file, _ := os.OpenFile(destination, os.O_WRONLY, 0644) + defer func(file *os.File) { + err = file.Close() + if err != nil { + panic(err) + } + }(file) + + // Start downloading + buffer := make([]byte, chunkSize) + var downloaded int64 + for { + // Controller pin + if controller.paused { + time.Sleep(1 * time.Second) + continue + } + + // Start read stream + var n int + n, err = resp.Body.Read(buffer) + if err != nil { + break + } + if n > 0 { + _, err = file.Write(buffer[:n]) + if err != nil { + return err + } + downloaded += int64(n) + controller.UpdateProgress(downloaded, "") + } + } + + // Error handler + if err != nil { + retries-- + time.Sleep(1 * time.Second) + continue + } else { + return err + } + } + + return nil +} + +// Download main download task creator +func Download(url, destination string, controller *Controller) (float64, int64, error) { + // Create http client + client := &http.Client{} + + // Get header info + req, err := http.NewRequest("HEAD", url, nil) + if err != nil { + controller.excepted = true + return 0, 0, err + } + req.Header.Add("User-Agent", userAgent) + + // Read http header + resp, err := client.Do(req) + if err != nil { + controller.excepted = true + return 0, 0, err + } + defer func(Body io.ReadCloser) { + err = Body.Close() + if err != nil { + panic(err) + } + }(resp.Body) + + sizeStr := resp.Header.Get("Content-Length") + fileSize, _ := strconv.ParseInt(sizeStr, 10, 64) + controller.totalSize = fileSize + + // Calc slices size + if fileSize <= sliceThreshold { + file, _ := os.Create(destination) + err = file.Truncate(fileSize) + if err != nil { + controller.excepted = true + return 0, 0, err + } + err = file.Close() + if err != nil { + controller.excepted = true + return 0, 0, err + } + + // Too small + startTime := time.Now() + err = downloadSingle(client, url, destination, controller) + if err != nil { + controller.excepted = true + return 0, 0, err + } + endTime := time.Now() + return endTime.Sub(startTime).Seconds(), fileSize, nil + } + + // Pre-create file + partSize := fileSize / coroutineLimit + file, _ := os.Create(destination) + err = file.Truncate(fileSize) + if err != nil { + controller.excepted = true + return 0, 0, err + } + err = file.Close() + if err != nil { + controller.excepted = true + return 0, 0, err + } + + // Start download goroutines + group := new(errgroup.Group) + for i := 0; i < coroutineLimit; i++ { + start := int64(i) * partSize + end := start + partSize - 1 + if i == coroutineLimit-1 { + end = fileSize - 1 + } + group.Go(func() error { + err = downloadRange(client, url, start, end, destination, controller) + return err + }) + } + + // Start all tasks + startTime := time.Now() + if err = group.Wait(); err != nil { + controller.excepted = true + return 0, 0, err + } + endTime := time.Now() + + return endTime.Sub(startTime).Seconds(), fileSize, nil +} diff --git a/richkago_test.go b/richkago_test.go new file mode 100644 index 0000000..dcc5c28 --- /dev/null +++ b/richkago_test.go @@ -0,0 +1,16 @@ +package richkago + +import ( + "testing" +) + +// TestDownload test main download +func TestDownload(t *testing.T) { + controller := NewController() + + _, _, err := Download("https://mirrors.tuna.tsinghua.edu.cn/github-release/git-for-windows/git/LatestRelease/Git-2.47.0.2-64-bit.exe", "Git-2.47.0.2-64-bit.exe", controller) + if err != nil { + t.Error("failed to download", err) + return + } +}