From 6f8ab8a2779101244c0ee0f4c0957b5e0ac17651 Mon Sep 17 00:00:00 2001 From: Pijus Kamandulis Date: Fri, 24 Jan 2020 19:02:50 +0200 Subject: [PATCH] Added option to download items from list --- .gitignore | 1 + README.md | 7 ++++-- main.go | 7 ++++++ models/config.go | 33 ++++++++++++++++++--------- utils/fileio.go | 21 +++++++++++++++++ workflows/downloadBatchFile.go | 41 ++++++++++++++++++++++++++++++++++ workflows/downloadVideo.go | 2 +- 7 files changed, 99 insertions(+), 13 deletions(-) create mode 100644 workflows/downloadBatchFile.go diff --git a/.gitignore b/.gitignore index 97d364b..178d90c 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ __debug_bin downloads *.exe tiktok-dl +batch_file.txt diff --git a/README.md b/README.md index 65fcb40..75f59f1 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,8 @@ A simple tiktok video downloader written in go ## Basic usage Download the executable from `https://github.com/pikami/tiktok-dl/releases`\ You can download all videos from user by running `./tiktok-dl [Options] TIKTOK_USERNAME`\ -You can download single video by running `./tiktok-dl [Options] VIDEO_URL` +You can download single video by running `./tiktok-dl [Options] VIDEO_URL`\ +You can download items listed in a text file by running `./tiktok-dl [OPTIONS] -batch-file path/to/items.txt` ## Build instructions Clone this repository and run `go build` to build the executable. @@ -17,6 +18,8 @@ Clone this repository and run `go build` to build the executable. * `-debug` - enables debug mode * `-output some_directory` - Output path (default "./downloads") * `-metadata` - Write video metadata to a .json file +* `-batch-file` - File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored. ## Acknowledgments -This software uses the chromedp for web scraping, it can be found here: https://github.com/chromedp/chromedp +This software uses the **chromedp** for web scraping, it can be found here: https://github.com/chromedp/chromedp\ +For releases the JS code is minified by using **terser** toolkit, it can be found here: https://github.com/terser/terser diff --git a/main.go b/main.go index 4018398..262f46f 100644 --- a/main.go +++ b/main.go @@ -8,6 +8,13 @@ import ( func main() { models.GetConfig() url := models.Config.URL + batchFilePath := models.Config.BatchFilePath + + // Batch file + if workflows.CanUseDownloadBatchFile(batchFilePath) { + workflows.DownloadBatchFile(batchFilePath) + return + } // Single video if workflows.CanUseDownloadSingleVideo(url) { diff --git a/models/config.go b/models/config.go index 450542a..0b68cb1 100644 --- a/models/config.go +++ b/models/config.go @@ -10,39 +10,52 @@ import ( // Config - Runtime configuration var Config struct { - URL string - OutputPath string - Debug bool - MetaData bool + URL string + OutputPath string + BatchFilePath string + Debug bool + MetaData bool } // GetConfig - Returns Config object func GetConfig() { outputPath := flag.String("output", "./downloads", "Output path") + batchFilePath := flag.String("batch-file", "", "File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored.") debug := flag.Bool("debug", false, "Enables debug mode") metadata := flag.Bool("metadata", false, "Write video metadata to a .json file") flag.Parse() args := flag.Args() - if len(args) < 1 { + if len(args) < 1 && *batchFilePath == "" { fmt.Println("Usage: tiktok-dl [OPTIONS] TIKTOK_USERNAME|TIKTOK_URL") + fmt.Println(" or: tiktok-dl [OPTIONS] -batch-file path/to/users.txt") os.Exit(2) } - Config.URL = flag.Args()[len(args)-1] + if len(args) > 0 { + Config.URL = flag.Args()[len(args)-1] + } else { + Config.URL = "" + } Config.OutputPath = *outputPath + Config.BatchFilePath = *batchFilePath Config.Debug = *debug Config.MetaData = *metadata } // GetUsername - Get's username from passed URL param func GetUsername() string { - if match := strings.Contains(Config.URL, "/"); !match { // Not url - return strings.Replace(Config.URL, "@", "", -1) + return GetUsernameFromString(Config.URL) +} + +// GetUsernameFromString - Get's username from passed param +func GetUsernameFromString(str string) string { + if match := strings.Contains(str, "/"); !match { // Not url + return strings.Replace(str, "@", "", -1) } - if match, _ := regexp.MatchString(".+tiktok\\.com/@.+", Config.URL); match { // URL - stripedSuffix := strings.Split(Config.URL, "@")[1] + if match, _ := regexp.MatchString(".+tiktok\\.com/@.+", str); match { // URL + stripedSuffix := strings.Split(str, "@")[1] return strings.Split(stripedSuffix, "/")[0] } diff --git a/utils/fileio.go b/utils/fileio.go index 42ef8ea..0a59e98 100644 --- a/utils/fileio.go +++ b/utils/fileio.go @@ -1,10 +1,13 @@ package utils import ( + "bufio" "io/ioutil" "os" ) +type delegateString func(string) + // CheckIfExists - Checks if file or directory exists func CheckIfExists(path string) bool { if _, err := os.Stat(path); os.IsNotExist(err) { @@ -30,3 +33,21 @@ func ReadFileToString(path string) string { return string(content) } + +// ReadFileLineByLine - Reads file line by line and calls delegate +func ReadFileLineByLine(path string, delegate delegateString) { + file, err := os.Open(path) + if err != nil { + panic(err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + delegate(scanner.Text()) + } + + if err := scanner.Err(); err != nil { + panic(err) + } +} diff --git a/workflows/downloadBatchFile.go b/workflows/downloadBatchFile.go new file mode 100644 index 0000000..d4f3c18 --- /dev/null +++ b/workflows/downloadBatchFile.go @@ -0,0 +1,41 @@ +package workflows + +import ( + models "../models" + utils "../utils" + "fmt" +) + +// CanUseDownloadBatchFile - Check's if DownloadBatchFile can be used +func CanUseDownloadBatchFile(batchFilePath string) bool { + return batchFilePath != "" +} + +// DownloadBatchFile - Download items from batch file +func DownloadBatchFile(batchFilePath string) { + if !utils.CheckIfExists(batchFilePath) { + panic(fmt.Sprintf("File path %s not found.", batchFilePath)) + } + + utils.ReadFileLineByLine(batchFilePath, downloadItem) +} + +func downloadItem(batchItem string) { + if batchItem[0] == '#' { + return + } + + // Single video + if CanUseDownloadSingleVideo(batchItem) { + DownloadSingleVideo(batchItem) + return + } + + // Tiktok user + if CanUseDownloadUser(batchItem) { + DownloadUser(models.GetUsernameFromString(batchItem)) + return + } + + panic(fmt.Sprintf("Could not recognise URL format of string %s", batchItem)) +} diff --git a/workflows/downloadVideo.go b/workflows/downloadVideo.go index 5ad1551..27da001 100644 --- a/workflows/downloadVideo.go +++ b/workflows/downloadVideo.go @@ -16,7 +16,7 @@ func CanUseDownloadSingleVideo(url string) bool { // DownloadSingleVideo - Downloads single video func DownloadSingleVideo(url string) { - username := models.GetUsername() + username := models.GetUsernameFromString(url) upload := client.GetVideoDetails(url) downloadDir := fmt.Sprintf("%s/%s", models.Config.OutputPath, username)