From af59659f41419d1aacdd3ff51c21c706bf55afa5 Mon Sep 17 00:00:00 2001 From: Pijus Kamandulis Date: Thu, 9 Apr 2020 18:10:33 +0300 Subject: [PATCH] TTDL-17 Added option to output failed items to file --- README.md | 26 ++++++++++++++++++-------- models/config/config.go | 3 +++ resources/strings.go | 7 +++++-- workflows/common.go | 20 ++++++++++++++++++++ workflows/downloadHashtag.go | 5 ++--- workflows/downloadMusic.go | 5 ++--- workflows/downloadShareLink.go | 8 ++++---- workflows/downloadUser.go | 5 ++--- workflows/downloadVideo.go | 3 +-- 9 files changed, 57 insertions(+), 25 deletions(-) create mode 100644 workflows/common.go diff --git a/README.md b/README.md index f76515f..a4c7bc3 100644 --- a/README.md +++ b/README.md @@ -5,27 +5,37 @@ A simple tiktok video downloader written in go -## Basic usage +## Basic usage examples Download the executable from `https://github.com/pikami/tiktok-dl/releases`\ -You can download all videos from user by running `./tiktok-dl [Options] TIKTOK_USERNAME`\ -You can download single video by running `./tiktok-dl [Options] VIDEO_URL`\ -You can download all videos by music by running `./tiktok-dl [Options] MUSIC_URL`\ -You can download items listed in a text file by running `./tiktok-dl [OPTIONS] -batch-file path/to/items.txt` +You can download all videos from user by running `./tiktok-dl TIKTOK_USERNAME`\ +You can download single video by running `./tiktok-dl VIDEO_URL`\ +You can download items listed in a text file by running `./tiktok-dl -batch-file path/to/items.txt` -## Build instructions -Clone this repository and run `go build` to build the executable. +## Usage Manual +``` +Usage: tiktok-dl [OPTION]... TARGET + or: tiktok-dl [OPTION]... -batch-file BATCH_FILE + +In the 1st form, download given `TARGET`. +In the 2nd form, download all targets listed in given `BATCH_FILE`. +``` ## Available options * `-archive` - Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it. -* `-batch-file` - File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored. +* `-batch-file some_file` - File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored. * `-deadline` - Sets the timout for scraper logic in seconds (used as a workaround for context deadline exceeded error) (default 1500) * `-debug` - enables debug mode +* `-fail-log some_file` - Write failed items to log file * `-json` - Returns whole data, that was scraped from TikTok, in json * `-limit` - Sets the max count of video that will be downloaded (default infinity) * `-metadata` - Write video metadata to a .json file * `-output some_directory` - Output path (default "./downloads") * `-quiet` - Supress output +## Build instructions +1. Clone this repository +2. Run `go build` to build the executable. + ## Acknowledgments This software uses the **chromedp** for web scraping, it can be found here: https://github.com/chromedp/chromedp \ For releases the JS code is minified by using **terser** toolkit, it can be found here: https://github.com/terser/terser diff --git a/models/config/config.go b/models/config/config.go index 38fa0fd..66ce415 100644 --- a/models/config/config.go +++ b/models/config/config.go @@ -12,6 +12,7 @@ var Config struct { OutputPath string BatchFilePath string ArchiveFilePath string + FailLogFilePath string Debug bool MetaData bool Quiet bool @@ -25,6 +26,7 @@ func GetConfig() { outputPath := flag.String("output", "./downloads", "Output path") batchFilePath := flag.String("batch-file", "", "File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored.") archive := flag.String("archive", "", "Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.") + failLogPath := flag.String("fail-log", "", "Write failed items to log file") debug := flag.Bool("debug", false, "Enables debug mode") metadata := flag.Bool("metadata", false, "Write video metadata to a .json file") quiet := flag.Bool("quiet", false, "Supress output") @@ -48,6 +50,7 @@ func GetConfig() { Config.OutputPath = *outputPath Config.BatchFilePath = *batchFilePath Config.ArchiveFilePath = *archive + Config.FailLogFilePath = *failLogPath Config.Debug = *debug Config.MetaData = *metadata Config.Quiet = *quiet diff --git a/resources/strings.go b/resources/strings.go index 15b87ca..b0baff1 100644 --- a/resources/strings.go +++ b/resources/strings.go @@ -6,8 +6,11 @@ var ErrorCouldNotSerializeJSON = "Could not serialize json for video: %s\n" // ErrorCouldNotRecogniseURL - var ErrorCouldNotRecogniseURL = "Could not recognise URL format of string %s" -// ErrorCouldNotGetUserUploads - -var ErrorCouldNotGetUserUploads = "Failed to get user uploads: %s\n" +// Error - +var Error = "Error : %s\n" // ErrorPathNotFound - var ErrorPathNotFound = "File path %s not found." + +// FailedOnItem - +var FailedOnItem = "Failed while scraping item: %s\n" diff --git a/workflows/common.go b/workflows/common.go new file mode 100644 index 0000000..bd4d77d --- /dev/null +++ b/workflows/common.go @@ -0,0 +1,20 @@ +package workflows + +import ( + config "../models/config" + res "../resources" + fileio "../utils/fileio" + log "../utils/log" +) + +// OnWorkflowFail - Funtion called when workflow fails +func OnWorkflowFail(err error, workItem string) { + failLogFilePath := config.Config.FailLogFilePath + + if failLogFilePath != "" { + fileio.AppendToFile(workItem, failLogFilePath) + } + + log.LogErr(res.Error, err.Error()) + log.LogErr(res.FailedOnItem, workItem) +} diff --git a/workflows/downloadHashtag.go b/workflows/downloadHashtag.go index 26e4a38..c3c9ce4 100644 --- a/workflows/downloadHashtag.go +++ b/workflows/downloadHashtag.go @@ -6,7 +6,6 @@ import ( client "../client" config "../models/config" - res "../resources" utils "../utils" fileio "../utils/fileio" log "../utils/log" @@ -22,7 +21,7 @@ func CanUseDownloadHashtag(url string) bool { func DownloadHashtag(url string) { uploads, err := client.GetHashtagUploads(url) if err != nil { - log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error()) + OnWorkflowFail(err, url) return } @@ -45,7 +44,7 @@ func DownloadHashtag(url string) { func GetHashtagJSON(url string) { uploads, err := client.GetHashtagUploadsJSON(url) if err != nil { - log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error()) + OnWorkflowFail(err, url) return } fmt.Printf("%s", uploads) diff --git a/workflows/downloadMusic.go b/workflows/downloadMusic.go index c8cfb23..d69c281 100644 --- a/workflows/downloadMusic.go +++ b/workflows/downloadMusic.go @@ -6,7 +6,6 @@ import ( client "../client" config "../models/config" - res "../resources" utils "../utils" fileio "../utils/fileio" log "../utils/log" @@ -22,7 +21,7 @@ func CanUseDownloadMusic(url string) bool { func DownloadMusic(url string) { uploads, err := client.GetMusicUploads(url) if err != nil { - log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error()) + OnWorkflowFail(err, url) return } @@ -44,7 +43,7 @@ func DownloadMusic(url string) { func GetMusicJSON(url string) { uploads, err := client.GetMusicUploadsJSON(url) if err != nil { - log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error()) + OnWorkflowFail(err, url) return } fmt.Printf("%s", uploads) diff --git a/workflows/downloadShareLink.go b/workflows/downloadShareLink.go index 6c0f7f7..099035d 100644 --- a/workflows/downloadShareLink.go +++ b/workflows/downloadShareLink.go @@ -1,10 +1,10 @@ package workflows import ( - client "../client" - res "../resources" - log "../utils/log" "regexp" + + client "../client" + log "../utils/log" ) // CanUseDownloadShareLink - Check's if DownloadShareLink can be used @@ -19,7 +19,7 @@ func DownloadShareLink(url string) { finalURL, err := client.GetRedirectUrl(url) if err != nil { - log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error()) + OnWorkflowFail(err, url) return } diff --git a/workflows/downloadUser.go b/workflows/downloadUser.go index 2ad0afc..50d2aff 100644 --- a/workflows/downloadUser.go +++ b/workflows/downloadUser.go @@ -7,7 +7,6 @@ import ( client "../client" config "../models/config" - res "../resources" utils "../utils" fileio "../utils/fileio" log "../utils/log" @@ -24,7 +23,7 @@ func CanUseDownloadUser(url string) bool { func DownloadUser(username string) { uploads, err := client.GetUserUploads(username) if err != nil { - log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error()) + OnWorkflowFail(err, username) return } @@ -46,7 +45,7 @@ func DownloadUser(username string) { func GetUserVideosJSON(username string) { uploads, err := client.GetUserUploadsJSON(username) if err != nil { - log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error()) + OnWorkflowFail(err, username) return } fmt.Printf("%s", uploads) diff --git a/workflows/downloadVideo.go b/workflows/downloadVideo.go index d433182..5d9ab25 100644 --- a/workflows/downloadVideo.go +++ b/workflows/downloadVideo.go @@ -7,7 +7,6 @@ import ( client "../client" models "../models" config "../models/config" - res "../resources" utils "../utils" fileio "../utils/fileio" log "../utils/log" @@ -24,7 +23,7 @@ func DownloadSingleVideo(url string) { username := utils.GetUsernameFromString(url) upload, err := client.GetVideoDetails(url) if err != nil { - log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error()) + OnWorkflowFail(err, url) return }