TTDL-17 Added option to output failed items to file

README.md

@@ -5,27 +5,37 @@
 
 A simple tiktok video downloader written in go
 
-## Basic usage
+## Basic usage examples
 Download the executable from `https://github.com/pikami/tiktok-dl/releases`\
-You can download all videos from user by running `./tiktok-dl [Options] TIKTOK_USERNAME`\
+You can download all videos from user by running `./tiktok-dl TIKTOK_USERNAME`\
-You can download single video by running `./tiktok-dl [Options] VIDEO_URL`\
+You can download single video by running `./tiktok-dl VIDEO_URL`\
-You can download all videos by music by running `./tiktok-dl [Options] MUSIC_URL`\
+You can download items listed in a text file by running `./tiktok-dl -batch-file path/to/items.txt`
-You can download items listed in a text file by running `./tiktok-dl [OPTIONS] -batch-file path/to/items.txt`
 
-## Build instructions
+## Usage Manual
-Clone this repository and run `go build` to build the executable.
+```
+Usage: tiktok-dl [OPTION]... TARGET
+  or:  tiktok-dl [OPTION]... -batch-file BATCH_FILE
+
+In the 1st form, download given `TARGET`.
+In the 2nd form, download all targets listed in given `BATCH_FILE`.
+```
 
 ## Available options
 * `-archive` - Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.
-* `-batch-file` - File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored.
+* `-batch-file some_file` - File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored.
 * `-deadline` - Sets the timout for scraper logic in seconds (used as a workaround for context deadline exceeded error) (default 1500)
 * `-debug` - enables debug mode
+* `-fail-log some_file` - Write failed items to log file
 * `-json` - Returns whole data, that was scraped from TikTok, in json
 * `-limit` - Sets the max count of video that will be downloaded (default infinity)
 * `-metadata` - Write video metadata to a .json file
 * `-output some_directory` - Output path (default "./downloads")
 * `-quiet` - Supress output
 
+## Build instructions
+1. Clone this repository
+2. Run `go build` to build the executable.
+
 ## Acknowledgments
 This software uses the **chromedp** for web scraping, it can be found here: https://github.com/chromedp/chromedp \
 For releases the JS code is minified by using **terser** toolkit, it can be found here: https://github.com/terser/terser

models/config/config.go

@@ -12,6 +12,7 @@ var Config struct {
 	OutputPath      string
 	BatchFilePath   string
 	ArchiveFilePath string
+	FailLogFilePath string
 	Debug           bool
 	MetaData        bool
 	Quiet           bool
@@ -25,6 +26,7 @@ func GetConfig() {
 	outputPath := flag.String("output", "./downloads", "Output path")
 	batchFilePath := flag.String("batch-file", "", "File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored.")
 	archive := flag.String("archive", "", "Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.")
+	failLogPath := flag.String("fail-log", "", "Write failed items to log file")
 	debug := flag.Bool("debug", false, "Enables debug mode")
 	metadata := flag.Bool("metadata", false, "Write video metadata to a .json file")
 	quiet := flag.Bool("quiet", false, "Supress output")
@@ -48,6 +50,7 @@ func GetConfig() {
 	Config.OutputPath = *outputPath
 	Config.BatchFilePath = *batchFilePath
 	Config.ArchiveFilePath = *archive
+	Config.FailLogFilePath = *failLogPath
 	Config.Debug = *debug
 	Config.MetaData = *metadata
 	Config.Quiet = *quiet

resources/strings.go

@@ -6,8 +6,11 @@ var ErrorCouldNotSerializeJSON = "Could not serialize json for video: %s\n"
 // ErrorCouldNotRecogniseURL -
 var ErrorCouldNotRecogniseURL = "Could not recognise URL format of string %s"
 
-// ErrorCouldNotGetUserUploads -
+// Error -
-var ErrorCouldNotGetUserUploads = "Failed to get user uploads: %s\n"
+var Error = "Error : %s\n"
 
 // ErrorPathNotFound -
 var ErrorPathNotFound = "File path %s not found."
+
+// FailedOnItem -
+var FailedOnItem = "Failed while scraping item: %s\n"

workflows/common.go

@@ -0,0 +1,20 @@
+package workflows
+
+import (
+	config "../models/config"
+	res "../resources"
+	fileio "../utils/fileio"
+	log "../utils/log"
+)
+
+// OnWorkflowFail - Funtion called when workflow fails
+func OnWorkflowFail(err error, workItem string) {
+	failLogFilePath := config.Config.FailLogFilePath
+
+	if failLogFilePath != "" {
+		fileio.AppendToFile(workItem, failLogFilePath)
+	}
+
+	log.LogErr(res.Error, err.Error())
+	log.LogErr(res.FailedOnItem, workItem)
+}

workflows/downloadHashtag.go

@@ -6,7 +6,6 @@ import (
 
 	client "../client"
 	config "../models/config"
-	res "../resources"
 	utils "../utils"
 	fileio "../utils/fileio"
 	log "../utils/log"
@@ -22,7 +21,7 @@ func CanUseDownloadHashtag(url string) bool {
 func DownloadHashtag(url string) {
 	uploads, err := client.GetHashtagUploads(url)
 	if err != nil {
-		log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
+		OnWorkflowFail(err, url)
 		return
 	}
 
@@ -45,7 +44,7 @@ func DownloadHashtag(url string) {
 func GetHashtagJSON(url string) {
 	uploads, err := client.GetHashtagUploadsJSON(url)
 	if err != nil {
-		log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
+		OnWorkflowFail(err, url)
 		return
 	}
 	fmt.Printf("%s", uploads)

workflows/downloadMusic.go

@@ -6,7 +6,6 @@ import (
 
 	client "../client"
 	config "../models/config"
-	res "../resources"
 	utils "../utils"
 	fileio "../utils/fileio"
 	log "../utils/log"
@@ -22,7 +21,7 @@ func CanUseDownloadMusic(url string) bool {
 func DownloadMusic(url string) {
 	uploads, err := client.GetMusicUploads(url)
 	if err != nil {
-		log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
+		OnWorkflowFail(err, url)
 		return
 	}
 
@@ -44,7 +43,7 @@ func DownloadMusic(url string) {
 func GetMusicJSON(url string) {
 	uploads, err := client.GetMusicUploadsJSON(url)
 	if err != nil {
-		log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
+		OnWorkflowFail(err, url)
 		return
 	}
 	fmt.Printf("%s", uploads)

workflows/downloadShareLink.go

@@ -1,10 +1,10 @@
 package workflows
 
 import (
-	client "../client"
-	res "../resources"
-	log "../utils/log"
 	"regexp"
+
+	client "../client"
+	log "../utils/log"
 )
 
 // CanUseDownloadShareLink - Check's if DownloadShareLink can be used
@@ -19,7 +19,7 @@ func DownloadShareLink(url string) {
 
 	finalURL, err := client.GetRedirectUrl(url)
 	if err != nil {
-		log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
+		OnWorkflowFail(err, url)
 		return
 	}
 

workflows/downloadUser.go

@@ -7,7 +7,6 @@ import (
 
 	client "../client"
 	config "../models/config"
-	res "../resources"
 	utils "../utils"
 	fileio "../utils/fileio"
 	log "../utils/log"
@@ -24,7 +23,7 @@ func CanUseDownloadUser(url string) bool {
 func DownloadUser(username string) {
 	uploads, err := client.GetUserUploads(username)
 	if err != nil {
-		log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
+		OnWorkflowFail(err, username)
 		return
 	}
 
@@ -46,7 +45,7 @@ func DownloadUser(username string) {
 func GetUserVideosJSON(username string) {
 	uploads, err := client.GetUserUploadsJSON(username)
 	if err != nil {
-		log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
+		OnWorkflowFail(err, username)
 		return
 	}
 	fmt.Printf("%s", uploads)

workflows/downloadVideo.go

@@ -7,7 +7,6 @@ import (
 	client "../client"
 	models "../models"
 	config "../models/config"
-	res "../resources"
 	utils "../utils"
 	fileio "../utils/fileio"
 	log "../utils/log"
@@ -24,7 +23,7 @@ func DownloadSingleVideo(url string) {
 	username := utils.GetUsernameFromString(url)
 	upload, err := client.GetVideoDetails(url)
 	if err != nil {
-		log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
+		OnWorkflowFail(err, url)
 		return
 	}