From 7a691ad32d06d7c057656f2951b9dbe48e4a4191 Mon Sep 17 00:00:00 2001 From: Pijus Kamandulis Date: Tue, 25 Feb 2020 20:12:01 +0200 Subject: [PATCH] TTDL-5 Added better error handling --- client/executeClientAction.go | 59 +++++++++++++++++++++-------------- client/getMusicUploads.go | 9 ++++-- client/getUserUploads.go | 9 ++++-- client/getVideoDetails.go | 9 ++++-- resources/strings.go | 3 ++ scraper.js | 32 ++++++++++++++++--- utils/log.go | 6 ++++ workflows/downloadMusic.go | 7 ++++- workflows/downloadUser.go | 13 ++++++-- workflows/downloadVideo.go | 7 ++++- 10 files changed, 113 insertions(+), 41 deletions(-) diff --git a/client/executeClientAction.go b/client/executeClientAction.go index 74fee36..14ca590 100644 --- a/client/executeClientAction.go +++ b/client/executeClientAction.go @@ -2,10 +2,12 @@ package client import ( "context" + "errors" "github.com/chromedp/chromedp" "io/ioutil" "log" "os" + "strings" "time" config "../models/config" @@ -13,9 +15,11 @@ import ( ) // GetMusicUploads - Get all uploads by given music -func executeClientAction(url string, jsAction string) string { +func executeClientAction(url string, jsAction string) (string, error) { dir, err := ioutil.TempDir("", "chromedp-example") - utils.CheckErr(err) + if err != nil { + return "", err + } defer os.RemoveAll(dir) opts := append(chromedp.DefaultExecAllocatorOptions[:], @@ -36,15 +40,16 @@ func executeClientAction(url string, jsAction string) string { ctx, cancel = context.WithTimeout(ctx, time.Duration(config.Config.Deadline)*time.Second) defer cancel() - var jsOutput string - jsOutput = runScrapeWithInfo(ctx, jsAction, url) - - return jsOutput + jsOutput, err := runScrapeWithInfo(ctx, jsAction, url) + if strings.HasPrefix(jsOutput, "\"ERR:") { + err = errors.New(jsOutput) + } + return jsOutput, err } -func runScrapeQuiet(ctx context.Context, jsAction string, url string) string { +func runScrapeQuiet(ctx context.Context, jsAction string, url string) (string, error) { var jsOutput string - err := chromedp.Run(ctx, + if err := chromedp.Run(ctx, // Navigate to user's page chromedp.Navigate(url), // Execute url grabber script @@ -54,33 +59,40 @@ func runScrapeQuiet(ctx context.Context, jsAction string, url string) string { chromedp.WaitVisible(`video_urls`), // Grab url links from our element chromedp.InnerHTML(`video_urls`, &jsOutput), - ) - utils.CheckErr(err) - return jsOutput + ); err != nil { + return "", err + } + + return jsOutput, nil } -func runScrapeWithInfo(ctx context.Context, jsAction string, url string) string { +func runScrapeWithInfo(ctx context.Context, jsAction string, url string) (string, error) { var jsOutput string - err := chromedp.Run(ctx, + if err := chromedp.Run(ctx, // Navigate to user's page chromedp.Navigate(url), // Execute url grabber script chromedp.EvaluateAsDevTools(utils.ReadFileAsString("scraper.js"), &jsOutput), chromedp.EvaluateAsDevTools(jsAction, &jsOutput), - ) - utils.CheckErr(err) + ); err != nil { + return "", err + } for { - err = chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.preloadCount.toString()", &jsOutput)) - utils.CheckErr(err) + if err := chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.preloadCount.toString()", &jsOutput)); err != nil { + return "", err + } + if jsOutput != "0" { utils.Logf("\rPreloading... Currently loaded %s items.", jsOutput) } else { utils.Logf("\rPreloading...") } - err = chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.finished.toString()", &jsOutput)) - utils.CheckErr(err) + if err := chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.finished.toString()", &jsOutput)); err != nil { + return "", err + } + if jsOutput == "true" { break } @@ -89,13 +101,14 @@ func runScrapeWithInfo(ctx context.Context, jsAction string, url string) string } utils.Log("\nRetrieving items...") - err = chromedp.Run(ctx, + if err := chromedp.Run(ctx, // Wait until custom js finishes chromedp.WaitVisible(`video_urls`), // Grab url links from our element chromedp.InnerHTML(`video_urls`, &jsOutput), - ) - utils.CheckErr(err) + ); err != nil { + return "", err + } - return jsOutput + return jsOutput, nil } diff --git a/client/getMusicUploads.go b/client/getMusicUploads.go index 4648940..a7fa108 100644 --- a/client/getMusicUploads.go +++ b/client/getMusicUploads.go @@ -5,7 +5,10 @@ import ( ) // GetMusicUploads - Get all uploads by given music -func GetMusicUploads(url string) []models.Upload { - actionOutput := executeClientAction(url, "bootstrapIteratingVideos()") - return models.ParseUploads(actionOutput) +func GetMusicUploads(url string) ([]models.Upload, error) { + actionOutput, err := executeClientAction(url, "bootstrapIteratingVideos()") + if err != nil { + return nil, err + } + return models.ParseUploads(actionOutput), nil } diff --git a/client/getUserUploads.go b/client/getUserUploads.go index f68d7b1..67c602d 100644 --- a/client/getUserUploads.go +++ b/client/getUserUploads.go @@ -5,7 +5,10 @@ import ( ) // GetUserUploads - Get all uploads by user -func GetUserUploads(username string) []models.Upload { - actionOutput := executeClientAction(`https://www.tiktok.com/@`+username, "bootstrapIteratingVideos()") - return models.ParseUploads(actionOutput) +func GetUserUploads(username string) ([]models.Upload, error) { + actionOutput, err := executeClientAction(`https://www.tiktok.com/@`+username, "bootstrapIteratingVideos()") + if err != nil { + return nil, err + } + return models.ParseUploads(actionOutput), nil } diff --git a/client/getVideoDetails.go b/client/getVideoDetails.go index 20a0d5d..673e9bb 100644 --- a/client/getVideoDetails.go +++ b/client/getVideoDetails.go @@ -5,7 +5,10 @@ import ( ) // GetVideoDetails - returns details of video -func GetVideoDetails(videoURL string) models.Upload { - actionOutput := executeClientAction(videoURL, "bootstrapGetCurrentVideo()") - return models.ParseUpload(actionOutput) +func GetVideoDetails(videoURL string) (models.Upload, error) { + actionOutput, err := executeClientAction(videoURL, "bootstrapGetCurrentVideo()") + if err != nil { + return models.Upload{}, err + } + return models.ParseUpload(actionOutput), nil } diff --git a/resources/strings.go b/resources/strings.go index 6438c07..15b87ca 100644 --- a/resources/strings.go +++ b/resources/strings.go @@ -6,5 +6,8 @@ var ErrorCouldNotSerializeJSON = "Could not serialize json for video: %s\n" // ErrorCouldNotRecogniseURL - var ErrorCouldNotRecogniseURL = "Could not recognise URL format of string %s" +// ErrorCouldNotGetUserUploads - +var ErrorCouldNotGetUserUploads = "Failed to get user uploads: %s\n" + // ErrorPathNotFound - var ErrorPathNotFound = "File path %s not found." diff --git a/scraper.js b/scraper.js index e45a9b6..5d2dd16 100644 --- a/scraper.js +++ b/scraper.js @@ -17,6 +17,7 @@ optStrings = { classes: { feedVideoItem: 'video-feed-item-wrapper', modalCloseDisabled: 'disabled', + titleMessage: 'title', }, tags: { resultTag: 'video_urls', @@ -25,6 +26,11 @@ optStrings = { attributes: { src: "src", }, + tiktokMessages: [ + "Couldn't find this account", + "No videos yet", + "Video currently unavailable", + ], }; currentState = { @@ -32,12 +38,25 @@ currentState = { finished: false, }; +checkForErrors = function() { + var titles = document.getElementsByClassName(optStrings.classes.titleMessage); + debugger; + if (titles && titles.length) { + var error = Array.from(titles).find(x => optStrings.tiktokMessages.includes(x.textContent)).textContent; + if (error) { + createVidUrlElement("ERR: " + error); + return true; + } + } + return false; +}; + createVidUrlElement = function(outputObj) { var urlSetElement = document.createElement(optStrings.tags.resultTag); urlSetElement.innerText = JSON.stringify(outputObj); document.getElementsByTagName(optStrings.tags.resultParentTag)[0].appendChild(urlSetElement); currentState.finished = true; -} +}; buldVidUrlArray = function(finishCallback) { var feedItem = document.getElementsByClassName(optStrings.classes.feedVideoItem)[0]; @@ -78,9 +97,10 @@ getCurrentModalVideo = function() { link: soundHref, }, }; -} +}; getCurrentVideo = function() { + if(checkForErrors()) return; var player = document.querySelector(optStrings.selectors.videoPlayer); var vidUrl = player.getAttribute(optStrings.attributes.src); var shareLink = document.querySelector(optStrings.selectors.videoShareInput).value; @@ -100,13 +120,17 @@ getCurrentVideo = function() { link: soundHref, }, }; -} +}; scrollWhileNew = function(finishCallback) { var state = { count: 0 }; var intervalID = window.setInterval(x => { var oldCount = state.count; state.count = document.getElementsByClassName(optStrings.classes.feedVideoItem).length; + if(checkForErrors()) { + window.clearInterval(intervalID); + return; + } if (oldCount !== state.count) { currentState.preloadCount = state.count; window.scrollTo(0, document.body.scrollHeight); @@ -130,7 +154,7 @@ bootstrapGetCurrentVideo = function() { var video = getCurrentVideo(); createVidUrlElement(video); return 'bootstrapGetCurrentVideo'; -} +}; init = () => { const newProto = navigator.__proto__; diff --git a/utils/log.go b/utils/log.go index 6d385f2..6ab790a 100644 --- a/utils/log.go +++ b/utils/log.go @@ -3,6 +3,7 @@ package utils import ( config "../models/config" "fmt" + "os" ) // Log - Write to std out @@ -23,3 +24,8 @@ func Logf(format string, a ...interface{}) { func LogFatal(format string, a ...interface{}) { panic(fmt.Sprintf(format, a...)) } + +// LogErr - Write error +func LogErr(format string, a ...interface{}) { + fmt.Fprintf(os.Stderr, format, a...) +} diff --git a/workflows/downloadMusic.go b/workflows/downloadMusic.go index 8fecd37..92a5e3a 100644 --- a/workflows/downloadMusic.go +++ b/workflows/downloadMusic.go @@ -3,6 +3,7 @@ package workflows import ( client "../client" config "../models/config" + res "../resources" utils "../utils" "fmt" "regexp" @@ -16,7 +17,11 @@ func CanUseDownloadMusic(url string) bool { // DownloadMusic - Download all videos by given music func DownloadMusic(url string) { - uploads := client.GetMusicUploads(url) + uploads, err := client.GetMusicUploads(url) + if err != nil { + utils.LogErr(res.ErrorCouldNotGetUserUploads, err.Error()) + return + } uploadCount := len(uploads) for index, upload := range uploads { diff --git a/workflows/downloadUser.go b/workflows/downloadUser.go index 980921c..5e77bd3 100644 --- a/workflows/downloadUser.go +++ b/workflows/downloadUser.go @@ -3,20 +3,27 @@ package workflows import ( client "../client" config "../models/config" + res "../resources" utils "../utils" "fmt" + "regexp" "strings" ) // CanUseDownloadUser - Test's if this workflow can be used for parameter func CanUseDownloadUser(url string) bool { - match := strings.Contains(url, "/") - return !match + isURL := strings.Contains(url, "/") + match, _ := regexp.MatchString(".+com\\/@[^\\/]+", url) + return !isURL || match } // DownloadUser - Download all user's videos func DownloadUser(username string) { - uploads := client.GetUserUploads(username) + uploads, err := client.GetUserUploads(username) + if err != nil { + utils.LogErr(res.ErrorCouldNotGetUserUploads, err.Error()) + return + } uploadCount := len(uploads) downloadDir := fmt.Sprintf("%s/%s", config.Config.OutputPath, username) diff --git a/workflows/downloadVideo.go b/workflows/downloadVideo.go index e538db1..24fd9c1 100644 --- a/workflows/downloadVideo.go +++ b/workflows/downloadVideo.go @@ -4,6 +4,7 @@ import ( client "../client" models "../models" config "../models/config" + res "../resources" utils "../utils" "fmt" "regexp" @@ -18,7 +19,11 @@ func CanUseDownloadSingleVideo(url string) bool { // DownloadSingleVideo - Downloads single video func DownloadSingleVideo(url string) { username := utils.GetUsernameFromString(url) - upload := client.GetVideoDetails(url) + upload, err := client.GetVideoDetails(url) + if err != nil { + utils.LogErr(res.ErrorCouldNotGetUserUploads, err.Error()) + return + } downloadDir := fmt.Sprintf("%s/%s", config.Config.OutputPath, username) utils.InitOutputDirectory(downloadDir)