diff --git a/client/executeClientAction.go b/client/executeClientAction.go index ca821c8..cb7bddc 100644 --- a/client/executeClientAction.go +++ b/client/executeClientAction.go @@ -2,10 +2,12 @@ package client import ( "context" + "errors" "github.com/chromedp/chromedp" "io/ioutil" "log" "os" + "strings" "time" config "../models/config" @@ -13,9 +15,11 @@ import ( ) // GetMusicUploads - Get all uploads by given music -func executeClientAction(url string, jsAction string) string { +func executeClientAction(url string, jsAction string) (string, error) { dir, err := ioutil.TempDir("", "chromedp-example") - utils.CheckErr(err) + if err != nil { + return "", err + } defer os.RemoveAll(dir) opts := append(chromedp.DefaultExecAllocatorOptions[:], @@ -36,15 +40,16 @@ func executeClientAction(url string, jsAction string) string { ctx, cancel = context.WithTimeout(ctx, time.Duration(config.Config.Deadline)*time.Second) defer cancel() - var jsOutput string - jsOutput = runScrapeWithInfo(ctx, jsAction, url) - - return jsOutput + jsOutput, err := runScrapeWithInfo(ctx, jsAction, url) + if strings.HasPrefix(jsOutput, "\"ERR:") { + err = errors.New(jsOutput) + } + return jsOutput, err } -func runScrapeQuiet(ctx context.Context, jsAction string, url string) string { +func runScrapeQuiet(ctx context.Context, jsAction string, url string) (string, error) { var jsOutput string - err := chromedp.Run(ctx, + if err := chromedp.Run(ctx, // Navigate to user's page chromedp.Navigate(url), // Execute url grabber script @@ -54,33 +59,40 @@ func runScrapeQuiet(ctx context.Context, jsAction string, url string) string { chromedp.WaitVisible(`video_urls`), // Grab url links from our element chromedp.InnerHTML(`video_urls`, &jsOutput), - ) - utils.CheckErr(err) - return jsOutput + ); err != nil { + return "", err + } + + return jsOutput, nil } -func runScrapeWithInfo(ctx context.Context, jsAction string, url string) string { +func runScrapeWithInfo(ctx context.Context, jsAction string, url string) (string, error) { var jsOutput string - err := chromedp.Run(ctx, + if err := chromedp.Run(ctx, // Navigate to user's page chromedp.Navigate(url), // Execute url grabber script chromedp.EvaluateAsDevTools(utils.ReadFileAsString("scraper.js"), &jsOutput), chromedp.EvaluateAsDevTools(jsAction, &jsOutput), - ) - utils.CheckErr(err) + ); err != nil { + return "", err + } for { - err = chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.preloadCount.toString()", &jsOutput)) - utils.CheckErr(err) + if err := chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.preloadCount.toString()", &jsOutput)); err != nil { + return "", err + } + if jsOutput != "0" { utils.Logf("\rPreloading... %s items have been founded.", jsOutput) } else { utils.Logf("\rPreloading...") } - err = chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.finished.toString()", &jsOutput)) - utils.CheckErr(err) + if err := chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.finished.toString()", &jsOutput)); err != nil { + return "", err + } + if jsOutput == "true" { break } @@ -89,13 +101,14 @@ func runScrapeWithInfo(ctx context.Context, jsAction string, url string) string } utils.Log("\nRetrieving items...") - err = chromedp.Run(ctx, + if err := chromedp.Run(ctx, // Wait until custom js finishes chromedp.WaitVisible(`video_urls`), // Grab url links from our element chromedp.InnerHTML(`video_urls`, &jsOutput), - ) - utils.CheckErr(err) + ); err != nil { + return "", err + } - return jsOutput + return jsOutput, nil } diff --git a/client/getMusicUploads.go b/client/getMusicUploads.go index b6f6a72..d980ed7 100644 --- a/client/getMusicUploads.go +++ b/client/getMusicUploads.go @@ -7,10 +7,13 @@ import ( ) // GetMusicUploads - Get all uploads by given music -func GetMusicUploads(url string) []models.Upload { - jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit) - actionOutput := executeClientAction(url, jsMethod) - return models.ParseUploads(actionOutput) +func GetMusicUploads(url string) ([]models.Upload, error) { + jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit) + actionOutput, err := executeClientAction(url, jsMethod) + if err != nil { + return nil, err + } + return models.ParseUploads(actionOutput), nil } func GetMusicUploadsJson(url string) string { diff --git a/client/getUserUploads.go b/client/getUserUploads.go index 4fb7d07..30f0d70 100644 --- a/client/getUserUploads.go +++ b/client/getUserUploads.go @@ -7,10 +7,13 @@ import ( ) // GetUserUploads - Get all uploads by user -func GetUserUploads(username string) []models.Upload { - jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit) - actionOutput := executeClientAction(`https://www.tiktok.com/@`+username, jsMethod) - return models.ParseUploads(actionOutput) +func GetUserUploads(username string) ([]models.Upload, error) { + jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit) + actionOutput, err := executeClientAction(`https://www.tiktok.com/@`+username, jsMethod) + if err != nil { + return nil, err + } + return models.ParseUploads(actionOutput), nil } func GetUserUploadsJson(username string) string { diff --git a/client/getVideoDetails.go b/client/getVideoDetails.go index 20a0d5d..673e9bb 100644 --- a/client/getVideoDetails.go +++ b/client/getVideoDetails.go @@ -5,7 +5,10 @@ import ( ) // GetVideoDetails - returns details of video -func GetVideoDetails(videoURL string) models.Upload { - actionOutput := executeClientAction(videoURL, "bootstrapGetCurrentVideo()") - return models.ParseUpload(actionOutput) +func GetVideoDetails(videoURL string) (models.Upload, error) { + actionOutput, err := executeClientAction(videoURL, "bootstrapGetCurrentVideo()") + if err != nil { + return models.Upload{}, err + } + return models.ParseUpload(actionOutput), nil } diff --git a/resources/strings.go b/resources/strings.go index 6438c07..15b87ca 100644 --- a/resources/strings.go +++ b/resources/strings.go @@ -6,5 +6,8 @@ var ErrorCouldNotSerializeJSON = "Could not serialize json for video: %s\n" // ErrorCouldNotRecogniseURL - var ErrorCouldNotRecogniseURL = "Could not recognise URL format of string %s" +// ErrorCouldNotGetUserUploads - +var ErrorCouldNotGetUserUploads = "Failed to get user uploads: %s\n" + // ErrorPathNotFound - var ErrorPathNotFound = "File path %s not found." diff --git a/scraper.js b/scraper.js index e2446c5..fc374e6 100644 --- a/scraper.js +++ b/scraper.js @@ -17,6 +17,7 @@ optStrings = { classes: { feedVideoItem: 'video-feed-item-wrapper', modalCloseDisabled: 'disabled', + titleMessage: 'title', }, tags: { resultTag: 'video_urls', @@ -25,6 +26,11 @@ optStrings = { attributes: { src: "src", }, + tiktokMessages: [ + "Couldn't find this account", + "No videos yet", + "Video currently unavailable", + ], }; currentState = { @@ -33,6 +39,19 @@ currentState = { limit: 0 }; +checkForErrors = function() { + var titles = document.getElementsByClassName(optStrings.classes.titleMessage); + debugger; + if (titles && titles.length) { + var error = Array.from(titles).find(x => optStrings.tiktokMessages.includes(x.textContent)).textContent; + if (error) { + createVidUrlElement("ERR: " + error); + return true; + } + } + return false; +}; + createVidUrlElement = function(outputObj) { var urlSetElement = document.createElement(optStrings.tags.resultTag); urlSetElement.innerText = JSON.stringify(outputObj); @@ -88,6 +107,7 @@ getCurrentModalVideo = function() { }; getCurrentVideo = function() { + if(checkForErrors()) return; var player = document.querySelector(optStrings.selectors.videoPlayer); var vidUrl = player.getAttribute(optStrings.attributes.src); var shareLink = document.querySelector(optStrings.selectors.videoShareInput).value; @@ -120,6 +140,10 @@ scrollWhileNew = function(finishCallback) { window.clearInterval(intervalID); } } + if(checkForErrors()) { + window.clearInterval(intervalID); + return; + } if (oldCount !== state.count) { currentState.preloadCount = state.count; window.scrollTo(0, document.body.scrollHeight); diff --git a/utils/log.go b/utils/log.go index 6d385f2..6ab790a 100644 --- a/utils/log.go +++ b/utils/log.go @@ -3,6 +3,7 @@ package utils import ( config "../models/config" "fmt" + "os" ) // Log - Write to std out @@ -23,3 +24,8 @@ func Logf(format string, a ...interface{}) { func LogFatal(format string, a ...interface{}) { panic(fmt.Sprintf(format, a...)) } + +// LogErr - Write error +func LogErr(format string, a ...interface{}) { + fmt.Fprintf(os.Stderr, format, a...) +} diff --git a/workflows/downloadMusic.go b/workflows/downloadMusic.go index ef1491e..d09717e 100644 --- a/workflows/downloadMusic.go +++ b/workflows/downloadMusic.go @@ -3,6 +3,7 @@ package workflows import ( client "../client" config "../models/config" + res "../resources" utils "../utils" "fmt" "regexp" @@ -16,7 +17,11 @@ func CanUseDownloadMusic(url string) bool { // DownloadMusic - Download all videos by given music func DownloadMusic(url string) { - uploads := client.GetMusicUploads(url) + uploads, err := client.GetMusicUploads(url) + if err != nil { + utils.LogErr(res.ErrorCouldNotGetUserUploads, err.Error()) + return + } uploadCount := len(uploads) for index, upload := range uploads { diff --git a/workflows/downloadUser.go b/workflows/downloadUser.go index 0967c75..51a8424 100644 --- a/workflows/downloadUser.go +++ b/workflows/downloadUser.go @@ -3,20 +3,27 @@ package workflows import ( client "../client" config "../models/config" + res "../resources" utils "../utils" "fmt" + "regexp" "strings" ) // CanUseDownloadUser - Test's if this workflow can be used for parameter func CanUseDownloadUser(url string) bool { - match := strings.Contains(url, "/") - return !match + isURL := strings.Contains(url, "/") + match, _ := regexp.MatchString(".+com\\/@[^\\/]+", url) + return !isURL || match } // DownloadUser - Download all user's videos func DownloadUser(username string) { - uploads := client.GetUserUploads(username) + uploads, err := client.GetUserUploads(username) + if err != nil { + utils.LogErr(res.ErrorCouldNotGetUserUploads, err.Error()) + return + } uploadCount := len(uploads) downloadDir := fmt.Sprintf("%s/%s", config.Config.OutputPath, username) diff --git a/workflows/downloadVideo.go b/workflows/downloadVideo.go index e538db1..24fd9c1 100644 --- a/workflows/downloadVideo.go +++ b/workflows/downloadVideo.go @@ -4,6 +4,7 @@ import ( client "../client" models "../models" config "../models/config" + res "../resources" utils "../utils" "fmt" "regexp" @@ -18,7 +19,11 @@ func CanUseDownloadSingleVideo(url string) bool { // DownloadSingleVideo - Downloads single video func DownloadSingleVideo(url string) { username := utils.GetUsernameFromString(url) - upload := client.GetVideoDetails(url) + upload, err := client.GetVideoDetails(url) + if err != nil { + utils.LogErr(res.ErrorCouldNotGetUserUploads, err.Error()) + return + } downloadDir := fmt.Sprintf("%s/%s", config.Config.OutputPath, username) utils.InitOutputDirectory(downloadDir)