Merge branch 'master' into master

client/executeClientAction.go

@@ -2,10 +2,12 @@ package client
 
 import (
 	"context"
+	"errors"
 	"github.com/chromedp/chromedp"
 	"io/ioutil"
 	"log"
 	"os"
+	"strings"
 	"time"
 
 	config "../models/config"
@@ -13,9 +15,11 @@ import (
 )
 
 // GetMusicUploads - Get all uploads by given music
-func executeClientAction(url string, jsAction string) string {
+func executeClientAction(url string, jsAction string) (string, error) {
 	dir, err := ioutil.TempDir("", "chromedp-example")
-	utils.CheckErr(err)
+	if err != nil {
+		return "", err
+	}
 	defer os.RemoveAll(dir)
 
 	opts := append(chromedp.DefaultExecAllocatorOptions[:],
@@ -36,15 +40,16 @@ func executeClientAction(url string, jsAction string) string {
 	ctx, cancel = context.WithTimeout(ctx, time.Duration(config.Config.Deadline)*time.Second)
 	defer cancel()
 
-	var jsOutput string
+	jsOutput, err := runScrapeWithInfo(ctx, jsAction, url)
-	jsOutput = runScrapeWithInfo(ctx, jsAction, url)
+	if strings.HasPrefix(jsOutput, "\"ERR:") {
-
+		err = errors.New(jsOutput)
-	return jsOutput
+	}
+	return jsOutput, err
 }
 
-func runScrapeQuiet(ctx context.Context, jsAction string, url string) string {
+func runScrapeQuiet(ctx context.Context, jsAction string, url string) (string, error) {
 	var jsOutput string
-	err := chromedp.Run(ctx,
+	if err := chromedp.Run(ctx,
 		// Navigate to user's page
 		chromedp.Navigate(url),
 		// Execute url grabber script
@@ -54,33 +59,40 @@ func runScrapeQuiet(ctx context.Context, jsAction string, url string) string {
 		chromedp.WaitVisible(`video_urls`),
 		// Grab url links from our element
 		chromedp.InnerHTML(`video_urls`, &jsOutput),
-	)
+	); err != nil {
-	utils.CheckErr(err)
+		return "", err
-	return jsOutput
+	}
+
+	return jsOutput, nil
 }
 
-func runScrapeWithInfo(ctx context.Context, jsAction string, url string) string {
+func runScrapeWithInfo(ctx context.Context, jsAction string, url string) (string, error) {
 	var jsOutput string
-	err := chromedp.Run(ctx,
+	if err := chromedp.Run(ctx,
 		// Navigate to user's page
 		chromedp.Navigate(url),
 		// Execute url grabber script
 		chromedp.EvaluateAsDevTools(utils.ReadFileAsString("scraper.js"), &jsOutput),
 		chromedp.EvaluateAsDevTools(jsAction, &jsOutput),
-	)
+	); err != nil {
-	utils.CheckErr(err)
+		return "", err
+	}
 
 	for {
-		err = chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.preloadCount.toString()", &jsOutput))
+		if err := chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.preloadCount.toString()", &jsOutput)); err != nil {
-		utils.CheckErr(err)
+			return "", err
+		}
+
 		if jsOutput != "0" {
 			utils.Logf("\rPreloading... %s items have been founded.", jsOutput)
 		} else {
 			utils.Logf("\rPreloading...")
 		}
 
-		err = chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.finished.toString()", &jsOutput))
+		if err := chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.finished.toString()", &jsOutput)); err != nil {
-		utils.CheckErr(err)
+			return "", err
+		}
+
 		if jsOutput == "true" {
 			break
 		}
@@ -89,13 +101,14 @@ func runScrapeWithInfo(ctx context.Context, jsAction string, url string) string
 	}
 
 	utils.Log("\nRetrieving items...")
-	err = chromedp.Run(ctx,
+	if err := chromedp.Run(ctx,
 		// Wait until custom js finishes
 		chromedp.WaitVisible(`video_urls`),
 		// Grab url links from our element
 		chromedp.InnerHTML(`video_urls`, &jsOutput),
-	)
+	); err != nil {
-	utils.CheckErr(err)
+		return "", err
+	}
 
-	return jsOutput
+	return jsOutput, nil
 }

client/getMusicUploads.go

@@ -7,10 +7,13 @@ import (
 )
 
 // GetMusicUploads - Get all uploads by given music
-func GetMusicUploads(url string) []models.Upload {
+func GetMusicUploads(url string) ([]models.Upload, error) {
-	jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit)
+  jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit)
-	actionOutput := executeClientAction(url, jsMethod)
+	actionOutput, err := executeClientAction(url, jsMethod)
-	return models.ParseUploads(actionOutput)
+	if err != nil {
+		return nil, err
+	}
+	return models.ParseUploads(actionOutput), nil
 }
 
 func GetMusicUploadsJson(url string) string {

client/getUserUploads.go

@@ -7,10 +7,13 @@ import (
 )
 
 // GetUserUploads - Get all uploads by user
-func GetUserUploads(username string) []models.Upload {
+func GetUserUploads(username string) ([]models.Upload, error) {
-	jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit)
+  jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit)
-	actionOutput := executeClientAction(`https://www.tiktok.com/@`+username, jsMethod)
+	actionOutput, err := executeClientAction(`https://www.tiktok.com/@`+username, jsMethod)
-	return models.ParseUploads(actionOutput)
+	if err != nil {
+		return nil, err
+	}
+	return models.ParseUploads(actionOutput), nil
 }
 
 func GetUserUploadsJson(username string) string {

client/getVideoDetails.go

@@ -5,7 +5,10 @@ import (
 )
 
 // GetVideoDetails - returns details of video
-func GetVideoDetails(videoURL string) models.Upload {
+func GetVideoDetails(videoURL string) (models.Upload, error) {
-	actionOutput := executeClientAction(videoURL, "bootstrapGetCurrentVideo()")
+	actionOutput, err := executeClientAction(videoURL, "bootstrapGetCurrentVideo()")
-	return models.ParseUpload(actionOutput)
+	if err != nil {
+		return models.Upload{}, err
+	}
+	return models.ParseUpload(actionOutput), nil
 }

resources/strings.go

@@ -6,5 +6,8 @@ var ErrorCouldNotSerializeJSON = "Could not serialize json for video: %s\n"
 // ErrorCouldNotRecogniseURL -
 var ErrorCouldNotRecogniseURL = "Could not recognise URL format of string %s"
 
+// ErrorCouldNotGetUserUploads -
+var ErrorCouldNotGetUserUploads = "Failed to get user uploads: %s\n"
+
 // ErrorPathNotFound -
 var ErrorPathNotFound = "File path %s not found."

scraper.js

@@ -17,6 +17,7 @@ optStrings = {
     classes: {
         feedVideoItem: 'video-feed-item-wrapper',
         modalCloseDisabled: 'disabled',
+        titleMessage: 'title',
     },
     tags: {
         resultTag: 'video_urls',
@@ -25,6 +26,11 @@ optStrings = {
     attributes: {
         src: "src",
     },
+    tiktokMessages: [
+        "Couldn't find this account",
+        "No videos yet",
+        "Video currently unavailable",
+    ],
 };
 
 currentState = {
@@ -33,6 +39,19 @@ currentState = {
     limit: 0
 };
 
+checkForErrors = function() {
+    var titles = document.getElementsByClassName(optStrings.classes.titleMessage);
+    debugger;
+    if (titles && titles.length) {
+        var error = Array.from(titles).find(x => optStrings.tiktokMessages.includes(x.textContent)).textContent;
+        if (error) {
+            createVidUrlElement("ERR: " + error);
+            return true;
+        }
+    }
+    return false;
+};
+
 createVidUrlElement = function(outputObj) {
     var urlSetElement = document.createElement(optStrings.tags.resultTag);
     urlSetElement.innerText = JSON.stringify(outputObj);
@@ -88,6 +107,7 @@ getCurrentModalVideo = function() {
 };
 
 getCurrentVideo = function() {
+    if(checkForErrors()) return;
     var player = document.querySelector(optStrings.selectors.videoPlayer);
     var vidUrl = player.getAttribute(optStrings.attributes.src);
     var shareLink = document.querySelector(optStrings.selectors.videoShareInput).value;
@@ -120,6 +140,10 @@ scrollWhileNew = function(finishCallback) {
                 window.clearInterval(intervalID);
             }
         }
+        if(checkForErrors()) {
+            window.clearInterval(intervalID);
+            return;
+        }
         if (oldCount !== state.count) {
             currentState.preloadCount = state.count;
             window.scrollTo(0, document.body.scrollHeight);

utils/log.go

@@ -3,6 +3,7 @@ package utils
 import (
 	config "../models/config"
 	"fmt"
+	"os"
 )
 
 // Log - Write to std out
@@ -23,3 +24,8 @@ func Logf(format string, a ...interface{}) {
 func LogFatal(format string, a ...interface{}) {
 	panic(fmt.Sprintf(format, a...))
 }
+
+// LogErr - Write error
+func LogErr(format string, a ...interface{}) {
+	fmt.Fprintf(os.Stderr, format, a...)
+}

workflows/downloadMusic.go

@@ -3,6 +3,7 @@ package workflows
 import (
 	client "../client"
 	config "../models/config"
+	res "../resources"
 	utils "../utils"
 	"fmt"
 	"regexp"
@@ -16,7 +17,11 @@ func CanUseDownloadMusic(url string) bool {
 
 // DownloadMusic - Download all videos by given music
 func DownloadMusic(url string) {
-	uploads := client.GetMusicUploads(url)
+	uploads, err := client.GetMusicUploads(url)
+	if err != nil {
+		utils.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
+		return
+	}
 	uploadCount := len(uploads)
 
 	for index, upload := range uploads {

workflows/downloadUser.go

@@ -3,20 +3,27 @@ package workflows
 import (
 	client "../client"
 	config "../models/config"
+	res "../resources"
 	utils "../utils"
 	"fmt"
+	"regexp"
 	"strings"
 )
 
 // CanUseDownloadUser - Test's if this workflow can be used for parameter
 func CanUseDownloadUser(url string) bool {
-	match := strings.Contains(url, "/")
+	isURL := strings.Contains(url, "/")
-	return !match
+	match, _ := regexp.MatchString(".+com\\/@[^\\/]+", url)
+	return !isURL || match
 }
 
 // DownloadUser - Download all user's videos
 func DownloadUser(username string) {
-	uploads := client.GetUserUploads(username)
+	uploads, err := client.GetUserUploads(username)
+	if err != nil {
+		utils.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
+		return
+	}
 	uploadCount := len(uploads)
 	downloadDir := fmt.Sprintf("%s/%s", config.Config.OutputPath, username)
 

workflows/downloadVideo.go

@@ -4,6 +4,7 @@ import (
 	client "../client"
 	models "../models"
 	config "../models/config"
+	res "../resources"
 	utils "../utils"
 	"fmt"
 	"regexp"
@@ -18,7 +19,11 @@ func CanUseDownloadSingleVideo(url string) bool {
 // DownloadSingleVideo - Downloads single video
 func DownloadSingleVideo(url string) {
 	username := utils.GetUsernameFromString(url)
-	upload := client.GetVideoDetails(url)
+	upload, err := client.GetVideoDetails(url)
+	if err != nil {
+		utils.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
+		return
+	}
 	downloadDir := fmt.Sprintf("%s/%s", config.Config.OutputPath, username)
 
 	utils.InitOutputDirectory(downloadDir)