TTDL-20 Wait for error/items beefore quiting

Create LICENSE

Update go.yml

.github/workflows/go.yml

@@ -1,5 +1,5 @@
 name: tiktok-dl_CI
-on: [push]
+on: [push, pull_request]
 jobs:
   build:
     strategy:
@@ -33,6 +33,9 @@ jobs:
 
     - name: Build
       run: npm run build:dist
+    
+    - name: Copy license
+      run: cp LICENSE out
 
     - name: Upload Unix Artifacts
       if: startsWith(matrix.os, 'ubuntu-')

.gitignore

@@ -5,3 +5,5 @@ downloads
 *.exe
 tiktok-dl
 batch_file.txt
+debug.log
+.scannerwork

.sonar/sonar-project.properties

@@ -0,0 +1,12 @@
+sonar.organization=pikami
+sonar.projectKey=tiktok-dl
+
+sonar.host.url=https://sonarcloud.io
+
+sonar.sources=.
+sonar.exclusions=**/*_test.go,**/node_modules/**
+
+sonar.tests=.
+sonar.test.inclusions=**/*_test.go
+sonar.test.exclusions=**/node_modules/**
+sonar.go.coverage.reportPaths=cov.out

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 pikami
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -16,10 +16,15 @@ You can download items listed in a text file by running `./tiktok-dl [OPTIONS] -
 Clone this repository and run `go build` to build the executable.
 
 ## Available options
-* `-debug` - enables debug mode
-* `-output some_directory` - Output path (default "./downloads")
-* `-metadata` - Write video metadata to a .json file
+* `-archive` - Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.
 * `-batch-file` - File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored.
+* `-deadline` - Sets the timout for scraper logic in seconds (used as a workaround for context deadline exceeded error) (default 1500)
+* `-debug` - enables debug mode
+* `-json` - Returns whole data, that was scraped from TikTok, in json
+* `-limit` - Sets the max count of video that will be downloaded (default infinity)
+* `-metadata` - Write video metadata to a .json file
+* `-output some_directory` - Output path (default "./downloads")
+* `-quiet` - Supress output
 
 ## Acknowledgments
 This software uses the **chromedp** for web scraping, it can be found here: https://github.com/chromedp/chromedp \

client/executeClientAction.go

@@ -0,0 +1,115 @@
+package client
+
+import (
+	"context"
+	"errors"
+	"io/ioutil"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/chromedp/chromedp"
+
+	config "../models/config"
+	utils "../utils"
+	log "../utils/log"
+)
+
+// GetMusicUploads - Get all uploads by given music
+func executeClientAction(url string, jsAction string) (string, error) {
+	dir, err := ioutil.TempDir("", "chromedp-example")
+	if err != nil {
+		return "", err
+	}
+	defer os.RemoveAll(dir)
+
+	opts := append(chromedp.DefaultExecAllocatorOptions[:],
+		chromedp.DisableGPU,
+		chromedp.UserDataDir(dir),
+		chromedp.Flag("headless", !config.Config.Debug),
+	)
+
+	allocCtx, cancel := chromedp.NewExecAllocator(context.Background(), opts...)
+	defer cancel()
+
+	ctx, cancel := chromedp.NewContext(
+		allocCtx,
+		chromedp.WithLogf(log.Logf),
+	)
+	defer cancel()
+
+	ctx, cancel = context.WithTimeout(ctx, time.Duration(config.Config.Deadline)*time.Second)
+	defer cancel()
+
+	jsOutput, err := runScrapeWithInfo(ctx, jsAction, url)
+	if strings.HasPrefix(jsOutput, "\"ERR:") {
+		err = errors.New(jsOutput)
+	}
+	return jsOutput, err
+}
+
+func runScrapeQuiet(ctx context.Context, jsAction string, url string) (string, error) {
+	var jsOutput string
+	if err := chromedp.Run(ctx,
+		// Navigate to user's page
+		chromedp.Navigate(url),
+		// Execute url grabber script
+		chromedp.EvaluateAsDevTools(utils.ReadFileAsString("scraper.js"), &jsOutput),
+		chromedp.EvaluateAsDevTools(jsAction, &jsOutput),
+		// Wait until custom js finishes
+		chromedp.WaitVisible(`video_urls`),
+		// Grab url links from our element
+		chromedp.InnerHTML(`video_urls`, &jsOutput),
+	); err != nil {
+		return "", err
+	}
+
+	return jsOutput, nil
+}
+
+func runScrapeWithInfo(ctx context.Context, jsAction string, url string) (string, error) {
+	var jsOutput string
+	if err := chromedp.Run(ctx,
+		// Navigate to user's page
+		chromedp.Navigate(url),
+		// Execute url grabber script
+		chromedp.EvaluateAsDevTools(utils.ReadFileAsString("scraper.js"), &jsOutput),
+		chromedp.EvaluateAsDevTools(jsAction, &jsOutput),
+	); err != nil {
+		return "", err
+	}
+
+	for {
+		if err := chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.preloadCount.toString()", &jsOutput)); err != nil {
+			return "", err
+		}
+
+		if jsOutput != "0" {
+			log.Logf("\rPreloading... %s items have been found.", jsOutput)
+		} else {
+			log.Logf("\rPreloading...")
+		}
+
+		if err := chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.finished.toString()", &jsOutput)); err != nil {
+			return "", err
+		}
+
+		if jsOutput == "true" {
+			break
+		}
+
+		time.Sleep(50 * time.Millisecond)
+	}
+
+	log.Log("\nRetrieving items...")
+	if err := chromedp.Run(ctx,
+		// Wait until custom js finishes
+		chromedp.WaitVisible(`video_urls`),
+		// Grab url links from our element
+		chromedp.InnerHTML(`video_urls`, &jsOutput),
+	); err != nil {
+		return "", err
+	}
+
+	return jsOutput, nil
+}

client/getHashtagUploads.go

@@ -0,0 +1,28 @@
+package client
+
+import (
+	"fmt"
+
+	models "../models"
+	config "../models/config"
+)
+
+// GetHashtagUploads - Get all uploads marked with given hashtag
+func GetHashtagUploads(hashtagURL string) ([]models.Upload, error) {
+	actionOutput, err := GetHashtagUploadsJSON(hashtagURL)
+	if err != nil {
+		return nil, err
+	}
+
+	return models.ParseUploads(actionOutput), nil
+}
+
+// GetHashtagUploadsJSON - Get hashtag uploads scrape
+func GetHashtagUploadsJSON(hashtagURL string) (string, error) {
+	jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit)
+	actionOutput, err := executeClientAction(hashtagURL, jsMethod)
+	if err != nil {
+		return "", err
+	}
+	return actionOutput, nil
+}

client/getMusicUploads.go

@@ -1,58 +1,27 @@
 package client
 
 import (
-	"context"
-	"github.com/chromedp/chromedp"
-	"io/ioutil"
-	"log"
-	"os"
-	"time"
+	"fmt"
 
 	models "../models"
-	utils "../utils"
+	config "../models/config"
 )
 
 // GetMusicUploads - Get all uploads by given music
-func GetMusicUploads(url string) []models.Upload {
-	dir, err := ioutil.TempDir("", "chromedp-example")
+func GetMusicUploads(url string) ([]models.Upload, error) {
+	actionOutput, err := GetMusicUploadsJSON(url)
 	if err != nil {
-		panic(err)
+		return nil, err
 	}
-	defer os.RemoveAll(dir)
-
-	opts := append(chromedp.DefaultExecAllocatorOptions[:],
-		chromedp.DisableGPU,
-		chromedp.UserDataDir(dir),
-		chromedp.Flag("headless", !models.Config.Debug),
-	)
-
-	allocCtx, cancel := chromedp.NewExecAllocator(context.Background(), opts...)
-	defer cancel()
-
-	ctx, cancel := chromedp.NewContext(
-		allocCtx,
-		chromedp.WithLogf(log.Printf),
-	)
-	defer cancel()
-
-	ctx, cancel = context.WithTimeout(ctx, 1500*time.Second)
-	defer cancel()
-
-	var jsOutput string
-	err = chromedp.Run(ctx,
-		// Navigate to user's page
-		chromedp.Navigate(url),
-		// Execute url grabber script
-		chromedp.EvaluateAsDevTools(utils.ReadFileAsString("scraper.js"), &jsOutput),
-		chromedp.EvaluateAsDevTools("bootstrapIteratingVideos()", &jsOutput),
-		// Wait until custom js finishes
-		chromedp.WaitVisible(`video_urls`),
-		// Grab url links from our element
-		chromedp.InnerHTML(`video_urls`, &jsOutput),
-	)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	return models.ParseUploads(jsOutput)
+	return models.ParseUploads(actionOutput), nil
+}
+
+// GetMusicUploadsJSON - Get music uploads scrape
+func GetMusicUploadsJSON(url string) (string, error) {
+	jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit)
+	actionOutput, err := executeClientAction(url, jsMethod)
+	if err != nil {
+		return "", err
+	}
+	return actionOutput, nil
 }

client/getRedirectUrl.go

@@ -0,0 +1,52 @@
+package client
+
+import (
+	"context"
+	"github.com/chromedp/chromedp"
+	"io/ioutil"
+	"os"
+	"time"
+
+	config "../models/config"
+	log "../utils/log"
+)
+
+func GetRedirectUrl(url string) (string, error) {
+	dir, err := ioutil.TempDir("", "chromedp-example")
+	if err != nil {
+		return "", err
+	}
+	defer os.RemoveAll(dir)
+
+	opts := append(chromedp.DefaultExecAllocatorOptions[:],
+		chromedp.DisableGPU,
+		chromedp.UserDataDir(dir),
+		chromedp.Flag("headless", !config.Config.Debug),
+	)
+
+	allocCtx, cancel := chromedp.NewExecAllocator(context.Background(), opts...)
+	defer cancel()
+
+	ctx, cancel := chromedp.NewContext(
+		allocCtx,
+		chromedp.WithLogf(log.Logf),
+	)
+	defer cancel()
+
+	ctx, cancel = context.WithTimeout(ctx, time.Duration(config.Config.Deadline)*time.Second)
+	defer cancel()
+
+	var jsOutput string
+	if err := chromedp.Run(ctx,
+		// Navigate to user's page
+		chromedp.Navigate(url),
+		// Wait until page loads
+		chromedp.WaitReady(`div`),
+		// Grab url links from our element
+		chromedp.EvaluateAsDevTools(`window.location.href`, &jsOutput),
+	); err != nil {
+		return "", err
+	}
+
+	return jsOutput, err
+}

client/getUserUploads.go

@@ -1,58 +1,27 @@
 package client
 
 import (
-	"context"
-	"github.com/chromedp/chromedp"
-	"io/ioutil"
-	"log"
-	"os"
-	"time"
+	"fmt"
 
 	models "../models"
-	utils "../utils"
+	config "../models/config"
 )
 
 // GetUserUploads - Get all uploads by user
-func GetUserUploads(username string) []models.Upload {
-	dir, err := ioutil.TempDir("", "chromedp-example")
+func GetUserUploads(username string) ([]models.Upload, error) {
+	actionOutput, err := GetUserUploadsJSON(username)
 	if err != nil {
-		panic(err)
+		return nil, err
 	}
-	defer os.RemoveAll(dir)
-
-	opts := append(chromedp.DefaultExecAllocatorOptions[:],
-		chromedp.DisableGPU,
-		chromedp.UserDataDir(dir),
-		chromedp.Flag("headless", !models.Config.Debug),
-	)
-
-	allocCtx, cancel := chromedp.NewExecAllocator(context.Background(), opts...)
-	defer cancel()
-
-	ctx, cancel := chromedp.NewContext(
-		allocCtx,
-		chromedp.WithLogf(log.Printf),
-	)
-	defer cancel()
-
-	ctx, cancel = context.WithTimeout(ctx, 1500*time.Second)
-	defer cancel()
-
-	var jsOutput string
-	err = chromedp.Run(ctx,
-		// Navigate to user's page
-		chromedp.Navigate(`https://www.tiktok.com/@`+username),
-		// Execute url grabber script
-		chromedp.EvaluateAsDevTools(utils.ReadFileAsString("scraper.js"), &jsOutput),
-		chromedp.EvaluateAsDevTools("bootstrapIteratingVideos()", &jsOutput),
-		// Wait until custom js finishes
-		chromedp.WaitVisible(`video_urls`),
-		// Grab url links from our element
-		chromedp.InnerHTML(`video_urls`, &jsOutput),
-	)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	return models.ParseUploads(jsOutput)
+	return models.ParseUploads(actionOutput), nil
+}
+
+// GetUserUploadsJSON - Get user uploads scrape
+func GetUserUploadsJSON(username string) (string, error) {
+	jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit)
+	actionOutput, err := executeClientAction(`https://www.tiktok.com/@`+username, jsMethod)
+	if err != nil {
+		return "", err
+	}
+	return actionOutput, nil
 }

client/getVideoDetails.go

@@ -1,58 +1,14 @@
 package client
 
 import (
-	"context"
-	"github.com/chromedp/chromedp"
-	"io/ioutil"
-	"log"
-	"os"
-	"time"
-
 	models "../models"
-	utils "../utils"
 )
 
 // GetVideoDetails - returns details of video
-func GetVideoDetails(videoURL string) models.Upload {
-	dir, err := ioutil.TempDir("", "chromedp-example")
+func GetVideoDetails(videoURL string) (models.Upload, error) {
+	actionOutput, err := executeClientAction(videoURL, "bootstrapGetCurrentVideo()")
 	if err != nil {
-		panic(err)
+		return models.Upload{}, err
 	}
-	defer os.RemoveAll(dir)
-
-	opts := append(chromedp.DefaultExecAllocatorOptions[:],
-		chromedp.DisableGPU,
-		chromedp.UserDataDir(dir),
-		chromedp.Flag("headless", !models.Config.Debug),
-	)
-
-	allocCtx, cancel := chromedp.NewExecAllocator(context.Background(), opts...)
-	defer cancel()
-
-	ctx, cancel := chromedp.NewContext(
-		allocCtx,
-		chromedp.WithLogf(log.Printf),
-	)
-	defer cancel()
-
-	ctx, cancel = context.WithTimeout(ctx, 1500*time.Second)
-	defer cancel()
-
-	var jsOutput string
-	err = chromedp.Run(ctx,
-		// Navigate to user's page
-		chromedp.Navigate(videoURL),
-		// Execute url grabber script
-		chromedp.EvaluateAsDevTools(utils.ReadFileAsString("scraper.js"), &jsOutput),
-		chromedp.EvaluateAsDevTools("bootstrapGetCurrentVideo()", &jsOutput),
-		// Wait until custom js finishes
-		chromedp.WaitVisible(`video_urls`),
-		// Grab url links from our element
-		chromedp.InnerHTML(`video_urls`, &jsOutput),
-	)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	return models.ParseUpload(jsOutput)
+	return models.ParseUpload(actionOutput), nil
 }

main.go

@@ -1,14 +1,14 @@
 package main
 
 import (
-	models "./models"
+	config "./models/config"
 	workflows "./workflows"
 )
 
 func main() {
-	models.GetConfig()
-	url := models.Config.URL
-	batchFilePath := models.Config.BatchFilePath
+	config.GetConfig()
+	url := config.Config.URL
+	batchFilePath := config.Config.BatchFilePath
 
 	// Batch file
 	if workflows.CanUseDownloadBatchFile(batchFilePath) {

models/config/config.go

@@ -1,28 +1,36 @@
-package models
+package config
 
 import (
 	"flag"
 	"fmt"
 	"os"
-	"regexp"
-	"strings"
 )
 
 // Config - Runtime configuration
 var Config struct {
-	URL           string
-	OutputPath    string
-	BatchFilePath string
-	Debug         bool
-	MetaData      bool
+	URL             string
+	OutputPath      string
+	BatchFilePath   string
+	ArchiveFilePath string
+	Debug           bool
+	MetaData        bool
+	Quiet           bool
+	JSONOnly        bool
+	Deadline        int
+	Limit           int
 }
 
 // GetConfig - Returns Config object
 func GetConfig() {
 	outputPath := flag.String("output", "./downloads", "Output path")
 	batchFilePath := flag.String("batch-file", "", "File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored.")
+	archive := flag.String("archive", "", "Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.")
 	debug := flag.Bool("debug", false, "Enables debug mode")
 	metadata := flag.Bool("metadata", false, "Write video metadata to a .json file")
+	quiet := flag.Bool("quiet", false, "Supress output")
+	jsonOnly := flag.Bool("json", false, "Just get JSON data from scraper (without video downloading)")
+	deadline := flag.Int("deadline", 1500, "Sets the timout for scraper logic in seconds (used as a workaround for 'context deadline exceeded' error)")
+	limit := flag.Int("limit", 0, "Sets the videos count limit (useful when there too many videos from the user or by hashtag)")
 	flag.Parse()
 
 	args := flag.Args()
@@ -39,25 +47,14 @@ func GetConfig() {
 	}
 	Config.OutputPath = *outputPath
 	Config.BatchFilePath = *batchFilePath
+	Config.ArchiveFilePath = *archive
 	Config.Debug = *debug
 	Config.MetaData = *metadata
-}
-
-// GetUsername - Get's username from passed URL param
-func GetUsername() string {
-	return GetUsernameFromString(Config.URL)
-}
-
-// GetUsernameFromString - Get's username from passed param
-func GetUsernameFromString(str string) string {
-	if match := strings.Contains(str, "/"); !match { // Not url
-		return strings.Replace(str, "@", "", -1)
+	Config.Quiet = *quiet
+	if *jsonOnly {
+		Config.Quiet = true
 	}
-
-	if match, _ := regexp.MatchString(".+tiktok\\.com/@.+", str); match { // URL
-		stripedSuffix := strings.Split(str, "@")[1]
-		return strings.Split(stripedSuffix, "/")[0]
-	}
-
-	panic("Could not recognise URL format")
+	Config.JSONOnly = *jsonOnly
+	Config.Deadline = *deadline
+	Config.Limit = *limit
 }

models/upload.go

@@ -2,9 +2,12 @@ package models
 
 import (
 	"encoding/json"
-	"fmt"
 	"os"
 	"strings"
+
+	res "../resources"
+	checkErr "../utils/checkErr"
+	log "../utils/log"
 )
 
 // Upload - Upload object
@@ -46,21 +49,16 @@ func (u Upload) GetUploadID() string {
 func (u Upload) WriteToFile(outputPath string) {
 	bytes, err := json.Marshal(u)
 	if err != nil {
-		fmt.Printf("Could not serialize json for video: %s", u.GetUploadID())
-		fmt.Println()
+		log.Logf(res.ErrorCouldNotSerializeJSON, u.GetUploadID())
 		panic(err)
 	}
 
 	// Create the file
 	out, err := os.Create(outputPath)
-	if err != nil {
-		panic(err)
-	}
+	checkErr.CheckErr(err)
 	defer out.Close()
 
 	// Write to file
 	_, err = out.Write(bytes)
-	if err != nil {
-		panic(err)
-	}
+	checkErr.CheckErr(err)
 }

models/upload_test.go

@@ -1,10 +1,11 @@
 package models
 
 import (
-	testUtil "../unitTestUtil"
-	utils "../utils"
 	"os"
 	"testing"
+
+	testUtil "../unitTestUtil"
+	fileio "../utils/fileio"
 )
 
 func TestParseUploads(t *testing.T) {
@@ -62,7 +63,7 @@ func TestWriteToFile(t *testing.T) {
 
 	upload.WriteToFile(filePath)
 
-	actual := utils.ReadFileToString(filePath)
+	actual := fileio.ReadFileToString(filePath)
 	tu.AssertString(actual, expected, "File content")
 
 	os.Remove(filePath)

package.json

@@ -3,12 +3,14 @@
     "version": "0.0.1",
     "scripts": {
         "install-dependencies": "go get -v -t -d ./...",
-        "test": "go test -v ./models",
+        "test:coverage": "go test -short -coverprofile=cov.out ./models ./utils",
+        "test": "go test -v ./models && go test -v ./utils",
         "clean": "rm -rf out",
         "build:scraper": "node node_modules/terser/bin/terser -c -m -- scraper.js > out/scraper.js",
         "build:app": "go build -o out/ -v .",
         "build:dist": "mkdir out && npm run build:app && npm run build:scraper",
-        "build": "go build -v ."
+        "build": "go build -v .",
+        "sonar": "sonar-scanner -Dsonar.login=${SONAR_LOGIN} -Dproject.settings=.sonar/sonar-project.properties"
     },
     "dependencies": {
         "terser": "^4.6.3"

resources/strings.go

@@ -0,0 +1,13 @@
+package resources
+
+// ErrorCouldNotSerializeJSON -
+var ErrorCouldNotSerializeJSON = "Could not serialize json for video: %s\n"
+
+// ErrorCouldNotRecogniseURL -
+var ErrorCouldNotRecogniseURL = "Could not recognise URL format of string %s"
+
+// ErrorCouldNotGetUserUploads -
+var ErrorCouldNotGetUserUploads = "Failed to get user uploads: %s\n"
+
+// ErrorPathNotFound -
+var ErrorPathNotFound = "File path %s not found."

scraper.js

@@ -1,7 +1,7 @@
 optStrings = {
     selectors: {
         feedLoading: 'div.tiktok-loading.feed-loading',
-        modalArrowLeft: 'div.video-card-modal > div > img.arrow-right',
+        modalArrowRight: 'div.video-card-modal > div > img.arrow-right',
         modalClose: '.video-card-modal > div > div.close',
         modalPlayer: 'div > div > main > div.video-card-modal > div > div.video-card-big > div.video-card-container > div > div > video',
         modalShareInput: '.copy-link-container > input',
@@ -17,6 +17,7 @@ optStrings = {
     classes: {
         feedVideoItem: 'video-feed-item-wrapper',
         modalCloseDisabled: 'disabled',
+        titleMessage: 'title',
     },
     tags: {
         resultTag: 'video_urls',
@@ -25,13 +26,38 @@ optStrings = {
     attributes: {
         src: "src",
     },
+    tiktokMessages: [
+        "Couldn't find this account",
+        "No videos yet",
+        "Video currently unavailable",
+    ],
+};
+
+currentState = {
+    preloadCount: 0,
+    finished: false,
+    limit: 0
+};
+
+checkForErrors = function() {
+    var titles = document.getElementsByClassName(optStrings.classes.titleMessage);
+    debugger;
+    if (titles && titles.length) {
+        var error = Array.from(titles).find(x => optStrings.tiktokMessages.includes(x.textContent)).textContent;
+        if (error) {
+            createVidUrlElement("ERR: " + error);
+            return true;
+        }
+    }
+    return false;
 };
 
 createVidUrlElement = function(outputObj) {
     var urlSetElement = document.createElement(optStrings.tags.resultTag);
     urlSetElement.innerText = JSON.stringify(outputObj);
     document.getElementsByTagName(optStrings.tags.resultParentTag)[0].appendChild(urlSetElement);
-}
+    currentState.finished = true;
+};
 
 buldVidUrlArray = function(finishCallback) {
     var feedItem = document.getElementsByClassName(optStrings.classes.feedVideoItem)[0];
@@ -40,8 +66,14 @@ buldVidUrlArray = function(finishCallback) {
     var videoArray = [];
     var intervalID = window.setInterval(x => {
         videoArray.push(getCurrentModalVideo());
-
-        var arrowRight = document.querySelectorAll(optStrings.selectors.modalArrowLeft)[0];
+        if(currentState.limit > 0) {
+            if (videoArray.length >= currentState.limit) {
+                window.clearInterval(intervalID);
+                document.querySelector(optStrings.selectors.modalClose).click();
+                finishCallback(videoArray);
+            }
+        }
+        var arrowRight = document.querySelectorAll(optStrings.selectors.modalArrowRight)[0];
         if (arrowRight.classList.contains(optStrings.classes.modalCloseDisabled)) {
             window.clearInterval(intervalID);
             document.querySelector(optStrings.selectors.modalClose).click();
@@ -72,9 +104,10 @@ getCurrentModalVideo = function() {
             link: soundHref,
         },
     };
-}
+};
 
 getCurrentVideo = function() {
+    if(checkForErrors()) return;
     var player = document.querySelector(optStrings.selectors.videoPlayer);
     var vidUrl = player.getAttribute(optStrings.attributes.src);
     var shareLink = document.querySelector(optStrings.selectors.videoShareInput).value;
@@ -94,18 +127,32 @@ getCurrentVideo = function() {
             link: soundHref,
         },
     };
-}
+};
+
+scrollBottom = () => window.scrollTo(0, document.body.scrollHeight);
 
 scrollWhileNew = function(finishCallback) {
     var state = { count: 0 };
     var intervalID = window.setInterval(x => {
+        scrollBottom();
         var oldCount = state.count;
         state.count = document.getElementsByClassName(optStrings.classes.feedVideoItem).length;
+        if(currentState.limit > 0) {
+            if (currentState.preloadCount >= currentState.limit || state.count >= currentState.limit) {
+                finishCallback(createVidUrlElement);
+                window.clearInterval(intervalID);
+            }
+        }
+        if(checkForErrors()) {
+            window.clearInterval(intervalID);
+            return;
+        } else if (state.count == 0) {
+            return;
+        }
         if (oldCount !== state.count) {
-            window.scrollTo(0, document.body.scrollHeight);
+            currentState.preloadCount = state.count;
         } else {
             if (document.querySelector(optStrings.selectors.feedLoading)) {
-                window.scrollTo(0, document.body.scrollHeight);
                 return;
             }
             window.clearInterval(intervalID);
@@ -114,7 +161,8 @@ scrollWhileNew = function(finishCallback) {
     }, 1000);
 };
 
-bootstrapIteratingVideos = function() {
+bootstrapIteratingVideos = function(limit) {
+    currentState.limit = limit;
     scrollWhileNew(buldVidUrlArray);
     return 'bootstrapIteratingVideos';
 };
@@ -123,7 +171,7 @@ bootstrapGetCurrentVideo = function() {
     var video = getCurrentVideo();
     createVidUrlElement(video);
     return 'bootstrapGetCurrentVideo';
-}
+};
 
 init = () => {
     const newProto = navigator.__proto__;

utils/archive.go

@@ -0,0 +1,54 @@
+package utils
+
+import (
+	models "../models"
+	config "../models/config"
+	fileio "./fileio"
+	log "./log"
+)
+
+// IsItemInArchive - Checks if the item is already archived
+func IsItemInArchive(upload models.Upload) bool {
+	if len(RemoveArchivedItems([]models.Upload{upload})) == 0 {
+		return true
+	}
+	return false
+}
+
+// RemoveArchivedItems - Returns items slice without archived items
+func RemoveArchivedItems(uploads []models.Upload) []models.Upload {
+	archiveFilePath := config.Config.ArchiveFilePath
+
+	if archiveFilePath == "" || !fileio.CheckIfExists(archiveFilePath) {
+		return uploads
+	}
+
+	removeArchivedItemsDelegate := func(archivedItem string) {
+		for i, upload := range uploads {
+			if upload.GetUploadID() == archivedItem {
+				uploads = append(uploads[:i], uploads[i+1:]...)
+			}
+		}
+	}
+
+	lenBeforeRemoval := len(uploads)
+	fileio.ReadFileLineByLine(archiveFilePath, removeArchivedItemsDelegate)
+
+	removedCount := lenBeforeRemoval - len(uploads)
+	if removedCount > 0 {
+		log.Logf("%d items, found in archive. Skipping...\n", removedCount)
+	}
+
+	return uploads
+}
+
+// AddItemToArchive - Adds item to archived list
+func AddItemToArchive(uploadID string) {
+	archiveFilePath := config.Config.ArchiveFilePath
+
+	if archiveFilePath == "" {
+		return
+	}
+
+	fileio.AppendToFile(uploadID, archiveFilePath)
+}

utils/checkErr/checkErr.go

@@ -0,0 +1,12 @@
+package utils
+
+import (
+	"log"
+)
+
+// CheckErr - Checks if error and log
+func CheckErr(err error) {
+	if err != nil {
+		log.Fatal(err)
+	}
+}

utils/downloadFile.go

@@ -4,28 +4,23 @@ import (
 	"io"
 	"net/http"
 	"os"
+
+	checkErr "./checkErr"
 )
 
 // DownloadFile - Downloads content from `url` and stores it in `outputPath`
 func DownloadFile(outputPath string, url string) {
 	// Get the data
 	resp, err := http.Get(url)
-	if err != nil {
-		panic(err)
-	}
+	checkErr.CheckErr(err)
 	defer resp.Body.Close()
 
 	// Create the file
 	out, err := os.Create(outputPath)
-	if err != nil {
-		panic(err)
-	}
+	checkErr.CheckErr(err)
 	defer out.Close()
 
 	// Write the body to file
 	_, err = io.Copy(out, resp.Body)
-
-	if err != nil {
-		panic(err)
-	}
+	checkErr.CheckErr(err)
 }

utils/fileio/fileio.go

@@ -4,6 +4,8 @@ import (
 	"bufio"
 	"io/ioutil"
 	"os"
+
+	checkErr "../checkErr"
 )
 
 type delegateString func(string)
@@ -37,9 +39,7 @@ func ReadFileToString(path string) string {
 // ReadFileLineByLine - Reads file line by line and calls delegate
 func ReadFileLineByLine(path string, delegate delegateString) {
 	file, err := os.Open(path)
-	if err != nil {
-		panic(err)
-	}
+	checkErr.CheckErr(err)
 	defer file.Close()
 
 	scanner := bufio.NewScanner(file)
@@ -51,3 +51,14 @@ func ReadFileLineByLine(path string, delegate delegateString) {
 		panic(err)
 	}
 }
+
+// AppendToFile - Appends line to file
+func AppendToFile(str string, filePath string) {
+	f, err := os.OpenFile(filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	checkErr.CheckErr(err)
+
+	defer f.Close()
+	if _, err := f.WriteString(str + "\n"); err != nil {
+		checkErr.CheckErr(err)
+	}
+}

utils/getHashtag.go

@@ -0,0 +1,16 @@
+package utils
+
+import (
+	res "../resources"
+	"fmt"
+	"strings"
+)
+
+// GetHashtagFromURL - Get's tag name from passed url
+func GetHashtagFromURL(str string) string {
+	if match := strings.Contains(str, "/tag/"); match {
+		return strings.Split(str, "/tag/")[1]
+	}
+
+	panic(fmt.Sprintf(res.ErrorCouldNotRecogniseURL, str))
+}

utils/getUsername.go

@@ -0,0 +1,28 @@
+package utils
+
+import (
+	config "../models/config"
+	res "../resources"
+	"fmt"
+	"regexp"
+	"strings"
+)
+
+// GetUsername - Get's username from passed URL param
+func GetUsername() string {
+	return GetUsernameFromString(config.Config.URL)
+}
+
+// GetUsernameFromString - Get's username from passed param
+func GetUsernameFromString(str string) string {
+	if match := strings.Contains(str, "/"); !match { // Not url
+		return strings.Replace(str, "@", "", -1)
+	}
+
+	if match, _ := regexp.MatchString(".+tiktok\\.com/@.+", str); match { // URL
+		stripedSuffix := strings.Split(str, "@")[1]
+		return strings.Split(stripedSuffix, "/")[0]
+	}
+
+	panic(fmt.Sprintf(res.ErrorCouldNotRecogniseURL, str))
+}

utils/getUsername_test.go

@@ -1,6 +1,7 @@
-package models
+package utils
 
 import (
+	config "../models/config"
 	testUtil "../unitTestUtil"
 	"testing"
 )
@@ -8,7 +9,7 @@ import (
 func TestGetUsername(t *testing.T) {
 	testCaseDelegate := func(t *testing.T, url string, username string) {
 		tu := testUtil.TestUtil{T: t}
-		Config.URL = url
+		config.Config.URL = url
 		actual := GetUsername()
 		tu.AssertString(actual, username, "Username")
 	}

utils/log/log.go

@@ -0,0 +1,32 @@
+package utils
+
+import (
+	"fmt"
+	"os"
+
+	config "../../models/config"
+)
+
+// Log - Write to std out
+func Log(a ...interface{}) {
+	if !config.Config.Quiet {
+		fmt.Println(a...)
+	}
+}
+
+// Logf - Write formated text
+func Logf(format string, a ...interface{}) {
+	if !config.Config.Quiet {
+		fmt.Printf(format, a...)
+	}
+}
+
+// LogFatal - Write error and panic
+func LogFatal(format string, a ...interface{}) {
+	panic(fmt.Sprintf(format, a...))
+}
+
+// LogErr - Write error
+func LogErr(format string, a ...interface{}) {
+	fmt.Fprintf(os.Stderr, format, a...)
+}

utils/readFileAsString.go

@@ -2,14 +2,13 @@ package utils
 
 import (
 	"io/ioutil"
-	"log"
+
+	checkErr "./checkErr"
 )
 
 // ReadFileAsString - Returns contents of given file
 func ReadFileAsString(fileName string) string {
 	content, err := ioutil.ReadFile(fileName)
-	if err != nil {
-		log.Fatal(err)
-	}
+	checkErr.CheckErr(err)
 	return string(content)
 }

workflows/downloadBatchFile.go

@@ -1,8 +1,9 @@
 package workflows
 
 import (
-	utils "../utils"
-	"fmt"
+	res "../resources"
+	fileio "../utils/fileio"
+	log "../utils/log"
 )
 
 // CanUseDownloadBatchFile - Check's if DownloadBatchFile can be used
@@ -12,11 +13,11 @@ func CanUseDownloadBatchFile(batchFilePath string) bool {
 
 // DownloadBatchFile - Download items from batch file
 func DownloadBatchFile(batchFilePath string) {
-	if !utils.CheckIfExists(batchFilePath) {
-		panic(fmt.Sprintf("File path %s not found.", batchFilePath))
+	if !fileio.CheckIfExists(batchFilePath) {
+		log.LogFatal(res.ErrorPathNotFound, batchFilePath)
 	}
 
-	utils.ReadFileLineByLine(batchFilePath, downloadItem)
+	fileio.ReadFileLineByLine(batchFilePath, downloadItem)
 }
 
 func downloadItem(batchItem string) {

workflows/downloadHashtag.go

@@ -0,0 +1,52 @@
+package workflows
+
+import (
+	"fmt"
+	"strings"
+
+	client "../client"
+	config "../models/config"
+	res "../resources"
+	utils "../utils"
+	fileio "../utils/fileio"
+	log "../utils/log"
+)
+
+// CanUseDownloadHashtag - Test's if this workflow can be used for parameter
+func CanUseDownloadHashtag(url string) bool {
+	match := strings.Contains(url, "/tag/")
+	return match
+}
+
+// DownloadHashtag - Download videos marked with given hashtag
+func DownloadHashtag(url string) {
+	uploads, err := client.GetHashtagUploads(url)
+	if err != nil {
+		log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
+		return
+	}
+
+	uploads = utils.RemoveArchivedItems(uploads)
+	uploadCount := len(uploads)
+
+	hashtag := utils.GetHashtagFromURL(url)
+	downloadDir := fmt.Sprintf("%s/%s", config.Config.OutputPath, hashtag)
+
+	fileio.InitOutputDirectory(downloadDir)
+
+	for index, upload := range uploads {
+		downloadVideo(upload, downloadDir)
+		log.Logf("\r[%d/%d] Downloaded", index+1, uploadCount)
+	}
+	log.Log()
+}
+
+// GetHashtagJSON - Prints scraped info from hashtag
+func GetHashtagJSON(url string) {
+	uploads, err := client.GetHashtagUploadsJSON(url)
+	if err != nil {
+		log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
+		return
+	}
+	fmt.Printf("%s", uploads)
+}

workflows/downloadMusic.go

@@ -1,11 +1,15 @@
 package workflows
 
 import (
-	client "../client"
-	models "../models"
-	utils "../utils"
 	"fmt"
 	"regexp"
+
+	client "../client"
+	config "../models/config"
+	res "../resources"
+	utils "../utils"
+	fileio "../utils/fileio"
+	log "../utils/log"
 )
 
 // CanUseDownloadMusic - Check's if DownloadMusic can be used for parameter
@@ -16,13 +20,32 @@ func CanUseDownloadMusic(url string) bool {
 
 // DownloadMusic - Download all videos by given music
 func DownloadMusic(url string) {
-	uploads := client.GetMusicUploads(url)
-
-	for _, upload := range uploads {
-		username := models.GetUsernameFromString(upload.Uploader)
-		downloadDir := fmt.Sprintf("%s/%s", models.Config.OutputPath, username)
-
-		utils.InitOutputDirectory(downloadDir)
-		downloadVideo(upload, downloadDir)
+	uploads, err := client.GetMusicUploads(url)
+	if err != nil {
+		log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
+		return
 	}
+
+	uploads = utils.RemoveArchivedItems(uploads)
+	uploadCount := len(uploads)
+
+	for index, upload := range uploads {
+		username := utils.GetUsernameFromString(upload.Uploader)
+		downloadDir := fmt.Sprintf("%s/%s", config.Config.OutputPath, username)
+
+		fileio.InitOutputDirectory(downloadDir)
+		downloadVideo(upload, downloadDir)
+		log.Logf("\r[%d/%d] Downloaded", index+1, uploadCount)
+	}
+	log.Log()
+}
+
+// GetMusicJSON - Prints scraped info from music
+func GetMusicJSON(url string) {
+	uploads, err := client.GetMusicUploadsJSON(url)
+	if err != nil {
+		log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
+		return
+	}
+	fmt.Printf("%s", uploads)
 }

workflows/downloadShareLink.go

@@ -0,0 +1,27 @@
+package workflows
+
+import (
+	client "../client"
+	res "../resources"
+	log "../utils/log"
+	"regexp"
+)
+
+// CanUseDownloadShareLink - Check's if DownloadShareLink can be used
+func CanUseDownloadShareLink(url string) bool {
+	match, _ := regexp.MatchString("vm.tiktok.com\\/.+", url)
+	return match
+}
+
+// DownloadShareLink - Download item by share link
+func DownloadShareLink(url string) {
+	log.Logf("Resolving share link: %s\n", url)
+
+	finalURL, err := client.GetRedirectUrl(url)
+	if err != nil {
+		log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
+		return
+	}
+
+	StartWorkflowByParameter(finalURL)
+}

workflows/downloadUser.go

@@ -1,27 +1,53 @@
 package workflows
 
 import (
-	client "../client"
-	models "../models"
-	utils "../utils"
 	"fmt"
+	"regexp"
 	"strings"
+
+	client "../client"
+	config "../models/config"
+	res "../resources"
+	utils "../utils"
+	fileio "../utils/fileio"
+	log "../utils/log"
 )
 
 // CanUseDownloadUser - Test's if this workflow can be used for parameter
 func CanUseDownloadUser(url string) bool {
-	match := strings.Contains(url, "/")
-	return !match
+	isURL := strings.Contains(url, "/")
+	match, _ := regexp.MatchString(".+com\\/@[^\\/]+", url)
+	return !isURL || match
 }
 
 // DownloadUser - Download all user's videos
 func DownloadUser(username string) {
-	downloadDir := fmt.Sprintf("%s/%s", models.Config.OutputPath, username)
-	uploads := client.GetUserUploads(username)
-
-	utils.InitOutputDirectory(downloadDir)
-
-	for _, upload := range uploads {
-		downloadVideo(upload, downloadDir)
+	uploads, err := client.GetUserUploads(username)
+	if err != nil {
+		log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
+		return
 	}
+
+	uploads = utils.RemoveArchivedItems(uploads)
+	uploadCount := len(uploads)
+
+	downloadDir := fmt.Sprintf("%s/%s", config.Config.OutputPath, username)
+
+	fileio.InitOutputDirectory(downloadDir)
+
+	for index, upload := range uploads {
+		downloadVideo(upload, downloadDir)
+		log.Logf("\r[%d/%d] Downloaded", index+1, uploadCount)
+	}
+	log.Log()
+}
+
+// GetUserVideosJSON - Prints scraped info from user
+func GetUserVideosJSON(username string) {
+	uploads, err := client.GetUserUploadsJSON(username)
+	if err != nil {
+		log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
+		return
+	}
+	fmt.Printf("%s", uploads)
 }

workflows/downloadVideo.go

@@ -1,11 +1,16 @@
 package workflows
 
 import (
-	client "../client"
-	models "../models"
-	utils "../utils"
 	"fmt"
 	"regexp"
+
+	client "../client"
+	models "../models"
+	config "../models/config"
+	res "../resources"
+	utils "../utils"
+	fileio "../utils/fileio"
+	log "../utils/log"
 )
 
 // CanUseDownloadSingleVideo - Check's if DownloadSingleVideo can be used for parameter
@@ -16,12 +21,21 @@ func CanUseDownloadSingleVideo(url string) bool {
 
 // DownloadSingleVideo - Downloads single video
 func DownloadSingleVideo(url string) {
-	username := models.GetUsernameFromString(url)
-	upload := client.GetVideoDetails(url)
-	downloadDir := fmt.Sprintf("%s/%s", models.Config.OutputPath, username)
+	username := utils.GetUsernameFromString(url)
+	upload, err := client.GetVideoDetails(url)
+	if err != nil {
+		log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
+		return
+	}
 
-	utils.InitOutputDirectory(downloadDir)
+	if utils.IsItemInArchive(upload) {
+		return
+	}
+	downloadDir := fmt.Sprintf("%s/%s", config.Config.OutputPath, username)
+
+	fileio.InitOutputDirectory(downloadDir)
 	downloadVideo(upload, downloadDir)
+	log.Log("[1/1] Downloaded\n")
 }
 
 // DownloadVideo - Downloads one video
@@ -29,16 +43,16 @@ func downloadVideo(upload models.Upload, downloadDir string) {
 	uploadID := upload.GetUploadID()
 	downloadPath := fmt.Sprintf("%s/%s.mp4", downloadDir, uploadID)
 
-	if utils.CheckIfExists(downloadPath) {
-		fmt.Println("Upload '" + uploadID + "' already downloaded, skipping")
+	if fileio.CheckIfExists(downloadPath) {
 		return
 	}
 
-	fmt.Println("Downloading upload item '" + uploadID + "' to " + downloadPath)
 	utils.DownloadFile(downloadPath, upload.URL)
 
-	if models.Config.MetaData {
+	if config.Config.MetaData {
 		metadataPath := fmt.Sprintf("%s/%s.json", downloadDir, uploadID)
 		upload.WriteToFile(metadataPath)
 	}
+
+	utils.AddItemToArchive(upload.GetUploadID())
 }

workflows/startWorkflowByParameter.go

@@ -1,8 +1,10 @@
 package workflows
 
 import (
-	models "../models"
-	"fmt"
+	config "../models/config"
+	res "../resources"
+	utils "../utils"
+	log "../utils/log"
 )
 
 // StartWorkflowByParameter - Start needed workflow by given parameter
@@ -10,7 +12,11 @@ func StartWorkflowByParameter(url string) {
 
 	// Music
 	if CanUseDownloadMusic(url) {
-		DownloadMusic(url)
+		if config.Config.JSONOnly {
+			GetMusicJSON(url)
+		} else {
+			DownloadMusic(url)
+		}
 		return
 	}
 
@@ -22,9 +28,30 @@ func StartWorkflowByParameter(url string) {
 
 	// Tiktok user
 	if CanUseDownloadUser(url) {
-		DownloadUser(models.GetUsernameFromString(url))
+		if config.Config.JSONOnly {
+			GetUserVideosJSON(utils.GetUsernameFromString(url))
+		} else {
+			DownloadUser(utils.GetUsernameFromString(url))
+		}
+
 		return
 	}
 
-	panic(fmt.Sprintf("Could not recognise URL format of string %s", url))
+	// Tiktok hashtag
+	if CanUseDownloadHashtag(url) {
+		if config.Config.JSONOnly {
+			GetHashtagJSON(url)
+		} else {
+			DownloadHashtag(url)
+		}
+		return
+	}
+
+	// Share URL
+	if CanUseDownloadShareLink(url) {
+		DownloadShareLink(url)
+		return
+	}
+
+	log.LogFatal(res.ErrorCouldNotRecogniseURL, url)
 }