TTDL-5 Added better error handling

Added `-quiet` flag

Move out error messages to separate file

.github/workflows/go.yml

@@ -15,30 +15,35 @@ jobs:
         go-version: 1.13
       id: go
 
+    - name: Set up node 10
+      uses: actions/setup-node@v1
+      with:
+        node-version: '10.x'
+
     - name: Check out code into the Go module directory
       uses: actions/checkout@v1
 
     - name: Get dependencies
       run: |
-        go get -v -t -d ./...
+        npm install
+        npm run install-dependencies
+
+    - name: Run unit tests
+      run: npm run test
 
     - name: Build
-      run: go build -v .
-
-    - name: Run tests
-      run: |
-        go test -v ./models
+      run: npm run build:dist
 
     - name: Upload Unix Artifacts
       if: startsWith(matrix.os, 'ubuntu-')
       uses: actions/upload-artifact@v1
       with:
         name: tiktok-dl_linux
-        path: tiktok-dl
+        path: out
         
     - name: Upload Windows Artifacts
       if: startsWith(matrix.os, 'windows-')
       uses: actions/upload-artifact@v1
       with:
         name: tiktok-dl_win64
-        path: tiktok-dl.exe
+        path: out

.gitignore

@@ -1,5 +1,8 @@
 .vscode
+node_modules
 __debug_bin
 downloads
 *.exe
 tiktok-dl
+batch_file.txt
+debug.log

README.md

@@ -1,17 +1,28 @@
 # TikTok-DL
 
 [![Go Report Card](https://goreportcard.com/badge/github.com/pikami/tiktok-dl)](https://goreportcard.com/report/github.com/pikami/tiktok-dl)
+![GitHub Workflow Status](https://img.shields.io/github/workflow/status/pikami/tiktok-dl/tiktok-dl_CI)
 
 A simple tiktok video downloader written in go
 
 ## Basic usage
-Clone this repository and run `go build` to build the executable.\
+Download the executable from `https://github.com/pikami/tiktok-dl/releases`\
 You can download all videos from user by running `./tiktok-dl [Options] TIKTOK_USERNAME`\
-You can download single video by running `./tiktok-dl [Options] VIDEO_URL`
+You can download single video by running `./tiktok-dl [Options] VIDEO_URL`\
+You can download all videos by music by running `./tiktok-dl [Options] MUSIC_URL`\
+You can download items listed in a text file by running `./tiktok-dl [OPTIONS] -batch-file path/to/items.txt`
+
+## Build instructions
+Clone this repository and run `go build` to build the executable.
 
 ## Available options
 * `-debug` - enables debug mode
 * `-output some_directory` - Output path (default "./downloads")
+* `-metadata` - Write video metadata to a .json file
+* `-batch-file` - File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored.
+* `-deadline` - Sets the timout for scraper logic in seconds (used as a workaround for context deadline exceeded error) (default 1500)
+* `-quiet` - Supress output
 
 ## Acknowledgments
-This software uses the chromedp for web scraping, it can be found here: https://github.com/chromedp/chromedp
+This software uses the **chromedp** for web scraping, it can be found here: https://github.com/chromedp/chromedp \
+For releases the JS code is minified by using **terser** toolkit, it can be found here: https://github.com/terser/terser

client/executeClientAction.go

@@ -0,0 +1,114 @@
+package client
+
+import (
+	"context"
+	"errors"
+	"github.com/chromedp/chromedp"
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+	"time"
+
+	config "../models/config"
+	utils "../utils"
+)
+
+// GetMusicUploads - Get all uploads by given music
+func executeClientAction(url string, jsAction string) (string, error) {
+	dir, err := ioutil.TempDir("", "chromedp-example")
+	if err != nil {
+		return "", err
+	}
+	defer os.RemoveAll(dir)
+
+	opts := append(chromedp.DefaultExecAllocatorOptions[:],
+		chromedp.DisableGPU,
+		chromedp.UserDataDir(dir),
+		chromedp.Flag("headless", !config.Config.Debug),
+	)
+
+	allocCtx, cancel := chromedp.NewExecAllocator(context.Background(), opts...)
+	defer cancel()
+
+	ctx, cancel := chromedp.NewContext(
+		allocCtx,
+		chromedp.WithLogf(log.Printf),
+	)
+	defer cancel()
+
+	ctx, cancel = context.WithTimeout(ctx, time.Duration(config.Config.Deadline)*time.Second)
+	defer cancel()
+
+	jsOutput, err := runScrapeWithInfo(ctx, jsAction, url)
+	if strings.HasPrefix(jsOutput, "\"ERR:") {
+		err = errors.New(jsOutput)
+	}
+	return jsOutput, err
+}
+
+func runScrapeQuiet(ctx context.Context, jsAction string, url string) (string, error) {
+	var jsOutput string
+	if err := chromedp.Run(ctx,
+		// Navigate to user's page
+		chromedp.Navigate(url),
+		// Execute url grabber script
+		chromedp.EvaluateAsDevTools(utils.ReadFileAsString("scraper.js"), &jsOutput),
+		chromedp.EvaluateAsDevTools(jsAction, &jsOutput),
+		// Wait until custom js finishes
+		chromedp.WaitVisible(`video_urls`),
+		// Grab url links from our element
+		chromedp.InnerHTML(`video_urls`, &jsOutput),
+	); err != nil {
+		return "", err
+	}
+
+	return jsOutput, nil
+}
+
+func runScrapeWithInfo(ctx context.Context, jsAction string, url string) (string, error) {
+	var jsOutput string
+	if err := chromedp.Run(ctx,
+		// Navigate to user's page
+		chromedp.Navigate(url),
+		// Execute url grabber script
+		chromedp.EvaluateAsDevTools(utils.ReadFileAsString("scraper.js"), &jsOutput),
+		chromedp.EvaluateAsDevTools(jsAction, &jsOutput),
+	); err != nil {
+		return "", err
+	}
+
+	for {
+		if err := chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.preloadCount.toString()", &jsOutput)); err != nil {
+			return "", err
+		}
+
+		if jsOutput != "0" {
+			utils.Logf("\rPreloading... Currently loaded %s items.", jsOutput)
+		} else {
+			utils.Logf("\rPreloading...")
+		}
+
+		if err := chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.finished.toString()", &jsOutput)); err != nil {
+			return "", err
+		}
+
+		if jsOutput == "true" {
+			break
+		}
+
+		time.Sleep(50 * time.Millisecond)
+	}
+
+	utils.Log("\nRetrieving items...")
+	if err := chromedp.Run(ctx,
+		// Wait until custom js finishes
+		chromedp.WaitVisible(`video_urls`),
+		// Grab url links from our element
+		chromedp.InnerHTML(`video_urls`, &jsOutput),
+	); err != nil {
+		return "", err
+	}
+
+	return jsOutput, nil
+}

client/getMusicUploads.go

@@ -0,0 +1,14 @@
+package client
+
+import (
+	models "../models"
+)
+
+// GetMusicUploads - Get all uploads by given music
+func GetMusicUploads(url string) ([]models.Upload, error) {
+	actionOutput, err := executeClientAction(url, "bootstrapIteratingVideos()")
+	if err != nil {
+		return nil, err
+	}
+	return models.ParseUploads(actionOutput), nil
+}

client/getUserUploads.go

@@ -1,58 +1,14 @@
 package client
 
 import (
-	"context"
-	"github.com/chromedp/chromedp"
-	"io/ioutil"
-	"log"
-	"os"
-	"time"
-
 	models "../models"
-	utils "../utils"
 )
 
 // GetUserUploads - Get all uploads by user
-func GetUserUploads(username string) []models.Upload {
-	dir, err := ioutil.TempDir("", "chromedp-example")
+func GetUserUploads(username string) ([]models.Upload, error) {
+	actionOutput, err := executeClientAction(`https://www.tiktok.com/@`+username, "bootstrapIteratingVideos()")
 	if err != nil {
-		panic(err)
+		return nil, err
 	}
-	defer os.RemoveAll(dir)
-
-	opts := append(chromedp.DefaultExecAllocatorOptions[:],
-		chromedp.DisableGPU,
-		chromedp.UserDataDir(dir),
-		chromedp.Flag("headless", !models.Config.Debug),
-	)
-
-	allocCtx, cancel := chromedp.NewExecAllocator(context.Background(), opts...)
-	defer cancel()
-
-	ctx, cancel := chromedp.NewContext(
-		allocCtx,
-		chromedp.WithLogf(log.Printf),
-	)
-	defer cancel()
-
-	ctx, cancel = context.WithTimeout(ctx, 1500*time.Second)
-	defer cancel()
-
-	var jsOutput string
-	err = chromedp.Run(ctx,
-		// Navigate to user's page
-		chromedp.Navigate(`https://www.tiktok.com/@`+username),
-		// Execute url grabber script
-		chromedp.EvaluateAsDevTools(utils.ReadFileAsString("scraper.js"), &jsOutput),
-		chromedp.EvaluateAsDevTools("bootstrapIteratingVideos()", &jsOutput),
-		// Wait until custom js finishes
-		chromedp.WaitVisible(`video_urls`),
-		// Grab url links from our element
-		chromedp.InnerHTML(`video_urls`, &jsOutput),
-	)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	return models.ParseUploads(jsOutput)
+	return models.ParseUploads(actionOutput), nil
 }

client/getVideoDetails.go

@@ -1,58 +1,14 @@
 package client
 
 import (
-	"context"
-	"github.com/chromedp/chromedp"
-	"io/ioutil"
-	"log"
-	"os"
-	"time"
-
 	models "../models"
-	utils "../utils"
 )
 
 // GetVideoDetails - returns details of video
-func GetVideoDetails(videoURL string) models.Upload {
-	dir, err := ioutil.TempDir("", "chromedp-example")
+func GetVideoDetails(videoURL string) (models.Upload, error) {
+	actionOutput, err := executeClientAction(videoURL, "bootstrapGetCurrentVideo()")
 	if err != nil {
-		panic(err)
+		return models.Upload{}, err
 	}
-	defer os.RemoveAll(dir)
-
-	opts := append(chromedp.DefaultExecAllocatorOptions[:],
-		chromedp.DisableGPU,
-		chromedp.UserDataDir(dir),
-		chromedp.Flag("headless", !models.Config.Debug),
-	)
-
-	allocCtx, cancel := chromedp.NewExecAllocator(context.Background(), opts...)
-	defer cancel()
-
-	ctx, cancel := chromedp.NewContext(
-		allocCtx,
-		chromedp.WithLogf(log.Printf),
-	)
-	defer cancel()
-
-	ctx, cancel = context.WithTimeout(ctx, 1500*time.Second)
-	defer cancel()
-
-	var jsOutput string
-	err = chromedp.Run(ctx,
-		// Navigate to user's page
-		chromedp.Navigate(videoURL),
-		// Execute url grabber script
-		chromedp.EvaluateAsDevTools(utils.ReadFileAsString("scraper.js"), &jsOutput),
-		chromedp.EvaluateAsDevTools("bootstrapGetCurrentVideo()", &jsOutput),
-		// Wait until custom js finishes
-		chromedp.WaitVisible(`video_urls`),
-		// Grab url links from our element
-		chromedp.InnerHTML(`video_urls`, &jsOutput),
-	)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	return models.ParseUpload(jsOutput)
+	return models.ParseUpload(actionOutput), nil
 }

main.go

@@ -1,57 +1,20 @@
 package main
 
 import (
-	client "./client"
-	models "./models"
-	utils "./utils"
-	"fmt"
-	"regexp"
-	"strings"
+	config "./models/config"
+	workflows "./workflows"
 )
 
 func main() {
-	models.GetConfig()
-	url := models.Config.URL
+	config.GetConfig()
+	url := config.Config.URL
+	batchFilePath := config.Config.BatchFilePath
 
-	// Single video
-	match, _ := regexp.MatchString("\\/@.+\\/video\\/[0-9]+", url)
-	if match {
-		getUsernameFromVidURLRegex, _ := regexp.Compile("com\\/@.*")
-		parts := strings.Split(getUsernameFromVidURLRegex.FindString(url), "/")
-		username := parts[1][1:]
-		upload := client.GetVideoDetails(url)
-		downloadDir := fmt.Sprintf("%s/%s", models.Config.OutputPath, username)
-
-		utils.InitOutputDirectory(downloadDir)
-		downloadVideo(upload, downloadDir)
+	// Batch file
+	if workflows.CanUseDownloadBatchFile(batchFilePath) {
+		workflows.DownloadBatchFile(batchFilePath)
 		return
 	}
 
-	// Tiktok user
-	downloadUser()
-}
-
-func downloadVideo(upload models.Upload, downloadDir string) {
-	uploadID := upload.GetUploadID()
-	downloadPath := fmt.Sprintf("%s/%s.mp4", downloadDir, uploadID)
-
-	if utils.CheckIfExists(downloadPath) {
-		fmt.Println("Upload '" + uploadID + "' already downloaded, skipping")
-		return
-	}
-
-	fmt.Println("Downloading upload item '" + uploadID + "' to " + downloadPath)
-	utils.DownloadFile(downloadPath, upload.URL)
-}
-
-func downloadUser() {
-	username := models.Config.URL
-	downloadDir := fmt.Sprintf("%s/%s", models.Config.OutputPath, username)
-	uploads := client.GetUserUploads(username)
-
-	utils.InitOutputDirectory(downloadDir)
-
-	for _, upload := range uploads {
-		downloadVideo(upload, downloadDir)
-	}
+	workflows.StartWorkflowByParameter(url)
 }

models/config.go

@@ -1,31 +0,0 @@
-package models
-
-import (
-	"flag"
-	"fmt"
-	"os"
-)
-
-// Config - Runtime configuration
-var Config struct {
-	URL        string
-	OutputPath string
-	Debug      bool
-}
-
-// GetConfig - Returns Config object
-func GetConfig() {
-	outputPath := flag.String("output", "./downloads", "Output path")
-	debug := flag.Bool("debug", false, "enables debug mode")
-	flag.Parse()
-
-	args := flag.Args()
-	if len(args) < 1 {
-		fmt.Println("Usage: tiktok-dl [OPTIONS] TIKTOK_USERNAME|TIKTOK_URL")
-		os.Exit(2)
-	}
-
-	Config.URL = flag.Args()[len(args)-1]
-	Config.OutputPath = *outputPath
-	Config.Debug = *debug
-}

models/config/config.go

@@ -0,0 +1,48 @@
+package config
+
+import (
+	"flag"
+	"fmt"
+	"os"
+)
+
+// Config - Runtime configuration
+var Config struct {
+	URL           string
+	OutputPath    string
+	BatchFilePath string
+	Debug         bool
+	MetaData      bool
+	Quiet         bool
+	Deadline      int
+}
+
+// GetConfig - Returns Config object
+func GetConfig() {
+	outputPath := flag.String("output", "./downloads", "Output path")
+	batchFilePath := flag.String("batch-file", "", "File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored.")
+	debug := flag.Bool("debug", false, "Enables debug mode")
+	metadata := flag.Bool("metadata", false, "Write video metadata to a .json file")
+	quiet := flag.Bool("quiet", false, "Supress output")
+	deadline := flag.Int("deadline", 1500, "Sets the timout for scraper logic in seconds (used as a workaround for 'context deadline exceeded' error)")
+	flag.Parse()
+
+	args := flag.Args()
+	if len(args) < 1 && *batchFilePath == "" {
+		fmt.Println("Usage: tiktok-dl [OPTIONS] TIKTOK_USERNAME|TIKTOK_URL")
+		fmt.Println("  or:  tiktok-dl [OPTIONS] -batch-file path/to/users.txt")
+		os.Exit(2)
+	}
+
+	if len(args) > 0 {
+		Config.URL = flag.Args()[len(args)-1]
+	} else {
+		Config.URL = ""
+	}
+	Config.OutputPath = *outputPath
+	Config.BatchFilePath = *batchFilePath
+	Config.Debug = *debug
+	Config.MetaData = *metadata
+	Config.Quiet = *quiet
+	Config.Deadline = *deadline
+}

models/upload.go

@@ -1,14 +1,26 @@
 package models
 
 import (
+	res "../resources"
+	utils "../utils"
 	"encoding/json"
+	"os"
 	"strings"
 )
 
 // Upload - Upload object
 type Upload struct {
-	ShareLink string `json:"shareLink"`
 	URL       string `json:"url"`
+	ShareLink string `json:"shareLink"`
+	Caption   string `json:"caption"`
+	Uploader  string `json:"uploader"`
+	Sound     Sound  `json:"sound"`
+}
+
+// Sound - Sound object
+type Sound struct {
+	Title string `json:"title"`
+	Link  string `json:"link"`
 }
 
 // ParseUploads - Parses json uploads array
@@ -30,3 +42,21 @@ func (u Upload) GetUploadID() string {
 	parts := strings.Split(u.ShareLink, "/")
 	return parts[len(parts)-1]
 }
+
+// WriteToFile - Writes object to file
+func (u Upload) WriteToFile(outputPath string) {
+	bytes, err := json.Marshal(u)
+	if err != nil {
+		utils.Logf(res.ErrorCouldNotSerializeJSON, u.GetUploadID())
+		panic(err)
+	}
+
+	// Create the file
+	out, err := os.Create(outputPath)
+	utils.CheckErr(err)
+	defer out.Close()
+
+	// Write to file
+	_, err = out.Write(bytes)
+	utils.CheckErr(err)
+}

models/upload_test.go

@@ -1,36 +1,69 @@
 package models
 
-import "testing"
+import (
+	testUtil "../unitTestUtil"
+	utils "../utils"
+	"os"
+	"testing"
+)
 
-// TestParseUploads - Test parsing
 func TestParseUploads(t *testing.T) {
-	jsonStr := "[{\"shareLink\":\"some_share_link\", \"url\": \"some_url\"}]"
+	tu := testUtil.TestUtil{T: t}
+	jsonStr := "[{\"url\":\"some_url\",\"shareLink\":\"some_share_link\",\"caption\":\"some_caption\", \"uploader\": \"some.uploader\",\"sound\":{\"title\":\"some_title\",\"link\":\"some_link\"}}]"
 	actual := ParseUploads(jsonStr)
 
-	expectedLen := 1
-	if len(actual) != expectedLen {
-		t.Errorf("Array len incorrect: Expected %d, but got %d", expectedLen, len(actual))
-	}
+	tu.AssertInt(len(actual), 1, "Array len")
 
-	expectedShareLink := "some_share_link"
-	if actual[0].ShareLink != expectedShareLink {
-		t.Errorf("ShareLink is incorrect: Expected %s, but got %s", expectedShareLink, actual[0].ShareLink)
-	}
+	tu.AssertString(actual[0].URL, "some_url", "URL")
+	tu.AssertString(actual[0].Caption, "some_caption", "Caption")
+	tu.AssertString(actual[0].ShareLink, "some_share_link", "ShareLink")
+	tu.AssertString(actual[0].Uploader, "some.uploader", "Uploader")
 
-	expectedURL := "some_url"
-	if actual[0].URL != expectedURL {
-		t.Errorf("URL is incorrect: Expected %s, but got %s", expectedURL, actual[0].URL)
-	}
+	tu.AssertString(actual[0].Sound.Link, "some_link", "Sound.Link")
+	tu.AssertString(actual[0].Sound.Title, "some_title", "Sound.Title")
+}
+
+func TestParseUpload(t *testing.T) {
+	tu := testUtil.TestUtil{T: t}
+	jsonStr := "{\"url\":\"some_url\",\"shareLink\":\"some_share_link\",\"caption\":\"some_caption\",\"sound\":{\"title\":\"some_title\",\"link\":\"some_link\"}}"
+	actual := ParseUpload(jsonStr)
+
+	tu.AssertString(actual.URL, "some_url", "URL")
+	tu.AssertString(actual.Caption, "some_caption", "Caption")
+	tu.AssertString(actual.ShareLink, "some_share_link", "ShareLink")
+
+	tu.AssertString(actual.Sound.Link, "some_link", "Sound.Link")
+	tu.AssertString(actual.Sound.Title, "some_title", "Sound.Title")
 }
 
 func TestGetUploadID(t *testing.T) {
+	tu := testUtil.TestUtil{T: t}
 	var upload Upload
 	upload.ShareLink = "http://pikami.org/some_thing/some_upload_id"
-	expected := "some_upload_id"
-
 	actual := upload.GetUploadID()
 
-	if actual != expected {
-		t.Errorf("UploadId is incorrect: Expected %s, but got %s", expected, actual)
-	}
+	tu.AssertString(actual, "some_upload_id", "Upload ID")
+}
+
+func TestWriteToFile(t *testing.T) {
+	tu := testUtil.TestUtil{T: t}
+	expected := "{\"url\":\"some_url\",\"shareLink\":\"some_share_link\",\"caption\":\"some_caption\",\"uploader\":\"some.uploader\",\"sound\":{\"title\":\"some_title\",\"link\":\"some_link\"}}"
+	filePath := "test_file.txt"
+	upload := Upload{
+		URL:       "some_url",
+		Caption:   "some_caption",
+		ShareLink: "some_share_link",
+		Uploader:  "some.uploader",
+		Sound: Sound{
+			Link:  "some_link",
+			Title: "some_title",
+		},
+	}
+
+	upload.WriteToFile(filePath)
+
+	actual := utils.ReadFileToString(filePath)
+	tu.AssertString(actual, expected, "File content")
+
+	os.Remove(filePath)
 }

package-lock.json

@@ -0,0 +1,42 @@
+{
+    "name": "tiktok-dl",
+    "version": "0.0.1",
+    "lockfileVersion": 1,
+    "requires": true,
+    "dependencies": {
+        "buffer-from": {
+            "version": "1.1.1",
+            "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz",
+            "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A=="
+        },
+        "commander": {
+            "version": "2.20.3",
+            "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
+            "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ=="
+        },
+        "source-map": {
+            "version": "0.6.1",
+            "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
+            "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
+        },
+        "source-map-support": {
+            "version": "0.5.16",
+            "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.16.tgz",
+            "integrity": "sha512-efyLRJDr68D9hBBNIPWFjhpFzURh+KJykQwvMyW5UiZzYwoF6l4YMMDIJJEyFWxWCqfyxLzz6tSfUFR+kXXsVQ==",
+            "requires": {
+                "buffer-from": "^1.0.0",
+                "source-map": "^0.6.0"
+            }
+        },
+        "terser": {
+            "version": "4.6.3",
+            "resolved": "https://registry.npmjs.org/terser/-/terser-4.6.3.tgz",
+            "integrity": "sha512-Lw+ieAXmY69d09IIc/yqeBqXpEQIpDGZqT34ui1QWXIUpR2RjbqEkT8X7Lgex19hslSqcWM5iMN2kM11eMsESQ==",
+            "requires": {
+                "commander": "^2.20.0",
+                "source-map": "~0.6.1",
+                "source-map-support": "~0.5.12"
+            }
+        }
+    }
+}

package.json

@@ -0,0 +1,16 @@
+{
+    "name": "tiktok-dl",
+    "version": "0.0.1",
+    "scripts": {
+        "install-dependencies": "go get -v -t -d ./...",
+        "test": "go test -v ./models && go test -v ./utils",
+        "clean": "rm -rf out",
+        "build:scraper": "node node_modules/terser/bin/terser -c -m -- scraper.js > out/scraper.js",
+        "build:app": "go build -o out/ -v .",
+        "build:dist": "mkdir out && npm run build:app && npm run build:scraper",
+        "build": "go build -v ."
+    },
+    "dependencies": {
+        "terser": "^4.6.3"
+    }
+}

resources/strings.go

@@ -0,0 +1,13 @@
+package resources
+
+// ErrorCouldNotSerializeJSON -
+var ErrorCouldNotSerializeJSON = "Could not serialize json for video: %s\n"
+
+// ErrorCouldNotRecogniseURL -
+var ErrorCouldNotRecogniseURL = "Could not recognise URL format of string %s"
+
+// ErrorCouldNotGetUserUploads -
+var ErrorCouldNotGetUserUploads = "Failed to get user uploads: %s\n"
+
+// ErrorPathNotFound -
+var ErrorPathNotFound = "File path %s not found."

scraper.js

@@ -1,15 +1,23 @@
 optStrings = {
     selectors: {
-        feedVideoItem: 'video-feed-item-wrapper',
+        feedLoading: 'div.tiktok-loading.feed-loading',
         modalArrowLeft: 'div.video-card-modal > div > img.arrow-right',
         modalClose: '.video-card-modal > div > div.close',
         modalPlayer: 'div > div > main > div.video-card-modal > div > div.video-card-big > div.video-card-container > div > div > video',
         modalShareInput: '.copy-link-container > input',
+        modalCaption: 'div.video-card-big > div.content-container > div.video-meta-info > h1',
+        modalSoundLink: 'div.content-container > div.video-meta-info > h2.music-info > a',
+        modalUploader: '.user-username',
         videoPlayer: 'div.video-card-container > div > div > video',
         videoShareInput: 'div.content-container.border > div.copy-link-container > input',
+        videoCaption: 'div.content-container.border > div.video-meta-info > h1',
+        videoSoundLink: 'div.content-container.border > div.video-meta-info > h2.music-info > a',
+        videoUploader: '.user-username',
     },
     classes: {
+        feedVideoItem: 'video-feed-item-wrapper',
         modalCloseDisabled: 'disabled',
+        titleMessage: 'title',
     },
     tags: {
         resultTag: 'video_urls',
@@ -18,16 +26,40 @@ optStrings = {
     attributes: {
         src: "src",
     },
+    tiktokMessages: [
+        "Couldn't find this account",
+        "No videos yet",
+        "Video currently unavailable",
+    ],
+};
+
+currentState = {
+    preloadCount: 0,
+    finished: false,
+};
+
+checkForErrors = function() {
+    var titles = document.getElementsByClassName(optStrings.classes.titleMessage);
+    debugger;
+    if (titles && titles.length) {
+        var error = Array.from(titles).find(x => optStrings.tiktokMessages.includes(x.textContent)).textContent;
+        if (error) {
+            createVidUrlElement("ERR: " + error);
+            return true;
+        }
+    }
+    return false;
 };
 
 createVidUrlElement = function(outputObj) {
     var urlSetElement = document.createElement(optStrings.tags.resultTag);
     urlSetElement.innerText = JSON.stringify(outputObj);
     document.getElementsByTagName(optStrings.tags.resultParentTag)[0].appendChild(urlSetElement);
-}
+    currentState.finished = true;
+};
 
 buldVidUrlArray = function(finishCallback) {
-    var feedItem = document.getElementsByClassName(optStrings.selectors.feedVideoItem)[0];
+    var feedItem = document.getElementsByClassName(optStrings.classes.feedVideoItem)[0];
     feedItem.click();
 
     var videoArray = [];
@@ -42,53 +74,79 @@ buldVidUrlArray = function(finishCallback) {
         } else {
             arrowRight.click();
         }
-    }, 500);
+    }, 20);
 };
 
 getCurrentModalVideo = function() {
     var modalPlayer = document.querySelector(optStrings.selectors.modalPlayer);
     var vidUrl = modalPlayer.getAttribute(optStrings.attributes.src);
     var shareLink = document.querySelector(optStrings.selectors.modalShareInput).value;
+    var caption = document.querySelector(optStrings.selectors.modalCaption).textContent;
+    var soundLink = document.querySelector(optStrings.selectors.modalSoundLink);
+    var uploader = document.querySelector(optStrings.selectors.modalUploader).textContent;
+    var soundHref = soundLink.getAttribute("href");
+    var soundText = soundLink.text;
 
     return {
         url: vidUrl,
-        shareLink: shareLink
+        shareLink: shareLink,
+        caption: caption,
+        uploader: uploader,
+        sound: {
+            title: soundText,
+            link: soundHref,
+        },
     };
-}
+};
 
 getCurrentVideo = function() {
+    if(checkForErrors()) return;
     var player = document.querySelector(optStrings.selectors.videoPlayer);
     var vidUrl = player.getAttribute(optStrings.attributes.src);
     var shareLink = document.querySelector(optStrings.selectors.videoShareInput).value;
+    var caption = document.querySelector(optStrings.selectors.videoCaption).textContent;
+    var soundLink = document.querySelector(optStrings.selectors.videoSoundLink);
+    var uploader = document.querySelector(optStrings.selectors.videoUploader).textContent;
+    var soundHref = soundLink.getAttribute("href");
+    var soundText = soundLink.text;
 
     return {
         url: vidUrl,
-        shareLink: shareLink
+        shareLink: shareLink,
+        caption: caption,
+        uploader: uploader,
+        sound: {
+            title: soundText,
+            link: soundHref,
+        },
     };
-}
+};
 
 scrollWhileNew = function(finishCallback) {
     var state = { count: 0 };
     var intervalID = window.setInterval(x => {
         var oldCount = state.count;
-        state.count = document.getElementsByClassName(optStrings.selectors.feedVideoItem).length;
+        state.count = document.getElementsByClassName(optStrings.classes.feedVideoItem).length;
+        if(checkForErrors()) {
+            window.clearInterval(intervalID);
+            return;
+        }
         if (oldCount !== state.count) {
+            currentState.preloadCount = state.count;
             window.scrollTo(0, document.body.scrollHeight);
         } else {
+            if (document.querySelector(optStrings.selectors.feedLoading)) {
+                window.scrollTo(0, document.body.scrollHeight);
+                return;
+            }
             window.clearInterval(intervalID);
-            finishCallback();
+            finishCallback(createVidUrlElement);
         }
     }, 1000);
 };
 
 bootstrapIteratingVideos = function() {
-    var intervalID = window.setInterval(() => {
-        window.scrollTo(0, document.body.scrollHeight);
-        if (document.getElementsByClassName(optStrings.selectors.feedVideoItem).length > 0) {
-            window.setTimeout(() => buldVidUrlArray(createVidUrlElement), 100);
-            window.clearInterval(intervalID);
-        }
-    }, 500);
+    scrollWhileNew(buldVidUrlArray);
     return 'bootstrapIteratingVideos';
 };
 
@@ -96,13 +154,13 @@ bootstrapGetCurrentVideo = function() {
     var video = getCurrentVideo();
     createVidUrlElement(video);
     return 'bootstrapGetCurrentVideo';
-}
+};
 
 init = () => {
     const newProto = navigator.__proto__;
     delete newProto.webdriver;
     navigator.__proto__ = newProto;
+    return 'script initialized';
 };
 
 init();
-'script initialized'

unitTestUtil/assert.go

@@ -0,0 +1,15 @@
+package unittestutil
+
+// AssertString - Check if two strings match
+func (tu *TestUtil) AssertString(actual string, expected string, name string) {
+	if actual != expected {
+		tu.T.Errorf("%s is incorrect: Expected '%s', but got '%s'", name, expected, actual)
+	}
+}
+
+// AssertInt - Check if two intagers match
+func (tu *TestUtil) AssertInt(actual int, expected int, name string) {
+	if actual != expected {
+		tu.T.Errorf("%s is incorrect: Expected '%d', but got '%d'", name, expected, actual)
+	}
+}

unitTestUtil/unitTestUtil.go

@@ -0,0 +1,10 @@
+package unittestutil
+
+import (
+	"testing"
+)
+
+// TestUtil - Utility for testing
+type TestUtil struct {
+	T *testing.T
+}

utils/checkErr.go

@@ -0,0 +1,12 @@
+package utils
+
+import (
+	"log"
+)
+
+// CheckErr - Checks if error and log
+func CheckErr(err error) {
+	if err != nil {
+		log.Fatal(err)
+	}
+}

utils/downloadFile.go

@@ -10,22 +10,15 @@ import (
 func DownloadFile(outputPath string, url string) {
 	// Get the data
 	resp, err := http.Get(url)
-	if err != nil {
-		panic(err)
-	}
+	CheckErr(err)
 	defer resp.Body.Close()
 
 	// Create the file
 	out, err := os.Create(outputPath)
-	if err != nil {
-		panic(err)
-	}
+	CheckErr(err)
 	defer out.Close()
 
 	// Write the body to file
 	_, err = io.Copy(out, resp.Body)
-
-	if err != nil {
-		panic(err)
-	}
+	CheckErr(err)
 }

utils/fileio.go

@@ -1,9 +1,13 @@
 package utils
 
 import (
+	"bufio"
+	"io/ioutil"
 	"os"
 )
 
+type delegateString func(string)
+
 // CheckIfExists - Checks if file or directory exists
 func CheckIfExists(path string) bool {
 	if _, err := os.Stat(path); os.IsNotExist(err) {
@@ -19,3 +23,29 @@ func InitOutputDirectory(path string) {
 		os.MkdirAll(path, os.ModePerm)
 	}
 }
+
+// ReadFileToString - Reads file and returns content
+func ReadFileToString(path string) string {
+	content, err := ioutil.ReadFile(path)
+	if err != nil {
+		panic(err)
+	}
+
+	return string(content)
+}
+
+// ReadFileLineByLine - Reads file line by line and calls delegate
+func ReadFileLineByLine(path string, delegate delegateString) {
+	file, err := os.Open(path)
+	CheckErr(err)
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		delegate(scanner.Text())
+	}
+
+	if err := scanner.Err(); err != nil {
+		panic(err)
+	}
+}

utils/getUsername.go

@@ -0,0 +1,28 @@
+package utils
+
+import (
+	config "../models/config"
+	res "../resources"
+	"fmt"
+	"regexp"
+	"strings"
+)
+
+// GetUsername - Get's username from passed URL param
+func GetUsername() string {
+	return GetUsernameFromString(config.Config.URL)
+}
+
+// GetUsernameFromString - Get's username from passed param
+func GetUsernameFromString(str string) string {
+	if match := strings.Contains(str, "/"); !match { // Not url
+		return strings.Replace(str, "@", "", -1)
+	}
+
+	if match, _ := regexp.MatchString(".+tiktok\\.com/@.+", str); match { // URL
+		stripedSuffix := strings.Split(str, "@")[1]
+		return strings.Split(stripedSuffix, "/")[0]
+	}
+
+	panic(fmt.Sprintf(res.ErrorCouldNotRecogniseURL, str))
+}

utils/getUsername_test.go

@@ -0,0 +1,37 @@
+package utils
+
+import (
+	config "../models/config"
+	testUtil "../unitTestUtil"
+	"testing"
+)
+
+func TestGetUsername(t *testing.T) {
+	testCaseDelegate := func(t *testing.T, url string, username string) {
+		tu := testUtil.TestUtil{T: t}
+		config.Config.URL = url
+		actual := GetUsername()
+		tu.AssertString(actual, username, "Username")
+	}
+
+	testVideoURL := func(t *testing.T) {
+		testCaseDelegate(t, "https://www.tiktok.com/@some_username/video/0000000000000000000", "some_username")
+	}
+
+	testProfileURL := func(t *testing.T) {
+		testCaseDelegate(t, "https://www.tiktok.com/@some_username", "some_username")
+	}
+
+	testPlainUsername := func(t *testing.T) {
+		testCaseDelegate(t, "some_username", "some_username")
+	}
+
+	testAtUsername := func(t *testing.T) {
+		testCaseDelegate(t, "@some_username", "some_username")
+	}
+
+	t.Run("Video URL", testVideoURL)
+	t.Run("Username URL", testProfileURL)
+	t.Run("Plain username", testPlainUsername)
+	t.Run("Username with @ suffix", testAtUsername)
+}

utils/log.go

@@ -0,0 +1,31 @@
+package utils
+
+import (
+	config "../models/config"
+	"fmt"
+	"os"
+)
+
+// Log - Write to std out
+func Log(a ...interface{}) {
+	if !config.Config.Quiet {
+		fmt.Println(a...)
+	}
+}
+
+// Logf - Write formated text
+func Logf(format string, a ...interface{}) {
+	if !config.Config.Quiet {
+		fmt.Printf(format, a...)
+	}
+}
+
+// LogFatal - Write error and panic
+func LogFatal(format string, a ...interface{}) {
+	panic(fmt.Sprintf(format, a...))
+}
+
+// LogErr - Write error
+func LogErr(format string, a ...interface{}) {
+	fmt.Fprintf(os.Stderr, format, a...)
+}

utils/readFileAsString.go

@@ -2,14 +2,11 @@ package utils
 
 import (
 	"io/ioutil"
-	"log"
 )
 
 // ReadFileAsString - Returns contents of given file
 func ReadFileAsString(fileName string) string {
 	content, err := ioutil.ReadFile(fileName)
-	if err != nil {
-		log.Fatal(err)
-	}
+	CheckErr(err)
 	return string(content)
 }

workflows/downloadBatchFile.go

@@ -0,0 +1,28 @@
+package workflows
+
+import (
+	res "../resources"
+	utils "../utils"
+)
+
+// CanUseDownloadBatchFile - Check's if DownloadBatchFile can be used
+func CanUseDownloadBatchFile(batchFilePath string) bool {
+	return batchFilePath != ""
+}
+
+// DownloadBatchFile - Download items from batch file
+func DownloadBatchFile(batchFilePath string) {
+	if !utils.CheckIfExists(batchFilePath) {
+		utils.LogFatal(res.ErrorPathNotFound, batchFilePath)
+	}
+
+	utils.ReadFileLineByLine(batchFilePath, downloadItem)
+}
+
+func downloadItem(batchItem string) {
+	if batchItem[0] == '#' {
+		return
+	}
+
+	StartWorkflowByParameter(batchItem)
+}

workflows/downloadMusic.go

@@ -0,0 +1,36 @@
+package workflows
+
+import (
+	client "../client"
+	config "../models/config"
+	res "../resources"
+	utils "../utils"
+	"fmt"
+	"regexp"
+)
+
+// CanUseDownloadMusic - Check's if DownloadMusic can be used for parameter
+func CanUseDownloadMusic(url string) bool {
+	match, _ := regexp.MatchString(".com\\/music\\/.+", url)
+	return match
+}
+
+// DownloadMusic - Download all videos by given music
+func DownloadMusic(url string) {
+	uploads, err := client.GetMusicUploads(url)
+	if err != nil {
+		utils.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
+		return
+	}
+	uploadCount := len(uploads)
+
+	for index, upload := range uploads {
+		username := utils.GetUsernameFromString(upload.Uploader)
+		downloadDir := fmt.Sprintf("%s/%s", config.Config.OutputPath, username)
+
+		utils.InitOutputDirectory(downloadDir)
+		downloadVideo(upload, downloadDir)
+		utils.Logf("\r[%d/%d] Downloaded", index+1, uploadCount)
+	}
+	utils.Log()
+}

workflows/downloadUser.go

@@ -0,0 +1,37 @@
+package workflows
+
+import (
+	client "../client"
+	config "../models/config"
+	res "../resources"
+	utils "../utils"
+	"fmt"
+	"regexp"
+	"strings"
+)
+
+// CanUseDownloadUser - Test's if this workflow can be used for parameter
+func CanUseDownloadUser(url string) bool {
+	isURL := strings.Contains(url, "/")
+	match, _ := regexp.MatchString(".+com\\/@[^\\/]+", url)
+	return !isURL || match
+}
+
+// DownloadUser - Download all user's videos
+func DownloadUser(username string) {
+	uploads, err := client.GetUserUploads(username)
+	if err != nil {
+		utils.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
+		return
+	}
+	uploadCount := len(uploads)
+	downloadDir := fmt.Sprintf("%s/%s", config.Config.OutputPath, username)
+
+	utils.InitOutputDirectory(downloadDir)
+
+	for index, upload := range uploads {
+		downloadVideo(upload, downloadDir)
+		utils.Logf("\r[%d/%d] Downloaded", index+1, uploadCount)
+	}
+	utils.Log()
+}

workflows/downloadVideo.go

@@ -0,0 +1,49 @@
+package workflows
+
+import (
+	client "../client"
+	models "../models"
+	config "../models/config"
+	res "../resources"
+	utils "../utils"
+	"fmt"
+	"regexp"
+)
+
+// CanUseDownloadSingleVideo - Check's if DownloadSingleVideo can be used for parameter
+func CanUseDownloadSingleVideo(url string) bool {
+	match, _ := regexp.MatchString("\\/@.+\\/video\\/[0-9]+", url)
+	return match
+}
+
+// DownloadSingleVideo - Downloads single video
+func DownloadSingleVideo(url string) {
+	username := utils.GetUsernameFromString(url)
+	upload, err := client.GetVideoDetails(url)
+	if err != nil {
+		utils.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
+		return
+	}
+	downloadDir := fmt.Sprintf("%s/%s", config.Config.OutputPath, username)
+
+	utils.InitOutputDirectory(downloadDir)
+	downloadVideo(upload, downloadDir)
+	utils.Log("[1/1] Downloaded\n")
+}
+
+// DownloadVideo - Downloads one video
+func downloadVideo(upload models.Upload, downloadDir string) {
+	uploadID := upload.GetUploadID()
+	downloadPath := fmt.Sprintf("%s/%s.mp4", downloadDir, uploadID)
+
+	if utils.CheckIfExists(downloadPath) {
+		return
+	}
+
+	utils.DownloadFile(downloadPath, upload.URL)
+
+	if config.Config.MetaData {
+		metadataPath := fmt.Sprintf("%s/%s.json", downloadDir, uploadID)
+		upload.WriteToFile(metadataPath)
+	}
+}

workflows/startWorkflowByParameter.go

@@ -0,0 +1,30 @@
+package workflows
+
+import (
+	res "../resources"
+	utils "../utils"
+)
+
+// StartWorkflowByParameter - Start needed workflow by given parameter
+func StartWorkflowByParameter(url string) {
+
+	// Music
+	if CanUseDownloadMusic(url) {
+		DownloadMusic(url)
+		return
+	}
+
+	// Single video
+	if CanUseDownloadSingleVideo(url) {
+		DownloadSingleVideo(url)
+		return
+	}
+
+	// Tiktok user
+	if CanUseDownloadUser(url) {
+		DownloadUser(utils.GetUsernameFromString(url))
+		return
+	}
+
+	utils.LogFatal(res.ErrorCouldNotRecogniseURL, url)
+}