Added option to download items from list

.gitignore

@@ -4,3 +4,4 @@ __debug_bin
 downloads
 *.exe
 tiktok-dl
+batch_file.txt

README.md

@@ -8,7 +8,8 @@ A simple tiktok video downloader written in go
 ## Basic usage
 Download the executable from `https://github.com/pikami/tiktok-dl/releases`\
 You can download all videos from user by running `./tiktok-dl [Options] TIKTOK_USERNAME`\
-You can download single video by running `./tiktok-dl [Options] VIDEO_URL`
+You can download single video by running `./tiktok-dl [Options] VIDEO_URL`\
+You can download items listed in a text file by running `./tiktok-dl [OPTIONS] -batch-file path/to/items.txt`
 
 ## Build instructions
 Clone this repository and run `go build` to build the executable.
@@ -16,6 +17,9 @@ Clone this repository and run `go build` to build the executable.
 ## Available options
 * `-debug` - enables debug mode
 * `-output some_directory` - Output path (default "./downloads")
+* `-metadata` - Write video metadata to a .json file
+* `-batch-file` - File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored.
 
 ## Acknowledgments
-This software uses the chromedp for web scraping, it can be found here: https://github.com/chromedp/chromedp
+This software uses the **chromedp** for web scraping, it can be found here: https://github.com/chromedp/chromedp\
+For releases the JS code is minified by using **terser** toolkit, it can be found here: https://github.com/terser/terser

main.go

@@ -1,62 +1,32 @@
 package main
 
 import (
-	client "./client"
 	models "./models"
-	utils "./utils"
-	"fmt"
-	"regexp"
-	"strings"
+	workflows "./workflows"
 )
 
 func main() {
 	models.GetConfig()
 	url := models.Config.URL
+	batchFilePath := models.Config.BatchFilePath
+
+	// Batch file
+	if workflows.CanUseDownloadBatchFile(batchFilePath) {
+		workflows.DownloadBatchFile(batchFilePath)
+		return
+	}
 
 	// Single video
-	match, _ := regexp.MatchString("\\/@.+\\/video\\/[0-9]+", url)
-	if match {
-		getUsernameFromVidURLRegex, _ := regexp.Compile("com\\/@.*")
-		parts := strings.Split(getUsernameFromVidURLRegex.FindString(url), "/")
-		username := parts[1][1:]
-		upload := client.GetVideoDetails(url)
-		downloadDir := fmt.Sprintf("%s/%s", models.Config.OutputPath, username)
-
-		utils.InitOutputDirectory(downloadDir)
-		downloadVideo(upload, downloadDir)
+	if workflows.CanUseDownloadSingleVideo(url) {
+		workflows.DownloadSingleVideo(url)
 		return
 	}
 
 	// Tiktok user
-	downloadUser()
-}
-
-func downloadVideo(upload models.Upload, downloadDir string) {
-	uploadID := upload.GetUploadID()
-	downloadPath := fmt.Sprintf("%s/%s.mp4", downloadDir, uploadID)
-
-	if utils.CheckIfExists(downloadPath) {
-		fmt.Println("Upload '" + uploadID + "' already downloaded, skipping")
+	if workflows.CanUseDownloadUser(url) {
+		workflows.DownloadUser(models.GetUsername())
 		return
 	}
 
-	fmt.Println("Downloading upload item '" + uploadID + "' to " + downloadPath)
-	utils.DownloadFile(downloadPath, upload.URL)
-
-	if models.Config.MetaData {
-		metadataPath := fmt.Sprintf("%s/%s.json", downloadDir, uploadID)
-		upload.WriteToFile(metadataPath)
-	}
-}
-
-func downloadUser() {
-	username := models.Config.URL
-	downloadDir := fmt.Sprintf("%s/%s", models.Config.OutputPath, username)
-	uploads := client.GetUserUploads(username)
-
-	utils.InitOutputDirectory(downloadDir)
-
-	for _, upload := range uploads {
-		downloadVideo(upload, downloadDir)
-	}
+	panic("Could not recognise URL format")
 }

models/config.go

@@ -4,31 +4,60 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"regexp"
+	"strings"
 )
 
 // Config - Runtime configuration
 var Config struct {
-	URL        string
-	OutputPath string
-	Debug      bool
-	MetaData   bool
+	URL           string
+	OutputPath    string
+	BatchFilePath string
+	Debug         bool
+	MetaData      bool
 }
 
 // GetConfig - Returns Config object
 func GetConfig() {
 	outputPath := flag.String("output", "./downloads", "Output path")
+	batchFilePath := flag.String("batch-file", "", "File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored.")
 	debug := flag.Bool("debug", false, "Enables debug mode")
 	metadata := flag.Bool("metadata", false, "Write video metadata to a .json file")
 	flag.Parse()
 
 	args := flag.Args()
-	if len(args) < 1 {
+	if len(args) < 1 && *batchFilePath == "" {
 		fmt.Println("Usage: tiktok-dl [OPTIONS] TIKTOK_USERNAME|TIKTOK_URL")
+		fmt.Println("  or:  tiktok-dl [OPTIONS] -batch-file path/to/users.txt")
 		os.Exit(2)
 	}
 
-	Config.URL = flag.Args()[len(args)-1]
+	if len(args) > 0 {
+		Config.URL = flag.Args()[len(args)-1]
+	} else {
+		Config.URL = ""
+	}
 	Config.OutputPath = *outputPath
+	Config.BatchFilePath = *batchFilePath
 	Config.Debug = *debug
 	Config.MetaData = *metadata
 }
+
+// GetUsername - Get's username from passed URL param
+func GetUsername() string {
+	return GetUsernameFromString(Config.URL)
+}
+
+// GetUsernameFromString - Get's username from passed param
+func GetUsernameFromString(str string) string {
+	if match := strings.Contains(str, "/"); !match { // Not url
+		return strings.Replace(str, "@", "", -1)
+	}
+
+	if match, _ := regexp.MatchString(".+tiktok\\.com/@.+", str); match { // URL
+		stripedSuffix := strings.Split(str, "@")[1]
+		return strings.Split(stripedSuffix, "/")[0]
+	}
+
+	panic("Could not recognise URL format")
+}

models/config_test.go

@@ -0,0 +1,36 @@
+package models
+
+import (
+	testUtil "../unitTestUtil"
+	"testing"
+)
+
+func TestGetUsername(t *testing.T) {
+	testCaseDelegate := func(t *testing.T, url string, username string) {
+		tu := testUtil.TestUtil{T: t}
+		Config.URL = url
+		actual := GetUsername()
+		tu.AssertString(actual, username, "Username")
+	}
+
+	testVideoURL := func(t *testing.T) {
+		testCaseDelegate(t, "https://www.tiktok.com/@some_username/video/0000000000000000000", "some_username")
+	}
+
+	testProfileURL := func(t *testing.T) {
+		testCaseDelegate(t, "https://www.tiktok.com/@some_username", "some_username")
+	}
+
+	testPlainUsername := func(t *testing.T) {
+		testCaseDelegate(t, "some_username", "some_username")
+	}
+
+	testAtUsername := func(t *testing.T) {
+		testCaseDelegate(t, "@some_username", "some_username")
+	}
+
+	t.Run("Video URL", testVideoURL)
+	t.Run("Username URL", testProfileURL)
+	t.Run("Plain username", testPlainUsername)
+	t.Run("Username with @ suffix", testAtUsername)
+}

models/upload_test.go

@@ -1,36 +1,67 @@
 package models
 
-import "testing"
+import (
+	testUtil "../unitTestUtil"
+	utils "../utils"
+	"os"
+	"testing"
+)
 
-// TestParseUploads - Test parsing
 func TestParseUploads(t *testing.T) {
-	jsonStr := "[{\"shareLink\":\"some_share_link\", \"url\": \"some_url\"}]"
+	tu := testUtil.TestUtil{T: t}
+	jsonStr := "[{\"url\":\"some_url\",\"shareLink\":\"some_share_link\",\"caption\":\"some_caption\",\"sound\":{\"title\":\"some_title\",\"link\":\"some_link\"}}]"
 	actual := ParseUploads(jsonStr)
 
-	expectedLen := 1
-	if len(actual) != expectedLen {
-		t.Errorf("Array len incorrect: Expected %d, but got %d", expectedLen, len(actual))
-	}
+	tu.AssertInt(len(actual), 1, "Array len")
 
-	expectedShareLink := "some_share_link"
-	if actual[0].ShareLink != expectedShareLink {
-		t.Errorf("ShareLink is incorrect: Expected %s, but got %s", expectedShareLink, actual[0].ShareLink)
-	}
+	tu.AssertString(actual[0].URL, "some_url", "URL")
+	tu.AssertString(actual[0].Caption, "some_caption", "Caption")
+	tu.AssertString(actual[0].ShareLink, "some_share_link", "ShareLink")
 
-	expectedURL := "some_url"
-	if actual[0].URL != expectedURL {
-		t.Errorf("URL is incorrect: Expected %s, but got %s", expectedURL, actual[0].URL)
-	}
+	tu.AssertString(actual[0].Sound.Link, "some_link", "Sound.Link")
+	tu.AssertString(actual[0].Sound.Title, "some_title", "Sound.Title")
+}
+
+func TestParseUpload(t *testing.T) {
+	tu := testUtil.TestUtil{T: t}
+	jsonStr := "{\"url\":\"some_url\",\"shareLink\":\"some_share_link\",\"caption\":\"some_caption\",\"sound\":{\"title\":\"some_title\",\"link\":\"some_link\"}}"
+	actual := ParseUpload(jsonStr)
+
+	tu.AssertString(actual.URL, "some_url", "URL")
+	tu.AssertString(actual.Caption, "some_caption", "Caption")
+	tu.AssertString(actual.ShareLink, "some_share_link", "ShareLink")
+
+	tu.AssertString(actual.Sound.Link, "some_link", "Sound.Link")
+	tu.AssertString(actual.Sound.Title, "some_title", "Sound.Title")
 }
 
 func TestGetUploadID(t *testing.T) {
+	tu := testUtil.TestUtil{T: t}
 	var upload Upload
 	upload.ShareLink = "http://pikami.org/some_thing/some_upload_id"
-	expected := "some_upload_id"
-
 	actual := upload.GetUploadID()
 
-	if actual != expected {
-		t.Errorf("UploadId is incorrect: Expected %s, but got %s", expected, actual)
-	}
+	tu.AssertString(actual, "some_upload_id", "Upload ID")
+}
+
+func TestWriteToFile(t *testing.T) {
+	tu := testUtil.TestUtil{T: t}
+	expected := "{\"url\":\"some_url\",\"shareLink\":\"some_share_link\",\"caption\":\"some_caption\",\"sound\":{\"title\":\"some_title\",\"link\":\"some_link\"}}"
+	filePath := "test_file.txt"
+	upload := Upload{
+		URL:       "some_url",
+		Caption:   "some_caption",
+		ShareLink: "some_share_link",
+		Sound: Sound{
+			Link:  "some_link",
+			Title: "some_title",
+		},
+	}
+
+	upload.WriteToFile(filePath)
+
+	actual := utils.ReadFileToString(filePath)
+	tu.AssertString(actual, expected, "File content")
+
+	os.Remove(filePath)
 }

unitTestUtil/assert.go

@@ -0,0 +1,15 @@
+package unittestutil
+
+// AssertString - Check if two strings match
+func (tu *TestUtil) AssertString(actual string, expected string, name string) {
+	if actual != expected {
+		tu.T.Errorf("%s is incorrect: Expected '%s', but got '%s'", name, expected, actual)
+	}
+}
+
+// AssertInt - Check if two intagers match
+func (tu *TestUtil) AssertInt(actual int, expected int, name string) {
+	if actual != expected {
+		tu.T.Errorf("%s is incorrect: Expected '%d', but got '%d'", name, expected, actual)
+	}
+}

unitTestUtil/unitTestUtil.go

@@ -0,0 +1,10 @@
+package unittestutil
+
+import (
+	"testing"
+)
+
+// TestUtil - Utility for testing
+type TestUtil struct {
+	T *testing.T
+}

utils/fileio.go

@@ -1,9 +1,13 @@
 package utils
 
 import (
+	"bufio"
+	"io/ioutil"
 	"os"
 )
 
+type delegateString func(string)
+
 // CheckIfExists - Checks if file or directory exists
 func CheckIfExists(path string) bool {
 	if _, err := os.Stat(path); os.IsNotExist(err) {
@@ -19,3 +23,31 @@ func InitOutputDirectory(path string) {
 		os.MkdirAll(path, os.ModePerm)
 	}
 }
+
+// ReadFileToString - Reads file and returns content
+func ReadFileToString(path string) string {
+	content, err := ioutil.ReadFile(path)
+	if err != nil {
+		panic(err)
+	}
+
+	return string(content)
+}
+
+// ReadFileLineByLine - Reads file line by line and calls delegate
+func ReadFileLineByLine(path string, delegate delegateString) {
+	file, err := os.Open(path)
+	if err != nil {
+		panic(err)
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		delegate(scanner.Text())
+	}
+
+	if err := scanner.Err(); err != nil {
+		panic(err)
+	}
+}

workflows/downloadBatchFile.go

@@ -0,0 +1,41 @@
+package workflows
+
+import (
+	models "../models"
+	utils "../utils"
+	"fmt"
+)
+
+// CanUseDownloadBatchFile - Check's if DownloadBatchFile can be used
+func CanUseDownloadBatchFile(batchFilePath string) bool {
+	return batchFilePath != ""
+}
+
+// DownloadBatchFile - Download items from batch file
+func DownloadBatchFile(batchFilePath string) {
+	if !utils.CheckIfExists(batchFilePath) {
+		panic(fmt.Sprintf("File path %s not found.", batchFilePath))
+	}
+
+	utils.ReadFileLineByLine(batchFilePath, downloadItem)
+}
+
+func downloadItem(batchItem string) {
+	if batchItem[0] == '#' {
+		return
+	}
+
+	// Single video
+	if CanUseDownloadSingleVideo(batchItem) {
+		DownloadSingleVideo(batchItem)
+		return
+	}
+
+	// Tiktok user
+	if CanUseDownloadUser(batchItem) {
+		DownloadUser(models.GetUsernameFromString(batchItem))
+		return
+	}
+
+	panic(fmt.Sprintf("Could not recognise URL format of string %s", batchItem))
+}

workflows/downloadUser.go

@@ -0,0 +1,27 @@
+package workflows
+
+import (
+	client "../client"
+	models "../models"
+	utils "../utils"
+	"fmt"
+	"strings"
+)
+
+// CanUseDownloadUser - Test's if this workflow can be used for parameter
+func CanUseDownloadUser(url string) bool {
+	match := strings.Contains(url, "/")
+	return !match
+}
+
+// DownloadUser - Download all user's videos
+func DownloadUser(username string) {
+	downloadDir := fmt.Sprintf("%s/%s", models.Config.OutputPath, username)
+	uploads := client.GetUserUploads(username)
+
+	utils.InitOutputDirectory(downloadDir)
+
+	for _, upload := range uploads {
+		downloadVideo(upload, downloadDir)
+	}
+}

workflows/downloadVideo.go

@@ -0,0 +1,44 @@
+package workflows
+
+import (
+	client "../client"
+	models "../models"
+	utils "../utils"
+	"fmt"
+	"regexp"
+)
+
+// CanUseDownloadSingleVideo - Check's if DownloadSingleVideo can be used for parameter
+func CanUseDownloadSingleVideo(url string) bool {
+	match, _ := regexp.MatchString("\\/@.+\\/video\\/[0-9]+", url)
+	return match
+}
+
+// DownloadSingleVideo - Downloads single video
+func DownloadSingleVideo(url string) {
+	username := models.GetUsernameFromString(url)
+	upload := client.GetVideoDetails(url)
+	downloadDir := fmt.Sprintf("%s/%s", models.Config.OutputPath, username)
+
+	utils.InitOutputDirectory(downloadDir)
+	downloadVideo(upload, downloadDir)
+}
+
+// DownloadVideo - Downloads one video
+func downloadVideo(upload models.Upload, downloadDir string) {
+	uploadID := upload.GetUploadID()
+	downloadPath := fmt.Sprintf("%s/%s.mp4", downloadDir, uploadID)
+
+	if utils.CheckIfExists(downloadPath) {
+		fmt.Println("Upload '" + uploadID + "' already downloaded, skipping")
+		return
+	}
+
+	fmt.Println("Downloading upload item '" + uploadID + "' to " + downloadPath)
+	utils.DownloadFile(downloadPath, upload.URL)
+
+	if models.Config.MetaData {
+		metadataPath := fmt.Sprintf("%s/%s.json", downloadDir, uploadID)
+		upload.WriteToFile(metadataPath)
+	}
+}