Added ability to download from scrapeData (json)

Create LICENSE

Update go.yml

.github/workflows/go.yml

@@ -33,6 +33,12 @@ jobs:
 
     - name: Build
       run: npm run build:dist
+    
+    - name: Copy license
+      run: cp LICENSE out
+
+    - name: Add execute permission
+      run: chmod +x out/tiktok-dl
 
     - name: Upload Unix Artifacts
       if: startsWith(matrix.os, 'ubuntu-')

.gitignore

@@ -6,3 +6,5 @@ downloads
 tiktok-dl
 batch_file.txt
 debug.log
+.scannerwork
+scraper.min.js

.sonar/sonar-project.properties

@@ -0,0 +1,12 @@
+sonar.organization=pikami
+sonar.projectKey=tiktok-dl
+
+sonar.host.url=https://sonarcloud.io
+
+sonar.sources=.
+sonar.exclusions=**/*_test.go,**/node_modules/**
+
+sonar.tests=.
+sonar.test.inclusions=**/*_test.go
+sonar.test.exclusions=**/node_modules/**
+sonar.go.coverage.reportPaths=cov.out

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 pikami
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,29 +1,44 @@
 # TikTok-DL
 
 [![Go Report Card](https://goreportcard.com/badge/github.com/pikami/tiktok-dl)](https://goreportcard.com/report/github.com/pikami/tiktok-dl)
-![GitHub Workflow Status](https://img.shields.io/github/workflow/status/pikami/tiktok-dl/tiktok-dl_CI)
+[![tiktok-dl_CI](https://github.com/pikami/tiktok-dl/workflows/tiktok-dl_CI/badge.svg?branch=master)](https://github.com/pikami/tiktok-dl/actions)
 
 A simple tiktok video downloader written in go
 
-## Basic usage
-Download the executable from `https://github.com/pikami/tiktok-dl/releases`\
-You can download all videos from user by running `./tiktok-dl [Options] TIKTOK_USERNAME`\
-You can download single video by running `./tiktok-dl [Options] VIDEO_URL`\
-You can download all videos by music by running `./tiktok-dl [Options] MUSIC_URL`\
-You can download items listed in a text file by running `./tiktok-dl [OPTIONS] -batch-file path/to/items.txt`
+```diff
+- This tool is not working currenly, I will revive it in the future when I have some free time
+```
 
-## Build instructions
-Clone this repository and run `go build` to build the executable.
+## Basic usage examples
+Download the executable from `https://github.com/pikami/tiktok-dl/releases`\
+You can download all videos from user by running `./tiktok-dl TIKTOK_USERNAME`\
+You can download single video by running `./tiktok-dl VIDEO_URL`\
+You can download items listed in a text file by running `./tiktok-dl -batch-file path/to/items.txt`
+
+## Usage Manual
+```
+Usage: tiktok-dl [OPTION]... TARGET
+  or:  tiktok-dl [OPTION]... -batch-file BATCH_FILE
+
+In the 1st form, download given `TARGET`.
+In the 2nd form, download all targets listed in given `BATCH_FILE`.
+```
 
 ## Available options
-* `-debug` - enables debug mode
-* `-output some_directory` - Output path (default "./downloads")
-* `-metadata` - Write video metadata to a .json file
-* `-batch-file` - File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored.
+* `-archive` - Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.
+* `-batch-file some_file` - File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored.
 * `-deadline` - Sets the timout for scraper logic in seconds (used as a workaround for context deadline exceeded error) (default 1500)
-* `-quiet` - Supress output
+* `-debug` - enables debug mode
+* `-fail-log some_file` - Write failed items to log file
 * `-json` - Returns whole data, that was scraped from TikTok, in json
 * `-limit` - Sets the max count of video that will be downloaded (default infinity)
+* `-metadata` - Write video metadata to a .json file
+* `-output some_directory` - Output path (default "./downloads")
+* `-quiet` - Suppress output
+
+## Build instructions
+1. Clone this repository
+2. Run `go build` to build the executable.
 
 ## Acknowledgments
 This software uses the **chromedp** for web scraping, it can be found here: https://github.com/chromedp/chromedp \

client/executeClientAction.go

@@ -3,15 +3,17 @@ package client
 import (
 	"context"
 	"errors"
-	"github.com/chromedp/chromedp"
 	"io/ioutil"
-	"log"
 	"os"
 	"strings"
 	"time"
 
-	config "../models/config"
-	utils "../utils"
+	"github.com/chromedp/chromedp"
+
+	config "github.com/pikami/tiktok-dl/models/config"
+	res "github.com/pikami/tiktok-dl/resources"
+	utils "github.com/pikami/tiktok-dl/utils"
+	log "github.com/pikami/tiktok-dl/utils/log"
 )
 
 // GetMusicUploads - Get all uploads by given music
@@ -33,7 +35,7 @@ func executeClientAction(url string, jsAction string) (string, error) {
 
 	ctx, cancel := chromedp.NewContext(
 		allocCtx,
-		chromedp.WithLogf(log.Printf),
+		chromedp.WithLogf(log.Logf),
 	)
 	defer cancel()
 
@@ -53,7 +55,7 @@ func runScrapeQuiet(ctx context.Context, jsAction string, url string) (string, e
 		// Navigate to user's page
 		chromedp.Navigate(url),
 		// Execute url grabber script
-		chromedp.EvaluateAsDevTools(utils.ReadFileAsString("scraper.js"), &jsOutput),
+		chromedp.EvaluateAsDevTools(utils.GetScraper(), &jsOutput),
 		chromedp.EvaluateAsDevTools(jsAction, &jsOutput),
 		// Wait until custom js finishes
 		chromedp.WaitVisible(`video_urls`),
@@ -72,7 +74,7 @@ func runScrapeWithInfo(ctx context.Context, jsAction string, url string) (string
 		// Navigate to user's page
 		chromedp.Navigate(url),
 		// Execute url grabber script
-		chromedp.EvaluateAsDevTools(utils.ReadFileAsString("scraper.js"), &jsOutput),
+		chromedp.EvaluateAsDevTools(utils.GetScraper(), &jsOutput),
 		chromedp.EvaluateAsDevTools(jsAction, &jsOutput),
 	); err != nil {
 		return "", err
@@ -84,9 +86,9 @@ func runScrapeWithInfo(ctx context.Context, jsAction string, url string) (string
 		}
 
 		if jsOutput != "0" {
-			utils.Logf("\rPreloading... %s items have been founded.", jsOutput)
+			log.Logf(res.PreloadingItemsFound, jsOutput)
 		} else {
-			utils.Logf("\rPreloading...")
+			log.Logf(res.Preloading)
 		}
 
 		if err := chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.finished.toString()", &jsOutput)); err != nil {
@@ -100,7 +102,7 @@ func runScrapeWithInfo(ctx context.Context, jsAction string, url string) (string
 		time.Sleep(50 * time.Millisecond)
 	}
 
-	utils.Log("\nRetrieving items...")
+	log.Log(res.Retrieving)
 	if err := chromedp.Run(ctx,
 		// Wait until custom js finishes
 		chromedp.WaitVisible(`video_urls`),

client/getHashtagUploads.go

@@ -1,22 +1,24 @@
 package client
 
 import (
-	models "../models"
-	config "../models/config"
 	"fmt"
+
+	models "github.com/pikami/tiktok-dl/models"
+	config "github.com/pikami/tiktok-dl/models/config"
 )
 
-// GetUserUploads - Get all uploads marked with given hashtag
+// GetHashtagUploads - Get all uploads marked with given hashtag
 func GetHashtagUploads(hashtagURL string) ([]models.Upload, error) {
-	jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit)
-	actionOutput, err := executeClientAction(hashtagURL, jsMethod)
+	actionOutput, err := GetHashtagUploadsJSON(hashtagURL)
 	if err != nil {
 		return nil, err
 	}
+
 	return models.ParseUploads(actionOutput), nil
 }
 
-func GetHashtagUploadsJson(hashtagURL string) (string, error) {
+// GetHashtagUploadsJSON - Get hashtag uploads scrape
+func GetHashtagUploadsJSON(hashtagURL string) (string, error) {
 	jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit)
 	actionOutput, err := executeClientAction(hashtagURL, jsMethod)
 	if err != nil {

client/getMusicUploads.go

@@ -1,22 +1,23 @@
 package client
 
 import (
-	models "../models"
-	config "../models/config"
 	"fmt"
+
+	models "github.com/pikami/tiktok-dl/models"
+	config "github.com/pikami/tiktok-dl/models/config"
 )
 
 // GetMusicUploads - Get all uploads by given music
 func GetMusicUploads(url string) ([]models.Upload, error) {
-	jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit)
-	actionOutput, err := executeClientAction(url, jsMethod)
+	actionOutput, err := GetMusicUploadsJSON(url)
 	if err != nil {
 		return nil, err
 	}
 	return models.ParseUploads(actionOutput), nil
 }
 
-func GetMusicUploadsJson(url string) (string, error) {
+// GetMusicUploadsJSON - Get music uploads scrape
+func GetMusicUploadsJSON(url string) (string, error) {
 	jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit)
 	actionOutput, err := executeClientAction(url, jsMethod)
 	if err != nil {

client/getRedirectUrl.go

@@ -0,0 +1,54 @@
+package client
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"time"
+
+	"github.com/chromedp/chromedp"
+
+	config "github.com/pikami/tiktok-dl/models/config"
+	log "github.com/pikami/tiktok-dl/utils/log"
+)
+
+// GetRedirectURL - Returns URL that the given URL redirects to
+func GetRedirectURL(url string) (string, error) {
+	dir, err := ioutil.TempDir("", "chromedp-example")
+	if err != nil {
+		return "", err
+	}
+	defer os.RemoveAll(dir)
+
+	opts := append(chromedp.DefaultExecAllocatorOptions[:],
+		chromedp.DisableGPU,
+		chromedp.UserDataDir(dir),
+		chromedp.Flag("headless", !config.Config.Debug),
+	)
+
+	allocCtx, cancel := chromedp.NewExecAllocator(context.Background(), opts...)
+	defer cancel()
+
+	ctx, cancel := chromedp.NewContext(
+		allocCtx,
+		chromedp.WithLogf(log.Logf),
+	)
+	defer cancel()
+
+	ctx, cancel = context.WithTimeout(ctx, time.Duration(config.Config.Deadline)*time.Second)
+	defer cancel()
+
+	var jsOutput string
+	if err := chromedp.Run(ctx,
+		// Navigate to user's page
+		chromedp.Navigate(url),
+		// Wait until page loads
+		chromedp.WaitReady(`div`),
+		// Grab url links from our element
+		chromedp.EvaluateAsDevTools(`window.location.href`, &jsOutput),
+	); err != nil {
+		return "", err
+	}
+
+	return jsOutput, err
+}

client/getUserUploads.go

@@ -1,22 +1,23 @@
 package client
 
 import (
-	models "../models"
-	config "../models/config"
 	"fmt"
+
+	models "github.com/pikami/tiktok-dl/models"
+	config "github.com/pikami/tiktok-dl/models/config"
 )
 
 // GetUserUploads - Get all uploads by user
 func GetUserUploads(username string) ([]models.Upload, error) {
-	jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit)
-	actionOutput, err := executeClientAction(`https://www.tiktok.com/@`+username, jsMethod)
+	actionOutput, err := GetUserUploadsJSON(username)
 	if err != nil {
 		return nil, err
 	}
 	return models.ParseUploads(actionOutput), nil
 }
 
-func GetUserUploadsJson(username string) (string, error) {
+// GetUserUploadsJSON - Get user uploads scrape
+func GetUserUploadsJSON(username string) (string, error) {
 	jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit)
 	actionOutput, err := executeClientAction(`https://www.tiktok.com/@`+username, jsMethod)
 	if err != nil {

client/getVideoDetails.go

@@ -1,7 +1,7 @@
 package client
 
 import (
-	models "../models"
+	models "github.com/pikami/tiktok-dl/models"
 )
 
 // GetVideoDetails - returns details of video

generator/generator.go

@@ -0,0 +1,64 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	checkErr "github.com/pikami/tiktok-dl/utils/checkErr"
+	fileio "github.com/pikami/tiktok-dl/utils/fileio"
+)
+
+type resource struct {
+	Package  string
+	FileName string
+	Values   map[string]string
+}
+
+func (r resource) generate() {
+	filename := fmt.Sprintf("%s/%s", outputDir, r.FileName)
+	f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
+	checkErr.CheckErr(err)
+	defer f.Close()
+
+	// Header
+	header := fmt.Sprintf("// Package %s - This file is automatically generated.\n"+
+		"// Do not edit this file manually.\n"+
+		"// Check `/generator/resources.go` to change generated content\n"+
+		"package %s\n", r.Package, r.Package)
+
+	if _, err := f.WriteString(header); err != nil {
+		checkErr.CheckErr(err)
+	}
+
+	// Values
+	for key, value := range r.Values {
+		value = strings.ReplaceAll(value, "\n", "\\n")
+		value = strings.ReplaceAll(value, "\r", "\\r")
+		valueLine := fmt.Sprintf("\n//%s -\nvar %s = \"%s\"\n", key, key, value)
+		if _, err := f.WriteString(valueLine); err != nil {
+			checkErr.CheckErr(err)
+		}
+	}
+}
+
+func fileContentsOrDefault(file string) string {
+	defer func() {
+		if r := recover(); r != nil {
+			fmt.Printf("Failed to load file: %s\n", file)
+		}
+	}()
+
+	return safeString(fileio.ReadFileToString(file))
+}
+
+func safeString(str string) string {
+	escaped := strings.ReplaceAll(str, "\"", "\\\"")
+	return strings.ReplaceAll(escaped, "\n", "")
+}
+
+func main() {
+	for _, i := range res {
+		i.generate()
+	}
+}

generator/resources.go

@@ -0,0 +1,92 @@
+package main
+
+var (
+	outputDir = "../resources"
+
+	res = []resource{
+		resource{
+			Package:  "resources",
+			FileName: "scraper.go",
+			Values: map[string]string{
+				"ScraperPath":   "scraper.js",
+				"ScraperScript": fileContentsOrDefault("../scraper.min.js"),
+			},
+		},
+		resource{
+			Package:  "resources",
+			FileName: "errorStrings.go",
+			Values: map[string]string{
+				"ErrorCouldNotSerializeJSON": "Could not serialize json for video: %s\n",
+				"ErrorCouldNotRecogniseURL":  "Could not recognise URL format of string %s",
+				"Error":                      "Error : %s\n",
+				"ErrorPathNotFound":          "File path %s not found.",
+				"FailedOnItem":               "Failed while scraping item: %s\n",
+				"FailedToLoadScraper":        "Failed to load scraper",
+			},
+		},
+		resource{
+			Package:  "resources",
+			FileName: "messages.go",
+			Values: map[string]string{
+				"PreloadingItemsFound": "\rPreloading... %s items have been found.",
+				"Preloading":           "\rPreloading...",
+				"Retrieving":           "\nRetrieving items...",
+				"ItemsFoundInArchive":  "%d items, found in archive. Skipping...\n",
+				"Downloaded":           "\r[%d/%d] Downloaded",
+				"UsageLine": "Usage: tiktok-dl [OPTIONS] TIKTOK_USERNAME|TIKTOK_URL\n" +
+					"  or:  tiktok-dl [OPTIONS] -batch-file path/to/users.txt\n" +
+					"  or:  tiktok-dl [OPTIONS] -scraped-data path/to/data.json",
+			},
+		},
+		resource{
+			Package:  "resources",
+			FileName: "flags.go",
+			Values: map[string]string{
+				// Output
+				"OutputFlag":        "output",
+				"OutputDefault":     "./downloads",
+				"OutputDescription": "Output path",
+				// Batch file
+				"BatchFlag":        "batch-file",
+				"BatchDefault":     "",
+				"BatchDescription": "File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored.",
+				// ScrapedData
+				"ScrapedDataFlag":        "scraped-data",
+				"ScrapedDataDefault":     "",
+				"ScrapedDataDescription": "Download videos from scrape file (json format)",
+				// Archive
+				"ArchiveFlag":        "archive",
+				"ArchiveDefault":     "",
+				"ArchiveDescription": "Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.",
+				// Fail log
+				"FailLogFlag":        "fail-log",
+				"FailLogDefault":     "",
+				"FailLogDescription": "Write failed items to log file",
+				// Debug
+				"DebugFlag":        "debug",
+				"DebugDefault":     "false",
+				"DebugDescription": "Enables debug mode",
+				// Metadata
+				"MetadataFlag":        "metadata",
+				"MetadataDefault":     "false",
+				"MetadataDescription": "Write video metadata to a .json file",
+				// Quiet
+				"QuietFlag":        "quiet",
+				"QuietDefault":     "false",
+				"QuietDescription": "Suppress output",
+				// JSON only
+				"JsonFlag":        "json",
+				"JsonDefault":     "false",
+				"JsonDescription": "Just get JSON data from scraper (without video downloading)",
+				// Deadline
+				"DeadlineFlag":        "deadline",
+				"DeadlineDefault":     "1500",
+				"DeadlineDescription": "Sets the timout for scraper logic in seconds (used as a workaround for 'context deadline exceeded' error)",
+				// Limit
+				"LimitFlag":        "limit",
+				"LimitDefault":     "0",
+				"LimitDescription": "Sets the videos count limit (useful when there too many videos from the user or by hashtag)",
+			},
+		},
+	}
+)

go.mod

@@ -0,0 +1,5 @@
+module github.com/pikami/tiktok-dl
+
+go 1.14
+
+require github.com/chromedp/chromedp v0.5.3

go.sum

@@ -0,0 +1,16 @@
+github.com/chromedp/cdproto v0.0.0-20200116234248-4da64dd111ac h1:T7V5BXqnYd55Hj/g5uhDYumg9Fp3rMTS6bykYtTIFX4=
+github.com/chromedp/cdproto v0.0.0-20200116234248-4da64dd111ac/go.mod h1:PfAWWKJqjlGFYJEidUM6aVIWPr0EpobeyVWEEmplX7g=
+github.com/chromedp/chromedp v0.5.3 h1:F9LafxmYpsQhWQBdCs+6Sret1zzeeFyHS5LkRF//Ffg=
+github.com/chromedp/chromedp v0.5.3/go.mod h1:YLdPtndaHQ4rCpSpBG+IPpy9JvX0VD+7aaLxYgYj28w=
+github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0=
+github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
+github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8=
+github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
+github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo=
+github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
+github.com/knq/sysutil v0.0.0-20191005231841-15668db23d08 h1:V0an7KRw92wmJysvFvtqtKMAPmvS5O0jtB0nYo6t+gs=
+github.com/knq/sysutil v0.0.0-20191005231841-15668db23d08/go.mod h1:dFWs1zEqDjFtnBXsd1vPOZaLsESovai349994nHx3e0=
+github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM=
+github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

main.go

@@ -1,14 +1,15 @@
 package main
 
 import (
-	config "./models/config"
-	workflows "./workflows"
+	config "github.com/pikami/tiktok-dl/models/config"
+	workflows "github.com/pikami/tiktok-dl/workflows"
 )
 
 func main() {
 	config.GetConfig()
 	url := config.Config.URL
 	batchFilePath := config.Config.BatchFilePath
+	scrapedDataFilePath := config.Config.ScrapedDataFilePath
 
 	// Batch file
 	if workflows.CanUseDownloadBatchFile(batchFilePath) {
@@ -16,5 +17,11 @@ func main() {
 		return
 	}
 
+	// Scraped data file
+	if workflows.CanUseDownloadScrapedData(scrapedDataFilePath) {
+		workflows.DownloadScrapedData(scrapedDataFilePath)
+		return
+	}
+
 	workflows.StartWorkflowByParameter(url)
 }

models/config/config.go

@@ -4,37 +4,45 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"strconv"
+
+	res "github.com/pikami/tiktok-dl/resources"
 )
 
 // Config - Runtime configuration
 var Config struct {
-	URL           string
-	OutputPath    string
-	BatchFilePath string
-	Debug         bool
-	MetaData      bool
-	Quiet         bool
-	Deadline      int
-	Limit         int
-	JSONOnly      bool
+	URL                 string
+	OutputPath          string
+	BatchFilePath       string
+	ScrapedDataFilePath string
+	ArchiveFilePath     string
+	FailLogFilePath     string
+	Debug               bool
+	MetaData            bool
+	Quiet               bool
+	JSONOnly            bool
+	Deadline            int
+	Limit               int
 }
 
 // GetConfig - Returns Config object
 func GetConfig() {
-	outputPath := flag.String("output", "./downloads", "Output path")
-	batchFilePath := flag.String("batch-file", "", "File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored.")
-	debug := flag.Bool("debug", false, "Enables debug mode")
-	metadata := flag.Bool("metadata", false, "Write video metadata to a .json file")
-	quiet := flag.Bool("quiet", false, "Supress output")
-	deadline := flag.Int("deadline", 1500, "Sets the timout for scraper logic in seconds (used as a workaround for 'context deadline exceeded' error)")
-	limit := flag.Int("limit", 0, "Sets the videos count limit (useful when there too many videos from the user or by hashtag)")
-	jsonOnly := flag.Bool("json", false, "Just get JSON data from scraper (without video downloading)")
+	outputPath := flag.String(res.OutputFlag, res.OutputDefault, res.OutputDescription)
+	batchFilePath := flag.String(res.BatchFlag, res.BatchDefault, res.BatchDescription)
+	scrapedDataFilePath := flag.String(res.ScrapedDataFlag, res.ScrapedDataDefault, res.ScrapedDataDescription)
+	archive := flag.String(res.ArchiveFlag, res.ArchiveDefault, res.ArchiveDescription)
+	failLogPath := flag.String(res.FailLogFlag, res.FailLogDefault, res.FailLogDescription)
+	debug := flag.Bool(res.DebugFlag, parseBool(res.DebugDefault), res.DebugDescription)
+	metadata := flag.Bool(res.MetadataFlag, parseBool(res.MetadataDefault), res.MetadataDescription)
+	quiet := flag.Bool(res.QuietFlag, parseBool(res.QuietDefault), res.QuietDescription)
+	jsonOnly := flag.Bool(res.JsonFlag, parseBool(res.JsonDefault), res.JsonDescription)
+	deadline := flag.Int(res.DeadlineFlag, parseInt(res.DeadlineDefault), res.DeadlineDescription)
+	limit := flag.Int(res.LimitFlag, parseInt(res.LimitDefault), res.LimitDescription)
 	flag.Parse()
 
 	args := flag.Args()
-	if len(args) < 1 && *batchFilePath == "" {
-		fmt.Println("Usage: tiktok-dl [OPTIONS] TIKTOK_USERNAME|TIKTOK_URL")
-		fmt.Println("  or:  tiktok-dl [OPTIONS] -batch-file path/to/users.txt")
+	if len(args) < 1 && *batchFilePath == "" && *scrapedDataFilePath == "" {
+		fmt.Println(res.UsageLine)
 		os.Exit(2)
 	}
 
@@ -45,13 +53,32 @@ func GetConfig() {
 	}
 	Config.OutputPath = *outputPath
 	Config.BatchFilePath = *batchFilePath
+	Config.ScrapedDataFilePath = *scrapedDataFilePath
+	Config.ArchiveFilePath = *archive
+	Config.FailLogFilePath = *failLogPath
 	Config.Debug = *debug
 	Config.MetaData = *metadata
 	Config.Quiet = *quiet
 	if *jsonOnly {
 		Config.Quiet = true
 	}
+	Config.JSONOnly = *jsonOnly
 	Config.Deadline = *deadline
 	Config.Limit = *limit
-	Config.JSONOnly = *jsonOnly
+}
+
+func parseBool(str string) bool {
+	val, err := strconv.ParseBool(str)
+	if err != nil {
+		panic(err)
+	}
+	return val
+}
+
+func parseInt(str string) int {
+	val, err := strconv.Atoi(str)
+	if err != nil {
+		panic(err)
+	}
+	return val
 }

models/upload.go

@@ -1,11 +1,13 @@
 package models
 
 import (
-	res "../resources"
-	utils "../utils"
 	"encoding/json"
 	"os"
 	"strings"
+
+	res "github.com/pikami/tiktok-dl/resources"
+	checkErr "github.com/pikami/tiktok-dl/utils/checkErr"
+	log "github.com/pikami/tiktok-dl/utils/log"
 )
 
 // Upload - Upload object
@@ -47,16 +49,16 @@ func (u Upload) GetUploadID() string {
 func (u Upload) WriteToFile(outputPath string) {
 	bytes, err := json.Marshal(u)
 	if err != nil {
-		utils.Logf(res.ErrorCouldNotSerializeJSON, u.GetUploadID())
+		log.Logf(res.ErrorCouldNotSerializeJSON, u.GetUploadID())
 		panic(err)
 	}
 
 	// Create the file
 	out, err := os.Create(outputPath)
-	utils.CheckErr(err)
+	checkErr.CheckErr(err)
 	defer out.Close()
 
 	// Write to file
 	_, err = out.Write(bytes)
-	utils.CheckErr(err)
+	checkErr.CheckErr(err)
 }

models/upload_test.go

@@ -1,10 +1,11 @@
 package models
 
 import (
-	testUtil "../unitTestUtil"
-	utils "../utils"
 	"os"
 	"testing"
+
+	testUtil "github.com/pikami/tiktok-dl/unitTestUtil"
+	fileio "github.com/pikami/tiktok-dl/utils/fileio"
 )
 
 func TestParseUploads(t *testing.T) {
@@ -62,7 +63,7 @@ func TestWriteToFile(t *testing.T) {
 
 	upload.WriteToFile(filePath)
 
-	actual := utils.ReadFileToString(filePath)
+	actual := fileio.ReadFileToString(filePath)
 	tu.AssertString(actual, expected, "File content")
 
 	os.Remove(filePath)

package.json

@@ -1,14 +1,17 @@
 {
     "name": "tiktok-dl",
-    "version": "0.0.1",
+    "version": "2.0.0",
     "scripts": {
         "install-dependencies": "go get -v -t -d ./...",
+        "test:coverage": "go test -short -coverprofile=cov.out ./models ./utils",
         "test": "go test -v ./models && go test -v ./utils",
-        "clean": "rm -rf out",
-        "build:scraper": "node node_modules/terser/bin/terser -c -m -- scraper.js > out/scraper.js",
-        "build:app": "go build -o out/ -v .",
-        "build:dist": "mkdir out && npm run build:app && npm run build:scraper",
-        "build": "go build -v ."
+        "clean": "rm -rf out && rm -f scraper.min.js",
+        "build:scraper": "node node_modules/terser/bin/terser -c -m -- scraper.js > scraper.min.js",
+        "build:res": "cd generator && go run . && cd ..",
+        "build:app": "go build -o out/tiktok-dl -v .",
+        "build:dist": "npm run clean && mkdir out && npm run build:scraper && npm run build:res && npm run build:app",
+        "build": "go build -v .",
+        "sonar": "sonar-scanner -Dsonar.login=${SONAR_LOGIN} -Dproject.settings=.sonar/sonar-project.properties"
     },
     "dependencies": {
         "terser": "^4.6.3"

resources/errorStrings.go

@@ -0,0 +1,22 @@
+// Package resources - This file is automatically generated.
+// Do not edit this file manually.
+// Check `/generator/resources.go` to change generated content
+package resources
+
+//ErrorCouldNotSerializeJSON -
+var ErrorCouldNotSerializeJSON = "Could not serialize json for video: %s\n"
+
+//ErrorCouldNotRecogniseURL -
+var ErrorCouldNotRecogniseURL = "Could not recognise URL format of string %s"
+
+//Error -
+var Error = "Error : %s\n"
+
+//ErrorPathNotFound -
+var ErrorPathNotFound = "File path %s not found."
+
+//FailedOnItem -
+var FailedOnItem = "Failed while scraping item: %s\n"
+
+//FailedToLoadScraper -
+var FailedToLoadScraper = "Failed to load scraper"

resources/flags.go

@@ -0,0 +1,103 @@
+// Package resources - This file is automatically generated.
+// Do not edit this file manually.
+// Check `/generator/resources.go` to change generated content
+package resources
+
+//FailLogDefault -
+var FailLogDefault = ""
+
+//QuietDescription -
+var QuietDescription = "Suppress output"
+
+//ScrapedDataDefault -
+var ScrapedDataDefault = ""
+
+//ArchiveDescription -
+var ArchiveDescription = "Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it."
+
+//FailLogFlag -
+var FailLogFlag = "fail-log"
+
+//QuietDefault -
+var QuietDefault = "false"
+
+//JsonFlag -
+var JsonFlag = "json"
+
+//ScrapedDataFlag -
+var ScrapedDataFlag = "scraped-data"
+
+//DebugFlag -
+var DebugFlag = "debug"
+
+//MetadataDefault -
+var MetadataDefault = "false"
+
+//DeadlineFlag -
+var DeadlineFlag = "deadline"
+
+//OutputFlag -
+var OutputFlag = "output"
+
+//BatchFlag -
+var BatchFlag = "batch-file"
+
+//ArchiveFlag -
+var ArchiveFlag = "archive"
+
+//ArchiveDefault -
+var ArchiveDefault = ""
+
+//JsonDefault -
+var JsonDefault = "false"
+
+//JsonDescription -
+var JsonDescription = "Just get JSON data from scraper (without video downloading)"
+
+//DeadlineDescription -
+var DeadlineDescription = "Sets the timout for scraper logic in seconds (used as a workaround for 'context deadline exceeded' error)"
+
+//OutputDefault -
+var OutputDefault = "./downloads"
+
+//OutputDescription -
+var OutputDescription = "Output path"
+
+//BatchDescription -
+var BatchDescription = "File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored."
+
+//LimitFlag -
+var LimitFlag = "limit"
+
+//LimitDescription -
+var LimitDescription = "Sets the videos count limit (useful when there too many videos from the user or by hashtag)"
+
+//ScrapedDataDescription -
+var ScrapedDataDescription = "Download videos from scrape file (json format)"
+
+//DebugDescription -
+var DebugDescription = "Enables debug mode"
+
+//MetadataFlag -
+var MetadataFlag = "metadata"
+
+//BatchDefault -
+var BatchDefault = ""
+
+//MetadataDescription -
+var MetadataDescription = "Write video metadata to a .json file"
+
+//LimitDefault -
+var LimitDefault = "0"
+
+//DeadlineDefault -
+var DeadlineDefault = "1500"
+
+//FailLogDescription -
+var FailLogDescription = "Write failed items to log file"
+
+//DebugDefault -
+var DebugDefault = "false"
+
+//QuietFlag -
+var QuietFlag = "quiet"

resources/messages.go

@@ -0,0 +1,22 @@
+// Package resources - This file is automatically generated.
+// Do not edit this file manually.
+// Check `/generator/resources.go` to change generated content
+package resources
+
+//PreloadingItemsFound -
+var PreloadingItemsFound = "\rPreloading... %s items have been found."
+
+//Preloading -
+var Preloading = "\rPreloading..."
+
+//Retrieving -
+var Retrieving = "\nRetrieving items..."
+
+//ItemsFoundInArchive -
+var ItemsFoundInArchive = "%d items, found in archive. Skipping...\n"
+
+//Downloaded -
+var Downloaded = "\r[%d/%d] Downloaded"
+
+//UsageLine -
+var UsageLine = "Usage: tiktok-dl [OPTIONS] TIKTOK_USERNAME|TIKTOK_URL\n  or:  tiktok-dl [OPTIONS] -batch-file path/to/users.txt\n  or:  tiktok-dl [OPTIONS] -scraped-data path/to/data.json"

resources/scraper.go

@@ -0,0 +1,10 @@
+// Package resources - This file is automatically generated.
+// Do not edit this file manually.
+// Check `/generator/resources.go` to change generated content
+package resources
+
+//ScraperPath -
+var ScraperPath = "scraper.js"
+
+//ScraperScript -
+var ScraperScript = "optStrings={selectors:{feedLoading:\".tiktok-ui-loading-container\",modalArrowRight:\"div > div.video-card-container > img.arrow-right\",modalClose:\"div > div.video-card-container > img.control-icon.close\",modalPlayer:\"div.video-card-container > div.video-card-browse > video\",modalCaption:\"div.content-container > div.video-infos-container > h1\",modalSoundLink:\"div.content-container > div.video-infos-container > h2.music-info > a\",modalUploader:\".user-username\",videoPlayer:\"div.video-card-container > div > video\",videoCaption:\"div.content-container > div.video-infos-container > h1\",videoSoundLink:\"div.content-container > div.video-infos-container > h2 > a\",videoUploader:\".user-username\"},classes:{feedVideoItem:\"video-feed-item-wrapper\",modalCloseDisabled:\"disabled\",titleMessage:\"title\"},tags:{resultTag:\"video_urls\",resultParentTag:\"body\"},attributes:{src:\"src\"},tiktokMessages:[\"Couldn't find this account\",\"No videos yet\",\"Video currently unavailable\"]},currentState={preloadCount:0,finished:!1,limit:0},checkForErrors=function(){var e=document.getElementsByClassName(optStrings.classes.titleMessage);if(e&&e.length){var t=Array.from(e).find(e=>optStrings.tiktokMessages.includes(e.textContent)).textContent;if(t)return createVidUrlElement(\"ERR: \"+t),!0}return!1},createVidUrlElement=function(e){var t=document.createElement(optStrings.tags.resultTag);t.innerText=JSON.stringify(e),document.getElementsByTagName(optStrings.tags.resultParentTag)[0].appendChild(t),currentState.finished=!0},buldVidUrlArray=function(e){document.getElementsByClassName(optStrings.classes.feedVideoItem)[0].click();var t=[],r=window.setInterval(o=>{t.push(getCurrentModalVideo()),currentState.limit>0&&t.length>=currentState.limit&&(window.clearInterval(r),document.querySelector(optStrings.selectors.modalClose).click(),e(t));var n=document.querySelectorAll(optStrings.selectors.modalArrowRight)[0];!n||n.classList.contains(optStrings.classes.modalCloseDisabled)?(window.clearInterval(r),document.querySelector(optStrings.selectors.modalClose).click(),e(t)):n.click()},20)},getCurrentModalVideo=function(){var e=document.querySelector(optStrings.selectors.modalPlayer).getAttribute(optStrings.attributes.src),t=window.location.href,r=document.querySelector(optStrings.selectors.modalCaption).textContent,o=document.querySelector(optStrings.selectors.modalSoundLink),n=document.querySelector(optStrings.selectors.modalUploader).textContent,i=o?o.getAttribute(\"href\"):\"\";return{url:e,shareLink:t,caption:r,uploader:n,sound:{title:o?o.text:\"\",link:i}}},getCurrentVideo=function(){if(!checkForErrors()){var e=document.querySelector(optStrings.selectors.videoPlayer).getAttribute(optStrings.attributes.src),t=window.location.href,r=document.querySelector(optStrings.selectors.videoCaption).textContent,o=document.querySelector(optStrings.selectors.videoSoundLink),n=document.querySelector(optStrings.selectors.videoUploader).textContent,i=o?o.getAttribute(\"href\"):\"\";return{url:e,shareLink:t,caption:r,uploader:n,sound:{title:o?o.text:\"\",link:i}}}},scrollBottom=()=>window.scrollTo(0,document.body.scrollHeight),scrollWhileNew=function(e){var t={count:0},r=window.setInterval(o=>{scrollBottom();var n=t.count;if(t.count=document.getElementsByClassName(optStrings.classes.feedVideoItem).length,currentState.limit>0&&(currentState.preloadCount>=currentState.limit||t.count>=currentState.limit)&&(e(createVidUrlElement),window.clearInterval(r)),checkForErrors())window.clearInterval(r);else if(0!=t.count)if(n!==t.count)currentState.preloadCount=t.count;else{if(isLoading())return;window.clearInterval(r),e(createVidUrlElement)}},1e3)},isLoading=function(){var e=document.querySelector(optStrings.selectors.feedLoading);return e&&0!=e.getClientRects().length},bootstrapIteratingVideos=function(e){return currentState.limit=e,scrollWhileNew(buldVidUrlArray),\"bootstrapIteratingVideos\"},bootstrapGetCurrentVideo=function(){var e=getCurrentVideo();return createVidUrlElement(e),\"bootstrapGetCurrentVideo\"},init=()=>{const e=navigator.__proto__;return delete e.webdriver,navigator.__proto__=e,\"script initialized\"},init();"

resources/strings.go

@@ -1,13 +0,0 @@
-package resources
-
-// ErrorCouldNotSerializeJSON -
-var ErrorCouldNotSerializeJSON = "Could not serialize json for video: %s\n"
-
-// ErrorCouldNotRecogniseURL -
-var ErrorCouldNotRecogniseURL = "Could not recognise URL format of string %s"
-
-// ErrorCouldNotGetUserUploads -
-var ErrorCouldNotGetUserUploads = "Failed to get user uploads: %s\n"
-
-// ErrorPathNotFound -
-var ErrorPathNotFound = "File path %s not found."

scraper.js

@@ -1,17 +1,15 @@
 optStrings = {
     selectors: {
-        feedLoading: 'div.tiktok-loading.feed-loading',
-        modalArrowRight: 'div.video-card-modal > div > img.arrow-right',
-        modalClose: '.video-card-modal > div > div.close',
-        modalPlayer: 'div > div > main > div.video-card-modal > div > div.video-card-big > div.video-card-container > div > div > video',
-        modalShareInput: '.copy-link-container > input',
-        modalCaption: 'div.video-card-big > div.content-container > div.video-meta-info > h1',
-        modalSoundLink: 'div.content-container > div.video-meta-info > h2.music-info > a',
+        feedLoading: '.tiktok-ui-loading-container',
+        modalArrowRight: 'div > div.video-card-container > img.arrow-right',
+        modalClose: 'div > div.video-card-container > img.control-icon.close',
+        modalPlayer: 'div.video-card-container > div.video-card-browse > video',
+        modalCaption: 'div.content-container > div.video-infos-container > h1',
+        modalSoundLink: 'div.content-container > div.video-infos-container > h2.music-info > a',
         modalUploader: '.user-username',
-        videoPlayer: 'div.video-card-container > div > div > video',
-        videoShareInput: 'div.content-container.border > div.copy-link-container > input',
-        videoCaption: 'div.content-container.border > div.video-meta-info > h1',
-        videoSoundLink: 'div.content-container.border > div.video-meta-info > h2.music-info > a',
+        videoPlayer: 'div.video-card-container > div > video',
+        videoCaption: 'div.content-container > div.video-infos-container > h1',
+        videoSoundLink: 'div.content-container > div.video-infos-container > h2 > a',
         videoUploader: '.user-username',
     },
     classes: {
@@ -39,9 +37,9 @@ currentState = {
     limit: 0
 };
 
-checkForErrors = function() {
+checkForErrors = function () {
     var titles = document.getElementsByClassName(optStrings.classes.titleMessage);
-    debugger;
+    //debugger;
     if (titles && titles.length) {
         var error = Array.from(titles).find(x => optStrings.tiktokMessages.includes(x.textContent)).textContent;
         if (error) {
@@ -52,21 +50,21 @@ checkForErrors = function() {
     return false;
 };
 
-createVidUrlElement = function(outputObj) {
+createVidUrlElement = function (outputObj) {
     var urlSetElement = document.createElement(optStrings.tags.resultTag);
     urlSetElement.innerText = JSON.stringify(outputObj);
     document.getElementsByTagName(optStrings.tags.resultParentTag)[0].appendChild(urlSetElement);
     currentState.finished = true;
 };
 
-buldVidUrlArray = function(finishCallback) {
+buldVidUrlArray = function (finishCallback) {
     var feedItem = document.getElementsByClassName(optStrings.classes.feedVideoItem)[0];
     feedItem.click();
 
     var videoArray = [];
     var intervalID = window.setInterval(x => {
         videoArray.push(getCurrentModalVideo());
-        if(currentState.limit > 0) {
+        if (currentState.limit > 0) {
             if (videoArray.length >= currentState.limit) {
                 window.clearInterval(intervalID);
                 document.querySelector(optStrings.selectors.modalClose).click();
@@ -74,7 +72,7 @@ buldVidUrlArray = function(finishCallback) {
             }
         }
         var arrowRight = document.querySelectorAll(optStrings.selectors.modalArrowRight)[0];
-        if (arrowRight.classList.contains(optStrings.classes.modalCloseDisabled)) {
+        if (!arrowRight || arrowRight.classList.contains(optStrings.classes.modalCloseDisabled)) {
             window.clearInterval(intervalID);
             document.querySelector(optStrings.selectors.modalClose).click();
             finishCallback(videoArray);
@@ -84,15 +82,15 @@ buldVidUrlArray = function(finishCallback) {
     }, 20);
 };
 
-getCurrentModalVideo = function() {
+getCurrentModalVideo = function () {
     var modalPlayer = document.querySelector(optStrings.selectors.modalPlayer);
     var vidUrl = modalPlayer.getAttribute(optStrings.attributes.src);
-    var shareLink = document.querySelector(optStrings.selectors.modalShareInput).value;
+    var shareLink = window.location.href;
     var caption = document.querySelector(optStrings.selectors.modalCaption).textContent;
     var soundLink = document.querySelector(optStrings.selectors.modalSoundLink);
     var uploader = document.querySelector(optStrings.selectors.modalUploader).textContent;
-    var soundHref = soundLink.getAttribute("href");
-    var soundText = soundLink.text;
+    var soundHref = soundLink ? soundLink.getAttribute("href") : '';
+    var soundText = soundLink ? soundLink.text : '';
 
     return {
         url: vidUrl,
@@ -106,16 +104,17 @@ getCurrentModalVideo = function() {
     };
 };
 
-getCurrentVideo = function() {
-    if(checkForErrors()) return;
+getCurrentVideo = function () {
+    //debugger;
+    if (checkForErrors()) return;
     var player = document.querySelector(optStrings.selectors.videoPlayer);
     var vidUrl = player.getAttribute(optStrings.attributes.src);
-    var shareLink = document.querySelector(optStrings.selectors.videoShareInput).value;
+    var shareLink = window.location.href;
     var caption = document.querySelector(optStrings.selectors.videoCaption).textContent;
     var soundLink = document.querySelector(optStrings.selectors.videoSoundLink);
     var uploader = document.querySelector(optStrings.selectors.videoUploader).textContent;
-    var soundHref = soundLink.getAttribute("href");
-    var soundText = soundLink.text;
+    var soundHref = soundLink ? soundLink.getAttribute("href") : '';
+    var soundText = soundLink ? soundLink.text : '';
 
     return {
         url: vidUrl,
@@ -129,27 +128,32 @@ getCurrentVideo = function() {
     };
 };
 
-scrollWhileNew = function(finishCallback) {
-    var state = { count: 0 };
+scrollBottom = () => window.scrollTo(0, document.body.scrollHeight);
+
+scrollWhileNew = function (finishCallback) {
+    var state = {
+        count: 0
+    };
     var intervalID = window.setInterval(x => {
+        scrollBottom();
         var oldCount = state.count;
         state.count = document.getElementsByClassName(optStrings.classes.feedVideoItem).length;
-        if(currentState.limit > 0) {
+        if (currentState.limit > 0) {
             if (currentState.preloadCount >= currentState.limit || state.count >= currentState.limit) {
                 finishCallback(createVidUrlElement);
                 window.clearInterval(intervalID);
             }
         }
-        if(checkForErrors()) {
+        if (checkForErrors()) {
             window.clearInterval(intervalID);
             return;
+        } else if (state.count == 0) {
+            return;
         }
         if (oldCount !== state.count) {
             currentState.preloadCount = state.count;
-            window.scrollTo(0, document.body.scrollHeight);
         } else {
-            if (document.querySelector(optStrings.selectors.feedLoading)) {
-                window.scrollTo(0, document.body.scrollHeight);
+            if (isLoading()) {
                 return;
             }
             window.clearInterval(intervalID);
@@ -158,13 +162,18 @@ scrollWhileNew = function(finishCallback) {
     }, 1000);
 };
 
-bootstrapIteratingVideos = function(limit) {
+isLoading = function () {
+    var loadingElement = document.querySelector(optStrings.selectors.feedLoading);
+    return loadingElement && loadingElement.getClientRects().length != 0;
+}
+
+bootstrapIteratingVideos = function (limit) {
     currentState.limit = limit;
     scrollWhileNew(buldVidUrlArray);
     return 'bootstrapIteratingVideos';
 };
 
-bootstrapGetCurrentVideo = function() {
+bootstrapGetCurrentVideo = function () {
     var video = getCurrentVideo();
     createVidUrlElement(video);
     return 'bootstrapGetCurrentVideo';

utils/archive.go

@@ -0,0 +1,55 @@
+package utils
+
+import (
+	models "github.com/pikami/tiktok-dl/models"
+	config "github.com/pikami/tiktok-dl/models/config"
+	res "github.com/pikami/tiktok-dl/resources"
+	fileio "github.com/pikami/tiktok-dl/utils/fileio"
+	log "github.com/pikami/tiktok-dl/utils/log"
+)
+
+// IsItemInArchive - Checks if the item is already archived
+func IsItemInArchive(upload models.Upload) bool {
+	if len(RemoveArchivedItems([]models.Upload{upload})) == 0 {
+		return true
+	}
+	return false
+}
+
+// RemoveArchivedItems - Returns items slice without archived items
+func RemoveArchivedItems(uploads []models.Upload) []models.Upload {
+	archiveFilePath := config.Config.ArchiveFilePath
+
+	if archiveFilePath == "" || !fileio.CheckIfExists(archiveFilePath) {
+		return uploads
+	}
+
+	removeArchivedItemsDelegate := func(archivedItem string) {
+		for i, upload := range uploads {
+			if upload.GetUploadID() == archivedItem {
+				uploads = append(uploads[:i], uploads[i+1:]...)
+			}
+		}
+	}
+
+	lenBeforeRemoval := len(uploads)
+	fileio.ReadFileLineByLine(archiveFilePath, removeArchivedItemsDelegate)
+
+	removedCount := lenBeforeRemoval - len(uploads)
+	if removedCount > 0 {
+		log.Logf(res.ItemsFoundInArchive, removedCount)
+	}
+
+	return uploads
+}
+
+// AddItemToArchive - Adds item to archived list
+func AddItemToArchive(uploadID string) {
+	archiveFilePath := config.Config.ArchiveFilePath
+
+	if archiveFilePath == "" {
+		return
+	}
+
+	fileio.AppendToFile(uploadID, archiveFilePath)
+}

utils/downloadFile.go

@@ -4,21 +4,23 @@ import (
 	"io"
 	"net/http"
 	"os"
+
+	checkErr "github.com/pikami/tiktok-dl/utils/checkErr"
 )
 
 // DownloadFile - Downloads content from `url` and stores it in `outputPath`
 func DownloadFile(outputPath string, url string) {
 	// Get the data
 	resp, err := http.Get(url)
-	CheckErr(err)
+	checkErr.CheckErr(err)
 	defer resp.Body.Close()
 
 	// Create the file
 	out, err := os.Create(outputPath)
-	CheckErr(err)
+	checkErr.CheckErr(err)
 	defer out.Close()
 
 	// Write the body to file
 	_, err = io.Copy(out, resp.Body)
-	CheckErr(err)
+	checkErr.CheckErr(err)
 }

utils/fileio/fileio.go

@@ -4,6 +4,8 @@ import (
 	"bufio"
 	"io/ioutil"
 	"os"
+
+	checkErr "github.com/pikami/tiktok-dl/utils/checkErr"
 )
 
 type delegateString func(string)
@@ -37,7 +39,7 @@ func ReadFileToString(path string) string {
 // ReadFileLineByLine - Reads file line by line and calls delegate
 func ReadFileLineByLine(path string, delegate delegateString) {
 	file, err := os.Open(path)
-	CheckErr(err)
+	checkErr.CheckErr(err)
 	defer file.Close()
 
 	scanner := bufio.NewScanner(file)
@@ -49,3 +51,14 @@ func ReadFileLineByLine(path string, delegate delegateString) {
 		panic(err)
 	}
 }
+
+// AppendToFile - Appends line to file
+func AppendToFile(str string, filePath string) {
+	f, err := os.OpenFile(filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	checkErr.CheckErr(err)
+
+	defer f.Close()
+	if _, err := f.WriteString(str + "\n"); err != nil {
+		checkErr.CheckErr(err)
+	}
+}

utils/getHashtag.go

@@ -1,9 +1,10 @@
 package utils
 
 import (
-	res "../resources"
 	"fmt"
 	"strings"
+
+	res "github.com/pikami/tiktok-dl/resources"
 )
 
 // GetHashtagFromURL - Get's tag name from passed url

utils/getScraper.go

@@ -0,0 +1,19 @@
+package utils
+
+import (
+	resources "github.com/pikami/tiktok-dl/resources"
+	fileio "github.com/pikami/tiktok-dl/utils/fileio"
+)
+
+// GetScraper - Retrieve scraper
+func GetScraper() string {
+	if fileio.CheckIfExists(resources.ScraperPath) {
+		return ReadFileAsString(resources.ScraperPath)
+	}
+
+	if resources.ScraperScript != "" {
+		return resources.ScraperScript
+	}
+
+	panic(resources.FailedToLoadScraper)
+}

utils/getUsername.go

@@ -1,11 +1,12 @@
 package utils
 
 import (
-	config "../models/config"
-	res "../resources"
 	"fmt"
 	"regexp"
 	"strings"
+
+	config "github.com/pikami/tiktok-dl/models/config"
+	res "github.com/pikami/tiktok-dl/resources"
 )
 
 // GetUsername - Get's username from passed URL param

utils/getUsername_test.go

@@ -1,9 +1,10 @@
 package utils
 
 import (
-	config "../models/config"
-	testUtil "../unitTestUtil"
 	"testing"
+
+	config "github.com/pikami/tiktok-dl/models/config"
+	testUtil "github.com/pikami/tiktok-dl/unitTestUtil"
 )
 
 func TestGetUsername(t *testing.T) {

utils/log/log.go

@@ -1,9 +1,10 @@
 package utils
 
 import (
-	config "../models/config"
 	"fmt"
 	"os"
+
+	config "github.com/pikami/tiktok-dl/models/config"
 )
 
 // Log - Write to std out

utils/readFileAsString.go

@@ -2,11 +2,13 @@ package utils
 
 import (
 	"io/ioutil"
+
+	checkErr "github.com/pikami/tiktok-dl/utils/checkErr"
 )
 
 // ReadFileAsString - Returns contents of given file
 func ReadFileAsString(fileName string) string {
 	content, err := ioutil.ReadFile(fileName)
-	CheckErr(err)
+	checkErr.CheckErr(err)
 	return string(content)
 }

workflows/common.go

@@ -0,0 +1,20 @@
+package workflows
+
+import (
+	config "github.com/pikami/tiktok-dl/models/config"
+	res "github.com/pikami/tiktok-dl/resources"
+	fileio "github.com/pikami/tiktok-dl/utils/fileio"
+	log "github.com/pikami/tiktok-dl/utils/log"
+)
+
+// OnWorkflowFail - Function called when workflow fails
+func OnWorkflowFail(err error, workItem string) {
+	failLogFilePath := config.Config.FailLogFilePath
+
+	if failLogFilePath != "" {
+		fileio.AppendToFile(workItem, failLogFilePath)
+	}
+
+	log.LogErr(res.Error, err.Error())
+	log.LogErr(res.FailedOnItem, workItem)
+}

workflows/downloadBatchFile.go

@@ -1,8 +1,9 @@
 package workflows
 
 import (
-	res "../resources"
-	utils "../utils"
+	res "github.com/pikami/tiktok-dl/resources"
+	fileio "github.com/pikami/tiktok-dl/utils/fileio"
+	log "github.com/pikami/tiktok-dl/utils/log"
 )
 
 // CanUseDownloadBatchFile - Check's if DownloadBatchFile can be used
@@ -12,11 +13,11 @@ func CanUseDownloadBatchFile(batchFilePath string) bool {
 
 // DownloadBatchFile - Download items from batch file
 func DownloadBatchFile(batchFilePath string) {
-	if !utils.CheckIfExists(batchFilePath) {
-		utils.LogFatal(res.ErrorPathNotFound, batchFilePath)
+	if !fileio.CheckIfExists(batchFilePath) {
+		log.LogFatal(res.ErrorPathNotFound, batchFilePath)
 	}
 
-	utils.ReadFileLineByLine(batchFilePath, downloadItem)
+	fileio.ReadFileLineByLine(batchFilePath, downloadItem)
 }
 
 func downloadItem(batchItem string) {

workflows/downloadHashtag.go

@@ -1,12 +1,15 @@
 package workflows
 
 import (
-	client "../client"
-	config "../models/config"
-	res "../resources"
-	utils "../utils"
 	"fmt"
 	"strings"
+
+	client "github.com/pikami/tiktok-dl/client"
+	config "github.com/pikami/tiktok-dl/models/config"
+	res "github.com/pikami/tiktok-dl/resources"
+	utils "github.com/pikami/tiktok-dl/utils"
+	fileio "github.com/pikami/tiktok-dl/utils/fileio"
+	log "github.com/pikami/tiktok-dl/utils/log"
 )
 
 // CanUseDownloadHashtag - Test's if this workflow can be used for parameter
@@ -19,26 +22,30 @@ func CanUseDownloadHashtag(url string) bool {
 func DownloadHashtag(url string) {
 	uploads, err := client.GetHashtagUploads(url)
 	if err != nil {
-		utils.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
+		OnWorkflowFail(err, url)
 		return
 	}
+
+	uploads = utils.RemoveArchivedItems(uploads)
 	uploadCount := len(uploads)
+
 	hashtag := utils.GetHashtagFromURL(url)
 	downloadDir := fmt.Sprintf("%s/%s", config.Config.OutputPath, hashtag)
 
-	utils.InitOutputDirectory(downloadDir)
+	fileio.InitOutputDirectory(downloadDir)
 
 	for index, upload := range uploads {
 		downloadVideo(upload, downloadDir)
-		utils.Logf("\r[%d/%d] Downloaded", index+1, uploadCount)
+		log.Logf(res.Downloaded, index+1, uploadCount)
 	}
-	utils.Log()
+	log.Log()
 }
 
-func GetHashtagJson(url string) {
-	uploads, err := client.GetHashtagUploads(url)
+// GetHashtagJSON - Prints scraped info from hashtag
+func GetHashtagJSON(url string) {
+	uploads, err := client.GetHashtagUploadsJSON(url)
 	if err != nil {
-		utils.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
+		OnWorkflowFail(err, url)
 		return
 	}
 	fmt.Printf("%s", uploads)

workflows/downloadMusic.go

@@ -1,12 +1,15 @@
 package workflows
 
 import (
-	client "../client"
-	config "../models/config"
-	res "../resources"
-	utils "../utils"
 	"fmt"
 	"regexp"
+
+	client "github.com/pikami/tiktok-dl/client"
+	config "github.com/pikami/tiktok-dl/models/config"
+	res "github.com/pikami/tiktok-dl/resources"
+	utils "github.com/pikami/tiktok-dl/utils"
+	fileio "github.com/pikami/tiktok-dl/utils/fileio"
+	log "github.com/pikami/tiktok-dl/utils/log"
 )
 
 // CanUseDownloadMusic - Check's if DownloadMusic can be used for parameter
@@ -19,26 +22,29 @@ func CanUseDownloadMusic(url string) bool {
 func DownloadMusic(url string) {
 	uploads, err := client.GetMusicUploads(url)
 	if err != nil {
-		utils.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
+		OnWorkflowFail(err, url)
 		return
 	}
+
+	uploads = utils.RemoveArchivedItems(uploads)
 	uploadCount := len(uploads)
 
 	for index, upload := range uploads {
 		username := utils.GetUsernameFromString(upload.Uploader)
 		downloadDir := fmt.Sprintf("%s/%s", config.Config.OutputPath, username)
 
-		utils.InitOutputDirectory(downloadDir)
+		fileio.InitOutputDirectory(downloadDir)
 		downloadVideo(upload, downloadDir)
-		utils.Logf("\r[%d/%d] Downloaded", index+1, uploadCount)
+		log.Logf(res.Downloaded, index+1, uploadCount)
 	}
-	utils.Log()
+	log.Log()
 }
 
-func GetMusicJson(url string) {
-	uploads, err := client.GetMusicUploadsJson(url)
+// GetMusicJSON - Prints scraped info from music
+func GetMusicJSON(url string) {
+	uploads, err := client.GetMusicUploadsJSON(url)
 	if err != nil {
-		utils.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
+		OnWorkflowFail(err, url)
 		return
 	}
 	fmt.Printf("%s", uploads)

workflows/downloadScrapedData.go

@@ -0,0 +1,39 @@
+package workflows
+
+import (
+	"fmt"
+	models "github.com/pikami/tiktok-dl/models"
+	config "github.com/pikami/tiktok-dl/models/config"
+	res "github.com/pikami/tiktok-dl/resources"
+	utils "github.com/pikami/tiktok-dl/utils"
+	fileio "github.com/pikami/tiktok-dl/utils/fileio"
+	log "github.com/pikami/tiktok-dl/utils/log"
+)
+
+// CanUseDownloadScrapedData - Check's if DownloadScrapedData can be used
+func CanUseDownloadScrapedData(scrapedDataFilePath string) bool {
+	return scrapedDataFilePath != ""
+}
+
+// DownloadScrapedData - Download items from scraped data file
+func DownloadScrapedData(scrapedDataFilePath string) {
+	if !fileio.CheckIfExists(scrapedDataFilePath) {
+		log.LogFatal(res.ErrorPathNotFound, scrapedDataFilePath)
+	}
+
+	dataFileContent := fileio.ReadFileToString(scrapedDataFilePath)
+	uploads := models.ParseUploads(dataFileContent)
+	uploads = utils.RemoveArchivedItems(uploads)
+
+	uploadCount := len(uploads)
+
+	for index, upload := range uploads {
+		username := utils.GetUsernameFromString(upload.Uploader)
+		downloadDir := fmt.Sprintf("%s/%s", config.Config.OutputPath, username)
+
+		fileio.InitOutputDirectory(downloadDir)
+		downloadVideo(upload, downloadDir)
+		log.Logf(res.Downloaded, index+1, uploadCount)
+	}
+	log.Log()
+}

workflows/downloadShareLink.go

@@ -0,0 +1,27 @@
+package workflows
+
+import (
+	"regexp"
+
+	client "github.com/pikami/tiktok-dl/client"
+	log "github.com/pikami/tiktok-dl/utils/log"
+)
+
+// CanUseDownloadShareLink - Check's if DownloadShareLink can be used
+func CanUseDownloadShareLink(url string) bool {
+	match, _ := regexp.MatchString("vm.tiktok.com\\/.+", url)
+	return match
+}
+
+// DownloadShareLink - Download item by share link
+func DownloadShareLink(url string) {
+	log.Logf("Resolving share link: %s\n", url)
+
+	finalURL, err := client.GetRedirectURL(url)
+	if err != nil {
+		OnWorkflowFail(err, url)
+		return
+	}
+
+	StartWorkflowByParameter(finalURL)
+}

workflows/downloadUser.go

@@ -1,13 +1,16 @@
 package workflows
 
 import (
-	client "../client"
-	config "../models/config"
-	res "../resources"
-	utils "../utils"
 	"fmt"
 	"regexp"
 	"strings"
+
+	client "github.com/pikami/tiktok-dl/client"
+	config "github.com/pikami/tiktok-dl/models/config"
+	res "github.com/pikami/tiktok-dl/resources"
+	utils "github.com/pikami/tiktok-dl/utils"
+	fileio "github.com/pikami/tiktok-dl/utils/fileio"
+	log "github.com/pikami/tiktok-dl/utils/log"
 )
 
 // CanUseDownloadUser - Test's if this workflow can be used for parameter
@@ -21,25 +24,29 @@ func CanUseDownloadUser(url string) bool {
 func DownloadUser(username string) {
 	uploads, err := client.GetUserUploads(username)
 	if err != nil {
-		utils.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
+		OnWorkflowFail(err, username)
 		return
 	}
+
+	uploads = utils.RemoveArchivedItems(uploads)
 	uploadCount := len(uploads)
+
 	downloadDir := fmt.Sprintf("%s/%s", config.Config.OutputPath, username)
 
-	utils.InitOutputDirectory(downloadDir)
+	fileio.InitOutputDirectory(downloadDir)
 
 	for index, upload := range uploads {
 		downloadVideo(upload, downloadDir)
-		utils.Logf("\r[%d/%d] Downloaded", index+1, uploadCount)
+		log.Logf(res.Downloaded, index+1, uploadCount)
 	}
-	utils.Log()
+	log.Log()
 }
 
-func GetUserVideosJson(username string) {
-	uploads, err := client.GetUserUploadsJson(username)
+// GetUserVideosJSON - Prints scraped info from user
+func GetUserVideosJSON(username string) {
+	uploads, err := client.GetUserUploadsJSON(username)
 	if err != nil {
-		utils.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
+		OnWorkflowFail(err, username)
 		return
 	}
 	fmt.Printf("%s", uploads)

workflows/downloadVideo.go

@@ -1,13 +1,16 @@
 package workflows
 
 import (
-	client "../client"
-	models "../models"
-	config "../models/config"
-	res "../resources"
-	utils "../utils"
 	"fmt"
 	"regexp"
+
+	client "github.com/pikami/tiktok-dl/client"
+	models "github.com/pikami/tiktok-dl/models"
+	config "github.com/pikami/tiktok-dl/models/config"
+	res "github.com/pikami/tiktok-dl/resources"
+	utils "github.com/pikami/tiktok-dl/utils"
+	fileio "github.com/pikami/tiktok-dl/utils/fileio"
+	log "github.com/pikami/tiktok-dl/utils/log"
 )
 
 // CanUseDownloadSingleVideo - Check's if DownloadSingleVideo can be used for parameter
@@ -21,14 +24,18 @@ func DownloadSingleVideo(url string) {
 	username := utils.GetUsernameFromString(url)
 	upload, err := client.GetVideoDetails(url)
 	if err != nil {
-		utils.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
+		OnWorkflowFail(err, url)
+		return
+	}
+
+	if utils.IsItemInArchive(upload) {
 		return
 	}
 	downloadDir := fmt.Sprintf("%s/%s", config.Config.OutputPath, username)
 
-	utils.InitOutputDirectory(downloadDir)
+	fileio.InitOutputDirectory(downloadDir)
 	downloadVideo(upload, downloadDir)
-	utils.Log("[1/1] Downloaded\n")
+	log.Logf(res.Downloaded, 1, 1)
 }
 
 // DownloadVideo - Downloads one video
@@ -36,7 +43,7 @@ func downloadVideo(upload models.Upload, downloadDir string) {
 	uploadID := upload.GetUploadID()
 	downloadPath := fmt.Sprintf("%s/%s.mp4", downloadDir, uploadID)
 
-	if utils.CheckIfExists(downloadPath) {
+	if fileio.CheckIfExists(downloadPath) {
 		return
 	}
 
@@ -46,4 +53,6 @@ func downloadVideo(upload models.Upload, downloadDir string) {
 		metadataPath := fmt.Sprintf("%s/%s.json", downloadDir, uploadID)
 		upload.WriteToFile(metadataPath)
 	}
+
+	utils.AddItemToArchive(upload.GetUploadID())
 }

workflows/startWorkflowByParameter.go

@@ -1,9 +1,10 @@
 package workflows
 
 import (
-	config "../models/config"
-	res "../resources"
-	utils "../utils"
+	config "github.com/pikami/tiktok-dl/models/config"
+	res "github.com/pikami/tiktok-dl/resources"
+	utils "github.com/pikami/tiktok-dl/utils"
+	log "github.com/pikami/tiktok-dl/utils/log"
 )
 
 // StartWorkflowByParameter - Start needed workflow by given parameter
@@ -12,7 +13,7 @@ func StartWorkflowByParameter(url string) {
 	// Music
 	if CanUseDownloadMusic(url) {
 		if config.Config.JSONOnly {
-			GetMusicJson(url)
+			GetMusicJSON(url)
 		} else {
 			DownloadMusic(url)
 		}
@@ -28,7 +29,7 @@ func StartWorkflowByParameter(url string) {
 	// Tiktok user
 	if CanUseDownloadUser(url) {
 		if config.Config.JSONOnly {
-			GetUserVideosJson(utils.GetUsernameFromString(url))
+			GetUserVideosJSON(utils.GetUsernameFromString(url))
 		} else {
 			DownloadUser(utils.GetUsernameFromString(url))
 		}
@@ -39,12 +40,18 @@ func StartWorkflowByParameter(url string) {
 	// Tiktok hashtag
 	if CanUseDownloadHashtag(url) {
 		if config.Config.JSONOnly {
-			GetHashtagJson(url)
+			GetHashtagJSON(url)
 		} else {
 			DownloadHashtag(url)
 		}
 		return
 	}
 
-	utils.LogFatal(res.ErrorCouldNotRecogniseURL, url)
+	// Share URL
+	if CanUseDownloadShareLink(url) {
+		DownloadShareLink(url)
+		return
+	}
+
+	log.LogFatal(res.ErrorCouldNotRecogniseURL, url)
 }