mirror of https://github.com/pikami/tiktok-dl.git
Embed scraper into compiled binary, extract strings to resources file
This commit is contained in:
parent
70c3faf17e
commit
ea2b866f9a
|
@ -7,3 +7,4 @@ tiktok-dl
|
||||||
batch_file.txt
|
batch_file.txt
|
||||||
debug.log
|
debug.log
|
||||||
.scannerwork
|
.scannerwork
|
||||||
|
scraper.min.js
|
||||||
|
|
|
@ -11,6 +11,7 @@ import (
|
||||||
"github.com/chromedp/chromedp"
|
"github.com/chromedp/chromedp"
|
||||||
|
|
||||||
config "github.com/pikami/tiktok-dl/models/config"
|
config "github.com/pikami/tiktok-dl/models/config"
|
||||||
|
res "github.com/pikami/tiktok-dl/resources"
|
||||||
utils "github.com/pikami/tiktok-dl/utils"
|
utils "github.com/pikami/tiktok-dl/utils"
|
||||||
log "github.com/pikami/tiktok-dl/utils/log"
|
log "github.com/pikami/tiktok-dl/utils/log"
|
||||||
)
|
)
|
||||||
|
@ -54,7 +55,7 @@ func runScrapeQuiet(ctx context.Context, jsAction string, url string) (string, e
|
||||||
// Navigate to user's page
|
// Navigate to user's page
|
||||||
chromedp.Navigate(url),
|
chromedp.Navigate(url),
|
||||||
// Execute url grabber script
|
// Execute url grabber script
|
||||||
chromedp.EvaluateAsDevTools(utils.ReadFileAsString("scraper.js"), &jsOutput),
|
chromedp.EvaluateAsDevTools(utils.GetScraper(), &jsOutput),
|
||||||
chromedp.EvaluateAsDevTools(jsAction, &jsOutput),
|
chromedp.EvaluateAsDevTools(jsAction, &jsOutput),
|
||||||
// Wait until custom js finishes
|
// Wait until custom js finishes
|
||||||
chromedp.WaitVisible(`video_urls`),
|
chromedp.WaitVisible(`video_urls`),
|
||||||
|
@ -73,7 +74,7 @@ func runScrapeWithInfo(ctx context.Context, jsAction string, url string) (string
|
||||||
// Navigate to user's page
|
// Navigate to user's page
|
||||||
chromedp.Navigate(url),
|
chromedp.Navigate(url),
|
||||||
// Execute url grabber script
|
// Execute url grabber script
|
||||||
chromedp.EvaluateAsDevTools(utils.ReadFileAsString("scraper.js"), &jsOutput),
|
chromedp.EvaluateAsDevTools(utils.GetScraper(), &jsOutput),
|
||||||
chromedp.EvaluateAsDevTools(jsAction, &jsOutput),
|
chromedp.EvaluateAsDevTools(jsAction, &jsOutput),
|
||||||
); err != nil {
|
); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
|
@ -85,9 +86,9 @@ func runScrapeWithInfo(ctx context.Context, jsAction string, url string) (string
|
||||||
}
|
}
|
||||||
|
|
||||||
if jsOutput != "0" {
|
if jsOutput != "0" {
|
||||||
log.Logf("\rPreloading... %s items have been found.", jsOutput)
|
log.Logf(res.PreloadingItemsFound, jsOutput)
|
||||||
} else {
|
} else {
|
||||||
log.Logf("\rPreloading...")
|
log.Logf(res.Preloading)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.finished.toString()", &jsOutput)); err != nil {
|
if err := chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.finished.toString()", &jsOutput)); err != nil {
|
||||||
|
@ -101,7 +102,7 @@ func runScrapeWithInfo(ctx context.Context, jsAction string, url string) (string
|
||||||
time.Sleep(50 * time.Millisecond)
|
time.Sleep(50 * time.Millisecond)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Log("\nRetrieving items...")
|
log.Log(res.Retrieving)
|
||||||
if err := chromedp.Run(ctx,
|
if err := chromedp.Run(ctx,
|
||||||
// Wait until custom js finishes
|
// Wait until custom js finishes
|
||||||
chromedp.WaitVisible(`video_urls`),
|
chromedp.WaitVisible(`video_urls`),
|
||||||
|
|
|
@ -0,0 +1,64 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
checkErr "github.com/pikami/tiktok-dl/utils/checkErr"
|
||||||
|
fileio "github.com/pikami/tiktok-dl/utils/fileio"
|
||||||
|
)
|
||||||
|
|
||||||
|
type resource struct {
|
||||||
|
Package string
|
||||||
|
FileName string
|
||||||
|
Values map[string]string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r resource) generate() {
|
||||||
|
filename := fmt.Sprintf("%s/%s", outputDir, r.FileName)
|
||||||
|
f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
|
||||||
|
checkErr.CheckErr(err)
|
||||||
|
defer f.Close()
|
||||||
|
|
||||||
|
// Header
|
||||||
|
header := fmt.Sprintf("// Package %s - This file is automatically generated.\n"+
|
||||||
|
"// Do not edit this file manually.\n"+
|
||||||
|
"// Check `/generator/resources.go` to change generated content\n"+
|
||||||
|
"package %s\n", r.Package, r.Package)
|
||||||
|
|
||||||
|
if _, err := f.WriteString(header); err != nil {
|
||||||
|
checkErr.CheckErr(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Values
|
||||||
|
for key, value := range r.Values {
|
||||||
|
value = strings.ReplaceAll(value, "\n", "\\n")
|
||||||
|
value = strings.ReplaceAll(value, "\r", "\\r")
|
||||||
|
valueLine := fmt.Sprintf("\n//%s -\nvar %s = \"%s\"\n", key, key, value)
|
||||||
|
if _, err := f.WriteString(valueLine); err != nil {
|
||||||
|
checkErr.CheckErr(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fileContentsOrDefault(file string) string {
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
fmt.Printf("Failed to load file: %s\n", file)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
return safeString(fileio.ReadFileToString(file))
|
||||||
|
}
|
||||||
|
|
||||||
|
func safeString(str string) string {
|
||||||
|
escaped := strings.ReplaceAll(str, "\"", "\\\"")
|
||||||
|
return strings.ReplaceAll(escaped, "\n", "")
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
for _, i := range res {
|
||||||
|
i.generate()
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,87 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
var (
|
||||||
|
outputDir = "../resources"
|
||||||
|
|
||||||
|
res = []resource{
|
||||||
|
resource{
|
||||||
|
Package: "resources",
|
||||||
|
FileName: "scraper.go",
|
||||||
|
Values: map[string]string{
|
||||||
|
"ScraperPath": "scraper.js",
|
||||||
|
"ScraperScript": fileContentsOrDefault("../scraper.min.js"),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
resource{
|
||||||
|
Package: "resources",
|
||||||
|
FileName: "errorStrings.go",
|
||||||
|
Values: map[string]string{
|
||||||
|
"ErrorCouldNotSerializeJSON": "Could not serialize json for video: %s\n",
|
||||||
|
"ErrorCouldNotRecogniseURL": "Could not recognise URL format of string %s",
|
||||||
|
"Error": "Error : %s\n",
|
||||||
|
"ErrorPathNotFound": "File path %s not found.",
|
||||||
|
"FailedOnItem": "Failed while scraping item: %s\n",
|
||||||
|
"FailedToLoadScraper": "Failed to load scraper",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
resource{
|
||||||
|
Package: "resources",
|
||||||
|
FileName: "messages.go",
|
||||||
|
Values: map[string]string{
|
||||||
|
"PreloadingItemsFound": "\rPreloading... %s items have been found.",
|
||||||
|
"Preloading": "\rPreloading...",
|
||||||
|
"Retrieving": "\nRetrieving items...",
|
||||||
|
"ItemsFoundInArchive": "%d items, found in archive. Skipping...\n",
|
||||||
|
"Downloaded": "\r[%d/%d] Downloaded",
|
||||||
|
"UsageLine": "Usage: tiktok-dl [OPTIONS] TIKTOK_USERNAME|TIKTOK_URL\n" +
|
||||||
|
" or: tiktok-dl [OPTIONS] -batch-file path/to/users.txt",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
resource{
|
||||||
|
Package: "resources",
|
||||||
|
FileName: "flags.go",
|
||||||
|
Values: map[string]string{
|
||||||
|
// Output
|
||||||
|
"OutputFlag": "output",
|
||||||
|
"OutputDefault": "./downloads",
|
||||||
|
"OutputDescription": "Output path",
|
||||||
|
// Batch file
|
||||||
|
"BatchFlag": "batch-file",
|
||||||
|
"BatchDefault": "",
|
||||||
|
"BatchDescription": "File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored.",
|
||||||
|
// Archive
|
||||||
|
"ArchiveFlag": "archive",
|
||||||
|
"ArchiveDefault": "",
|
||||||
|
"ArchiveDescription": "Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.",
|
||||||
|
// Fail log
|
||||||
|
"FailLogFlag": "fail-log",
|
||||||
|
"FailLogDefault": "",
|
||||||
|
"FailLogDescription": "Write failed items to log file",
|
||||||
|
// Debug
|
||||||
|
"DebugFlag": "debug",
|
||||||
|
"DebugDefault": "false",
|
||||||
|
"DebugDescription": "Enables debug mode",
|
||||||
|
// Metadata
|
||||||
|
"MetadataFlag": "metadata",
|
||||||
|
"MetadataDefault": "false",
|
||||||
|
"MetadataDescription": "Write video metadata to a .json file",
|
||||||
|
// Quiet
|
||||||
|
"QuietFlag": "quiet",
|
||||||
|
"QuietDefault": "false",
|
||||||
|
"QuietDescription": "Suppress output",
|
||||||
|
// JSON only
|
||||||
|
"JsonFlag": "json",
|
||||||
|
"JsonDefault": "false",
|
||||||
|
"JsonDescription": "Just get JSON data from scraper (without video downloading)",
|
||||||
|
// Deadline
|
||||||
|
"DeadlineFlag": "deadline",
|
||||||
|
"DeadlineDefault": "1500",
|
||||||
|
"DeadlineDescription": "Sets the timout for scraper logic in seconds (used as a workaround for 'context deadline exceeded' error)",
|
||||||
|
// Limit
|
||||||
|
"LimitFlag": "limit",
|
||||||
|
"LimitDefault": "0",
|
||||||
|
"LimitDescription": "Sets the videos count limit (useful when there too many videos from the user or by hashtag)",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
)
|
|
@ -4,6 +4,9 @@ import (
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
"strconv"
|
||||||
|
|
||||||
|
res "github.com/pikami/tiktok-dl/resources"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Config - Runtime configuration
|
// Config - Runtime configuration
|
||||||
|
@ -23,22 +26,21 @@ var Config struct {
|
||||||
|
|
||||||
// GetConfig - Returns Config object
|
// GetConfig - Returns Config object
|
||||||
func GetConfig() {
|
func GetConfig() {
|
||||||
outputPath := flag.String("output", "./downloads", "Output path")
|
outputPath := flag.String(res.OutputFlag, res.OutputDefault, res.OutputDescription)
|
||||||
batchFilePath := flag.String("batch-file", "", "File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored.")
|
batchFilePath := flag.String(res.BatchFlag, res.BatchDefault, res.BatchDescription)
|
||||||
archive := flag.String("archive", "", "Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.")
|
archive := flag.String(res.ArchiveFlag, res.ArchiveDefault, res.ArchiveDescription)
|
||||||
failLogPath := flag.String("fail-log", "", "Write failed items to log file")
|
failLogPath := flag.String(res.FailLogFlag, res.FailLogDefault, res.FailLogDescription)
|
||||||
debug := flag.Bool("debug", false, "Enables debug mode")
|
debug := flag.Bool(res.DebugFlag, parseBool(res.DebugDefault), res.DebugDescription)
|
||||||
metadata := flag.Bool("metadata", false, "Write video metadata to a .json file")
|
metadata := flag.Bool(res.MetadataFlag, parseBool(res.MetadataDefault), res.MetadataDescription)
|
||||||
quiet := flag.Bool("quiet", false, "Suppress output")
|
quiet := flag.Bool(res.QuietFlag, parseBool(res.QuietDefault), res.QuietDescription)
|
||||||
jsonOnly := flag.Bool("json", false, "Just get JSON data from scraper (without video downloading)")
|
jsonOnly := flag.Bool(res.JsonFlag, parseBool(res.JsonDefault), res.JsonDescription)
|
||||||
deadline := flag.Int("deadline", 1500, "Sets the timout for scraper logic in seconds (used as a workaround for 'context deadline exceeded' error)")
|
deadline := flag.Int(res.DeadlineFlag, parseInt(res.DeadlineDefault), res.DeadlineDescription)
|
||||||
limit := flag.Int("limit", 0, "Sets the videos count limit (useful when there too many videos from the user or by hashtag)")
|
limit := flag.Int(res.LimitFlag, parseInt(res.LimitDefault), res.LimitDescription)
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
|
||||||
args := flag.Args()
|
args := flag.Args()
|
||||||
if len(args) < 1 && *batchFilePath == "" {
|
if len(args) < 1 && *batchFilePath == "" {
|
||||||
fmt.Println("Usage: tiktok-dl [OPTIONS] TIKTOK_USERNAME|TIKTOK_URL")
|
fmt.Println(res.UsageLine)
|
||||||
fmt.Println(" or: tiktok-dl [OPTIONS] -batch-file path/to/users.txt")
|
|
||||||
os.Exit(2)
|
os.Exit(2)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -61,3 +63,19 @@ func GetConfig() {
|
||||||
Config.Deadline = *deadline
|
Config.Deadline = *deadline
|
||||||
Config.Limit = *limit
|
Config.Limit = *limit
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func parseBool(str string) bool {
|
||||||
|
val, err := strconv.ParseBool(str)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
return val
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseInt(str string) int {
|
||||||
|
val, err := strconv.Atoi(str)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
return val
|
||||||
|
}
|
||||||
|
|
|
@ -1,14 +1,15 @@
|
||||||
{
|
{
|
||||||
"name": "tiktok-dl",
|
"name": "tiktok-dl",
|
||||||
"version": "0.0.1",
|
"version": "2.0.0",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"install-dependencies": "go get -v -t -d ./...",
|
"install-dependencies": "go get -v -t -d ./...",
|
||||||
"test:coverage": "go test -short -coverprofile=cov.out ./models ./utils",
|
"test:coverage": "go test -short -coverprofile=cov.out ./models ./utils",
|
||||||
"test": "go test -v ./models && go test -v ./utils",
|
"test": "go test -v ./models && go test -v ./utils",
|
||||||
"clean": "rm -rf out",
|
"clean": "rm -rf out && rm -f scraper.min.js",
|
||||||
"build:scraper": "node node_modules/terser/bin/terser -c -m -- scraper.js > out/scraper.js",
|
"build:scraper": "node node_modules/terser/bin/terser -c -m -- scraper.js > scraper.min.js",
|
||||||
|
"build:res": "cd generator && go run . && cd ..",
|
||||||
"build:app": "go build -o out/tiktok-dl -v .",
|
"build:app": "go build -o out/tiktok-dl -v .",
|
||||||
"build:dist": "mkdir out && npm run build:app && npm run build:scraper",
|
"build:dist": "npm run clean && mkdir out && npm run build:scraper && npm run build:res && npm run build:app",
|
||||||
"build": "go build -v .",
|
"build": "go build -v .",
|
||||||
"sonar": "sonar-scanner -Dsonar.login=${SONAR_LOGIN} -Dproject.settings=.sonar/sonar-project.properties"
|
"sonar": "sonar-scanner -Dsonar.login=${SONAR_LOGIN} -Dproject.settings=.sonar/sonar-project.properties"
|
||||||
},
|
},
|
||||||
|
|
|
@ -0,0 +1,22 @@
|
||||||
|
// Package resources - This file is automatically generated.
|
||||||
|
// Do not edit this file manually.
|
||||||
|
// Check `/generator/resources.go` to change generated content
|
||||||
|
package resources
|
||||||
|
|
||||||
|
//ErrorCouldNotSerializeJSON -
|
||||||
|
var ErrorCouldNotSerializeJSON = "Could not serialize json for video: %s\n"
|
||||||
|
|
||||||
|
//ErrorCouldNotRecogniseURL -
|
||||||
|
var ErrorCouldNotRecogniseURL = "Could not recognise URL format of string %s"
|
||||||
|
|
||||||
|
//Error -
|
||||||
|
var Error = "Error : %s\n"
|
||||||
|
|
||||||
|
//ErrorPathNotFound -
|
||||||
|
var ErrorPathNotFound = "File path %s not found."
|
||||||
|
|
||||||
|
//FailedOnItem -
|
||||||
|
var FailedOnItem = "Failed while scraping item: %s\n"
|
||||||
|
|
||||||
|
//FailedToLoadScraper -
|
||||||
|
var FailedToLoadScraper = "Failed to load scraper"
|
|
@ -0,0 +1,94 @@
|
||||||
|
// Package resources - This file is automatically generated.
|
||||||
|
// Do not edit this file manually.
|
||||||
|
// Check `/generator/resources.go` to change generated content
|
||||||
|
package resources
|
||||||
|
|
||||||
|
//BatchDescription -
|
||||||
|
var BatchDescription = "File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored."
|
||||||
|
|
||||||
|
//LimitDefault -
|
||||||
|
var LimitDefault = "0"
|
||||||
|
|
||||||
|
//DebugDescription -
|
||||||
|
var DebugDescription = "Enables debug mode"
|
||||||
|
|
||||||
|
//QuietDescription -
|
||||||
|
var QuietDescription = "Suppress output"
|
||||||
|
|
||||||
|
//MetadataDefault -
|
||||||
|
var MetadataDefault = "false"
|
||||||
|
|
||||||
|
//LimitDescription -
|
||||||
|
var LimitDescription = "Sets the videos count limit (useful when there too many videos from the user or by hashtag)"
|
||||||
|
|
||||||
|
//OutputDescription -
|
||||||
|
var OutputDescription = "Output path"
|
||||||
|
|
||||||
|
//BatchDefault -
|
||||||
|
var BatchDefault = ""
|
||||||
|
|
||||||
|
//FailLogDefault -
|
||||||
|
var FailLogDefault = ""
|
||||||
|
|
||||||
|
//MetadataFlag -
|
||||||
|
var MetadataFlag = "metadata"
|
||||||
|
|
||||||
|
//ArchiveFlag -
|
||||||
|
var ArchiveFlag = "archive"
|
||||||
|
|
||||||
|
//MetadataDescription -
|
||||||
|
var MetadataDescription = "Write video metadata to a .json file"
|
||||||
|
|
||||||
|
//QuietDefault -
|
||||||
|
var QuietDefault = "false"
|
||||||
|
|
||||||
|
//DeadlineDefault -
|
||||||
|
var DeadlineDefault = "1500"
|
||||||
|
|
||||||
|
//JsonFlag -
|
||||||
|
var JsonFlag = "json"
|
||||||
|
|
||||||
|
//JsonDefault -
|
||||||
|
var JsonDefault = "false"
|
||||||
|
|
||||||
|
//DeadlineDescription -
|
||||||
|
var DeadlineDescription = "Sets the timout for scraper logic in seconds (used as a workaround for 'context deadline exceeded' error)"
|
||||||
|
|
||||||
|
//OutputFlag -
|
||||||
|
var OutputFlag = "output"
|
||||||
|
|
||||||
|
//OutputDefault -
|
||||||
|
var OutputDefault = "./downloads"
|
||||||
|
|
||||||
|
//FailLogFlag -
|
||||||
|
var FailLogFlag = "fail-log"
|
||||||
|
|
||||||
|
//DebugDefault -
|
||||||
|
var DebugDefault = "false"
|
||||||
|
|
||||||
|
//ArchiveDefault -
|
||||||
|
var ArchiveDefault = ""
|
||||||
|
|
||||||
|
//FailLogDescription -
|
||||||
|
var FailLogDescription = "Write failed items to log file"
|
||||||
|
|
||||||
|
//BatchFlag -
|
||||||
|
var BatchFlag = "batch-file"
|
||||||
|
|
||||||
|
//QuietFlag -
|
||||||
|
var QuietFlag = "quiet"
|
||||||
|
|
||||||
|
//DeadlineFlag -
|
||||||
|
var DeadlineFlag = "deadline"
|
||||||
|
|
||||||
|
//LimitFlag -
|
||||||
|
var LimitFlag = "limit"
|
||||||
|
|
||||||
|
//ArchiveDescription -
|
||||||
|
var ArchiveDescription = "Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it."
|
||||||
|
|
||||||
|
//DebugFlag -
|
||||||
|
var DebugFlag = "debug"
|
||||||
|
|
||||||
|
//JsonDescription -
|
||||||
|
var JsonDescription = "Just get JSON data from scraper (without video downloading)"
|
|
@ -0,0 +1,22 @@
|
||||||
|
// Package resources - This file is automatically generated.
|
||||||
|
// Do not edit this file manually.
|
||||||
|
// Check `/generator/resources.go` to change generated content
|
||||||
|
package resources
|
||||||
|
|
||||||
|
//PreloadingItemsFound -
|
||||||
|
var PreloadingItemsFound = "\rPreloading... %s items have been found."
|
||||||
|
|
||||||
|
//Preloading -
|
||||||
|
var Preloading = "\rPreloading..."
|
||||||
|
|
||||||
|
//Retrieving -
|
||||||
|
var Retrieving = "\nRetrieving items..."
|
||||||
|
|
||||||
|
//ItemsFoundInArchive -
|
||||||
|
var ItemsFoundInArchive = "%d items, found in archive. Skipping...\n"
|
||||||
|
|
||||||
|
//Downloaded -
|
||||||
|
var Downloaded = "\r[%d/%d] Downloaded"
|
||||||
|
|
||||||
|
//UsageLine -
|
||||||
|
var UsageLine = "Usage: tiktok-dl [OPTIONS] TIKTOK_USERNAME|TIKTOK_URL\n or: tiktok-dl [OPTIONS] -batch-file path/to/users.txt"
|
|
@ -0,0 +1,10 @@
|
||||||
|
// Package resources - This file is automatically generated.
|
||||||
|
// Do not edit this file manually.
|
||||||
|
// Check `/generator/resources.go` to change generated content
|
||||||
|
package resources
|
||||||
|
|
||||||
|
//ScraperScript -
|
||||||
|
var ScraperScript = "optStrings={selectors:{feedLoading:\"div.tiktok-loading.feed-loading\",modalArrowRight:\"div.video-card-modal > div > img.arrow-right\",modalClose:\".video-card-modal > div > div.close\",modalPlayer:\"div > div > main > div.video-card-modal > div > div.video-card-big > div.video-card-container > div > div > video\",modalShareInput:\".copy-link-container > input\",modalCaption:\"div.video-card-big > div.content-container > div.video-meta-info > h1\",modalSoundLink:\"div.content-container > div.video-meta-info > h2.music-info > a\",modalUploader:\".user-username\",videoPlayer:\"div.video-card-container > div > div > video\",videoShareInput:\"div.content-container.border > div.copy-link-container > input\",videoCaption:\"div.content-container.border > div.video-meta-info > h1\",videoSoundLink:\"div.content-container.border > div.video-meta-info > h2.music-info > a\",videoUploader:\".user-username\"},classes:{feedVideoItem:\"video-feed-item-wrapper\",modalCloseDisabled:\"disabled\",titleMessage:\"title\"},tags:{resultTag:\"video_urls\",resultParentTag:\"body\"},attributes:{src:\"src\"},tiktokMessages:[\"Couldn't find this account\",\"No videos yet\",\"Video currently unavailable\"]},currentState={preloadCount:0,finished:!1,limit:0},checkForErrors=function(){var e=document.getElementsByClassName(optStrings.classes.titleMessage);if(e&&e.length){var t=Array.from(e).find(e=>optStrings.tiktokMessages.includes(e.textContent)).textContent;if(t)return createVidUrlElement(\"ERR: \"+t),!0}return!1},createVidUrlElement=function(e){var t=document.createElement(optStrings.tags.resultTag);t.innerText=JSON.stringify(e),document.getElementsByTagName(optStrings.tags.resultParentTag)[0].appendChild(t),currentState.finished=!0},buldVidUrlArray=function(e){document.getElementsByClassName(optStrings.classes.feedVideoItem)[0].click();var t=[],r=window.setInterval(o=>{t.push(getCurrentModalVideo()),currentState.limit>0&&t.length>=currentState.limit&&(window.clearInterval(r),document.querySelector(optStrings.selectors.modalClose).click(),e(t));var i=document.querySelectorAll(optStrings.selectors.modalArrowRight)[0];i.classList.contains(optStrings.classes.modalCloseDisabled)?(window.clearInterval(r),document.querySelector(optStrings.selectors.modalClose).click(),e(t)):i.click()},20)},getCurrentModalVideo=function(){var e=document.querySelector(optStrings.selectors.modalPlayer).getAttribute(optStrings.attributes.src),t=document.querySelector(optStrings.selectors.modalShareInput).value,r=document.querySelector(optStrings.selectors.modalCaption).textContent,o=document.querySelector(optStrings.selectors.modalSoundLink),i=document.querySelector(optStrings.selectors.modalUploader).textContent,n=o.getAttribute(\"href\");return{url:e,shareLink:t,caption:r,uploader:i,sound:{title:o.text,link:n}}},getCurrentVideo=function(){if(!checkForErrors()){var e=document.querySelector(optStrings.selectors.videoPlayer).getAttribute(optStrings.attributes.src),t=document.querySelector(optStrings.selectors.videoShareInput).value,r=document.querySelector(optStrings.selectors.videoCaption).textContent,o=document.querySelector(optStrings.selectors.videoSoundLink),i=document.querySelector(optStrings.selectors.videoUploader).textContent,n=o.getAttribute(\"href\");return{url:e,shareLink:t,caption:r,uploader:i,sound:{title:o.text,link:n}}}},scrollBottom=()=>window.scrollTo(0,document.body.scrollHeight),scrollWhileNew=function(e){var t={count:0},r=window.setInterval(o=>{scrollBottom();var i=t.count;if(t.count=document.getElementsByClassName(optStrings.classes.feedVideoItem).length,currentState.limit>0&&(currentState.preloadCount>=currentState.limit||t.count>=currentState.limit)&&(e(createVidUrlElement),window.clearInterval(r)),checkForErrors())window.clearInterval(r);else if(0!=t.count)if(i!==t.count)currentState.preloadCount=t.count;else{if(document.querySelector(optStrings.selectors.feedLoading))return;window.clearInterval(r),e(createVidUrlElement)}},1e3)},bootstrapIteratingVideos=function(e){return currentState.limit=e,scrollWhileNew(buldVidUrlArray),\"bootstrapIteratingVideos\"},bootstrapGetCurrentVideo=function(){var e=getCurrentVideo();return createVidUrlElement(e),\"bootstrapGetCurrentVideo\"},init=()=>{const e=navigator.__proto__;return delete e.webdriver,navigator.__proto__=e,\"script initialized\"},init();"
|
||||||
|
|
||||||
|
//ScraperPath -
|
||||||
|
var ScraperPath = "scraper.js"
|
|
@ -1,16 +0,0 @@
|
||||||
package resources
|
|
||||||
|
|
||||||
// ErrorCouldNotSerializeJSON -
|
|
||||||
var ErrorCouldNotSerializeJSON = "Could not serialize json for video: %s\n"
|
|
||||||
|
|
||||||
// ErrorCouldNotRecogniseURL -
|
|
||||||
var ErrorCouldNotRecogniseURL = "Could not recognise URL format of string %s"
|
|
||||||
|
|
||||||
// Error -
|
|
||||||
var Error = "Error : %s\n"
|
|
||||||
|
|
||||||
// ErrorPathNotFound -
|
|
||||||
var ErrorPathNotFound = "File path %s not found."
|
|
||||||
|
|
||||||
// FailedOnItem -
|
|
||||||
var FailedOnItem = "Failed while scraping item: %s\n"
|
|
|
@ -3,6 +3,7 @@ package utils
|
||||||
import (
|
import (
|
||||||
models "github.com/pikami/tiktok-dl/models"
|
models "github.com/pikami/tiktok-dl/models"
|
||||||
config "github.com/pikami/tiktok-dl/models/config"
|
config "github.com/pikami/tiktok-dl/models/config"
|
||||||
|
res "github.com/pikami/tiktok-dl/resources"
|
||||||
fileio "github.com/pikami/tiktok-dl/utils/fileio"
|
fileio "github.com/pikami/tiktok-dl/utils/fileio"
|
||||||
log "github.com/pikami/tiktok-dl/utils/log"
|
log "github.com/pikami/tiktok-dl/utils/log"
|
||||||
)
|
)
|
||||||
|
@ -36,7 +37,7 @@ func RemoveArchivedItems(uploads []models.Upload) []models.Upload {
|
||||||
|
|
||||||
removedCount := lenBeforeRemoval - len(uploads)
|
removedCount := lenBeforeRemoval - len(uploads)
|
||||||
if removedCount > 0 {
|
if removedCount > 0 {
|
||||||
log.Logf("%d items, found in archive. Skipping...\n", removedCount)
|
log.Logf(res.ItemsFoundInArchive, removedCount)
|
||||||
}
|
}
|
||||||
|
|
||||||
return uploads
|
return uploads
|
||||||
|
|
|
@ -0,0 +1,19 @@
|
||||||
|
package utils
|
||||||
|
|
||||||
|
import (
|
||||||
|
resources "github.com/pikami/tiktok-dl/resources"
|
||||||
|
fileio "github.com/pikami/tiktok-dl/utils/fileio"
|
||||||
|
)
|
||||||
|
|
||||||
|
// GetScraper - Retrieve scraper
|
||||||
|
func GetScraper() string {
|
||||||
|
if fileio.CheckIfExists(resources.ScraperPath) {
|
||||||
|
return ReadFileAsString(resources.ScraperPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
if resources.ScraperScript != "" {
|
||||||
|
return resources.ScraperScript
|
||||||
|
}
|
||||||
|
|
||||||
|
panic(resources.FailedToLoadScraper)
|
||||||
|
}
|
|
@ -6,6 +6,7 @@ import (
|
||||||
|
|
||||||
client "github.com/pikami/tiktok-dl/client"
|
client "github.com/pikami/tiktok-dl/client"
|
||||||
config "github.com/pikami/tiktok-dl/models/config"
|
config "github.com/pikami/tiktok-dl/models/config"
|
||||||
|
res "github.com/pikami/tiktok-dl/resources"
|
||||||
utils "github.com/pikami/tiktok-dl/utils"
|
utils "github.com/pikami/tiktok-dl/utils"
|
||||||
fileio "github.com/pikami/tiktok-dl/utils/fileio"
|
fileio "github.com/pikami/tiktok-dl/utils/fileio"
|
||||||
log "github.com/pikami/tiktok-dl/utils/log"
|
log "github.com/pikami/tiktok-dl/utils/log"
|
||||||
|
@ -35,7 +36,7 @@ func DownloadHashtag(url string) {
|
||||||
|
|
||||||
for index, upload := range uploads {
|
for index, upload := range uploads {
|
||||||
downloadVideo(upload, downloadDir)
|
downloadVideo(upload, downloadDir)
|
||||||
log.Logf("\r[%d/%d] Downloaded", index+1, uploadCount)
|
log.Logf(res.Downloaded, index+1, uploadCount)
|
||||||
}
|
}
|
||||||
log.Log()
|
log.Log()
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,6 +6,7 @@ import (
|
||||||
|
|
||||||
client "github.com/pikami/tiktok-dl/client"
|
client "github.com/pikami/tiktok-dl/client"
|
||||||
config "github.com/pikami/tiktok-dl/models/config"
|
config "github.com/pikami/tiktok-dl/models/config"
|
||||||
|
res "github.com/pikami/tiktok-dl/resources"
|
||||||
utils "github.com/pikami/tiktok-dl/utils"
|
utils "github.com/pikami/tiktok-dl/utils"
|
||||||
fileio "github.com/pikami/tiktok-dl/utils/fileio"
|
fileio "github.com/pikami/tiktok-dl/utils/fileio"
|
||||||
log "github.com/pikami/tiktok-dl/utils/log"
|
log "github.com/pikami/tiktok-dl/utils/log"
|
||||||
|
@ -34,7 +35,7 @@ func DownloadMusic(url string) {
|
||||||
|
|
||||||
fileio.InitOutputDirectory(downloadDir)
|
fileio.InitOutputDirectory(downloadDir)
|
||||||
downloadVideo(upload, downloadDir)
|
downloadVideo(upload, downloadDir)
|
||||||
log.Logf("\r[%d/%d] Downloaded", index+1, uploadCount)
|
log.Logf(res.Downloaded, index+1, uploadCount)
|
||||||
}
|
}
|
||||||
log.Log()
|
log.Log()
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,6 +7,7 @@ import (
|
||||||
|
|
||||||
client "github.com/pikami/tiktok-dl/client"
|
client "github.com/pikami/tiktok-dl/client"
|
||||||
config "github.com/pikami/tiktok-dl/models/config"
|
config "github.com/pikami/tiktok-dl/models/config"
|
||||||
|
res "github.com/pikami/tiktok-dl/resources"
|
||||||
utils "github.com/pikami/tiktok-dl/utils"
|
utils "github.com/pikami/tiktok-dl/utils"
|
||||||
fileio "github.com/pikami/tiktok-dl/utils/fileio"
|
fileio "github.com/pikami/tiktok-dl/utils/fileio"
|
||||||
log "github.com/pikami/tiktok-dl/utils/log"
|
log "github.com/pikami/tiktok-dl/utils/log"
|
||||||
|
@ -36,7 +37,7 @@ func DownloadUser(username string) {
|
||||||
|
|
||||||
for index, upload := range uploads {
|
for index, upload := range uploads {
|
||||||
downloadVideo(upload, downloadDir)
|
downloadVideo(upload, downloadDir)
|
||||||
log.Logf("\r[%d/%d] Downloaded", index+1, uploadCount)
|
log.Logf(res.Downloaded, index+1, uploadCount)
|
||||||
}
|
}
|
||||||
log.Log()
|
log.Log()
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,6 +7,7 @@ import (
|
||||||
client "github.com/pikami/tiktok-dl/client"
|
client "github.com/pikami/tiktok-dl/client"
|
||||||
models "github.com/pikami/tiktok-dl/models"
|
models "github.com/pikami/tiktok-dl/models"
|
||||||
config "github.com/pikami/tiktok-dl/models/config"
|
config "github.com/pikami/tiktok-dl/models/config"
|
||||||
|
res "github.com/pikami/tiktok-dl/resources"
|
||||||
utils "github.com/pikami/tiktok-dl/utils"
|
utils "github.com/pikami/tiktok-dl/utils"
|
||||||
fileio "github.com/pikami/tiktok-dl/utils/fileio"
|
fileio "github.com/pikami/tiktok-dl/utils/fileio"
|
||||||
log "github.com/pikami/tiktok-dl/utils/log"
|
log "github.com/pikami/tiktok-dl/utils/log"
|
||||||
|
@ -34,7 +35,7 @@ func DownloadSingleVideo(url string) {
|
||||||
|
|
||||||
fileio.InitOutputDirectory(downloadDir)
|
fileio.InitOutputDirectory(downloadDir)
|
||||||
downloadVideo(upload, downloadDir)
|
downloadVideo(upload, downloadDir)
|
||||||
log.Log("[1/1] Downloaded\n")
|
log.Logf(res.Downloaded, 1, 1)
|
||||||
}
|
}
|
||||||
|
|
||||||
// DownloadVideo - Downloads one video
|
// DownloadVideo - Downloads one video
|
||||||
|
|
Loading…
Reference in New Issue